Add routing_method experiment variable for A/B/C hydra experiment

extensions/copilot/src/platform/configuration/common/configurationService.ts

@@ -830,6 +830,14 @@ export namespace ConfigKey {
 		export const InstantApplyModelName = defineTeamInternalSetting<string>('chat.advanced.instantApply.modelName', ConfigType.ExperimentBased, CHAT_MODEL.GPT4OPROXY);
 		export const VerifyTextDocumentChanges = defineTeamInternalSetting<boolean>('chat.advanced.inlineEdits.verifyTextDocumentChanges', ConfigType.ExperimentBased, false);
 		export const UseAutoModeRouting = defineTeamInternalSetting<boolean>('chat.advanced.useAutoModeRouter', ConfigType.ExperimentBased, false);
+		/** Controls which `routing_method` value is sent to the auto-intent-service per request
+		 * when `UseAutoModeRouting` is enabled.
+		 * '' (empty/default) = omit `routing_method` and use the server default.
+		 * 'binary' = binary classifier v1.
+		 * 'hydra' = HYDRA multi-head capability matching.
+		 * For experiments, this setting selects the routing method only when router usage is enabled;
+		 * it does not by itself determine whether the router is called. */
+		export const AutoModeRoutingMethod = defineTeamInternalSetting<string>('chat.advanced.autoModeRoutingMethod', ConfigType.ExperimentBased, '', undefined, undefined, { experimentName: 'copilotchat.autoModeRoutingMethod' });
 
 		/** Inline Completions */
 		export const InlineCompletionsDefaultDiagnosticsOptions = defineTeamInternalSetting<string | undefined>('chat.advanced.inlineCompletions.defaultDiagnosticsOptionsString', ConfigType.ExperimentBased, undefined);

extensions/copilot/src/platform/endpoint/node/automodeService.ts

@@ -277,7 +277,13 @@ export class AutomodeService extends Disposable implements IAutomodeService {
 				previous_model: entry?.endpoint?.model,
 				turn_number: (entry?.turnCount ?? 0) + 1,
 			};
-			const result = await this._routerDecisionFetcher.getRouterDecision(prompt, token.session_token, token.available_models, undefined, contextSignals, chatRequest?.sessionId, chatRequest?.id);
+			const routingMethod = this._configurationService.getExperimentBasedConfig(ConfigKey.TeamInternal.AutoModeRoutingMethod, this._expService) || undefined;
+			const result = await this._routerDecisionFetcher.getRouterDecision(prompt, token.session_token, token.available_models, undefined, contextSignals, chatRequest?.sessionId, chatRequest?.id, routingMethod);
+
+			if (result.fallback) {
+				this._logService.info(`[AutomodeService] Router signaled fallback: ${result.fallback_reason ?? 'unknown'}, routing_method=${result.routing_method ?? 'n/a'}`);
+				return { lastRoutedPrompt: prompt, fallbackReason: 'routerFallback' };
+			}
 
 			if (!result.candidate_models.length) {
 				return { lastRoutedPrompt: prompt, fallbackReason: 'emptyCandidateList' };

extensions/copilot/src/platform/endpoint/node/routerDecisionFetcher.ts

@@ -13,7 +13,7 @@ import { ITelemetryService } from '../../telemetry/common/telemetry';
 import { ICAPIClientService } from '../common/capiClient';
 
 export interface RouterDecisionResponse {
-	predicted_label: 'needs_reasoning' | 'no_reasoning';
+	predicted_label: 'needs_reasoning' | 'no_reasoning' | 'fallback';
 	confidence: number;
 	latency_ms: number;
 	candidate_models: string[];
@@ -22,6 +22,12 @@ export interface RouterDecisionResponse {
 		no_reasoning: number;
 	};
 	sticky_override?: boolean;
+	routing_method?: string;
+	fallback?: boolean;
+	fallback_reason?: string;
+	hydra_scores?: Record<string, number>;
+	chosen_model?: string;
+	chosen_shortfall?: number;
 }
 
 export interface RoutingContextSignals {
@@ -48,12 +54,15 @@ export class RouterDecisionFetcher {
 	) {
 	}
 
-	async getRouterDecision(query: string, autoModeToken: string, availableModels: string[], stickyThreshold?: number, contextSignals?: RoutingContextSignals, conversationId?: string, vscodeRequestId?: string): Promise<RouterDecisionResponse> {
+	async getRouterDecision(query: string, autoModeToken: string, availableModels: string[], stickyThreshold?: number, contextSignals?: RoutingContextSignals, conversationId?: string, vscodeRequestId?: string, routingMethod?: string): Promise<RouterDecisionResponse> {
 		const startTime = Date.now();
 		const requestBody: Record<string, unknown> = { prompt: query, available_models: availableModels, ...contextSignals };
 		if (stickyThreshold !== undefined) {
 			requestBody.sticky_threshold = stickyThreshold;
 		}
+		if (routingMethod) {
+			requestBody.routing_method = routingMethod;
+		}
 		const copilotToken = (await this._authService.getCopilotToken()).token;
 		const abortController = new AbortController();
 		const timeout = setTimeout(() => abortController.abort(), 1000);
@@ -79,7 +88,7 @@ export class RouterDecisionFetcher {
 		const text = await response.text();
 		const result: RouterDecisionResponse = JSON.parse(text);
 		const e2eLatencyMs = Date.now() - startTime;
-		this._logService.trace(`[RouterDecisionFetcher] Prediction: ${result.predicted_label}, (confidence: ${(result.confidence * 100).toFixed(1)}%, scores: needs_reasoning=${(result.scores.needs_reasoning * 100).toFixed(1)}%, no_reasoning=${(result.scores.no_reasoning * 100).toFixed(1)}%) (latency_ms: ${result.latency_ms}, e2e_latency_ms: ${e2eLatencyMs}, candidate models: ${result.candidate_models.join(', ')}, sticky_override: ${result.sticky_override ?? false})`);
+		this._logService.trace(`[RouterDecisionFetcher] Prediction: ${result.predicted_label}, (confidence: ${(result.confidence * 100).toFixed(1)}%, scores: needs_reasoning=${(result.scores.needs_reasoning * 100).toFixed(1)}%, no_reasoning=${(result.scores.no_reasoning * 100).toFixed(1)}%) (latency_ms: ${result.latency_ms}, e2e_latency_ms: ${e2eLatencyMs}, candidate models: ${result.candidate_models.join(', ')}, sticky_override: ${result.sticky_override ?? false}, routing_method: ${result.routing_method ?? 'n/a'}, fallback: ${result.fallback ?? false})`);
 
 		this._requestLogger.addEntry({
 			type: LoggedRequestKind.MarkdownContentRequest,
@@ -111,7 +120,10 @@ export class RouterDecisionFetcher {
 				"comment": "Reports the routing decision made by the auto mode router API",
 				"conversationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The conversation ID in which the routing decision was made." },
 				"vscodeRequestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The VS Code chat request id in which the routing decision was made." },
-				"predictedLabel": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The predicted classification label (needs_reasoning or no_reasoning)" },
+				"predictedLabel": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The predicted classification label (needs_reasoning, no_reasoning, or fallback)" },
+				"routingMethod": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The routing method used for this request (empty=server default, binary, hydra). Identifies the A/B/C experiment path." },
+				"fallback": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the router signaled a fallback to default automod selection." },
+				"fallbackReason": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The reason provided by the server when fallback is true." },
 				"confidence": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The confidence score of the routing decision" },
 				"latencyMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "The latency of the router API call in milliseconds" },
 				"e2eLatencyMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "The end-to-end latency of the router request in milliseconds, including network overhead" }
@@ -122,6 +134,9 @@ export class RouterDecisionFetcher {
 				conversationId: conversationId ?? '',
 				vscodeRequestId: vscodeRequestId ?? '',
 				predictedLabel: result.predicted_label,
+				routingMethod: result.routing_method ?? '',
+				fallback: String(result.fallback ?? false),
+				fallbackReason: result.fallback_reason ?? '',
 			},
 			{
 				confidence: result.confidence,