feat(server): add fallback client proxy

Signed-off-by: nachiketb <nachiketb@nvidia.com>

Author: nachiketb-nvidia

Cargo.lock

@@ -255,17 +255,19 @@ impl Backend {
         self.config().max_retries
     }
 
-    /// Whether this backend speaks the Anthropic Messages wire format — the only
-    /// one with a `count_tokens` endpoint.
-    pub fn is_anthropic(&self) -> bool {
-        matches!(self, Backend::Anthropic(_))
-    }
-
-    /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
-    /// URL join as [`url`](Self::url).
-    pub fn count_tokens_url(&self) -> String {
+    /// Resolves an unmatched provider path against this backend's API root.
+    pub(crate) fn forwarding_url(&self, path_and_query: &str) -> String {
         let base_url = self.config().base_url.trim_end_matches('/');
-        format!("{}/count_tokens", anthropic_url(base_url))
+        let root = [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages",
+            "/v1",
+        ]
+        .iter()
+        .find_map(|suffix| base_url.strip_suffix(suffix))
+        .unwrap_or(base_url);
+        format!("{root}{path_and_query}")
     }
 
     /// Whether an upstream 400 `body` looks like a context-window overflow for
@@ -391,34 +393,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn count_tokens_url_joins_every_base_url_shape() {
-        assert_eq!(
-            Backend::Anthropic(config("https://host")).count_tokens_url(),
-            "https://host/v1/messages/count_tokens"
-        );
-        assert_eq!(
-            Backend::Anthropic(config("https://host/v1")).count_tokens_url(),
-            "https://host/v1/messages/count_tokens"
-        );
-        assert_eq!(
-            Backend::Anthropic(config("https://host/v1/messages")).count_tokens_url(),
-            "https://host/v1/messages/count_tokens"
-        );
-        // Trailing slash is trimmed before the join.
-        assert_eq!(
-            Backend::Anthropic(config("https://host/v1/")).count_tokens_url(),
-            "https://host/v1/messages/count_tokens"
-        );
-    }
-
-    #[test]
-    fn only_anthropic_backend_is_anthropic() {
-        assert!(Backend::Anthropic(config("x")).is_anthropic());
-        assert!(!Backend::OpenAiChat(config("x")).is_anthropic());
-        assert!(!Backend::OpenAiResponses(config("x")).is_anthropic());
-    }
-
     #[test]
     fn wire_format_matches_variant() {
         assert_eq!(

crates/libsy-llm-client/src/backend.rs

@@ -10,9 +10,9 @@ use std::time::{Duration, SystemTime};
 
 use async_trait::async_trait;
 use futures_util::{StreamExt, stream};
-use http::StatusCode;
+use http::{Method, StatusCode};
 use reqwest::RequestBuilder;
-use reqwest::header::{HeaderMap, RETRY_AFTER};
+use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE, HeaderMap, RETRY_AFTER};
 use serde_json::{Map, Value};
 use switchyard_protocol::{
     LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Metadata, ModelId, Request,
@@ -146,49 +146,35 @@ impl TranslatingLlmClient {
         })
     }
 
-    /// Whether `model` has an Anthropic backend that supports token counting.
-    pub fn supports_count_tokens(&self, model: &ModelId) -> bool {
-        self.backend_for(model, WireFormat::AnthropicMessages)
-            .is_some()
-    }
-
-    /// Counts input tokens with `model`'s Anthropic backend.
-    ///
-    /// Returns an error when the model has no Anthropic backend or the upstream
-    /// request fails or returns invalid JSON.
-    pub async fn count_tokens(&self, model: &ModelId, request: Request) -> Result<Value> {
-        let backend = self
-            .backend_for(model, WireFormat::AnthropicMessages)
-            .ok_or_else(|| LlmClientError::Configuration {
-                message: format!("model {model} has no Anthropic backend for count_tokens"),
-            })?;
-        let Request {
-            mut llm_request,
-            metadata,
-            ..
-        } = request;
-        llm_request.model = Some(model.to_string());
-        let http_response = self
-            .send_encoded(
-                backend,
-                WireFormat::AnthropicMessages,
-                llm_request,
-                metadata.as_ref(),
-                model,
-                UpstreamEndpoint::CountTokens,
-            )
-            .await?;
-        let body = match http_response {
-            EncodedResponse::Buffered { body, .. } => body,
-            EncodedResponse::Streaming(_) => {
-                return Err(LlmClientError::InvalidRequest {
-                    message: "count_tokens does not support streaming requests".to_string(),
-                });
-            }
+    /// Forwards a provider-native request through `backend` without translation.
+    pub async fn forward(
+        &self,
+        backend: &Backend,
+        method: Method,
+        path_and_query: &str,
+        body: reqwest::Body,
+        metadata: Option<&Metadata>,
+    ) -> Result<reqwest::Response> {
+        let client = if backend.is_forwarding_auth() {
+            &self.forward_auth_client
+        } else {
+            &self.client
         };
-        serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {
-            source: Box::new(error),
-        })
+        let mut builder = client
+            .request(method, backend.forwarding_url(path_and_query))
+            .body(body);
+        builder = forward_metadata_headers(builder, metadata);
+        if let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) {
+            for name in [CONTENT_TYPE, CONTENT_LENGTH] {
+                if let Some(value) = headers.get(&name) {
+                    builder = builder.header(name, value);
+                }
+            }
+        }
+        builder = backend.apply_forwarded_auth(builder, metadata);
+        builder = apply_extra_headers(builder, backend);
+        builder = backend.apply_auth(builder);
+        builder.send().await.map_err(convert_reqwest_error)
     }
 
     /// Encode `llm_request` for `wire_format`, POST it to `url` with the request's
@@ -198,18 +184,15 @@ impl TranslatingLlmClient {
     /// response is returned as soon as its successful headers arrive. A non-success
     /// status maps to a typed error — a 400 is classified as a context-window
     /// overflow via the backend's provider rules. Shared by
-    /// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the
-    /// backend's completion URL and decodes a response) and
-    /// [`count_tokens`](Self::count_tokens) (which POSTs to the `count_tokens`
-    /// URL and returns the raw JSON).
+    /// [`call_rewrite_model`](Self::call_rewrite_model), which POSTs to the
+    /// backend's completion URL and decodes a response.
     async fn send_encoded(
         &self,
         backend: &Backend,
         wire_format: WireFormat,
         llm_request: LlmRequest,
         metadata: Option<&Metadata>,
         model: &ModelId,
-        endpoint: UpstreamEndpoint,
     ) -> Result<EncodedResponse> {
         let mut body = encode_request(&llm_request, wire_format)
             .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
@@ -230,9 +213,8 @@ impl TranslatingLlmClient {
         if matches!(backend, Backend::OpenAiChat(_)) {
             ensure_openai_stream_usage(&mut body);
         }
-        let streaming = endpoint.allows_streaming()
-            && body.get("stream").and_then(Value::as_bool).unwrap_or(false);
-        let url = endpoint.url(backend);
+        let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
+        let url = backend.url();
         record_gen_ai_request(&url, model, streaming);
 
         let max_retries = u64::from(backend.max_retries());
@@ -426,7 +408,6 @@ impl TranslatingLlmClient {
                 llm_request,
                 metadata.as_ref(),
                 &model_id,
-                UpstreamEndpoint::Completion,
             )
             .await?;
 
@@ -559,25 +540,6 @@ impl RoutedLlmClient for TranslatingLlmClient {
     }
 }
 
-#[derive(Clone, Copy)]
-enum UpstreamEndpoint {
-    Completion,
-    CountTokens,
-}
-
-impl UpstreamEndpoint {
-    fn url(self, backend: &Backend) -> String {
-        match self {
-            UpstreamEndpoint::Completion => backend.url(),
-            UpstreamEndpoint::CountTokens => backend.count_tokens_url(),
-        }
-    }
-
-    fn allows_streaming(self) -> bool {
-        matches!(self, UpstreamEndpoint::Completion)
-    }
-}
-
 enum EncodedResponse {
     Buffered { status: u16, body: Vec<u8> },
     Streaming(reqwest::Response),

crates/libsy-llm-client/src/client.rs

@@ -467,7 +467,7 @@ impl AlgorithmSpec {
                 names
             }
             // The advisor is judge-only: reviews go through its own client,
-            // so it is not a completion (or count_tokens) destination.
+            // so it is not a completion destination.
             Self::Advisor {
                 executor_target, ..
             } => vec![executor_target],

crates/switchyard-runner/src/algorithm.rs

@@ -17,9 +17,9 @@ use switchyard_llm_client::{
 };
 use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};
 
+use crate::runner::FallbackClient;
 use crate::{
-    AlgorithmSpec, CallerAuthKind, CountTokensTarget, DecisionTarget, ModelCapabilities, Route,
-    Runner, RunnerError,
+    AlgorithmSpec, CallerAuthKind, DecisionTarget, ModelCapabilities, Route, Runner, RunnerError,
 };
 
 const SUPPORTED_SCHEMA_VERSION: u32 = 1;
@@ -54,6 +54,7 @@ pub(crate) fn runner_from_toml(source: &str) -> RunnerResult<Runner> {
 #[serde(deny_unknown_fields)]
 pub(crate) struct DeploymentConfig {
     schema_version: u32,
+    fallback_client: Option<String>,
     #[serde(default)]
     llm_clients: BTreeMap<String, LlmClientConfig>,
     targets: BTreeMap<String, TargetConfig>,
@@ -165,6 +166,7 @@ impl DeploymentConfig {
 
         let clients = self.build_clients()?;
         let targets = self.build_targets();
+        let fallback_client = self.build_fallback_client(&clients)?;
         let mut routes = Vec::with_capacity(self.routes.len());
         for (route_name, config) in &self.routes {
             validate_value("route name", route_name)?;
@@ -188,7 +190,6 @@ impl DeploymentConfig {
                 .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;
             let (route_clients, caller_auth) =
                 self.build_route_clients(route_name, config, &clients)?;
-            let count_tokens_target = self.build_count_tokens_target(config, &clients);
             let decision_targets = config
                 .routing_target_names()
                 .into_iter()
@@ -199,12 +200,11 @@ impl DeploymentConfig {
                 route_clients,
                 caller_auth,
                 capabilities,
-                count_tokens_target,
                 decision_targets,
             );
             routes.push((config.id.clone(), route));
         }
-        Ok(Runner::new(routes))
+        Ok(Runner::new(routes).with_fallback_client(fallback_client))
     }
 
     fn build_clients(&self) -> RunnerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {
@@ -291,42 +291,28 @@ impl DeploymentConfig {
         Ok((ClientRouter::new(by_model), caller_auth))
     }
 
-    fn build_count_tokens_target(
+    fn build_fallback_client(
         &self,
-        route: &RouteConfig,
         clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
-    ) -> Option<CountTokensTarget> {
-        route
-            .routing_target_names()
-            .into_iter()
-            .enumerate()
-            .filter_map(|(index, name)| {
-                let target = self.targets.get(name)?;
-                let client = clients.get(&target.llm_client)?;
-                client.supports_count_tokens(&target.id).then_some((
-                    count_tokens_priority(name, &target.id),
-                    index,
-                    target,
-                    client,
-                ))
-            })
-            .min_by_key(|(priority, index, _, _)| (*priority, *index))
-            .map(|(_, _, target, client)| CountTokensTarget {
-                model: target.id.clone(),
-                client: client.clone(),
-            })
+    ) -> RunnerResult<Option<FallbackClient>> {
+        let Some(name) = &self.fallback_client else {
+            return Ok(None);
+        };
+        let config = self.llm_clients.get(name).ok_or_else(|| {
+            RunnerError::configuration(format!(
+                "fallback_client references unknown llm client {name}"
+            ))
+        })?;
+        let client = clients.get(name).ok_or_else(|| {
+            RunnerError::configuration("validated fallback client was not initialized")
+        })?;
+        Ok(Some(FallbackClient {
+            backend: build_backend(name, config, &BTreeMap::new())?,
+            client: client.clone(),
+        }))
     }
 }
 
-fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
-    let target_name = target_name.to_ascii_lowercase();
-    let model_id = model_id.to_ascii_lowercase();
-    ["opus", "sonnet", "haiku"]
-        .iter()
-        .position(|hint| target_name.contains(hint) || model_id.contains(hint))
-        .unwrap_or(3)
-}
-
 /// A client endpoint, parsed when the config loads rather than checked afterwards.
 ///
 /// Holding a `HttpBaseUrl` is proof the value is an absolute HTTP(S) URL, so no

crates/switchyard-runner/src/config.rs

@@ -98,9 +98,7 @@ impl RunnerError {
                 None,
                 None,
             ),
-            Self::UnknownRouteModel(_)
-            | Self::IncompatibleCallerFormat(_)
-            | Self::CountTokensUnsupported => summary(
+            Self::UnknownRouteModel(_) | Self::IncompatibleCallerFormat(_) => summary(
                 RouteErrorKind::InvalidRequest,
                 RouteErrorPhase::BeforeResponse,
                 None,
@@ -328,11 +326,5 @@ mod tests {
             configuration.execution_error_summary().kind,
             RouteErrorKind::Configuration
         ));
-
-        let unsupported = RunnerError::CountTokensUnsupported;
-        assert!(matches!(
-            unsupported.execution_error_summary().kind,
-            RouteErrorKind::InvalidRequest
-        ));
     }
 }

crates/switchyard-runner/src/failure.rs

@@ -14,7 +14,5 @@ pub use algorithm::{
     ClassifierPolicyConfig, LlmClassifierRouteConfig, StageClassifierConfig, SubagentRouteConfig,
 };
 pub use failure::{RouteErrorKind, RouteErrorPhase, RouteErrorSummary, stream_error_summary};
-pub use route::{
-    CallerAuthKind, CountTokensTarget, ModelCapabilities, Route, RunOutput, RunnerError,
-};
+pub use route::{CallerAuthKind, ModelCapabilities, Route, RunOutput, RunnerError};
 pub use runner::{DecisionDescription, DecisionTarget, ModelInfo, Runner};