From b6053718fb67b89b0efaa389f086c7f6b93c35f6 Mon Sep 17 00:00:00 2001 From: Brian Glusman Date: Wed, 13 May 2026 00:37:19 -0400 Subject: [PATCH 1/2] Generalize OpenAI-compatible provider adapters --- crates/calciforge/Cargo.toml | 5 +- crates/calciforge/src/config.rs | 26 +- crates/calciforge/src/config/validator.rs | 1618 +---------------- .../src/config/validator_test_support.rs | 38 + .../src/config/validator_tests_1.rs | 501 +++++ .../src/config/validator_tests_2.rs | 508 ++++++ .../src/config/validator_tests_3.rs | 526 ++++++ crates/calciforge/src/doctor.rs | 84 +- crates/calciforge/src/proxy/backend.rs | 95 +- crates/calciforge/src/proxy/gateway.rs | 619 ++----- crates/calciforge/src/proxy/gateway_tests.rs | 539 ++++++ .../calciforge/src/proxy/helicone_router.rs | 384 ---- .../src/proxy/helicone_router_tests.rs | 382 ---- crates/calciforge/src/proxy/mod.rs | 112 +- ...icone_streaming.rs => openai_streaming.rs} | 26 +- crates/calciforge/src/proxy/routing.rs | 94 +- ...0001-model-gateway-and-agent-boundaries.md | 37 +- docs/adr/0002-provider-adapter-boundary.md | 4 + docs/model-gateway.md | 167 +- .../roadmap/litellm-gateway-owned-provider.md | 14 +- scripts/boundary-aggression.sh | 2 +- scripts/boundary-explore-long.sh | 2 +- scripts/model-gateway-litellm-smoke.py | 2 +- tests/boundaries/integration-surfaces.json | 14 +- tests/scenarios/high-risk-scenarios.json | 9 +- 25 files changed, 2708 insertions(+), 3100 deletions(-) create mode 100644 crates/calciforge/src/config/validator_test_support.rs create mode 100644 crates/calciforge/src/config/validator_tests_1.rs create mode 100644 crates/calciforge/src/config/validator_tests_2.rs create mode 100644 crates/calciforge/src/config/validator_tests_3.rs create mode 100644 crates/calciforge/src/proxy/gateway_tests.rs delete mode 100644 crates/calciforge/src/proxy/helicone_router.rs delete mode 100644 crates/calciforge/src/proxy/helicone_router_tests.rs rename crates/calciforge/src/proxy/{helicone_streaming.rs => openai_streaming.rs} (94%) diff --git a/crates/calciforge/Cargo.toml b/crates/calciforge/Cargo.toml index ad26b434..6960bc0c 100644 --- a/crates/calciforge/Cargo.toml +++ b/crates/calciforge/Cargo.toml @@ -10,11 +10,10 @@ name = "calciforge" path = "src/main.rs" [features] -default = ["helicone"] +default = [] channel-matrix = [] hegel = ["dep:hegeltest"] persistent-context = ["dep:rusqlite"] -helicone = [] tiktoken-estimator = ["dep:tiktoken-rs"] test = [] @@ -108,7 +107,7 @@ uuid = { version = "1", features = ["v4"] } tempfile = "3" -# Retry logic (Helicone-style) +# Retry logic backon = "0.4" # Optional exact-ish OpenAI tokenizer for context-window routing. diff --git a/crates/calciforge/src/config.rs b/crates/calciforge/src/config.rs index 9ff77271..9f319f3f 100644 --- a/crates/calciforge/src/config.rs +++ b/crates/calciforge/src/config.rs @@ -685,18 +685,19 @@ pub struct ProxyConfig { #[serde(default = "default_proxy_default_policy")] pub default_policy: ProxyAccessPolicy, - /// Legacy root provider adapter type for proxy: "http", "helicone", or "mock". + /// Legacy root provider adapter type for proxy. /// - /// "http" is Calciforge's minimal builtin OpenAI-compatible upstream - /// adapter. It is a compatibility path, not a provider-owned boundary with - /// its own registry or observability UI. + /// Supported values include "http", "helicone", "litellm", "portkey", + /// "tensorzero", "future-agi", "openrouter", and "mock". All non-mock + /// values use the same OpenAI-compatible HTTP core plus engine-specific + /// policy/metadata overlays. /// /// Provider-specific routes under `[[proxy.providers]]` have their own /// narrower `backend_type` surface. #[serde(default = "default_proxy_backend_type")] pub backend_type: String, - /// Optional operator UI URL for the selected gateway engine. + /// Optional operator UI URL for the selected provider adapter. /// /// External engines such as Helicone may expose their own dashboard. Engines /// without a built-in dashboard can point this at a lightweight Calciforge @@ -704,7 +705,7 @@ pub struct ProxyConfig { #[serde(default)] pub gateway_ui_url: Option, - /// Backend API key (for HTTP backend) + /// Root provider adapter API key. #[serde(default)] pub backend_api_key: Option, @@ -1044,13 +1045,12 @@ pub struct ProxyProviderConfig { /// Unique identifier for this provider (e.g. "kimi", "local-mlx"). pub id: String, - /// Provider adapter kind. "http" uses Calciforge's builtin - /// OpenAI-compatible HTTP transport. With `model_credential_owner = "provider"`, - /// that endpoint is treated as an external OpenAI-compatible gateway such - /// as LiteLLM. With the default `model_credential_owner = "calciforge"`, it is a - /// raw upstream-provider compatibility path. "helicone" forwards through a - /// Helicone AI Gateway with Helicone auth headers. CLI-backed - /// subscriptions are configured as `[[agents]]`, not gateway providers. + /// Provider adapter kind. Supported OpenAI-compatible engine overlays + /// include "http", "helicone", "litellm", "portkey", "tensorzero", + /// "future-agi", and "openrouter". They share the same request core; + /// the kind chooses engine metadata, dashboard capability, and any + /// provider-specific headers. CLI-backed subscriptions are configured as + /// `[[agents]]`, not gateway providers. #[serde(default = "default_proxy_provider_backend")] pub backend_type: String, diff --git a/crates/calciforge/src/config/validator.rs b/crates/calciforge/src/config/validator.rs index 11c50ee0..51064b4b 100644 --- a/crates/calciforge/src/config/validator.rs +++ b/crates/calciforge/src/config/validator.rs @@ -589,7 +589,12 @@ fn validate_proxy_config(proxy: &crate::config::ProxyConfig, result: &mut Valida ); } - if matches!(proxy.backend_type.as_str(), "http" | "helicone") + let root_gateway_type = proxy + .backend_type + .parse::() + .ok(); + + if root_gateway_type.is_some_and(|gateway_type| gateway_type.requires_backend_url()) && proxy.backend_url.trim().is_empty() { result.add_error(format!( @@ -623,14 +628,19 @@ fn validate_proxy_config(proxy: &crate::config::ProxyConfig, result: &mut Valida ); } - if proxy.backend_type == "helicone" { + if root_gateway_type.is_some_and(|gateway_type| gateway_type.requires_backend_url()) { let backend_url = proxy.backend_url.trim(); if backend_url.is_empty() { - result.add_error("Helicone backend_url cannot be blank".to_string()); + result.add_error(format!( + "Proxy backend_url cannot be blank for backend_type='{}'", + proxy.backend_type + )); } else { - validate_http_url("Helicone backend_url", backend_url, result, false); + validate_http_url("Proxy backend_url", backend_url, result, false); } + } + if proxy.backend_type == "helicone" { let has_inline_key = proxy .backend_api_key .as_deref() @@ -645,16 +655,34 @@ fn validate_proxy_config(proxy: &crate::config::ProxyConfig, result: &mut Valida } for provider in &proxy.providers { - match provider.backend_type.as_str() { - "http" | "helicone" => {} - "exec" => result.add_error(format!( + let provider_gateway_type = provider + .backend_type + .parse::() + .ok(); + if provider.backend_type == "exec" { + result.add_error(format!( "Proxy provider '{}' uses deprecated backend_type = \"exec\". CLI-backed subscriptions must be configured as [[agents]], not gateway providers.", provider.id - )), - other => result.add_error(format!( - "Proxy provider '{}' backend_type '{}' is invalid. Use: http, helicone", - provider.id, other - )), + )); + } else if !provider_gateway_type + .is_some_and(|gateway_type| gateway_type.uses_openai_compatible_http_core()) + { + result.add_error(format!( + "Proxy provider '{}' backend_type '{}' is invalid. Use one of: {}", + provider.id, + provider.backend_type, + crate::proxy::gateway::GatewayType::SUPPORTED_PROVIDER_CONFIG_NAMES.join(", ") + )); + } + + if provider_gateway_type + .is_some_and(|gateway_type| gateway_type.uses_openai_compatible_http_core()) + && provider.url.trim().is_empty() + { + result.add_error(format!( + "Proxy provider '{}' with backend_type='{}' requires url", + provider.id, provider.backend_type + )); } if provider.backend_type == "http" { @@ -862,1558 +890,14 @@ pub fn validate_config_file(path: &std::path::PathBuf) -> Result CalciforgeConfig { - toml::from_str(toml).expect("fixture should parse") - } - - /// Given a minimal config with no violations, - /// when validate_config runs, - /// then `is_valid()` is true. Positive baseline. - #[test] - fn baseline_minimum_config_validates_clean() { - let config = parse(MIN_VALID); - let result = validate_config(&config); - assert!( - result.is_valid(), - "baseline fixture should validate clean; errors: {:?}", - result.errors - ); - } - - #[test] - fn security_profile_validation_matches_runtime_parser() { - let invalid = parse(&format!("{MIN_VALID}\n[security]\nprofile = \"minimal\"\n")); - let invalid_result = validate_config(&invalid); - assert!( - invalid_result - .errors - .iter() - .any(|error| error.contains("Security profile 'minimal' is invalid")), - "unsupported profile must fail validation before runtime fallback; errors: {:?}", - invalid_result.errors - ); - - let maximum = parse(&format!("{MIN_VALID}\n[security]\nprofile = \"maximum\"\n")); - let maximum_result = validate_config(&maximum); - assert!( - maximum_result.is_valid(), - "maximum is a runtime-supported alias for paranoid; errors: {:?}", - maximum_result.errors - ); - } - - #[test] - fn model_shortcut_cycles_are_config_errors() { - let fixture = format!( - r#" -{MIN_VALID} - -[[model_shortcuts]] -alias = "local" -model = "balanced" - -[[model_shortcuts]] -alias = "balanced" -model = "local" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "cyclic model aliases should fail validation before runtime" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("model shortcut cycle") - && e.contains("local -> balanced -> local")), - "error should identify the shortcut cycle; errors: {:?}", - result.errors - ); - } - - #[test] - fn model_roles_share_shortcut_resolution_and_cycle_checks() { - let fixture = format!( - r#" -{MIN_VALID} - -[[model_roles]] -role = "fast" -model = "balanced" - -[[model_shortcuts]] -alias = "balanced" -model = "fast" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "roles must share shortcut cycle validation" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("model shortcut cycle") - && e.contains("fast -> balanced -> fast")), - "error should identify role/shortcut cycle; errors: {:?}", - result.errors - ); - } - - #[test] - fn duplicate_model_role_and_shortcut_names_are_config_errors() { - let fixture = format!( - r#" -{MIN_VALID} - -[[model_roles]] -role = "security.screening" -model = "local/qwen" - -[[model_shortcuts]] -alias = "security.screening" -model = "cloud/gpt" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "roles and shortcuts share one public selector namespace" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("Duplicate model shortcut alias or role") - && e.contains("security.screening")), - "error should identify duplicate role/shortcut name; errors: {:?}", - result.errors - ); - } - - #[test] - fn model_shortcut_alias_cannot_shadow_synthetic_model_id() { - let fixture = format!( - r#" -{MIN_VALID} - -[[dispatchers]] -id = "balanced" - -[[dispatchers.models]] -model = "qwen-test:small" -context_window = 60000 - -[[model_shortcuts]] -alias = "balanced" -model = "qwen-test:small" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "shortcut aliases must not silently shadow synthetic model IDs" - ); - assert!( - result.errors.iter().any(|e| e.contains("balanced") - && e.contains("Ambiguous model shortcut alias") - && e.contains("synthetic model ID")), - "error should identify the colliding alias and synthetic ID; errors: {:?}", - result.errors - ); - } - - #[test] - fn model_shortcut_alias_cannot_shadow_local_model_id() { - let fixture = format!( - r#" -{MIN_VALID} - -[local_models] -enabled = true - -[[local_models.models]] -id = "local" -hf_id = "example/local" - -[[model_shortcuts]] -alias = "local" -model = "qwen-test:small" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "shortcut aliases must not silently shadow local model IDs" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("local") - && e.contains("Ambiguous model shortcut alias") - && e.contains("local model ID") - }), - "error should identify the colliding alias and local model ID; errors: {:?}", - result.errors - ); - } - - #[test] - fn model_shortcut_alias_cannot_shadow_exact_provider_model_id() { - let fixture = format!( - r#" -{MIN_VALID} - -[[proxy.providers]] -id = "remote" -backend_type = "http" -url = "https://example.invalid/v1" -models = ["openai/gpt-5.5", "openai/*"] - -[[model_shortcuts]] -alias = "openai/gpt-5.5" -model = "kimi-cli" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "shortcut aliases must not silently shadow exact provider model IDs" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("openai/gpt-5.5") - && e.contains("Ambiguous model shortcut alias") - && e.contains("provider model ID") - }), - "error should identify the colliding alias and provider model ID; errors: {:?}", - result.errors - ); - } - - #[test] - fn provider_strip_model_prefix_must_not_be_empty() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "opencode-go" -backend_type = "http" -url = "https://opencode.ai/zen/go/v1" -models = ["opencode-go/kimi-k2.6"] -strip_model_prefix = " " -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "empty strip_model_prefix should fail validation" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("strip_model_prefix cannot be empty")), - "error should mention strip_model_prefix; errors: {:?}", - result.errors - ); - } - - #[test] - fn provider_strip_model_prefix_warns_when_no_models_use_it() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "opencode-go" -backend_type = "http" -url = "https://opencode.ai/zen/go/v1" -api_key = "test-provider-key" -models = ["kimi-k2.6"] -strip_model_prefix = "opencode-go/" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - result.is_valid(), - "mismatched strip prefix should warn, not block unrelated direct models" - ); - assert!( - result - .warnings - .iter() - .any(|w| w.contains("strips model prefix")), - "warning should identify useless strip_model_prefix; warnings: {:?}", - result.warnings - ); - } - - #[test] - fn gateway_retry_config_rejects_invalid_backoff_bounds() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[proxy.retry] -enabled = true -min_timeout_ms = 2000 -max_timeout_ms = 1000 -factor = 0 -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "invalid retry timing should fail validation before runtime" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("min_timeout_ms") && e.contains("max_timeout_ms")), - "error should identify inverted retry bounds; errors: {:?}", - result.errors - ); - assert!( - result.errors.iter().any(|e| e.contains("factor")), - "error should reject zero retry factor; errors: {:?}", - result.errors - ); - } - - #[test] - fn provider_owned_provider_key_is_endpoint_auth_warning_not_error() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "managed-gateway" -backend_type = "http" -url = "http://127.0.0.1:4000/v1" -model_credential_owner = "provider" -api_key = "sk-local-gateway-client" -models = ["gateway/default"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - result.is_valid(), - "provider-owned provider keys should be a supported config shape; errors: {:?}", - result.errors - ); - assert!( - result.warnings.iter().any(|w| { - w.contains("managed-gateway") - && w.contains("model_credential_owner='provider'") - && w.contains("provider boundary endpoint") - }), - "warning should clarify api_key is provider-boundary transport auth; warnings: {:?}", - result.warnings - ); - assert!( - result.warnings.iter().any(|w| { - w.contains("managed-gateway") - && w.contains("provider-owned OpenAI-compatible boundary") - && w.contains("LiteLLM") - }), - "warning should distinguish provider-owned HTTP endpoints from raw upstream providers; warnings: {:?}", - result.warnings - ); - } - - #[test] - fn builtin_http_provider_warns_that_it_is_not_provider_owned_boundary() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true -backend_type = "helicone" -backend_url = "http://127.0.0.1:8787/ai" - -[[proxy.providers]] -id = "raw-upstream" -backend_type = "http" -url = "https://example.invalid/v1" -api_key = "test-provider-key" -models = ["raw/model"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - result.is_valid(), - "builtin HTTP providers remain supported but should be explicit; errors: {:?}", - result.errors - ); - assert!( - result.warnings.iter().any(|w| { - w.contains("raw-upstream") - && w.contains("builtin HTTP upstream adapter") - && w.contains("not a provider-owned boundary") - }), - "warning should prevent treating raw HTTP provider routes as equal to Helicone/LiteLLM; warnings: {:?}", - result.warnings - ); - } - - #[test] - fn mock_proxy_backend_warns_for_real_deployments() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true -backend_type = "mock" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!(!result.is_valid(), "mock without providers must fail now"); - assert!( - result - .errors - .iter() - .any(|e| e.contains("mock provider adapter")), - "error should make mock/no-provider invalid; errors: {:?}", - result.errors - ); - assert!( - result - .warnings - .iter() - .any(|w| w.contains("backend_type='mock'") && w.contains("test-only")), - "warning should keep mock out of production configs; warnings: {:?}", - result.warnings - ); - } - - #[test] - fn explicit_providers_do_not_make_blank_http_root_valid() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true -backend_type = "http" -backend_url = "" - -[[proxy.providers]] -id = "managed" -backend_type = "http" -url = "http://127.0.0.1:4000/v1" -model_credential_owner = "provider" -models = ["managed/default"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "blank root HTTP backend should be invalid even when providers are configured" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("backend_type='http'") && e.contains("requires backend_url")), - "error should identify blank root backend_url; errors: {:?}", - result.errors - ); - } - - #[test] - fn calciforge_owned_provider_requires_key_or_file() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "raw-upstream" -backend_type = "http" -url = "https://example.invalid/v1" -model_credential_owner = "calciforge" -models = ["raw/model"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "Calciforge-owned provider credentials must be explicit" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("raw-upstream") - && e.contains("model_credential_owner='calciforge'") - && e.contains("model_api_key/model_api_key_file") - }), - "error should identify provider and missing credential; errors: {:?}", - result.errors - ); - } - - #[test] - fn calciforge_owned_provider_accepts_model_key_without_endpoint_key() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "raw-upstream" -backend_type = "http" -url = "https://example.invalid/v1" -model_credential_owner = "calciforge" -model_api_key = "upstream-model-key" -models = ["raw/model"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - result.is_valid(), - "separate provider-auth and model-auth credentials should be valid; errors: {:?}", - result.errors - ); - assert!( - !result - .warnings - .iter() - .any(|w| w.contains("legacy api_key/api_key_file as both")), - "explicit model credentials should avoid legacy dual-use warning; warnings: {:?}", - result.warnings - ); - } - - #[test] - fn calciforge_owned_provider_rejects_two_bearer_credentials() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "raw-upstream" -backend_type = "http" -url = "https://example.invalid/v1" -api_key = "endpoint-virtual-key" -model_credential_owner = "calciforge" -model_api_key = "upstream-model-key" -models = ["raw/model"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "current provider adapters cannot send two independent bearer credentials" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("raw-upstream") - && e.contains("both provider adapter auth") - && e.contains("one bearer credential") - }), - "error should identify the two-credential conflict; errors: {:?}", - result.errors - ); - } - - #[test] - fn provider_owned_model_credentials_reject_model_key() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "managed" -backend_type = "http" -url = "https://managed.example.invalid/v1" -api_key = "adapter-client-key" -model_credential_owner = "provider" -model_api_key = "wrong-place" -models = ["managed/default"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "provider-owned final model credentials must not also configure model_api_key" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("managed") && e.contains("model_api_key")), - "error should identify provider-owned/model-key conflict; errors: {:?}", - result.errors - ); - } - - #[test] - fn deprecated_model_credential_owner_none_aliases_to_provider() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true - -[[proxy.providers]] -id = "local" -backend_type = "http" -url = "http://127.0.0.1:11434/v1" -model_credential_owner = "none" -models = ["ollama/qwen3.6:27b"] -"# - ); - let config = parse(&fixture); - let owner = config - .proxy - .as_ref() - .and_then(|proxy| proxy.providers.first()) - .map(|provider| provider.model_credential_owner); - assert_eq!(owner, Some(CredentialOwner::Provider)); - - let result = validate_config(&config); - - assert!( - result.is_valid(), - "deprecated model_credential_owner='none' should parse as provider-owned/no Calciforge model credentials" - ); - } - - #[test] - fn model_shortcut_alias_cannot_shadow_exact_model_route_id() { - let fixture = format!( - r#" -{MIN_VALID} - -[[proxy.providers]] -id = "remote" -backend_type = "http" -url = "https://example.invalid/v1" - -[[proxy.model_routes]] -pattern = "coding/default" -provider = "remote" - -[[model_shortcuts]] -alias = "coding/default" -model = "kimi-cli" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "shortcut aliases must not silently shadow exact model-route IDs" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("coding/default") - && e.contains("Ambiguous model shortcut alias") - && e.contains("model-route ID") - }), - "error should identify the colliding alias and model-route ID; errors: {:?}", - result.errors - ); - } - - #[test] - fn exact_model_route_cannot_shadow_agent_selector() { - let fixture = format!( - r#" -{MIN_VALID} - -[[agents]] -id = "local-dispatcher" -kind = "openai-compat" -endpoint = "http://127.0.0.1:18083" -model = "local-cloud-coding" - -[[proxy.providers]] -id = "remote" -backend_type = "http" -url = "https://example.invalid/v1" - -[[proxy.model_routes]] -pattern = "local-dispatcher" -provider = "remote" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "exact model routes must not silently treat agent names as concrete gateway models" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("local-dispatcher") - && e.contains("model-route ID") - && e.contains("agent ID") - }), - "error should identify the model route / agent selector collision; errors: {:?}", - result.errors - ); - } - - #[test] - fn provider_model_id_cannot_shadow_agent_alias() { - let fixture = format!( - r#" -{MIN_VALID} - -[[agents]] -id = "gateway-agent" -kind = "openai-compat" -endpoint = "http://127.0.0.1:18083" -model = "local-cloud-coding" -aliases = ["premium"] - -[[proxy.providers]] -id = "remote" -backend_type = "http" -url = "https://example.invalid/v1" -models = ["premium"] -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "exact provider model IDs must not silently reuse agent aliases" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("premium") - && e.contains("provider model ID") - && e.contains("agent alias") - }), - "error should identify the provider model / agent alias collision; errors: {:?}", - result.errors - ); - } - - #[test] - fn model_shortcut_alias_cannot_shadow_agent_selector() { - let fixture = format!( - r#" -{MIN_VALID} - -[[agents]] -id = "gateway-agent" -kind = "openai-compat" -endpoint = "http://127.0.0.1:18083" -model = "local-cloud-coding" -aliases = ["premium"] - -[[model_shortcuts]] -alias = "premium" -model = "openai/gpt-5.5" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "model shortcut aliases must not silently reuse agent selectors" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("premium") - && e.contains("model shortcut alias") - && e.contains("agent alias") - }), - "error should identify shortcut / agent alias collision; errors: {:?}", - result.errors - ); - } - - #[test] - fn exec_models_are_deprecated_config_errors() { - let fixture = format!( - r#" -{MIN_VALID} - -[[exec_models]] -id = "codex/gpt-5.5" -context_window = 262144 -command = "codex" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "deprecated exec model shims should not silently register as gateway models" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("[[exec_models]]") - && e.contains("deprecated") - && e.contains("kind = \"exec\"") - }), - "error should explain the agent migration path; errors: {:?}", - result.errors - ); - } - - #[test] - fn exec_proxy_providers_are_deprecated_config_errors() { - let fixture = format!( - r#" -{MIN_VALID} - -[proxy] -enabled = true -bind = "127.0.0.1:18083" - -[[proxy.providers]] -id = "codex-cli" -backend_type = "exec" -models = ["codex/gpt-5.5"] -command = "codex" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "exec proxy providers must not silently register as gateway models" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("codex-cli") - && e.contains("backend_type = \"exec\"") - && e.contains("[[agents]]") - }), - "error should explain that CLI-backed subscriptions are agents; errors: {:?}", - result.errors - ); - } - - #[test] - fn duplicate_first_class_model_ids_are_config_errors() { - let fixture = format!( - r#" -{MIN_VALID} - -[[dispatchers]] -id = "balanced" - -[[dispatchers.models]] -model = "qwen-test:small" -context_window = 60000 - -[[proxy.model_routes]] -pattern = "balanced" -provider = "missing-provider" -"# - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "configured first-class model IDs must not collide across namespaces" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("Ambiguous model selector 'balanced'")), - "error should identify duplicate first-class model ID; errors: {:?}", - result.errors - ); - } - - #[test] - fn whatsapp_legacy_webhook_fields_are_config_errors() { - let fixture = r#" -[calciforge] -version = 2 - -[[channels]] -kind = "whatsapp" -enabled = true -whatsapp_session_path = "/tmp/calciforge-wa.db" -webhook_listen = "0.0.0.0:18795" -webhook_path = "/webhooks/whatsapp" -zeroclaw_endpoint = "http://127.0.0.1:18796" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "legacy webhook fields must fail before startup" - ); - assert!( - result.errors.iter().any(|e| e.contains("WhatsApp") - && e.contains("webhook_listen") - && e.contains("embedded channel schema")), - "error should explain the embedded-channel migration; errors: {:?}", - result.errors - ); - } - - #[test] - fn enabled_whatsapp_requires_session_path() { - let fixture = r#" -[calciforge] -version = 2 - -[[channels]] -kind = "whatsapp" -enabled = true -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "enabled WhatsApp without session storage should fail" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("whatsapp_session_path")), - "error should name whatsapp_session_path; errors: {:?}", - result.errors - ); - } - - #[test] - fn signal_legacy_webhook_fields_are_config_errors() { - let fixture = r#" -[calciforge] -version = 2 - -[[channels]] -kind = "signal" -enabled = true -signal_cli_url = "http://127.0.0.1:8080" -signal_account = "+15555550001" -webhook_path = "/webhooks/signal" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "legacy webhook fields must fail before startup" - ); - assert!( - result.errors.iter().any(|e| e.contains("Signal") - && e.contains("webhook_path") - && e.contains("embedded channel schema")), - "error should explain the embedded-channel migration; errors: {:?}", - result.errors - ); - } - - #[test] - fn removed_openclaw_http_agent_is_an_error() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "custodian" -kind = "openclaw-http" -endpoint = "http://127.0.0.1:18789" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!(!result.is_valid(), "openclaw-http must fail validation"); - assert!( - result - .errors - .iter() - .any(|e| e.contains("openclaw-http") && e.contains("openclaw-channel")), - "error should name the removed kind and migration target; errors: {:?}", - result.errors - ); - } - - #[test] - fn openclaw_channel_agent_validates_with_callback_auth() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "custodian" -kind = "openclaw-channel" -endpoint = "http://127.0.0.1:18789" -api_key = "test-gateway-token" -reply_auth_token = "test-reply-token" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - result.is_valid(), - "openclaw-channel should validate; errors: {:?}", - result.errors - ); - } - - #[test] - fn openclaw_channel_agent_validates_with_callback_auth_file() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "custodian" -kind = "openclaw-channel" -endpoint = "http://127.0.0.1:18789" -api_key = "test-gateway-token" -reply_auth_token_file = "/tmp/calciforge-test-reply-token" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - result.is_valid(), - "openclaw-channel should validate; errors: {:?}", - result.errors - ); - } - - #[test] - fn openai_compat_agent_validates() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "gateway" -kind = "openai-compat" -endpoint = "http://127.0.0.1:8083" -api_key = "test-gateway-token" -model = "local-kimi-gpt55" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - result.is_valid(), - "openai-compat should validate; errors: {:?}", - result.errors - ); - } - - #[test] - fn openai_compat_rejects_openclaw_model_ids() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "librarian" -kind = "openai-compat" -endpoint = "http://127.0.0.1:18789" -api_key = "test-gateway-token" -model = "openclaw/main" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "OpenClaw model IDs should require openclaw-channel" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("OpenClaw") && e.contains("openclaw-channel")), - "error should point to openclaw-channel; errors: {:?}", - result.errors - ); - } - - #[test] - fn openai_compat_without_model_requires_override_opt_in() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "gateway" -kind = "openai-compat" -endpoint = "http://127.0.0.1:8083" -api_key = "test-gateway-token" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "openai-compat without model or allow_model_override must fail" - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("allow_model_override")), - "error should mention allow_model_override; errors: {:?}", - result.errors - ); - } - - #[test] - fn openai_compat_without_model_validates_when_override_is_explicit() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "gateway" -kind = "openai-compat" -endpoint = "http://127.0.0.1:8083" -api_key = "test-gateway-token" -allow_model_override = true -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - result.is_valid(), - "openai-compat with explicit model override should validate; errors: {:?}", - result.errors - ); - } - - #[test] - fn zeroclaw_agent_requires_api_key_or_file() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "librarianzero" -kind = "zeroclaw" -endpoint = "http://127.0.0.1:18799" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!(!result.is_valid(), "zeroclaw without key must fail"); - assert!( - result.errors.iter().any(|e| e.contains("api_key")), - "error should mention missing api_key/api_key_file; errors: {:?}", - result.errors - ); - } - - #[test] - fn hermes_without_auth_fails_before_runtime() { - let fixture = r#" -[calciforge] -version = 2 - -[[agents]] -id = "hermes" -kind = "hermes" -endpoint = "http://127.0.0.1:8642" -"#; - let config = parse(fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "Hermes config without auth should fail before adapter construction; errors: {:?}", - result.errors - ); - assert!( - result - .errors - .iter() - .any(|error| error.contains("hermes") && error.contains("api_key")), - "error should mention missing auth for Hermes; errors: {:?}", - result.errors - ); - } - - /// Given a config with two agents sharing the same id, - /// when validate_config runs, - /// then an error naming the duplicated id is produced. - #[test] - fn duplicate_agent_id_is_an_error() { - let fixture = format!( - "{MIN_VALID}\n[[agents]]\nid = \"bot\"\nkind = \"cli\"\ncommand = \"/bin/echo\"\nargs = []\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!(!result.is_valid(), "duplicate agent id must fail"); - assert!( - result.errors.iter().any(|e| e.contains("bot")), - "error must name the duplicated id 'bot'; errors: {:?}", - result.errors - ); - } - - #[test] - fn agent_alias_cannot_shadow_another_agent_id() { - let fixture = format!( - "{MIN_VALID}\n[[agents]]\nid = \"helper\"\nkind = \"cli\"\ncommand = \"/bin/echo\"\nargs = []\naliases = [\"bot\"]\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "agent aliases must not silently shadow another agent id" - ); - assert!( - result.errors.iter().any(|e| { - e.contains("Ambiguous agent selector 'bot'") - && e.contains("agent 'bot'") - && e.contains("agent 'helper'") - }), - "error should identify both agent selector owners; errors: {:?}", - result.errors - ); - } - - /// Given a config with two identities sharing the same id, - /// when validate_config runs, - /// then an error naming the duplicated id is produced. - #[test] - fn duplicate_identity_id_is_an_error() { - let fixture = format!( - "{MIN_VALID}\n[[identities]]\nid = \"alice\"\naliases = [{{ channel = \"signal\", id = \"7000000099\" }}]\nrole = \"user\"\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!(!result.is_valid(), "duplicate identity id must fail"); - assert!( - result.errors.iter().any(|e| e.contains("alice")), - "error must name the duplicated id 'alice'; errors: {:?}", - result.errors - ); - } - - /// Given a proxy config with an unparseable bind address, - /// when validate_config runs, - /// then an error is produced naming the bind/address problem. - #[test] - fn malformed_proxy_bind_is_an_error() { - let fixture = format!( - "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"not-an-address\"\nbackend_type = \"http\"\nbackend_url = \"https://api.example.com\"\ntimeout_seconds = 10\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "invalid bind address must fail; errors: {:?}", - result.errors - ); - assert!( - result.errors.iter().any(|e| { - let lower = e.to_lowercase(); - lower.contains("bind") || lower.contains("address") - }), - "error should name the bind/address problem; errors: {:?}", - result.errors - ); - } - - /// Given a proxy config with `timeout_seconds = 0`, - /// when validate_config runs, - /// then an error is produced — a zero timeout means requests - /// hang indefinitely, which is never the intent. - #[test] - fn zero_proxy_timeout_is_an_error() { - let fixture = format!( - "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"http\"\nbackend_url = \"https://api.example.com\"\ntimeout_seconds = 0\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "zero proxy timeout must fail; errors: {:?}", - result.errors - ); - } - - /// Given a proxy config with a gateway UI link that chat users may open, - /// when validate_config runs, - /// then non-HTTP links are rejected before they appear in `!help`. - #[test] - fn gateway_ui_url_requires_http_url() { - let fixture = format!( - "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"http\"\nbackend_url = \"https://api.example.com\"\ngateway_ui_url = \"localhost:8585\"\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "gateway UI URL without scheme must fail; errors: {:?}", - result.errors - ); - assert!( - result.errors.iter().any(|e| e.contains("gateway_ui_url")), - "error should name gateway_ui_url; errors: {:?}", - result.errors - ); - } - - /// Given a proxy backend type outside the runtime allow-list, - /// when validate_config runs, - /// then validation rejects it and reports the supported values. - #[test] - fn unsupported_proxy_backend_type_is_rejected_by_allowlist() { - let backend_type = "experimental-gateway"; - let fixture = format!( - "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"{backend_type}\"\nbackend_url = \"https://api.example.com\"\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "{backend_type} must not validate as a supported proxy backend; errors: {:?}", - result.errors - ); - assert!( - result.errors.iter().any(|e| { - e.contains("backend_type") && e.contains(backend_type) && e.contains("http") - }), - "error should name unsupported backend and supported values; errors: {:?}", - result.errors - ); - } - - /// Given a disabled proxy with a configured gateway UI link, - /// when validate_config runs, - /// then the UI URL is still validated because chat help can surface it from - /// config even when the model gateway listener is disabled. - #[test] - fn disabled_proxy_still_validates_gateway_ui_url() { - let fixture = format!( - "{MIN_VALID}\n[proxy]\nenabled = false\ngateway_ui_url = \"javascript:alert(1)\"\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "disabled proxy must still validate displayed gateway UI URLs; errors: {:?}", - result.errors - ); - assert!( - result.errors.iter().any(|e| e.contains("gateway_ui_url")), - "error should name gateway_ui_url; errors: {:?}", - result.errors - ); - } - - /// Given Helicone is selected as the model gateway engine, - /// when the backend URL is malformed or contains request modifiers, - /// then validation rejects it before runtime path construction can fail. - #[test] - fn helicone_backend_url_must_be_plain_http_base_url() { - let fixture = format!( - "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"helicone\"\nbackend_url = \"https://ai-gateway.helicone.ai/v1?debug=true\"\nbackend_api_key = \"test-key\"\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - !result.is_valid(), - "Helicone backend URL with query string must fail; errors: {:?}", - result.errors - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("Helicone backend_url") && e.contains("query")), - "error should name Helicone backend_url and query/fragment issue; errors: {:?}", - result.errors - ); - } - - /// Given Helicone is selected for a likely local unauthenticated gateway, - /// when no backend key is configured, - /// then validation warns instead of silently accepting a surprising empty - /// `Authorization: Bearer` header. - #[test] - fn helicone_without_backend_key_warns_operator() { - let fixture = format!( - "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"helicone\"\nbackend_url = \"http://127.0.0.1:8787\"\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - - assert!( - result.is_valid(), - "local unauthenticated Helicone should remain possible: {:?}", - result.errors - ); - assert!( - result - .warnings - .iter() - .any(|w| w.contains("Helicone backend has no backend_api_key")), - "missing Helicone key should produce an operator warning; warnings: {:?}", - result.warnings - ); - } - - /// Given a routing rule referencing an agent id that doesn't - /// exist in the agent list, - /// when validate_config runs, - /// then an error naming the missing agent is produced. - /// - /// Catches the most common cause of silent "agent unavailable" - /// at runtime: typo in a routing rule. - #[test] - fn routing_rule_default_to_nonexistent_agent_is_an_error() { - let fixture = - format!("{MIN_VALID}\n[[routing]]\nidentity = \"alice\"\ndefault_agent = \"ghost\"\n"); - let config = parse(&fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "routing default_agent pointing at a non-existent agent must fail; \ - errors: {:?}", - result.errors - ); - assert!( - result.errors.iter().any(|e| e.contains("ghost")), - "error should name the missing agent id 'ghost'; errors: {:?}", - result.errors - ); - } - - /// Given a routing rule whose `default_agent` is valid but whose - /// `allowed_agents` list contains an id not in the agent list, - /// when validate_config runs, - /// then an error naming both the identity and the missing agent - /// is produced. - /// - /// Validates the branch in `validate_routing_rules` that walks - /// each entry of `allowed_agents` — a test for - /// `default_agent` alone wouldn't exercise this code path. - #[test] - fn routing_rule_allowed_list_with_nonexistent_agent_is_an_error() { - let fixture = format!( - "{MIN_VALID}\n[[routing]]\nidentity = \"alice\"\ndefault_agent = \"bot\"\nallowed_agents = [\"bot\", \"ghost\"]\n" - ); - let config = parse(&fixture); - let result = validate_config(&config); - assert!( - !result.is_valid(), - "allowed_agents pointing at a non-existent agent must fail; \ - errors: {:?}", - result.errors - ); - assert!( - result - .errors - .iter() - .any(|e| e.contains("ghost") && e.contains("alice")), - "error should name both the identity 'alice' and missing agent \ - 'ghost'; errors: {:?}", - result.errors - ); - } -} +#[path = "validator_test_support.rs"] +mod validator_test_support; +#[cfg(test)] +#[path = "validator_tests_1.rs"] +mod validator_tests_1; +#[cfg(test)] +#[path = "validator_tests_2.rs"] +mod validator_tests_2; +#[cfg(test)] +#[path = "validator_tests_3.rs"] +mod validator_tests_3; diff --git a/crates/calciforge/src/config/validator_test_support.rs b/crates/calciforge/src/config/validator_test_support.rs new file mode 100644 index 00000000..4b241c3e --- /dev/null +++ b/crates/calciforge/src/config/validator_test_support.rs @@ -0,0 +1,38 @@ +// Behavioral tests for `validate_config`. Round-2 test quality +// audit (2026-04-24) flagged this module as having zero tests +// despite ~290 lines of validation logic. These tests close the +// most important invariants (duplicates, dangling references, +// out-of-range fields) so future refactors can't silently regress. +use crate::config::CalciforgeConfig; + +/// Minimal TOML that passes validation. Each negative test +/// derives from this by prepending/appending ONE targeted +/// deviation so the failing invariant is the only difference +/// between the valid fixture and the test under test. +pub(super) const MIN_VALID: &str = r#" +[calciforge] +version = 2 + +[context] +buffer_size = 20 +inject_depth = 5 + +[[identities]] +id = "alice" +aliases = [{ channel = "telegram", id = "7000000001" }] +role = "owner" + +[[agents]] +id = "bot" +kind = "cli" +command = "/bin/echo" +args = [] + +[[channels]] +kind = "telegram" +bot_token_file = "/tmp/nope" +"#; + +pub(super) fn parse(toml: &str) -> CalciforgeConfig { + toml::from_str(toml).expect("fixture should parse") +} diff --git a/crates/calciforge/src/config/validator_tests_1.rs b/crates/calciforge/src/config/validator_tests_1.rs new file mode 100644 index 00000000..dd67d053 --- /dev/null +++ b/crates/calciforge/src/config/validator_tests_1.rs @@ -0,0 +1,501 @@ +use super::validator_test_support::{MIN_VALID, parse}; +use super::*; + +/// Given a minimal config with no violations, +/// when validate_config runs, +/// then `is_valid()` is true. Positive baseline. +#[test] +fn baseline_minimum_config_validates_clean() { + let config = parse(MIN_VALID); + let result = validate_config(&config); + assert!( + result.is_valid(), + "baseline fixture should validate clean; errors: {:?}", + result.errors + ); +} + +#[test] +fn security_profile_validation_matches_runtime_parser() { + let invalid = parse(&format!("{MIN_VALID}\n[security]\nprofile = \"minimal\"\n")); + let invalid_result = validate_config(&invalid); + assert!( + invalid_result + .errors + .iter() + .any(|error| error.contains("Security profile 'minimal' is invalid")), + "unsupported profile must fail validation before runtime fallback; errors: {:?}", + invalid_result.errors + ); + + let maximum = parse(&format!("{MIN_VALID}\n[security]\nprofile = \"maximum\"\n")); + let maximum_result = validate_config(&maximum); + assert!( + maximum_result.is_valid(), + "maximum is a runtime-supported alias for paranoid; errors: {:?}", + maximum_result.errors + ); +} + +#[test] +fn model_shortcut_cycles_are_config_errors() { + let fixture = format!( + r#" +{MIN_VALID} + +[[model_shortcuts]] +alias = "local" +model = "balanced" + +[[model_shortcuts]] +alias = "balanced" +model = "local" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "cyclic model aliases should fail validation before runtime" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("model shortcut cycle") + && e.contains("local -> balanced -> local")), + "error should identify the shortcut cycle; errors: {:?}", + result.errors + ); +} + +#[test] +fn model_roles_share_shortcut_resolution_and_cycle_checks() { + let fixture = format!( + r#" +{MIN_VALID} + +[[model_roles]] +role = "fast" +model = "balanced" + +[[model_shortcuts]] +alias = "balanced" +model = "fast" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "roles must share shortcut cycle validation" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("model shortcut cycle") && e.contains("fast -> balanced -> fast")), + "error should identify role/shortcut cycle; errors: {:?}", + result.errors + ); +} + +#[test] +fn duplicate_model_role_and_shortcut_names_are_config_errors() { + let fixture = format!( + r#" +{MIN_VALID} + +[[model_roles]] +role = "security.screening" +model = "local/qwen" + +[[model_shortcuts]] +alias = "security.screening" +model = "cloud/gpt" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "roles and shortcuts share one public selector namespace" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("Duplicate model shortcut alias or role") + && e.contains("security.screening")), + "error should identify duplicate role/shortcut name; errors: {:?}", + result.errors + ); +} + +#[test] +fn model_shortcut_alias_cannot_shadow_synthetic_model_id() { + let fixture = format!( + r#" +{MIN_VALID} + +[[dispatchers]] +id = "balanced" + +[[dispatchers.models]] +model = "qwen-test:small" +context_window = 60000 + +[[model_shortcuts]] +alias = "balanced" +model = "qwen-test:small" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "shortcut aliases must not silently shadow synthetic model IDs" + ); + assert!( + result.errors.iter().any(|e| e.contains("balanced") + && e.contains("Ambiguous model shortcut alias") + && e.contains("synthetic model ID")), + "error should identify the colliding alias and synthetic ID; errors: {:?}", + result.errors + ); +} + +#[test] +fn model_shortcut_alias_cannot_shadow_local_model_id() { + let fixture = format!( + r#" +{MIN_VALID} + +[local_models] +enabled = true + +[[local_models.models]] +id = "local" +hf_id = "example/local" + +[[model_shortcuts]] +alias = "local" +model = "qwen-test:small" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "shortcut aliases must not silently shadow local model IDs" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("local") + && e.contains("Ambiguous model shortcut alias") + && e.contains("local model ID") + }), + "error should identify the colliding alias and local model ID; errors: {:?}", + result.errors + ); +} + +#[test] +fn model_shortcut_alias_cannot_shadow_exact_provider_model_id() { + let fixture = format!( + r#" +{MIN_VALID} + +[[proxy.providers]] +id = "remote" +backend_type = "http" +url = "https://example.invalid/v1" +models = ["openai/gpt-5.5", "openai/*"] + +[[model_shortcuts]] +alias = "openai/gpt-5.5" +model = "kimi-cli" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "shortcut aliases must not silently shadow exact provider model IDs" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("openai/gpt-5.5") + && e.contains("Ambiguous model shortcut alias") + && e.contains("provider model ID") + }), + "error should identify the colliding alias and provider model ID; errors: {:?}", + result.errors + ); +} + +#[test] +fn provider_strip_model_prefix_must_not_be_empty() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "opencode-go" +backend_type = "http" +url = "https://opencode.ai/zen/go/v1" +models = ["opencode-go/kimi-k2.6"] +strip_model_prefix = " " +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "empty strip_model_prefix should fail validation" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("strip_model_prefix cannot be empty")), + "error should mention strip_model_prefix; errors: {:?}", + result.errors + ); +} + +#[test] +fn provider_strip_model_prefix_warns_when_no_models_use_it() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "opencode-go" +backend_type = "http" +url = "https://opencode.ai/zen/go/v1" +api_key = "test-provider-key" +models = ["kimi-k2.6"] +strip_model_prefix = "opencode-go/" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + result.is_valid(), + "mismatched strip prefix should warn, not block unrelated direct models" + ); + assert!( + result + .warnings + .iter() + .any(|w| w.contains("strips model prefix")), + "warning should identify useless strip_model_prefix; warnings: {:?}", + result.warnings + ); +} + +#[test] +fn gateway_retry_config_rejects_invalid_backoff_bounds() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[proxy.retry] +enabled = true +min_timeout_ms = 2000 +max_timeout_ms = 1000 +factor = 0 +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "invalid retry timing should fail validation before runtime" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("min_timeout_ms") && e.contains("max_timeout_ms")), + "error should identify inverted retry bounds; errors: {:?}", + result.errors + ); + assert!( + result.errors.iter().any(|e| e.contains("factor")), + "error should reject zero retry factor; errors: {:?}", + result.errors + ); +} + +#[test] +fn provider_owned_provider_key_is_endpoint_auth_warning_not_error() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "managed-gateway" +backend_type = "http" +url = "http://127.0.0.1:4000/v1" +model_credential_owner = "provider" +api_key = "sk-local-gateway-client" +models = ["gateway/default"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + result.is_valid(), + "provider-owned provider keys should be a supported config shape; errors: {:?}", + result.errors + ); + assert!( + result.warnings.iter().any(|w| { + w.contains("managed-gateway") + && w.contains("model_credential_owner='provider'") + && w.contains("provider boundary endpoint") + }), + "warning should clarify api_key is provider-boundary transport auth; warnings: {:?}", + result.warnings + ); + assert!( + result.warnings.iter().any(|w| { + w.contains("managed-gateway") + && w.contains("provider-owned OpenAI-compatible boundary") + && w.contains("LiteLLM") + }), + "warning should distinguish provider-owned HTTP endpoints from raw upstream providers; warnings: {:?}", + result.warnings + ); +} + +#[test] +fn builtin_http_provider_warns_that_it_is_not_provider_owned_boundary() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true +backend_type = "helicone" +backend_url = "http://127.0.0.1:8787/ai" + +[[proxy.providers]] +id = "raw-upstream" +backend_type = "http" +url = "https://example.invalid/v1" +api_key = "test-provider-key" +models = ["raw/model"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + result.is_valid(), + "builtin HTTP providers remain supported but should be explicit; errors: {:?}", + result.errors + ); + assert!( + result.warnings.iter().any(|w| { + w.contains("raw-upstream") + && w.contains("builtin HTTP upstream adapter") + && w.contains("not a provider-owned boundary") + }), + "warning should prevent treating raw HTTP provider routes as equal to Helicone/LiteLLM; warnings: {:?}", + result.warnings + ); +} + +#[test] +fn mock_proxy_backend_warns_for_real_deployments() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true +backend_type = "mock" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!(!result.is_valid(), "mock without providers must fail now"); + assert!( + result + .errors + .iter() + .any(|e| e.contains("mock provider adapter")), + "error should make mock/no-provider invalid; errors: {:?}", + result.errors + ); + assert!( + result + .warnings + .iter() + .any(|w| w.contains("backend_type='mock'") && w.contains("test-only")), + "warning should keep mock out of production configs; warnings: {:?}", + result.warnings + ); +} + +#[test] +fn explicit_providers_do_not_make_blank_http_root_valid() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true +backend_type = "http" +backend_url = "" + +[[proxy.providers]] +id = "managed" +backend_type = "http" +url = "http://127.0.0.1:4000/v1" +model_credential_owner = "provider" +models = ["managed/default"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "blank root HTTP backend should be invalid even when providers are configured" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("backend_type='http'") && e.contains("requires backend_url")), + "error should identify blank root backend_url; errors: {:?}", + result.errors + ); +} diff --git a/crates/calciforge/src/config/validator_tests_2.rs b/crates/calciforge/src/config/validator_tests_2.rs new file mode 100644 index 00000000..713e11ec --- /dev/null +++ b/crates/calciforge/src/config/validator_tests_2.rs @@ -0,0 +1,508 @@ +use super::validator_test_support::{MIN_VALID, parse}; +use super::*; + +#[test] +fn calciforge_owned_provider_requires_key_or_file() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "raw-upstream" +backend_type = "http" +url = "https://example.invalid/v1" +model_credential_owner = "calciforge" +models = ["raw/model"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "Calciforge-owned provider credentials must be explicit" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("raw-upstream") + && e.contains("model_credential_owner='calciforge'") + && e.contains("model_api_key/model_api_key_file") + }), + "error should identify provider and missing credential; errors: {:?}", + result.errors + ); +} + +#[test] +fn calciforge_owned_provider_accepts_model_key_without_endpoint_key() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "raw-upstream" +backend_type = "http" +url = "https://example.invalid/v1" +model_credential_owner = "calciforge" +model_api_key = "upstream-model-key" +models = ["raw/model"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + result.is_valid(), + "separate provider-auth and model-auth credentials should be valid; errors: {:?}", + result.errors + ); + assert!( + !result + .warnings + .iter() + .any(|w| w.contains("legacy api_key/api_key_file as both")), + "explicit model credentials should avoid legacy dual-use warning; warnings: {:?}", + result.warnings + ); +} + +#[test] +fn calciforge_owned_provider_rejects_two_bearer_credentials() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "raw-upstream" +backend_type = "http" +url = "https://example.invalid/v1" +api_key = "endpoint-virtual-key" +model_credential_owner = "calciforge" +model_api_key = "upstream-model-key" +models = ["raw/model"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "current provider adapters cannot send two independent bearer credentials" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("raw-upstream") + && e.contains("both provider adapter auth") + && e.contains("one bearer credential") + }), + "error should identify the two-credential conflict; errors: {:?}", + result.errors + ); +} + +#[test] +fn provider_owned_model_credentials_reject_model_key() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "managed" +backend_type = "http" +url = "https://managed.example.invalid/v1" +api_key = "adapter-client-key" +model_credential_owner = "provider" +model_api_key = "wrong-place" +models = ["managed/default"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "provider-owned final model credentials must not also configure model_api_key" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("managed") && e.contains("model_api_key")), + "error should identify provider-owned/model-key conflict; errors: {:?}", + result.errors + ); +} + +#[test] +fn deprecated_model_credential_owner_none_aliases_to_provider() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true + +[[proxy.providers]] +id = "local" +backend_type = "http" +url = "http://127.0.0.1:11434/v1" +model_credential_owner = "none" +models = ["ollama/qwen3.6:27b"] +"# + ); + let config = parse(&fixture); + let owner = config + .proxy + .as_ref() + .and_then(|proxy| proxy.providers.first()) + .map(|provider| provider.model_credential_owner); + assert_eq!(owner, Some(CredentialOwner::Provider)); + + let result = validate_config(&config); + + assert!( + result.is_valid(), + "deprecated model_credential_owner='none' should parse as provider-owned/no Calciforge model credentials" + ); +} + +#[test] +fn model_shortcut_alias_cannot_shadow_exact_model_route_id() { + let fixture = format!( + r#" +{MIN_VALID} + +[[proxy.providers]] +id = "remote" +backend_type = "http" +url = "https://example.invalid/v1" + +[[proxy.model_routes]] +pattern = "coding/default" +provider = "remote" + +[[model_shortcuts]] +alias = "coding/default" +model = "kimi-cli" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "shortcut aliases must not silently shadow exact model-route IDs" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("coding/default") + && e.contains("Ambiguous model shortcut alias") + && e.contains("model-route ID") + }), + "error should identify the colliding alias and model-route ID; errors: {:?}", + result.errors + ); +} + +#[test] +fn exact_model_route_cannot_shadow_agent_selector() { + let fixture = format!( + r#" +{MIN_VALID} + +[[agents]] +id = "local-dispatcher" +kind = "openai-compat" +endpoint = "http://127.0.0.1:18083" +model = "local-cloud-coding" + +[[proxy.providers]] +id = "remote" +backend_type = "http" +url = "https://example.invalid/v1" + +[[proxy.model_routes]] +pattern = "local-dispatcher" +provider = "remote" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "exact model routes must not silently treat agent names as concrete gateway models" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("local-dispatcher") && e.contains("model-route ID") && e.contains("agent ID") + }), + "error should identify the model route / agent selector collision; errors: {:?}", + result.errors + ); +} + +#[test] +fn provider_model_id_cannot_shadow_agent_alias() { + let fixture = format!( + r#" +{MIN_VALID} + +[[agents]] +id = "gateway-agent" +kind = "openai-compat" +endpoint = "http://127.0.0.1:18083" +model = "local-cloud-coding" +aliases = ["premium"] + +[[proxy.providers]] +id = "remote" +backend_type = "http" +url = "https://example.invalid/v1" +models = ["premium"] +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "exact provider model IDs must not silently reuse agent aliases" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("premium") && e.contains("provider model ID") && e.contains("agent alias") + }), + "error should identify the provider model / agent alias collision; errors: {:?}", + result.errors + ); +} + +#[test] +fn model_shortcut_alias_cannot_shadow_agent_selector() { + let fixture = format!( + r#" +{MIN_VALID} + +[[agents]] +id = "gateway-agent" +kind = "openai-compat" +endpoint = "http://127.0.0.1:18083" +model = "local-cloud-coding" +aliases = ["premium"] + +[[model_shortcuts]] +alias = "premium" +model = "openai/gpt-5.5" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "model shortcut aliases must not silently reuse agent selectors" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("premium") && e.contains("model shortcut alias") && e.contains("agent alias") + }), + "error should identify shortcut / agent alias collision; errors: {:?}", + result.errors + ); +} + +#[test] +fn exec_models_are_deprecated_config_errors() { + let fixture = format!( + r#" +{MIN_VALID} + +[[exec_models]] +id = "codex/gpt-5.5" +context_window = 262144 +command = "codex" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "deprecated exec model shims should not silently register as gateway models" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("[[exec_models]]") + && e.contains("deprecated") + && e.contains("kind = \"exec\"") + }), + "error should explain the agent migration path; errors: {:?}", + result.errors + ); +} + +#[test] +fn exec_proxy_providers_are_deprecated_config_errors() { + let fixture = format!( + r#" +{MIN_VALID} + +[proxy] +enabled = true +bind = "127.0.0.1:18083" + +[[proxy.providers]] +id = "codex-cli" +backend_type = "exec" +models = ["codex/gpt-5.5"] +command = "codex" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "exec proxy providers must not silently register as gateway models" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("codex-cli") + && e.contains("backend_type = \"exec\"") + && e.contains("[[agents]]") + }), + "error should explain that CLI-backed subscriptions are agents; errors: {:?}", + result.errors + ); +} + +#[test] +fn duplicate_first_class_model_ids_are_config_errors() { + let fixture = format!( + r#" +{MIN_VALID} + +[[dispatchers]] +id = "balanced" + +[[dispatchers.models]] +model = "qwen-test:small" +context_window = 60000 + +[[proxy.model_routes]] +pattern = "balanced" +provider = "missing-provider" +"# + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "configured first-class model IDs must not collide across namespaces" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("Ambiguous model selector 'balanced'")), + "error should identify duplicate first-class model ID; errors: {:?}", + result.errors + ); +} + +#[test] +fn whatsapp_legacy_webhook_fields_are_config_errors() { + let fixture = r#" +[calciforge] +version = 2 + +[[channels]] +kind = "whatsapp" +enabled = true +whatsapp_session_path = "/tmp/calciforge-wa.db" +webhook_listen = "0.0.0.0:18795" +webhook_path = "/webhooks/whatsapp" +zeroclaw_endpoint = "http://127.0.0.1:18796" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "legacy webhook fields must fail before startup" + ); + assert!( + result.errors.iter().any(|e| e.contains("WhatsApp") + && e.contains("webhook_listen") + && e.contains("embedded channel schema")), + "error should explain the embedded-channel migration; errors: {:?}", + result.errors + ); +} + +#[test] +fn enabled_whatsapp_requires_session_path() { + let fixture = r#" +[calciforge] +version = 2 + +[[channels]] +kind = "whatsapp" +enabled = true +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "enabled WhatsApp without session storage should fail" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("whatsapp_session_path")), + "error should name whatsapp_session_path; errors: {:?}", + result.errors + ); +} + +#[test] +fn signal_legacy_webhook_fields_are_config_errors() { + let fixture = r#" +[calciforge] +version = 2 + +[[channels]] +kind = "signal" +enabled = true +signal_cli_url = "http://127.0.0.1:8080" +signal_account = "+15555550001" +webhook_path = "/webhooks/signal" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "legacy webhook fields must fail before startup" + ); + assert!( + result.errors.iter().any(|e| e.contains("Signal") + && e.contains("webhook_path") + && e.contains("embedded channel schema")), + "error should explain the embedded-channel migration; errors: {:?}", + result.errors + ); +} diff --git a/crates/calciforge/src/config/validator_tests_3.rs b/crates/calciforge/src/config/validator_tests_3.rs new file mode 100644 index 00000000..e19253c6 --- /dev/null +++ b/crates/calciforge/src/config/validator_tests_3.rs @@ -0,0 +1,526 @@ +use super::validator_test_support::{MIN_VALID, parse}; +use super::*; + +#[test] +fn removed_openclaw_http_agent_is_an_error() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "custodian" +kind = "openclaw-http" +endpoint = "http://127.0.0.1:18789" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!(!result.is_valid(), "openclaw-http must fail validation"); + assert!( + result + .errors + .iter() + .any(|e| e.contains("openclaw-http") && e.contains("openclaw-channel")), + "error should name the removed kind and migration target; errors: {:?}", + result.errors + ); +} + +#[test] +fn openclaw_channel_agent_validates_with_callback_auth() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "custodian" +kind = "openclaw-channel" +endpoint = "http://127.0.0.1:18789" +api_key = "test-gateway-token" +reply_auth_token = "test-reply-token" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + result.is_valid(), + "openclaw-channel should validate; errors: {:?}", + result.errors + ); +} + +#[test] +fn openclaw_channel_agent_validates_with_callback_auth_file() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "custodian" +kind = "openclaw-channel" +endpoint = "http://127.0.0.1:18789" +api_key = "test-gateway-token" +reply_auth_token_file = "/tmp/calciforge-test-reply-token" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + result.is_valid(), + "openclaw-channel should validate; errors: {:?}", + result.errors + ); +} + +#[test] +fn openai_compat_agent_validates() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "gateway" +kind = "openai-compat" +endpoint = "http://127.0.0.1:8083" +api_key = "test-gateway-token" +model = "local-kimi-gpt55" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + result.is_valid(), + "openai-compat should validate; errors: {:?}", + result.errors + ); +} + +#[test] +fn openai_compat_rejects_openclaw_model_ids() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "librarian" +kind = "openai-compat" +endpoint = "http://127.0.0.1:18789" +api_key = "test-gateway-token" +model = "openclaw/main" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "OpenClaw model IDs should require openclaw-channel" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("OpenClaw") && e.contains("openclaw-channel")), + "error should point to openclaw-channel; errors: {:?}", + result.errors + ); +} + +#[test] +fn openai_compat_without_model_requires_override_opt_in() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "gateway" +kind = "openai-compat" +endpoint = "http://127.0.0.1:8083" +api_key = "test-gateway-token" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "openai-compat without model or allow_model_override must fail" + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("allow_model_override")), + "error should mention allow_model_override; errors: {:?}", + result.errors + ); +} + +#[test] +fn openai_compat_without_model_validates_when_override_is_explicit() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "gateway" +kind = "openai-compat" +endpoint = "http://127.0.0.1:8083" +api_key = "test-gateway-token" +allow_model_override = true +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + result.is_valid(), + "openai-compat with explicit model override should validate; errors: {:?}", + result.errors + ); +} + +#[test] +fn zeroclaw_agent_requires_api_key_or_file() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "librarianzero" +kind = "zeroclaw" +endpoint = "http://127.0.0.1:18799" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!(!result.is_valid(), "zeroclaw without key must fail"); + assert!( + result.errors.iter().any(|e| e.contains("api_key")), + "error should mention missing api_key/api_key_file; errors: {:?}", + result.errors + ); +} + +#[test] +fn hermes_without_auth_fails_before_runtime() { + let fixture = r#" +[calciforge] +version = 2 + +[[agents]] +id = "hermes" +kind = "hermes" +endpoint = "http://127.0.0.1:8642" +"#; + let config = parse(fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "Hermes config without auth should fail before adapter construction; errors: {:?}", + result.errors + ); + assert!( + result + .errors + .iter() + .any(|error| error.contains("hermes") && error.contains("api_key")), + "error should mention missing auth for Hermes; errors: {:?}", + result.errors + ); +} + +/// Given a config with two agents sharing the same id, +/// when validate_config runs, +/// then an error naming the duplicated id is produced. +#[test] +fn duplicate_agent_id_is_an_error() { + let fixture = format!( + "{MIN_VALID}\n[[agents]]\nid = \"bot\"\nkind = \"cli\"\ncommand = \"/bin/echo\"\nargs = []\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!(!result.is_valid(), "duplicate agent id must fail"); + assert!( + result.errors.iter().any(|e| e.contains("bot")), + "error must name the duplicated id 'bot'; errors: {:?}", + result.errors + ); +} + +#[test] +fn agent_alias_cannot_shadow_another_agent_id() { + let fixture = format!( + "{MIN_VALID}\n[[agents]]\nid = \"helper\"\nkind = \"cli\"\ncommand = \"/bin/echo\"\nargs = []\naliases = [\"bot\"]\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "agent aliases must not silently shadow another agent id" + ); + assert!( + result.errors.iter().any(|e| { + e.contains("Ambiguous agent selector 'bot'") + && e.contains("agent 'bot'") + && e.contains("agent 'helper'") + }), + "error should identify both agent selector owners; errors: {:?}", + result.errors + ); +} + +/// Given a config with two identities sharing the same id, +/// when validate_config runs, +/// then an error naming the duplicated id is produced. +#[test] +fn duplicate_identity_id_is_an_error() { + let fixture = format!( + "{MIN_VALID}\n[[identities]]\nid = \"alice\"\naliases = [{{ channel = \"signal\", id = \"7000000099\" }}]\nrole = \"user\"\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!(!result.is_valid(), "duplicate identity id must fail"); + assert!( + result.errors.iter().any(|e| e.contains("alice")), + "error must name the duplicated id 'alice'; errors: {:?}", + result.errors + ); +} + +/// Given a proxy config with an unparseable bind address, +/// when validate_config runs, +/// then an error is produced naming the bind/address problem. +#[test] +fn malformed_proxy_bind_is_an_error() { + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"not-an-address\"\nbackend_type = \"http\"\nbackend_url = \"https://api.example.com\"\ntimeout_seconds = 10\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "invalid bind address must fail; errors: {:?}", + result.errors + ); + assert!( + result.errors.iter().any(|e| { + let lower = e.to_lowercase(); + lower.contains("bind") || lower.contains("address") + }), + "error should name the bind/address problem; errors: {:?}", + result.errors + ); +} + +/// Given a proxy config with `timeout_seconds = 0`, +/// when validate_config runs, +/// then an error is produced — a zero timeout means requests +/// hang indefinitely, which is never the intent. +#[test] +fn zero_proxy_timeout_is_an_error() { + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"http\"\nbackend_url = \"https://api.example.com\"\ntimeout_seconds = 0\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "zero proxy timeout must fail; errors: {:?}", + result.errors + ); +} + +/// Given a proxy config with a gateway UI link that chat users may open, +/// when validate_config runs, +/// then non-HTTP links are rejected before they appear in `!help`. +#[test] +fn gateway_ui_url_requires_http_url() { + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"http\"\nbackend_url = \"https://api.example.com\"\ngateway_ui_url = \"localhost:8585\"\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "gateway UI URL without scheme must fail; errors: {:?}", + result.errors + ); + assert!( + result.errors.iter().any(|e| e.contains("gateway_ui_url")), + "error should name gateway_ui_url; errors: {:?}", + result.errors + ); +} + +/// Given a proxy backend type outside the runtime allow-list, +/// when validate_config runs, +/// then validation rejects it and reports the supported values. +#[test] +fn unsupported_proxy_backend_type_is_rejected_by_allowlist() { + let backend_type = "experimental-gateway"; + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"{backend_type}\"\nbackend_url = \"https://api.example.com\"\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "{backend_type} must not validate as a supported proxy backend; errors: {:?}", + result.errors + ); + assert!( + result.errors.iter().any(|e| { + e.contains("backend_type") && e.contains(backend_type) && e.contains("http") + }), + "error should name unsupported backend and supported values; errors: {:?}", + result.errors + ); +} + +#[test] +fn named_openai_compatible_backend_types_are_validated_from_shared_allowlist() { + for backend_type in [ + "litellm", + "portkey", + "tensorzero", + "future-agi", + "openrouter", + ] { + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"{backend_type}\"\nbackend_url = \"https://gateway.example.invalid/v1\"\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + result.is_valid(), + "{backend_type} should be accepted as a supported OpenAI-compatible provider adapter; errors: {:?}", + result.errors + ); + } +} + +/// Given a disabled proxy with a configured gateway UI link, +/// when validate_config runs, +/// then the UI URL is still validated because chat help can surface it from +/// config even when the model gateway listener is disabled. +#[test] +fn disabled_proxy_still_validates_gateway_ui_url() { + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = false\ngateway_ui_url = \"javascript:alert(1)\"\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "disabled proxy must still validate displayed gateway UI URLs; errors: {:?}", + result.errors + ); + assert!( + result.errors.iter().any(|e| e.contains("gateway_ui_url")), + "error should name gateway_ui_url; errors: {:?}", + result.errors + ); +} + +/// Given an OpenAI-compatible provider adapter is selected, +/// when the backend URL is malformed or contains request modifiers, +/// then validation rejects it before runtime path construction can fail. +#[test] +fn provider_adapter_backend_url_must_be_plain_http_base_url() { + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"litellm\"\nbackend_url = \"https://litellm.example.invalid/v1?debug=true\"\nbackend_api_key = \"test-key\"\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + !result.is_valid(), + "provider adapter backend URL with query string must fail; errors: {:?}", + result.errors + ); + assert!( + result + .errors + .iter() + .any(|e| { e.contains("Proxy backend_url") && e.contains("query") }), + "error should name Proxy backend_url and query/fragment issue; errors: {:?}", + result.errors + ); +} + +/// Given Helicone is selected for a likely local unauthenticated gateway, +/// when no backend key is configured, +/// then validation warns instead of silently accepting a surprising empty +/// `Authorization: Bearer` header. +#[test] +fn helicone_without_backend_key_warns_operator() { + let fixture = format!( + "{MIN_VALID}\n[proxy]\nenabled = true\nbind = \"127.0.0.1:18083\"\nbackend_type = \"helicone\"\nbackend_url = \"http://127.0.0.1:8787\"\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + + assert!( + result.is_valid(), + "local unauthenticated Helicone should remain possible: {:?}", + result.errors + ); + assert!( + result + .warnings + .iter() + .any(|w| w.contains("Helicone backend has no backend_api_key")), + "missing Helicone key should produce an operator warning; warnings: {:?}", + result.warnings + ); +} + +/// Given a routing rule referencing an agent id that doesn't +/// exist in the agent list, +/// when validate_config runs, +/// then an error naming the missing agent is produced. +/// +/// Catches the most common cause of silent "agent unavailable" +/// at runtime: typo in a routing rule. +#[test] +fn routing_rule_default_to_nonexistent_agent_is_an_error() { + let fixture = + format!("{MIN_VALID}\n[[routing]]\nidentity = \"alice\"\ndefault_agent = \"ghost\"\n"); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "routing default_agent pointing at a non-existent agent must fail; \ + errors: {:?}", + result.errors + ); + assert!( + result.errors.iter().any(|e| e.contains("ghost")), + "error should name the missing agent id 'ghost'; errors: {:?}", + result.errors + ); +} + +/// Given a routing rule whose `default_agent` is valid but whose +/// `allowed_agents` list contains an id not in the agent list, +/// when validate_config runs, +/// then an error naming both the identity and the missing agent +/// is produced. +/// +/// Validates the branch in `validate_routing_rules` that walks +/// each entry of `allowed_agents` — a test for +/// `default_agent` alone wouldn't exercise this code path. +#[test] +fn routing_rule_allowed_list_with_nonexistent_agent_is_an_error() { + let fixture = format!( + "{MIN_VALID}\n[[routing]]\nidentity = \"alice\"\ndefault_agent = \"bot\"\nallowed_agents = [\"bot\", \"ghost\"]\n" + ); + let config = parse(&fixture); + let result = validate_config(&config); + assert!( + !result.is_valid(), + "allowed_agents pointing at a non-existent agent must fail; \ + errors: {:?}", + result.errors + ); + assert!( + result + .errors + .iter() + .any(|e| e.contains("ghost") && e.contains("alice")), + "error should name both the identity 'alice' and missing agent \ + 'ghost'; errors: {:?}", + result.errors + ); +} diff --git a/crates/calciforge/src/doctor.rs b/crates/calciforge/src/doctor.rs index 922a011b..9d7ba2a2 100644 --- a/crates/calciforge/src/doctor.rs +++ b/crates/calciforge/src/doctor.rs @@ -898,19 +898,36 @@ fn report_model_gateway_provider_boundaries( report: &mut DoctorReport, ) { for provider in &proxy.providers { - if provider.backend_type != "http" { + let gateway_type = provider + .backend_type + .parse::(); + let Ok(gateway_type) = gateway_type else { + report.error(format!( + "provider '{}' uses unsupported backend_type '{}'", + provider.id, provider.backend_type + )); + continue; + }; + if !gateway_type.uses_openai_compatible_http_core() { continue; } - match provider.model_credential_owner { - crate::config::CredentialOwner::Provider => report.ok(format!( - "provider '{}' uses builtin HTTP transport to a provider-owned endpoint", - provider.id - )), - crate::config::CredentialOwner::Calciforge => report.warn(format!( - "provider '{}' uses Calciforge-owned builtin HTTP upstream credentials; this route is not handled by an external provider dashboard or registry", - provider.id - )), + if provider.backend_type == "http" { + match provider.model_credential_owner { + crate::config::CredentialOwner::Provider => report.ok(format!( + "provider '{}' uses plain HTTP transport to a provider-owned endpoint", + provider.id + )), + crate::config::CredentialOwner::Calciforge => report.warn(format!( + "provider '{}' uses Calciforge-owned plain HTTP upstream credentials; this route is not handled by a named provider dashboard or registry", + provider.id + )), + } + } else { + report.ok(format!( + "provider '{}' uses {} provider adapter boundary", + provider.id, provider.backend_type + )); } } } @@ -3165,10 +3182,53 @@ mod tests { && finding.message.contains("provider 'opencode-go'") && finding .message - .contains("Calciforge-owned builtin HTTP upstream credentials") + .contains("Calciforge-owned plain HTTP upstream credentials") + && finding + .message + .contains("not handled by a named provider dashboard or registry") + })); + } + + #[test] + fn model_gateway_config_reports_named_provider_adapter_boundaries() { + let mut config = base_config(); + let proxy = config.proxy.as_mut().expect("proxy"); + proxy.providers = vec![ProxyProviderConfig { + id: "litellm-local".to_string(), + backend_type: "litellm".to_string(), + url: "http://127.0.0.1:4000/v1".to_string(), + api_key: None, + api_key_file: None, + models: vec!["local/qwen".to_string()], + strip_model_prefix: None, + add_model_prefix: None, + timeout_seconds: Some(60), + headers: HashMap::new(), + on_switch: None, + command: None, + args: Vec::new(), + env: HashMap::new(), + ..Default::default() + }]; + proxy.model_routes = vec![ProxyModelRoute { + pattern: "local/qwen".to_string(), + provider: "litellm-local".to_string(), + }]; + let mut report = DoctorReport::default(); + + check_model_gateway_config(&config, &mut report); + + assert!(report.findings.iter().any(|finding| { + finding.severity == Severity::Ok + && finding + .message + .contains("provider 'litellm-local' uses litellm provider adapter boundary") + })); + assert!(!report.findings.iter().any(|finding| { + finding.severity == Severity::Warn && finding .message - .contains("not handled by an external provider dashboard or registry") + .contains("Calciforge-owned plain HTTP upstream credentials") })); } diff --git a/crates/calciforge/src/proxy/backend.rs b/crates/calciforge/src/proxy/backend.rs index e1fc2b01..9afaf999 100644 --- a/crates/calciforge/src/proxy/backend.rs +++ b/crates/calciforge/src/proxy/backend.rs @@ -1,9 +1,9 @@ //! Unified backend interface for the model gateway //! //! Provides the runtime abstraction used by supported model-provider methods. -//! The production root gateway surface is intentionally small: Calciforge's -//! builtin OpenAI-compatible HTTP upstream adapter, Helicone's external HTTP -//! gateway, and a mock backend for tests. +//! The production root gateway surface is intentionally small: one shared +//! OpenAI-compatible HTTP core with engine-specific policy overlays, plus a +//! mock backend for tests. use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -13,10 +13,6 @@ use crate::sync::Arc; use crate::config::GatewayFailureKind; use crate::proxy::openai::{ChatCompletionResponse, MessageContent}; -// Helicone router (HTTP adapter) -#[cfg(feature = "helicone")] -use super::helicone_router; - /// Errors that can occur in backend operations #[derive(Error, Debug)] #[allow(dead_code)] @@ -126,8 +122,6 @@ pub trait SecretsBackend: Send + Sync { pub enum BackendType { /// HTTP to an OpenAI-compatible provider. Http, - /// HTTP to Helicone AI Gateway - Helicone, /// Mock backend for testing Mock, } @@ -136,7 +130,6 @@ impl std::fmt::Display for BackendType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { BackendType::Http => write!(f, "http"), - BackendType::Helicone => write!(f, "helicone"), BackendType::Mock => write!(f, "mock"), } } @@ -172,11 +165,6 @@ pub struct BackendConfig { pub api_key: Option, pub timeout_seconds: Option, pub headers: Option>, - - // Helicone backend config - pub helicone_url: Option, - pub helicone_api_key: Option, - pub helicone_router_name: Option, } impl Default for BackendConfig { @@ -187,9 +175,6 @@ impl Default for BackendConfig { api_key: None, timeout_seconds: Some(30), headers: None, - helicone_url: Some("http://localhost:8080".to_string()), - helicone_api_key: None, - helicone_router_name: None, } } } @@ -206,50 +191,10 @@ pub fn create_backend(config: &BackendConfig) -> Result, let headers = config.headers.clone(); Ok(Arc::new(HttpBackend::new(url, api_key, timeout, headers))) } - BackendType::Helicone => create_helicone_backend(config), BackendType::Mock => Ok(Arc::new(MockBackend::new())), } } -#[cfg(feature = "helicone")] -fn create_helicone_backend( - config: &BackendConfig, -) -> Result, BackendError> { - let url = config.helicone_url.clone().ok_or_else(|| { - BackendError::ConfigError("Missing helicone_url for Helicone backend".to_string()) - })?; - let api_key = config.helicone_api_key.clone().unwrap_or_default(); - let timeout = config.timeout_seconds.unwrap_or(120); - let router_name = config - .helicone_router_name - .clone() - .unwrap_or_else(|| "helicone".to_string()); - let helicone_config = helicone_router::HeliconeRouterConfig { - base_url: url, - api_key, - timeout_seconds: timeout, - router_name, - enable_caching: true, - cache_ttl_seconds: 300, - headers: std::collections::HashMap::new(), - retry: crate::config::GatewayRetryConfig::default(), - }; - let router = helicone_router::HeliconeRouter::new(helicone_config).map_err(|e| { - BackendError::ConfigError(format!("Failed to create Helicone router: {}", e)) - })?; - Ok(Arc::new(router)) -} - -#[cfg(not(feature = "helicone"))] -fn create_helicone_backend( - _config: &BackendConfig, -) -> Result, BackendError> { - Err(BackendError::ConfigError( - "Helicone backend selected but calciforge was built without the helicone feature" - .to_string(), - )) -} - // Mock backend implementation pub struct MockBackend { responses: std::collections::HashMap, @@ -411,13 +356,10 @@ impl HttpBackend { async fn send_chat_completion_request( &self, - mut request: crate::proxy::openai::ChatCompletionRequest, + request: crate::proxy::openai::ChatCompletionRequest, ) -> Result { let url = format!("{}/chat/completions", self.base_url); - // Force non-streaming until this backend grows SSE support. - request.stream = Some(false); - let model = request.model.clone(); let mut request_body = serde_json::to_value(&request).map_err(|e| { BackendError::InvalidResponse(format!("Failed to serialize request: {e}")) @@ -458,10 +400,31 @@ impl HttpBackend { )); } - response - .json() - .await - .map_err(|e| BackendError::InvalidResponse(format!("Failed to parse response: {}", e))) + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_ascii_lowercase(); + if content_type.contains("text/event-stream") { + let body = response.text().await.map_err(|e| { + BackendError::transport( + format!( + "Failed to read streaming response for model '{}': {}", + model, e + ), + e.is_timeout(), + ) + })?; + return crate::proxy::openai_streaming::parse_streaming_chat_completion(&body, &model); + } + + response.json().await.map_err(|e| { + BackendError::InvalidResponse(format!( + "Failed to parse response for model '{}': {}", + model, e + )) + }) } } diff --git a/crates/calciforge/src/proxy/gateway.rs b/crates/calciforge/src/proxy/gateway.rs index 84140f43..492f5940 100644 --- a/crates/calciforge/src/proxy/gateway.rs +++ b/crates/calciforge/src/proxy/gateway.rs @@ -9,6 +9,7 @@ //! runtime code should treat these as adapter kinds. use async_trait::async_trait; +use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; use std::time::Duration; @@ -91,6 +92,16 @@ pub enum GatewayType { Helicone, /// Calciforge's minimal builtin OpenAI-compatible HTTP upstream adapter. BuiltinHttp, + /// LiteLLM OpenAI-compatible proxy/gateway. + LiteLlm, + /// Portkey OpenAI-compatible gateway. + Portkey, + /// TensorZero OpenAI-compatible gateway. + TensorZero, + /// Future AGI OpenAI-compatible gateway. + FutureAgi, + /// OpenRouter OpenAI-compatible provider boundary. + OpenRouter, /// Mock adapter for tests only. Mock, } @@ -102,6 +113,11 @@ impl std::str::FromStr for GatewayType { match s.to_lowercase().as_str() { "helicone" => Ok(GatewayType::Helicone), "http" | "builtin-http" | "builtin_http" | "direct" => Ok(GatewayType::BuiltinHttp), + "litellm" | "lite-llm" | "lite_llm" => Ok(GatewayType::LiteLlm), + "portkey" => Ok(GatewayType::Portkey), + "tensorzero" | "tensor-zero" | "tensor_zero" => Ok(GatewayType::TensorZero), + "future-agi" | "future_agi" | "futureagi" => Ok(GatewayType::FutureAgi), + "openrouter" | "open-router" | "open_router" => Ok(GatewayType::OpenRouter), "mock" => Ok(GatewayType::Mock), _ => Err(format!("Unknown gateway type: {}", s)), } @@ -113,16 +129,47 @@ impl std::fmt::Display for GatewayType { match self { GatewayType::Helicone => write!(f, "helicone"), GatewayType::BuiltinHttp => write!(f, "builtin-http"), + GatewayType::LiteLlm => write!(f, "litellm"), + GatewayType::Portkey => write!(f, "portkey"), + GatewayType::TensorZero => write!(f, "tensorzero"), + GatewayType::FutureAgi => write!(f, "future-agi"), + GatewayType::OpenRouter => write!(f, "openrouter"), GatewayType::Mock => write!(f, "mock"), } } } impl GatewayType { + pub const SUPPORTED_CONFIG_NAMES: &'static [&'static str] = &[ + "http", + "helicone", + "litellm", + "portkey", + "tensorzero", + "future-agi", + "openrouter", + "mock", + ]; + + pub const SUPPORTED_PROVIDER_CONFIG_NAMES: &'static [&'static str] = &[ + "http", + "helicone", + "litellm", + "portkey", + "tensorzero", + "future-agi", + "openrouter", + ]; + pub fn display_name(self) -> &'static str { match self { GatewayType::Helicone => "Helicone AI Gateway", GatewayType::BuiltinHttp => "Calciforge builtin HTTP upstream adapter", + GatewayType::LiteLlm => "LiteLLM gateway", + GatewayType::Portkey => "Portkey gateway", + GatewayType::TensorZero => "TensorZero gateway", + GatewayType::FutureAgi => "Future AGI gateway", + GatewayType::OpenRouter => "OpenRouter", GatewayType::Mock => "Mock provider adapter", } } @@ -145,6 +192,46 @@ impl GatewayType { observability: false, operator_ui: false, }, + GatewayType::LiteLlm => GatewayCapabilities { + openai_chat_completions: true, + model_listing: true, + tool_call_transcripts: false, + config_validation: false, + observability: true, + operator_ui: true, + }, + GatewayType::Portkey => GatewayCapabilities { + openai_chat_completions: true, + model_listing: false, + tool_call_transcripts: false, + config_validation: false, + observability: true, + operator_ui: true, + }, + GatewayType::TensorZero => GatewayCapabilities { + openai_chat_completions: true, + model_listing: false, + tool_call_transcripts: false, + config_validation: false, + observability: true, + operator_ui: true, + }, + GatewayType::FutureAgi => GatewayCapabilities { + openai_chat_completions: true, + model_listing: false, + tool_call_transcripts: false, + config_validation: false, + observability: true, + operator_ui: true, + }, + GatewayType::OpenRouter => GatewayCapabilities { + openai_chat_completions: true, + model_listing: true, + tool_call_transcripts: false, + config_validation: false, + observability: false, + operator_ui: true, + }, GatewayType::Mock => GatewayCapabilities { openai_chat_completions: true, model_listing: true, @@ -155,6 +242,27 @@ impl GatewayType { }, } } + + pub fn uses_openai_compatible_http_core(self) -> bool { + matches!( + self, + GatewayType::BuiltinHttp + | GatewayType::Helicone + | GatewayType::LiteLlm + | GatewayType::Portkey + | GatewayType::TensorZero + | GatewayType::FutureAgi + | GatewayType::OpenRouter + ) + } + + pub fn requires_backend_url(self) -> bool { + !matches!(self, GatewayType::Mock) + } + + pub fn delegates_retry_to_adapter(self) -> bool { + matches!(self, GatewayType::Helicone) + } } impl GatewayConfig { @@ -168,6 +276,46 @@ impl GatewayConfig { } } +/// Apply engine-specific OpenAI-compatible HTTP headers while keeping all +/// engines on the same request/response core. +pub(crate) fn openai_compatible_headers( + gateway_type: GatewayType, + api_key: Option<&str>, + retry: &GatewayRetryConfig, + configured: Option<&HashMap>, +) -> Option> { + let mut headers = configured.cloned().unwrap_or_default(); + if gateway_type == GatewayType::Helicone { + if let Some(key) = api_key.map(str::trim).filter(|key| !key.is_empty()) { + headers.insert("helicone-auth".to_string(), format!("Bearer {key}")); + } + if retry.enabled { + headers.insert("helicone-retry-enabled".to_string(), "true".to_string()); + headers.insert( + "helicone-retry-num".to_string(), + retry.max_retries.to_string(), + ); + headers.insert( + "helicone-retry-min-timeout".to_string(), + retry.min_timeout_ms.to_string(), + ); + headers.insert( + "helicone-retry-max-timeout".to_string(), + retry.max_timeout_ms.to_string(), + ); + headers.insert( + "helicone-retry-factor".to_string(), + retry.factor.to_string(), + ); + } + } + if headers.is_empty() { + None + } else { + Some(headers) + } +} + /// Main trait for provider adapters #[async_trait] #[allow(dead_code)] @@ -199,37 +347,6 @@ pub fn create_gateway( backend: Option>, ) -> Result, BackendError> { match config.backend_type { - #[cfg(feature = "helicone")] - GatewayType::Helicone => { - use crate::proxy::helicone_router::{HeliconeRouter, HeliconeRouterConfig}; - - let helicone_config = HeliconeRouterConfig { - base_url: config - .base_url - .clone() - .unwrap_or_else(|| "http://localhost:8787".to_string()), - api_key: config.api_key.clone().unwrap_or_default(), - timeout_seconds: config.timeout_seconds, - router_name: "helicone".to_string(), - enable_caching: false, - cache_ttl_seconds: 300, - headers: config.headers.clone().unwrap_or_default(), - retry: config.retry.clone(), - }; - - let router = HeliconeRouter::new(helicone_config).map_err(|e| { - BackendError::ConfigError(format!("Failed to create Helicone router: {}", e)) - })?; - - let inner_gateway = Arc::new(HeliconeGateway { - config: config.clone(), - router, - }); - - // Wrap with logging for debugging - Ok(Arc::new(LoggingGateway::new(config, inner_gateway))) - } - GatewayType::Mock => { let inner_gateway = Arc::new(MockGateway::new(config.clone())); @@ -237,13 +354,20 @@ pub fn create_gateway( Ok(Arc::new(LoggingGateway::new(config, inner_gateway))) } - GatewayType::BuiltinHttp => { + GatewayType::Helicone + | GatewayType::BuiltinHttp + | GatewayType::LiteLlm + | GatewayType::Portkey + | GatewayType::TensorZero + | GatewayType::FutureAgi + | GatewayType::OpenRouter => { // Builtin HTTP upstream calls // This requires a backend to be passed in let backend = backend.ok_or_else(|| { - BackendError::ConfigError( - "Builtin HTTP gateway requires a backend parameter".to_string(), - ) + BackendError::ConfigError(format!( + "{} gateway requires a backend parameter", + config.backend_type + )) })?; let inner_gateway = Arc::new(BuiltinHttpGateway::new(config.clone(), backend)); @@ -251,46 +375,6 @@ pub fn create_gateway( // Wrap with logging for debugging Ok(Arc::new(LoggingGateway::new(config, inner_gateway))) } - - #[cfg(not(feature = "helicone"))] - GatewayType::Helicone => Err(BackendError::ConfigError( - "Helicone feature not enabled".to_string(), - )), - } -} - -// --------------------------------------------------------------------------- -// Helicone Gateway Implementation -// --------------------------------------------------------------------------- - -#[cfg(feature = "helicone")] -#[derive(Debug)] -#[allow(dead_code)] -pub struct HeliconeGateway { - config: GatewayConfig, - router: crate::proxy::helicone_router::HeliconeRouter, -} - -#[cfg(feature = "helicone")] -#[async_trait] -impl ProviderAdapter for HeliconeGateway { - fn gateway_type(&self) -> GatewayType { - GatewayType::Helicone - } - - async fn chat_completion( - &self, - request: ChatCompletionRequest, - ) -> Result { - self.router.chat_completion_request(request).await - } - - async fn list_models(&self) -> Result, BackendError> { - self.router.list_models().await - } - - fn config(&self) -> &GatewayConfig { - &self.config } } @@ -465,14 +549,14 @@ impl ProviderAdapter for LoggingGateway { } } -fn should_retry_locally( +pub(super) fn should_retry_locally( gateway_type: GatewayType, policy: &GatewayRetryConfig, error: &BackendError, attempt: u32, ) -> bool { - if gateway_type == GatewayType::Helicone { - // Helicone retry policy is passed to the provider adapter as request + if gateway_type.delegates_retry_to_adapter() { + // Some gateway retry policies are passed to the provider adapter as request // headers. Retrying again here would multiply attempts and costs. return false; } @@ -524,7 +608,7 @@ impl BuiltinHttpGateway { #[async_trait] impl ProviderAdapter for BuiltinHttpGateway { fn gateway_type(&self) -> GatewayType { - GatewayType::BuiltinHttp + self.config.backend_type } async fn chat_completion( @@ -622,380 +706,3 @@ impl ProviderAdapter for MockGateway { &self.config } } - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_gateway_type_parsing() { - assert_eq!( - "helicone".parse::().unwrap(), - GatewayType::Helicone - ); - assert_eq!( - "direct".parse::().unwrap(), - GatewayType::BuiltinHttp - ); - assert_eq!( - "http".parse::().unwrap(), - GatewayType::BuiltinHttp - ); - assert_eq!( - "builtin-http".parse::().unwrap(), - GatewayType::BuiltinHttp - ); - assert_eq!("mock".parse::().unwrap(), GatewayType::Mock); - assert!("unknown".parse::().is_err()); - } - - #[test] - fn test_gateway_type_display() { - assert_eq!(GatewayType::Helicone.to_string(), "helicone"); - assert_eq!(GatewayType::BuiltinHttp.to_string(), "builtin-http"); - assert_eq!(GatewayType::Mock.to_string(), "mock"); - } - - #[test] - fn test_mock_gateway() { - use super::MockGateway; - - let config = GatewayConfig { - backend_type: GatewayType::Mock, - base_url: None, - api_key: None, - timeout_seconds: 30, - extra_config: None, - headers: None, - retry: GatewayRetryConfig::default(), - ui_url: None, - }; - - let gateway = MockGateway::new(config); - assert_eq!(gateway.gateway_type(), GatewayType::Mock); - } - - #[tokio::test] - async fn mock_gateway_returns_openai_compatible_chat_choice() { - let gateway = MockGateway::new(GatewayConfig { - backend_type: GatewayType::Mock, - ..Default::default() - }); - - let response = gateway - .chat_completion(ChatCompletionRequest { - model: "gpt-4".to_string(), - messages: vec![ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("short".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }], - max_tokens: Some(2), - ..Default::default() - }) - .await - .unwrap(); - - assert_eq!(response.model, "gpt-4"); - let choice = response - .choices - .first() - .expect("mock gateway should return an assistant choice"); - assert_eq!(choice.message.role, "assistant"); - let Some(MessageContent::Text(content)) = choice.message.content.as_ref() else { - panic!("mock gateway choice should contain text content"); - }; - assert!( - content.contains("gpt-4") && content.to_lowercase().contains("mock"), - "mock response content should identify the routed model: {content}" - ); - } - - #[test] - fn gateway_engine_info_carries_operator_ui_link() { - let config = GatewayConfig { - backend_type: GatewayType::Helicone, - ui_url: Some("http://127.0.0.1:8585".to_string()), - ..Default::default() - }; - - let info = config.engine_info(GatewayType::Helicone); - - assert_eq!(info.id, "helicone"); - assert_eq!(info.display_name, "Helicone AI Gateway"); - assert_eq!(info.ui_url.as_deref(), Some("http://127.0.0.1:8585")); - assert!(info.capabilities.operator_ui); - assert!(info.capabilities.observability); - assert!(!info.capabilities.model_listing); - assert!(!info.capabilities.tool_call_transcripts); - assert!(!info.capabilities.config_validation); - } - - #[test] - fn builtin_http_gateway_retries_only_configured_failure_kinds() { - let retry = GatewayRetryConfig { - enabled: true, - max_retries: 2, - min_timeout_ms: 1, - max_timeout_ms: 10, - factor: 2, - retry_on: vec![crate::config::GatewayFailureKind::ServerError], - }; - let server_error = - BackendError::http_status_error(reqwest::StatusCode::SERVICE_UNAVAILABLE, "down"); - let auth_error = - BackendError::http_status_error(reqwest::StatusCode::UNAUTHORIZED, "bad key"); - - assert!(should_retry_locally( - GatewayType::BuiltinHttp, - &retry, - &server_error, - 0 - )); - assert!(!should_retry_locally( - GatewayType::BuiltinHttp, - &retry, - &auth_error, - 0 - )); - assert!(!should_retry_locally( - GatewayType::BuiltinHttp, - &retry, - &server_error, - 2 - )); - } - - #[test] - fn helicone_retry_policy_is_not_applied_twice_locally() { - let retry = GatewayRetryConfig { - enabled: true, - max_retries: 2, - min_timeout_ms: 1, - max_timeout_ms: 10, - factor: 2, - retry_on: vec![crate::config::GatewayFailureKind::ServerError], - }; - let server_error = - BackendError::http_status_error(reqwest::StatusCode::SERVICE_UNAVAILABLE, "down"); - - assert!( - !should_retry_locally(GatewayType::Helicone, &retry, &server_error, 0), - "Helicone receives retry headers, so Calciforge must not multiply attempts locally" - ); - } - - #[tokio::test] - async fn builtin_http_gateway_forwards_complete_chat_request_options() { - use crate::proxy::backend::{BackendConfig, BackendType, create_backend}; - use crate::proxy::openai::{ChatMessage, Choice, MessageContent, Usage}; - use mockito::Matcher; - use std::collections::HashMap; - - let mut server = mockito::Server::new_async().await; - let response = ChatCompletionResponse { - id: "chatcmpl-test".to_string(), - object: "chat.completion".to_string(), - created: 1, - model: "kimi-for-coding".to_string(), - choices: vec![Choice { - index: 0, - message: ChatMessage { - role: "assistant".to_string(), - content: Some(MessageContent::Text("ok".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }, - finish_reason: Some("stop".to_string()), - logprobs: None, - }], - usage: Usage { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - }, - system_fingerprint: None, - }; - let mock = server - .mock("POST", "/v1/chat/completions") - .match_header("x-client-family", "kimi-cli") - .match_body(Matcher::PartialJson(serde_json::json!({ - "model": "kimi-for-coding", - "max_tokens": 16, - "temperature": 0.5, - "thinking": {"type": "enabled"}, - "stream": false, - "messages": [{"role": "user", "content": "hello"}] - }))) - .with_status(200) - .with_header("content-type", "application/json") - .with_body(serde_json::to_string(&response).unwrap()) - .create_async() - .await; - - let mut headers = HashMap::new(); - headers.insert("x-client-family".to_string(), "kimi-cli".to_string()); - let backend = create_backend(&BackendConfig { - backend_type: BackendType::Http, - url: Some(format!("{}/v1", server.url())), - api_key: Some("provider-key".to_string()), - timeout_seconds: Some(30), - headers: Some(headers.clone()), - ..Default::default() - }) - .unwrap(); - let gateway = create_gateway( - GatewayConfig { - backend_type: GatewayType::BuiltinHttp, - base_url: Some(format!("{}/v1", server.url())), - api_key: Some("provider-key".to_string()), - timeout_seconds: 30, - headers: Some(headers), - ..Default::default() - }, - Some(backend), - ) - .unwrap(); - - let result = gateway - .chat_completion( - serde_json::from_value(serde_json::json!({ - "model": "kimi-for-coding", - "messages": [{"role": "user", "content": "hello"}], - "max_tokens": 16, - "temperature": 0.5, - "thinking": {"type": "enabled"} - })) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(result.model, "kimi/kimi-for-coding"); - mock.assert_async().await; - } - - #[cfg(feature = "helicone")] - #[test] - fn create_helicone_gateway_preserves_engine_metadata_through_logging_wrapper() { - let gateway = create_gateway( - GatewayConfig { - backend_type: GatewayType::Helicone, - base_url: Some("https://ai-gateway.helicone.ai".to_string()), - api_key: Some("helicone-test-key".to_string()), - ui_url: Some("https://us.helicone.ai/requests".to_string()), - ..Default::default() - }, - None, - ) - .unwrap(); - - let info = gateway.engine_info(); - - assert_eq!(gateway.gateway_type(), GatewayType::Helicone); - assert_eq!(info.id, "helicone"); - assert_eq!(info.display_name, "Helicone AI Gateway"); - assert_eq!( - info.ui_url.as_deref(), - Some("https://us.helicone.ai/requests") - ); - assert!(info.capabilities.openai_chat_completions); - assert!(info.capabilities.operator_ui); - assert!(info.capabilities.observability); - } - - #[cfg(feature = "helicone")] - #[tokio::test] - async fn helicone_gateway_forwards_complete_chat_request_options() { - use crate::proxy::openai::{ChatMessage, Choice, MessageContent, Usage}; - use mockito::Matcher; - - let mut server = mockito::Server::new_async().await; - let response = ChatCompletionResponse { - id: "chatcmpl-test".to_string(), - object: "chat.completion".to_string(), - created: 1, - model: "ollama/qwen3.6:27b".to_string(), - choices: vec![Choice { - index: 0, - message: ChatMessage { - role: "assistant".to_string(), - content: Some(MessageContent::Text("ok".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }, - finish_reason: Some("stop".to_string()), - logprobs: None, - }], - usage: Usage { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - }, - system_fingerprint: None, - }; - let mock = server - .mock("POST", "/v1/chat/completions") - .match_body(Matcher::PartialJson(serde_json::json!({ - "model": "ollama/qwen3.6:27b", - "max_tokens": 16, - "temperature": 0.2, - "stream": false, - "messages": [{"role": "user", "content": "hello"}] - }))) - .with_status(200) - .with_header("content-type", "application/json") - .with_body(serde_json::to_string(&response).unwrap()) - .create_async() - .await; - - let gateway = create_gateway( - GatewayConfig { - backend_type: GatewayType::Helicone, - base_url: Some(format!("{}/v1/", server.url())), - api_key: Some("helicone-test-key".to_string()), - timeout_seconds: 30, - ..Default::default() - }, - None, - ) - .unwrap(); - - let result = gateway - .chat_completion(ChatCompletionRequest { - model: "ollama/qwen3.6:27b".to_string(), - messages: vec![ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("hello".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }], - max_tokens: Some(16), - temperature: Some(0.2), - stream: Some(false), - ..Default::default() - }) - .await - .unwrap(); - - assert_eq!(result.model, "ollama/qwen3.6:27b"); - mock.assert_async().await; - } -} diff --git a/crates/calciforge/src/proxy/gateway_tests.rs b/crates/calciforge/src/proxy/gateway_tests.rs new file mode 100644 index 00000000..49b4c2c6 --- /dev/null +++ b/crates/calciforge/src/proxy/gateway_tests.rs @@ -0,0 +1,539 @@ +use super::backend::{BackendConfig, BackendError, BackendType, create_backend}; +use super::gateway::*; +use super::openai::{ + ChatCompletionRequest, ChatCompletionResponse, ChatMessage, Choice, MessageContent, Usage, +}; +use crate::config::GatewayRetryConfig; +use mockito::Matcher; +use std::collections::HashMap; + +#[test] +fn test_gateway_type_parsing() { + assert_eq!( + "helicone".parse::().unwrap(), + GatewayType::Helicone + ); + assert_eq!( + "direct".parse::().unwrap(), + GatewayType::BuiltinHttp + ); + assert_eq!( + "http".parse::().unwrap(), + GatewayType::BuiltinHttp + ); + assert_eq!( + "builtin-http".parse::().unwrap(), + GatewayType::BuiltinHttp + ); + assert_eq!( + "litellm".parse::().unwrap(), + GatewayType::LiteLlm + ); + assert_eq!( + "lite-llm".parse::().unwrap(), + GatewayType::LiteLlm + ); + assert_eq!( + "portkey".parse::().unwrap(), + GatewayType::Portkey + ); + assert_eq!( + "tensor-zero".parse::().unwrap(), + GatewayType::TensorZero + ); + assert_eq!( + "futureagi".parse::().unwrap(), + GatewayType::FutureAgi + ); + assert_eq!( + "open-router".parse::().unwrap(), + GatewayType::OpenRouter + ); + assert_eq!("mock".parse::().unwrap(), GatewayType::Mock); + assert!("unknown".parse::().is_err()); +} + +#[test] +fn test_gateway_type_display() { + assert_eq!(GatewayType::Helicone.to_string(), "helicone"); + assert_eq!(GatewayType::BuiltinHttp.to_string(), "builtin-http"); + assert_eq!(GatewayType::LiteLlm.to_string(), "litellm"); + assert_eq!(GatewayType::Portkey.to_string(), "portkey"); + assert_eq!(GatewayType::TensorZero.to_string(), "tensorzero"); + assert_eq!(GatewayType::FutureAgi.to_string(), "future-agi"); + assert_eq!(GatewayType::OpenRouter.to_string(), "openrouter"); + assert_eq!(GatewayType::Mock.to_string(), "mock"); +} + +#[test] +fn named_gateway_engines_share_openai_compatible_http_core() { + for gateway_type in [ + GatewayType::Helicone, + GatewayType::BuiltinHttp, + GatewayType::LiteLlm, + GatewayType::Portkey, + GatewayType::TensorZero, + GatewayType::FutureAgi, + GatewayType::OpenRouter, + ] { + assert!( + gateway_type.uses_openai_compatible_http_core(), + "{gateway_type} should use the shared OpenAI-compatible HTTP core" + ); + } + assert!(!GatewayType::Mock.uses_openai_compatible_http_core()); +} + +#[test] +fn helicone_policy_headers_are_overlay_not_separate_gateway_core() { + let retry = GatewayRetryConfig { + enabled: true, + max_retries: 4, + min_timeout_ms: 250, + max_timeout_ms: 3_000, + factor: 3, + retry_on: vec![], + }; + + let headers = openai_compatible_headers(GatewayType::Helicone, Some("test-key"), &retry, None) + .expect("helicone overlay should add headers"); + + assert_eq!( + headers.get("helicone-auth"), + Some(&"Bearer test-key".to_string()) + ); + assert_eq!( + headers.get("helicone-retry-enabled"), + Some(&"true".to_string()) + ); + assert_eq!(headers.get("helicone-retry-num"), Some(&"4".to_string())); + assert_eq!( + headers.get("helicone-retry-min-timeout"), + Some(&"250".to_string()) + ); + assert_eq!( + headers.get("helicone-retry-max-timeout"), + Some(&"3000".to_string()) + ); + assert_eq!(headers.get("helicone-retry-factor"), Some(&"3".to_string())); + + assert!( + openai_compatible_headers( + GatewayType::LiteLlm, + Some("test-key"), + &GatewayRetryConfig::default(), + None + ) + .is_none(), + "LiteLLM should not inherit Helicone-specific headers" + ); +} + +#[test] +fn test_mock_gateway() { + let config = GatewayConfig { + backend_type: GatewayType::Mock, + base_url: None, + api_key: None, + timeout_seconds: 30, + extra_config: None, + headers: None, + retry: GatewayRetryConfig::default(), + ui_url: None, + }; + + let gateway = MockGateway::new(config); + assert_eq!(gateway.gateway_type(), GatewayType::Mock); +} + +#[tokio::test] +async fn mock_gateway_returns_openai_compatible_chat_choice() { + let gateway = MockGateway::new(GatewayConfig { + backend_type: GatewayType::Mock, + ..Default::default() + }); + + let response = gateway + .chat_completion(ChatCompletionRequest { + model: "gpt-4".to_string(), + messages: vec![ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("short".to_string())), + name: None, + tool_calls: None, + tool_call_id: None, + reasoning: None, + reasoning_content: None, + }], + max_tokens: Some(2), + ..Default::default() + }) + .await + .unwrap(); + + assert_eq!(response.model, "gpt-4"); + let choice = response + .choices + .first() + .expect("mock gateway should return an assistant choice"); + assert_eq!(choice.message.role, "assistant"); + let Some(MessageContent::Text(content)) = choice.message.content.as_ref() else { + panic!("mock gateway choice should contain text content"); + }; + assert!( + content.contains("gpt-4") && content.to_lowercase().contains("mock"), + "mock response content should identify the routed model: {content}" + ); +} + +#[test] +fn gateway_engine_info_carries_operator_ui_link() { + let config = GatewayConfig { + backend_type: GatewayType::Helicone, + ui_url: Some("http://127.0.0.1:8585".to_string()), + ..Default::default() + }; + + let info = config.engine_info(GatewayType::Helicone); + + assert_eq!(info.id, "helicone"); + assert_eq!(info.display_name, "Helicone AI Gateway"); + assert_eq!(info.ui_url.as_deref(), Some("http://127.0.0.1:8585")); + assert!(info.capabilities.operator_ui); + assert!(info.capabilities.observability); + assert!(!info.capabilities.model_listing); + assert!(!info.capabilities.tool_call_transcripts); + assert!(!info.capabilities.config_validation); +} + +#[test] +fn builtin_http_gateway_retries_only_configured_failure_kinds() { + let retry = GatewayRetryConfig { + enabled: true, + max_retries: 2, + min_timeout_ms: 1, + max_timeout_ms: 10, + factor: 2, + retry_on: vec![crate::config::GatewayFailureKind::ServerError], + }; + let server_error = + BackendError::http_status_error(reqwest::StatusCode::SERVICE_UNAVAILABLE, "down"); + let auth_error = BackendError::http_status_error(reqwest::StatusCode::UNAUTHORIZED, "bad key"); + + assert!(should_retry_locally( + GatewayType::BuiltinHttp, + &retry, + &server_error, + 0 + )); + assert!(!should_retry_locally( + GatewayType::BuiltinHttp, + &retry, + &auth_error, + 0 + )); + assert!(!should_retry_locally( + GatewayType::BuiltinHttp, + &retry, + &server_error, + 2 + )); +} + +#[test] +fn helicone_retry_policy_is_not_applied_twice_locally() { + let retry = GatewayRetryConfig { + enabled: true, + max_retries: 2, + min_timeout_ms: 1, + max_timeout_ms: 10, + factor: 2, + retry_on: vec![crate::config::GatewayFailureKind::ServerError], + }; + let server_error = + BackendError::http_status_error(reqwest::StatusCode::SERVICE_UNAVAILABLE, "down"); + + assert!( + !should_retry_locally(GatewayType::Helicone, &retry, &server_error, 0), + "Helicone receives retry headers, so Calciforge must not multiply attempts locally" + ); +} + +#[tokio::test] +async fn builtin_http_gateway_forwards_complete_chat_request_options() { + let mut server = mockito::Server::new_async().await; + let response = ChatCompletionResponse { + id: "chatcmpl-test".to_string(), + object: "chat.completion".to_string(), + created: 1, + model: "kimi-for-coding".to_string(), + choices: vec![Choice { + index: 0, + message: ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("ok".to_string())), + name: None, + tool_calls: None, + tool_call_id: None, + reasoning: None, + reasoning_content: None, + }, + finish_reason: Some("stop".to_string()), + logprobs: None, + }], + usage: Usage { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + }, + system_fingerprint: None, + }; + let mock = server + .mock("POST", "/v1/chat/completions") + .match_header("x-client-family", "kimi-cli") + .match_body(Matcher::PartialJson(serde_json::json!({ + "model": "kimi-for-coding", + "max_tokens": 16, + "temperature": 0.5, + "thinking": {"type": "enabled"}, + "stream": false, + "messages": [{"role": "user", "content": "hello"}] + }))) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(serde_json::to_string(&response).unwrap()) + .create_async() + .await; + + let mut headers = HashMap::new(); + headers.insert("x-client-family".to_string(), "kimi-cli".to_string()); + let backend = create_backend(&BackendConfig { + backend_type: BackendType::Http, + url: Some(format!("{}/v1", server.url())), + api_key: Some("provider-key".to_string()), + timeout_seconds: Some(30), + headers: Some(headers.clone()), + }) + .unwrap(); + let gateway = create_gateway( + GatewayConfig { + backend_type: GatewayType::BuiltinHttp, + base_url: Some(format!("{}/v1", server.url())), + api_key: Some("provider-key".to_string()), + timeout_seconds: 30, + headers: Some(headers), + ..Default::default() + }, + Some(backend), + ) + .unwrap(); + + let result = gateway + .chat_completion( + serde_json::from_value(serde_json::json!({ + "model": "kimi-for-coding", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 16, + "temperature": 0.5, + "stream": false, + "thinking": {"type": "enabled"} + })) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(result.model, "kimi/kimi-for-coding"); + mock.assert_async().await; +} + +#[test] +fn create_openai_compatible_gateway_preserves_engine_metadata_through_logging_wrapper() { + let backend = create_backend(&BackendConfig { + backend_type: BackendType::Http, + url: Some("http://127.0.0.1:8787/v1".to_string()), + api_key: Some("helicone-test-key".to_string()), + timeout_seconds: Some(30), + ..Default::default() + }) + .unwrap(); + let gateway = create_gateway( + GatewayConfig { + backend_type: GatewayType::Helicone, + base_url: Some("https://ai-gateway.helicone.ai".to_string()), + api_key: Some("helicone-test-key".to_string()), + ui_url: Some("https://us.helicone.ai/requests".to_string()), + ..Default::default() + }, + Some(backend), + ) + .unwrap(); + + let info = gateway.engine_info(); + + assert_eq!(gateway.gateway_type(), GatewayType::Helicone); + assert_eq!(info.id, "helicone"); + assert_eq!(info.display_name, "Helicone AI Gateway"); + assert_eq!( + info.ui_url.as_deref(), + Some("https://us.helicone.ai/requests") + ); + assert!(info.capabilities.openai_chat_completions); + assert!(info.capabilities.operator_ui); + assert!(info.capabilities.observability); +} + +#[tokio::test] +async fn helicone_engine_uses_shared_http_core_with_engine_headers() { + let mut server = mockito::Server::new_async().await; + let response = ChatCompletionResponse { + id: "chatcmpl-test".to_string(), + object: "chat.completion".to_string(), + created: 1, + model: "ollama/qwen3.6:27b".to_string(), + choices: vec![Choice { + index: 0, + message: ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("ok".to_string())), + name: None, + tool_calls: None, + tool_call_id: None, + reasoning: None, + reasoning_content: None, + }, + finish_reason: Some("stop".to_string()), + logprobs: None, + }], + usage: Usage { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + }, + system_fingerprint: None, + }; + let mock = server + .mock("POST", "/v1/chat/completions") + .match_header("authorization", "Bearer helicone-test-key") + .match_header("helicone-auth", "Bearer helicone-test-key") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(serde_json::to_string(&response).unwrap()) + .create_async() + .await; + + let headers = openai_compatible_headers( + GatewayType::Helicone, + Some("helicone-test-key"), + &GatewayRetryConfig::default(), + None, + ); + let backend = create_backend(&BackendConfig { + backend_type: BackendType::Http, + url: Some(format!("{}/v1", server.url())), + api_key: Some("helicone-test-key".to_string()), + timeout_seconds: Some(30), + headers: headers.clone(), + }) + .unwrap(); + let gateway = create_gateway( + GatewayConfig { + backend_type: GatewayType::Helicone, + base_url: Some(format!("{}/v1/", server.url())), + api_key: Some("helicone-test-key".to_string()), + timeout_seconds: 30, + headers, + ..Default::default() + }, + Some(backend), + ) + .unwrap(); + + let result = gateway + .chat_completion(ChatCompletionRequest { + model: "ollama/qwen3.6:27b".to_string(), + messages: vec![ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("hello".to_string())), + name: None, + tool_calls: None, + tool_call_id: None, + reasoning: None, + reasoning_content: None, + }], + max_tokens: Some(16), + temperature: Some(0.2), + stream: Some(false), + ..Default::default() + }) + .await + .unwrap(); + + assert_eq!(result.model, "ollama/qwen3.6:27b"); + mock.assert_async().await; +} + +#[tokio::test] +async fn litellm_engine_parses_streaming_response_from_shared_http_core() { + let mut server = mockito::Server::new_async().await; + let body = concat!( + "data: {\"id\":\"chatcmpl-litellm-stream\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"qwen3.6\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"po\"},\"finish_reason\":null}]}\n\n", + "data: {\"id\":\"chatcmpl-litellm-stream\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"qwen3.6\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ng\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n", + "data: [DONE]\n\n", + ); + let mock = server + .mock("POST", "/v1/chat/completions") + .with_status(200) + .with_header("content-type", "text/event-stream; charset=utf-8") + .with_body(body) + .create_async() + .await; + + let backend = create_backend(&BackendConfig { + backend_type: BackendType::Http, + url: Some(format!("{}/v1", server.url())), + timeout_seconds: Some(30), + ..Default::default() + }) + .unwrap(); + let gateway = create_gateway( + GatewayConfig { + backend_type: GatewayType::LiteLlm, + base_url: Some(format!("{}/v1", server.url())), + timeout_seconds: 30, + ..Default::default() + }, + Some(backend), + ) + .unwrap(); + + let result = gateway + .chat_completion(ChatCompletionRequest { + model: "qwen3.6".to_string(), + messages: vec![ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("hello".to_string())), + name: None, + tool_calls: None, + tool_call_id: None, + reasoning: None, + reasoning_content: None, + }], + stream: Some(true), + ..Default::default() + }) + .await + .unwrap(); + + assert_eq!(result.model, "qwen3.6"); + assert_eq!( + result.choices[0] + .message + .content + .as_ref() + .and_then(MessageContent::to_text) + .as_deref(), + Some("pong") + ); + mock.assert_async().await; +} diff --git a/crates/calciforge/src/proxy/helicone_router.rs b/crates/calciforge/src/proxy/helicone_router.rs deleted file mode 100644 index 5448388b..00000000 --- a/crates/calciforge/src/proxy/helicone_router.rs +++ /dev/null @@ -1,384 +0,0 @@ -//! Helicone Router - HTTP-based router for Helicone AI Gateway -//! -//! This module provides a router that sends requests to a Helicone AI Gateway -//! instance via HTTP. This is the recommended approach since ai-gateway is -//! designed as a server application, not an embedded library. - -use async_trait::async_trait; -use reqwest::{ - Client, - header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue}, -}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::time::Duration; -use thiserror::Error; -use url::Url; - -use crate::{ - config::GatewayRetryConfig, - proxy::backend::{BackendError, BackendType, ModelInfo, SecretsBackend}, - proxy::helicone_streaming::parse_streaming_chat_completion, - proxy::openai::{ - ChatCompletionRequest, ChatCompletionResponse, ChatMessage, ToolChoice, ToolDefinition, - }, -}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HeliconeRouterConfig { - /// Base URL of the Helicone AI Gateway instance - pub base_url: String, - /// API key for Helicone - pub api_key: String, - /// Timeout in seconds for requests - pub timeout_seconds: u64, - /// Router name for identification - pub router_name: String, - /// Enable response caching - pub enable_caching: bool, - /// Cache TTL in seconds - pub cache_ttl_seconds: u64, - /// Custom headers forwarded to the Helicone AI Gateway. - #[serde(default)] - pub headers: HashMap, - /// Gateway retry policy. Mapped to Helicone retry headers when enabled. - #[serde(default)] - pub retry: GatewayRetryConfig, -} - -impl Default for HeliconeRouterConfig { - fn default() -> Self { - Self { - base_url: "http://localhost:8787".to_string(), - api_key: "".to_string(), - timeout_seconds: 30, - router_name: "helicone".to_string(), - enable_caching: false, - cache_ttl_seconds: 300, - headers: HashMap::new(), - retry: GatewayRetryConfig::default(), - } - } -} - -#[derive(Debug, Error)] -#[allow(dead_code)] -pub enum HeliconeError { - #[error("Configuration error: {0}")] - Config(String), - #[error("HTTP client error: {0}")] - HttpClient(String), - #[error("Request error: {0}")] - Request(String), - #[error("Response error: {0}")] - Response(String), - #[error("Serialization error: {0}")] - Serialization(#[from] serde_json::Error), - #[error("Timeout error: {0}")] - Timeout(String), -} - -impl From for BackendError { - fn from(err: HeliconeError) -> Self { - BackendError::ConfigError(err.to_string()) - } -} - -#[derive(Debug)] -pub struct HeliconeRouter { - config: HeliconeRouterConfig, - client: Client, -} - -impl HeliconeRouter { - pub fn new(config: HeliconeRouterConfig) -> Result { - let client = Client::builder() - .timeout(Duration::from_secs(config.timeout_seconds)) - .build() - .map_err(|e| { - HeliconeError::HttpClient(format!("Failed to create HTTP client: {}", e)) - })?; - - Ok(Self { config, client }) - } - - fn chat_completions_url(&self) -> Result { - helicone_chat_completions_url(&self.config.base_url) - } - - /// Create a default router with standard configuration - #[allow(dead_code)] - pub fn default() -> Result { - Self::new(HeliconeRouterConfig::default()) - } - - pub async fn chat_completion( - &self, - model: String, - messages: Vec, - stream: bool, - tools: Option>, - tool_choice: Option, - ) -> Result { - self.chat_completion_request(ChatCompletionRequest { - model, - messages, - stream: Some(stream), - tools, - tool_choice, - ..Default::default() - }) - .await - } - - pub async fn chat_completion_request( - &self, - request_body: ChatCompletionRequest, - ) -> Result { - let url = self.chat_completions_url().map_err(BackendError::from)?; - let url_for_error = url.as_str().to_string(); - let model_for_error = request_body.model.clone(); - - let mut headers = HeaderMap::new(); - for (name, value) in &self.config.headers { - let header_name = HeaderName::from_bytes(name.as_bytes()).map_err(|e| { - BackendError::ConfigError(format!("Invalid Helicone custom header '{name}': {e}")) - })?; - let header_value = HeaderValue::from_str(value).map_err(|e| { - BackendError::ConfigError(format!( - "Invalid value for Helicone custom header '{name}': {e}" - )) - })?; - headers.insert(header_name, header_value); - } - if self.config.retry.enabled { - headers.insert( - HeaderName::from_static("helicone-retry-enabled"), - HeaderValue::from_static("true"), - ); - headers.insert( - HeaderName::from_static("helicone-retry-num"), - HeaderValue::from_str(&self.config.retry.max_retries.to_string()).map_err(|e| { - BackendError::ConfigError(format!("Invalid Helicone retry count: {e}")) - })?, - ); - headers.insert( - HeaderName::from_static("helicone-retry-min-timeout"), - HeaderValue::from_str(&self.config.retry.min_timeout_ms.to_string()).map_err( - |e| { - BackendError::ConfigError(format!( - "Invalid Helicone retry minimum timeout: {e}" - )) - }, - )?, - ); - headers.insert( - HeaderName::from_static("helicone-retry-max-timeout"), - HeaderValue::from_str(&self.config.retry.max_timeout_ms.to_string()).map_err( - |e| { - BackendError::ConfigError(format!( - "Invalid Helicone retry maximum timeout: {e}" - )) - }, - )?, - ); - headers.insert( - HeaderName::from_static("helicone-retry-factor"), - HeaderValue::from_str(&self.config.retry.factor.to_string()).map_err(|e| { - BackendError::ConfigError(format!("Invalid Helicone retry factor: {e}")) - })?, - ); - } - let bearer = - HeaderValue::from_str(&format!("Bearer {}", self.config.api_key)).map_err(|e| { - BackendError::ConfigError(format!("Invalid Helicone API key for auth header: {e}")) - })?; - headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - headers.insert(AUTHORIZATION, bearer.clone()); - headers.insert(HeaderName::from_static("helicone-auth"), bearer); - - let response = self - .client - .post(url) - .headers(headers) - .json(&request_body) - .send() - .await - .map_err(|e| { - BackendError::transport( - format!( - "Helicone request to {} for model '{}' failed: {}", - url_for_error, model_for_error, e - ), - e.is_timeout(), - ) - })?; - - if !response.status().is_success() { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - return Err(BackendError::http_status_error( - status, - format!( - "Helicone gateway returned {} for model '{}': {}", - status, - model_for_error, - truncate_error_body(error_text.trim()) - ), - )); - } - - let content_type = response - .headers() - .get(CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_ascii_lowercase(); - if content_type.contains("text/event-stream") { - let body = response.text().await.map_err(|e| { - BackendError::transport( - format!( - "Failed to read Helicone streaming response for model '{}': {}", - model_for_error, e - ), - e.is_timeout(), - ) - })?; - return parse_streaming_chat_completion(&body, &model_for_error); - } - - let completion_response: ChatCompletionResponse = response.json().await.map_err(|e| { - BackendError::InvalidResponse(format!( - "Failed to parse Helicone response for model '{}': {}", - model_for_error, e - )) - })?; - - Ok(completion_response) - } - - pub async fn list_models(&self) -> Result, BackendError> { - // Helicone doesn't have a standard models endpoint, so we return - // a placeholder list or fetch from the underlying provider - // For now, return an empty list - Ok(vec![]) - } -} - -pub(super) fn helicone_chat_completions_url(base_url: &str) -> Result { - let trimmed = base_url.trim(); - if trimmed.is_empty() { - return Err(HeliconeError::Config( - "Helicone base_url cannot be blank".to_string(), - )); - } - - let mut url = Url::parse(trimmed).map_err(|e| { - HeliconeError::Config(format!( - "Helicone base_url '{}' is invalid: {}", - base_url, e - )) - })?; - if url.query().is_some() || url.fragment().is_some() { - return Err(HeliconeError::Config( - "Helicone base_url must not include query parameters or fragments".to_string(), - )); - } - - let path = url.path().trim_end_matches('/'); - let chat_path = if path.is_empty() { - "/v1/chat/completions".to_string() - } else if path.ends_with("/chat/completions") { - path.to_string() - } else { - format!("{path}/chat/completions") - }; - url.set_path(&chat_path); - Ok(url) -} - -fn truncate_error_body(body: &str) -> String { - const MAX_ERROR_BODY_CHARS: usize = 1024; - let mut chars = body.chars(); - let truncated: String = chars.by_ref().take(MAX_ERROR_BODY_CHARS).collect(); - if chars.next().is_some() { - format!("{truncated}...") - } else { - truncated - } -} - -// --------------------------------------------------------------------------- -// Router trait implementation -// --------------------------------------------------------------------------- - -#[async_trait] -#[allow(dead_code)] -pub trait Router: Send + Sync { - async fn chat_completion( - &self, - model: String, - messages: Vec, - stream: bool, - tools: Option>, - tool_choice: Option, - ) -> Result; - - async fn list_models(&self) -> Result, BackendError>; -} - -#[async_trait] -impl Router for HeliconeRouter { - async fn chat_completion( - &self, - model: String, - messages: Vec, - stream: bool, - tools: Option>, - tool_choice: Option, - ) -> Result { - self.chat_completion(model, messages, stream, tools, tool_choice) - .await - } - - async fn list_models(&self) -> Result, BackendError> { - self.list_models().await - } -} - -// --------------------------------------------------------------------------- -// SecretsBackend implementation -// --------------------------------------------------------------------------- - -#[async_trait] -impl SecretsBackend for HeliconeRouter { - async fn chat_completion( - &self, - model: String, - messages: Vec, - stream: bool, - tools: Option>, - tool_choice: Option, - ) -> Result { - self.chat_completion(model, messages, stream, tools, tool_choice) - .await - } - - async fn chat_completion_request( - &self, - request: ChatCompletionRequest, - ) -> Result { - HeliconeRouter::chat_completion_request(self, request).await - } - - async fn list_models(&self) -> Result, BackendError> { - self.list_models().await - } - - fn backend_type(&self) -> BackendType { - BackendType::Helicone - } -} diff --git a/crates/calciforge/src/proxy/helicone_router_tests.rs b/crates/calciforge/src/proxy/helicone_router_tests.rs deleted file mode 100644 index 09da34be..00000000 --- a/crates/calciforge/src/proxy/helicone_router_tests.rs +++ /dev/null @@ -1,382 +0,0 @@ -use std::collections::HashMap; - -use super::helicone_router::{HeliconeRouter, HeliconeRouterConfig, helicone_chat_completions_url}; -use super::openai::{ChatCompletionResponse, ChatMessage, Choice, MessageContent, Usage}; -use crate::config::GatewayRetryConfig; -use mockito::Matcher; - -fn config(base_url: String) -> HeliconeRouterConfig { - HeliconeRouterConfig { - base_url, - api_key: "helicone-test-key".to_string(), - timeout_seconds: 30, - router_name: "test".to_string(), - enable_caching: false, - cache_ttl_seconds: 300, - headers: HashMap::new(), - retry: GatewayRetryConfig::default(), - } -} - -#[test] -fn test_helicone_router_creation() { - let router = HeliconeRouter::new(config("http://localhost:8787".to_string())); - assert!(router.is_ok()); -} - -#[test] -fn test_default_router() { - let router = HeliconeRouter::default(); - assert!(router.is_ok()); -} - -#[test] -fn helicone_url_adds_v1_path_for_origin_base() { - let url = helicone_chat_completions_url("https://ai-gateway.helicone.ai").unwrap(); - assert_eq!( - url.as_str(), - "https://ai-gateway.helicone.ai/v1/chat/completions" - ); -} - -#[test] -fn helicone_url_uses_configured_gateway_base_path() { - let url = helicone_chat_completions_url("https://gateway.example.invalid/router/calciforge/") - .unwrap(); - assert_eq!( - url.as_str(), - "https://gateway.example.invalid/router/calciforge/chat/completions" - ); -} - -#[test] -fn helicone_url_does_not_duplicate_v1_path() { - let url = helicone_chat_completions_url("https://ai-gateway.helicone.ai/v1/").unwrap(); - assert_eq!( - url.as_str(), - "https://ai-gateway.helicone.ai/v1/chat/completions" - ); -} - -#[test] -fn helicone_url_rejects_query_or_fragment_base() { - let err = helicone_chat_completions_url("https://ai-gateway.helicone.ai/v1?debug=true") - .unwrap_err() - .to_string(); - assert!( - err.contains("query parameters or fragments"), - "unexpected error: {err}" - ); - - let err = helicone_chat_completions_url("https://ai-gateway.helicone.ai/v1#dashboard") - .unwrap_err() - .to_string(); - assert!( - err.contains("query parameters or fragments"), - "unexpected error: {err}" - ); -} - -#[tokio::test] -async fn chat_completion_posts_to_configured_v1_path_without_duplication() { - let mut server = mockito::Server::new_async().await; - let response = ChatCompletionResponse { - id: "chatcmpl-test".to_string(), - object: "chat.completion".to_string(), - created: 1, - model: "openai/gpt-4o-mini".to_string(), - choices: vec![Choice { - index: 0, - message: ChatMessage { - role: "assistant".to_string(), - content: Some(MessageContent::Text("ok".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }, - finish_reason: Some("stop".to_string()), - logprobs: None, - }], - usage: Usage { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - }, - system_fingerprint: None, - }; - let mock = server - .mock("POST", "/v1/chat/completions") - .match_header("authorization", "Bearer helicone-test-key") - .match_header("helicone-auth", "Bearer helicone-test-key") - .match_body(Matcher::PartialJson(serde_json::json!({ - "model": "openai/gpt-4o-mini", - "messages": [{"role": "user", "content": "hello"}] - }))) - .with_status(200) - .with_header("content-type", "application/json") - .with_body(serde_json::to_string(&response).unwrap()) - .create_async() - .await; - - let router = HeliconeRouter::new(config(format!("{}/v1/", server.url()))).unwrap(); - let result = router - .chat_completion( - "openai/gpt-4o-mini".to_string(), - vec![ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("hello".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }], - false, - None, - None, - ) - .await - .unwrap(); - - assert_eq!(result.model, "openai/gpt-4o-mini"); - mock.assert_async().await; -} - -#[tokio::test] -async fn chat_completion_accepts_helicone_streaming_response() { - let mut server = mockito::Server::new_async().await; - let body = concat!( - "data: {\"id\":\"chatcmpl-stream\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"openai/gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"po\"},\"finish_reason\":null}]}\n\n", - "data: {\"id\":\"chatcmpl-stream\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"openai/gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ng\"},\"finish_reason\":\"stop\"}]}\n\n", - "data: [DONE]\n\n", - ); - let mock = server - .mock("POST", "/v1/chat/completions") - .match_body(Matcher::PartialJson(serde_json::json!({ - "model": "openai/gpt-4o-mini", - "stream": true - }))) - .with_status(200) - .with_header("content-type", "text/event-stream; charset=utf-8") - .with_body(body) - .create_async() - .await; - - let router = HeliconeRouter::new(config(format!("{}/v1/", server.url()))).unwrap(); - let result = router - .chat_completion( - "openai/gpt-4o-mini".to_string(), - vec![ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("hello".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }], - true, - None, - None, - ) - .await - .unwrap(); - - assert_eq!( - result.choices[0] - .message - .content - .as_ref() - .and_then(MessageContent::to_text) - .as_deref(), - Some("pong") - ); - mock.assert_async().await; -} - -#[tokio::test] -async fn chat_completion_forwards_custom_headers() { - let mut server = mockito::Server::new_async().await; - let response = ChatCompletionResponse { - id: "chatcmpl-test".to_string(), - object: "chat.completion".to_string(), - created: 1, - model: "openai/gpt-4o-mini".to_string(), - choices: vec![Choice { - index: 0, - message: ChatMessage { - role: "assistant".to_string(), - content: Some(MessageContent::Text("ok".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }, - finish_reason: Some("stop".to_string()), - logprobs: None, - }], - usage: Usage { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - }, - system_fingerprint: None, - }; - let mock = server - .mock("POST", "/v1/chat/completions") - .match_header("authorization", "Bearer helicone-test-key") - .match_header("helicone-auth", "Bearer helicone-test-key") - .match_header("x-provider-scope", "local-ollama") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(serde_json::to_string(&response).unwrap()) - .create_async() - .await; - - let mut cfg = config(format!("{}/v1/", server.url())); - cfg.headers - .insert("x-provider-scope".to_string(), "local-ollama".to_string()); - cfg.headers - .insert("authorization".to_string(), "Bearer wrong".to_string()); - cfg.headers - .insert("helicone-auth".to_string(), "Bearer wrong".to_string()); - let router = HeliconeRouter::new(cfg).unwrap(); - router - .chat_completion( - "openai/gpt-4o-mini".to_string(), - vec![ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("hello".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }], - false, - None, - None, - ) - .await - .unwrap(); - - mock.assert_async().await; -} - -#[tokio::test] -async fn chat_completion_maps_retry_config_to_helicone_headers() { - let mut server = mockito::Server::new_async().await; - let response = ChatCompletionResponse { - id: "chatcmpl-test".to_string(), - object: "chat.completion".to_string(), - created: 1, - model: "openai/gpt-4o-mini".to_string(), - choices: vec![Choice { - index: 0, - message: ChatMessage { - role: "assistant".to_string(), - content: Some(MessageContent::Text("ok".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }, - finish_reason: Some("stop".to_string()), - logprobs: None, - }], - usage: Usage { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - }, - system_fingerprint: None, - }; - let mock = server - .mock("POST", "/v1/chat/completions") - .match_header("helicone-retry-enabled", "true") - .match_header("helicone-retry-num", "4") - .match_header("helicone-retry-min-timeout", "250") - .match_header("helicone-retry-max-timeout", "3000") - .match_header("helicone-retry-factor", "3") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(serde_json::to_string(&response).unwrap()) - .create_async() - .await; - - let mut cfg = config(format!("{}/v1/", server.url())); - cfg.retry.enabled = true; - cfg.retry.max_retries = 4; - cfg.retry.min_timeout_ms = 250; - cfg.retry.max_timeout_ms = 3000; - cfg.retry.factor = 3; - let router = HeliconeRouter::new(cfg).unwrap(); - router - .chat_completion( - "openai/gpt-4o-mini".to_string(), - vec![ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("hello".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }], - false, - None, - None, - ) - .await - .unwrap(); - - mock.assert_async().await; -} - -#[tokio::test] -async fn chat_completion_error_names_gateway_and_model_without_full_body_dump() { - let mut server = mockito::Server::new_async().await; - let long_body = format!("{}{}", "denied: ", "x".repeat(2048)); - let mock = server - .mock("POST", "/v1/chat/completions") - .with_status(503) - .with_header("content-type", "text/plain") - .with_body(long_body) - .create_async() - .await; - - let router = HeliconeRouter::new(config(format!("{}/v1/", server.url()))).unwrap(); - let err = router - .chat_completion( - "openai/gpt-4o-mini".to_string(), - vec![ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("hello".to_string())), - name: None, - tool_calls: None, - tool_call_id: None, - reasoning: None, - reasoning_content: None, - }], - false, - None, - None, - ) - .await - .unwrap_err() - .to_string(); - - assert!(err.contains("503 Service Unavailable"), "{err}"); - assert!(err.contains("openai/gpt-4o-mini"), "{err}"); - assert!(err.contains("denied:"), "{err}"); - assert!( - err.len() < 1300, - "error should be truncated instead of dumping full upstream body: {} bytes", - err.len() - ); - mock.assert_async().await; -} diff --git a/crates/calciforge/src/proxy/mod.rs b/crates/calciforge/src/proxy/mod.rs index 112d6fd6..5302e931 100644 --- a/crates/calciforge/src/proxy/mod.rs +++ b/crates/calciforge/src/proxy/mod.rs @@ -23,23 +23,18 @@ use crate::providers::alloy::AlloyManager; mod auth; mod backend; mod control_auth; -mod gateway; +pub(crate) mod gateway; +#[cfg(test)] +mod gateway_tests; mod handlers; pub(crate) mod model_resolver; mod openai; +mod openai_streaming; pub(crate) mod routing; mod streaming; mod token_estimator; mod voice_handlers; -// Helicone AI Gateway router (HTTP-based) -#[cfg(feature = "helicone")] -mod helicone_router; -#[cfg(all(test, feature = "helicone"))] -mod helicone_router_tests; -#[cfg(feature = "helicone")] -mod helicone_streaming; - pub use openai::ChatCompletionRequest; pub use routing::ProviderEntry; @@ -91,24 +86,19 @@ fn resolve_api_key( Ok(api_key.and_then(normalize_api_key)) } -const SUPPORTED_ROOT_GATEWAY_BACKEND_TYPES: &[&str] = &["http", "helicone", "mock"]; - pub(crate) fn supported_root_gateway_backend_types() -> &'static [&'static str] { - SUPPORTED_ROOT_GATEWAY_BACKEND_TYPES + gateway::GatewayType::SUPPORTED_CONFIG_NAMES } -fn gateway_type_for_backend_type(backend_type: &str) -> gateway::GatewayType { - match backend_type { - "helicone" => gateway::GatewayType::Helicone, - "mock" => gateway::GatewayType::Mock, - _ => gateway::GatewayType::BuiltinHttp, - } +pub(crate) fn gateway_type_for_backend_type(backend_type: &str) -> Option { + backend_type.parse().ok() } /// Return true when the configured root gateway is authoritative for model IDs /// that are not enumerated in Calciforge provider routes. pub(crate) fn backend_accepts_unlisted_models(backend_type: &str) -> bool { - matches!(backend_type, "http" | "helicone") + gateway_type_for_backend_type(backend_type) + .is_some_and(|gateway_type| gateway_type.requires_backend_url()) } fn validate_explicit_provider_selection(config: &ProxyConfig) -> anyhow::Result<()> { @@ -117,8 +107,7 @@ fn validate_explicit_provider_selection(config: &ProxyConfig) -> anyhow::Result< "proxy.enabled=true requires at least one explicit [[proxy.providers]] adapter or an explicit non-mock root backend_type. The mock adapter is test-only and is not a production default." ); } - if matches!(config.backend_type.as_str(), "http" | "helicone") - && config.backend_url.trim().is_empty() + if backend_accepts_unlisted_models(&config.backend_type) && config.backend_url.trim().is_empty() { anyhow::bail!( "proxy.enabled=true with root backend_type='{}' requires backend_url. Use backend_type='mock' for explicit-provider-only configs where unmatched models should fail instead of falling back to a root provider.", @@ -182,33 +171,41 @@ pub async fn start_proxy_server( )?; // Create backend based on config - let backend_config = match config.backend_type.as_str() { - "http" => backend::BackendConfig { - backend_type: backend::BackendType::Http, - url: Some(config.backend_url.clone()), - api_key: default_api_key.clone(), - timeout_seconds: Some(config.timeout_seconds), - headers: config.headers.clone(), - ..Default::default() - }, - "helicone" => backend::BackendConfig { - backend_type: backend::BackendType::Helicone, - helicone_url: Some(config.backend_url.clone()), - helicone_api_key: default_api_key.clone(), - timeout_seconds: Some(config.timeout_seconds), - headers: config.headers.clone(), - ..Default::default() - }, - "mock" => backend::BackendConfig { + let gateway_type = gateway_type_for_backend_type(&config.backend_type).ok_or_else(|| { + anyhow::anyhow!( + "Unsupported proxy backend_type '{}'. Supported root provider adapters: {}", + config.backend_type, + supported_root_gateway_backend_types().join(", ") + ) + })?; + + let backend_config = match gateway_type { + gateway::GatewayType::Helicone + | gateway::GatewayType::BuiltinHttp + | gateway::GatewayType::LiteLlm + | gateway::GatewayType::Portkey + | gateway::GatewayType::TensorZero + | gateway::GatewayType::FutureAgi + | gateway::GatewayType::OpenRouter => { + let headers = gateway::openai_compatible_headers( + gateway_type, + default_api_key.as_deref(), + &config.retry, + config.headers.as_ref(), + ); + backend::BackendConfig { + backend_type: backend::BackendType::Http, + url: Some(config.backend_url.clone()), + api_key: default_api_key.clone(), + timeout_seconds: Some(config.timeout_seconds), + headers, + } + } + gateway::GatewayType::Mock => backend::BackendConfig { backend_type: backend::BackendType::Mock, headers: config.headers.clone(), ..Default::default() }, - other => anyhow::bail!( - "Unsupported proxy backend_type '{}'. Supported root provider adapters: {}", - other, - supported_root_gateway_backend_types().join(", ") - ), }; info!( @@ -220,9 +217,6 @@ pub async fn start_proxy_server( let backend = backend::create_backend(&backend_config) .map_err(|e| anyhow::anyhow!("Failed to create backend: {}", e))?; - // Determine gateway type based on configuration - let gateway_type = gateway_type_for_backend_type(&config.backend_type); - let gateway_config = gateway::GatewayConfig { backend_type: gateway_type, base_url: Some(config.backend_url.clone()), @@ -313,22 +307,29 @@ mod tests { fn gateway_type_for_backend_type_preserves_supported_external_engines() { assert_eq!( gateway_type_for_backend_type("helicone"), - gateway::GatewayType::Helicone + Some(gateway::GatewayType::Helicone) ); assert_eq!( gateway_type_for_backend_type("http"), - gateway::GatewayType::BuiltinHttp + Some(gateway::GatewayType::BuiltinHttp) + ); + assert_eq!( + gateway_type_for_backend_type("litellm"), + Some(gateway::GatewayType::LiteLlm) ); assert_eq!( gateway_type_for_backend_type("mock"), - gateway::GatewayType::Mock + Some(gateway::GatewayType::Mock) ); + assert_eq!(gateway_type_for_backend_type("unknown"), None); } #[test] fn unlisted_model_acceptance_is_shared_for_runtime_and_doctor() { assert!(backend_accepts_unlisted_models("helicone")); assert!(backend_accepts_unlisted_models("http")); + assert!(backend_accepts_unlisted_models("litellm")); + assert!(backend_accepts_unlisted_models("openrouter")); assert!(!backend_accepts_unlisted_models("mock")); } @@ -336,7 +337,16 @@ mod tests { fn supported_root_backend_allowlist_is_small_and_explicit() { assert_eq!( supported_root_gateway_backend_types(), - ["http", "helicone", "mock"] + [ + "http", + "helicone", + "litellm", + "portkey", + "tensorzero", + "future-agi", + "openrouter", + "mock" + ] ); } } diff --git a/crates/calciforge/src/proxy/helicone_streaming.rs b/crates/calciforge/src/proxy/openai_streaming.rs similarity index 94% rename from crates/calciforge/src/proxy/helicone_streaming.rs rename to crates/calciforge/src/proxy/openai_streaming.rs index af0f456d..2f34fa47 100644 --- a/crates/calciforge/src/proxy/helicone_streaming.rs +++ b/crates/calciforge/src/proxy/openai_streaming.rs @@ -60,7 +60,7 @@ pub(super) fn parse_streaming_chat_completion( saw_chunk = true; let value: serde_json::Value = serde_json::from_str(data).map_err(|e| { BackendError::InvalidResponse(format!( - "Failed to parse Helicone streaming chunk for model '{}': {}", + "Failed to parse OpenAI-compatible streaming chunk for model '{}': {}", requested_model, e )) })?; @@ -94,7 +94,7 @@ pub(super) fn parse_streaming_chat_completion( .and_then(|v| v.as_array()) .ok_or_else(|| { BackendError::InvalidResponse(format!( - "Helicone streaming chunk for model '{}' did not include choices", + "OpenAI-compatible streaming chunk for model '{}' did not include choices", requested_model )) })?; @@ -104,13 +104,13 @@ pub(super) fn parse_streaming_chat_completion( .and_then(|v| v.as_u64()) .ok_or_else(|| { BackendError::InvalidResponse(format!( - "Helicone streaming choice for model '{}' did not include a numeric index", + "OpenAI-compatible streaming choice for model '{}' did not include a numeric index", requested_model )) })?; let index = u32::try_from(index).map_err(|_| { BackendError::InvalidResponse(format!( - "Helicone streaming choice index for model '{}' exceeded u32", + "OpenAI-compatible streaming choice index for model '{}' exceeded u32", requested_model )) })?; @@ -146,13 +146,13 @@ pub(super) fn parse_streaming_chat_completion( .transpose() .map_err(|_| { BackendError::InvalidResponse(format!( - "Helicone streaming tool call index for model '{}' exceeded u32", + "OpenAI-compatible streaming tool call index for model '{}' exceeded u32", requested_model )) })? .unwrap_or(u32::try_from(fallback_index).map_err(|_| { BackendError::InvalidResponse(format!( - "Helicone streaming tool call index for model '{}' exceeded u32", + "OpenAI-compatible streaming tool call index for model '{}' exceeded u32", requested_model )) })?); @@ -178,20 +178,20 @@ pub(super) fn parse_streaming_chat_completion( if !saw_chunk { return Err(BackendError::InvalidResponse(format!( - "Helicone streaming response for model '{}' did not include any chunks", + "OpenAI-compatible streaming response for model '{}' did not include any chunks", requested_model ))); } let Some(id) = id else { return Err(BackendError::InvalidResponse(format!( - "Helicone streaming response for model '{}' did not include an id", + "OpenAI-compatible streaming response for model '{}' did not include an id", requested_model ))); }; let Some(created) = created else { return Err(BackendError::InvalidResponse(format!( - "Helicone streaming response for model '{}' did not include created", + "OpenAI-compatible streaming response for model '{}' did not include created", requested_model ))); }; @@ -202,13 +202,13 @@ pub(super) fn parse_streaming_chat_completion( for (tool_index, call) in accumulator.tool_calls { let id = call.id.ok_or_else(|| { BackendError::InvalidResponse(format!( - "Helicone streaming tool call for model '{}' did not include an id", + "OpenAI-compatible streaming tool call for model '{}' did not include an id", requested_model )) })?; let name = call.name.ok_or_else(|| { BackendError::InvalidResponse(format!( - "Helicone streaming tool call for model '{}' did not include a function name", + "OpenAI-compatible streaming tool call for model '{}' did not include a function name", requested_model )) })?; @@ -269,7 +269,7 @@ pub(super) fn parse_streaming_chat_completion( if choice_entries.is_empty() { return Err(BackendError::InvalidResponse(format!( - "Helicone streaming response for model '{}' did not include any choices", + "OpenAI-compatible streaming response for model '{}' did not include any choices", requested_model ))); } @@ -300,7 +300,7 @@ fn parse_usage_count( }; u32::try_from(value).map_err(|_| { BackendError::InvalidResponse(format!( - "Helicone streaming usage field '{}' exceeded u32", + "OpenAI-compatible streaming usage field '{}' exceeded u32", field )) }) diff --git a/crates/calciforge/src/proxy/routing.rs b/crates/calciforge/src/proxy/routing.rs index d8208fd0..830aee33 100644 --- a/crates/calciforge/src/proxy/routing.rs +++ b/crates/calciforge/src/proxy/routing.rs @@ -143,63 +143,46 @@ pub fn build_provider_entries( provider_switch_state .entry(p.id.clone()) .or_insert_with(|| Arc::new(ProviderSwitchState::default())); - if p.backend_type == "helicone" { - if p.url.trim().is_empty() { - anyhow::bail!( - "provider '{}' with backend_type 'helicone' requires non-empty url", - p.id - ); - } - let api_key = resolve_provider_api_key(p)?; - let timeout = p.timeout_seconds.unwrap_or(default_timeout); - let headers: Option> = if p.headers.is_empty() { - None - } else { - Some(p.headers.clone()) - }; - let gw_cfg = GatewayConfig { - backend_type: GatewayType::Helicone, - base_url: Some(p.url.clone()), - api_key, - timeout_seconds: timeout, - extra_config: None, - headers, - retry: p.retry.clone().unwrap_or_else(|| config.retry.clone()), - ui_url: None, - }; - let gw = gateway::create_gateway(gw_cfg, None) - .with_context(|| format!("creating Helicone gateway for provider '{}'", p.id))?; - info!(id = %p.id, url = %p.url, models = ?p.models, "Helicone provider loaded"); - provider_gateways.insert(p.id.clone(), gw); - provider_on_switch.insert(p.id.clone(), p.on_switch.clone()); - provider_strip_prefix.insert(p.id.clone(), normalized_strip_prefix(p)); - provider_add_prefix.insert(p.id.clone(), normalized_add_prefix(p)); - provider_request_body.insert(p.id.clone(), request_body_map(p)); - continue; - } - if p.backend_type != "http" { + let gateway_type: GatewayType = p.backend_type.parse().map_err(|_| { + anyhow::anyhow!( + "provider '{}' has unsupported backend_type '{}'; use one of: {}. CLI-backed subscriptions must be configured as [[agents]], not gateway providers.", + p.id, + p.backend_type, + GatewayType::SUPPORTED_PROVIDER_CONFIG_NAMES.join(", ") + ) + })?; + if !gateway_type.uses_openai_compatible_http_core() { anyhow::bail!( - "provider '{}' has unsupported backend_type '{}'; use 'http' or 'helicone'. CLI-backed subscriptions must be configured as [[agents]], not gateway providers.", + "provider '{}' has unsupported backend_type '{}'; use an OpenAI-compatible provider adapter such as {}. CLI-backed subscriptions must be configured as [[agents]], not gateway providers.", p.id, - p.backend_type + p.backend_type, + GatewayType::SUPPORTED_PROVIDER_CONFIG_NAMES.join(", ") ); } if p.url.trim().is_empty() { anyhow::bail!( - "provider '{}' with backend_type 'http' requires non-empty url", - p.id + "provider '{}' with backend_type '{}' requires non-empty url", + p.id, + p.backend_type ); } let api_key = resolve_provider_api_key(p)?; let timeout = p.timeout_seconds.unwrap_or(default_timeout); - let headers: Option> = if p.headers.is_empty() { + let retry = p.retry.clone().unwrap_or_else(|| config.retry.clone()); + let configured_headers = if p.headers.is_empty() { None } else { - Some(p.headers.clone()) + Some(&p.headers) }; + let headers = gateway::openai_compatible_headers( + gateway_type, + api_key.as_deref(), + &retry, + configured_headers, + ); let backend_cfg = BackendConfig { backend_type: BackendType::Http, @@ -207,27 +190,26 @@ pub fn build_provider_entries( api_key: api_key.clone(), timeout_seconds: Some(timeout), headers: headers.clone(), - ..Default::default() }; let backend = super::backend::create_backend(&backend_cfg) .with_context(|| format!("creating backend for provider '{}'", p.id))?; let gw_cfg = GatewayConfig { - backend_type: GatewayType::BuiltinHttp, + backend_type: gateway_type, base_url: Some(p.url.clone()), api_key, timeout_seconds: timeout, extra_config: None, headers, - retry: p.retry.clone().unwrap_or_else(|| config.retry.clone()), + retry, ui_url: None, }; let gw = gateway::create_gateway(gw_cfg, Some(backend)) .with_context(|| format!("creating gateway for provider '{}'", p.id))?; - info!(id = %p.id, url = %p.url, models = ?p.models, "Provider loaded"); + info!(id = %p.id, backend_type = %gateway_type, url = %p.url, models = ?p.models, "Provider loaded"); provider_gateways.insert(p.id.clone(), gw); provider_on_switch.insert(p.id.clone(), p.on_switch.clone()); provider_strip_prefix.insert(p.id.clone(), normalized_strip_prefix(p)); @@ -562,9 +544,8 @@ mod tests { ); } - #[cfg(feature = "helicone")] #[test] - fn helicone_provider_uses_helicone_gateway_auth_path() { + fn helicone_provider_uses_shared_http_core_with_helicone_engine_metadata() { let config = ProxyConfig { providers: vec![provider( "helicone-local", @@ -580,6 +561,25 @@ mod tests { assert_eq!(entries[0].gateway.gateway_type(), GatewayType::Helicone); } + #[test] + fn litellm_provider_uses_shared_http_core_with_litellm_engine_metadata() { + let config = ProxyConfig { + providers: vec![provider( + "litellm-local", + "litellm", + "http://127.0.0.1:4000/v1", + )], + ..Default::default() + }; + + let entries = build_provider_entries(&config, 30).unwrap(); + + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].patterns, vec!["test-model"]); + assert_eq!(entries[0].gateway.gateway_type(), GatewayType::LiteLlm); + assert_eq!(entries[0].gateway.engine_info().id, "litellm"); + } + proptest! { #[test] fn prefix_slash_wildcard_only_matches_names_inside_namespace( diff --git a/docs/adr/0001-model-gateway-and-agent-boundaries.md b/docs/adr/0001-model-gateway-and-agent-boundaries.md index 56bf5970..a076170a 100644 --- a/docs/adr/0001-model-gateway-and-agent-boundaries.md +++ b/docs/adr/0001-model-gateway-and-agent-boundaries.md @@ -40,24 +40,17 @@ flowchart TD User["Human channel"] --> Router["Calciforge channel/router"] Router --> Adapter["Agent adapter"] Adapter -->|"only when runtime is configured for it"| Gateway["Calciforge model gateway"] - Gateway --> Engine["ProviderAdapter: builtin HTTP, Helicone, external HTTP boundary, or mock"] + Gateway --> Engine["ProviderAdapter: OpenAI-compatible engine or mock"] Engine --> Provider["Model provider"] Adapter -->|"otherwise"| AgentEgress["Agent-owned model/tool egress"] ``` -The root model gateway has a small supported backend set: - -- `http`: Calciforge's builtin HTTP upstream adapter. It is a minimal - compatibility path for OpenAI-compatible endpoints, not a provider-owned - boundary. -- `helicone`: Calciforge forwards through a Helicone AI Gateway process. -- External OpenAI-compatible provider boundary endpoints such as LiteLLM are currently - configured as provider routes with `backend_type = "http"` and - `model_credential_owner = "provider"`. In that shape, the builtin HTTP adapter is - only the transport to the provider boundary; that boundary owns its - model/provider registry and upstream keys. -- `mock`: deterministic local/test behavior. +The root model gateway has a small supported backend set. `http`, `helicone`, +`litellm`, `portkey`, `tensorzero`, `future-agi`, and `openrouter` use the same +OpenAI-compatible HTTP core. The engine name selects metadata, dashboard hints, +and small policy overlays such as Helicone auth/retry headers. `mock` is +deterministic local/test behavior. Experimental or stale root backends such as `embedded`, `library`, and `traceloop` are not supported in production config. They can return later only @@ -94,8 +87,9 @@ Operators get fewer false promises: This also narrows supported configuration. Configs that used `backend_type = "embedded"`, `backend_type = "library"`, or -`backend_type = "traceloop"` as the root `[proxy]` backend must move to `http`, -`helicone`, or `mock`, or use an agent adapter/recipe instead. +`backend_type = "traceloop"` as the root `[proxy]` backend must move to one of +the supported OpenAI-compatible provider adapter kinds, `mock`, or an agent +adapter/recipe instead. ## Follow-Up Refactor Plan @@ -115,8 +109,13 @@ This also narrows supported configuration. Configs that used ## Follow-Through -2026-05-08: The production code path now exposes only the shared root gateway -allowlist (`http`, `helicone`, and `mock`). The old Traceloop feature module and -unimplemented embedded/library backend stubs were removed so validation, -runtime startup, and selectable gateway engine types cannot drift apart around +2026-05-08: The production code path narrowed the gateway allowlist and removed +old Traceloop plus unimplemented embedded/library backend stubs so validation, +runtime startup, and selectable gateway engine types could not drift around unsupported names. + +2026-05-13: The provider adapter implementation stopped treating Helicone as a +privileged runtime path. `http`, `helicone`, `litellm`, `portkey`, +`tensorzero`, `future-agi`, and `openrouter` now share the same +OpenAI-compatible HTTP core, with named engines supplying policy/metadata +overlays. diff --git a/docs/adr/0002-provider-adapter-boundary.md b/docs/adr/0002-provider-adapter-boundary.md index d554517b..35a8e16f 100644 --- a/docs/adr/0002-provider-adapter-boundary.md +++ b/docs/adr/0002-provider-adapter-boundary.md @@ -29,6 +29,10 @@ Calciforge will use provider adapters as the primary model-call abstraction. - `[[proxy.providers]]` is the preferred operational config surface. - A deployment may configure multiple adapters: Ollama, OpenRouter, LiteLLM, Helicone, direct OpenAI-compatible HTTP, or future native/library adapters. +- OpenAI-compatible engine adapters share one HTTP request/response core. + Engine names such as `litellm`, `helicone`, `portkey`, `tensorzero`, + `future-agi`, and `openrouter` supply metadata, dashboard hints, and small + policy overlays; they are not separate copied gateways. - Aliases and nested model resolution should be scoped to the selected provider where provider-owned routing exists. - The legacy root `[proxy].backend_type` remains for compatibility, but it is diff --git a/docs/model-gateway.md b/docs/model-gateway.md index 684b8f30..4ecacac3 100644 --- a/docs/model-gateway.md +++ b/docs/model-gateway.md @@ -116,44 +116,47 @@ warns because that path bypasses provider-specific prefixes, API keys, and | Dispatchers | Working | `[[dispatchers]]` picks the smallest configured context window that fits, then uses larger eligible models as fallbacks. | | Token estimators | Working | `char_ratio`, `byte_ratio`, and optional `tiktoken-rs` support for OpenAI-compatible BPE counts. BPE means byte-pair encoding, a common way model APIs count tokens. | | CLI-backed subscription agents | Working | Codex, Claude Code, Kimi Code, Dirac, and generic executable adapters are agent routes, not gateway model selectors. | -| External gateway metadata | Working | `/gateway`, `/gateway/ui`, and `!gateway` expose the selected gateway engine and operator dashboard link after sender identity resolution. | -| Helicone external gateway adapter | Working | `backend_type = "helicone"` forwards OpenAI-compatible requests to a Helicone AI Gateway while preserving Calciforge auth, routing, and command UX. | -| Builtin HTTP upstream adapter | Compatibility path | `backend_type = "http"` uses Calciforge's minimal OpenAI-compatible HTTP client. It is useful for tests, local development, and explicit operator escape hatches, but it is not equivalent to a mature gateway engine such as Helicone or LiteLLM. | +| External gateway metadata | Working | `/gateway`, `/gateway/ui`, and `!gateway` expose the selected provider adapter and operator dashboard link after sender identity resolution. | +| OpenAI-compatible provider adapter core | Working | `backend_type = "http"`, `"helicone"`, `"litellm"`, `"portkey"`, `"tensorzero"`, `"future-agi"`, and `"openrouter"` share the same `/v1/chat/completions` request path. Engine names select metadata, dashboard hints, and small policy overlays, not separate gateway implementations. | +| Builtin HTTP upstream adapter | Compatibility path | `backend_type = "http"` is the plain OpenAI-compatible HTTP shape. It is useful for direct providers, tests, and local development. Prefer a named engine such as `litellm`, `helicone`, or `openrouter` when that boundary owns provider registry, keys, retries, or dashboard state. | ## External Provider Adapters -Calciforge's gateway layer is pluggable at the engine boundary. The built-in -`mock` engine is for tests. The built-in `http` engine is a minimal upstream -adapter: it sends OpenAI-compatible HTTP requests directly from Calciforge to a -configured endpoint. Treat it as a compatibility path, not as a peer to mature -gateway engines. External engines such as Helicone and LiteLLM can add -operator-facing dashboards, provider registries, virtual keys, retries, load -balancing, and provider-specific request translation without changing how +Calciforge's gateway layer is pluggable at the provider-adapter boundary. The +`mock` engine is for tests. Every non-mock engine uses the same +OpenAI-compatible HTTP core, then applies a small engine policy for metadata, +dashboard hints, and headers. `helicone` is no longer a privileged code path; +it is one adapter kind beside `litellm`, `portkey`, `tensorzero`, `future-agi`, +`openrouter`, and plain `http`. + +That split matters. Request plumbing should be boring and shared. Provider +engines can add operator dashboards, provider registries, virtual keys, retries, +load balancing, request translation, or evaluation tooling without changing how channels and agents talk to Calciforge. Calciforge intentionally treats external provider-boundary model IDs as opaque -when that boundary owns provider configuration. OpenRouter, Helicone, and -LiteLLM all support provider/key/model registries; Calciforge should not -duplicate that registry when the operator has chosen that shape. In Calciforge config, set +when that boundary owns provider configuration. If LiteLLM, Helicone, Portkey, +TensorZero, Future AGI, OpenRouter, or another gateway owns provider/key/model +state, Calciforge should not duplicate that registry. In Calciforge config, set `model_credential_owner = "provider"` on the provider route. The provider's -`api_key`/`api_key_file`, if present, then authenticates Calciforge to the +`api_key`/`api_key_file`, if present, then authenticates Calciforge to that provider boundary; it is not the upstream OpenAI, Anthropic, Ollama, or other final provider key. ```toml [[proxy.providers]] id = "managed-gateway" -backend_type = "http" +backend_type = "litellm" url = "http://127.0.0.1:4000/v1" model_credential_owner = "provider" api_key_file = "/etc/calciforge/secrets/managed-gateway-client-key" models = ["managed/*"] ``` -In that shape, `backend_type = "http"` is just the transport used to reach a -provider-owned OpenAI-compatible boundary endpoint. `model_credential_owner = "provider"` -is the important ownership boundary: `managed/default`, `managed/cheap`, or -`managed/coding` are Calciforge-visible selectors but provider-owned model names. +In that shape, `backend_type` names the provider boundary Calciforge is calling. +`model_credential_owner = "provider"` is the important ownership boundary: +`managed/default`, `managed/cheap`, or `managed/coding` are Calciforge-visible +selectors but provider-owned model names. Calciforge still owns aliases, synthetic selectors, access policy, sender identity, security scanning, and command UX. The external gateway owns upstream provider API keys, provider-specific model IDs, load balancing, and any @@ -170,12 +173,12 @@ should use the explicit model credential fields when those concepts differ. This is intentionally separate from substitution-protected fnox secrets because agents should never need to request these model-provider credentials directly. -Current built-in HTTP/Helicone adapters have one first-class bearer credential -slot. Do not configure both provider endpoint auth (`api_key`/`api_key_file`) -and Calciforge-owned final model auth (`model_api_key`/`model_api_key_file`) on -the same provider route unless that adapter has a documented second auth -channel. For direct upstream providers, use `model_api_key_file`. For external -gateway boundaries such as LiteLLM, OpenRouter, or Helicone, use +Current OpenAI-compatible adapters have one first-class bearer credential slot. +Do not configure both provider endpoint auth (`api_key`/`api_key_file`) and +Calciforge-owned final model auth (`model_api_key`/`model_api_key_file`) on the +same provider route unless that adapter has a documented second auth channel. +For direct upstream providers, use `model_api_key_file`. For external gateway +boundaries such as LiteLLM, OpenRouter, or Helicone, use `model_credential_owner = "provider"` and put the gateway/client credential in `api_key_file`. @@ -225,35 +228,49 @@ path, that route may still be operationally cleaner because it preserves the provider's expected request shape and session behavior without extra gateway translation. -Helicone is the first external gateway adapter and the default batteries-included -observability path we ship today. It gives operators a real request dashboard, -provider routing surface, and persisted gateway logs, while Calciforge remains -the local identity, command, alias, alloy, dispatcher, and policy boundary. -That convenience has a cost: the local stack is heavier than a plain HTTP -forwarder because it includes dashboard, Postgres, ClickHouse, Jawn, and -S3-compatible object storage pieces. A lighter external gateway or a smaller -Calciforge-native observability engine may be desirable later; the adapter -boundary is intentionally where future PRs can plug in those alternatives. - -Calciforge's installer can -provision a local Helicone deployment when `CALCIFORGE_HELICONE_ENABLED=true`. -The tested local setup uses Helicone's all-in-one Docker image for the -dashboard, bundled MinIO S3-compatible storage, and Jawn API, plus the standalone -`@helicone/ai-gateway` package for request routing. The standalone gateway is -intentional: current all-in-one images may start a bundled gateway supervisor -that exits before routing traffic. -The installer pins the dashboard image with `CALCIFORGE_HELICONE_IMAGE` -(`helicone/helicone-all-in-one:v2025.08.21` by default) so local installs do -not drift when upstream retags `latest`. - -Configure Calciforge manually by setting `backend_type = "helicone"` and -pointing `backend_url` at the Helicone AI Gateway OpenAI-compatible base URL. +## Gateway Engines Vs Observability Sinks + +`backend_type` chooses the inline provider engine that receives model traffic. +Observability is a separate concern. Some engines, such as Helicone, Portkey, or +TensorZero, may provide both a request path and a dashboard. Others, such as +LiteLLM or OpenRouter, may be useful primarily as provider boundaries. Pure +observability tools should not need to become model gateways just to receive +events. + +The intended next config shape is a separate observability block, for example: + +```toml +[[proxy.observability]] +kind = "otel" +endpoint = "http://127.0.0.1:4318/v1/traces" +format = "openinference" +``` + +That block is roadmap, not a guarantee in the current release. Today, +`gateway_ui_url` is the stable operator link exposed by `!gateway` and +`/gateway/ui`, and provider adapters expose coarse capability metadata. + +LiteLLM is the lightest current candidate for the default local provider +boundary. It can sit in front of Ollama and remote providers without pulling in +Helicone's dashboard stack. Helicone remains supported when you want its UI and +request log, but it is optional: useful, not sacred. Calcifer may like a bright +fire, but your laptop does not need to run a small castle just to route one +model request. + +Calciforge's installer can provision a local Helicone deployment when +`CALCIFORGE_HELICONE_ENABLED=true`. That path is heavier because it includes a +dashboard, Postgres, ClickHouse, Jawn, and S3-compatible object storage pieces. +The adapter boundary is intentionally where LiteLLM, Helicone, Portkey, +TensorZero, Future AGI, OpenRouter, and future PRs plug in without changing +agent/channel behavior. + +Configure Calciforge manually by setting `backend_type` to the adapter kind and +pointing `backend_url` at that engine's OpenAI-compatible base URL. `backend_url` must be a plain `http` or `https` base URL without query -parameters or fragments. -If it has no path, Calciforge posts to `/v1/chat/completions`; if it already -includes a path such as `/v1`, `/ai`, or `/router/`, Calciforge appends -`/chat/completions` to that configured base path instead of injecting another -`/v1`. +parameters or fragments. If it has no path, Calciforge posts to +`/v1/chat/completions`; if it already includes a path such as `/v1`, `/ai`, or +`/router/`, Calciforge appends `/chat/completions` to that configured base +path instead of injecting another `/v1`. ```toml [proxy] @@ -266,6 +283,19 @@ backend_api_key_file = "/etc/calciforge/secrets/helicone-gateway-key" gateway_ui_url = "http://127.0.0.1:3300" ``` +The same shape works for LiteLLM: + +```toml +[proxy] +enabled = true +bind = "127.0.0.1:8080" +api_key_file = "/etc/calciforge/secrets/model-gateway-client-key" +backend_type = "litellm" +backend_url = "http://127.0.0.1:4000/v1" +backend_api_key_file = "/etc/calciforge/secrets/litellm-client-key" +gateway_ui_url = "http://127.0.0.1:4000/ui" +``` + ### Retry and Fallback Policy There are two distinct failure-handling layers: @@ -393,21 +423,20 @@ does not require Calciforge to own the tunnel, DNS name, certificate, firewall, or reverse proxy. If `CALCIFORGE_GATEWAY_UI_URL` is unset, the installer only records a local dashboard URL when it actually starts the local dashboard container. When a dashboard URL is configured, `!gateway` and `/gateway` expose -it so the operator can jump from Calciforge into Helicone's observability UI. +it so the operator can jump from Calciforge into the selected provider adapter's +UI. Use the same pattern for other local web surfaces: keep the service bind conservative, then configure the advertised public URL separately. Paste-server links use `CALCIFORGE_PASTE_PUBLIC_BASE_URL` for reverse proxies or tunnels and `CALCIFORGE_PASTE_PUBLIC_HOST` for a stable LAN/Tailscale host. -The Helicone gateway is currently strongest for providers that Helicone knows -how to route directly, such as Ollama via the `/ai` router with -provider-qualified model IDs. Keep user-facing local selectors such as -`qwen3.6:27b` in Calciforge, then set `add_model_prefix = "ollama/"` on the -Helicone provider so upstream requests send `ollama/qwen3.6:27b`. Arbitrary -OpenAI-compatible providers may still be configured through Calciforge's builtin -HTTP upstream adapter until their Helicone provider/converter support is -validated, but that should be treated as an explicit compatibility path. +For provider boundaries that expect provider-qualified model IDs, keep +user-facing local selectors such as `qwen3.6:27b` in Calciforge, then set +`add_model_prefix = "ollama/"` on that provider so upstream requests send +`ollama/qwen3.6:27b`. For boundaries with their own registry, such as LiteLLM +or OpenRouter, prefer selectors that make sense in that registry and use +`strip_model_prefix` / `add_model_prefix` only at the Calciforge edge. Large local Ollama models usually cannot stay resident together. For Ollama providers, configure `on_switch` so Calciforge unloads any other resident model @@ -477,12 +506,14 @@ For process-boundary coverage, run: ```bash python3 scripts/model-gateway-helicone-smoke.py +python3 scripts/model-gateway-litellm-smoke.py ``` -That script starts a local Helicone-shaped gateway, starts Calciforge in -`--proxy-only` mode, checks `/gateway` metadata and `/gateway/ui`, and sends a -real `/v1/chat/completions` request through Calciforge to prove the adapter -forwards the expected auth headers, path, and model. +Those scripts start local external-gateway-shaped processes, start Calciforge in +`--proxy-only` mode, check `/gateway` metadata and `/gateway/ui`, and send real +`/v1/chat/completions` requests through Calciforge to prove the shared +OpenAI-compatible adapter core forwards the expected auth headers, path, and +model. For a live deployment smoke against configured provider routes, run: @@ -519,8 +550,8 @@ requests send the provider's concrete model ID. Calciforge's model gateway currently speaks the OpenAI-compatible `/v1/chat/completions` request shape. Use OpenCode Go models that are exposed on that shape, such as Kimi and Qwen. Models that require Anthropic-compatible -`/v1/messages` are not supported by the builtin HTTP upstream adapter yet; route -them through a CLI, ACP adapter, LiteLLM, or another gateway that converts +`/v1/messages` are not supported by the shared OpenAI-compatible adapter yet; +route them through a CLI, ACP adapter, LiteLLM, or another gateway that converts OpenAI-compatible requests to Anthropic-compatible upstream calls. ```toml diff --git a/docs/roadmap/litellm-gateway-owned-provider.md b/docs/roadmap/litellm-gateway-owned-provider.md index ba08873d..dcbf5bbe 100644 --- a/docs/roadmap/litellm-gateway-owned-provider.md +++ b/docs/roadmap/litellm-gateway-owned-provider.md @@ -7,12 +7,12 @@ title: LiteLLM Gateway-Owned Provider Spike Status: recipe/proof path. -Calciforge does not need a LiteLLM-specific adapter to exercise the external -gateway contract. LiteLLM's proxy is an OpenAI-compatible gateway process, so -Calciforge can route to it with the existing builtin HTTP transport and mark -the custody boundary with `model_credential_owner = "provider"`. In this recipe, -`backend_type = "http"` is only the transport from Calciforge to LiteLLM; it is -not a raw upstream-provider route. +Calciforge routes to LiteLLM through the shared OpenAI-compatible provider +adapter core. Use `backend_type = "litellm"` so Calciforge records the right +engine metadata and dashboard expectations, then mark the custody boundary with +`model_credential_owner = "provider"`. The underlying HTTP request plumbing is +shared with `http`, `helicone`, `openrouter`, and the other OpenAI-compatible +adapter kinds. In this shape, Calciforge owns channel identity, aliases, synthetic selectors, access policy, command UX, and security scanning. LiteLLM owns upstream provider @@ -84,7 +84,7 @@ timeout_seconds = 60 [[proxy.providers]] id = "litellm-managed" -backend_type = "http" +backend_type = "litellm" url = "http://127.0.0.1:4000/v1" model_credential_owner = "provider" api_key_file = "/etc/calciforge/secrets/litellm-virtual-key" diff --git a/scripts/boundary-aggression.sh b/scripts/boundary-aggression.sh index 1a13ea1e..08048fa2 100755 --- a/scripts/boundary-aggression.sh +++ b/scripts/boundary-aggression.sh @@ -54,7 +54,7 @@ ensure_uv_for_hegel() { echo "boundary aggression mode: $mode" echo "PROPTEST_CASES=$PROPTEST_CASES" -run cargo test -p calciforge proxy::helicone_streaming::tests -- --nocapture +run cargo test -p calciforge proxy::openai_streaming::tests -- --nocapture run cargo test -p calciforge proxy::routing::tests -- --nocapture run cargo test -p calciforge proxy::auth::tests -- --nocapture run cargo test -p calciforge adapters::openclaw_channel::openclaw_channel_reply_tests -- --nocapture diff --git a/scripts/boundary-explore-long.sh b/scripts/boundary-explore-long.sh index 45dd43c8..bcfbd06a 100755 --- a/scripts/boundary-explore-long.sh +++ b/scripts/boundary-explore-long.sh @@ -65,7 +65,7 @@ run_logged() { } run_gateway() { - run_logged gateway-helicone env PROPTEST_CASES="$property_cases" cargo test -p calciforge proxy::helicone_streaming::tests -- --nocapture || return 1 + run_logged gateway-openai-streaming env PROPTEST_CASES="$property_cases" cargo test -p calciforge proxy::openai_streaming::tests -- --nocapture || return 1 run_logged gateway-routing env PROPTEST_CASES="$property_cases" cargo test -p calciforge proxy::routing::tests -- --nocapture || return 1 run_logged gateway-auth env PROPTEST_CASES="$property_cases" cargo test -p calciforge proxy::auth::tests -- --nocapture || return 1 } diff --git a/scripts/model-gateway-litellm-smoke.py b/scripts/model-gateway-litellm-smoke.py index 485ca4a9..e34f5af6 100755 --- a/scripts/model-gateway-litellm-smoke.py +++ b/scripts/model-gateway-litellm-smoke.py @@ -246,7 +246,7 @@ def write_calciforge_config(tmp: Path, gateway_port: int, litellm_port: int) -> [[proxy.providers]] id = "litellm-local" -backend_type = "http" +backend_type = "litellm" url = "http://127.0.0.1:{litellm_port}/v1" model_credential_owner = "provider" api_key = "{LITELLM_GATEWAY_KEY}" diff --git a/tests/boundaries/integration-surfaces.json b/tests/boundaries/integration-surfaces.json index 49c49f0a..28ee44fe 100644 --- a/tests/boundaries/integration-surfaces.json +++ b/tests/boundaries/integration-surfaces.json @@ -10,10 +10,9 @@ "crates/calciforge/src/proxy/control_auth.rs", "crates/calciforge/src/proxy/gateway.rs", "crates/calciforge/src/proxy/handlers.rs", - "crates/calciforge/src/proxy/helicone_router.rs", - "crates/calciforge/src/proxy/helicone_streaming.rs", "crates/calciforge/src/proxy/model_resolver.rs", "crates/calciforge/src/proxy/openai.rs", + "crates/calciforge/src/proxy/openai_streaming.rs", "crates/calciforge/src/proxy/routing.rs", "crates/calciforge/src/proxy/streaming.rs", "crates/calciforge/src/proxy/token_estimator.rs", @@ -41,15 +40,16 @@ "fuzz_targets": [], "automation": [ "scripts/boundary-aggression.sh pr", - "cargo test -p calciforge proxy::helicone_streaming::tests", + "cargo test -p calciforge proxy::openai_streaming::tests", "cargo test -p calciforge proxy::routing::tests", "cargo test -p calciforge proxy::auth::tests", "cargo test -p calciforge --test e2e property_tests", "scripts/boundary-fuzz.sh nightly", - "scripts/model-gateway-helicone-smoke.py" + "scripts/model-gateway-helicone-smoke.py", + "scripts/model-gateway-litellm-smoke.py" ], "scenario_ids": [ - "helicone-streaming-tools", + "openai-compatible-streaming-tools", "first-class-agent-gateway-egress", "local-model-latency-envelope" ] @@ -157,6 +157,10 @@ "crates/calciforge/src/commands.rs", "crates/calciforge/src/config.rs", "crates/calciforge/src/config/validator.rs", + "crates/calciforge/src/config/validator_test_support.rs", + "crates/calciforge/src/config/validator_tests_1.rs", + "crates/calciforge/src/config/validator_tests_2.rs", + "crates/calciforge/src/config/validator_tests_3.rs", "crates/calciforge/src/model_names.rs", "crates/calciforge/src/providers/alloy.rs", "crates/calciforge/src/providers/mod.rs", diff --git a/tests/scenarios/high-risk-scenarios.json b/tests/scenarios/high-risk-scenarios.json index 1a6ff308..e37f02c5 100644 --- a/tests/scenarios/high-risk-scenarios.json +++ b/tests/scenarios/high-risk-scenarios.json @@ -1,7 +1,7 @@ [ { - "id": "helicone-streaming-tools", - "title": "Helicone returns SSE for a tool-enabled chat completion", + "id": "openai-compatible-streaming-tools", + "title": "OpenAI-compatible provider adapter returns SSE for a tool-enabled chat completion", "user_action": "A first-class agent sends a tool-enabled chat-completions request through Calciforge with stream=true.", "components": [ "first-class agent adapter", @@ -17,8 +17,9 @@ "return an SSE content type with parameters" ], "current_automation": [ - "crates/calciforge/src/proxy/helicone_streaming.rs unit tests", - "scripts/model-gateway-helicone-smoke.py" + "crates/calciforge/src/proxy/openai_streaming.rs unit tests", + "scripts/model-gateway-helicone-smoke.py", + "scripts/model-gateway-litellm-smoke.py" ], "status": "automated" }, From 96dc76d4744c2c1afd4049644e2962b6503808c5 Mon Sep 17 00:00:00 2001 From: Brian Glusman Date: Wed, 13 May 2026 00:48:28 -0400 Subject: [PATCH 2/2] Harden shared gateway header handling --- crates/calciforge/src/proxy/backend.rs | 71 +++++++++--------- crates/calciforge/src/proxy/gateway_tests.rs | 79 ++++++++++++++++++++ 2 files changed, 116 insertions(+), 34 deletions(-) diff --git a/crates/calciforge/src/proxy/backend.rs b/crates/calciforge/src/proxy/backend.rs index 9afaf999..e3261e8a 100644 --- a/crates/calciforge/src/proxy/backend.rs +++ b/crates/calciforge/src/proxy/backend.rs @@ -327,23 +327,10 @@ impl HttpBackend { timeout_seconds: u64, headers: Option>, ) -> Self { - let mut client_builder = - reqwest::Client::builder().timeout(std::time::Duration::from_secs(timeout_seconds)); - - // Add default headers if provided - if let Some(headers) = &headers { - let mut header_map = reqwest::header::HeaderMap::new(); - for (key, value) in headers { - if let Ok(header_name) = reqwest::header::HeaderName::from_bytes(key.as_bytes()) - && let Ok(header_value) = reqwest::header::HeaderValue::from_str(value) - { - header_map.insert(header_name, header_value); - } - } - client_builder = client_builder.default_headers(header_map); - } - - let client = client_builder.build().expect("Failed to build HTTP client"); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(timeout_seconds)) + .build() + .expect("Failed to build HTTP client"); Self { client, @@ -354,6 +341,30 @@ impl HttpBackend { } } + fn apply_configured_headers( + &self, + mut request_builder: reqwest::RequestBuilder, + ) -> reqwest::RequestBuilder { + for (key, value) in &self.headers { + if !self.api_key.is_empty() && key.eq_ignore_ascii_case("authorization") { + continue; + } + request_builder = request_builder.header(key, value); + } + request_builder + } + + fn apply_authorization_header( + &self, + mut request_builder: reqwest::RequestBuilder, + ) -> reqwest::RequestBuilder { + if !self.api_key.is_empty() { + request_builder = + request_builder.header("Authorization", format!("Bearer {}", self.api_key)); + } + request_builder + } + async fn send_chat_completion_request( &self, request: crate::proxy::openai::ChatCompletionRequest, @@ -366,19 +377,13 @@ impl HttpBackend { })?; apply_kimi_compat(&self.base_url, &model, &mut request_body); - let mut request_builder = self - .client - .post(&url) - .header("Content-Type", "application/json"); - - if !self.api_key.is_empty() { - request_builder = - request_builder.header("Authorization", format!("Bearer {}", self.api_key)); - } - - for (key, value) in &self.headers { - request_builder = request_builder.header(key, value); - } + let request_builder = self.apply_authorization_header( + self.apply_configured_headers( + self.client + .post(&url) + .header("Content-Type", "application/json"), + ), + ); let response = request_builder .json(&request_body) @@ -482,10 +487,8 @@ impl SecretsBackend for HttpBackend { async fn list_models(&self) -> Result, BackendError> { let url = format!("{}/models", self.base_url); - let mut req = self.client.get(&url); - if !self.api_key.is_empty() { - req = req.header("Authorization", format!("Bearer {}", self.api_key)); - } + let req = + self.apply_authorization_header(self.apply_configured_headers(self.client.get(&url))); let response = req.send().await.map_err(|e| { BackendError::transport(format!("Request failed: {}", e), e.is_timeout()) })?; diff --git a/crates/calciforge/src/proxy/gateway_tests.rs b/crates/calciforge/src/proxy/gateway_tests.rs index 49b4c2c6..54be0b29 100644 --- a/crates/calciforge/src/proxy/gateway_tests.rs +++ b/crates/calciforge/src/proxy/gateway_tests.rs @@ -347,6 +347,85 @@ async fn builtin_http_gateway_forwards_complete_chat_request_options() { mock.assert_async().await; } +#[tokio::test] +async fn configured_authorization_header_cannot_override_backend_api_key() { + let mut server = mockito::Server::new_async().await; + let response = ChatCompletionResponse { + id: "chatcmpl-auth-order".to_string(), + object: "chat.completion".to_string(), + created: 1, + model: "managed/default".to_string(), + choices: vec![Choice { + index: 0, + message: ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("pong".to_string())), + name: None, + tool_calls: None, + tool_call_id: None, + reasoning: None, + reasoning_content: None, + }, + finish_reason: Some("stop".to_string()), + logprobs: None, + }], + usage: Usage { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + }, + system_fingerprint: None, + }; + let mock = server + .mock("POST", "/v1/chat/completions") + .match_header("authorization", "Bearer backend-key") + .match_header("x-provider-boundary", "litellm") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(serde_json::to_string(&response).unwrap()) + .create_async() + .await; + + let mut headers = HashMap::new(); + headers.insert( + "Authorization".to_string(), + "Bearer wrong-configured-key".to_string(), + ); + headers.insert("x-provider-boundary".to_string(), "litellm".to_string()); + let backend = create_backend(&BackendConfig { + backend_type: BackendType::Http, + url: Some(format!("{}/v1", server.url())), + api_key: Some("backend-key".to_string()), + timeout_seconds: Some(30), + headers: Some(headers), + }) + .unwrap(); + let gateway = create_gateway( + GatewayConfig { + backend_type: GatewayType::LiteLlm, + base_url: Some(format!("{}/v1", server.url())), + api_key: Some("backend-key".to_string()), + timeout_seconds: 30, + ..Default::default() + }, + Some(backend), + ) + .unwrap(); + + gateway + .chat_completion( + serde_json::from_value(serde_json::json!({ + "model": "managed/default", + "messages": [{"role": "user", "content": "ping"}] + })) + .unwrap(), + ) + .await + .unwrap(); + + mock.assert_async().await; +} + #[test] fn create_openai_compatible_gateway_preserves_engine_metadata_through_logging_wrapper() { let backend = create_backend(&BackendConfig {