diff --git a/config/catalog/README.md b/config/catalog/README.md index 28c07ea404..9b2a381e34 100644 --- a/config/catalog/README.md +++ b/config/catalog/README.md @@ -107,7 +107,7 @@ without duplicating its intrinsic identity. Virtual recipes are materialized from packaged assets and keep their own evaluation directory. The built-in physical inventory is curated at the creator-company level. The -current baseline contains 83 physical cards from 22 mainstream creators and +current baseline contains 84 physical cards from 22 mainstream creators and five separately stored virtual cards. For each creator, prefer roughly the latest three generations or representative product lines over accumulating a shallow long tail of lesser-known creators. This policy is about Model Cards, diff --git a/config/catalog/catalog.yaml b/config/catalog/catalog.yaml index a951cffffb..28c890f970 100644 --- a/config/catalog/catalog.yaml +++ b/config/catalog/catalog.yaml @@ -106,9 +106,9 @@ inventory: - nvidia/nemotron-cascade-2-30b-a3b - publisher: OpenAI representative_models: + - openai/gpt-6-astra - openai/gpt-5.6-sol - openai/gpt-5.5 - - openai/gpt-5.4 - publisher: StepFun representative_models: - stepfun/step-3.7-flash diff --git a/config/catalog/resources/evaluations/single/openai.yaml b/config/catalog/resources/evaluations/single/openai.yaml index fc30ce0a5a..4284df3495 100644 --- a/config/catalog/resources/evaluations/single/openai.yaml +++ b/config/catalog/resources/evaluations/single/openai.yaml @@ -4912,3 +4912,94 @@ verification: claimed source: https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf redistributable: true +- id: openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 + model: openai/gpt-6-astra + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.96 + status: available + observed_at: 2026-09-07 + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-humanitys-last-exam-tools@1.0.0 + model: openai/gpt-6-astra + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: with-tools + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + tool_policy: tools_enabled + source_kind: official_launch_evaluation + metrics: + accuracy: 0.572 + status: available + observed_at: 2026-09-07 + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-deep-swe-1.1@1.0.0 + model: openai/gpt-6-astra + benchmark: datacurve/deep-swe@1.1.0 + benchmark_profile: published-agent + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + resolved: 0.741 + status: available + observed_at: 2026-09-07 + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-arc-agi-2@1.0.0 + model: openai/gpt-6-astra + benchmark: arc-prize/arc-agi-2@1.0.0 + benchmark_profile: semi-private + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.95 + status: available + observed_at: 2026-09-07 + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-arc-agi-1@1.0.0 + model: openai/gpt-6-astra + benchmark: arc-prize/arc-agi-1@1.0.0 + benchmark_profile: semi-private + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.985 + status: available + observed_at: 2026-09-07 + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true diff --git a/config/catalog/resources/models/single/openai.yaml b/config/catalog/resources/models/single/openai.yaml index 146f155d75..ca9bfdf095 100644 --- a/config/catalog/resources/models/single/openai.yaml +++ b/config/catalog/resources/models/single/openai.yaml @@ -1,3 +1,45 @@ +- id: openai/gpt-6-astra + display_name: GPT-6 Astra + description: OpenAI flagship model for complex reasoning, coding, computer use, research, and document creation. + kind: physical + publisher: OpenAI + presentation: + logo: package:openai + monogram: O + monochrome: true + distribution: + type: proprietary_api + source: https://developers.openai.com/api/docs/models/gpt-6-astra + family: gpt-6 + lifecycle: active + limits: + context_window_size: 1050000 + max_output_tokens: 128000 + capabilities: + - chat + - reasoning + - tools + - structured_output + - vision + - long_context + modalities: + input: + - text + - image + output: + - text + reasoning_family: gpt-6-astra + verification: + authority: OpenAI + status: claimed + verified_at: 2026-09-07 + source: https://developers.openai.com/api/docs/models/gpt-6-astra + tags: + - frontier + - proprietary + - agentic + released_at: '2026-09-03' + knowledge_cutoff: '2026-04-30' - id: openai/gpt-5.6-sol display_name: GPT-5.6 Sol description: OpenAI high-capability GPT-5.6 model for demanding reasoning and agentic work. diff --git a/config/catalog/resources/providers/openai.yaml b/config/catalog/resources/providers/openai.yaml index b29dfb152f..cafb4552c2 100644 --- a/config/catalog/resources/providers/openai.yaml +++ b/config/catalog/resources/providers/openai.yaml @@ -28,6 +28,48 @@ conformance: status: fixture_verified verified_at: 2026-09-04 models: +- catalog: openai/gpt-6-astra + relationship: first_party + id: gpt-6-astra + protocols: + - openai/chat-completions@1 + - openai/responses@1 + reasoning_modes: [enabled] + reasoning_efforts: [low, medium, high, xhigh, max] + reasoning_efforts_by_protocol: + openai/chat-completions@1: [low, medium, high, xhigh] + pricing: + currency: USD + prompt_per_1m: 10.0 + cached_input_per_1m: 1.0 + cache_write_per_1m: 12.5 + completion_per_1m: 50.0 + restrictions: + tools_protocols: + - openai/responses@1 + long_context_pricing: + input_threshold_tokens: 272000 + prompt_multiplier: 2.0 + cached_input_multiplier: 2.0 + cache_write_multiplier: 2.0 + completion_multiplier: 1.5 + unsupported_request_fields: + openai/chat-completions@1: + - temperature + - top_p + - top_logprobs + - logprobs + openai/responses@1: + - temperature + - top_p + - top_logprobs + unsupported_include_values: + - message.output_text.logprobs + lifecycle: active + verification: + status: claimed + verified_at: 2026-09-07 + source: https://developers.openai.com/api/docs/models/gpt-6-astra - catalog: openai/gpt-5.4 relationship: first_party id: gpt-5.4 diff --git a/config/catalog/resources/reasoning-families.yaml b/config/catalog/resources/reasoning-families.yaml index bda7898c1e..aa7c5b882b 100644 --- a/config/catalog/resources/reasoning-families.yaml +++ b/config/catalog/resources/reasoning-families.yaml @@ -87,6 +87,12 @@ disabled: none modes: [enabled, disabled] default_mode: enabled +- id: gpt-6-astra + type: reasoning_effort + parameter: reasoning_effort + levels: [low, medium, high, xhigh, max] + modes: [enabled] + default_mode: enabled - id: gpt-5.6 type: reasoning_effort parameter: reasoning_effort diff --git a/config/catalog/schemas/catalog-resources-v1.schema.json b/config/catalog/schemas/catalog-resources-v1.schema.json index c60f2316f9..0911b8c448 100644 --- a/config/catalog/schemas/catalog-resources-v1.schema.json +++ b/config/catalog/schemas/catalog-resources-v1.schema.json @@ -112,6 +112,11 @@ "items": {"enum": ["enabled", "disabled", "adaptive"]} }, "reasoning_efforts": {"$ref": "#/$defs/stringSet"}, + "reasoning_efforts_by_protocol": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"$ref": "#/$defs/stringSet"} + }, "pricing": { "type": "object", "additionalProperties": {"type": ["number", "string", "boolean"]} diff --git a/config/recipes/built-in/latest/catalog.yaml b/config/recipes/built-in/latest/catalog.yaml index 31924489de..99f9839ac2 100644 --- a/config/recipes/built-in/latest/catalog.yaml +++ b/config/recipes/built-in/latest/catalog.yaml @@ -1563,6 +1563,58 @@ providers: status: fixture_verified verified_at: '2026-09-04' models: + - catalog: openai/gpt-6-astra + relationship: first_party + id: gpt-6-astra + protocols: + - openai/chat-completions@1 + - openai/responses@1 + reasoning_modes: + - enabled + reasoning_efforts: + - low + - medium + - high + - xhigh + - max + reasoning_efforts_by_protocol: + openai/chat-completions@1: + - low + - medium + - high + - xhigh + pricing: + currency: USD + prompt_per_1m: 10.0 + cached_input_per_1m: 1.0 + cache_write_per_1m: 12.5 + completion_per_1m: 50.0 + restrictions: + tools_protocols: + - openai/responses@1 + long_context_pricing: + input_threshold_tokens: 272000 + prompt_multiplier: 2.0 + cached_input_multiplier: 2.0 + cache_write_multiplier: 2.0 + completion_multiplier: 1.5 + unsupported_request_fields: + openai/chat-completions@1: + - temperature + - top_p + - top_logprobs + - logprobs + openai/responses@1: + - temperature + - top_p + - top_logprobs + unsupported_include_values: + - message.output_text.logprobs + lifecycle: active + verification: + status: claimed + verified_at: '2026-09-07' + source: https://developers.openai.com/api/docs/models/gpt-6-astra - catalog: openai/gpt-5.4 relationship: first_party id: gpt-5.4 @@ -3602,6 +3654,18 @@ reasoning_families: - enabled - disabled default_mode: enabled +- id: gpt-6-astra + type: reasoning_effort + parameter: reasoning_effort + levels: + - low + - medium + - high + - xhigh + - max + modes: + - enabled + default_mode: enabled - id: gpt-5.6 type: reasoning_effort parameter: reasoning_effort @@ -6069,6 +6133,48 @@ models: - moe - efficient released_at: '2026-03-19' +- id: openai/gpt-6-astra + display_name: GPT-6 Astra + description: OpenAI flagship model for complex reasoning, coding, computer use, research, and document creation. + kind: physical + publisher: OpenAI + presentation: + logo: package:openai + monogram: O + monochrome: true + distribution: + type: proprietary_api + source: https://developers.openai.com/api/docs/models/gpt-6-astra + family: gpt-6 + lifecycle: active + limits: + context_window_size: 1050000 + max_output_tokens: 128000 + capabilities: + - chat + - reasoning + - tools + - structured_output + - vision + - long_context + modalities: + input: + - text + - image + output: + - text + reasoning_family: gpt-6-astra + verification: + authority: OpenAI + status: claimed + verified_at: '2026-09-07' + source: https://developers.openai.com/api/docs/models/gpt-6-astra + tags: + - frontier + - proprietary + - agentic + released_at: '2026-09-03' + knowledge_cutoff: '2026-04-30' - id: openai/gpt-5.6-sol display_name: GPT-5.6 Sol description: OpenAI high-capability GPT-5.6 model for demanding reasoning and agentic work. @@ -27217,6 +27323,97 @@ evaluations: verification: claimed source: https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf redistributable: true +- id: openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 + model: openai/gpt-6-astra + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.96 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-humanitys-last-exam-tools@1.0.0 + model: openai/gpt-6-astra + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: with-tools + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + tool_policy: tools_enabled + source_kind: official_launch_evaluation + metrics: + accuracy: 0.572 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-deep-swe-1.1@1.0.0 + model: openai/gpt-6-astra + benchmark: datacurve/deep-swe@1.1.0 + benchmark_profile: published-agent + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + resolved: 0.741 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-arc-agi-2@1.0.0 + model: openai/gpt-6-astra + benchmark: arc-prize/arc-agi-2@1.0.0 + benchmark_profile: semi-private + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.95 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-arc-agi-1@1.0.0 + model: openai/gpt-6-astra + benchmark: arc-prize/arc-agi-1@1.0.0 + benchmark_profile: semi-private + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.985 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true - id: independent/qwen3-7-max-critpt@1.0.0 model: qwen/qwen3.7-max benchmark: artificial-analysis/critpt@1.0.0 @@ -38224,6 +38421,188 @@ evaluation_coverage: benchmark_profile: published-agent metric: resolved status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: available + value: 0.96 + evaluation: openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing - model: openai/gpt-5.6-sol reasoning_effort: low benchmark: tiger-ai-lab/mmlu-pro@1.0.0 @@ -48173,6 +48552,266 @@ index_results: general_reasoning: 79.80000000000001 scientific_reasoning: 76.1 software_engineering: 50.2 +- model: openai/gpt-6-astra + reasoning_effort: low + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: medium + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: high + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: xhigh + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: max + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: unspecified + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.2 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: available + value: 0.96 + normalized: 0.96 + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + evaluation: openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: + - openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 - model: openai/gpt-5.6-sol reasoning_effort: low index: vllm-sr/intelligence@1.0.0 diff --git a/dashboard/backend/handlers/model_catalog_test.go b/dashboard/backend/handlers/model_catalog_test.go index 45f5a95f97..567ab464ec 100644 --- a/dashboard/backend/handlers/model_catalog_test.go +++ b/dashboard/backend/handlers/model_catalog_test.go @@ -360,7 +360,7 @@ func TestGeneratedPublicModelCatalogSatisfiesDashboardContract(t *testing.T) { if unmarshalErr := json.Unmarshal(normalized, &document); unmarshalErr != nil { t.Fatalf("decode normalized public catalog: %v", unmarshalErr) } - if len(document.Models) != 88 || len(document.Providers) != 60 || len(document.Evaluations) != 1360 { + if len(document.Models) != 89 || len(document.Providers) != 60 || len(document.Evaluations) != 1365 { t.Fatalf( "unexpected generated inventory: models=%d providers=%d evaluations=%d", len(document.Models), diff --git a/dashboard/frontend/src/generated/modelCatalog.json b/dashboard/frontend/src/generated/modelCatalog.json index 5de49bb0b2..d69421a16e 100644 --- a/dashboard/frontend/src/generated/modelCatalog.json +++ b/dashboard/frontend/src/generated/modelCatalog.json @@ -7547,6 +7547,248 @@ "reasoning_effort": "unspecified", "status": "missing" }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evaluation": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "available", + "value": 0.96 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", @@ -35665,6 +35907,122 @@ "variant": "gpt-oss-20b" } }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", + "metrics": { + "accuracy": 0.96 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "with-tools", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-humanitys-last-exam-tools@1.0.0", + "metrics": { + "accuracy": 0.572 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "tool_policy": "tools_enabled", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "datacurve/deep-swe@1.1.0", + "benchmark_profile": "published-agent", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-deep-swe-1.1@1.0.0", + "metrics": { + "resolved": 0.741 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "arc-prize/arc-agi-2@1.0.0", + "benchmark_profile": "semi-private", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-arc-agi-2@1.0.0", + "metrics": { + "accuracy": 0.95 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "arc-prize/arc-agi-1@1.0.0", + "benchmark_profile": "semi-private", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-arc-agi-1@1.0.0", + "metrics": { + "accuracy": 0.985 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, { "benchmark": "artificial-analysis/critpt@1.0.0", "benchmark_profile": "independent-standard", @@ -51597,9 +51955,360 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3.5-lightning", + "model": "nvidia/nemotron-3.5-lightning", + "provenance": [], + "reasoning_effort": "disabled", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [], + "reasoning_effort": "medium", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", + "metric": "accuracy", + "normalized": 0.879, + "status": "available", + "value": 0.879, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", + "metric": "accuracy", + "normalized": 0.261, + "status": "available", + "value": 0.261, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0", + "metric": "resolved", + "normalized": 0.695, + "status": "available", + "value": 0.695, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", + "metric": "resolved", + "normalized": 0.539, + "status": "available", + "value": 0.539, + "weight": 0.2 + } + ], + "coverage": 0.8, + "domains": { + "agentic_systems": 53.900000000000006, + "frontier_reasoning": 26.1, + "scientific_reasoning": 87.9, + "software_engineering": 69.49999999999999 + }, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [ + "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0" + ], + "reasoning_effort": "high", + "score": 59.349999999999994, + "status": "available" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [], + "reasoning_effort": "enabled", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", + "provenance": [], + "reasoning_effort": "low", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", + "provenance": [], + "reasoning_effort": "high", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "enabled", "score": null, "status": "missing" }, @@ -51653,9 +52362,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "provenance": [], - "reasoning_effort": "medium", + "reasoning_effort": "enabled", "score": null, "status": "missing" }, @@ -51673,62 +52382,47 @@ { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", "metric": "accuracy", - "normalized": 0.879, - "status": "available", - "value": 0.879, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": 0.261, - "status": "available", - "value": 0.261, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0", "metric": "resolved", - "normalized": 0.695, - "status": "available", - "value": 0.695, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "harbor/terminal-bench@2.1.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", "metric": "resolved", - "normalized": 0.539, - "status": "available", - "value": 0.539, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 } ], - "coverage": 0.8, - "domains": { - "agentic_systems": 53.900000000000006, - "frontier_reasoning": 26.1, - "scientific_reasoning": 87.9, - "software_engineering": 69.49999999999999 - }, + "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", - "provenance": [ - "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0" - ], - "reasoning_effort": "high", - "score": 59.349999999999994, - "status": "available" + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "provenance": [], + "reasoning_effort": "disabled", + "score": null, + "status": "missing" }, { "components": [ @@ -51780,7 +52474,7 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", + "model": "nvidia/nemotron-cascade-2-30b-a3b", "provenance": [], "reasoning_effort": "enabled", "score": null, @@ -51836,9 +52530,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", + "model": "nvidia/nemotron-cascade-2-30b-a3b", "provenance": [], - "reasoning_effort": "low", + "reasoning_effort": "disabled", "score": null, "status": "missing" }, @@ -51847,37 +52541,41 @@ { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.798, + "status": "available", + "value": 0.798, "weight": 0.2 }, { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.761, + "status": "available", + "value": 0.761, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", + "evaluation": "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.177, + "status": "available", + "value": 0.177, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0", "metric": "resolved", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.502, + "status": "available", + "value": 0.502, "weight": 0.2 }, { @@ -51890,13 +52588,24 @@ "weight": 0.2 } ], - "coverage": 0.0, + "coverage": 0.8, + "domains": { + "frontier_reasoning": 17.7, + "general_reasoning": 79.80000000000001, + "scientific_reasoning": 76.1, + "software_engineering": 50.2 + }, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", - "provenance": [], - "reasoning_effort": "high", - "score": null, - "status": "missing" + "model": "nvidia/nemotron-cascade-2-30b-a3b", + "provenance": [ + "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", + "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", + "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", + "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0" + ], + "reasoning_effort": "unspecified", + "score": 55.95, + "status": "available" }, { "components": [ @@ -51948,9 +52657,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "low", "score": null, "status": "missing" }, @@ -52004,9 +52713,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "medium", "score": null, "status": "missing" }, @@ -52060,9 +52769,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "high", "score": null, "status": "missing" }, @@ -52116,9 +52825,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "xhigh", "score": null, "status": "missing" }, @@ -52172,9 +52881,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "max", "score": null, "status": "missing" }, @@ -52183,41 +52892,38 @@ { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", "metric": "accuracy", - "normalized": 0.798, - "status": "available", - "value": 0.798, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", + "evaluation": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", "metric": "accuracy", - "normalized": 0.761, + "normalized": 0.96, "status": "available", - "value": 0.761, + "value": 0.96, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", - "evaluation": "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": 0.177, - "status": "available", - "value": 0.177, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0", "metric": "resolved", - "normalized": 0.502, - "status": "available", - "value": 0.502, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { @@ -52230,24 +52936,15 @@ "weight": 0.2 } ], - "coverage": 0.8, - "domains": { - "frontier_reasoning": 17.7, - "general_reasoning": 79.80000000000001, - "scientific_reasoning": 76.1, - "software_engineering": 50.2 - }, + "coverage": 0.2, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [ - "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", - "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", - "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", - "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0" + "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0" ], "reasoning_effort": "unspecified", - "score": 55.95, - "status": "available" + "score": null, + "status": "missing" }, { "components": [ @@ -61923,6 +62620,59 @@ "verified_at": "2026-09-05" } }, + { + "capabilities": [ + "chat", + "reasoning", + "tools", + "structured_output", + "vision", + "long_context" + ], + "description": "OpenAI flagship model for complex reasoning, coding, computer use, research, and document creation.", + "display_name": "GPT-6 Astra", + "distribution": { + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "type": "proprietary_api" + }, + "family": "gpt-6", + "id": "openai/gpt-6-astra", + "kind": "physical", + "knowledge_cutoff": "2026-04-30", + "lifecycle": "active", + "limits": { + "context_window_size": 1050000, + "max_output_tokens": 128000 + }, + "modalities": { + "input": [ + "text", + "image" + ], + "output": [ + "text" + ] + }, + "presentation": { + "logo": "package:openai", + "monochrome": true, + "monogram": "O" + }, + "publisher": "OpenAI", + "reasoning_family": "gpt-6-astra", + "released_at": "2026-09-03", + "tags": [ + "frontier", + "proprietary", + "agentic" + ], + "verification": { + "authority": "OpenAI", + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "status": "claimed", + "verified_at": "2026-09-07" + } + }, { "capabilities": [ "chat", @@ -66133,6 +66883,74 @@ "display_name": "OpenAI", "id": "openai", "models": [ + { + "catalog": "openai/gpt-6-astra", + "id": "gpt-6-astra", + "lifecycle": "active", + "pricing": { + "cache_write_per_1m": 12.5, + "cached_input_per_1m": 1.0, + "completion_per_1m": 50.0, + "currency": "USD", + "prompt_per_1m": 10.0 + }, + "protocols": [ + "openai/chat-completions@1", + "openai/responses@1" + ], + "reasoning_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "reasoning_efforts_by_protocol": { + "openai/chat-completions@1": [ + "low", + "medium", + "high", + "xhigh" + ] + }, + "reasoning_modes": [ + "enabled" + ], + "relationship": "first_party", + "restrictions": { + "long_context_pricing": { + "cache_write_multiplier": 2.0, + "cached_input_multiplier": 2.0, + "completion_multiplier": 1.5, + "input_threshold_tokens": 272000, + "prompt_multiplier": 2.0 + }, + "tools_protocols": [ + "openai/responses@1" + ], + "unsupported_include_values": [ + "message.output_text.logprobs" + ], + "unsupported_request_fields": { + "openai/chat-completions@1": [ + "temperature", + "top_p", + "top_logprobs", + "logprobs" + ], + "openai/responses@1": [ + "temperature", + "top_p", + "top_logprobs" + ] + } + }, + "verification": { + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "status": "claimed", + "verified_at": "2026-09-07" + } + }, { "catalog": "openai/gpt-5.4", "id": "gpt-5.4", @@ -68910,6 +69728,22 @@ "parameter": "reasoning_effort", "type": "reasoning_effort" }, + { + "default_mode": "enabled", + "id": "gpt-6-astra", + "levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "modes": [ + "enabled" + ], + "parameter": "reasoning_effort", + "type": "reasoning_effort" + }, { "default": "medium", "default_mode": "enabled", diff --git a/dashboard/frontend/src/types/modelCatalog.ts b/dashboard/frontend/src/types/modelCatalog.ts index a148449f87..c7f19a6e02 100644 --- a/dashboard/frontend/src/types/modelCatalog.ts +++ b/dashboard/frontend/src/types/modelCatalog.ts @@ -89,6 +89,7 @@ export interface CatalogModelBinding { | 'deepseek_thinking' reasoning_modes?: Array<'enabled' | 'disabled' | 'adaptive'> reasoning_efforts?: string[] + reasoning_efforts_by_protocol?: Record pricing?: Record restrictions?: Record lifecycle: ModelCatalogLifecycle diff --git a/e2e/profiles/response-api/profile.go b/e2e/profiles/response-api/profile.go index 35e1cc4786..f69768c3f4 100644 --- a/e2e/profiles/response-api/profile.go +++ b/e2e/profiles/response-api/profile.go @@ -72,6 +72,7 @@ func (p *Profile) GetTestCases() []string { "response-api-edge-concurrent-requests", "response-api-image-file-id", "input-modality-cross-protocol", + "model-catalog-astra", "protocol-codec-chat-backend-buffered-matrix", "protocol-codec-chat-backend-streaming-matrix", "protocol-codec-responses-backend-buffered-matrix", diff --git a/e2e/profiles/response-api/values.yaml b/e2e/profiles/response-api/values.yaml index fc1cdb3ffc..ae29b62eb5 100644 --- a/e2e/profiles/response-api/values.yaml +++ b/e2e/profiles/response-api/values.yaml @@ -23,6 +23,23 @@ config: provider: vllm endpoint: mock-vllm.default.svc.cluster.local:8000 weight: 1 + # Built-in catalog coverage: the card supplies the provider-native model ID, + # reasoning family, pricing, limits, capabilities, and presentation. + - name: astra-chat + catalog: openai/gpt-6-astra + backend_refs: + - name: mock-openai + provider: openai + base_url: http://mock-vllm.default.svc.cluster.local:8000/v1 + api_key_env: VSR_E2E_OPENAI_API_KEY + - name: astra-responses + catalog: openai/gpt-6-astra + api_format: responses + backend_refs: + - name: mock-openai + provider: openai + base_url: http://mock-vllm.default.svc.cluster.local:8000/v1 + api_key_env: VSR_E2E_OPENAI_API_KEY routing: decisions: - name: default_decision @@ -130,3 +147,7 @@ resources: # E2E/Kind: disable PVC for reliable CI (same as other e2e profiles). persistence: enabled: false + +extraEnv: + - name: VSR_E2E_OPENAI_API_KEY + value: astra-fixture-key diff --git a/e2e/testcases/model_catalog_astra.go b/e2e/testcases/model_catalog_astra.go new file mode 100644 index 0000000000..16a7a0fe04 --- /dev/null +++ b/e2e/testcases/model_catalog_astra.go @@ -0,0 +1,211 @@ +package testcases + +import ( + "context" + "encoding/json" + "fmt" + + "k8s.io/client-go/kubernetes" + + "github.com/vllm-project/semantic-router/e2e/pkg/fixtures" + pkgtestcases "github.com/vllm-project/semantic-router/e2e/pkg/testcases" +) + +const ( + astraChatAlias = "astra-chat" + astraResponsesAlias = "astra-responses" + astraProviderModel = "gpt-6-astra" +) + +func init() { + pkgtestcases.Register("model-catalog-astra", pkgtestcases.TestCase{ + Description: "A catalog-backed built-in model materializes and emits exact OpenAI Chat and Responses reasoning controls", + Tags: []string{"model-catalog", "built-in", "openai", "reasoning", "response-api"}, + Fn: testModelCatalogAstra, + }) +} + +func testModelCatalogAstra( + ctx context.Context, + client *kubernetes.Clientset, + opts pkgtestcases.TestCaseOptions, +) error { + routerSession, err := fixtures.OpenServiceSession(ctx, client, opts) + if err != nil { + return err + } + defer routerSession.Close() + + providerSession, err := openProtocolCodecProviderSession(ctx, client, opts, "openai.responses.v1") + if err != nil { + return err + } + defer providerSession.Close() + + chatSessionID := "astra-chat-contract" + if _, err := sendProtocolMatrixRequestWithHeaders( + ctx, + routerSession, + "/v1/chat/completions", + map[string]any{ + "model": astraChatAlias, + "messages": []map[string]string{{ + "role": "user", "content": "Astra Chat contract", + }}, + "reasoning_effort": "xhigh", + }, + false, + map[string]string{"x-vsr-test-session-id": chatSessionID}, + ); err != nil { + return fmt.Errorf("astra Chat request: %w", err) + } + if err := verifyAstraProviderRequest(ctx, providerSession, chatSessionID, false, "xhigh", false); err != nil { + return err + } + + responsesSessionID := "astra-responses-contract" + if _, err := sendProtocolMatrixRequestWithHeaders( + ctx, + routerSession, + "/v1/responses", + map[string]any{ + "model": astraResponsesAlias, + "input": "Astra Responses contract", + "reasoning": map[string]any{"effort": "max"}, + "store": false, + }, + false, + map[string]string{"x-vsr-test-session-id": responsesSessionID}, + ); err != nil { + return fmt.Errorf("astra Responses request: %w", err) + } + if err := verifyAstraProviderRequest(ctx, providerSession, responsesSessionID, true, "max", false); err != nil { + return err + } + + toolSessionID := "astra-responses-tools-contract" + if _, err := sendProtocolMatrixRequestWithHeaders( + ctx, + routerSession, + "/v1/responses", + map[string]any{ + "model": astraResponsesAlias, + "input": "__mock_tool_call__", + "reasoning": map[string]any{"effort": "high"}, + "store": false, + "tools": []any{protocolLookupTool()}, + "tool_choice": map[string]any{ + "type": "function", + "name": "lookup", + }, + }, + false, + map[string]string{"x-vsr-test-session-id": toolSessionID}, + ); err != nil { + return fmt.Errorf("astra Responses tool request: %w", err) + } + if err := verifyAstraProviderRequest(ctx, providerSession, toolSessionID, true, "high", true); err != nil { + return err + } + + if opts.SetDetails != nil { + opts.SetDetails(map[string]interface{}{ + "catalog": "openai/gpt-6-astra", + "provider_model_id": astraProviderModel, + "protocols_verified": []string{"openai.chat.v1", "openai.responses.v1"}, + "responses_tools": true, + }) + } + return nil +} + +func verifyAstraProviderRequest( + ctx context.Context, + providerSession *fixtures.ServiceSession, + sessionID string, + responses bool, + wantEffort string, + wantTools bool, +) error { + observed, err := lastProviderSimulatorRequest(ctx, providerSession, sessionID) + if err != nil { + return fmt.Errorf("read Astra provider request: %w", err) + } + var debug struct { + Body map[string]json.RawMessage `json:"body"` + } + if err := json.Unmarshal(observed, &debug); err != nil { + return fmt.Errorf("decode Astra provider request: %w", err) + } + + var model string + if err := json.Unmarshal(debug.Body["model"], &model); err != nil || model != astraProviderModel { + return fmt.Errorf("astra provider model = %q, want %q: %s", model, astraProviderModel, truncateString(string(observed), 600)) + } + if responses { + if _, found := debug.Body["input"]; !found { + return fmt.Errorf("astra Responses request lost input: %s", truncateString(string(observed), 600)) + } + if _, found := debug.Body["messages"]; found { + return fmt.Errorf("astra Responses request leaked messages: %s", truncateString(string(observed), 600)) + } + if _, found := debug.Body["reasoning_effort"]; found { + return fmt.Errorf("astra Responses request leaked top-level reasoning_effort: %s", truncateString(string(observed), 600)) + } + var reasoning struct { + Effort string `json:"effort"` + } + if err := json.Unmarshal(debug.Body["reasoning"], &reasoning); err != nil || reasoning.Effort != wantEffort { + return fmt.Errorf("astra Responses effort = %q, want %q: %s", reasoning.Effort, wantEffort, truncateString(string(observed), 600)) + } + _, hasTools := debug.Body["tools"] + if hasTools != wantTools { + return fmt.Errorf("astra Responses tools present = %t, want %t: %s", hasTools, wantTools, truncateString(string(observed), 600)) + } + if wantTools { + var tools []struct { + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Parameters struct { + Type string `json:"type"` + Properties map[string]struct { + Type string `json:"type"` + } `json:"properties"` + Required []string `json:"required"` + } `json:"parameters"` + } + if err := json.Unmarshal(debug.Body["tools"], &tools); err != nil || len(tools) != 1 || + tools[0].Type != "function" || tools[0].Name != "lookup" || + tools[0].Description != "Look up a value" || tools[0].Parameters.Type != "object" || + tools[0].Parameters.Properties["query"].Type != "string" || + len(tools[0].Parameters.Required) != 1 || tools[0].Parameters.Required[0] != "query" { + return fmt.Errorf("astra Responses tool schema was not preserved: %s", truncateString(string(observed), 600)) + } + var toolChoice struct { + Type string `json:"type"` + Name string `json:"name"` + } + if err := json.Unmarshal(debug.Body["tool_choice"], &toolChoice); err != nil || + toolChoice.Type != "function" || toolChoice.Name != "lookup" { + return fmt.Errorf("astra Responses tool choice was not preserved: %s", truncateString(string(observed), 600)) + } + } + return nil + } + + if _, found := debug.Body["messages"]; !found { + return fmt.Errorf("astra Chat request lost messages: %s", truncateString(string(observed), 600)) + } + if _, found := debug.Body["input"]; found { + return fmt.Errorf("astra Chat request leaked input: %s", truncateString(string(observed), 600)) + } + if _, found := debug.Body["reasoning"]; found { + return fmt.Errorf("astra Chat request leaked a reasoning object: %s", truncateString(string(observed), 600)) + } + var effort string + if err := json.Unmarshal(debug.Body["reasoning_effort"], &effort); err != nil || effort != wantEffort { + return fmt.Errorf("astra Chat effort = %q, want %q: %s", effort, wantEffort, truncateString(string(observed), 600)) + } + return nil +} diff --git a/src/semantic-router/pkg/catalog/compiler_model_provider.go b/src/semantic-router/pkg/catalog/compiler_model_provider.go index 95f1796f00..803d94c256 100644 --- a/src/semantic-router/pkg/catalog/compiler_model_provider.go +++ b/src/semantic-router/pkg/catalog/compiler_model_provider.go @@ -169,6 +169,7 @@ func (registry *Registry) findCatalogBinding(providerID, modelID, nativeModelID, } copy := catalogBinding copy.Protocols = append([]string(nil), catalogBinding.Protocols...) + copy.ReasoningEffortsByProtocol = cloneStringSliceMap(catalogBinding.ReasoningEffortsByProtocol) copy.Restrictions = cloneMap(catalogBinding.Restrictions) return © } diff --git a/src/semantic-router/pkg/catalog/registry.go b/src/semantic-router/pkg/catalog/registry.go index 927306231f..cae8cf5cc1 100644 --- a/src/semantic-router/pkg/catalog/registry.go +++ b/src/semantic-router/pkg/catalog/registry.go @@ -260,6 +260,7 @@ func cloneProvider(value ProviderDefinition) ProviderDefinition { value.Models[index].Protocols = append([]string(nil), value.Models[index].Protocols...) value.Models[index].ReasoningModes = append([]string(nil), value.Models[index].ReasoningModes...) value.Models[index].ReasoningEfforts = append([]string(nil), value.Models[index].ReasoningEfforts...) + value.Models[index].ReasoningEffortsByProtocol = cloneStringSliceMap(value.Models[index].ReasoningEffortsByProtocol) value.Models[index].Restrictions = cloneArbitraryMap(value.Models[index].Restrictions) value.Models[index].Pricing.CacheWritePer1M = cloneFloatPointer(value.Models[index].Pricing.CacheWritePer1M) } @@ -386,3 +387,14 @@ func cloneMap[Value any](source map[string]Value) map[string]Value { } return result } + +func cloneStringSliceMap(source map[string][]string) map[string][]string { + if source == nil { + return nil + } + result := make(map[string][]string, len(source)) + for key, value := range source { + result[key] = append([]string(nil), value...) + } + return result +} diff --git a/src/semantic-router/pkg/catalog/types.go b/src/semantic-router/pkg/catalog/types.go index 6582d661e1..e71143a959 100644 --- a/src/semantic-router/pkg/catalog/types.go +++ b/src/semantic-router/pkg/catalog/types.go @@ -75,13 +75,15 @@ type CatalogModelBinding struct { Protocols []string `json:"protocols"` ReasoningTransport ReasoningTransport `json:"reasoning_transport,omitempty"` // ReasoningModes and ReasoningEfforts narrow the model-level capability to - // values accepted by this provider's API. They never expand the family. - ReasoningModes []string `json:"reasoning_modes,omitempty"` - ReasoningEfforts []string `json:"reasoning_efforts,omitempty"` - Pricing Pricing `json:"pricing,omitempty"` - Restrictions map[string]any `json:"restrictions,omitempty"` - Lifecycle string `json:"lifecycle,omitempty"` - Verification CatalogBindingVerification `json:"verification"` + // values accepted by this provider's API. A protocol override can narrow the + // common effort set further; neither form may expand the model family. + ReasoningModes []string `json:"reasoning_modes,omitempty"` + ReasoningEfforts []string `json:"reasoning_efforts,omitempty"` + ReasoningEffortsByProtocol map[string][]string `json:"reasoning_efforts_by_protocol,omitempty"` + Pricing Pricing `json:"pricing,omitempty"` + Restrictions map[string]any `json:"restrictions,omitempty"` + Lifecycle string `json:"lifecycle,omitempty"` + Verification CatalogBindingVerification `json:"verification"` } type ReasoningTransport string diff --git a/src/semantic-router/pkg/catalog/zz_generated_catalog.go b/src/semantic-router/pkg/catalog/zz_generated_catalog.go index bca6b38b38..f6b1a9ffe9 100644 --- a/src/semantic-router/pkg/catalog/zz_generated_catalog.go +++ b/src/semantic-router/pkg/catalog/zz_generated_catalog.go @@ -2,7 +2,7 @@ package catalog -const builtInCatalogDigest = "sha256:1252383abd22640370153c56390b1972744171e167991a60dc9fb3f750728c33" +const builtInCatalogDigest = "sha256:0e4500fb858d55a85b21c028eabb69d5b8b13eb894a98e0ec3e7931cd50ed607" const builtInCatalogJSON = `{ "benchmarks": [ @@ -7553,6 +7553,248 @@ const builtInCatalogJSON = `{ "reasoning_effort": "unspecified", "status": "missing" }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evaluation": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "available", + "value": 0.96 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", @@ -35671,6 +35913,122 @@ const builtInCatalogJSON = `{ "variant": "gpt-oss-20b" } }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", + "metrics": { + "accuracy": 0.96 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "with-tools", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-humanitys-last-exam-tools@1.0.0", + "metrics": { + "accuracy": 0.572 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "tool_policy": "tools_enabled", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "datacurve/deep-swe@1.1.0", + "benchmark_profile": "published-agent", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-deep-swe-1.1@1.0.0", + "metrics": { + "resolved": 0.741 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "arc-prize/arc-agi-2@1.0.0", + "benchmark_profile": "semi-private", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-arc-agi-2@1.0.0", + "metrics": { + "accuracy": 0.95 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "arc-prize/arc-agi-1@1.0.0", + "benchmark_profile": "semi-private", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-arc-agi-1@1.0.0", + "metrics": { + "accuracy": 0.985 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, { "benchmark": "artificial-analysis/critpt@1.0.0", "benchmark_profile": "independent-standard", @@ -51603,9 +51961,360 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3.5-lightning", + "model": "nvidia/nemotron-3.5-lightning", + "provenance": [], + "reasoning_effort": "disabled", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [], + "reasoning_effort": "medium", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", + "metric": "accuracy", + "normalized": 0.879, + "status": "available", + "value": 0.879, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", + "metric": "accuracy", + "normalized": 0.261, + "status": "available", + "value": 0.261, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0", + "metric": "resolved", + "normalized": 0.695, + "status": "available", + "value": 0.695, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", + "metric": "resolved", + "normalized": 0.539, + "status": "available", + "value": 0.539, + "weight": 0.2 + } + ], + "coverage": 0.8, + "domains": { + "agentic_systems": 53.900000000000006, + "frontier_reasoning": 26.1, + "scientific_reasoning": 87.9, + "software_engineering": 69.49999999999999 + }, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [ + "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0" + ], + "reasoning_effort": "high", + "score": 59.349999999999994, + "status": "available" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [], + "reasoning_effort": "enabled", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", + "provenance": [], + "reasoning_effort": "low", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", + "provenance": [], + "reasoning_effort": "high", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "enabled", "score": null, "status": "missing" }, @@ -51659,9 +52368,9 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "provenance": [], - "reasoning_effort": "medium", + "reasoning_effort": "enabled", "score": null, "status": "missing" }, @@ -51679,62 +52388,47 @@ const builtInCatalogJSON = `{ { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", "metric": "accuracy", - "normalized": 0.879, - "status": "available", - "value": 0.879, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": 0.261, - "status": "available", - "value": 0.261, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0", "metric": "resolved", - "normalized": 0.695, - "status": "available", - "value": 0.695, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "harbor/terminal-bench@2.1.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", "metric": "resolved", - "normalized": 0.539, - "status": "available", - "value": 0.539, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 } ], - "coverage": 0.8, - "domains": { - "agentic_systems": 53.900000000000006, - "frontier_reasoning": 26.1, - "scientific_reasoning": 87.9, - "software_engineering": 69.49999999999999 - }, + "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", - "provenance": [ - "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0" - ], - "reasoning_effort": "high", - "score": 59.349999999999994, - "status": "available" + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "provenance": [], + "reasoning_effort": "disabled", + "score": null, + "status": "missing" }, { "components": [ @@ -51786,7 +52480,7 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", + "model": "nvidia/nemotron-cascade-2-30b-a3b", "provenance": [], "reasoning_effort": "enabled", "score": null, @@ -51842,9 +52536,9 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", + "model": "nvidia/nemotron-cascade-2-30b-a3b", "provenance": [], - "reasoning_effort": "low", + "reasoning_effort": "disabled", "score": null, "status": "missing" }, @@ -51853,37 +52547,41 @@ const builtInCatalogJSON = `{ { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.798, + "status": "available", + "value": 0.798, "weight": 0.2 }, { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.761, + "status": "available", + "value": 0.761, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", + "evaluation": "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.177, + "status": "available", + "value": 0.177, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0", "metric": "resolved", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.502, + "status": "available", + "value": 0.502, "weight": 0.2 }, { @@ -51896,13 +52594,24 @@ const builtInCatalogJSON = `{ "weight": 0.2 } ], - "coverage": 0.0, + "coverage": 0.8, + "domains": { + "frontier_reasoning": 17.7, + "general_reasoning": 79.80000000000001, + "scientific_reasoning": 76.1, + "software_engineering": 50.2 + }, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", - "provenance": [], - "reasoning_effort": "high", - "score": null, - "status": "missing" + "model": "nvidia/nemotron-cascade-2-30b-a3b", + "provenance": [ + "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", + "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", + "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", + "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0" + ], + "reasoning_effort": "unspecified", + "score": 55.95, + "status": "available" }, { "components": [ @@ -51954,9 +52663,9 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "low", "score": null, "status": "missing" }, @@ -52010,9 +52719,9 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "medium", "score": null, "status": "missing" }, @@ -52066,9 +52775,9 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "high", "score": null, "status": "missing" }, @@ -52122,9 +52831,9 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "xhigh", "score": null, "status": "missing" }, @@ -52178,9 +52887,9 @@ const builtInCatalogJSON = `{ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "max", "score": null, "status": "missing" }, @@ -52189,41 +52898,38 @@ const builtInCatalogJSON = `{ { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", "metric": "accuracy", - "normalized": 0.798, - "status": "available", - "value": 0.798, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", + "evaluation": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", "metric": "accuracy", - "normalized": 0.761, + "normalized": 0.96, "status": "available", - "value": 0.761, + "value": 0.96, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", - "evaluation": "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": 0.177, - "status": "available", - "value": 0.177, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0", "metric": "resolved", - "normalized": 0.502, - "status": "available", - "value": 0.502, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { @@ -52236,24 +52942,15 @@ const builtInCatalogJSON = `{ "weight": 0.2 } ], - "coverage": 0.8, - "domains": { - "frontier_reasoning": 17.7, - "general_reasoning": 79.80000000000001, - "scientific_reasoning": 76.1, - "software_engineering": 50.2 - }, + "coverage": 0.2, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [ - "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", - "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", - "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", - "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0" + "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0" ], "reasoning_effort": "unspecified", - "score": 55.95, - "status": "available" + "score": null, + "status": "missing" }, { "components": [ @@ -61929,6 +62626,59 @@ const builtInCatalogJSON = `{ "verified_at": "2026-09-05" } }, + { + "capabilities": [ + "chat", + "reasoning", + "tools", + "structured_output", + "vision", + "long_context" + ], + "description": "OpenAI flagship model for complex reasoning, coding, computer use, research, and document creation.", + "display_name": "GPT-6 Astra", + "distribution": { + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "type": "proprietary_api" + }, + "family": "gpt-6", + "id": "openai/gpt-6-astra", + "kind": "physical", + "knowledge_cutoff": "2026-04-30", + "lifecycle": "active", + "limits": { + "context_window_size": 1050000, + "max_output_tokens": 128000 + }, + "modalities": { + "input": [ + "text", + "image" + ], + "output": [ + "text" + ] + }, + "presentation": { + "logo": "package:openai", + "monochrome": true, + "monogram": "O" + }, + "publisher": "OpenAI", + "reasoning_family": "gpt-6-astra", + "released_at": "2026-09-03", + "tags": [ + "frontier", + "proprietary", + "agentic" + ], + "verification": { + "authority": "OpenAI", + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "status": "claimed", + "verified_at": "2026-09-07" + } + }, { "capabilities": [ "chat", @@ -66139,6 +66889,74 @@ const builtInCatalogJSON = `{ "display_name": "OpenAI", "id": "openai", "models": [ + { + "catalog": "openai/gpt-6-astra", + "id": "gpt-6-astra", + "lifecycle": "active", + "pricing": { + "cache_write_per_1m": 12.5, + "cached_input_per_1m": 1.0, + "completion_per_1m": 50.0, + "currency": "USD", + "prompt_per_1m": 10.0 + }, + "protocols": [ + "openai/chat-completions@1", + "openai/responses@1" + ], + "reasoning_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "reasoning_efforts_by_protocol": { + "openai/chat-completions@1": [ + "low", + "medium", + "high", + "xhigh" + ] + }, + "reasoning_modes": [ + "enabled" + ], + "relationship": "first_party", + "restrictions": { + "long_context_pricing": { + "cache_write_multiplier": 2.0, + "cached_input_multiplier": 2.0, + "completion_multiplier": 1.5, + "input_threshold_tokens": 272000, + "prompt_multiplier": 2.0 + }, + "tools_protocols": [ + "openai/responses@1" + ], + "unsupported_include_values": [ + "message.output_text.logprobs" + ], + "unsupported_request_fields": { + "openai/chat-completions@1": [ + "temperature", + "top_p", + "top_logprobs", + "logprobs" + ], + "openai/responses@1": [ + "temperature", + "top_p", + "top_logprobs" + ] + } + }, + "verification": { + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "status": "claimed", + "verified_at": "2026-09-07" + } + }, { "catalog": "openai/gpt-5.4", "id": "gpt-5.4", @@ -68916,6 +69734,22 @@ const builtInCatalogJSON = `{ "parameter": "reasoning_effort", "type": "reasoning_effort" }, + { + "default_mode": "enabled", + "id": "gpt-6-astra", + "levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "modes": [ + "enabled" + ], + "parameter": "reasoning_effort", + "type": "reasoning_effort" + }, { "default": "medium", "default_mode": "enabled", diff --git a/src/semantic-router/pkg/config/canonical_catalog.go b/src/semantic-router/pkg/config/canonical_catalog.go index b845e32286..e4e57e6aff 100644 --- a/src/semantic-router/pkg/config/canonical_catalog.go +++ b/src/semantic-router/pkg/config/canonical_catalog.go @@ -253,6 +253,9 @@ func materializedProviderProfile( if effective.CatalogBinding != nil { reasoningModes = append([]string(nil), effective.CatalogBinding.ReasoningModes...) reasoningEfforts = append([]string(nil), effective.CatalogBinding.ReasoningEfforts...) + if protocolEfforts, ok := effective.CatalogBinding.ReasoningEffortsByProtocol[effective.Binding.Protocol]; ok { + reasoningEfforts = append([]string(nil), protocolEfforts...) + } } return ProviderProfile{ Type: provider.Definition.ID, Protocol: effective.Binding.Protocol, BaseURL: baseURL, diff --git a/src/semantic-router/pkg/config/canonical_router_replay_test.go b/src/semantic-router/pkg/config/canonical_router_replay_test.go index 055f23dd83..cb6466eeb7 100644 --- a/src/semantic-router/pkg/config/canonical_router_replay_test.go +++ b/src/semantic-router/pkg/config/canonical_router_replay_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "strings" "testing" @@ -302,6 +303,125 @@ routing: {} } } +func TestGPT6AstraCatalogMaterializesItsExactReasoningContract(t *testing.T) { + cfg, err := ParseYAMLBytes([]byte(` +version: v0.3 +providers: + models: + - name: astra + catalog: openai/gpt-6-astra + api_format: responses + backend_refs: + - name: primary + provider: openai +routing: + decisions: + - name: astra_default + priority: 1 + rules: + operator: AND + conditions: [] + modelRefs: + - model: astra + use_reasoning: true + reasoning_effort: max +`)) + if err != nil { + t.Fatal(err) + } + + family := cfg.GetModelReasoningFamily("astra") + if family == nil { + t.Fatal("Astra reasoning family is missing") + } + if got := strings.Join(family.Levels, ","); got != "low,medium,high,xhigh,max" { + t.Fatalf("Astra reasoning levels = %q", got) + } + if family.Default != "" || family.Disabled != "" || + len(family.Modes) != 1 || family.Modes[0] != ReasoningModeEnabled { + t.Fatalf("Astra reasoning contract = %+v", *family) + } + profile, ok := cfg.ProviderProfiles["astra_primary"] + if !ok || profile.Protocol != "openai/responses@1" { + t.Fatalf("Astra provider profile = %+v", profile) + } + if got := strings.Join(profile.ReasoningEfforts, ","); got != "low,medium,high,xhigh,max" { + t.Fatalf("Astra provider efforts = %q", got) + } + if got := cfg.ResolveExternalModelID("astra", "astra_primary"); got != "gpt-6-astra" { + t.Fatalf("Astra provider model ID = %q", got) + } +} + +func TestGPT6AstraCatalogNarrowsChatReasoningEfforts(t *testing.T) { + configForEffort := func(effort string) []byte { + return []byte(fmt.Sprintf(` +version: v0.3 +providers: + models: + - name: astra + catalog: openai/gpt-6-astra + backend_refs: + - name: primary + provider: openai +routing: + decisions: + - name: astra_default + priority: 1 + rules: + operator: AND + conditions: [] + modelRefs: + - model: astra + use_reasoning: true + reasoning_effort: %s +`, effort)) + } + + cfg, err := ParseYAMLBytes(configForEffort("xhigh")) + if err != nil { + t.Fatal(err) + } + profile, ok := cfg.ProviderProfiles["astra_primary"] + if !ok || profile.Protocol != "openai/chat-completions@1" { + t.Fatalf("Astra Chat provider profile = %+v", profile) + } + if got := strings.Join(profile.ReasoningEfforts, ","); got != "low,medium,high,xhigh" { + t.Fatalf("Astra Chat provider efforts = %q", got) + } + + _, err = ParseYAMLBytes(configForEffort("max")) + if err == nil || !strings.Contains(err.Error(), `reasoning_effort "max" is not supported by provider "openai"`) { + t.Fatalf("expected Astra Chat max-effort rejection, got %v", err) + } +} + +func TestGPT6AstraCatalogRejectsDisabledReasoning(t *testing.T) { + _, err := ParseYAMLBytes([]byte(` +version: v0.3 +providers: + models: + - name: astra + catalog: openai/gpt-6-astra + backend_refs: + - name: primary + provider: openai +routing: + decisions: + - name: astra_disabled + priority: 1 + rules: + operator: AND + conditions: [] + modelRefs: + - model: astra + use_reasoning: false +`)) + if err == nil || !strings.Contains(err.Error(), "always-on reasoning family") { + t.Fatalf("expected Astra disable rejection, got %v", err) + } +} + func TestRouterOwnedListenerRejectsBackendlessPhysicalModel(t *testing.T) { _, err := ParseYAMLBytes([]byte(` version: v0.3 diff --git a/src/semantic-router/pkg/extproc/provider_request_catalog_contract_test.go b/src/semantic-router/pkg/extproc/provider_request_catalog_contract_test.go index f68f88e2a0..2d94b4c691 100644 --- a/src/semantic-router/pkg/extproc/provider_request_catalog_contract_test.go +++ b/src/semantic-router/pkg/extproc/provider_request_catalog_contract_test.go @@ -32,6 +32,17 @@ type catalogReasoningWireCase struct { // runtime adapter from drifting independently. func TestBuiltInCatalogReasoningWireContracts(t *testing.T) { tests := []catalogReasoningWireCase{ + { + name: "OpenAI Astra Chat xhigh effort", catalog: "openai/gpt-6-astra", provider: "openai", + enabled: true, effort: "xhigh", wantTransport: modelcatalog.ReasoningTransportTopLevelEffort, + wantControls: map[string]interface{}{"reasoning_effort": "xhigh"}, + }, + { + name: "OpenAI Astra Responses max effort", catalog: "openai/gpt-6-astra", provider: "openai", + apiFormat: config.APIFormatResponses, enabled: true, effort: "max", + wantTransport: modelcatalog.ReasoningTransportTopLevelEffort, + wantControls: map[string]interface{}{"reasoning": map[string]interface{}{"effort": "max"}}, + }, { name: "OpenAI Chat effort", catalog: "openai/gpt-5.6-sol", provider: "openai", enabled: true, effort: "high", wantTransport: modelcatalog.ReasoningTransportTopLevelEffort, diff --git a/src/vllm-sr/cli/model_assets/latest/catalog.yaml b/src/vllm-sr/cli/model_assets/latest/catalog.yaml index 31924489de..99f9839ac2 100644 --- a/src/vllm-sr/cli/model_assets/latest/catalog.yaml +++ b/src/vllm-sr/cli/model_assets/latest/catalog.yaml @@ -1563,6 +1563,58 @@ providers: status: fixture_verified verified_at: '2026-09-04' models: + - catalog: openai/gpt-6-astra + relationship: first_party + id: gpt-6-astra + protocols: + - openai/chat-completions@1 + - openai/responses@1 + reasoning_modes: + - enabled + reasoning_efforts: + - low + - medium + - high + - xhigh + - max + reasoning_efforts_by_protocol: + openai/chat-completions@1: + - low + - medium + - high + - xhigh + pricing: + currency: USD + prompt_per_1m: 10.0 + cached_input_per_1m: 1.0 + cache_write_per_1m: 12.5 + completion_per_1m: 50.0 + restrictions: + tools_protocols: + - openai/responses@1 + long_context_pricing: + input_threshold_tokens: 272000 + prompt_multiplier: 2.0 + cached_input_multiplier: 2.0 + cache_write_multiplier: 2.0 + completion_multiplier: 1.5 + unsupported_request_fields: + openai/chat-completions@1: + - temperature + - top_p + - top_logprobs + - logprobs + openai/responses@1: + - temperature + - top_p + - top_logprobs + unsupported_include_values: + - message.output_text.logprobs + lifecycle: active + verification: + status: claimed + verified_at: '2026-09-07' + source: https://developers.openai.com/api/docs/models/gpt-6-astra - catalog: openai/gpt-5.4 relationship: first_party id: gpt-5.4 @@ -3602,6 +3654,18 @@ reasoning_families: - enabled - disabled default_mode: enabled +- id: gpt-6-astra + type: reasoning_effort + parameter: reasoning_effort + levels: + - low + - medium + - high + - xhigh + - max + modes: + - enabled + default_mode: enabled - id: gpt-5.6 type: reasoning_effort parameter: reasoning_effort @@ -6069,6 +6133,48 @@ models: - moe - efficient released_at: '2026-03-19' +- id: openai/gpt-6-astra + display_name: GPT-6 Astra + description: OpenAI flagship model for complex reasoning, coding, computer use, research, and document creation. + kind: physical + publisher: OpenAI + presentation: + logo: package:openai + monogram: O + monochrome: true + distribution: + type: proprietary_api + source: https://developers.openai.com/api/docs/models/gpt-6-astra + family: gpt-6 + lifecycle: active + limits: + context_window_size: 1050000 + max_output_tokens: 128000 + capabilities: + - chat + - reasoning + - tools + - structured_output + - vision + - long_context + modalities: + input: + - text + - image + output: + - text + reasoning_family: gpt-6-astra + verification: + authority: OpenAI + status: claimed + verified_at: '2026-09-07' + source: https://developers.openai.com/api/docs/models/gpt-6-astra + tags: + - frontier + - proprietary + - agentic + released_at: '2026-09-03' + knowledge_cutoff: '2026-04-30' - id: openai/gpt-5.6-sol display_name: GPT-5.6 Sol description: OpenAI high-capability GPT-5.6 model for demanding reasoning and agentic work. @@ -27217,6 +27323,97 @@ evaluations: verification: claimed source: https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf redistributable: true +- id: openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 + model: openai/gpt-6-astra + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.96 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-humanitys-last-exam-tools@1.0.0 + model: openai/gpt-6-astra + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: with-tools + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + tool_policy: tools_enabled + source_kind: official_launch_evaluation + metrics: + accuracy: 0.572 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-deep-swe-1.1@1.0.0 + model: openai/gpt-6-astra + benchmark: datacurve/deep-swe@1.1.0 + benchmark_profile: published-agent + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + resolved: 0.741 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-arc-agi-2@1.0.0 + model: openai/gpt-6-astra + benchmark: arc-prize/arc-agi-2@1.0.0 + benchmark_profile: semi-private + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.95 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true +- id: openai/gpt-6-astra-launch-arc-agi-1@1.0.0 + model: openai/gpt-6-astra + benchmark: arc-prize/arc-agi-1@1.0.0 + benchmark_profile: semi-private + reasoning_effort: unspecified + subject: + variant: GPT-6 Astra + result_selection: maximum_across_supported_efforts + source_kind: official_launch_evaluation + metrics: + accuracy: 0.985 + status: available + observed_at: '2026-09-07' + evidence: + provenance: vendor_claimed + verification: claimed + source: https://openai.com/index/gpt-6-astra/ + redistributable: true - id: independent/qwen3-7-max-critpt@1.0.0 model: qwen/qwen3.7-max benchmark: artificial-analysis/critpt@1.0.0 @@ -38224,6 +38421,188 @@ evaluation_coverage: benchmark_profile: published-agent metric: resolved status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: low + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: medium + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: high + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: xhigh + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: max + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: idavidrein/gpqa-diamond@1.0.0 + benchmark_profile: published-standard + metric: accuracy + status: available + value: 0.96 + evaluation: openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: cais/humanitys-last-exam@1.0.0 + benchmark_profile: no-tools + metric: accuracy + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: swe-bench/verified@1.0.0 + benchmark_profile: published-agent + metric: resolved + status: missing +- model: openai/gpt-6-astra + reasoning_effort: unspecified + benchmark: harbor/terminal-bench@2.1.0 + benchmark_profile: published-agent + metric: resolved + status: missing - model: openai/gpt-5.6-sol reasoning_effort: low benchmark: tiger-ai-lab/mmlu-pro@1.0.0 @@ -48173,6 +48552,266 @@ index_results: general_reasoning: 79.80000000000001 scientific_reasoning: 76.1 software_engineering: 50.2 +- model: openai/gpt-6-astra + reasoning_effort: low + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: medium + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: high + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: xhigh + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: max + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.0 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: [] +- model: openai/gpt-6-astra + reasoning_effort: unspecified + index: vllm-sr/intelligence@1.0.0 + status: missing + score: null + coverage: 0.2 + components: + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: tiger-ai-lab/mmlu-pro@1.0.0 + metric: accuracy + benchmark_profile: published-standard + - weight: 0.2 + status: available + value: 0.96 + normalized: 0.96 + benchmark: idavidrein/gpqa-diamond@1.0.0 + metric: accuracy + benchmark_profile: published-standard + evaluation: openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: cais/humanitys-last-exam@1.0.0 + metric: accuracy + benchmark_profile: no-tools + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: swe-bench/verified@1.0.0 + metric: resolved + benchmark_profile: published-agent + - weight: 0.2 + status: missing + value: null + normalized: null + benchmark: harbor/terminal-bench@2.1.0 + metric: resolved + benchmark_profile: published-agent + provenance: + - openai/gpt-6-astra-launch-gpqa-diamond@1.0.0 - model: openai/gpt-5.6-sol reasoning_effort: low index: vllm-sr/intelligence@1.0.0 diff --git a/tools/catalog/catalog_validation.py b/tools/catalog/catalog_validation.py index 644e4a96be..2c48a5b327 100644 --- a/tools/catalog/catalog_validation.py +++ b/tools/catalog/catalog_validation.py @@ -155,6 +155,7 @@ def _validate_provider_binding( "reasoning_transport", "reasoning_modes", "reasoning_efforts", + "reasoning_efforts_by_protocol", "pricing", "restrictions", "lifecycle", @@ -174,6 +175,8 @@ def _validate_provider_binding( raise CatalogBuildError(f"{path}.reasoning_transport is unsupported") if model_id not in models: raise CatalogBuildError(f"{path}.catalog references an unknown model") + protocols = _sequence(item.get("protocols"), f"{path}.protocols") + _validate_binding_protocols(protocols, path, provider, protocol_ids) _validate_reasoning_transport_family( item, path, @@ -181,7 +184,13 @@ def _validate_provider_binding( models[model_id], reasoning_families, ) - _validate_reasoning_binding_values(item, path, models[model_id], reasoning_families) + _validate_reasoning_binding_values( + item, + path, + models[model_id], + reasoning_families, + protocols, + ) if native_id in native_ids: raise CatalogBuildError( f"{path}.id duplicates a provider-native model identifier" @@ -192,8 +201,7 @@ def _validate_provider_binding( if verification.get("source") is not None: _validate_https_url(verification["source"], f"{path}.verification.source") _validate_provider_model_id_policy(item, path) - protocols = _sequence(item.get("protocols"), f"{path}.protocols") - _validate_binding_protocols(protocols, path, provider, protocol_ids) + _validate_provider_model_api_restrictions(item, path, protocols) for protocol in protocols: pair = (model_id, str(protocol)) if pair in pairs: @@ -265,15 +273,21 @@ def _validated_reasoning_modes(item: dict[str, Any], path: str) -> list[Any] | N return values -def _validated_reasoning_efforts(item: dict[str, Any], path: str) -> list[Any] | None: +def _validated_reasoning_efforts(item: dict[str, Any], path: str) -> list[str] | None: efforts = item.get("reasoning_efforts") if efforts is None: return None - values = _sequence(efforts, f"{path}.reasoning_efforts") + return _validated_reasoning_effort_values(efforts, f"{path}.reasoning_efforts") + + +def _validated_reasoning_effort_values(value: Any, path: str) -> list[str]: + raw_values = _sequence(value, path) + values = [ + _nonempty_string(raw_value, f"{path}[{index}]") + for index, raw_value in enumerate(raw_values) + ] if not values or len(values) != len(set(values)): - raise CatalogBuildError(f"{path}.reasoning_efforts is invalid") - for index, value in enumerate(values): - _nonempty_string(value, f"{path}.reasoning_efforts[{index}]") + raise CatalogBuildError(f"{path} is invalid") return values @@ -296,7 +310,7 @@ def _validate_reasoning_mode_subset( def _validate_reasoning_effort_subset( - efforts: list[Any] | None, + efforts: list[str] | None, family: dict[str, Any], path: str, ) -> None: @@ -319,10 +333,12 @@ def _validate_reasoning_binding_values( path: str, model: dict[str, Any], reasoning_families: dict[str, dict[str, Any]], + protocols: list[Any], ) -> None: modes = _validated_reasoning_modes(item, path) efforts = _validated_reasoning_efforts(item, path) - if modes is None and efforts is None: + efforts_by_protocol = item.get("reasoning_efforts_by_protocol") + if modes is None and efforts is None and efforts_by_protocol is None: return family_id = model.get("reasoning_family") family = reasoning_families.get(str(family_id)) @@ -332,6 +348,31 @@ def _validate_reasoning_binding_values( ) _validate_reasoning_mode_subset(modes, family, path) _validate_reasoning_effort_subset(efforts, family, path) + if efforts_by_protocol is None: + return + if efforts is None: + raise CatalogBuildError( + f"{path}.reasoning_efforts_by_protocol requires reasoning_efforts" + ) + overrides = _mapping( + efforts_by_protocol, + f"{path}.reasoning_efforts_by_protocol", + ) + if not overrides: + raise CatalogBuildError( + f"{path}.reasoning_efforts_by_protocol must not be empty" + ) + supported_protocols = {str(protocol) for protocol in protocols} + for protocol, raw_values in overrides.items(): + override_path = f"{path}.reasoning_efforts_by_protocol.{protocol}" + if protocol not in supported_protocols: + raise CatalogBuildError(f"{override_path} references an unbound protocol") + values = _validated_reasoning_effort_values(raw_values, override_path) + if not set(values).issubset(efforts): + raise CatalogBuildError( + f"{override_path} must narrow the provider reasoning_efforts" + ) + _validate_reasoning_effort_subset(values, family, override_path) def _validate_provider_model_id_policy(item: dict[str, Any], path: str) -> None: @@ -351,6 +392,113 @@ def _validate_provider_model_id_policy(item: dict[str, Any], path: str) -> None: ) +def _validate_provider_model_api_restrictions( + item: dict[str, Any], path: str, protocols: list[Any] +) -> None: + """Validate protocol-scoped model constraints without closing extensions.""" + + if "restrictions" not in item: + return + restrictions = _mapping(item["restrictions"], f"{path}.restrictions") + supported_protocols = {str(protocol) for protocol in protocols} + + tools_protocols = restrictions.get("tools_protocols") + if tools_protocols is not None: + values = _sequence(tools_protocols, f"{path}.restrictions.tools_protocols") + normalized_values = [ + _nonempty_string(value, f"{path}.restrictions.tools_protocols[{index}]") + for index, value in enumerate(values) + ] + if not normalized_values or len(normalized_values) != len( + set(normalized_values) + ): + raise CatalogBuildError( + f"{path}.restrictions.tools_protocols must be a non-empty unique list" + ) + unknown = sorted( + value for value in normalized_values if value not in supported_protocols + ) + if unknown: + raise CatalogBuildError( + f"{path}.restrictions.tools_protocols references an unbound protocol: " + + ", ".join(unknown) + ) + + long_context_pricing = restrictions.get("long_context_pricing") + if long_context_pricing is not None: + pricing_path = f"{path}.restrictions.long_context_pricing" + pricing = _mapping(long_context_pricing, pricing_path) + multiplier_fields = { + "prompt_multiplier", + "cached_input_multiplier", + "cache_write_multiplier", + "completion_multiplier", + } + _reject_unknown( + pricing, + {"input_threshold_tokens", *multiplier_fields}, + pricing_path, + ) + threshold = pricing.get("input_threshold_tokens") + if ( + not isinstance(threshold, int) + or isinstance(threshold, bool) + or threshold <= 0 + ): + raise CatalogBuildError( + f"{pricing_path}.input_threshold_tokens must be a positive integer" + ) + present_multipliers = multiplier_fields & pricing.keys() + if not present_multipliers: + raise CatalogBuildError( + f"{pricing_path} must declare at least one price multiplier" + ) + for field in sorted(present_multipliers): + value = pricing[field] + if not _is_finite_number(value) or value <= 0: + raise CatalogBuildError(f"{pricing_path}.{field} must be positive") + + unsupported_include_values = restrictions.get("unsupported_include_values") + if unsupported_include_values is not None: + include_path = f"{path}.restrictions.unsupported_include_values" + raw_values = _sequence(unsupported_include_values, include_path) + values = [ + _nonempty_string(value, f"{include_path}[{index}]") + for index, value in enumerate(raw_values) + ] + if not values or len(values) != len(set(values)): + raise CatalogBuildError(f"{include_path} must be a non-empty unique list") + for index, value in enumerate(values): + if not re.fullmatch(r"[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*", value): + raise CatalogBuildError( + f"{include_path}[{index}] must be a dotted JSON field path" + ) + + unsupported_fields = restrictions.get("unsupported_request_fields") + if unsupported_fields is None: + return + fields_by_protocol = _mapping( + unsupported_fields, + f"{path}.restrictions.unsupported_request_fields", + ) + for protocol, raw_fields in fields_by_protocol.items(): + protocol_path = f"{path}.restrictions.unsupported_request_fields.{protocol}" + if protocol not in supported_protocols: + raise CatalogBuildError(f"{protocol_path} references an unbound protocol") + raw_field_values = _sequence(raw_fields, protocol_path) + fields = [ + _nonempty_string(field, f"{protocol_path}[{index}]") + for index, field in enumerate(raw_field_values) + ] + if not fields or len(fields) != len(set(fields)): + raise CatalogBuildError(f"{protocol_path} must be a non-empty unique list") + for index, name in enumerate(fields): + if not re.fullmatch(r"[a-z][a-z0-9_]*", name): + raise CatalogBuildError( + f"{protocol_path}[{index}] must be a JSON field name" + ) + + def _validate_binding_protocols( protocols: list[Any], path: str, diff --git a/tools/catalog/tests/test_generate_model_catalog.py b/tools/catalog/tests/test_generate_model_catalog.py index e698f386c1..f0df6f66ed 100644 --- a/tools/catalog/tests/test_generate_model_catalog.py +++ b/tools/catalog/tests/test_generate_model_catalog.py @@ -178,7 +178,142 @@ def test_physical_inventory_is_the_curated_mainstream_creator_set(self) -> None: self.assertIn( physical_models[model_id]["lifecycle"], {"active", "experimental"} ) - self.assertNotIn("openai/gpt-6-astra", physical_models) + self.assertIn("openai/gpt-6-astra", physical_models) + + def test_gpt_6_astra_day_zero_contract_is_complete(self) -> None: + manifest, resources, _ = catalog.load_and_validate() + models = {model["id"]: model for model in resources["models"]} + astra = models["openai/gpt-6-astra"] + self.assertEqual( + astra["limits"], + { + "context_window_size": 1_050_000, + "max_output_tokens": 128_000, + }, + ) + self.assertEqual(astra["knowledge_cutoff"], "2026-04-30") + self.assertEqual(astra["reasoning_family"], "gpt-6-astra") + self.assertTrue( + {"chat", "reasoning", "tools", "structured_output", "vision"}.issubset( + astra["capabilities"] + ) + ) + + families = {family["id"]: family for family in resources["reasoning_families"]} + reasoning = families["gpt-6-astra"] + self.assertEqual(reasoning["levels"], ["low", "medium", "high", "xhigh", "max"]) + self.assertEqual(reasoning["modes"], ["enabled"]) + self.assertNotIn("disabled", reasoning) + self.assertNotIn( + "default", + reasoning, + "the official Astra model page does not publish a default effort", + ) + + providers = {provider["id"]: provider for provider in resources["providers"]} + binding = next( + item + for item in providers["openai"]["models"] + if item["catalog"] == "openai/gpt-6-astra" + ) + self.assertEqual(binding["id"], "gpt-6-astra") + self.assertEqual( + binding["protocols"], + ["openai/chat-completions@1", "openai/responses@1"], + ) + self.assertEqual(binding["reasoning_modes"], ["enabled"]) + self.assertEqual(binding["reasoning_efforts"], reasoning["levels"]) + self.assertEqual( + binding["reasoning_efforts_by_protocol"], + { + "openai/chat-completions@1": [ + "low", + "medium", + "high", + "xhigh", + ] + }, + ) + self.assertEqual( + binding["pricing"], + { + "currency": "USD", + "prompt_per_1m": 10.0, + "cached_input_per_1m": 1.0, + "cache_write_per_1m": 12.5, + "completion_per_1m": 50.0, + }, + ) + self.assertEqual( + binding["restrictions"], + { + "tools_protocols": ["openai/responses@1"], + "long_context_pricing": { + "input_threshold_tokens": 272000, + "prompt_multiplier": 2.0, + "cached_input_multiplier": 2.0, + "cache_write_multiplier": 2.0, + "completion_multiplier": 1.5, + }, + "unsupported_request_fields": { + "openai/chat-completions@1": [ + "temperature", + "top_p", + "top_logprobs", + "logprobs", + ], + "openai/responses@1": [ + "temperature", + "top_p", + "top_logprobs", + ], + }, + "unsupported_include_values": ["message.output_text.logprobs"], + }, + ) + + evaluations = [ + item + for item in resources["evaluations"] + if item["model"] == "openai/gpt-6-astra" + ] + self.assertEqual(len(evaluations), 5) + self.assertEqual( + {item["benchmark"] for item in evaluations}, + { + "idavidrein/gpqa-diamond@1.0.0", + "cais/humanitys-last-exam@1.0.0", + "datacurve/deep-swe@1.1.0", + "arc-prize/arc-agi-1@1.0.0", + "arc-prize/arc-agi-2@1.0.0", + }, + ) + self.assertTrue( + all(item["reasoning_effort"] == "unspecified" for item in evaluations) + ) + self.assertTrue( + all( + item["subject"]["result_selection"] + == "maximum_across_supported_efforts" + for item in evaluations + ) + ) + self.assertTrue( + all( + item["evidence"]["provenance"] == "vendor_claimed" + for item in evaluations + ) + ) + + openai_inventory = next( + creator + for creator in manifest["inventory"]["physical"]["creators"] + if creator["publisher"] == "OpenAI" + ) + self.assertEqual( + openai_inventory["representative_models"], + ["openai/gpt-6-astra", "openai/gpt-5.6-sol", "openai/gpt-5.5"], + ) def test_baidu_creator_has_exact_evidence_buckets_and_real_bindings(self) -> None: manifest, resources, _ = catalog.load_and_validate() @@ -618,6 +753,7 @@ def test_frontier_context_limits_keep_exact_published_units(self) -> None: { "openai/gpt-5.4", "openai/gpt-5.5", + "openai/gpt-6-astra", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", diff --git a/tools/catalog/tests/test_generate_model_catalog_validation.py b/tools/catalog/tests/test_generate_model_catalog_validation.py index 18f671f1da..81d95b30ff 100644 --- a/tools/catalog/tests/test_generate_model_catalog_validation.py +++ b/tools/catalog/tests/test_generate_model_catalog_validation.py @@ -501,6 +501,174 @@ def test_provider_model_id_kind_policy_is_closed(self) -> None: providers, models, {"openai/chat-completions@1"} ) + def test_provider_model_api_restrictions_are_protocol_scoped(self) -> None: + protocols = { + "openai/chat-completions@1", + "openai/responses@1", + } + binding = { + "catalog": "example/model", + "relationship": "first_party", + "id": "native-model", + "protocols": sorted(protocols), + "restrictions": { + "tools_protocols": ["openai/responses@1"], + "long_context_pricing": { + "input_threshold_tokens": 272000, + "prompt_multiplier": 2.0, + "completion_multiplier": 1.5, + }, + "unsupported_request_fields": { + "openai/chat-completions@1": ["temperature", "logprobs"], + }, + "unsupported_include_values": ["message.output_text.logprobs"], + }, + } + providers = { + "example": { + "protocols": sorted(protocols), + "supported_operations": [ + f"{protocol}#create" for protocol in sorted(protocols) + ], + "models": [binding], + } + } + models = {"example/model": {"kind": "physical", "lifecycle": "active"}} + + catalog._validate_provider_bindings(providers, models, protocols) + + binding["restrictions"]["tools_protocols"] = ["anthropic/messages@1"] + with self.assertRaisesRegex( + catalog.CatalogBuildError, "tools_protocols references an unbound protocol" + ): + catalog._validate_provider_bindings(providers, models, protocols) + + binding["restrictions"]["tools_protocols"] = ["openai/responses@1"] + binding["restrictions"]["unsupported_request_fields"] = { + "openai/responses@1": ["temperature", "temperature"] + } + with self.assertRaisesRegex( + catalog.CatalogBuildError, "must be a non-empty unique list" + ): + catalog._validate_provider_bindings(providers, models, protocols) + + binding["restrictions"]["unsupported_request_fields"] = { + "openai/responses@1": ["reasoning.effort"] + } + with self.assertRaisesRegex( + catalog.CatalogBuildError, "must be a JSON field name" + ): + catalog._validate_provider_bindings(providers, models, protocols) + + binding["restrictions"]["unsupported_request_fields"] = { + "openai/responses@1": ["temperature"] + } + binding["restrictions"]["long_context_pricing"] = { + "input_threshold_tokens": 0, + "prompt_multiplier": 2.0, + } + with self.assertRaisesRegex( + catalog.CatalogBuildError, + "input_threshold_tokens must be a positive integer", + ): + catalog._validate_provider_bindings(providers, models, protocols) + + binding["restrictions"]["long_context_pricing"] = { + "input_threshold_tokens": 272000, + "prompt_multiplier": float("inf"), + } + with self.assertRaisesRegex( + catalog.CatalogBuildError, "prompt_multiplier must be positive" + ): + catalog._validate_provider_bindings(providers, models, protocols) + + binding["restrictions"]["long_context_pricing"] = { + "input_threshold_tokens": 272000, + "prompt_multiplier": 2.0, + } + binding["restrictions"]["unsupported_include_values"] = [ + "message.output_text.logprobs", + "message.output_text.logprobs", + ] + with self.assertRaisesRegex( + catalog.CatalogBuildError, "must be a non-empty unique list" + ): + catalog._validate_provider_bindings(providers, models, protocols) + + binding["restrictions"]["unsupported_include_values"] = [ + "message[0].output_text" + ] + with self.assertRaisesRegex( + catalog.CatalogBuildError, "must be a dotted JSON field path" + ): + catalog._validate_provider_bindings(providers, models, protocols) + + def test_provider_reasoning_efforts_can_be_narrowed_by_protocol(self) -> None: + protocols = { + "openai/chat-completions@1", + "openai/responses@1", + } + binding = { + "catalog": "example/model", + "relationship": "first_party", + "id": "native-model", + "protocols": sorted(protocols), + "reasoning_efforts": ["low", "max"], + "reasoning_efforts_by_protocol": { + "openai/chat-completions@1": ["low"], + }, + } + providers = { + "example": { + "protocols": sorted(protocols), + "supported_operations": [ + f"{protocol}#create" for protocol in sorted(protocols) + ], + "reasoning_transport": "top_level_effort", + "models": [binding], + } + } + models = { + "example/model": { + "kind": "physical", + "lifecycle": "active", + "reasoning_family": "example", + } + } + families = { + "example": { + "type": "reasoning_effort", + "parameter": "reasoning_effort", + "levels": ["low", "max"], + "modes": ["enabled"], + "default_mode": "enabled", + } + } + + catalog._validate_provider_bindings(providers, models, protocols, families) + + binding["reasoning_efforts_by_protocol"] = {"anthropic/messages@1": ["low"]} + with self.assertRaisesRegex( + catalog.CatalogBuildError, "references an unbound protocol" + ): + catalog._validate_provider_bindings(providers, models, protocols, families) + + binding["reasoning_efforts_by_protocol"] = { + "openai/chat-completions@1": ["medium"] + } + with self.assertRaisesRegex( + catalog.CatalogBuildError, + "must narrow the provider reasoning_efforts", + ): + catalog._validate_provider_bindings(providers, models, protocols, families) + + del binding["reasoning_efforts"] + with self.assertRaisesRegex( + catalog.CatalogBuildError, + "requires reasoning_efforts", + ): + catalog._validate_provider_bindings(providers, models, protocols, families) + def test_provider_binding_relationship_is_required_and_closed(self) -> None: binding = { "catalog": "example/model", @@ -586,9 +754,12 @@ def test_benchmark_scoped_subject_keys_are_rejected_elsewhere(self) -> None: "terminal_harness", "swe_harness", ): - with self.subTest(subject_key=subject_key), self.assertRaisesRegex( - catalog.CatalogBuildError, - rf"subject\.{subject_key} is only valid for benchmark families", + with ( + self.subTest(subject_key=subject_key), + self.assertRaisesRegex( + catalog.CatalogBuildError, + rf"subject\.{subject_key} is only valid for benchmark families", + ), ): catalog._validate_evaluations( [ diff --git a/website/docs/community/model-provider-day-0-support.md b/website/docs/community/model-provider-day-0-support.md index 103f00e255..c439689b91 100644 --- a/website/docs/community/model-provider-day-0-support.md +++ b/website/docs/community/model-provider-day-0-support.md @@ -170,6 +170,97 @@ metadata endpoints are rejected. Redirects and environment proxies are disabled, and resolved addresses are checked again by the dialer before credentials are sent. +## Worked example: GPT-6 Astra + +GPT-6 Astra is an existing-provider model addition, so its implementation stays +inside the shared catalog and the existing OpenAI protocol adapters. It is a +complete worked example of the contribution path above, without claiming that +the change landed on the model's launch day. Its source packet is the official +[model reference](https://developers.openai.com/api/docs/models/gpt-6-astra), +[latest-model guide](https://developers.openai.com/api/docs/guides/latest-model), +and [launch evaluation report](https://openai.com/index/gpt-6-astra/): + +| Contract | Source of truth | +| --- | --- | +| Identity, limits, modalities, capabilities, and presentation | `config/catalog/resources/models/single/openai.yaml` | +| OpenAI model ID, Chat/Responses availability, pricing, and API constraints | `config/catalog/resources/providers/openai.yaml` | +| Always-on `low`, `medium`, `high`, `xhigh`, and `max` reasoning | `config/catalog/resources/reasoning-families.yaml` | +| Exact vendor-published benchmark results | `config/catalog/resources/evaluations/single/openai.yaml` | +| Final Chat and Responses request shapes | `src/semantic-router/pkg/extproc/provider_request_catalog_contract_test.go` | +| Runnable aliases and mock-provider credentials | `e2e/profiles/response-api/values.yaml` | +| Default CI profile membership | `e2e/profiles/response-api/profile.go` | +| Black-box model-ID and reasoning projection | `e2e/testcases/model_catalog_astra.go` | + +The official model page does not name a default reasoning effort, so the +catalog does not invent one. It also does not expose a disabled mode: omitting +an effort lets the API choose its default, while `use_reasoning: false` is +rejected during configuration validation. Published launch scores are marked +`unspecified` because the source reports the maximum result across supported +efforts rather than attributing each score to one effort. + +The OpenAI binding records that tools require Responses; requests beyond +272,000 input tokens use the published long-context multipliers; and +`temperature`, `top_p`, and `top_logprobs` are unsupported (`logprobs` is also +unsupported for Chat Completions, and Responses cannot include +`message.output_text.logprobs`). It also narrows Chat Completions to `low`, +`medium`, `high`, and `xhigh`, because `max` is Responses-only. These are +discoverable catalog constraints, +not silent request rewriting: the Router projects protocol and reasoning +fields exactly, while OpenAI remains authoritative for rejecting unsupported +request fields. + +Use Responses for Astra tool calls. A minimal routed configuration needs no +handwritten Model Card or reasoning family: + +```yaml +version: v0.3 +providers: + models: + - name: astra + catalog: openai/gpt-6-astra + api_format: responses + backend_refs: + - name: primary + provider: openai + api_key_env: OPENAI_API_KEY + +routing: + decisions: + - name: astra_default + priority: 1 + rules: + operator: AND + conditions: [] + modelRefs: + - model: astra + use_reasoning: true + reasoning_effort: high +``` + +For a plain Chat Completions workload without tools, omit +`api_format: responses`; the same catalog binding emits top-level +`reasoning_effort` and rejects a Responses-only effort during startup. +Responses emits `reasoning.effort`. The black-box E2E also +proves that a Responses tool definition survives the Router-to-provider hop. +The generated Router, CLI, Dashboard, and website views all come from these +authored resources. + +Run the worked example by name: + +```bash +make e2e-test-specific \ + E2E_PROFILE=response-api \ + E2E_TESTS=model-catalog-astra +``` + +The profile uses a fixture credential and local mock provider; it never calls +the external model API. The test sends Chat with `xhigh`, Responses with +Responses-only `max`, and a Responses `high` tool request. It then reads the +mock provider's captured request and checks the provider-native model ID, +protocol-specific reasoning field, absence of the competing protocol shape, +and tool preservation. Keep this final-hop assertion when copying the example +for another model whose protocol or reasoning contract differs. + ## Built-in and custom user configuration A built-in card is selected with one optional `catalog` reference. The model @@ -272,12 +363,17 @@ versioned, while metric values must be finite. ```bash make model-catalog-generate make model-catalog-check -make agent-report ENV=cpu CHANGED_FILES="config/catalog/resources/models/organization.yaml" +make impact ENV=cpu BASE_REF=upstream/main +make check BASE_REF=upstream/main +make verify PROFILE=response-api ``` -Commit the authored resources and every generated projection together. Then -run the gates reported for the actual changed files. A complete Day-0 pull -request demonstrates: +Commit the authored resources and every generated projection together. Use the +E2E profile reported by `impact` instead of `response-api` when the model or +provider belongs to another profile; use `make verify DOMAIN=` for an +explicit integration domain. Run `make ci-full BASE_REF=upstream/main` when a +complete local PR baseline is required. A complete Day-0 pull request +demonstrates: - stable identities and valid references; - protocol and capability conformance for every support claim; diff --git a/website/docs/proposals/unified-model-catalog-and-evaluation-index.md b/website/docs/proposals/unified-model-catalog-and-evaluation-index.md index b7a4b8cc80..e01ada9ea1 100644 --- a/website/docs/proposals/unified-model-catalog-and-evaluation-index.md +++ b/website/docs/proposals/unified-model-catalog-and-evaluation-index.md @@ -48,15 +48,15 @@ and a recommended model reference is not a complete built-in model card. The old Dashboard therefore contained 40 provider presets, while the Router had seven hard-coded runtime types and the packaged catalog had no general-purpose physical-model registry. The implemented snapshot compiles 60 -serving providers, three protocol definitions, 83 physical Model Cards, five -virtual Model Cards, 168 provider-owned model mappings, 64 benchmark -definitions, and 1,360 exact evaluation records. The five default benchmark -components produce 1,155 explicit slots over 231 model/effort rows; 124 slots +serving providers, three protocol definitions, 84 physical Model Cards, five +virtual Model Cards, 169 provider-owned model mappings, 64 benchmark +definitions, and 1,365 exact evaluation records. The five default benchmark +components produce 1,325 explicit slots over 265 model/effort rows; 125 slots are currently measured and every other slot stays explicitly missing. Support tier, lifecycle, and conformance remain independent, so catalog inclusion is not flattened into a native-support or benchmark claim. -All 83 physical cards pass the hard admission rule: at least one exact +All 84 physical cards pass the hard admission rule: at least one exact model, reasoning-effort, and evidence-provenance bucket contains five distinct benchmark identities. That does not mean every runtime-selectable effort has five published results. @@ -74,8 +74,9 @@ first 20 individual models from any ranking or endpoint inventory. It focuses on roughly twenty mainstream creator companies (22 in this snapshot) and represents roughly their latest three generations or product lines. Closely related sizes or reasoning variants are included only when they are separately -selectable and materially useful to operators. GPT-6 Astra remains -intentionally absent for the separate Day-0 example change. +selectable and materially useful to operators. GPT-6 Astra is added separately +as the focused model-onboarding example rather than being folded into the +baseline change. | Model creator (`publisher`) | Recent generations and representative lines | Models | | --- | --- | ---: | @@ -163,8 +164,8 @@ and representative. model-and-reasoning-effort record. - It does not require every new model to have a composite score on release day; missing evidence remains explicitly unavailable. -- GPT-6 Astra is intentionally excluded. It is the separate representative - Day-0 contribution after this architecture and baseline-catalog change. +- GPT-6 Astra was intentionally excluded from the architecture baseline and is + added by the separate representative model-onboarding contribution. ## Design principles @@ -551,7 +552,11 @@ A provider-model mapping may further declare `reasoning_modes` or new user configuration. They prevent an API-specific surface from accepting a mode that is valid for a self-hosted runtime but invalid on that provider; the materializer rejects a configured decision before startup when any selected -backend cannot carry its requested control. +backend cannot carry its requested control. When one provider exposes the same +model through several protocols, `reasoning_efforts_by_protocol` can only +narrow that common set for a named bound protocol. For example, an effort that +is Responses-only is rejected during startup for a Chat binding instead of +being sent upstream as a known-invalid request. The public decision contract remains only `use_reasoning`, optional `reasoning_mode`, and optional `reasoning_effort`. At the final dispatch @@ -708,8 +713,8 @@ The initial population audit makes both coverage and gaps visible. The 64 benchmark definitions retain all exact measurements as source records, while public Hub surfaces remove every exact benchmark/profile/metric tuple measured on fewer than ten distinct models. The default five-component matrix -materializes 1,155 slots over 231 -model/effort rows. At this snapshot, 124 of those slots have an exact +materializes 1,325 slots over 265 +model/effort rows. At this snapshot, 125 of those slots have an exact measurement. Other rows remain explicitly `missing`, `failed`, `not_applicable`, or `withheld`; none is fabricated as zero. @@ -1113,8 +1118,9 @@ Protocol and provider adapters remain in narrow runtime packages. | 5 | Dashboard catalog API/Add Model migration and website Models page | Logos, forms, Model Hub, and benchmark comparisons consume generated data | | 6 | Day-0 contributor guide and repository gates | A compatible model/provider change has one authored source path | -The architecture PR also establishes the initial physical-model baseline. A -separate follow-up adds GPT-6 Astra as the focused, reviewable Day-0 example. +The architecture change establishes the initial physical-model baseline. The +focused GPT-6 Astra follow-up demonstrates the complete, reviewable +model-onboarding path. ## Acceptance criteria diff --git a/website/src/data/modelHubCatalogTypes.ts b/website/src/data/modelHubCatalogTypes.ts index 0170d4f03a..46c0f7b031 100644 --- a/website/src/data/modelHubCatalogTypes.ts +++ b/website/src/data/modelHubCatalogTypes.ts @@ -34,6 +34,9 @@ export interface CatalogModelBinding { id: string protocols: string[] reasoning_transport?: ReasoningTransport + reasoning_modes?: Array<'enabled' | 'disabled' | 'adaptive'> + reasoning_efforts?: string[] + reasoning_efforts_by_protocol?: Record lifecycle: string } diff --git a/website/static/model-catalog/catalog.json b/website/static/model-catalog/catalog.json index 5de49bb0b2..d69421a16e 100644 --- a/website/static/model-catalog/catalog.json +++ b/website/static/model-catalog/catalog.json @@ -7547,6 +7547,248 @@ "reasoning_effort": "unspecified", "status": "missing" }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "low", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "medium", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "high", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "xhigh", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "max", + "status": "missing" + }, + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evaluation": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "available", + "value": 0.96 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "model": "openai/gpt-6-astra", + "reasoning_effort": "unspecified", + "status": "missing" + }, { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", @@ -35665,6 +35907,122 @@ "variant": "gpt-oss-20b" } }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", + "metrics": { + "accuracy": 0.96 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "with-tools", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-humanitys-last-exam-tools@1.0.0", + "metrics": { + "accuracy": 0.572 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "tool_policy": "tools_enabled", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "datacurve/deep-swe@1.1.0", + "benchmark_profile": "published-agent", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-deep-swe-1.1@1.0.0", + "metrics": { + "resolved": 0.741 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "arc-prize/arc-agi-2@1.0.0", + "benchmark_profile": "semi-private", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-arc-agi-2@1.0.0", + "metrics": { + "accuracy": 0.95 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, + { + "benchmark": "arc-prize/arc-agi-1@1.0.0", + "benchmark_profile": "semi-private", + "evidence": { + "provenance": "vendor_claimed", + "redistributable": true, + "source": "https://openai.com/index/gpt-6-astra/", + "verification": "claimed" + }, + "id": "openai/gpt-6-astra-launch-arc-agi-1@1.0.0", + "metrics": { + "accuracy": 0.985 + }, + "model": "openai/gpt-6-astra", + "observed_at": "2026-09-07", + "reasoning_effort": "unspecified", + "status": "available", + "subject": { + "result_selection": "maximum_across_supported_efforts", + "source_kind": "official_launch_evaluation", + "variant": "GPT-6 Astra" + } + }, { "benchmark": "artificial-analysis/critpt@1.0.0", "benchmark_profile": "independent-standard", @@ -51597,9 +51955,360 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3.5-lightning", + "model": "nvidia/nemotron-3.5-lightning", + "provenance": [], + "reasoning_effort": "disabled", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [], + "reasoning_effort": "medium", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", + "metric": "accuracy", + "normalized": 0.879, + "status": "available", + "value": 0.879, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", + "metric": "accuracy", + "normalized": 0.261, + "status": "available", + "value": 0.261, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0", + "metric": "resolved", + "normalized": 0.695, + "status": "available", + "value": 0.695, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", + "metric": "resolved", + "normalized": 0.539, + "status": "available", + "value": 0.539, + "weight": 0.2 + } + ], + "coverage": 0.8, + "domains": { + "agentic_systems": 53.900000000000006, + "frontier_reasoning": 26.1, + "scientific_reasoning": 87.9, + "software_engineering": 69.49999999999999 + }, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [ + "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", + "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0" + ], + "reasoning_effort": "high", + "score": 59.349999999999994, + "status": "available" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-ultra", + "provenance": [], + "reasoning_effort": "enabled", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", + "provenance": [], + "reasoning_effort": "low", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", + "provenance": [], + "reasoning_effort": "high", + "score": null, + "status": "missing" + }, + { + "components": [ + { + "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "idavidrein/gpqa-diamond@1.0.0", + "benchmark_profile": "published-standard", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "cais/humanitys-last-exam@1.0.0", + "benchmark_profile": "no-tools", + "metric": "accuracy", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "swe-bench/verified@1.0.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + }, + { + "benchmark": "harbor/terminal-bench@2.1.0", + "benchmark_profile": "published-agent", + "metric": "resolved", + "normalized": null, + "status": "missing", + "value": null, + "weight": 0.2 + } + ], + "coverage": 0.0, + "index": "vllm-sr/intelligence@1.0.0", + "model": "nvidia/nemotron-3-super", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "enabled", "score": null, "status": "missing" }, @@ -51653,9 +52362,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "provenance": [], - "reasoning_effort": "medium", + "reasoning_effort": "enabled", "score": null, "status": "missing" }, @@ -51673,62 +52382,47 @@ { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", "metric": "accuracy", - "normalized": 0.879, - "status": "available", - "value": 0.879, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": 0.261, - "status": "available", - "value": 0.261, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0", "metric": "resolved", - "normalized": 0.695, - "status": "available", - "value": 0.695, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "harbor/terminal-bench@2.1.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", "metric": "resolved", - "normalized": 0.539, - "status": "available", - "value": 0.539, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 } ], - "coverage": 0.8, - "domains": { - "agentic_systems": 53.900000000000006, - "frontier_reasoning": 26.1, - "scientific_reasoning": 87.9, - "software_engineering": 69.49999999999999 - }, + "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", - "provenance": [ - "nvidia/nemotron-3-ultra-nvfp4-model-card-gpqa@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-humanitys-last-exam@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-terminal-bench@1.0.0", - "nvidia/nemotron-3-ultra-nvfp4-model-card-verified@1.0.0" - ], - "reasoning_effort": "high", - "score": 59.349999999999994, - "status": "available" + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "provenance": [], + "reasoning_effort": "disabled", + "score": null, + "status": "missing" }, { "components": [ @@ -51780,7 +52474,7 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-ultra", + "model": "nvidia/nemotron-cascade-2-30b-a3b", "provenance": [], "reasoning_effort": "enabled", "score": null, @@ -51836,9 +52530,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", + "model": "nvidia/nemotron-cascade-2-30b-a3b", "provenance": [], - "reasoning_effort": "low", + "reasoning_effort": "disabled", "score": null, "status": "missing" }, @@ -51847,37 +52541,41 @@ { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.798, + "status": "available", + "value": 0.798, "weight": 0.2 }, { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", + "evaluation": "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.761, + "status": "available", + "value": 0.761, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", + "evaluation": "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.177, + "status": "available", + "value": 0.177, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", + "evaluation": "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0", "metric": "resolved", - "normalized": null, - "status": "missing", - "value": null, + "normalized": 0.502, + "status": "available", + "value": 0.502, "weight": 0.2 }, { @@ -51890,13 +52588,24 @@ "weight": 0.2 } ], - "coverage": 0.0, + "coverage": 0.8, + "domains": { + "frontier_reasoning": 17.7, + "general_reasoning": 79.80000000000001, + "scientific_reasoning": 76.1, + "software_engineering": 50.2 + }, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", - "provenance": [], - "reasoning_effort": "high", - "score": null, - "status": "missing" + "model": "nvidia/nemotron-cascade-2-30b-a3b", + "provenance": [ + "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", + "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", + "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", + "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0" + ], + "reasoning_effort": "unspecified", + "score": 55.95, + "status": "available" }, { "components": [ @@ -51948,9 +52657,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-super", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "low", "score": null, "status": "missing" }, @@ -52004,9 +52713,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "medium", "score": null, "status": "missing" }, @@ -52060,9 +52769,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "high", "score": null, "status": "missing" }, @@ -52116,9 +52825,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "enabled", + "reasoning_effort": "xhigh", "score": null, "status": "missing" }, @@ -52172,9 +52881,9 @@ ], "coverage": 0.0, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [], - "reasoning_effort": "disabled", + "reasoning_effort": "max", "score": null, "status": "missing" }, @@ -52183,41 +52892,38 @@ { "benchmark": "tiger-ai-lab/mmlu-pro@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", "metric": "accuracy", - "normalized": 0.798, - "status": "available", - "value": 0.798, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "idavidrein/gpqa-diamond@1.0.0", "benchmark_profile": "published-standard", - "evaluation": "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", + "evaluation": "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0", "metric": "accuracy", - "normalized": 0.761, + "normalized": 0.96, "status": "available", - "value": 0.761, + "value": 0.96, "weight": 0.2 }, { "benchmark": "cais/humanitys-last-exam@1.0.0", "benchmark_profile": "no-tools", - "evaluation": "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", "metric": "accuracy", - "normalized": 0.177, - "status": "available", - "value": 0.177, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { "benchmark": "swe-bench/verified@1.0.0", "benchmark_profile": "published-agent", - "evaluation": "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0", "metric": "resolved", - "normalized": 0.502, - "status": "available", - "value": 0.502, + "normalized": null, + "status": "missing", + "value": null, "weight": 0.2 }, { @@ -52230,24 +52936,15 @@ "weight": 0.2 } ], - "coverage": 0.8, - "domains": { - "frontier_reasoning": 17.7, - "general_reasoning": 79.80000000000001, - "scientific_reasoning": 76.1, - "software_engineering": 50.2 - }, + "coverage": 0.2, "index": "vllm-sr/intelligence@1.0.0", - "model": "nvidia/nemotron-cascade-2-30b-a3b", + "model": "openai/gpt-6-astra", "provenance": [ - "nvidia/nemotron-cascade-2-model-card-gpqa-diamond@1.0.0", - "nvidia/nemotron-cascade-2-model-card-humanitys-last-exam@1.0.0", - "nvidia/nemotron-cascade-2-model-card-mmlu-pro@1.0.0", - "nvidia/nemotron-cascade-2-model-card-swe-bench-verified@1.0.0" + "openai/gpt-6-astra-launch-gpqa-diamond@1.0.0" ], "reasoning_effort": "unspecified", - "score": 55.95, - "status": "available" + "score": null, + "status": "missing" }, { "components": [ @@ -61923,6 +62620,59 @@ "verified_at": "2026-09-05" } }, + { + "capabilities": [ + "chat", + "reasoning", + "tools", + "structured_output", + "vision", + "long_context" + ], + "description": "OpenAI flagship model for complex reasoning, coding, computer use, research, and document creation.", + "display_name": "GPT-6 Astra", + "distribution": { + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "type": "proprietary_api" + }, + "family": "gpt-6", + "id": "openai/gpt-6-astra", + "kind": "physical", + "knowledge_cutoff": "2026-04-30", + "lifecycle": "active", + "limits": { + "context_window_size": 1050000, + "max_output_tokens": 128000 + }, + "modalities": { + "input": [ + "text", + "image" + ], + "output": [ + "text" + ] + }, + "presentation": { + "logo": "package:openai", + "monochrome": true, + "monogram": "O" + }, + "publisher": "OpenAI", + "reasoning_family": "gpt-6-astra", + "released_at": "2026-09-03", + "tags": [ + "frontier", + "proprietary", + "agentic" + ], + "verification": { + "authority": "OpenAI", + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "status": "claimed", + "verified_at": "2026-09-07" + } + }, { "capabilities": [ "chat", @@ -66133,6 +66883,74 @@ "display_name": "OpenAI", "id": "openai", "models": [ + { + "catalog": "openai/gpt-6-astra", + "id": "gpt-6-astra", + "lifecycle": "active", + "pricing": { + "cache_write_per_1m": 12.5, + "cached_input_per_1m": 1.0, + "completion_per_1m": 50.0, + "currency": "USD", + "prompt_per_1m": 10.0 + }, + "protocols": [ + "openai/chat-completions@1", + "openai/responses@1" + ], + "reasoning_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "reasoning_efforts_by_protocol": { + "openai/chat-completions@1": [ + "low", + "medium", + "high", + "xhigh" + ] + }, + "reasoning_modes": [ + "enabled" + ], + "relationship": "first_party", + "restrictions": { + "long_context_pricing": { + "cache_write_multiplier": 2.0, + "cached_input_multiplier": 2.0, + "completion_multiplier": 1.5, + "input_threshold_tokens": 272000, + "prompt_multiplier": 2.0 + }, + "tools_protocols": [ + "openai/responses@1" + ], + "unsupported_include_values": [ + "message.output_text.logprobs" + ], + "unsupported_request_fields": { + "openai/chat-completions@1": [ + "temperature", + "top_p", + "top_logprobs", + "logprobs" + ], + "openai/responses@1": [ + "temperature", + "top_p", + "top_logprobs" + ] + } + }, + "verification": { + "source": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "status": "claimed", + "verified_at": "2026-09-07" + } + }, { "catalog": "openai/gpt-5.4", "id": "gpt-5.4", @@ -68910,6 +69728,22 @@ "parameter": "reasoning_effort", "type": "reasoning_effort" }, + { + "default_mode": "enabled", + "id": "gpt-6-astra", + "levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "modes": [ + "enabled" + ], + "parameter": "reasoning_effort", + "type": "reasoning_effort" + }, { "default": "medium", "default_mode": "enabled",