vllm-project · duchangyong · Jul 28, 2026 · Aug 1, 2026 · Aug 1, 2026 · Aug 1, 2026
@@ -18,6 +18,7 @@ Inside canonical `config.yaml`:
 - `routing.projections.partitions` is the canonical runtime home for exclusive domain or embedding partitions; DSL authoring uses `PROJECTION partition`
 - `routing.projections.scores` and `routing.projections.mappings` let maintained configs turn learned and heuristic signals into named routing bands that decisions can reference with `type: projection`
 - `routing.decisions[].candidateIterations` carries bounded DSL `FOR ... IN` metadata for candidate-model authoring; it is declarative selection policy input, not a general scripting runtime
+- `routing.decisions[].modelRefs[].quality_score` optionally overrides the model-card quality score for that candidate in the owning decision; `multi_factor` falls back to the model-card value only when this field is omitted
 - `routing.decisions[].emits[]` carries typed side-effect directives from DSL `EMIT` blocks; the current supported kind is `retention`, where `drop: true` skips response-side semantic-cache writes and the remaining fields stay structured/auditable for follow-up runtime consumers such as turn-aware cache TTL, current-model affinity, prefix/KV-cache warmth, and session transition telemetry
 - request-shape detectors such as `routing.signals.structure` stay in the signal layer as typed named facts; numeric thresholds live inside the detector config instead of turning decisions into a free-form expression language
 - `routing.signals.embeddings[].query_modality` declares which modality of incoming request payload the embedding rule's query is computed from. Defaults to `"text"`; `"image"` and `"audio"` require `global.model_catalog.embeddings.semantic.embedding_config.model_type=multimodal` so the query and candidate embeddings land in the same shared space. See `website/docs/tutorials/signal/learned/embedding.md` for the worked multimodal example.
@@ -46,7 +47,7 @@ Inside canonical `config.yaml`:
 - `not/`: exclusion examples
 - `composite/`: nested AND/OR/NOT cases
 
-Decision fragments may reference `modelRefs[].lora_name`, but those adapter names must be declared in the base config's `routing.modelCards[].loras`.
+Decision fragments may reference `modelRefs[].lora_name`, but those adapter names must be declared in the base config's `routing.modelCards[].loras`. A `modelRefs[].quality_score` value is decision-scoped, must be between `0` and `1`, and preserves an explicit `0`.
 Candidate iteration fragments must stay bounded to `decision.candidates` or an explicit model list and feed existing decision outputs such as `MODEL <iterator>`.
 
 `config/algorithm/` is organized by routing policy:

@@ -571,6 +571,7 @@ routing:
             - model: qwen3-8b
               lora_name: business-adapter
               weight: 1.0
+              quality_score: 0.83
               use_reasoning: false
               reasoning_description: Candidate iteration reference model for DSL authoring coverage.
               reasoning_effort: low
@@ -1135,7 +1136,9 @@ routing:
             name: other
       modelRefs:
         - model: qwen3-8b
+          quality_score: 0.72
         - model: qwen3-32b
+          quality_score: 0.94
       algorithm:
         type: multi_factor
         multi_factor:

@@ -0,0 +1,15 @@
+import { readFileSync } from 'node:fs'
+import { describe, expect, it } from 'vitest'
+
+describe('decision model quality editor', () => {
+  it('preserves and edits decision-scoped quality scores', () => {
+    const source = readFileSync(
+      new URL('./ConfigPageDecisionsSection.tsx', import.meta.url),
+      'utf8',
+    )
+
+    expect(source).toContain('Decision quality score')
+    expect(source).toContain("| 'quality_score'")
+    expect(source).toContain('modelRef.quality_score = modelRefValue.quality_score')
+  })
+})
@@ -114,6 +114,7 @@ export default function ConfigPageDecisionsSection({
       ref.reasoning_effort ? `Effort: ${ref.reasoning_effort}` : null,
       ref.lora_name ? `LoRA: ${ref.lora_name}` : null,
       typeof ref.weight === 'number' ? `Weight: ${ref.weight}` : null,
+      typeof ref.quality_score === 'number' ? `Quality: ${ref.quality_score}` : null,
     ].filter((value): value is string => Boolean(value))
 
     const details = [
@@ -355,6 +356,7 @@ export default function ConfigPageDecisionsSection({
               reasoning_effort: ref.reasoning_effort || '',
               lora_name: ref.lora_name || '',
               weight: typeof ref.weight === 'number' ? ref.weight : undefined,
+              quality_score: typeof ref.quality_score === 'number' ? ref.quality_score : undefined,
             })),
             plugins: (decision.plugins || []).map((plugin) => ({
               type: plugin.type,
@@ -471,7 +473,8 @@ export default function ConfigPageDecisionsSection({
           | 'reasoning_description'
           | 'reasoning_effort'
           | 'lora_name'
-          | 'weight',
+          | 'weight'
+          | 'quality_score',
         val: string | boolean | number | undefined,
       ) => {
         const next = rows.map((item, idx) => (idx === index ? { ...item, [key]: val } : item))
@@ -592,6 +595,27 @@ export default function ConfigPageDecisionsSection({
                     className={decisionStyles.editorInput}
                   />
                 </label>
+                <label className={decisionStyles.editorControlLabel}>
+                  <span className={decisionStyles.editorControlLabelText}>
+                    Decision quality score
+                  </span>
+                  <input
+                    type="number"
+                    value={typeof ref?.quality_score === 'number' ? ref.quality_score : ''}
+                    onChange={(e) =>
+                      updateItem(
+                        idx,
+                        'quality_score',
+                        e.target.value === '' ? undefined : Number(e.target.value),
+                      )
+                    }
+                    placeholder="Optional 0–1 override"
+                    step="0.01"
+                    min="0"
+                    max="1"
+                    className={decisionStyles.editorInput}
+                  />
+                </label>
               </div>
 
               <label className={decisionStyles.editorControlLabel}>
@@ -732,6 +756,15 @@ export default function ConfigPageDecisionsSection({
         if (typeof modelRefValue?.weight === 'number' && Number.isFinite(modelRefValue.weight)) {
           modelRef.weight = modelRefValue.weight
         }
+        if (
+          typeof modelRefValue?.quality_score === 'number' &&
+          Number.isFinite(modelRefValue.quality_score)
+        ) {
+          if (modelRefValue.quality_score < 0 || modelRefValue.quality_score > 1) {
+            throw new Error(`Model reference #${idx + 1} quality score must be between 0 and 1.`)
+          }
+          modelRef.quality_score = modelRefValue.quality_score
+        }
         return modelRef
       })
 

@@ -206,6 +206,7 @@ export interface DecisionModelRef {
   reasoning_effort?: string
   lora_name?: string
   weight?: number
+  quality_score?: number
 }
 
 export interface DecisionPluginConfig {

@@ -33,6 +33,8 @@ spec:
         - '{"name": "humanities-expert"}'
         - '{"name": "law-expert"}'
         - '{"name": "general-expert"}'
+        - '{"name": "premium-model"}'
+        - '{"name": "economy-model"}'
         env:
         - name: POD_NAME
           valueFrom:

@@ -62,6 +62,27 @@ config:
               latency: 0
               cost: 0
               load: 0
+      - name: quality_override_policy_decision
+        description: Quality-only policy with decision-scoped score overrides
+        priority: 250
+        rules:
+          operator: AND
+          conditions:
+            - type: keyword
+              name: quality_override_policy
+        modelRefs:
+          - model: premium-model
+            quality_score: 0.1
+          - model: economy-model
+            quality_score: 0.9
+        algorithm:
+          type: multi_factor
+          multi_factor:
+            weights:
+              quality: 1
+              latency: 0
+              cost: 0
+              load: 0
       - name: cost_policy_decision
         description: Cost-only multi-factor policy
         priority: 250
@@ -178,6 +199,11 @@ config:
           keywords:
             - decision scoped quality policy
           case_sensitive: false
+        - name: quality_override_policy
+          operator: OR
+          keywords:
+            - decision scoped quality override policy
+          case_sensitive: false
         - name: cost_policy
           operator: OR
           keywords:

@@ -15,7 +15,7 @@ import (
 
 func init() {
 	pkgtestcases.Register("decision-scoped-multi-factor", pkgtestcases.TestCase{
-		Description: "Verify each matched decision uses its own multi-factor policy",
+		Description: "Verify decision-scoped multi-factor quality fallback and overrides",
 		Tags:        []string{"routing", "selection", "multi-factor", "regression"},
 		Fn:          testDecisionScopedMultiFactor,
 	})
@@ -42,6 +42,11 @@ func testDecisionScopedMultiFactor(
 			wantDecision: "quality_policy_decision",
 			wantModel:    "premium-model",
 		},
+		{
+			query:        "Apply the decision scoped quality override policy",
+			wantDecision: "quality_override_policy_decision",
+			wantModel:    "economy-model",
+		},
 		{
 			query:        "Apply the decision scoped cost policy",
 			wantDecision: "cost_policy_decision",

@@ -236,6 +236,15 @@ func validateCanonicalDecisions(decisions []Decision, modelsByName map[string]Ro
 		if err := validateCanonicalDecisionModelRefs(decision, modelsByName, modelCards); err != nil {
 			return err
 		}
+		for i, iteration := range decision.CandidateIterations {
+			if strings.TrimSpace(iteration.Source) != "models" {
+				continue
+			}
+			context := fmt.Sprintf("routing.decisions[%s].candidateIterations[%d]", decision.Name, i)
+			if err := validateDecisionCandidateIterationModels(iteration.Models, context); err != nil {
+				return err
+			}
+		}
 	}
 
 	return nil
@@ -246,6 +255,9 @@ func validateCanonicalDecisionModelRefs(decision Decision, modelsByName map[stri
 		if modelRef.Model == "" {
 			continue
 		}
+		if err := validateModelRefQualityScore(modelRef, fmt.Sprintf("routing.decisions[%s].modelRefs[%s]", decision.Name, modelRef.Model)); err != nil {
+			return err
+		}
 		// Reject modelRefs that point at a model the config does not define.
 		// Previously this was only checked when a lora_name was also set, so a
 		// plain modelRef to an unknown model slipped through and only failed at

@@ -139,9 +139,12 @@ type ModelReasoningControl struct {
 }
 
 type ModelRef struct {
-	Model                 string  `yaml:"model"`
-	LoRAName              string  `yaml:"lora_name,omitempty"`
-	Weight                float64 `yaml:"weight,omitempty"`
+	Model    string  `yaml:"model"`
+	LoRAName string  `yaml:"lora_name,omitempty"`
+	Weight   float64 `yaml:"weight,omitempty"`
+	// QualityScore optionally overrides the model-card quality score for this
+	// candidate in the owning decision. A pointer preserves explicit zero.
+	QualityScore          *float64 `yaml:"quality_score,omitempty" json:"quality_score,omitempty"`
 	ModelReasoningControl `yaml:",inline"`
 }
 

@@ -3,6 +3,8 @@ package config
 import (
 	"strings"
 	"testing"
+
+	"gopkg.in/yaml.v3"
 )
 
 // buildDecisionRefConfig wraps a decisions block in an otherwise-valid v0.3
@@ -91,6 +93,104 @@ func TestDecisionModelRefKnownModelAccepted(t *testing.T) {
 	}
 }
 
+func TestDecisionModelRefQualityScoreRoundTripsExplicitZero(t *testing.T) {
+	cfg, err := ParseYAMLBytes(buildDecisionRefConfig(`    - name: d1
+      priority: 1
+      rules: {operator: AND, conditions: []}
+      modelRefs:
+        - model: m1
+          quality_score: 0
+          use_reasoning: false
+`))
+	if err != nil {
+		t.Fatalf("parse decision quality score: %v", err)
+	}
+	quality := cfg.Decisions[0].ModelRefs[0].QualityScore
+	if quality == nil || *quality != 0 {
+		t.Fatalf("expected explicit zero quality score, got %#v", quality)
+	}
+
+	exported, err := yaml.Marshal(CanonicalConfigFromRouterConfig(cfg))
+	if err != nil {
+		t.Fatalf("marshal canonical config: %v", err)
+	}
+	if !strings.Contains(string(exported), "quality_score: 0") {
+		t.Fatalf("exported canonical config dropped explicit zero quality score:\n%s", exported)
+	}
+}
+
+func TestDecisionModelRefQualityScoreRejectsOutOfRange(t *testing.T) {
+	_, err := ParseYAMLBytes(buildDecisionRefConfig(`    - name: d1
+      priority: 1
+      rules: {operator: AND, conditions: []}
+      modelRefs:
+        - model: m1
+          quality_score: 1.1
+          use_reasoning: false
+`))
+	if err == nil {
+		t.Fatal("expected out-of-range decision quality score to be rejected")
+	}
+	if !strings.Contains(err.Error(), "quality_score") {
+		t.Fatalf("error should name quality_score, got: %v", err)
+	}
+}
+
+func TestDecisionModelRefQualityScoreRejectsNaN(t *testing.T) {
+	_, err := ParseYAMLBytes(buildDecisionRefConfig(`    - name: d1
+      priority: 1
+      rules: {operator: AND, conditions: []}
+      modelRefs:
+        - model: m1
+          quality_score: .nan
+          use_reasoning: false
+`))
+	if err == nil {
+		t.Fatal("expected NaN decision quality score to be rejected")
+	}
+	if !strings.Contains(err.Error(), "quality_score") {
+		t.Fatalf("error should name quality_score, got: %v", err)
+	}
+}
+
+func TestDecisionCandidateIterationModelQualityScoreRejectsInvalidValues(t *testing.T) {
+	tests := []struct {
+		name  string
+		score string
+	}{
+		{name: "negative", score: "-0.1"},
+		{name: "above one", score: "2"},
+		{name: "NaN", score: ".nan"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			payload := buildDecisionRefConfig(`    - name: d1
+      priority: 1
+      rules: {operator: AND, conditions: []}
+      candidateIterations:
+        - variable: candidate
+          source: models
+          models:
+            - model: m1
+              quality_score: ` + tt.score + `
+              use_reasoning: false
+`)
+			var canonical CanonicalConfig
+			if err := yaml.Unmarshal(payload, &canonical); err != nil {
+				t.Fatalf("unmarshal canonical config: %v", err)
+			}
+			err := validateCanonicalContract(&canonical)
+			if err == nil {
+				t.Fatalf("expected candidate iteration quality score %s to be rejected", tt.score)
+			}
+			if !strings.Contains(err.Error(), "candidateIterations[0]") || !strings.Contains(err.Error(), "quality_score") {
+				t.Fatalf("error should identify candidate iteration quality_score, got: %v", err)
+			}
+		})
+	}
+}
+
 // G2: two decisions sharing the same name are ambiguous and must be rejected.
 func TestDuplicateDecisionNamesRejected(t *testing.T) {
 	cfg := buildDecisionRefConfig(`    - name: dup

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"math"
 	"strings"
 
 	"github.com/vllm-project/semantic-router/src/semantic-router/pkg/observability/logging"
@@ -46,6 +47,9 @@ func validateDecisionModelRefs(cfg *RouterConfig, decision Decision) error {
 		if modelRef.UseReasoning == nil {
 			return fmt.Errorf("decision '%s', model '%s': missing required field 'use_reasoning'", decision.Name, modelRef.Model)
 		}
+		if err := validateModelRefQualityScore(modelRef, fmt.Sprintf("decision '%s', modelRefs[%d]", decision.Name, i)); err != nil {
+			return err
+		}
 		if modelRef.LoRAName == "" {
 			continue
 		}
@@ -56,6 +60,16 @@ func validateDecisionModelRefs(cfg *RouterConfig, decision Decision) error {
 	return nil
 }
 
+func validateModelRefQualityScore(modelRef ModelRef, context string) error {
+	if modelRef.QualityScore == nil {
+		return nil
+	}
+	if math.IsNaN(*modelRef.QualityScore) || *modelRef.QualityScore < 0 || *modelRef.QualityScore > 1 {
+		return fmt.Errorf("%s.quality_score must be between 0 and 1", context)
+	}
+	return nil
+}
+
 func validateDecisionWorkflowModelRefs(decision Decision) error {
 	if decision.Algorithm == nil || decision.Algorithm.Workflows == nil {
 		return nil
@@ -147,6 +161,9 @@ func validateDecisionCandidateIterationModels(models []ModelRef, context string)
 		if strings.TrimSpace(modelRef.Model) == "" {
 			return fmt.Errorf("%s, models[%d]: model name cannot be empty", context, j)
 		}
+		if err := validateModelRefQualityScore(modelRef, fmt.Sprintf("%s, models[%d]", context, j)); err != nil {
+			return err
+		}
 	}
 	return nil
 }