diff --git a/.github/workflows/integration-test-k8s.yml b/.github/workflows/integration-test-k8s.yml index 1a2ce1965f..2297196845 100644 --- a/.github/workflows/integration-test-k8s.yml +++ b/.github/workflows/integration-test-k8s.yml @@ -87,7 +87,7 @@ jobs: set +e if [ "${{ matrix.profile }}" = "envoy-ai-gateway" ]; then # Temporarily skip the stress / pressure coverage until the suite is stable again. - ENVOY_AI_GATEWAY_CI_TESTS="chat-completions-request,apiserver-runtime-config-endpoints,domain-classify,semantic-cache,semantic-cache-polarity,pii-detection,pii-entity-offsets,pii-long-text,jailbreak-detection,security-long-text,decision-priority-selection,plugin-chain-execution,tool-selection,rule-condition-logic,decision-fallback-behavior,plugin-config-variations,event-routing,plugin-short-circuit-no-dispatch" + ENVOY_AI_GATEWAY_CI_TESTS="chat-completions-request,apiserver-runtime-config-endpoints,domain-classify,semantic-cache,semantic-cache-polarity,pii-detection,pii-entity-offsets,pii-long-text,jailbreak-detection,security-long-text,decision-priority-selection,plugin-chain-execution,tool-selection,rule-condition-logic,decision-fallback-behavior,plugin-config-variations,event-routing,plugin-short-circuit-no-dispatch,language-routing" make e2e-test E2E_PROFILE=${{ matrix.profile }} E2E_TESTS="${ENVOY_AI_GATEWAY_CI_TESTS}" E2E_VERBOSE=true E2E_KEEP_CLUSTER=false TEST_EXIT_CODE=$? set -e diff --git a/e2e/pkg/testmatrix/testcases.go b/e2e/pkg/testmatrix/testcases.go index 164d5a3679..2c224e0418 100644 --- a/e2e/pkg/testmatrix/testcases.go +++ b/e2e/pkg/testmatrix/testcases.go @@ -53,6 +53,8 @@ var BaselineRouterContract = []string{ "session-pricing-response-api", // Event signal rule matching and routing (issue #3178) "event-routing", + // Language signal rule matching and routing (issue #3178) + "language-routing", } // DashboardContract is the canonical E2E contract for the dashboard API surface. diff --git a/e2e/profiles/ai-gateway/values.yaml b/e2e/profiles/ai-gateway/values.yaml index 47bc2748f4..501cee35ae 100644 --- a/e2e/profiles/ai-gateway/values.yaml +++ b/e2e/profiles/ai-gateway/values.yaml @@ -581,6 +581,24 @@ config: enabled: true system_prompt: You are handling a critical event alert. Prioritize a fast, precise, and factual response. mode: replace + - name: spanish_language + description: Spanish-language requests requiring specialized handling + priority: 25 + rules: + operator: OR + conditions: + - type: language + name: es + modelRefs: + - model: base-model + lora_name: general-expert + use_reasoning: false + plugins: + - type: system_prompt + configuration: + enabled: true + system_prompt: You are responding to a Spanish-language query. Respond in Spanish. + mode: replace - name: other_decision description: General knowledge and miscellaneous topics priority: 1 @@ -669,6 +687,9 @@ config: - critical action_codes: - TXN_DECLINE + language: + - name: es + description: Spanish-language requests. domains: - name: business description: Business, corporate strategy, management, finance, marketing diff --git a/e2e/testcases/event_routing.go b/e2e/testcases/event_routing.go index 485ac35033..9270ce4f76 100644 --- a/e2e/testcases/event_routing.go +++ b/e2e/testcases/event_routing.go @@ -2,19 +2,14 @@ package testcases import ( "context" - "encoding/json" - "fmt" - "net/http" - "os" - "strings" - "time" - pkgtestcases "github.com/vllm-project/semantic-router/e2e/pkg/testcases" "k8s.io/client-go/kubernetes" + + pkgtestcases "github.com/vllm-project/semantic-router/e2e/pkg/testcases" ) -// targetEventDecision is the decision configured to route on the -// critical_payment_event rule (see e2e/profiles/ai-gateway/values.yaml). +// targetEventDecision is the decision configured to route on the event +// signal's critical_payment_event rule (see e2e/profiles/ai-gateway/values.yaml). const targetEventDecision = "critical_event" func init() { @@ -25,168 +20,12 @@ func init() { }) } -// EventRoutingCase represents a test case for event signal routing. -type EventRoutingCase struct { - Name string `json:"name"` - Description string `json:"description"` - Query string `json:"query"` - ExpectedDecision string `json:"expected_decision"` - ExpectedMatchedEvent string `json:"expected_matched_event"` - ShouldMatch bool `json:"should_match"` -} - -// EventRoutingResult tracks the result of a single event routing test. -type EventRoutingResult struct { - Name string - Query string - ExpectedDecision string - ActualDecision string - ExpectedMatchedEvent string - ActualMatchedEvent string - ShouldMatch bool - DecisionCorrect bool - MatchCorrect bool - Error string -} - func testEventRouting(ctx context.Context, client *kubernetes.Clientset, opts pkgtestcases.TestCaseOptions) error { - if opts.Verbose { - fmt.Println("[Test] Testing event signal routing") - } - - localPort, stopPortForward, err := setupServiceConnection(ctx, client, opts) - if err != nil { - return err - } - defer stopPortForward() - - testCases, err := loadEventRoutingCases("e2e/testcases/testdata/event_routing_cases.json") - if err != nil { - return fmt.Errorf("failed to load test cases: %w", err) - } - - var results []EventRoutingResult - totalTests := 0 - correctTests := 0 - - for _, testCase := range testCases { - totalTests++ - result := testSingleEventRouting(ctx, testCase, localPort, opts.Verbose) - results = append(results, result) - if result.DecisionCorrect && result.MatchCorrect { - correctTests++ - } - } - - accuracy := float64(correctTests) / float64(totalTests) * 100 - - if opts.SetDetails != nil { - opts.SetDetails(map[string]interface{}{ - "total_tests": totalTests, - "correct_tests": correctTests, - "accuracy_rate": fmt.Sprintf("%.2f%%", accuracy), - "failed_tests": totalTests - correctTests, - }) - } - - printEventRoutingResults(results, totalTests, correctTests, accuracy) - - if opts.Verbose { - fmt.Printf("[Test] Event routing test completed: %d/%d correct (%.2f%% accuracy)\n", - correctTests, totalTests, accuracy) - } - - if correctTests != totalTests { - return fmt.Errorf("event routing test failed: %d/%d correct", correctTests, totalTests) - } - - return nil -} - -func loadEventRoutingCases(filepath string) ([]EventRoutingCase, error) { - data, err := os.ReadFile(filepath) - if err != nil { - return nil, fmt.Errorf("failed to read test cases file: %w", err) - } - - var cases []EventRoutingCase - if err := json.Unmarshal(data, &cases); err != nil { - return nil, fmt.Errorf("failed to parse test cases: %w", err) - } - - return cases, nil -} - -func testSingleEventRouting(ctx context.Context, testCase EventRoutingCase, localPort string, verbose bool) EventRoutingResult { - result := EventRoutingResult{ - Name: testCase.Name, - Query: testCase.Query, - ExpectedDecision: testCase.ExpectedDecision, - ExpectedMatchedEvent: testCase.ExpectedMatchedEvent, - ShouldMatch: testCase.ShouldMatch, - } - - response, err := sendLocalChatCompletion(ctx, localPort, "MoM", testCase.Query, 30*time.Second) - if err != nil { - result.Error = err.Error() - return result - } - - if response.StatusCode != http.StatusOK { - result.Error = formatUnexpectedChatCompletionStatus(response) - logUnexpectedChatCompletionStatus(verbose, response, "test case: "+testCase.Name, - "Query: "+testCase.Query, - "Should match: "+fmt.Sprintf("%v", testCase.ShouldMatch)) - return result - } - - decision := response.Headers.Get("x-vsr-selected-decision") - result.ActualDecision = strings.TrimSuffix(decision, "_decision") - result.ActualMatchedEvent = response.Headers.Get("x-vsr-matched-event") - - if testCase.ShouldMatch { - result.DecisionCorrect = result.ActualDecision == testCase.ExpectedDecision - result.MatchCorrect = result.ActualMatchedEvent == testCase.ExpectedMatchedEvent - } else { - result.DecisionCorrect = result.ActualDecision != targetEventDecision - result.MatchCorrect = result.ActualMatchedEvent == "" - } - - if verbose && (!result.DecisionCorrect || !result.MatchCorrect) { - fmt.Printf("[Test] Test case failed: %s\n", testCase.Name) - if !result.DecisionCorrect { - fmt.Printf(" Decision mismatch: query='%s', expected=%s, actual=%s\n", - testCase.Query, testCase.ExpectedDecision, result.ActualDecision) - } - if !result.MatchCorrect { - fmt.Printf(" Matched-event mismatch: expected=%q, actual=%q\n", - testCase.ExpectedMatchedEvent, result.ActualMatchedEvent) - } - } - - return result -} - -func printEventRoutingResults(results []EventRoutingResult, totalTests, correctTests int, accuracy float64) { - separator := "================================================================================" - fmt.Println("\n" + separator) - fmt.Println("EVENT ROUTING TEST RESULTS") - fmt.Println(separator) - fmt.Printf("Total Tests: %d\n", totalTests) - fmt.Printf("Correct: %d (%.2f%%)\n", correctTests, accuracy) - fmt.Println(separator) - - for _, result := range results { - if result.Error != "" { - fmt.Printf(" - Test: %s\n Query: %s\n Error: %s\n", result.Name, result.Query, result.Error) - continue - } - if !result.DecisionCorrect || !result.MatchCorrect { - fmt.Printf(" - Test: %s\n Query: %s\n Expected decision: %s, actual: %s\n Expected matched event: %q, actual: %q\n", - result.Name, result.Query, result.ExpectedDecision, result.ActualDecision, - result.ExpectedMatchedEvent, result.ActualMatchedEvent) - } - } - - fmt.Println(separator + "\n") + return runSignalRoutingTest(ctx, client, opts, signalRoutingConfig{ + TestDataPath: "e2e/testcases/testdata/event_routing_cases.json", + MatchedHeader: "x-vsr-matched-event", + TargetDecision: targetEventDecision, + ResultsTitle: "EVENT ROUTING TEST RESULTS", + LogLabel: "Event", + }) } diff --git a/e2e/testcases/language_routing.go b/e2e/testcases/language_routing.go new file mode 100644 index 0000000000..e57b6977a1 --- /dev/null +++ b/e2e/testcases/language_routing.go @@ -0,0 +1,31 @@ +package testcases + +import ( + "context" + + "k8s.io/client-go/kubernetes" + + pkgtestcases "github.com/vllm-project/semantic-router/e2e/pkg/testcases" +) + +// targetLanguageDecision is the decision configured to route on the +// language signal matching Spanish (see e2e/profiles/ai-gateway/values.yaml). +const targetLanguageDecision = "spanish_language" + +func init() { + pkgtestcases.Register("language-routing", pkgtestcases.TestCase{ + Description: "Test language signal rule matching and routing", + Tags: []string{"kubernetes", "routing", "language"}, + Fn: testLanguageRouting, + }) +} + +func testLanguageRouting(ctx context.Context, client *kubernetes.Clientset, opts pkgtestcases.TestCaseOptions) error { + return runSignalRoutingTest(ctx, client, opts, signalRoutingConfig{ + TestDataPath: "e2e/testcases/testdata/language_routing_cases.json", + MatchedHeader: "x-vsr-matched-language", + TargetDecision: targetLanguageDecision, + ResultsTitle: "LANGUAGE ROUTING TEST RESULTS", + LogLabel: "Language", + }) +} diff --git a/e2e/testcases/signal_routing_contract_test.go b/e2e/testcases/signal_routing_contract_test.go index 8eee3017fc..6a60ca573b 100644 --- a/e2e/testcases/signal_routing_contract_test.go +++ b/e2e/testcases/signal_routing_contract_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/vllm-project/semantic-router/e2e/pkg/framework" - // Registers every profile (and, transitively, every testcase in this // package) so framework.NewProfileByName resolves the same way it does // inside the real e2e binary. @@ -19,6 +18,7 @@ var signalRoutingContracts = []struct { testCase string }{ {profile: "envoy-ai-gateway", testCase: "event-routing"}, + {profile: "envoy-ai-gateway", testCase: "language-routing"}, } func TestProfilesSelectSignalRoutingContracts(t *testing.T) { diff --git a/e2e/testcases/signal_routing_helpers.go b/e2e/testcases/signal_routing_helpers.go new file mode 100644 index 0000000000..3d9520f986 --- /dev/null +++ b/e2e/testcases/signal_routing_helpers.go @@ -0,0 +1,199 @@ +package testcases + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "time" + + "k8s.io/client-go/kubernetes" + + pkgtestcases "github.com/vllm-project/semantic-router/e2e/pkg/testcases" +) + +// SignalRoutingCase is a single request/expectation pair for a signal that is +// evaluated from one standalone request (e.g. event, language): the signal +// either matches a configured rule on that request or it doesn't. +type SignalRoutingCase struct { + Name string `json:"name"` + Description string `json:"description"` + Query string `json:"query"` + ExpectedDecision string `json:"expected_decision"` + ExpectedMatchedSignal string `json:"expected_matched_signal"` + ShouldMatch bool `json:"should_match"` +} + +// SignalRoutingResult tracks the result of a single SignalRoutingCase. +type SignalRoutingResult struct { + Name string + Query string + ExpectedDecision string + ActualDecision string + ExpectedMatchedSignal string + ActualMatchedSignal string + ShouldMatch bool + DecisionCorrect bool + MatchCorrect bool + Error string +} + +// signalRoutingConfig parameterizes runSignalRoutingTest for one signal. +type signalRoutingConfig struct { + // TestDataPath is the JSON file of SignalRoutingCase entries. + TestDataPath string + // MatchedHeader is the response header carrying the matched rule name + // (e.g. "x-vsr-matched-event", "x-vsr-matched-language"). + MatchedHeader string + // TargetDecision is the decision this signal's rule routes to; used to + // check negative cases without pinning the exact fallback decision. + TargetDecision string + // ResultsTitle is printed as the results-table header. + ResultsTitle string + // LogLabel names the signal in progress/summary log lines (e.g. "Event"). + LogLabel string +} + +func runSignalRoutingTest(ctx context.Context, client *kubernetes.Clientset, opts pkgtestcases.TestCaseOptions, cfg signalRoutingConfig) error { + if opts.Verbose { + fmt.Printf("[Test] Testing %s signal routing\n", strings.ToLower(cfg.LogLabel)) + } + + localPort, stopPortForward, err := setupServiceConnection(ctx, client, opts) + if err != nil { + return err + } + defer stopPortForward() + + testCases, err := loadSignalRoutingCases(cfg.TestDataPath) + if err != nil { + return fmt.Errorf("failed to load test cases: %w", err) + } + + var results []SignalRoutingResult + totalTests := 0 + correctTests := 0 + + for _, testCase := range testCases { + totalTests++ + result := testSingleSignalRouting(ctx, testCase, localPort, opts.Verbose, cfg) + results = append(results, result) + if result.DecisionCorrect && result.MatchCorrect { + correctTests++ + } + } + + accuracy := float64(correctTests) / float64(totalTests) * 100 + + if opts.SetDetails != nil { + opts.SetDetails(map[string]interface{}{ + "total_tests": totalTests, + "correct_tests": correctTests, + "accuracy_rate": fmt.Sprintf("%.2f%%", accuracy), + "failed_tests": totalTests - correctTests, + }) + } + + printSignalRoutingResults(cfg.ResultsTitle, results, totalTests, correctTests, accuracy) + + if opts.Verbose { + fmt.Printf("[Test] %s routing test completed: %d/%d correct (%.2f%% accuracy)\n", + cfg.LogLabel, correctTests, totalTests, accuracy) + } + + if correctTests != totalTests { + return fmt.Errorf("%s routing test failed: %d/%d correct", strings.ToLower(cfg.LogLabel), correctTests, totalTests) + } + + return nil +} + +func loadSignalRoutingCases(filepath string) ([]SignalRoutingCase, error) { + data, err := os.ReadFile(filepath) + if err != nil { + return nil, fmt.Errorf("failed to read test cases file: %w", err) + } + + var cases []SignalRoutingCase + if err := json.Unmarshal(data, &cases); err != nil { + return nil, fmt.Errorf("failed to parse test cases: %w", err) + } + + return cases, nil +} + +func testSingleSignalRouting(ctx context.Context, testCase SignalRoutingCase, localPort string, verbose bool, cfg signalRoutingConfig) SignalRoutingResult { + result := SignalRoutingResult{ + Name: testCase.Name, + Query: testCase.Query, + ExpectedDecision: testCase.ExpectedDecision, + ExpectedMatchedSignal: testCase.ExpectedMatchedSignal, + ShouldMatch: testCase.ShouldMatch, + } + + response, err := sendLocalChatCompletion(ctx, localPort, "MoM", testCase.Query, 30*time.Second) + if err != nil { + result.Error = err.Error() + return result + } + + if response.StatusCode != http.StatusOK { + result.Error = formatUnexpectedChatCompletionStatus(response) + logUnexpectedChatCompletionStatus(verbose, response, "test case: "+testCase.Name, + "Query: "+testCase.Query, + "Should match: "+fmt.Sprintf("%v", testCase.ShouldMatch)) + return result + } + + decision := response.Headers.Get("x-vsr-selected-decision") + result.ActualDecision = strings.TrimSuffix(decision, "_decision") + result.ActualMatchedSignal = response.Headers.Get(cfg.MatchedHeader) + + if testCase.ShouldMatch { + result.DecisionCorrect = result.ActualDecision == testCase.ExpectedDecision + result.MatchCorrect = result.ActualMatchedSignal == testCase.ExpectedMatchedSignal + } else { + result.DecisionCorrect = result.ActualDecision != cfg.TargetDecision + result.MatchCorrect = result.ActualMatchedSignal == "" + } + + if verbose && (!result.DecisionCorrect || !result.MatchCorrect) { + fmt.Printf("[Test] Test case failed: %s\n", testCase.Name) + if !result.DecisionCorrect { + fmt.Printf(" Decision mismatch: query='%s', expected=%s, actual=%s\n", + testCase.Query, testCase.ExpectedDecision, result.ActualDecision) + } + if !result.MatchCorrect { + fmt.Printf(" Matched-signal mismatch: expected=%q, actual=%q\n", + testCase.ExpectedMatchedSignal, result.ActualMatchedSignal) + } + } + + return result +} + +func printSignalRoutingResults(title string, results []SignalRoutingResult, totalTests, correctTests int, accuracy float64) { + separator := "================================================================================" + fmt.Println("\n" + separator) + fmt.Println(title) + fmt.Println(separator) + fmt.Printf("Total Tests: %d\n", totalTests) + fmt.Printf("Correct: %d (%.2f%%)\n", correctTests, accuracy) + fmt.Println(separator) + + for _, result := range results { + if result.Error != "" { + fmt.Printf(" - Test: %s\n Query: %s\n Error: %s\n", result.Name, result.Query, result.Error) + continue + } + if !result.DecisionCorrect || !result.MatchCorrect { + fmt.Printf(" - Test: %s\n Query: %s\n Expected decision: %s, actual: %s\n Expected matched signal: %q, actual: %q\n", + result.Name, result.Query, result.ExpectedDecision, result.ActualDecision, + result.ExpectedMatchedSignal, result.ActualMatchedSignal) + } + } + + fmt.Println(separator + "\n") +} diff --git a/e2e/testcases/testdata/event_routing_cases.json b/e2e/testcases/testdata/event_routing_cases.json index 1084c93b3d..8a16637297 100644 --- a/e2e/testcases/testdata/event_routing_cases.json +++ b/e2e/testcases/testdata/event_routing_cases.json @@ -4,7 +4,7 @@ "description": "critical_payment_event matches on the payment_failed event type alone", "query": "Our system just received a payment_failed webhook, please advise", "expected_decision": "critical_event", - "expected_matched_event": "critical_payment_event", + "expected_matched_signal": "critical_payment_event", "should_match": true }, { @@ -12,7 +12,7 @@ "description": "critical_payment_event matches on the critical severity and TXN_DECLINE action code", "query": "We have a critical TXN_DECLINE that needs review right away", "expected_decision": "critical_event", - "expected_matched_event": "critical_payment_event", + "expected_matched_signal": "critical_payment_event", "should_match": true }, { @@ -20,7 +20,7 @@ "description": "A generic query with no event_type, severity, action_code, or temporal marker should not trigger critical_payment_event", "query": "What's a good recipe for pasta tonight?", "expected_decision": "", - "expected_matched_event": "", + "expected_matched_signal": "", "should_match": false } ] diff --git a/e2e/testcases/testdata/language_routing_cases.json b/e2e/testcases/testdata/language_routing_cases.json new file mode 100644 index 0000000000..9e810dd349 --- /dev/null +++ b/e2e/testcases/testdata/language_routing_cases.json @@ -0,0 +1,26 @@ +[ + { + "name": "Spanish query - vacation planning", + "description": "spanish_language matches a clearly Spanish-language request (English translation: \"Could you help me plan a beach vacation this summer?\")", + "query": "¿Podrías ayudarme a planificar unas vacaciones en la playa este verano?", + "expected_decision": "spanish_language", + "expected_matched_signal": "es", + "should_match": true + }, + { + "name": "Spanish query - houseplant care", + "description": "spanish_language matches a second, unrelated Spanish-language request (English translation: \"I need advice on how to better care for my houseplants.\")", + "query": "Necesito consejos sobre cómo cuidar mejor mis plantas de interior.", + "expected_decision": "spanish_language", + "expected_matched_signal": "es", + "should_match": true + }, + { + "name": "English query - not Spanish", + "description": "An English request should not trigger spanish_language. expected_decision is left empty because the test only checks that the actual decision is not spanish_language, not what it actually is", + "query": "What's the best way to organize a bookshelf by color?", + "expected_decision": "", + "expected_matched_signal": "", + "should_match": false + } +]