diff --git a/.golangci.yml b/.golangci.yml index 93e43b91a3..bcdf22a741 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -89,6 +89,8 @@ linters: alias: fwkgaie - pkg: github.com/llm-d/llm-d-router/test/framework/epp alias: fwkepp + - pkg: github.com/llm-d/llm-d-router/test/framework/epp/harness + alias: eppharness revive: # see https://github.com/mgechev/revive#available-rules for all options rules: - name: blank-imports diff --git a/Makefile b/Makefile index f40294c565..1b11ab65c2 100644 --- a/Makefile +++ b/Makefile @@ -167,7 +167,10 @@ sidecar_TEST_PACKAGES = ./pkg/sidecar/... # block (epp+sidecar only); CI's baseline cache unions all coverage/*.out, so # framework enters the CI baseline once this is on main. The PR run reports # coverage/framework.out as a new component. -framework_TEST_PACKAGES = ./test/framework/... +# The EPP harness is excluded: it is envtest-driven and exercised by +# make test-integration-hermetic, so unit coverage would report 0% for code +# that is in fact covered. +framework_TEST_PACKAGES = $$(go list ./test/framework/... | grep -vE '/test/framework/epp/harness($$|/)' | tr '\n' ' ') # Internal variables for generic targets epp_IMAGE = $(EPP_IMAGE) diff --git a/test/integration/epp/harness.go b/test/framework/epp/harness/harness.go similarity index 88% rename from test/integration/epp/harness.go rename to test/framework/epp/harness/harness.go index 6a3d42e4a4..adb736524d 100644 --- a/test/integration/epp/harness.go +++ b/test/framework/epp/harness/harness.go @@ -14,7 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package epp +// Package harness boots a shared envtest environment and per-test EPP servers for +// integration suites. +// +// It sits at the top of the framework layering and may import anything, including +// pkg/epp/server and cmd/epp/runner; only integration and e2e suites import it. +package harness import ( "context" @@ -24,6 +29,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" "time" @@ -585,3 +591,60 @@ func (h *TestHarness) ExpectMetrics(expected map[string]string) { } } } + +// --- Data Structures & Metrics Helpers --- + +type PodState struct { + index int + queueSize int + kvCacheUsage float64 + activeModels []string +} + +// P constructs a Pod State: Index, Queue, KV%, Models... +// Usage: P(0, 5, 0.2, "model-a") +func P(idx int, q int, kv float64, models ...string) PodState { + return PodState{index: idx, queueSize: q, kvCacheUsage: kv, activeModels: models} +} + +type label struct{ name, value string } + +func labelsToString(labels []label) string { + parts := make([]string, len(labels)) + for i, l := range labels { + parts[i] = fmt.Sprintf("%s=%q", l.name, l.value) + } + return strings.Join(parts, ",") +} + +// MetricReqTotal renders the expected inference_objective_request_total exposition text. +func MetricReqTotal(model, target string, priority int) string { + return fmt.Sprintf(` + # HELP inference_objective_request_total [ALPHA] [Deprecated: Use llm_d_epp_request_total] Counter of inference objective requests broken out for each model and target model. + # TYPE inference_objective_request_total counter + inference_objective_request_total{%s} 1 + `, labelsToString([]label{{"model_name", model}, {"priority", strconv.Itoa(priority)}, {"target_model_name", target}})) +} + +// MetricReadyPods renders the expected inference_pool_ready_pods exposition text. +func MetricReadyPods(count int) string { + return fmt.Sprintf(` + # HELP inference_pool_ready_pods [ALPHA] [Deprecated: Use llm_d_epp_ready_endpoints] The number of ready pods in the inference server pool. + # TYPE inference_pool_ready_pods gauge + inference_pool_ready_pods{%s} %d + `, labelsToString([]label{{"name", TestPoolName}}), count) +} + +// CleanMetric removes indentation from multiline metric strings and ensures a trailing newline exists, which is +// required by the Prometheus text parser. +func CleanMetric(s string) string { + lines := strings.Split(s, "\n") + var cleaned []string + for _, l := range lines { + trimmed := strings.TrimSpace(l) + if trimmed != "" { + cleaned = append(cleaned, trimmed) + } + } + return strings.Join(cleaned, "\n") + "\n" +} diff --git a/test/integration/epp/testdata/datalayer-config.yaml b/test/framework/epp/harness/testdata/datalayer-config.yaml similarity index 100% rename from test/integration/epp/testdata/datalayer-config.yaml rename to test/framework/epp/harness/testdata/datalayer-config.yaml diff --git a/test/integration/epp/common_tests.go b/test/integration/epp/common_tests.go index cf09cb0685..30df53f424 100644 --- a/test/integration/epp/common_tests.go +++ b/test/integration/epp/common_tests.go @@ -18,9 +18,6 @@ package epp import ( "encoding/json" - "fmt" - "strconv" - "strings" envoyCorev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" extProcPb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" @@ -28,6 +25,7 @@ import ( reqcommon "github.com/llm-d/llm-d-router/pkg/common/request" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) // Model name constants shared across test suites. @@ -52,7 +50,7 @@ func buildEnvoyHeaders(headers map[string]string) []*envoyCorev3.HeaderValue { // This simulates the "Subset Load Balancing" flow where EPP picks a specific pod IP. func ReqSubset(prompt, model, target string, subsets ...string) []*extProcPb.ProcessingRequest { // Uses the shared low-level generator which handles the metadata construction - return fwkepp.GenerateStreamedRequestSet(Logger(), prompt, model, target, subsets) + return fwkepp.GenerateStreamedRequestSet(eppharness.Logger(), prompt, model, target, subsets) } // ReqResponseOnly creates a sequence simulating only the response phase from Envoy. @@ -276,7 +274,7 @@ func ExpectGRPCStreamResp(chunks ...string) []*extProcPb.ProcessingResponse { type testCase struct { name string requests []*extProcPb.ProcessingRequest - pods []PodState + pods []eppharness.PodState configText string wantResponses []*extProcPb.ProcessingResponse wantMetrics map[string]string @@ -294,30 +292,30 @@ func commonTestCases(prio func(int) int) []testCase { return []testCase{ { name: "select lower queue and kv cache", - requests: fwkepp.ReqLLM(Logger(), "test1", modelMyModel, modelMyModelTarget), - pods: []PodState{ - P(0, 3, 0.2), - P(1, 0, 0.1), // Winner (Low Queue, Low KV) - P(2, 10, 0.2), + requests: fwkepp.ReqLLM(eppharness.Logger(), "test1", modelMyModel, modelMyModelTarget), + pods: []eppharness.PodState{ + eppharness.P(0, 3, 0.2), + eppharness.P(1, 0, 0.1), // Winner (Low Queue, Low KV) + eppharness.P(2, 10, 0.2), }, wantResponses: ExpectRouteTo("192.168.1.2:8000", modelMyModelTarget, "test1"), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal(modelMyModel, modelMyModelTarget, prio(2))), - "inference_pool_ready_pods": CleanMetric(MetricReadyPods(3)), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal(modelMyModel, modelMyModelTarget, prio(2))), + "inference_pool_ready_pods": eppharness.CleanMetric(eppharness.MetricReadyPods(3)), }, wantSpans: []string{"gateway.request", "gateway.request_orchestration"}, }, { name: "select active lora, low queue", - requests: fwkepp.ReqLLM(Logger(), "test2", modelSQLLora, modelSQLLoraTarget), - pods: []PodState{ - P(0, 0, 0.2, "foo", "bar"), - P(1, 0, 0.1, "foo", modelSQLLoraTarget), // Winner (Has LoRA) - P(2, 10, 0.2, "foo", "bar"), + requests: fwkepp.ReqLLM(eppharness.Logger(), "test2", modelSQLLora, modelSQLLoraTarget), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo", "bar"), + eppharness.P(1, 0, 0.1, "foo", modelSQLLoraTarget), // Winner (Has LoRA) + eppharness.P(2, 10, 0.2, "foo", "bar"), }, wantResponses: ExpectRouteTo("192.168.1.2:8000", modelSQLLoraTarget, "test2"), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal(modelSQLLora, modelSQLLoraTarget, prio(2))), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal(modelSQLLora, modelSQLLoraTarget, prio(2))), }, }, { @@ -338,60 +336,3 @@ func commonTestCases(prio func(int) int) []testCase { }, } } - -// --- Data Structures & Metrics Helpers --- - -type PodState struct { - index int - queueSize int - kvCacheUsage float64 - activeModels []string -} - -// P constructs a Pod State: Index, Queue, KV%, Models... -// Usage: P(0, 5, 0.2, "model-a") -func P(idx int, q int, kv float64, models ...string) PodState { - return PodState{index: idx, queueSize: q, kvCacheUsage: kv, activeModels: models} -} - -type label struct{ name, value string } - -func labelsToString(labels []label) string { - parts := make([]string, len(labels)) - for i, l := range labels { - parts[i] = fmt.Sprintf("%s=%q", l.name, l.value) - } - return strings.Join(parts, ",") -} - -// MetricReqTotal renders the expected inference_objective_request_total exposition text. -func MetricReqTotal(model, target string, priority int) string { - return fmt.Sprintf(` - # HELP inference_objective_request_total [ALPHA] [Deprecated: Use llm_d_epp_request_total] Counter of inference objective requests broken out for each model and target model. - # TYPE inference_objective_request_total counter - inference_objective_request_total{%s} 1 - `, labelsToString([]label{{"model_name", model}, {"priority", strconv.Itoa(priority)}, {"target_model_name", target}})) -} - -// MetricReadyPods renders the expected inference_pool_ready_pods exposition text. -func MetricReadyPods(count int) string { - return fmt.Sprintf(` - # HELP inference_pool_ready_pods [ALPHA] [Deprecated: Use llm_d_epp_ready_endpoints] The number of ready pods in the inference server pool. - # TYPE inference_pool_ready_pods gauge - inference_pool_ready_pods{%s} %d - `, labelsToString([]label{{"name", TestPoolName}}), count) -} - -// CleanMetric removes indentation from multiline metric strings and ensures a trailing newline exists, which is -// required by the Prometheus text parser. -func CleanMetric(s string) string { - lines := strings.Split(s, "\n") - var cleaned []string - for _, l := range lines { - trimmed := strings.TrimSpace(l) - if trimmed != "" { - cleaned = append(cleaned, trimmed) - } - } - return strings.Join(cleaned, "\n") + "\n" -} diff --git a/test/integration/epp/datalayer_integration_test.go b/test/integration/epp/datalayer_integration_test.go index d0f6aa37e9..bc0ee137c6 100644 --- a/test/integration/epp/datalayer_integration_test.go +++ b/test/integration/epp/datalayer_integration_test.go @@ -26,6 +26,7 @@ import ( "google.golang.org/protobuf/testing/protocmp" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) // TestFullDuplexStreamed_DataLayer runs integration tests through the datalayer metrics pipeline. @@ -39,7 +40,7 @@ func TestFullDuplexStreamed_DataLayer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - h := NewTestHarness(ctx, t, WithStandardMode()) + h := eppharness.NewTestHarness(ctx, t, eppharness.WithStandardMode()) h.WithBaseResources().WithPods(tc.pods).WaitForSync(len(tc.pods), modelMyModel) if len(tc.pods) > 0 { h.WaitForReadyPodsMetric(len(tc.pods)) diff --git a/test/integration/epp/dynamic_attributes_test.go b/test/integration/epp/dynamic_attributes_test.go index 0c4172ba43..3e9c51a54c 100644 --- a/test/integration/epp/dynamic_attributes_test.go +++ b/test/integration/epp/dynamic_attributes_test.go @@ -27,6 +27,7 @@ import ( reqcommon "github.com/llm-d/llm-d-router/pkg/common/request" "github.com/llm-d/llm-d-router/pkg/epp/metadata" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) func TestDynamicAttributes_Concurrency(t *testing.T) { @@ -73,12 +74,12 @@ flowControl: ` ctx := t.Context() - h := NewTestHarness(ctx, t, WithConfigText(configText), WithStandardMode()) + h := eppharness.NewTestHarness(ctx, t, eppharness.WithConfigText(configText), eppharness.WithStandardMode()) h = h.WithBaseResources() // Add one pod. We need it to support modelMyModelTarget. - pods := []PodState{ - P(0, 0, 0.0, modelMyModelTarget), + pods := []eppharness.PodState{ + eppharness.P(0, 0, 0.0, modelMyModelTarget), } h.WithPods(pods).WaitForSync(len(pods), modelMyModel) h.WaitForReadyPodsMetric(len(pods)) diff --git a/test/integration/epp/e2e_config_smoke_test.go b/test/integration/epp/e2e_config_smoke_test.go index 33aec8a68b..2a6547ebbd 100644 --- a/test/integration/epp/e2e_config_smoke_test.go +++ b/test/integration/epp/e2e_config_smoke_test.go @@ -10,7 +10,11 @@ You may obtain a copy of the License at package epp -import "testing" +import ( + "testing" + + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" +) // e2eConfigsForSmoke mirrors the YAML strings in test/e2e/configs_test.go so // the hermetic harness validates that the e2e fixtures actually parse + @@ -171,9 +175,9 @@ schedulingProfiles: func TestE2EConfigs_StartupParseSmoke(t *testing.T) { for name, yaml := range e2eConfigsForSmoke { t.Run(name, func(t *testing.T) { - // NewTestHarness fails the test if config-parse / plugin-init + // eppharness.NewTestHarness fails the test if config-parse / plugin-init // errors — exactly what we want to catch before pushing. - _ = NewTestHarness(t.Context(), t, WithConfigText(yaml)) + _ = eppharness.NewTestHarness(t.Context(), t, eppharness.WithConfigText(yaml)) }) } } diff --git a/test/integration/epp/grpc_test.go b/test/integration/epp/grpc_test.go index 1d4d64da78..818a23f07b 100644 --- a/test/integration/epp/grpc_test.go +++ b/test/integration/epp/grpc_test.go @@ -30,6 +30,7 @@ import ( reqcommon "github.com/llm-d/llm-d-router/pkg/common/request" pb "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requesthandling/parsers/vllmgrpc/api/gen" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) const ( @@ -63,7 +64,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { tests := []struct { name string requests []*extProcPb.ProcessingRequest - pods []PodState + pods []eppharness.PodState wantResponses []*extProcPb.ProcessingResponse wantMetrics map[string]string // requiresCRDs indicates that this test case relies on specific Gateway API CRD features (like @@ -73,57 +74,57 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { // --- Standard Routing Logic --- { name: "select lower queue and kv cache", - requests: fwkepp.ReqGRPCLLM(Logger(), "test1", inferenceObjectiveWithPriority4, fwkepp.GenerateGRPCMethodName), - pods: []PodState{ - P(0, 3, 0.2), - P(1, 0, 0.1), // Winner (Low Queue, Low KV) - P(2, 10, 0.2), + requests: fwkepp.ReqGRPCLLM(eppharness.Logger(), "test1", inferenceObjectiveWithPriority4, fwkepp.GenerateGRPCMethodName), + pods: []eppharness.PodState{ + eppharness.P(0, 3, 0.2), + eppharness.P(1, 0, 0.1), // Winner (Low Queue, Low KV) + eppharness.P(2, 10, 0.2), }, wantResponses: ExpectGRPCRouteTo("192.168.1.2:8000", "test1", fwkepp.GenerateGRPCMethodName), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal("", "", 4)), - "inference_pool_ready_pods": CleanMetric(MetricReadyPods(3)), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal("", "", 4)), + "inference_pool_ready_pods": eppharness.CleanMetric(eppharness.MetricReadyPods(3)), }, }, { name: "select lower queue and kv cache for embedRequest", - requests: fwkepp.ReqGRPCLLM(Logger(), "test1", inferenceObjectiveWithPriority4, fwkepp.EmbedGRPCMethodName), - pods: []PodState{ - P(0, 3, 0.2), - P(1, 0, 0.1), // Winner (Low Queue, Low KV) - P(2, 10, 0.2), + requests: fwkepp.ReqGRPCLLM(eppharness.Logger(), "test1", inferenceObjectiveWithPriority4, fwkepp.EmbedGRPCMethodName), + pods: []eppharness.PodState{ + eppharness.P(0, 3, 0.2), + eppharness.P(1, 0, 0.1), // Winner (Low Queue, Low KV) + eppharness.P(2, 10, 0.2), }, wantResponses: ExpectGRPCRouteTo("192.168.1.2:8000", "test1", fwkepp.EmbedGRPCMethodName), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal("", "", 4)), - "inference_pool_ready_pods": CleanMetric(MetricReadyPods(3)), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal("", "", 4)), + "inference_pool_ready_pods": eppharness.CleanMetric(eppharness.MetricReadyPods(3)), }, }, { name: "select lower queue with streaming request", - requests: fwkepp.ReqGRPCLLMWithStream(Logger(), "test-stream", inferenceObjectiveWithPriority4, fwkepp.GenerateGRPCMethodName), - pods: []PodState{ - P(0, 3, 0.2), - P(1, 0, 0.1), // Winner - P(2, 10, 0.2), + requests: fwkepp.ReqGRPCLLMWithStream(eppharness.Logger(), "test-stream", inferenceObjectiveWithPriority4, fwkepp.GenerateGRPCMethodName), + pods: []eppharness.PodState{ + eppharness.P(0, 3, 0.2), + eppharness.P(1, 0, 0.1), // Winner + eppharness.P(2, 10, 0.2), }, wantResponses: ExpectGRPCRouteToWithStream("192.168.1.2:8000", "test-stream", fwkepp.GenerateGRPCMethodName), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal("", "", 4)), - "inference_pool_ready_pods": CleanMetric(MetricReadyPods(3)), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal("", "", 4)), + "inference_pool_ready_pods": eppharness.CleanMetric(eppharness.MetricReadyPods(3)), }, }, { name: "do not shed requests by default", - requests: fwkepp.ReqGRPCLLM(Logger(), "test2", "", fwkepp.GenerateGRPCMethodName), - pods: []PodState{ - P(0, 6, 0.2, "foo", "bar"), // Winner (Lowest saturated) - P(1, 0, 0.85, "foo"), - P(2, 10, 0.9, "foo"), + requests: fwkepp.ReqGRPCLLM(eppharness.Logger(), "test2", "", fwkepp.GenerateGRPCMethodName), + pods: []eppharness.PodState{ + eppharness.P(0, 6, 0.2, "foo", "bar"), // Winner (Lowest saturated) + eppharness.P(1, 0, 0.85, "foo"), + eppharness.P(2, 10, 0.9, "foo"), }, wantResponses: ExpectGRPCRouteTo("192.168.1.1:8000", "test2", fwkepp.GenerateGRPCMethodName), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal("", "", 0)), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal("", "", 0)), }, }, @@ -157,13 +158,13 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { string(gRPCPayload[len(gRPCPayload)/2:]), ) }(), - pods: []PodState{ - P(0, 4, 0.2, "foo", "bar"), - P(1, 4, 0.85, "foo"), + pods: []eppharness.PodState{ + eppharness.P(0, 4, 0.2, "foo", "bar"), + eppharness.P(1, 4, 0.85, "foo"), }, wantResponses: ExpectGRPCRouteTo("192.168.1.1:8000", "test3", fwkepp.GenerateGRPCMethodName), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal("", "", 0)), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal("", "", 0)), }, }, { @@ -178,31 +179,31 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { { name: "subsetting: select best from subset", // Only pods in the subset list are eligible. - requests: fwkepp.GenerateStreamedGRPCRequestSet(Logger(), "test2", "", + requests: fwkepp.GenerateStreamedGRPCRequestSet(eppharness.Logger(), "test2", "", []string{"192.168.1.1:8000", "192.168.1.2:8000", "192.168.1.3:8000"}, fwkepp.GenerateGRPCMethodName), - pods: []PodState{ - P(0, 0, 0.2, "foo"), - P(1, 0, 0.1, "foo", modelSQLLoraTarget), // Winner (Low Queue + Matches Subset) - P(2, 10, 0.2, "foo"), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo"), + eppharness.P(1, 0, 0.1, "foo", modelSQLLoraTarget), // Winner (Low Queue + Matches Subset) + eppharness.P(2, 10, 0.2, "foo"), }, wantResponses: ExpectGRPCRouteTo("192.168.1.2:8000", "test2", fwkepp.GenerateGRPCMethodName), }, { name: "subsetting: partial match", - requests: fwkepp.GenerateStreamedGRPCRequestSet(Logger(), "test2", "", []string{"192.168.1.3:8000"}, fwkepp.GenerateGRPCMethodName), - pods: []PodState{ - P(0, 0, 0.2, "foo"), - P(1, 0, 0.1, "foo", modelSQLLoraTarget), - P(2, 10, 0.2, "foo"), // Winner (Matches Subset, despite load) + requests: fwkepp.GenerateStreamedGRPCRequestSet(eppharness.Logger(), "test2", "", []string{"192.168.1.3:8000"}, fwkepp.GenerateGRPCMethodName), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo"), + eppharness.P(1, 0, 0.1, "foo", modelSQLLoraTarget), + eppharness.P(2, 10, 0.2, "foo"), // Winner (Matches Subset, despite load) }, wantResponses: ExpectGRPCRouteTo("192.168.1.3:8000", "test2", fwkepp.GenerateGRPCMethodName), }, { name: "subsetting: no pods match", - requests: fwkepp.GenerateStreamedGRPCRequestSet(Logger(), "test2", "", []string{"192.168.1.99:8000"}, fwkepp.GenerateGRPCMethodName), - pods: []PodState{ - P(0, 0, 0.2, "foo"), - P(1, 0, 0.1, "foo", modelSQLLoraTarget), + requests: fwkepp.GenerateStreamedGRPCRequestSet(eppharness.Logger(), "test2", "", []string{"192.168.1.99:8000"}, fwkepp.GenerateGRPCMethodName), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo"), + eppharness.P(1, 0, 0.1, "foo", modelSQLLoraTarget), }, wantResponses: ExpectReject(envoyTypePb.StatusCode_ServiceUnavailable, "inference error: ServiceUnavailable - failed to find endpoint candidates for serving the request"), @@ -227,7 +228,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { gRPCPayload[len(gRPCPayload)/2:], ) }(), - pods: []PodState{P(0, 4, 0.2)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2)}, wantResponses: func() []*extProcPb.ProcessingResponse { resp := &pb.GenerateResponse{ Response: &pb.GenerateResponse_Chunk{ @@ -247,7 +248,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { map[string]string{"content-type": "application/grpc"}, []byte("no healthy upstream"), ), - pods: []PodState{P(0, 4, 0.2)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2)}, wantResponses: ExpectBufferResp("no healthy upstream", "application/grpc"), }, { @@ -269,7 +270,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { gRPCPayload[len(gRPCPayload)/2:], ) }(), - pods: []PodState{P(0, 4, 0.2)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2)}, wantResponses: func() []*extProcPb.ProcessingResponse { resp := &pb.GenerateResponse{ Response: &pb.GenerateResponse_Complete{ @@ -284,7 +285,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { }(), // Labels are empty because we skipped the Request phase. wantMetrics: map[string]string{ - "inference_objective_input_tokens": CleanMetric(` + "inference_objective_input_tokens": eppharness.CleanMetric(` # HELP inference_objective_input_tokens [ALPHA] [Deprecated: Use llm_d_epp_request_input_tokens] Inference objective input token count distribution for requests in each model. # TYPE inference_objective_input_tokens histogram inference_objective_input_tokens_bucket{model_name="",target_model_name="",le="1"} 0 @@ -315,7 +316,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { { name: "response streaming with token usage", requests: func() []*extProcPb.ProcessingRequest { - reqs := fwkepp.ReqGRPCLLMWithStream(Logger(), "test-stream", inferenceObjectiveWithPriority4, fwkepp.GenerateGRPCMethodName) + reqs := fwkepp.ReqGRPCLLMWithStream(eppharness.Logger(), "test-stream", inferenceObjectiveWithPriority4, fwkepp.GenerateGRPCMethodName) resp1 := &pb.GenerateResponse{ Response: &pb.GenerateResponse_Chunk{ @@ -367,7 +368,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { return append(reqs, respHeaders, respBody1, respBody2, respTrailers) }(), - pods: []PodState{P(0, 4, 0.2)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2)}, wantResponses: func() []*extProcPb.ProcessingResponse { reqs := ExpectGRPCRouteToWithStream("192.168.1.1:8000", "test-stream", fwkepp.GenerateGRPCMethodName) @@ -408,7 +409,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { return append(reqs, respRespHeaders, respChunk1, respChunk2) }(), wantMetrics: map[string]string{ - "inference_objective_input_tokens": CleanMetric(` + "inference_objective_input_tokens": eppharness.CleanMetric(` # HELP inference_objective_input_tokens [ALPHA] [Deprecated: Use llm_d_epp_request_input_tokens] Inference objective input token count distribution for requests in each model. # TYPE inference_objective_input_tokens histogram inference_objective_input_tokens_bucket{model_name="",target_model_name="",le="1"} 0 @@ -434,7 +435,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { inference_objective_input_tokens_sum{model_name="",target_model_name=""} 7 inference_objective_input_tokens_count{model_name="",target_model_name=""} 1 `), - "inference_objective_output_tokens": CleanMetric(` + "inference_objective_output_tokens": eppharness.CleanMetric(` # HELP inference_objective_output_tokens [ALPHA] [Deprecated: Use llm_d_epp_request_output_tokens] Inference objective output token count distribution for requests in each model. # TYPE inference_objective_output_tokens histogram inference_objective_output_tokens_bucket{model_name="",target_model_name="",le="1"} 0 @@ -461,7 +462,7 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() - h := NewTestHarness(ctx, t, WithStandardMode(), WithConfigText(testConfigWithVllmGRPCParser)).WithBaseResources() + h := eppharness.NewTestHarness(ctx, t, eppharness.WithStandardMode(), eppharness.WithConfigText(testConfigWithVllmGRPCParser)).WithBaseResources() h.WithPods(tc.pods).WaitForSync(len(tc.pods), modelMyModel) if len(tc.pods) > 0 { diff --git a/test/integration/epp/hermetic_test.go b/test/integration/epp/hermetic_test.go index 284a3b7ed4..7d1d872e6f 100644 --- a/test/integration/epp/hermetic_test.go +++ b/test/integration/epp/hermetic_test.go @@ -33,6 +33,7 @@ import ( reqcommon "github.com/llm-d/llm-d-router/pkg/common/request" "github.com/llm-d/llm-d-router/pkg/epp/metadata" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) const ( @@ -44,24 +45,24 @@ const ( inferenceObjectiveWithPriority4 = "inference-objective-with-priority-4" ) -func TestMain(m *testing.M) { os.Exit(Run(m)) } +func TestMain(m *testing.M) { os.Exit(eppharness.Run(m)) } func TestFullDuplexStreamed_KubeInferenceObjectiveRequest(t *testing.T) { // executionModes defines the permutations of EPP deployment modes to test. executionModes := []struct { name string - mode RunMode - strategy StandaloneStrategy + mode eppharness.RunMode + strategy eppharness.StandaloneStrategy }{ - {name: "Standard", mode: ModeStandard}, - {name: "Standalone-NoCRD", mode: ModeStandalone, strategy: StrategyNoCRD}, - {name: "Standalone-WithCRD", mode: ModeStandalone, strategy: StrategyWithCRD}, + {name: "Standard", mode: eppharness.ModeStandard}, + {name: "Standalone-NoCRD", mode: eppharness.ModeStandalone, strategy: eppharness.StrategyNoCRD}, + {name: "Standalone-WithCRD", mode: eppharness.ModeStandalone, strategy: eppharness.StrategyWithCRD}, } for _, executionMode := range executionModes { t.Run(executionMode.name, func(t *testing.T) { // Determine if we are running in the standalone mode without CRDs - isNoCRD := executionMode.mode == ModeStandalone && executionMode.strategy == StrategyNoCRD + isNoCRD := executionMode.mode == eppharness.ModeStandalone && executionMode.strategy == eppharness.StrategyNoCRD // Helper function to override priority to 0 when in NoCRD mode prio := func(p int) int { @@ -74,15 +75,15 @@ func TestFullDuplexStreamed_KubeInferenceObjectiveRequest(t *testing.T) { hermeticTests := []testCase{ { name: "select lora despite higher kv cache (affinity)", - requests: fwkepp.ReqLLM(Logger(), "test3", modelSQLLora, modelSQLLoraTarget), - pods: []PodState{ - P(0, 10, 0.2, "foo", "bar"), - P(1, 10, 0.4, "foo", modelSQLLoraTarget), // Winner (Affinity overrides KV) - P(2, 10, 0.3, "foo"), + requests: fwkepp.ReqLLM(eppharness.Logger(), "test3", modelSQLLora, modelSQLLoraTarget), + pods: []eppharness.PodState{ + eppharness.P(0, 10, 0.2, "foo", "bar"), + eppharness.P(1, 10, 0.4, "foo", modelSQLLoraTarget), // Winner (Affinity overrides KV) + eppharness.P(2, 10, 0.3, "foo"), }, wantResponses: ExpectRouteTo("192.168.1.2:8000", modelSQLLoraTarget, "test3"), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal(modelSQLLora, modelSQLLoraTarget, prio(2))), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal(modelSQLLora, modelSQLLoraTarget, prio(2))), }, }, { @@ -115,28 +116,28 @@ dataLayer: }, "passthrough-parser", ), - pods: []PodState{ - P(0, 3, 0.2), - P(1, 0, 0.1), // Winner - P(2, 10, 0.2), + pods: []eppharness.PodState{ + eppharness.P(0, 3, 0.2), + eppharness.P(1, 0, 0.1), // Winner + eppharness.P(2, 10, 0.2), }, wantResponses: ExpectPassthroughRouteTo("192.168.1.2:8000", []byte("passthrough-parser")), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal("", "", prio(2))), - "inference_pool_ready_pods": CleanMetric(MetricReadyPods(3)), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal("", "", prio(2))), + "inference_pool_ready_pods": eppharness.CleanMetric(eppharness.MetricReadyPods(3)), }, }, { name: "do not shed requests by default", - requests: fwkepp.ReqLLM(Logger(), "test4", modelSQLLora, modelSQLLoraTarget), - pods: []PodState{ - P(0, 6, 0.2, "foo", "bar", modelSQLLoraTarget), // Winner (Lowest saturated) - P(1, 0, 0.85, "foo"), - P(2, 10, 0.9, "foo"), + requests: fwkepp.ReqLLM(eppharness.Logger(), "test4", modelSQLLora, modelSQLLoraTarget), + pods: []eppharness.PodState{ + eppharness.P(0, 6, 0.2, "foo", "bar", modelSQLLoraTarget), // Winner (Lowest saturated) + eppharness.P(1, 0, 0.85, "foo"), + eppharness.P(2, 10, 0.9, "foo"), }, wantResponses: ExpectRouteTo("192.168.1.1:8000", modelSQLLoraTarget, "test4"), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal(modelSQLLora, modelSQLLoraTarget, prio(2))), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal(modelSQLLora, modelSQLLoraTarget, prio(2))), }, }, @@ -147,8 +148,8 @@ dataLayer: map[string]string{"hi": "mom"}, "no healthy upstream", ), - pods: []PodState{ - P(0, 0, 0.2, "foo", "bar"), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo", "bar"), }, wantResponses: ExpectReject( envoyTypePb.StatusCode_BadRequest, @@ -167,13 +168,13 @@ dataLayer: `{"max_tokens":100,"model":"sql-lo`, `ra-sheddable","prompt":"test6","temperature":0}`, ), - pods: []PodState{ - P(0, 4, 0.2, "foo", "bar", modelSheddableTarget), - P(1, 4, 0.85, "foo", modelSheddableTarget), + pods: []eppharness.PodState{ + eppharness.P(0, 4, 0.2, "foo", "bar", modelSheddableTarget), + eppharness.P(1, 4, 0.85, "foo", modelSheddableTarget), }, wantResponses: ExpectRouteTo("192.168.1.1:8000", modelSheddableTarget, "test6"), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal(modelSheddable, modelSheddableTarget, prio(0))), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal(modelSheddable, modelSheddableTarget, prio(0))), }, }, { @@ -199,29 +200,29 @@ dataLayer: // Only pods in the subset list are eligible. requests: ReqSubset("test2", modelSQLLora, modelSQLLoraTarget, "192.168.1.1:8000", "192.168.1.2:8000", "192.168.1.3:8000"), - pods: []PodState{ - P(0, 0, 0.2, "foo"), - P(1, 0, 0.1, "foo", modelSQLLoraTarget), // Winner (Low Queue + Matches Subset) - P(2, 10, 0.2, "foo"), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo"), + eppharness.P(1, 0, 0.1, "foo", modelSQLLoraTarget), // Winner (Low Queue + Matches Subset) + eppharness.P(2, 10, 0.2, "foo"), }, wantResponses: ExpectRouteTo("192.168.1.2:8000", modelSQLLoraTarget, "test2"), }, { name: "subsetting: partial match", requests: ReqSubset("test2", modelSQLLora, modelSQLLoraTarget, "192.168.1.3:8000"), - pods: []PodState{ - P(0, 0, 0.2, "foo"), - P(1, 0, 0.1, "foo", modelSQLLoraTarget), - P(2, 10, 0.2, "foo"), // Winner (Matches Subset, despite load) + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo"), + eppharness.P(1, 0, 0.1, "foo", modelSQLLoraTarget), + eppharness.P(2, 10, 0.2, "foo"), // Winner (Matches Subset, despite load) }, wantResponses: ExpectRouteTo("192.168.1.3:8000", modelSQLLoraTarget, "test2"), }, { name: "subsetting: no pods match", requests: ReqSubset("test2", modelSQLLora, modelSQLLoraTarget, "192.168.1.99:8000"), - pods: []PodState{ - P(0, 0, 0.2, "foo"), - P(1, 0, 0.1, "foo", modelSQLLoraTarget), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.2, "foo"), + eppharness.P(1, 0, 0.1, "foo", modelSQLLoraTarget), }, wantResponses: ExpectReject(envoyTypePb.StatusCode_ServiceUnavailable, "inference error: ServiceUnavailable - failed to find endpoint candidates for serving the request"), @@ -240,23 +241,23 @@ dataLayer: `{"max_tokens":100,"model":"direct-`, `model","prompt":"test6","temperature":0}`, ), - pods: []PodState{ - P(0, 4, 0.2, "foo", "bar", modelSheddableTarget), + pods: []eppharness.PodState{ + eppharness.P(0, 4, 0.2, "foo", "bar", modelSheddableTarget), }, wantResponses: ExpectRouteTo("192.168.1.1:8000", modelDirect, "test6"), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal(modelDirect, modelDirect, prio(2))), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal(modelDirect, modelDirect, prio(2))), }, }, { name: "rewrite request model", - requests: fwkepp.ReqLLM(Logger(), "test-rewrite", modelToBeWritten, modelToBeWritten), - pods: []PodState{ - P(0, 0, 0.1, "foo", modelAfterRewrite), + requests: fwkepp.ReqLLM(eppharness.Logger(), "test-rewrite", modelToBeWritten, modelToBeWritten), + pods: []eppharness.PodState{ + eppharness.P(0, 0, 0.1, "foo", modelAfterRewrite), }, wantResponses: ExpectRouteTo("192.168.1.1:8000", modelAfterRewrite, "test-rewrite"), wantMetrics: map[string]string{ - "inference_objective_request_total": CleanMetric(MetricReqTotal(modelToBeWritten, modelAfterRewrite, prio(0))), + "inference_objective_request_total": eppharness.CleanMetric(eppharness.MetricReqTotal(modelToBeWritten, modelAfterRewrite, prio(0))), }, requiresCRDs: true, }, @@ -266,7 +267,7 @@ dataLayer: "content-type": "text/event-stream", "status": "200", }), - pods: []PodState{P(0, 0, 0, "foo")}, + pods: []eppharness.PodState{eppharness.P(0, 0, 0, "foo")}, wantResponses: nil, }, @@ -278,7 +279,7 @@ dataLayer: `{"max_tokens":100,"model":"sql-lo`, `ra-sheddable","prompt":"test6","temperature":0}`, ), - pods: []PodState{P(0, 4, 0.2, modelSheddableTarget)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2, modelSheddableTarget)}, wantResponses: ExpectBufferResp( fmt.Sprintf(`{"max_tokens":100,"model":%q,"prompt":"test6","temperature":0}`, modelSheddable), "application/json"), @@ -289,7 +290,7 @@ dataLayer: map[string]string{"content-type": "application/json"}, "no healthy upstream", ), - pods: []PodState{P(0, 4, 0.2, modelSheddableTarget)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2, modelSheddableTarget)}, wantResponses: ExpectBufferResp("no healthy upstream", "application/json"), }, { @@ -299,7 +300,7 @@ dataLayer: `{"max_tokens":100,"model":"sql-lora-sheddable","prompt":"test6","temperature":0}`, "", ), - pods: []PodState{P(0, 4, 0.2, modelSheddableTarget)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2, modelSheddableTarget)}, wantResponses: ExpectBufferResp( fmt.Sprintf(`{"max_tokens":100,"model":%q,"prompt":"test6","temperature":0}`, modelSheddable), "application/json"), @@ -314,7 +315,7 @@ dataLayer: `data: {"usage":{"prompt_tokens":7,"total_tokens":17,"completion_tokens":10}}`+"\n"+`data: [DONE]`, "", // EndOfStream ), - pods: []PodState{P(0, 4, 0.2, modelSheddableTarget)}, + pods: []eppharness.PodState{eppharness.P(0, 4, 0.2, modelSheddableTarget)}, waitForModel: modelSheddable, wantResponses: ExpectStreamResp( `data: {}`, @@ -323,7 +324,7 @@ dataLayer: ), // Labels are empty because we skipped the Request phase. wantMetrics: map[string]string{ - "inference_objective_input_tokens": CleanMetric(` + "inference_objective_input_tokens": eppharness.CleanMetric(` # HELP inference_objective_input_tokens [ALPHA] [Deprecated: Use llm_d_epp_request_input_tokens] Inference objective input token count distribution for requests in each model. # TYPE inference_objective_input_tokens histogram inference_objective_input_tokens_bucket{model_name="",target_model_name="",le="1"} 0 @@ -362,26 +363,26 @@ dataLayer: ctx := t.Context() - var h *TestHarness - var harnessOpts []HarnessOption + var h *eppharness.TestHarness + var harnessOpts []eppharness.HarnessOption if len(tc.wantSpans) > 0 { - harnessOpts = append(harnessOpts, WithTracing()) + harnessOpts = append(harnessOpts, eppharness.WithTracing()) } - if executionMode.mode == ModeStandalone { - harnessOpts = append(harnessOpts, WithStandaloneMode(executionMode.strategy)) + if executionMode.mode == eppharness.ModeStandalone { + harnessOpts = append(harnessOpts, eppharness.WithStandaloneMode(executionMode.strategy)) } else { - harnessOpts = append(harnessOpts, WithStandardMode()) + harnessOpts = append(harnessOpts, eppharness.WithStandardMode()) } if tc.configText != "" { - harnessOpts = append(harnessOpts, WithConfigText(tc.configText)) + harnessOpts = append(harnessOpts, eppharness.WithConfigText(tc.configText)) } - h = NewTestHarness(ctx, t, harnessOpts...) + h = eppharness.NewTestHarness(ctx, t, harnessOpts...) - if executionMode.mode == ModeStandard || executionMode.strategy == StrategyWithCRD { + if executionMode.mode == eppharness.ModeStandard || executionMode.strategy == eppharness.StrategyWithCRD { h = h.WithBaseResources() } diff --git a/test/integration/epp/k8s_datasource_integration_test.go b/test/integration/epp/k8s_datasource_integration_test.go index 0e54051c70..08dc48c89d 100644 --- a/test/integration/epp/k8s_datasource_integration_test.go +++ b/test/integration/epp/k8s_datasource_integration_test.go @@ -39,6 +39,7 @@ import ( "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/mocks" "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/notifications" "github.com/llm-d/llm-d-router/pkg/epp/metrics" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) const ( @@ -125,14 +126,14 @@ func setupIntegrationTest(t *testing.T, withReconciler bool) *testSetup { uid := uuid.New().String()[:8] // create unique namespace nsName := "epp-test-" + uid ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}} - require.NoError(t, K8sClient().Create(context.Background(), ns)) + require.NoError(t, eppharness.K8sClient().Create(context.Background(), ns)) t.Cleanup(func() { - _ = K8sClient().Delete(context.Background(), ns) + _ = eppharness.K8sClient().Delete(context.Background(), ns) metrics.Reset() }) - mgr, mgrClient := setupTestManager(t, Config(), nsName) + mgr, mgrClient := setupTestManager(t, eppharness.Config(), nsName) var reconciler *testPodReconciler if withReconciler { @@ -168,7 +169,7 @@ func setupTestManager(t *testing.T, cfg *rest.Config, nsName string) (ctrl.Manag skipValidation := true mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: Scheme(), + Scheme: eppharness.Scheme(), Cache: cache.Options{ DefaultNamespaces: map[string]cache.Config{ nsName: {}, @@ -202,7 +203,7 @@ func startManagerAndWaitForSync(ctx context.Context, t *testing.T, mgr ctrl.Mana t.Errorf("Manager failed to start: %v", err) errChan <- err } else { - Logger().Info("Manager stopped due to context cancellation", "error", err) + eppharness.Logger().Info("Manager stopped due to context cancellation", "error", err) } } close(errChan) diff --git a/test/integration/epp/request_attribute_reporter_streaming_test.go b/test/integration/epp/request_attribute_reporter_streaming_test.go index ef0fb924d9..79a6955b41 100644 --- a/test/integration/epp/request_attribute_reporter_streaming_test.go +++ b/test/integration/epp/request_attribute_reporter_streaming_test.go @@ -22,14 +22,15 @@ import ( "github.com/stretchr/testify/require" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) func TestRequestAttributeReporterStreaming(t *testing.T) { ctx := t.Context() - h := NewTestHarness(ctx, t, WithStandardMode(), WithConfigText(requestAttributeReporterTestConfig)).WithBaseResources() + h := eppharness.NewTestHarness(ctx, t, eppharness.WithStandardMode(), eppharness.WithConfigText(requestAttributeReporterTestConfig)).WithBaseResources() - pods := []PodState{P(0, 0, 0.1, modelMyModelTarget)} + pods := []eppharness.PodState{eppharness.P(0, 0, 0.1, modelMyModelTarget)} h.WithPods(pods).WaitForSync(len(pods), modelMyModel) h.WaitForReadyPodsMetric(len(pods)) diff --git a/test/integration/epp/request_attribute_reporter_test.go b/test/integration/epp/request_attribute_reporter_test.go index 2ce9b05916..fd1a1a3a89 100644 --- a/test/integration/epp/request_attribute_reporter_test.go +++ b/test/integration/epp/request_attribute_reporter_test.go @@ -26,6 +26,7 @@ import ( logutil "github.com/llm-d/llm-d-router/pkg/common/observability/logging" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) var reqLogger = zap.New(zap.UseDevMode(true), zap.Level(-1*zapcore.Level(logutil.DEFAULT))) @@ -36,11 +37,11 @@ var requestAttributeReporterTestConfig string func TestRequestAttributeReporter(t *testing.T) { ctx := t.Context() - // Use our new WithConfigText harness option to provide the custom config. - h := NewTestHarness(ctx, t, WithStandardMode(), WithConfigText(requestAttributeReporterTestConfig)).WithBaseResources() + // Use our new eppharness.WithConfigText harness option to provide the custom config. + h := eppharness.NewTestHarness(ctx, t, eppharness.WithStandardMode(), eppharness.WithConfigText(requestAttributeReporterTestConfig)).WithBaseResources() // Pods must exist in datastore so that there's no early failure - pods := []PodState{P(0, 0, 0.1, modelMyModelTarget)} + pods := []eppharness.PodState{eppharness.P(0, 0, 0.1, modelMyModelTarget)} h.WithPods(pods).WaitForSync(len(pods), modelMyModel) h.WaitForReadyPodsMetric(len(pods)) diff --git a/test/integration/epp/runtime_notification_test.go b/test/integration/epp/runtime_notification_test.go index 4a1e9aa839..6ab9d8d373 100644 --- a/test/integration/epp/runtime_notification_test.go +++ b/test/integration/epp/runtime_notification_test.go @@ -31,6 +31,7 @@ import ( fwkplugin "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/plugin" "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/mocks" "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/notifications" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) var ( @@ -49,7 +50,7 @@ func setupRuntimeWithExtractor(r *datalayer.Runtime, extractorName string) (*moc {Plugin: src, Extractors: []fwkplugin.Plugin{ext}}, }, } - return ext, r.Configure(cfg, Logger()) + return ext, r.Configure(cfg, eppharness.Logger()) } func TestRuntimeNotificationDispatch(t *testing.T) { @@ -157,7 +158,7 @@ func TestRuntimeNotificationDispatch(t *testing.T) { {Plugin: src, Extractors: []fwkplugin.Plugin{ext1, ext2}}, }, } - return ext1, r.Configure(cfg, Logger()) + return ext1, r.Configure(cfg, eppharness.Logger()) }, trigger: func(_ *testing.T, s *testSetup, _ *mocks.NotificationExtractor) error { pod := newTestPod("test-pod-multi", s.namespace) @@ -180,7 +181,7 @@ func TestRuntimeNotificationDispatch(t *testing.T) { {Plugin: src, Extractors: []fwkplugin.Plugin{errExtractor, workingExtractor}}, }, } - return workingExtractor, r.Configure(cfg, Logger()) + return workingExtractor, r.Configure(cfg, eppharness.Logger()) }, trigger: func(_ *testing.T, s *testSetup, _ *mocks.NotificationExtractor) error { pod := newTestPod("test-pod-error", s.namespace) @@ -232,7 +233,7 @@ func TestRuntimeNotificationWithRuntime(t *testing.T) { {Plugin: src, Extractors: []fwkplugin.Plugin{extractor}}, }, } - require.NoError(t, r.Configure(cfg, Logger())) + require.NoError(t, r.Configure(cfg, eppharness.Logger())) require.NoError(t, r.Start(setup.ctx, setup.mgr)) @@ -261,7 +262,7 @@ func TestRuntimeNotificationDifferentGVKs(t *testing.T) { }, } - require.NoError(t, r.Configure(cfg, Logger())) + require.NoError(t, r.Configure(cfg, eppharness.Logger())) require.NoError(t, r.Start(setup.ctx, setup.mgr)) pod := newTestPod("test-pod-gvk", setup.namespace) diff --git a/test/integration/epp/runtime_polling_test.go b/test/integration/epp/runtime_polling_test.go index f35eb8fa7d..d35bfee4d9 100644 --- a/test/integration/epp/runtime_polling_test.go +++ b/test/integration/epp/runtime_polling_test.go @@ -36,6 +36,7 @@ import ( fwkplugin "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/plugin" "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/mocks" httpds "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/http" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) func TestRuntimePollingDispatch(t *testing.T) { @@ -96,7 +97,7 @@ func TestRuntimePollingDispatch(t *testing.T) { }, } - require.NoError(t, r.Configure(cfg, Logger())) + require.NoError(t, r.Configure(cfg, eppharness.Logger())) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -148,7 +149,7 @@ func TestRuntimePollingMultipleExtractors(t *testing.T) { }, } - require.NoError(t, r.Configure(cfg, Logger())) + require.NoError(t, r.Configure(cfg, eppharness.Logger())) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -197,7 +198,7 @@ func TestRuntimePollingEndpointLifecycle(t *testing.T) { }, } - require.NoError(t, r.Configure(cfg, Logger())) + require.NoError(t, r.Configure(cfg, eppharness.Logger())) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -251,7 +252,7 @@ func TestRuntimePollingWithoutExtractors(t *testing.T) { }, } - require.NoError(t, r.Configure(cfg, Logger())) + require.NoError(t, r.Configure(cfg, eppharness.Logger())) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -290,7 +291,7 @@ func TestRuntimePollingHTTPError(t *testing.T) { }, } - require.NoError(t, r.Configure(cfg, Logger())) + require.NoError(t, r.Configure(cfg, eppharness.Logger())) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) diff --git a/test/integration/epp/session_affinity_test.go b/test/integration/epp/session_affinity_test.go index 35eb6b017f..7efe72dd27 100644 --- a/test/integration/epp/session_affinity_test.go +++ b/test/integration/epp/session_affinity_test.go @@ -31,6 +31,7 @@ import ( sessionutil "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/scheduling/util/sessionaffinity" "github.com/llm-d/llm-d-router/pkg/epp/metadata" fwkepp "github.com/llm-d/llm-d-router/test/framework/epp" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" fwkk8s "github.com/llm-d/llm-d-router/test/framework/k8s" ) @@ -61,12 +62,12 @@ dataLayer: ` ctx := t.Context() - h := NewTestHarness(ctx, t, WithStandardMode(), WithConfigText(configText)).WithBaseResources() + h := eppharness.NewTestHarness(ctx, t, eppharness.WithStandardMode(), eppharness.WithConfigText(configText)).WithBaseResources() // Two ready pods so the filter has a real choice to pin to. - pods := []PodState{ - P(0, 0, 0.1, modelMyModelTarget), - P(1, 0, 0.1, modelMyModelTarget), + pods := []eppharness.PodState{ + eppharness.P(0, 0, 0.1, modelMyModelTarget), + eppharness.P(1, 0, 0.1, modelMyModelTarget), } h.WithPods(pods).WaitForSync(len(pods), modelMyModel) h.WaitForReadyPodsMetric(len(pods)) @@ -86,7 +87,7 @@ dataLayer: // EPP and returns the routed destination endpoint and the session token set on // the response headers. When sessionToken is non-empty it is sent as the // session header on the request. -func sendSessionRequest(t *testing.T, h *TestHarness, sessionToken string) (endpoint, token string) { +func sendSessionRequest(t *testing.T, h *eppharness.TestHarness, sessionToken string) (endpoint, token string) { t.Helper() reqHeaders := map[string]string{ @@ -166,10 +167,10 @@ schedulingProfiles: // setupPDHarness creates a test harness with prefill and decode pods for PD // session-affinity tests. -func setupPDHarness(t *testing.T, configText string) *TestHarness { +func setupPDHarness(t *testing.T, configText string) *eppharness.TestHarness { t.Helper() ctx := t.Context() - h := NewTestHarness(ctx, t, WithStandardMode(), WithConfigText(configText)).WithBaseResources() + h := eppharness.NewTestHarness(ctx, t, eppharness.WithStandardMode(), eppharness.WithConfigText(configText)).WithBaseResources() metricsMap := make(map[types.NamespacedName]*fwkdl.Metrics) type podDef struct { @@ -192,7 +193,7 @@ func setupPDHarness(t *testing.T, configText string) *TestHarness { Namespace(h.Namespace). ReadyCondition(). Labels(map[string]string{ - "app": TestPoolName, + "app": eppharness.TestPoolName, bylabel.RoleLabel: p.role, }). IP(fmt.Sprintf("192.168.1.%d", p.index+1)). @@ -200,9 +201,9 @@ func setupPDHarness(t *testing.T, configText string) *TestHarness { ObjRef() intendedStatus := pod.Status - require.NoError(t, K8sClient().Create(ctx, pod)) + require.NoError(t, eppharness.K8sClient().Create(ctx, pod)) pod.Status = intendedStatus - require.NoError(t, K8sClient().Status().Update(ctx, pod)) + require.NoError(t, eppharness.K8sClient().Status().Update(ctx, pod)) } h.SetPodMetrics(metricsMap) h.WaitForSync(len(pods), modelMyModel) @@ -260,7 +261,7 @@ func TestSessionAffinityFilter_DecodeOnly(t *testing.T) { // sendSessionRequestGetHeader drives a request through the EPP and returns the // value of the named response header. It extends sendSessionRequest to support // the prefill session token header. -func sendSessionRequestGetHeader(t *testing.T, h *TestHarness, decodeToken, prefillToken, headerKey string) string { +func sendSessionRequestGetHeader(t *testing.T, h *eppharness.TestHarness, decodeToken, prefillToken, headerKey string) string { t.Helper() reqHeaders := map[string]string{ diff --git a/test/integration/epp/wellknown_configs_test.go b/test/integration/epp/wellknown_configs_test.go index 2970f1dc9e..148bed92b1 100644 --- a/test/integration/epp/wellknown_configs_test.go +++ b/test/integration/epp/wellknown_configs_test.go @@ -20,6 +20,7 @@ import ( "testing" configapi "github.com/llm-d/llm-d-router/apix/config/v1alpha1" + eppharness "github.com/llm-d/llm-d-router/test/framework/epp/harness" ) // wellKnownConfigs are configs in the llm-d well-lit path guides. @@ -403,7 +404,7 @@ schedulingProfiles: func TestWellKnownConfigs(t *testing.T) { for name, tc := range wellKnownConfigs { t.Run(name, func(t *testing.T) { - runner := NewTestHarness(t.Context(), t, WithConfigText(tc.yaml)).Runner + runner := eppharness.NewTestHarness(t.Context(), t, eppharness.WithConfigText(tc.yaml)).Runner t.Logf("All plugins: %v", runner.PluginHandle.GetAllPluginsWithNames()) // Validate that the expected plugins exist in runner.PluginHandle.