diff --git a/Makefile b/Makefile index 3919c14c9d..175cd2261f 100644 --- a/Makefile +++ b/Makefile @@ -119,12 +119,14 @@ BUILDER_SOCK_FLAGS = $(DOCKER_GROUP_PARAM) \ -e CONTAINER_RUNTIME=docker endif +E2E_NUM_PROCS ?= 5 + # Env vars forwarded into the e2e test container. # Add new image vars here so they are automatically passed through. # Should we pass ALL env vars here? E2E_ENV_VARS = EPP_IMAGE VLLM_IMAGE SIDECAR_IMAGE VLLM_RENDER_IMAGE \ E2E_KEEP_CLUSTER_ON_FAILURE E2E_PORT E2E_METRICS_PORT K8S_CONTEXT READY_TIMEOUT \ - E2E_LABEL_FILTER LOAD_VLLM_RENDER_IMAGE HF_TOKEN + E2E_NUM_PROCS E2E_LABEL_FILTER LOAD_VLLM_RENDER_IMAGE HF_TOKEN BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))')) ifneq ($(filter command line environment,$(origin NAMESPACE)),) BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE) diff --git a/deploy/environments/dev/e2e-infra/envoy.yaml b/deploy/environments/dev/e2e-infra/envoy.yaml index d84b1d25cb..84b31777e8 100644 --- a/deploy/environments/dev/e2e-infra/envoy.yaml +++ b/deploy/environments/dev/e2e-infra/envoy.yaml @@ -299,7 +299,7 @@ spec: port: 8081 protocol: TCP targetPort: 8081 - nodePort: 30080 + nodePort: ${ENVOY_NODE_PORT} selector: app: envoy type: NodePort diff --git a/deploy/environments/dev/e2e-infra/services.yaml b/deploy/environments/dev/e2e-infra/services.yaml index 544657ad9c..9b19d88d8d 100644 --- a/deploy/environments/dev/e2e-infra/services.yaml +++ b/deploy/environments/dev/e2e-infra/services.yaml @@ -30,5 +30,5 @@ spec: protocol: TCP port: 9090 targetPort: 9090 - nodePort: 32090 + nodePort: ${METRICS_NODE_PORT} type: NodePort diff --git a/test/e2e/README.md b/test/e2e/README.md index 4b147374a5..47e8175081 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -18,6 +18,14 @@ The end-to-end tests are designed to validate end-to-end Gateway API Inference E export HF_TOKEN= ``` +## Running the End-to-End Tests in Parallel + +By default the end to end tests run in groups that run in parallel to each other on the same Kubernetes cluster. +As each group is setup various Kubernetes objects are created for the test group. They include the Namespace, +the Envoy Deployment and Service, ServiceAccount, EPP Service, and RBAC. When running on Kind each Namespace is +assigned its own pair of NodePorts for Envoy and the EPP's metrics port. When the test group ends the created +Kubernetes objects are delete. + ## Running the End-to-End Tests Follow these steps to run the end-to-end tests: @@ -30,6 +38,10 @@ Follow these steps to run the end-to-end tests: 1. **Optional Settings** + - **Running all of the tests serially** By default the end to end tests are run in groups that are parallel to + each other. The number of groups running in parallel at any time is controlled via the E2E_NUM_PROCS environment + variable, which defaults to five. To run all of the tests in a serial fashion, use `E2E_NUM_PROCS=1`. + - **Run the tests on a real cluster**: By default the end to end tests are run on a kind cluster that is created and torn down by the test code. If you want to run the tests on a real Kubernetes cluster, set the following environment variable: @@ -43,8 +55,19 @@ Follow these steps to run the end-to-end tests: **Note:** When running on a real cluster the tests will start a pair of `kubectl port-forward` processes to sent various requests to the cluster under test. - - **Set the test namespace**: By default, the e2e test creates resources in the `default` namespace. - If you would like to change this namespace, set the following environment variable: + - **Set the test namespace**: The namespace(s) in which the tests run vary based on whether or not the tests + are being run in parallel or not. + + If the tests are being run in parallel, the e2e test creates resources in namespaces of the form -N, + where by default is `e2e` and N is the process number of the process running the test. can + changed by setting the following environment variable: + + ```sh + export NAMESPACE= + ``` + + If the test are not being run in parallel, then by default, the e2e test creates resources in the `default` + namespace. If you would like to change this namespace, set the following environment variable: ```sh export NAMESPACE= diff --git a/test/e2e/disruption_test.go b/test/e2e/disruption_test.go index 7382b8ded1..718126e825 100644 --- a/test/e2e/disruption_test.go +++ b/test/e2e/disruption_test.go @@ -50,9 +50,9 @@ func sendRawCompletion() (int, error) { // podGone returns true when the named pod no longer appears in the pod list // and at least minRemaining pods exist. -func podGone(podName string, minRemaining int) func() bool { +func podGone(podName string, nsName string, minRemaining int) func() bool { return func() bool { - _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector) + _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) for _, pod := range currentDecode { if pod == podName { return false @@ -63,9 +63,9 @@ func podGone(podName string, minRemaining int) func() bool { } // eppPodReady returns true when a new EPP pod (not oldPodName) is Running and Ready. -func eppPodReady(oldPodName string) func() bool { +func eppPodReady(oldPodName string, nsName string) func() bool { return func() bool { - pods := getPods(map[string]string{"app": "e2e-epp"}) + pods := getPods(map[string]string{"app": "e2e-epp"}, nsName) for _, p := range pods { if p.Name == oldPodName { continue @@ -96,20 +96,17 @@ func completionRoutedToNamespace(nsName string) error { return nil } -var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disruptiveTestLabel), func() { - ginkgo.When("A decode pod is killed mid-request", func() { +var _ = ginkgo.Describe("Disruption tests", ginkgo.Label(disruptiveTestLabel), func() { + ginkgo.When("A decode pod is killed mid-request", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should recover and route to surviving pods", func() { - infPoolObjects = createInferencePool(1, true) - nsName := getNamespace() - modelServers := createModelServersDecode(2) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers, nsName) + infPoolObjects := createInferencePool(1) + modelServers := createModelServersDecode(2) epp := createEndPointPicker(simpleConfig) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp, nsName) - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(2)) @@ -123,7 +120,7 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt deletePodByName(targetPod, 0) ginkgo.By("Waiting for killed pod to be replaced") - gomega.Eventually(podGone(targetPod, 1), podRemovalTimeout, 1*time.Second).Should(gomega.BeTrue()) + gomega.Eventually(podGone(targetPod, nsName, 1), podRemovalTimeout, 1*time.Second).Should(gomega.BeTrue()) ginkgo.By("Verifying new requests eventually route to a pod other than the killed one") gomega.Eventually(func() error { @@ -142,29 +139,31 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt ginkgo.By("Waiting for replacement pod to become ready") gomega.Eventually(func() int { - _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector) + _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) return len(currentDecode) }, readyTimeout, 2*time.Second).Should(gomega.Equal(2)) ginkgo.By("Verifying requests succeed consistently after recovery") gomega.Eventually(completionRoutedToNamespace, eppRecoveryTimeout, 1*time.Second).WithArguments(nsName). MustPassRepeatedly(3).Should(gomega.Succeed()) + + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, epp, nsName) }) - }) + })) - ginkgo.When("A decode pod is killed while a streaming request is in-flight", func() { + ginkgo.When("A decode pod is killed while a streaming request is in-flight", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should not hang and should recover routing", func() { - infPoolObjects = createInferencePool(1, true) - nsName := getNamespace() + infPoolObjects := createInferencePool(1) + modelServers := createModelServersDecode(2) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers, nsName) epp := createEndPointPicker(simpleConfig) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp, nsName) - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(2)) @@ -194,29 +193,31 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt ginkgo.By("Waiting for replacement pod") gomega.Eventually(func() int { - _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector) + _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) return len(currentDecode) }, readyTimeout, 2*time.Second).Should(gomega.Equal(2)) ginkgo.By("Verifying requests succeed consistently after recovery") gomega.Eventually(completionRoutedToNamespace, eppRecoveryTimeout, 1*time.Second).WithArguments(nsName). MustPassRepeatedly(3).Should(gomega.Succeed()) + + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, epp, nsName) }) - }) + })) - ginkgo.When("All pods are gone", func() { + ginkgo.When("All pods are gone", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should return 503 to the client", func() { - infPoolObjects = createInferencePool(1, true) - nsName := getNamespace() + infPoolObjects := createInferencePool(1) + modelServers := createModelServersDecode(1) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers, nsName) epp := createEndPointPicker(simpleConfig) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp, nsName) - _, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + _, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(decodePods).Should(gomega.HaveLen(1)) ginkgo.By("Verifying requests succeed before disruption") @@ -228,7 +229,7 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt ginkgo.By("Waiting for all pods to be removed") gomega.Eventually(func() int { - _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector) + _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) return len(currentDecode) }, podRemovalTimeout, 1*time.Second).Should(gomega.Equal(0)) @@ -249,27 +250,29 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt nsHdr, _, _ := runCompletion(simplePrompt, simModelName) return nsHdr }, eppRecoveryTimeout, 2*time.Second).Should(gomega.Equal(nsName)) + + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, epp, nsName) }) - }) + })) - ginkgo.When("The EPP is killed while requests are in flight", func() { + ginkgo.When("The EPP is killed while requests are in flight", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should recover and resume routing after restart", func() { - infPoolObjects = createInferencePool(1, true) - nsName := getNamespace() + infPoolObjects := createInferencePool(1) + modelServers := createModelServersDecode(1) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers, nsName) epp := createEndPointPicker(simpleConfig) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp, nsName) ginkgo.By("Verifying requests succeed before EPP disruption") nsHdr, _, _ := runCompletion(simplePrompt, simModelName) gomega.Expect(nsHdr).Should(gomega.Equal(getNamespace())) ginkgo.By("Finding EPP pod") - eppPods := getPods(map[string]string{"app": "e2e-epp"}) + eppPods := getPods(map[string]string{"app": "e2e-epp"}, nsName) gomega.Expect(eppPods).Should(gomega.HaveLen(1)) eppPodName := eppPods[0].Name @@ -287,7 +290,7 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt "requests should fail while EPP is down") ginkgo.By("Waiting for EPP to recover") - gomega.Eventually(eppPodReady(eppPodName), readyTimeout, 2*time.Second).Should(gomega.BeTrue()) + gomega.Eventually(eppPodReady(eppPodName, nsName), readyTimeout, 2*time.Second).Should(gomega.BeTrue()) ginkgo.By("Verifying requests succeed after EPP recovery") gomega.Eventually(func() error { @@ -300,19 +303,22 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt } return nil }, eppRecoveryTimeout, 2*time.Second).Should(gomega.Succeed()) + + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, epp, nsName) }) - }) + })) - ginkgo.When("Traffic is flowing during scale-to-zero and back", func() { + ginkgo.When("Traffic is flowing during scale-to-zero and back", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should return 503s when empty and recover when scaled back", func() { - infPoolObjects = createInferencePool(1, true) nsName := getNamespace() + infPoolObjects := createInferencePool(1) + modelServers := createModelServersDecode(1) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers, nsName) epp := createEndPointPicker(simpleConfig) - ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp, nsName) ginkgo.By("Verifying requests succeed before disruption") nsHdr, _, _ := runCompletion(simplePrompt, simModelName) @@ -332,7 +338,7 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt ginkgo.By("Waiting for all pods to be removed") gomega.Eventually(func() int { - _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector) + _, currentDecode := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) return len(currentDecode) }, podRemovalTimeout, 1*time.Second).Should(gomega.Equal(0)) @@ -352,8 +358,12 @@ var _ = ginkgo.Describe("Disruption tests", ginkgo.Ordered, ginkgo.Label(disrupt <-done ginkgo.By(fmt.Sprintf("Traffic results: %d successes, %d failures", tc.successes(), tc.failures())) + + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, epp, nsName) }) - }) + })) }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 8d6f3b1b53..ea0eceac95 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -14,7 +14,7 @@ import ( "github.com/onsi/gomega/gexec" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -87,7 +87,12 @@ var ( vllmRenderImage = env.GetEnvString("VLLM_RENDER_IMAGE", "vllm/vllm-openai-cpu:v0.21.0", ginkgo.GinkgoLogr) vllmRenderPort = env.GetEnvString("VLLM_RENDER_PORT", "8082", ginkgo.GinkgoLogr) loadRenderImage = env.GetEnvBool("LOAD_VLLM_RENDER_IMAGE", true, ginkgo.GinkgoLogr) - numProcesses = env.GetEnvInt("E2E_NUM_PROCS", 1, ginkgo.GinkgoLogr) + + // numProcesses is the number of parallel processes that will be used by the tests. + // Theoretically this value could be gotten from the Ginkgo Suite Configuration. + // However, that needs to be done while the tests are running and has shown to not + // always allow the baseNsName field to be set correctly. + numProcesses = env.GetEnvInt("E2E_NUM_PROCS", 1, ginkgo.GinkgoLogr) // baseNsName is the base of the namespace in which the K8S objects will be created baseNsName = env.GetEnvString("NAMESPACE", testutils.DefaultNsName(numProcesses, "e2e"), ginkgo.GinkgoLogr) @@ -97,16 +102,10 @@ var ( readyTimeout = env.GetEnvDuration("READY_TIMEOUT", defaultReadyTimeout, ginkgo.GinkgoLogr) interval = defaultInterval - crdObjects []string - envoyObjects []string - rbacObjects []string - serviceAccountObjects []string - serviceObjects []string - renderObjects []string - infPoolObjects []string - createdNameSpace bool + crdObjects []string + renderObjects []string + createdRendererNS bool - portForwardSession *gexec.Session eppPortForwardSession *gexec.Session ) @@ -117,7 +116,8 @@ func TestEndToEnd(t *testing.T) { ) } -var _ = ginkgo.BeforeSuite(func() { +// There is only special setup to be done before process #1 +var _ = ginkgo.SynchronizedBeforeSuite(func() { testutils.RequireParallelProcessesMatch(numProcesses) if k8sContext == "" { @@ -125,35 +125,14 @@ var _ = ginkgo.BeforeSuite(func() { } testConfig = testutils.NewTestConfig(k8sContext) setupK8sClient() - setupNameSpace() createCRDs() - nsName := getNamespace() - createEnvoy(nsName) - infraSubs := map[string]string{ - "${EPP_NAME}": "e2e-epp", - } - rbacYamls := substituteMany(testutils.ReadYaml(rbacManifest), infraSubs) - rbacObjects = testutils.CreateObjsFromYaml(testConfig, rbacYamls, nsName) - saYamls := substituteMany(testutils.ReadYaml(serviceAccountManifest), infraSubs) - serviceAccountObjects = testutils.CreateObjsFromYaml(testConfig, saYamls, nsName) - serviceObjects = testutils.ApplyYAMLFile(testConfig, servicesManifest, nsName) - renderObjects = createRender(nsName) - - // Prevent failure in tests due to InferencePool not existing before the test - infPoolObjects = createInferencePool(1, false) -}) - -var _ = ginkgo.AfterSuite(func() { - // Stop port-forwards when using an existing cluster context; they must be - // terminated before the process exits regardless of pass/fail status. - if k8sContext != "" { - if portForwardSession != nil { - portForwardSession.Terminate() - } - if eppPortForwardSession != nil { - eppPortForwardSession.Terminate() - eppPortForwardSession = nil - } + // If we are running tests in parallel, create the renderer in the "base namespace" + createdRendererNS = setupNameSpaceHelper(baseNsName) + renderObjects = createRender(baseNsName) +}, func() { + if ginkgo.GinkgoParallelProcess() != 1 { + testConfig = testutils.NewTestConfig(k8sContext) + setupK8sClient() } }) @@ -188,21 +167,12 @@ var _ = ginkgo.ReportAfterSuite("cleanup", func(report ginkgo.Report) { if shouldKeep { ginkgo.By("Keeping created Kubernetes objects due to suite failure (E2E_KEEP_CLUSTER_ON_FAILURE=true)") } else { - nsName := getNamespace() ginkgo.By("Deleting created Kubernetes objects") - testutils.DeleteObjects(testConfig, infPoolObjects, nsName) - testutils.DeleteObjects(testConfig, renderObjects, nsName) - testutils.DeleteObjects(testConfig, serviceObjects, nsName) - testutils.DeleteObjects(testConfig, serviceAccountObjects, nsName) - testutils.DeleteObjects(testConfig, rbacObjects, nsName) - testutils.DeleteObjects(testConfig, envoyObjects, nsName) - testutils.DeleteObjects(testConfig, crdObjects, "") - - if createdNameSpace { - ginkgo.By("Deleting namespace " + getNamespace()) - err := testConfig.KubeCli.CoreV1().Namespaces().Delete(testConfig.Context, getNamespace(), metav1.DeleteOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + testutils.DeleteObjects(testConfig, renderObjects, baseNsName) + if createdRendererNS { + deleteNameSpace(baseNsName) } + testutils.DeleteObjects(testConfig, crdObjects, "") } } }) @@ -290,28 +260,44 @@ func setupK8sClient() { } // setupNameSpace sets up the specified namespace if it doesn't exist -func setupNameSpace() { - _, err := testConfig.KubeCli.CoreV1().Namespaces().Get(testConfig.Context, getNamespace(), metav1.GetOptions{}) +func setupNameSpace() bool { + return setupNameSpaceHelper(getNamespace()) +} + +func setupNameSpaceHelper(nsName string) bool { + ginkgo.By("Setup namespace " + nsName) + _, err := testConfig.KubeCli.CoreV1().Namespaces().Get(testConfig.Context, nsName, metav1.GetOptions{}) if err == nil { - return + return false } - gomega.Expect(errors.IsNotFound(err)).To(gomega.BeTrue()) + gomega.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue()) - ginkgo.By("Creating namespace " + getNamespace()) + ginkgo.By("Creating namespace " + nsName) namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: getNamespace(), + Name: nsName, }, } _, err = testConfig.KubeCli.CoreV1().Namespaces().Create(testConfig.Context, namespace, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - createdNameSpace = true - ginkgo.By("Ensuring namespace exists: " + getNamespace()) + ginkgo.By("Ensuring namespace exists: " + nsName) testutils.EventuallyExists(testConfig, func() error { return testConfig.K8sClient.Get(testConfig.Context, - types.NamespacedName{Name: getNamespace()}, &corev1.Namespace{}) + types.NamespacedName{Name: nsName}, &corev1.Namespace{}) }) + + return true +} + +func deleteNameSpace(nsName string) { + ginkgo.By("Deleting namespace " + nsName) + err := testConfig.KubeCli.CoreV1().Namespaces().Delete(testConfig.Context, nsName, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Eventually(func() bool { + _, err := testConfig.KubeCli.CoreV1().Namespaces().Get(testConfig.Context, nsName, metav1.GetOptions{}) + return apierrors.IsNotFound(err) + }, testConfig.ExistsTimeout, testConfig.Interval).Should(gomega.BeTrue()) } // createCRDs creates the Inference Extension CRDs used for testing. @@ -320,11 +306,16 @@ func createCRDs() { crdObjects = testutils.CreateObjsFromYaml(testConfig, crds, "") } -func createEnvoy(nsName string) { +func createEnvoy(nsName string) ([]string, *gexec.Session) { + infraSubs := map[string]string{ + "${NAMESPACE}": nsName, + "${ENVOY_NODE_PORT}": strconv.Itoa(getPort()), + } manifests := testutils.ReadYaml(envoyManifest) - manifests = substituteMany(manifests, map[string]string{"${NAMESPACE}": nsName}) + manifests = substituteMany(manifests, infraSubs) ginkgo.By("Creating envoy proxy resources from manifest: " + envoyManifest) - envoyObjects = testutils.CreateObjsFromYaml(testConfig, manifests, nsName) + envoyObjects := testutils.CreateObjsFromYaml(testConfig, manifests, nsName) + var portForwardSession *gexec.Session if k8sContext != "" { envoyName := "" @@ -342,17 +333,13 @@ func createEnvoy(nsName string) { portForwardSession, err = gexec.Start(command, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) } + return envoyObjects, portForwardSession } -func createInferencePool(numTargetPorts int, toDelete bool) []string { +func createInferencePool(numTargetPorts int) []string { poolName := simModelName + "-inference-pool" nsName := getNamespace() - if toDelete { - objName := []string{"inferencepool/" + poolName} - testutils.DeleteObjects(testConfig, objName, nsName) - } - infPoolYaml := testutils.ReadYaml(inferExtManifest) // targetPorts is substituted into `targetPorts: ${TARGET_PORTS}` in inference-pools.yaml. // Each item must use 2-space indentation to match that field's level in the YAML. diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 273e5bd487..b5871d30ac 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -61,10 +61,10 @@ var ( doubleEmbedding = []string{"First sentence to embed.", "Second sentence to embed."} ) -var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { - ginkgo.When("Running simple non-PD configuration", func() { +var _ = ginkgo.Describe("Run end to end tests", func() { + ginkgo.When("Running simple non-PD configuration", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) modelServers := createModelServersDecode(1) @@ -75,11 +75,12 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) ginkgo.It("should report metrics", func() { numTargetPorts := 1 - infPoolObjects = createInferencePool(numTargetPorts, true) + infPoolObjects := createInferencePool(numTargetPorts) temp := strings.Split(infPoolObjects[0], "/") infPoolName := temp[1] @@ -92,15 +93,16 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running leader election", func() { + ginkgo.When("Running leader election", ginkgo.Ordered, testWrapper(func() { ginkgo.It("Should elect one leader and have other pods as not ready", func() { numOfPods := 3 numTargetPorts := 1 - infPoolObjects = createInferencePool(numTargetPorts, true) + infPoolObjects := createInferencePool(numTargetPorts) modelServers := createModelServersDecode(1) @@ -112,13 +114,14 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) ginkgo.It("Should successfully failover and serve traffic after the leader pod is deleted", func() { numOfPods := 3 numTargetPorts := 1 - infPoolObjects = createInferencePool(numTargetPorts, true) + infPoolObjects := createInferencePool(numTargetPorts) temp := strings.Split(infPoolObjects[0], "/") infPoolName := temp[1] @@ -153,13 +156,14 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running a PD configuration with nixlv2 connector(deprecated pd-profile-handler)", ginkgo.Label(metricsTestLabel, deprecatedPDTestLabel), func() { + ginkgo.When("Running a PD configuration with nixlv2 connector(deprecated pd-profile-handler)", ginkgo.Label(metricsTestLabel, deprecatedPDTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 4 @@ -172,7 +176,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { startEPPMetricsPortForward() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -222,8 +226,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) for _, tc := range []struct { name string @@ -235,9 +240,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { } { config := tc.config // capture for closure label := tc.label - ginkgo.When("Running a PD configuration with shared-storage connector using "+tc.name, ginkgo.Label(sharedStorageTestLabel, label), func() { + ginkgo.When("Running a PD configuration with shared-storage connector using "+tc.name, ginkgo.Label(sharedStorageTestLabel, label), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run regular (non-streaming) requests successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 2 @@ -246,7 +251,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { epp := createEndPointPicker(config) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -273,10 +278,11 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) ginkgo.It("should run streaming requests successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 2 @@ -285,7 +291,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { epp := createEndPointPicker(config) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -306,6 +312,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) ginkgo.It("should handle decode-first success scenario with cache_hit_threshold", func() { @@ -313,7 +320,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { // When cache_hit_threshold is set and the decode succeeds (cache hit), // the request should complete without falling back to P/D. // IMPORTANT: The prefill pod should NOT process any requests in this scenario. - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 2 @@ -322,7 +329,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { epp := createEndPointPicker(config) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -354,6 +361,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) ginkgo.It("should handle decode-first fallback to P/D when cache threshold not met", func() { @@ -361,7 +369,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { // When cache_hit_threshold is set and the decode returns cache_threshold finish_reason, // the sidecar should fall back to P/D disaggregation. // IMPORTANT: The prefill pod SHOULD process requests in this scenario. - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 2 @@ -370,7 +378,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { epp := createEndPointPicker(config) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -406,13 +414,14 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) } - ginkgo.When("Running a PD configuration with mooncake connector (disagg-profile-handler)", func() { + ginkgo.When("Running a PD configuration with mooncake connector (disagg-profile-handler)", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run regular (non-streaming) requests successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 2 @@ -421,7 +430,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { epp := createEndPointPicker(pdConfig) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -435,10 +444,11 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) ginkgo.It("should run streaming requests successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 2 @@ -447,7 +457,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { epp := createEndPointPicker(pdConfig) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -461,13 +471,14 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running a PD configuration with disagg-profile-handler and metrics validation", ginkgo.Label(metricsTestLabel, disaggTestLabel), func() { + ginkgo.When("Running a PD configuration with disagg-profile-handler and metrics validation", ginkgo.Label(metricsTestLabel, disaggTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 4 @@ -480,7 +491,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { startEPPMetricsPortForward() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -530,19 +541,20 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running simple non-PD configuration with disagg-profile-handler", func() { + ginkgo.When("Running simple non-PD configuration with disagg-profile-handler", ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) modelServers := createModelServersDecode(1) epp := createEndPointPicker(decodeOnlyConfig) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(1)) @@ -556,12 +568,13 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running an E/PD (Encode/Prefill-Decode) configuration", ginkgo.Label(extendedTestLabel), func() { + ginkgo.When("Running an E/PD (Encode/Prefill-Decode) configuration", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should route multimodal requests through encode and decode pods", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) encodeReplicas := 2 decodeReplicas := 1 @@ -575,8 +588,8 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { startEPPMetricsPortForward() } - encodePods := getPodNames(encodeSelector) - prefillDecodePods := getPodNames(prefillDecodeSelector) + encodePods := getPodNames(encodeSelector, nsName) + prefillDecodePods := getPodNames(prefillDecodeSelector, nsName) gomega.Expect(encodePods).Should(gomega.HaveLen(encodeReplicas)) gomega.Expect(prefillDecodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -630,12 +643,13 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running an E/P/D (encode/prefill/decode) configuration", ginkgo.Label(extendedTestLabel), func() { + ginkgo.When("Running an E/P/D (encode/prefill/decode) configuration", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should route multimodal requests through encode, prefill, and decode pods", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) encodeReplicas := 2 prefillReplicas := 1 @@ -650,9 +664,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { startEPPMetricsPortForward() } - encodePods := getPodNames(encodeSelector) - prefillPods := getPodNames(prefillSelector) - decodePods := getPodNames(decodeSelector) + encodePods := getPodNames(encodeSelector, nsName) + prefillPods := getPodNames(prefillSelector, nsName) + decodePods := getPodNames(decodeSelector, nsName) gomega.Expect(encodePods).Should(gomega.HaveLen(encodeReplicas)) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -714,12 +728,13 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running an EPD (no disaggregation) configuration", ginkgo.Label(extendedTestLabel), func() { + ginkgo.When("Running an EPD (no disaggregation) configuration", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should route text and multimodal requests to the single deployment", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) // Single deployment labeled encode-prefill-decode: matches encode-filter, prefill-filter, // and decode-filter, so all EPD stages are handled by the same deployment. @@ -736,7 +751,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { startEPPMetricsPortForward() } - epdPods := getPodNames(epdSingleSelector) + epdPods := getPodNames(epdSingleSelector, nsName) gomega.Expect(epdPods).Should(gomega.HaveLen(replicas)) // Text completion: encode skipped, routes to decode profile -> single deployment @@ -781,18 +796,19 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running simple non-PD KV enabled configuration", ginkgo.Label(extendedTestLabel), func() { + ginkgo.When("Running simple non-PD KV enabled configuration", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) modelServers := createModelServersDecodeKV(1) epp := createEndPointPicker(kvConfig()) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(1)) @@ -804,18 +820,19 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running KV configuration with external tokenizer DataProducer plugin", ginkgo.Label(extendedTestLabel), func() { + ginkgo.When("Running KV configuration with external tokenizer DataProducer plugin", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should run successfully", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) modelServers := createModelServersDecodeKV(1) epp := createEndPointPicker(kvExternalTokenizerConfig()) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(1)) @@ -838,19 +855,20 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Scaling up and down the model servers", ginkgo.Label(extendedTestLabel), func() { + ginkgo.When("Scaling up and down the model servers", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should distribute inference requests across all model servers", func() { - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) modelServers := createModelServersDecode(1) epp := createEndPointPicker(scaleConfig) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(1)) @@ -863,7 +881,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { scaleDeployment(nsName, modelServers, 1) - scaledUpPrefillPods, scaledUpDecodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + scaledUpPrefillPods, scaledUpDecodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(scaledUpPrefillPods).Should(gomega.BeEmpty()) gomega.Expect(scaledUpDecodePods).Should(gomega.HaveLen(2)) @@ -881,7 +899,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { scaleDeployment(nsName, modelServers, -1) - scaledDownPrefillPods, scaledDownDecodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + scaledDownPrefillPods, scaledDownDecodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(scaledDownPrefillPods).Should(gomega.BeEmpty()) gomega.Expect(scaledDownDecodePods).Should(gomega.HaveLen(1)) gomega.Expect(scaledDownDecodePods[0]).Should(gomega.BeElementOf(scaledUpDecodePods)) @@ -895,19 +913,20 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) - ginkgo.When("Running a vLLM Data Parallel configuration", ginkgo.Label(extendedTestLabel), func() { + ginkgo.When("Running a vLLM Data Parallel configuration", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { ginkgo.It("should schedule inference on all ranks", func() { - infPoolObjects = createInferencePool(2, true) + infPoolObjects := createInferencePool(2) modelServers := createModelServersDecodeDP(1) epp := createEndPointPicker(dataParallelConfig) nsName := getNamespace() - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(1)) @@ -945,8 +964,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { testutils.DeleteObjects(testConfig, epp, nsName) testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) - }) + })) }) func waitForReadyLeader(numOfPods int, nsName string) *corev1.Pod { diff --git a/test/e2e/generate_endpoint_test.go b/test/e2e/generate_endpoint_test.go index e0c30663c2..5e6d8a4ad6 100644 --- a/test/e2e/generate_endpoint_test.go +++ b/test/e2e/generate_endpoint_test.go @@ -43,22 +43,18 @@ var twoImages = []imageSpec{ {Hash: "e2e-image-hash-1", Offset: 4, Length: 5}, } -var _ = ginkgo.Describe("Direct gateway /inference/v1/generate encode against encode-only", ginkgo.Label(extendedTestLabel), func() { +var _ = ginkgo.Describe("Direct gateway /inference/v1/generate encode against encode-only", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { // Uses single-profile-handler (generateEncodeConfig) so the EPP routes // directly to encode pods without requiring a decode stage first. ginkgo.It("returns ec_transfer_params for encode bodies", func() { nsName := getNamespace() - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) encodeReplicas := 1 modelServers := createModelServersEncodeOnly(encodeReplicas) epp := createEndPointPicker(generateEncodeConfig) - ginkgo.DeferCleanup(func() { - testutils.DeleteObjects(testConfig, epp, nsName) - testutils.DeleteObjects(testConfig, modelServers, nsName) - }) - encodePods := getPodNames(encodeSelector) + encodePods := getPodNames(encodeSelector, nsName) gomega.Expect(encodePods).Should(gomega.HaveLen(encodeReplicas)) ginkgo.By("Encode_Generate: single-image encode body returns ec_transfer_params") @@ -74,25 +70,25 @@ var _ = ginkgo.Describe("Direct gateway /inference/v1/generate encode against en parsed := expectGenerateOK(resp, raw) expectECTransferParams(parsed, raw) } + + testutils.DeleteObjects(testConfig, epp, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) -}) +})) -var _ = ginkgo.Describe("Direct gateway /inference/v1/generate prefill against prefill-only", ginkgo.Label(extendedTestLabel), func() { +var _ = ginkgo.Describe("Direct gateway /inference/v1/generate prefill against prefill-only", ginkgo.Label(extendedTestLabel), ginkgo.Ordered, testWrapper(func() { // Uses single-profile-handler (generatePrefillConfig) so the EPP routes // directly to prefill pods without requiring a decode stage first. ginkgo.It("returns kv_transfer_params for prefill bodies", func() { nsName := getNamespace() - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 modelServers := createModelServersPrefillOnly(prefillReplicas) epp := createEndPointPicker(generatePrefillConfig) - ginkgo.DeferCleanup(func() { - testutils.DeleteObjects(testConfig, epp, nsName) - testutils.DeleteObjects(testConfig, modelServers, nsName) - }) - prefillPods := getPodNames(prefillSelector) + prefillPods := getPodNames(prefillSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) ginkgo.By("TwoImages_Prefill: combined two-image prefill body returns kv_transfer_params") @@ -102,8 +98,12 @@ var _ = ginkgo.Describe("Direct gateway /inference/v1/generate prefill against p parsed := expectGenerateOK(resp, raw) expectKVTransferParams(parsed, raw) } + + testutils.DeleteObjects(testConfig, epp, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) -}) +})) // imageFeatures builds the features map that encode and prefill share: // mm_hashes, mm_placeholders, kwargs_data, all keyed by modality @@ -237,7 +237,7 @@ func expectKVTransferParams(parsed map[string]any, raw []byte) { "kv_transfer_params is empty: %s", string(raw)) } -var _ = ginkgo.Describe("P/D gateway /inference/v1/generate disaggregates via sidecar", ginkgo.Label(sharedStorageTestLabel, disaggTestLabel), func() { +var _ = ginkgo.Describe("P/D gateway /inference/v1/generate disaggregates via sidecar", ginkgo.Label(sharedStorageTestLabel, disaggTestLabel), ginkgo.Ordered, testWrapper(func() { // Regression test for https://github.com/llm-d/llm-d-router/issues/1461: // the pd-sidecar previously had no route for /inference/v1/generate, so // token-in P/D requests silently fell through to decode-only. This test @@ -246,18 +246,14 @@ var _ = ginkgo.Describe("P/D gateway /inference/v1/generate disaggregates via si // disaggregatedPrefillHandler rather than the decoder catch-all. ginkgo.It("routes token-in generate to the prefill pod", func() { nsName := getNamespace() - infPoolObjects = createInferencePool(1, true) + infPoolObjects := createInferencePool(1) prefillReplicas := 1 decodeReplicas := 1 modelServers := createModelServersPDSharedStorage(decodeReplicas) epp := createEndPointPicker(pdConfig) - ginkgo.DeferCleanup(func() { - testutils.DeleteObjects(testConfig, epp, nsName) - testutils.DeleteObjects(testConfig, modelServers, nsName) - }) - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas)) gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas)) @@ -275,8 +271,12 @@ var _ = ginkgo.Describe("P/D gateway /inference/v1/generate disaggregates via si gomega.Expect(prefillCountAfter).To(gomega.BeNumerically(">", prefillCountBefore), "prefill pod should have received the generate request; sidecar must route "+ "/inference/v1/generate through disaggregatedPrefillHandler, not the decoder catch-all") + + testutils.DeleteObjects(testConfig, epp, nsName) + testutils.DeleteObjects(testConfig, modelServers, nsName) + testutils.DeleteObjects(testConfig, infPoolObjects, nsName) }) -}) +})) // simpleTokenGenerateBody builds a minimal /inference/v1/generate body with // enough token IDs to exceed the prefix-based-pd-decider nonCachedTokens diff --git a/test/e2e/requests_test.go b/test/e2e/requests_test.go index ab53062b0c..e7af53b926 100644 --- a/test/e2e/requests_test.go +++ b/test/e2e/requests_test.go @@ -28,7 +28,7 @@ func extractInferenceHeaders(httpResp *http.Response) (string, string, string) { func generateAndCheckLoad(count int) { nsName := getNamespace() for range count { - prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName) gomega.Expect(prefillPods).Should(gomega.BeEmpty()) gomega.Expect(decodePods).Should(gomega.HaveLen(1)) @@ -310,7 +310,7 @@ func verifyMetrics(infPoolName string, numTargetPorts int) { gomega.Expect(theMetrics).ShouldNot(gomega.BeEmpty()) metricsAsString := strings.Join(theMetrics, "\n") - _, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) + _, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, getNamespace()) // Define the metrics we expect to see preset := []string{ //nolint:prealloc diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index d4ed7aba0e..d3efc76de6 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -10,6 +10,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -35,7 +36,7 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin "${VLLM_EXTRA_ARGS_E}": "--force-dummy-tokenizer", "${VLLM_EXTRA_ARGS_P}": "--force-dummy-tokenizer", "${VLLM_EXTRA_ARGS_D}": "--force-dummy-tokenizer", - "${VLLM_RENDER_URL}": fmt.Sprintf("http://vllm-render.%s.svc.cluster.local:%s", nsName, vllmRenderPort), + "${VLLM_RENDER_URL}": fmt.Sprintf("http://vllm-render.%s.svc.cluster.local:%s", baseNsName, vllmRenderPort), "${VLLM_RENDER_PORT}": vllmRenderPort, } for k, v := range extra { @@ -212,5 +213,80 @@ func createEndPointPickerHelper(eppConfig string, replicas int, isLeaderElection return append(objects, testutils.CreateObjsFromYaml(testConfig, eppYamls, nsName)...) } objs := testutils.CreateUnstructuredObjs(testConfig, eppYamls) - return append(objects, testutils.CreateObjsWithVerifier(testConfig, objs, nsName, func(kind string, clientObj client.Object) {})...) + objects = append(objects, testutils.CreateObjsWithVerifier(testConfig, objs, nsName, func(kind string, clientObj client.Object) {})...) + + gomega.Eventually(func() error { + _, _, err := tryCompletion(simplePrompt, simModelName) + return err + }, readyTimeout, 1*time.Second).Should(gomega.Succeed()) + + return objects +} + +// testWrapper wraps tests with the setup and teardown code needed. +// It is used as a wrapper of the function passed to ginkgo.When calls that +// setup the tests. It is important that the ginkgo.Ordered decorator is used +// to enable the use of the ginkgo.BeforeAll and ginkgo.AfterAll functions +// to inject the setup and teardown code. +func testWrapper(test func()) func() { + var ( + nsName string + createdNameSpace bool + + rbacObjects []string + serviceAccountObjects []string + serviceObjects []string + envoyObjects []string + portForwardSession *gexec.Session + ) + return func() { + ginkgo.BeforeAll(func() { + nsName = getNamespace() + createdNameSpace = setupNameSpace() + + envoyObjects, portForwardSession = createEnvoy(nsName) + + infraSubs := map[string]string{ + "${EPP_NAME}": "e2e-epp", + "${METRICS_NODE_PORT}": strconv.Itoa(getMetricsPort()), + } + rbacYamls := substituteMany(testutils.ReadYaml(rbacManifest), infraSubs) + rbacObjects = testutils.CreateObjsFromYaml(testConfig, rbacYamls, nsName) + saYamls := substituteMany(testutils.ReadYaml(serviceAccountManifest), infraSubs) + serviceAccountObjects = testutils.CreateObjsFromYaml(testConfig, saYamls, nsName) + svcYamls := substituteMany(testutils.ReadYaml(servicesManifest), infraSubs) + serviceObjects = testutils.CreateObjsFromYaml(testConfig, svcYamls, nsName) + }) + + ginkgo.AfterEach(func() { + // The starting of the EPP can launch a port-forwarder to access the metrics port. + if eppPortForwardSession != nil { + eppPortForwardSession.Terminate() + eppPortForwardSession = nil + } + }) + + ginkgo.AfterAll(func() { + if ginkgo.CurrentSpecReport().Failed() && keepClusterOnFailure { + // The test failed + testutils.DumpPodsAndLogs(testConfig, nsName) + } else { + // Only cleanup if the test succeeded + testutils.DeleteObjects(testConfig, rbacObjects, nsName) + testutils.DeleteObjects(testConfig, serviceObjects, nsName) + testutils.DeleteObjects(testConfig, serviceAccountObjects, nsName) + if portForwardSession != nil { + portForwardSession.Terminate() + portForwardSession = nil + } + testutils.DeleteObjects(testConfig, envoyObjects, nsName) + + if createdNameSpace { + deleteNameSpace(nsName) + } + } + }) + + test() + } } diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 2a1d7b4644..81189cff5a 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -52,10 +52,10 @@ func scaleDeployment(nsName string, objects []string, increment int) { } // getModelServerPods Returns the list of Prefill and Decode vLLM pods separately -func getModelServerPods(podLabels, prefillLabels, decodeLabels map[string]string) ([]string, []string) { +func getModelServerPods(podLabels, prefillLabels, decodeLabels map[string]string, nsName string) ([]string, []string) { ginkgo.By("Getting Model server pods") - pods := getPods(podLabels) + pods := getPods(podLabels, nsName) prefillValidator, err := apilabels.ValidatedSelectorFromSet(prefillLabels) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) @@ -90,10 +90,11 @@ func getModelServerPods(podLabels, prefillLabels, decodeLabels map[string]string return prefillPods, decodePods } -func getPods(labels map[string]string) []corev1.Pod { +func getPods(labels map[string]string, nsName string) []corev1.Pod { podList := corev1.PodList{} selector := apilabels.SelectorFromSet(labels) - err := testConfig.K8sClient.List(testConfig.Context, &podList, &client.ListOptions{LabelSelector: selector}) + err := testConfig.K8sClient.List(testConfig.Context, &podList, + &client.ListOptions{LabelSelector: selector, Namespace: nsName}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) pods := []corev1.Pod{} @@ -107,8 +108,8 @@ func getPods(labels map[string]string) []corev1.Pod { } // getPodNames returns the names of all running pods matching the given label selector. -func getPodNames(labels map[string]string) []string { - pods := getPods(labels) +func getPodNames(labels map[string]string, nsName string) []string { + pods := getPods(labels, nsName) names := make([]string, 0, len(pods)) for _, pod := range pods { names = append(names, pod.Name) diff --git a/test/scripts/e2e-common.sh b/test/scripts/e2e-common.sh index 91ab2d7854..3fed999fb6 100644 --- a/test/scripts/e2e-common.sh +++ b/test/scripts/e2e-common.sh @@ -35,8 +35,9 @@ run_ginkgo_suite() { local pkg="$1" if [ -n "${E2E_LABEL_FILTER:-}" ]; then echo "Label filter: ${E2E_LABEL_FILTER}" - go test -v -timeout 45m "${pkg}" -ginkgo.v -ginkgo.fail-fast "-ginkgo.label-filter=${E2E_LABEL_FILTER}" + # Tests not being run in parallel using Ginkgo + E2E_NUM_PROCS=1 go test -v -timeout 45m "${pkg}" -ginkgo.v -ginkgo.fail-fast "-ginkgo.label-filter=${E2E_LABEL_FILTER}" else - go test -v -timeout 45m "${pkg}" -ginkgo.v -ginkgo.fail-fast + ginkgo run --procs="${E2E_NUM_PROCS}" --timeout 45m -v --fail-fast "${pkg}" fi }