From 3a4e79bc61cc55f7d72979dac5b4569186489f60 Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Thu, 11 Jun 2026 09:21:51 +0300 Subject: [PATCH 1/8] deploy vllm render as a dedicated service, decoupled from EPP and simulator pods Signed-off-by: Maya Barnea --- .../inference-gateway/deployment.yaml | 31 ---- .../overlays/simulator/kustomization.yaml | 96 ------------ deploy/components/vllm-decode/deployment.yaml | 1 + deploy/components/vllm-encode/deployment.yaml | 1 + .../components/vllm-prefill/deployment.yaml | 1 + .../dev/e2e-infra/vllm-render.yaml | 72 +++++++++ deploy/environments/dev/epd/patch-decode.yaml | 1 + test/e2e/configs_test.go | 4 +- test/e2e/e2e_suite_test.go | 5 + test/e2e/setup_test.go | 42 ++--- test/e2e/utils_test.go | 147 ------------------ test/testdata/sim-deployment.yaml | 4 +- 12 files changed, 99 insertions(+), 306 deletions(-) create mode 100644 deploy/environments/dev/e2e-infra/vllm-render.yaml diff --git a/deploy/components/inference-gateway/deployment.yaml b/deploy/components/inference-gateway/deployment.yaml index 0e34342e4f..3117469d75 100644 --- a/deploy/components/inference-gateway/deployment.yaml +++ b/deploy/components/inference-gateway/deployment.yaml @@ -60,40 +60,9 @@ spec: mountPath: /etc/epp - name: cache mountPath: /cache - # vLLM render sidecar — exposes /v1/completions/render and - # /v1/chat/completions/render over loopback for the token-producer plugin. - - name: vllm-render - image: ${VLLM_RENDER_IMAGE} - imagePullPolicy: IfNotPresent - command: ["vllm", "launch", "render"] - args: ["${MODEL_NAME}", "--port=8000"] - ports: - - name: render-http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: 8000 - initialDelaySeconds: 2 - periodSeconds: 1 - timeoutSeconds: 1 - failureThreshold: 600 - readinessProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 5 - timeoutSeconds: 1 - failureThreshold: 1 - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface volumes: - name: epp-config configMap: name: epp-config - name: cache emptyDir: {} - # HuggingFace cache for the render sidecar's model weights. - - name: model-cache - emptyDir: {} diff --git a/deploy/components/overlays/simulator/kustomization.yaml b/deploy/components/overlays/simulator/kustomization.yaml index a250389271..b2cce3cb98 100644 --- a/deploy/components/overlays/simulator/kustomization.yaml +++ b/deploy/components/overlays/simulator/kustomization.yaml @@ -40,99 +40,3 @@ patches: - op: add path: /spec/template/spec/containers/0/args/4 value: "--zmq-endpoint=tcp://${EPP_NAME}.${NAMESPACE}.svc.cluster.local:5557" - - op: add - path: /spec/template/spec/initContainers/0 - value: - name: vllm-render - image: ${VLLM_RENDER_IMAGE} - imagePullPolicy: IfNotPresent - restartPolicy: Always - command: ["vllm"] - args: ["launch", "render", "${MODEL_NAME}", "--port=8082"] - env: - - name: HF_TOKEN - value: "${HF_TOKEN}" - ports: - - name: http - containerPort: 8082 - startupProbe: - httpGet: - path: /health - port: 8082 - initialDelaySeconds: 2 - periodSeconds: 1 - failureThreshold: 600 - timeoutSeconds: 1 - livenessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 1 - readinessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 5 - failureThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface - - op: add - path: /spec/template/spec/volumes/- - value: - name: model-cache - emptyDir: {} -# --- Prefill and Encode deployments (vllm-p, vllm-e) --- -# initContainers: [] in base, so append with /- -- target: - kind: Deployment - name: "vllm-[pe]" - patch: | - - op: add - path: /spec/template/spec/initContainers/- - value: - name: vllm-render - image: ${VLLM_RENDER_IMAGE} - imagePullPolicy: IfNotPresent - restartPolicy: Always - command: ["vllm"] - args: ["launch", "render", "${MODEL_NAME}", "--port=8082"] - env: - - name: HF_TOKEN - value: "${HF_TOKEN}" - ports: - - name: http - containerPort: 8082 - startupProbe: - httpGet: - path: /health - port: 8082 - initialDelaySeconds: 2 - periodSeconds: 1 - failureThreshold: 600 - timeoutSeconds: 1 - livenessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 1 - readinessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 5 - failureThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface - - op: add - path: /spec/template/spec/volumes/- - value: - name: model-cache - emptyDir: {} diff --git a/deploy/components/vllm-decode/deployment.yaml b/deploy/components/vllm-decode/deployment.yaml index 69920f8e0e..56af4be9ff 100644 --- a/deploy/components/vllm-decode/deployment.yaml +++ b/deploy/components/vllm-decode/deployment.yaml @@ -67,6 +67,7 @@ spec: - "--port=8200" - "--model=${MODEL_NAME}" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" + - "--render-url=http://vllm-render:8082" - "${VLLM_EXTRA_ARGS_D}" ports: - name: http diff --git a/deploy/components/vllm-encode/deployment.yaml b/deploy/components/vllm-encode/deployment.yaml index 927b13e5e5..ad233d1d3a 100644 --- a/deploy/components/vllm-encode/deployment.yaml +++ b/deploy/components/vllm-encode/deployment.yaml @@ -30,6 +30,7 @@ spec: - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - "--no-enable-prefix-caching" - "--mm-encoder-only" + - "--render-url=http://vllm-render:8082" - "${VLLM_EXTRA_ARGS_E}" ports: - name: encode-http diff --git a/deploy/components/vllm-prefill/deployment.yaml b/deploy/components/vllm-prefill/deployment.yaml index bf7d30dd4e..f2bb86d67d 100644 --- a/deploy/components/vllm-prefill/deployment.yaml +++ b/deploy/components/vllm-prefill/deployment.yaml @@ -28,6 +28,7 @@ spec: - "--model=${MODEL_NAME}" - "--port=8000" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" + - "--render-url=http://vllm-render:8082" - "${VLLM_EXTRA_ARGS_P}" ports: - name: prefill-http diff --git a/deploy/environments/dev/e2e-infra/vllm-render.yaml b/deploy/environments/dev/e2e-infra/vllm-render.yaml new file mode 100644 index 0000000000..4dbe264bd6 --- /dev/null +++ b/deploy/environments/dev/e2e-infra/vllm-render.yaml @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + replicas: 1 + selector: + matchLabels: + app: vllm-render + template: + metadata: + labels: + app: vllm-render + spec: + containers: + - name: vllm-render + image: ${VLLM_RENDER_IMAGE} + imagePullPolicy: IfNotPresent + command: ["vllm", "launch", "render"] + args: ["${MODEL_NAME}", "--port=8082"] + env: + - name: HF_TOKEN + value: "${HF_TOKEN}" + ports: + - name: http + containerPort: 8082 + startupProbe: + httpGet: + path: /health + port: 8082 + initialDelaySeconds: 2 + periodSeconds: 1 + failureThreshold: 600 + timeoutSeconds: 1 + livenessProbe: + httpGet: + path: /health + port: 8082 + periodSeconds: 10 + failureThreshold: 5 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /health + port: 8082 + periodSeconds: 5 + failureThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - name: model-cache + mountPath: /root/.cache/huggingface + volumes: + - name: model-cache + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + selector: + app: vllm-render + ports: + - name: http + protocol: TCP + port: 8082 + targetPort: 8082 + type: ClusterIP diff --git a/deploy/environments/dev/epd/patch-decode.yaml b/deploy/environments/dev/epd/patch-decode.yaml index e10a96beeb..498e79323d 100644 --- a/deploy/environments/dev/epd/patch-decode.yaml +++ b/deploy/environments/dev/epd/patch-decode.yaml @@ -22,6 +22,7 @@ spec: - "--block-size=16" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - "--zmq-endpoint=tcp://${EPP_NAME}.${NAMESPACE}.svc.cluster.local:5557" + - "--render-url=http://vllm-render:8082" - "${VLLM_EXTRA_ARGS_D}" ports: - $patch: replace diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index 6760a66242..1f0a30a257 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -187,7 +187,7 @@ plugins: parameters: modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: - url: http://localhost:8000 + url: http://vllm-render:8082 - type: precise-prefix-cache-scorer parameters: tokenProcessorConfig: @@ -219,7 +219,7 @@ plugins: parameters: modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: - url: http://localhost:8000 + url: http://vllm-render:8082 - type: precise-prefix-cache-scorer parameters: tokenProcessorConfig: diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 71076fc8ab..0bb118c6ce 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -54,6 +54,8 @@ const ( serviceAccountManifest = "../../deploy/components/inference-gateway/service-accounts.yaml" // servicesManifest is the manifest for the EPP's service resources. servicesManifest = "../../deploy/environments/dev/e2e-infra/services.yaml" + // renderManifest is the manifest for the standalone vLLM render deployment and service. + renderManifest = "../../deploy/environments/dev/e2e-infra/vllm-render.yaml" // CI shards scheduler e2e specs with label filters. extendedTestLabel = "Extended" @@ -95,6 +97,7 @@ var ( rbacObjects []string serviceAccountObjects []string serviceObjects []string + renderObjects []string infPoolObjects []string createdNameSpace bool @@ -133,6 +136,7 @@ var _ = ginkgo.BeforeSuite(func() { saYamls := substituteMany(testutils.ReadYaml(serviceAccountManifest), infraSubs) serviceAccountObjects = testutils.CreateObjsFromYaml(testConfig, saYamls, nsName) serviceObjects = testutils.ApplyYAMLFile(testConfig, servicesManifest, nsName) + renderObjects = createRender(nsName) // Prevent failure in tests due to InferencePool not existing before the test infPoolObjects = createInferencePool(1, false) @@ -190,6 +194,7 @@ var _ = ginkgo.ReportAfterSuite("cleanup", func(report ginkgo.Report) { nsName := getNamespace() ginkgo.By("Deleting created Kubernetes objects") testutils.DeleteObjects(testConfig, infPoolObjects, nsName) + testutils.DeleteObjects(testConfig, renderObjects, nsName) testutils.DeleteObjects(testConfig, serviceObjects, nsName) testutils.DeleteObjects(testConfig, serviceAccountObjects, nsName) testutils.DeleteObjects(testConfig, rbacObjects, nsName) diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 578e938475..13b3c85dfd 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -14,8 +14,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - configloader "github.com/llm-d/llm-d-router/pkg/epp/config/loader" - "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer" "github.com/llm-d/llm-d-router/pkg/sidecar/proxy" testutils "github.com/llm-d/llm-d-router/test/utils" ) @@ -26,7 +24,6 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin "${MODEL_NAME}": simModelName, "${POOL_NAME}": poolName, "${VLLM_IMAGE}": vllmSimImage, - "${VLLM_RENDER_IMAGE}": vllmRenderImage, "${SIDECAR_IMAGE}": sideCarImage, "${VLLM_DATA_PARALLEL_SIZE}": "1", "${VLLM_SIM_MODE}": "echo", @@ -48,10 +45,6 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin // Remove labels with empty values (produced when ${DECODE_ROLE} is empty) manifests = removeEmptyLabels(manifests) manifests = removeEmptyArgs(manifests) - // remove render sidecar if model is simulated - if !isModelReal(subs["${MODEL_NAME}"]) { - manifests = removeRenderSidecar(manifests) - } objects := testutils.CreateObjsFromYaml(testConfig, manifests, nsName) podsInDeploymentsReady(nsName, objects) return objects @@ -130,6 +123,18 @@ func createModelServersEPDUnified(replicas int) []string { }) } +func createRender(nsName string) []string { + renderYamls := substituteMany(testutils.ReadYaml(renderManifest), + map[string]string{ + "${MODEL_NAME}": kvModelName, + "${VLLM_RENDER_IMAGE}": vllmRenderImage, + "${HF_TOKEN}": "", + }) + objects := testutils.CreateObjsFromYaml(testConfig, renderYamls, nsName) + podsInDeploymentsReady(objects) + return objects +} + func createEndPointPicker(eppConfig string) []string { objects := createEndPointPickerHelper(eppConfig, 1, false, true) podsInDeploymentsReady(getNamespace(), objects) @@ -175,22 +180,14 @@ func createEndPointPickerHelper(eppConfig string, replicas int, isLeaderElection eppYamls := testutils.ReadYaml(eppManifest) eppYamls = substituteMany(eppYamls, map[string]string{ - "${EPP_NAME}": eppName, - "${EPP_IMAGE}": eppImage, - "${VLLM_RENDER_IMAGE}": vllmRenderImage, - // The render sidecar needs a real, fetchable model. Sim tests - // don't query it; the cost is paying weights-load on every EPP. - "${MODEL_NAME}": kvModelName, + "${EPP_NAME}": eppName, + "${EPP_IMAGE}": eppImage, "${NAMESPACE}": nsName, "${POOL_NAME}": simModelName + "-inference-pool", "${METRICS_ENDPOINT_AUTH}": "false", "${EPP_REPLICA_COUNT}": strconv.Itoa(replicas), "${ENABLE_LEADER_ELECTION}": strconv.FormatBool(isLeaderElectionEnabled), }) - if !usesTokenProducer(eppConfig) { - eppYamls = removeRenderSidecar(eppYamls) - } - eppYamls = appendEppArgs(eppYamls, eppExtraArgs) if waitForReady { return append(objects, testutils.CreateObjsFromYaml(testConfig, eppYamls, nsName)...) @@ -198,14 +195,3 @@ func createEndPointPickerHelper(eppConfig string, replicas int, isLeaderElection objs := testutils.CreateUnstructuredObjs(testConfig, eppYamls) return append(objects, testutils.CreateObjsWithVerifier(testConfig, objs, nsName, func(kind string, clientObj client.Object) {})...) } - -func usesTokenProducer(eppConfig string) bool { - cfg, _, err := configloader.LoadRawConfig([]byte(eppConfig), ginkgo.GinkgoLogr) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - for _, plugin := range cfg.Plugins { - if plugin.Type == tokenizer.PluginType { - return true - } - } - return false -} diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 4e3195e3d4..8005cbfe90 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/http" - "os" "os/exec" "strconv" "strings" @@ -18,11 +17,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" apilabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" ) const ( @@ -212,150 +209,6 @@ func removeEmptyLabels(inputs []string) []string { return outputs } -func isModelReal(modelName string) bool { - req, err := http.NewRequest("GET", "https://huggingface.co/api/models/"+modelName, nil) - if err != nil { - return false - } - if token := os.Getenv("HF_TOKEN"); token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return false - } - defer resp.Body.Close() - - return resp.StatusCode == http.StatusOK -} - -// removeRenderSidecar takes a slice of YAML strings (each may contain multiple -// objects separated by "---") and returns the same slice with the vllm-render -// container and the model-cache volume stripped from any Deployment. -func removeRenderSidecar(inputs []string) []string { - outputs := make([]string, len(inputs)) - for idx, input := range inputs { - docs := strings.Split(input, "\n---") - rendered := make([]string, 0, len(docs)) - for _, doc := range docs { - if strings.TrimSpace(doc) == "" { - continue - } - rendered = append(rendered, filterDocument(doc)) - } - outputs[idx] = strings.Join(rendered, "\n---\n") - } - return outputs -} - -func filterDocument(doc string) string { - obj := &unstructured.Unstructured{} - err := yaml.Unmarshal([]byte(doc), &obj.Object) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - if len(obj.Object) == 0 { - return doc - } - if obj.GetKind() == kubernetesDeploymentKind { - removePodSpecListItem(obj, "containers", "vllm-render") - removePodSpecListItem(obj, "initContainers", "vllm-render") - removePodSpecListItem(obj, "volumes", "model-cache") - } - out, err := yaml.Marshal(obj.Object) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - return strings.TrimRight(string(out), "\n") -} - -func removePodSpecListItem(obj *unstructured.Unstructured, fieldName, itemName string) { - path := []string{"spec", "template", "spec", fieldName} - items, found, err := unstructured.NestedSlice(obj.Object, path...) - if err != nil || !found { - return - } - filtered := make([]any, 0, len(items)) - for _, item := range items { - if m, ok := item.(map[string]any); ok && m["name"] == itemName { - continue - } - filtered = append(filtered, item) - } - if len(filtered) == 0 { - unstructured.RemoveNestedField(obj.Object, path...) - return - } - err = unstructured.SetNestedSlice(obj.Object, filtered, path...) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) -} - -// eppExtraArgs are appended to the "epp" container's args in the Deployment -// created by createEndPointPickerHelper. -// -// Graceful drain is disabled (--drain-timeout=0) for all e2e EPPs. The suites -// create and delete the shared "e2e-epp" Deployment in sequence behind a single -// Envoy; with a drain window a deleted EPP keeps serving ext_proc on its -// existing connection, so Envoy lingers on the terminating pod (stale datastore) -// and the next spec's requests fail. The graceful-drain behavior itself is -// covered by unit tests. -var eppExtraArgs = []string{"--drain-timeout=0"} - -// appendEppArgs returns the input YAML docs with args appended to the "epp" -// container of any Deployment. It is a no-op when args is empty. -func appendEppArgs(inputs []string, args []string) []string { - if len(args) == 0 { - return inputs - } - outputs := make([]string, len(inputs)) - for idx, input := range inputs { - docs := strings.Split(input, "\n---") - rendered := make([]string, 0, len(docs)) - for _, doc := range docs { - if strings.TrimSpace(doc) == "" { - continue - } - rendered = append(rendered, appendArgsToEppContainer(doc, args)) - } - outputs[idx] = strings.Join(rendered, "\n---\n") - } - return outputs -} - -func appendArgsToEppContainer(doc string, args []string) string { - obj := &unstructured.Unstructured{} - err := yaml.Unmarshal([]byte(doc), &obj.Object) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - if len(obj.Object) == 0 || obj.GetKind() != kubernetesDeploymentKind { - return doc - } - path := []string{"spec", "template", "spec", "containers"} - containers, found, err := unstructured.NestedSlice(obj.Object, path...) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - if !found { - return doc - } - for i, c := range containers { - m, ok := c.(map[string]any) - if !ok || m["name"] != "epp" { - continue - } - existing, _, err := unstructured.NestedStringSlice(m, "args") - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - merged := make([]any, 0, len(existing)+len(args)) - for _, a := range existing { - merged = append(merged, a) - } - for _, a := range args { - merged = append(merged, a) - } - m["args"] = merged - containers[i] = m - } - err = unstructured.SetNestedSlice(obj.Object, containers, path...) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - out, err := yaml.Marshal(obj.Object) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - return strings.TrimRight(string(out), "\n") -} - func substituteMany(inputs []string, substitutions map[string]string) []string { outputs := make([]string, len(inputs)) for idx, input := range inputs { diff --git a/test/testdata/sim-deployment.yaml b/test/testdata/sim-deployment.yaml index 0292281317..c75cb9853a 100644 --- a/test/testdata/sim-deployment.yaml +++ b/test/testdata/sim-deployment.yaml @@ -19,7 +19,7 @@ spec: imagePullPolicy: IfNotPresent restartPolicy: Always command: ["vllm"] - args: ["launch", "render", "Qwen/Qwen3-32B", "--port=8082"] + args: ["launch", "render", "Qwen/Qwen2.5-1.5B-Instruct", "--port=8082"] ports: - name: http containerPort: 8082 @@ -57,7 +57,7 @@ spec: imagePullPolicy: IfNotPresent args: - --model - - Qwen/Qwen3-32B + - Qwen/Qwen2.5-1.5B-Instruct - --port - "8000" - --max-loras From acfe9ea9e6e38ca0dc46bef1c531ea0c7979950f Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Thu, 11 Jun 2026 14:15:14 +0300 Subject: [PATCH 2/8] use simulated render for e2e test Signed-off-by: Maya Barnea --- .../dev/base-kind-istio/kustomization.yaml | 1 + .../dev/base-kind-istio/vllm-render.yaml | 69 +++++++++++++++++++ .../dev/e2e-infra/vllm-render.yaml | 17 ++--- test/e2e/e2e_suite_test.go | 2 +- test/e2e/setup_test.go | 7 +- test/scripts/test-e2e-router.sh | 2 + 6 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 deploy/environments/dev/base-kind-istio/vllm-render.yaml diff --git a/deploy/environments/dev/base-kind-istio/kustomization.yaml b/deploy/environments/dev/base-kind-istio/kustomization.yaml index eda61c4f6d..9cffaff0bb 100644 --- a/deploy/environments/dev/base-kind-istio/kustomization.yaml +++ b/deploy/environments/dev/base-kind-istio/kustomization.yaml @@ -16,6 +16,7 @@ kind: Kustomization resources: - destination-rules.yaml - services.yaml +- vllm-render.yaml - ../../../components/istio-control-plane/ - ../../../components/inference-gateway/ diff --git a/deploy/environments/dev/base-kind-istio/vllm-render.yaml b/deploy/environments/dev/base-kind-istio/vllm-render.yaml new file mode 100644 index 0000000000..05e2413caa --- /dev/null +++ b/deploy/environments/dev/base-kind-istio/vllm-render.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + replicas: 1 + selector: + matchLabels: + app: vllm-render + template: + metadata: + labels: + app: vllm-render + spec: + containers: + - name: vllm-render + image: ${VLLM_RENDER_IMAGE} + imagePullPolicy: IfNotPresent + command: ["vllm", "launch", "render"] + args: ["${MODEL_NAME}", "--port=8082"] + ports: + - name: render-http + containerPort: 8082 + startupProbe: + httpGet: + path: /health + port: 8082 + initialDelaySeconds: 2 + periodSeconds: 1 + failureThreshold: 600 + timeoutSeconds: 1 + livenessProbe: + httpGet: + path: /health + port: 8082 + periodSeconds: 10 + failureThreshold: 5 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /health + port: 8082 + periodSeconds: 5 + failureThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - name: model-cache + mountPath: /root/.cache/huggingface + volumes: + - name: model-cache + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + selector: + app: vllm-render + ports: + - name: http + protocol: TCP + port: 8082 + targetPort: 8082 + type: ClusterIP diff --git a/deploy/environments/dev/e2e-infra/vllm-render.yaml b/deploy/environments/dev/e2e-infra/vllm-render.yaml index 4dbe264bd6..6023dd1ea7 100644 --- a/deploy/environments/dev/e2e-infra/vllm-render.yaml +++ b/deploy/environments/dev/e2e-infra/vllm-render.yaml @@ -18,11 +18,11 @@ spec: - name: vllm-render image: ${VLLM_RENDER_IMAGE} imagePullPolicy: IfNotPresent - command: ["vllm", "launch", "render"] - args: ["${MODEL_NAME}", "--port=8082"] - env: - - name: HF_TOKEN - value: "${HF_TOKEN}" + args: + - "--model=${MODEL_NAME}" + - "--port=8082" + - "--mode=echo" + - "--force-dummy-tokenizer" ports: - name: http containerPort: 8082 @@ -48,12 +48,7 @@ spec: periodSeconds: 5 failureThreshold: 1 timeoutSeconds: 1 - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface - volumes: - - name: model-cache - emptyDir: {} + volumes: [] --- apiVersion: v1 kind: Service diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0bb118c6ce..f31ce34fb1 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -247,7 +247,7 @@ func setupK8sCluster() { kindLoadImage(vllmSimImage) kindLoadImage(eppImage) kindLoadImage(sideCarImage) - if loadRenderImage { + if loadRenderImage && vllmRenderImage != vllmSimImage { kindLoadImage(vllmRenderImage) } } diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 13b3c85dfd..65e5dfbcc4 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -32,9 +32,9 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin "${EPP_NAME}": "e2e-epp", "${NAMESPACE}": nsName, "${HF_TOKEN}": os.Getenv("HF_TOKEN"), - "${VLLM_EXTRA_ARGS_E}": "", - "${VLLM_EXTRA_ARGS_P}": "", - "${VLLM_EXTRA_ARGS_D}": "", + "${VLLM_EXTRA_ARGS_E}": "--force-dummy-tokenizer", + "${VLLM_EXTRA_ARGS_P}": "--force-dummy-tokenizer", + "${VLLM_EXTRA_ARGS_D}": "--force-dummy-tokenizer", } for k, v := range extra { subs[k] = v @@ -128,7 +128,6 @@ func createRender(nsName string) []string { map[string]string{ "${MODEL_NAME}": kvModelName, "${VLLM_RENDER_IMAGE}": vllmRenderImage, - "${HF_TOKEN}": "", }) objects := testutils.CreateObjsFromYaml(testConfig, renderYamls, nsName) podsInDeploymentsReady(objects) diff --git a/test/scripts/test-e2e-router.sh b/test/scripts/test-e2e-router.sh index 491cbdda3e..623f9a1ade 100755 --- a/test/scripts/test-e2e-router.sh +++ b/test/scripts/test-e2e-router.sh @@ -31,4 +31,6 @@ trap 'e2e_handle_interrupt "e2e-tests"' INT TERM echo "Running end to end tests" +export VLLM_RENDER_IMAGE=${VLLM_IMAGE} + run_ginkgo_suite "${DIR}/../e2e/" From 6d4eff10c80ea9da7d8dba5de9758feeb12a09e2 Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Wed, 17 Jun 2026 14:58:21 +0300 Subject: [PATCH 3/8] update readme Signed-off-by: Maya Barnea --- DEVELOPMENT.md | 4 ++-- deploy/environments/dev/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c465fc634e..8ec6847802 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -447,7 +447,7 @@ The `deploy/components/` directory contains all reusable Kustomize components: - `vllm-decode/` — Decode pod with routing sidecar **Deployment overlays** — applied on top of the base components: -- `overlays/simulator/` — adds `--mode=${VLLM_SIM_MODE}`, vllm renderer sidecar, KV cache args, and `--zmq-endpoint` on Decode. Included by default in all dev scenario overlays. +- `overlays/simulator/` — adds `--mode=${VLLM_SIM_MODE}`, KV cache args, and `--zmq-endpoint` on Decode. Included by default in all dev scenario overlays. - `overlays/real-vllm/` — adds `--kv-events-config` on Decode, `--ec-transfer-config` on Encode, and a shared PVC for encoder embeddings. **Infrastructure components** — shared cluster infrastructure: @@ -523,7 +523,7 @@ a shared PVC (`ec-cache-pvc`) for encoder embeddings transfer. | Component | What it adds | When to use | |---|---|---| -| `overlays/simulator/` | `--mode=${VLLM_SIM_MODE}`, vLLM renderer, KV cache args, `--zmq-endpoint` on Decode | Dev/test with simulator image | +| `overlays/simulator/` | `--mode=${VLLM_SIM_MODE}`, KV cache args, `--zmq-endpoint` on Decode | Dev/test with simulator image | | `overlays/real-vllm/` | `--kv-events-config` on Decode (per-pod ZMQ publisher), `--ec-transfer-config` on Encode, ec-cache PVC | Production with real vLLM image | | Variable | Default | Description | diff --git a/deploy/environments/dev/README.md b/deploy/environments/dev/README.md index 7fe9ef9174..87668d6159 100644 --- a/deploy/environments/dev/README.md +++ b/deploy/environments/dev/README.md @@ -9,7 +9,7 @@ scenario-specific patches. The atomic components live in `deploy/components/`: | `vllm-decode/` | Decode pod — always deployed, includes routing sidecar (removed in EPD scenario) | | `vllm-prefill/` | Prefill pod — deployed when `DISAGG_P=true` | | `vllm-encode/` | Encoder pod — deployed when `DISAGG_E=true` | -| `overlays/simulator/` | Adds `--mode=${VLLM_SIM_MODE}`, vLLM render, KV cache and ZMQ args (included by all scenario overlays) | +| `overlays/simulator/` | Adds `--mode=${VLLM_SIM_MODE}`, KV cache and ZMQ args (included by all scenario overlays) | These overlays are used by both `scripts/kind-dev-env.sh` (for local KIND clusters) and e2e tests (via `kustomize build` + env var substitution). From b077a20781002a3cd42d6d01714046be3af4c7a5 Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Mon, 22 Jun 2026 14:10:31 +0300 Subject: [PATCH 4/8] review - use VLLM_RENDER_URL env variable, placeholder for hf token Signed-off-by: Maya Barnea --- Makefile | 2 ++ deploy/components/vllm-decode/deployment.yaml | 2 +- deploy/components/vllm-encode/deployment.yaml | 2 +- .../components/vllm-prefill/deployment.yaml | 2 +- deploy/environments/dev/README.md | 2 ++ .../dev/base-kind-istio/vllm-render.yaml | 23 ++++++++++++------- .../dev/e2e-infra/vllm-render.yaml | 15 ++++++------ deploy/environments/dev/epd/patch-decode.yaml | 2 +- scripts/kind-dev-env.sh | 6 +++-- test/e2e/e2e_suite_test.go | 4 ++++ test/e2e/setup_test.go | 1 + 11 files changed, 39 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index a4deaf313d..95977cb3e3 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,8 @@ export VLLM_IMAGE ?= $(VLLM_SIMULATOR_TAG_BASE):$(VLLM_SIMULATOR_TAG) # CPU-only vLLM image that exposes `vllm launch render` for the token-producer # plugin's HTTP backend. export VLLM_RENDER_IMAGE ?= vllm/vllm-openai-cpu:v0.21.0 +export VLLM_RENDER_PORT ?= 8082 +export VLLM_RENDER_URL ?= http://vllm-render:$(VLLM_RENDER_PORT) BUILDER_TAG ?= dev BUILDER_TAG_BASE ?= $(IMAGE_REGISTRY)/$(BUILDER_IMAGE_NAME) diff --git a/deploy/components/vllm-decode/deployment.yaml b/deploy/components/vllm-decode/deployment.yaml index 56af4be9ff..dd6715fcf9 100644 --- a/deploy/components/vllm-decode/deployment.yaml +++ b/deploy/components/vllm-decode/deployment.yaml @@ -67,7 +67,7 @@ spec: - "--port=8200" - "--model=${MODEL_NAME}" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - - "--render-url=http://vllm-render:8082" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_D}" ports: - name: http diff --git a/deploy/components/vllm-encode/deployment.yaml b/deploy/components/vllm-encode/deployment.yaml index ad233d1d3a..6c287a684a 100644 --- a/deploy/components/vllm-encode/deployment.yaml +++ b/deploy/components/vllm-encode/deployment.yaml @@ -30,7 +30,7 @@ spec: - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - "--no-enable-prefix-caching" - "--mm-encoder-only" - - "--render-url=http://vllm-render:8082" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_E}" ports: - name: encode-http diff --git a/deploy/components/vllm-prefill/deployment.yaml b/deploy/components/vllm-prefill/deployment.yaml index f2bb86d67d..a97a7b6464 100644 --- a/deploy/components/vllm-prefill/deployment.yaml +++ b/deploy/components/vllm-prefill/deployment.yaml @@ -28,7 +28,7 @@ spec: - "--model=${MODEL_NAME}" - "--port=8000" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - - "--render-url=http://vllm-render:8082" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_P}" ports: - name: prefill-http diff --git a/deploy/environments/dev/README.md b/deploy/environments/dev/README.md index 87668d6159..0f77ea988f 100644 --- a/deploy/environments/dev/README.md +++ b/deploy/environments/dev/README.md @@ -70,6 +70,8 @@ Variables substituted at deploy time via `envsubst` or Go test `substituteMany`: | `VLLM_IMAGE` | vLLM container image (simulator or real) | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` | | `SIDECAR_IMAGE` | Routing sidecar image | `ghcr.io/llm-d/llm-d-router-disagg-sidecar:dev` | | `VLLM_RENDER_IMAGE` | vLLM render sidecar image | `vllm/vllm-openai-cpu:v0.21.0` | +| `VLLM_RENDER_PORT` | Port the vllm-render Service listens on | `8082` | +| `VLLM_RENDER_URL` | URL of the vllm-render Service used by decode/encode/prefill pods | `http://vllm-render:${VLLM_RENDER_PORT}` | | `MODEL_NAME` | Model name passed to vLLM. Can be a real HuggingFace model (e.g. `TinyLlama/TinyLlama-1.1B-Chat-v1.0`, `Qwen/Qwen3-VL-2B-Instruct`) or an arbitrary name when using the simulator (e.g. `food-review`) | `food-review` | | `POOL_NAME` | InferencePool name | `food-review-inference-pool` | | `VLLM_REPLICA_COUNT_E` | Encode deployment replicas | `1` | diff --git a/deploy/environments/dev/base-kind-istio/vllm-render.yaml b/deploy/environments/dev/base-kind-istio/vllm-render.yaml index 05e2413caa..b09d71b204 100644 --- a/deploy/environments/dev/base-kind-istio/vllm-render.yaml +++ b/deploy/environments/dev/base-kind-istio/vllm-render.yaml @@ -19,14 +19,21 @@ spec: image: ${VLLM_RENDER_IMAGE} imagePullPolicy: IfNotPresent command: ["vllm", "launch", "render"] - args: ["${MODEL_NAME}", "--port=8082"] + args: ["${MODEL_NAME}", "--port=${VLLM_RENDER_PORT}"] + # For gated models, inject HF_TOKEN via a Secret reference: + # env: + # - name: HF_TOKEN + # valueFrom: + # secretKeyRef: + # name: hf-token + # key: token ports: - name: render-http - containerPort: 8082 + containerPort: ${VLLM_RENDER_PORT} startupProbe: httpGet: path: /health - port: 8082 + port: ${VLLM_RENDER_PORT} initialDelaySeconds: 2 periodSeconds: 1 failureThreshold: 600 @@ -34,21 +41,21 @@ spec: livenessProbe: httpGet: path: /health - port: 8082 + port: ${VLLM_RENDER_PORT} periodSeconds: 10 failureThreshold: 5 timeoutSeconds: 1 readinessProbe: httpGet: path: /health - port: 8082 + port: ${VLLM_RENDER_PORT} periodSeconds: 5 failureThreshold: 1 timeoutSeconds: 1 volumeMounts: - name: model-cache mountPath: /root/.cache/huggingface - volumes: + volumes: - name: model-cache emptyDir: {} --- @@ -64,6 +71,6 @@ spec: ports: - name: http protocol: TCP - port: 8082 - targetPort: 8082 + port: ${VLLM_RENDER_PORT} + targetPort: ${VLLM_RENDER_PORT} type: ClusterIP diff --git a/deploy/environments/dev/e2e-infra/vllm-render.yaml b/deploy/environments/dev/e2e-infra/vllm-render.yaml index 6023dd1ea7..f3b425f562 100644 --- a/deploy/environments/dev/e2e-infra/vllm-render.yaml +++ b/deploy/environments/dev/e2e-infra/vllm-render.yaml @@ -20,16 +20,16 @@ spec: imagePullPolicy: IfNotPresent args: - "--model=${MODEL_NAME}" - - "--port=8082" + - "--port=${VLLM_RENDER_PORT}" - "--mode=echo" - "--force-dummy-tokenizer" ports: - name: http - containerPort: 8082 + containerPort: ${VLLM_RENDER_PORT} startupProbe: httpGet: path: /health - port: 8082 + port: ${VLLM_RENDER_PORT} initialDelaySeconds: 2 periodSeconds: 1 failureThreshold: 600 @@ -37,18 +37,17 @@ spec: livenessProbe: httpGet: path: /health - port: 8082 + port: ${VLLM_RENDER_PORT} periodSeconds: 10 failureThreshold: 5 timeoutSeconds: 1 readinessProbe: httpGet: path: /health - port: 8082 + port: ${VLLM_RENDER_PORT} periodSeconds: 5 failureThreshold: 1 timeoutSeconds: 1 - volumes: [] --- apiVersion: v1 kind: Service @@ -62,6 +61,6 @@ spec: ports: - name: http protocol: TCP - port: 8082 - targetPort: 8082 + port: ${VLLM_RENDER_PORT} + targetPort: ${VLLM_RENDER_PORT} type: ClusterIP diff --git a/deploy/environments/dev/epd/patch-decode.yaml b/deploy/environments/dev/epd/patch-decode.yaml index 498e79323d..a48914ae15 100644 --- a/deploy/environments/dev/epd/patch-decode.yaml +++ b/deploy/environments/dev/epd/patch-decode.yaml @@ -22,7 +22,7 @@ spec: - "--block-size=16" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - "--zmq-endpoint=tcp://${EPP_NAME}.${NAMESPACE}.svc.cluster.local:5557" - - "--render-url=http://vllm-render:8082" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_D}" ports: - $patch: replace diff --git a/scripts/kind-dev-env.sh b/scripts/kind-dev-env.sh index 91edb0147d..611eee3e48 100755 --- a/scripts/kind-dev-env.sh +++ b/scripts/kind-dev-env.sh @@ -65,6 +65,8 @@ export SIDECAR_IMAGE # Set a default VLLM_RENDER_IMAGE if not provided (CPU-only vLLM image that # runs `vllm launch render` for the token-producer plugin's HTTP backend). export VLLM_RENDER_IMAGE="${VLLM_RENDER_IMAGE:-vllm/vllm-openai-cpu:v0.21.0}" +export VLLM_RENDER_PORT="${VLLM_RENDER_PORT:-8082}" +export VLLM_RENDER_URL="${VLLM_RENDER_URL:-http://vllm-render:${VLLM_RENDER_PORT}}" # Set the inference pool name for the deployment export POOL_NAME="${POOL_NAME:-${MODEL_NAME_SAFE}-inference-pool}" @@ -414,7 +416,7 @@ export ENABLE_LEADER_ELECTION=false # Deploy Istio base (shared infrastructure) kubectl kustomize --enable-helm deploy/environments/dev/base-kind-istio \ | envsubst '${POOL_NAME} ${MODEL_NAME} ${MODEL_NAME_SAFE} ${EPP_NAME} ${EPP_IMAGE} ${VLLM_IMAGE} \ - ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${TARGET_PORTS} ${NAMESPACE} ${METRICS_ENDPOINT_AUTH} \ + ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${VLLM_RENDER_PORT} ${VLLM_RENDER_URL} ${TARGET_PORTS} ${NAMESPACE} ${METRICS_ENDPOINT_AUTH} \ ${EPP_REPLICA_COUNT} ${VLLM_REPLICA_COUNT_E} ${VLLM_REPLICA_COUNT_P} ${VLLM_REPLICA_COUNT_D} \ ${VLLM_DATA_PARALLEL_SIZE} ${ENABLE_LEADER_ELECTION}' \ | kubectl --context ${KUBE_CONTEXT} apply -f - @@ -422,7 +424,7 @@ kubectl kustomize --enable-helm deploy/environments/dev/base-kind-istio \ # Deploy scenario-specific vLLM components kubectl kustomize --enable-helm ${KUSTOMIZE_DIR} \ | envsubst '${POOL_NAME} ${MODEL_NAME} ${MODEL_NAME_SAFE} ${EPP_NAME} ${EPP_IMAGE} ${VLLM_IMAGE} \ - ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${TARGET_PORTS} ${NAMESPACE} \ + ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${VLLM_RENDER_PORT} ${VLLM_RENDER_URL} ${TARGET_PORTS} ${NAMESPACE} \ ${VLLM_REPLICA_COUNT_E} ${VLLM_REPLICA_COUNT_P} ${VLLM_REPLICA_COUNT_D} ${VLLM_DATA_PARALLEL_SIZE} \ ${KV_CONNECTOR_TYPE} ${EC_CONNECTOR_TYPE} ${CONNECTOR_TYPE} ${KV_CACHE_ENABLED} ${HF_TOKEN} ${VLLM_SIM_MODE} \ ${DECODE_ROLE} ${VLLM_EXTRA_ARGS_E} ${VLLM_EXTRA_ARGS_P} ${VLLM_EXTRA_ARGS_D}' \ diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f31ce34fb1..4046b7672e 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -64,6 +64,9 @@ const ( metricsTestLabel = "Metrics" deprecatedPDTestLabel = "DeprecatedPD" disaggTestLabel = "Disagg" + + // images + simulatorImg = "ghcr.io/llm-d/llm-d-inference-sim:v0.9.2" ) var ( @@ -81,6 +84,7 @@ var ( vllmSimImage = env.GetEnvString("VLLM_IMAGE", "ghcr.io/llm-d/llm-d-inference-sim:v0.10.2", ginkgo.GinkgoLogr) sideCarImage = env.GetEnvString("SIDECAR_IMAGE", "ghcr.io/llm-d/llm-d-router-disagg-sidecar:dev", ginkgo.GinkgoLogr) vllmRenderImage = env.GetEnvString("VLLM_RENDER_IMAGE", "vllm/vllm-openai-cpu:v0.21.0", ginkgo.GinkgoLogr) + vllmRenderPort = env.GetEnvString("VLLM_RENDER_PORT", "8082", ginkgo.GinkgoLogr) loadRenderImage = env.GetEnvBool("LOAD_VLLM_RENDER_IMAGE", true, ginkgo.GinkgoLogr) numProcesses = env.GetEnvInt("E2E_NUM_PROCS", 1, ginkgo.GinkgoLogr) // baseNsName is the base of the namespace in which the K8S objects will be created diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 65e5dfbcc4..ed2f90c3f0 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -128,6 +128,7 @@ func createRender(nsName string) []string { map[string]string{ "${MODEL_NAME}": kvModelName, "${VLLM_RENDER_IMAGE}": vllmRenderImage, + "${VLLM_RENDER_PORT}": vllmRenderPort, }) objects := testutils.CreateObjsFromYaml(testConfig, renderYamls, nsName) podsInDeploymentsReady(objects) From f9377bc572a2de011e9c95ab1da97e31d09df884 Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Tue, 7 Jul 2026 14:54:38 +0300 Subject: [PATCH 5/8] review Signed-off-by: Maya Barnea --- test/e2e/configs_test.go | 24 ++++++++++++++++-------- test/e2e/e2e_test.go | 6 +++++- test/e2e/setup_test.go | 2 ++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index 1f0a30a257..0afbb4abba 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -1,5 +1,7 @@ package e2e +import "fmt" + // Simple EPP configuration for running without P/D const simpleConfig = `apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -179,15 +181,17 @@ schedulingProfiles: weight: 2 ` -// EPP config for running with precise prefix scoring (i.e. KV events). -const kvConfig = `apiVersion: llm-d.ai/v1alpha1 +// kvConfig returns the EPP config for running with precise prefix scoring (i.e. KV events). +// The render URL is built from vllmRenderPort so VLLM_RENDER_PORT is respected. +func kvConfig() string { + return fmt.Sprintf(`apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig plugins: - type: token-producer parameters: modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: - url: http://vllm-render:8082 + url: http://vllm-render:%s - type: precise-prefix-cache-scorer parameters: tokenProcessorConfig: @@ -209,17 +213,20 @@ schedulingProfiles: - pluginRef: max-score-picker - pluginRef: precise-prefix-cache-scorer weight: 10 -` +`, vllmRenderPort) +} -// Alias of kvConfig retained for tests that reference the external-tokenizer name. -const kvExternalTokenizerConfig = `apiVersion: llm-d.ai/v1alpha1 +// kvExternalTokenizerConfig returns the EPP config for the external-tokenizer DataProducer variant. +// The render URL is built from vllmRenderPort so VLLM_RENDER_PORT is respected. +func kvExternalTokenizerConfig() string { + return fmt.Sprintf(`apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig plugins: - type: token-producer parameters: modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: - url: http://vllm-render:8082 + url: http://vllm-render:%s - type: precise-prefix-cache-scorer parameters: tokenProcessorConfig: @@ -240,7 +247,8 @@ schedulingProfiles: - pluginRef: max-score-picker - pluginRef: precise-prefix-cache-scorer weight: 10 -` +`, vllmRenderPort) +} // EPP configuration for running scale model server test const scaleConfig = `apiVersion: llm-d.ai/v1alpha1 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 428a6821a6..21d8e33c7a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -784,7 +784,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { infPoolObjects = createInferencePool(1, true) modelServers := createModelServersDecodeKV(1) - epp := createEndPointPicker(kvConfig) + epp := createEndPointPicker(kvConfig()) nsName := getNamespace() prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) @@ -806,8 +806,12 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { ginkgo.It("should run successfully", func() { infPoolObjects = createInferencePool(1, true) +<<<<<<< HEAD modelServers := createModelServersDecodeKV(1) epp := createEndPointPicker(kvExternalTokenizerConfig) +======= + epp := createEndPointPicker(kvExternalTokenizerConfig()) +>>>>>>> 574c8b00 (review) nsName := getNamespace() prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index ed2f90c3f0..807ac2c510 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -35,6 +35,8 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin "${VLLM_EXTRA_ARGS_E}": "--force-dummy-tokenizer", "${VLLM_EXTRA_ARGS_P}": "--force-dummy-tokenizer", "${VLLM_EXTRA_ARGS_D}": "--force-dummy-tokenizer", + "${VLLM_RENDER_URL}": fmt.Sprintf("http://vllm-render.%s.svc.cluster.local:%s", nsName, vllmRenderPort), + "${VLLM_RENDER_PORT}": vllmRenderPort, } for k, v := range extra { subs[k] = v From b49b9c98152f924eefc508cfe0f4490c908566dc Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Wed, 15 Jul 2026 14:22:27 +0300 Subject: [PATCH 6/8] fix merge Signed-off-by: Maya Barnea --- test/e2e/setup_test.go | 3 +- test/e2e/utils_test.go | 73 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 807ac2c510..52e106a812 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -133,7 +133,7 @@ func createRender(nsName string) []string { "${VLLM_RENDER_PORT}": vllmRenderPort, }) objects := testutils.CreateObjsFromYaml(testConfig, renderYamls, nsName) - podsInDeploymentsReady(objects) + podsInDeploymentsReady(nsName, objects) return objects } @@ -190,6 +190,7 @@ func createEndPointPickerHelper(eppConfig string, replicas int, isLeaderElection "${EPP_REPLICA_COUNT}": strconv.Itoa(replicas), "${ENABLE_LEADER_ELECTION}": strconv.FormatBool(isLeaderElectionEnabled), }) + eppYamls = appendEppArgs(eppYamls, eppExtraArgs) if waitForReady { return append(objects, testutils.CreateObjsFromYaml(testConfig, eppYamls, nsName)...) diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 8005cbfe90..a71273600a 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -14,9 +14,12 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" + "sigs.k8s.io/yaml" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" apilabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -209,6 +212,76 @@ func removeEmptyLabels(inputs []string) []string { return outputs } +// eppExtraArgs are appended to the "epp" container's args in the Deployment +// created by createEndPointPickerHelper. +// +// Graceful drain is disabled (--drain-timeout=0) for all e2e EPPs. The suites +// create and delete the shared "e2e-epp" Deployment in sequence behind a single +// Envoy; with a drain window a deleted EPP keeps serving ext_proc on its +// existing connection, so Envoy lingers on the terminating pod (stale datastore) +// and the next spec's requests fail. The graceful-drain behavior itself is +// covered by unit tests. +var eppExtraArgs = []string{"--drain-timeout=0"} + +// appendEppArgs returns the input YAML docs with args appended to the "epp" +// container of any Deployment. It is a no-op when args is empty. +func appendEppArgs(inputs []string, args []string) []string { + if len(args) == 0 { + return inputs + } + outputs := make([]string, len(inputs)) + for idx, input := range inputs { + docs := strings.Split(input, "\n---") + rendered := make([]string, 0, len(docs)) + for _, doc := range docs { + if strings.TrimSpace(doc) == "" { + continue + } + rendered = append(rendered, appendArgsToEppContainer(doc, args)) + } + outputs[idx] = strings.Join(rendered, "\n---\n") + } + return outputs +} + +func appendArgsToEppContainer(doc string, args []string) string { + obj := &unstructured.Unstructured{} + err := yaml.Unmarshal([]byte(doc), &obj.Object) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + if len(obj.Object) == 0 || obj.GetKind() != kubernetesDeploymentKind { + return doc + } + path := []string{"spec", "template", "spec", "containers"} + containers, found, err := unstructured.NestedSlice(obj.Object, path...) + + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + if !found { + return doc + } + for i, c := range containers { + m, ok := c.(map[string]any) + if !ok || m["name"] != "epp" { + continue + } + existing, _, err := unstructured.NestedStringSlice(m, "args") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + merged := make([]any, 0, len(existing)+len(args)) + for _, a := range existing { + merged = append(merged, a) + } + for _, a := range args { + merged = append(merged, a) + } + m["args"] = merged + containers[i] = m + } + err = unstructured.SetNestedSlice(obj.Object, containers, path...) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + out, err := yaml.Marshal(obj.Object) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + return strings.TrimRight(string(out), "\n") +} + func substituteMany(inputs []string, substitutions map[string]string) []string { outputs := make([]string, len(inputs)) for idx, input := range inputs { From 1ca7ded1bd524854632da51ac2bcd69b4b7c08fb Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Wed, 15 Jul 2026 21:07:25 +0300 Subject: [PATCH 7/8] fix compilation Signed-off-by: Maya Barnea --- test/e2e/e2e_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 21d8e33c7a..5e406a0912 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -806,12 +806,8 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { ginkgo.It("should run successfully", func() { infPoolObjects = createInferencePool(1, true) -<<<<<<< HEAD modelServers := createModelServersDecodeKV(1) - epp := createEndPointPicker(kvExternalTokenizerConfig) -======= epp := createEndPointPicker(kvExternalTokenizerConfig()) ->>>>>>> 574c8b00 (review) nsName := getNamespace() prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) From f74e4661bcb001463a389d41ddd29ec9c5e4c309 Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Wed, 15 Jul 2026 21:12:31 +0300 Subject: [PATCH 8/8] remove unused constant Signed-off-by: Maya Barnea --- test/e2e/e2e_suite_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 4046b7672e..9a1b05b9df 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -64,9 +64,6 @@ const ( metricsTestLabel = "Metrics" deprecatedPDTestLabel = "DeprecatedPD" disaggTestLabel = "Disagg" - - // images - simulatorImg = "ghcr.io/llm-d/llm-d-inference-sim:v0.9.2" ) var (