diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c465fc634e..8ec6847802 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -447,7 +447,7 @@ The `deploy/components/` directory contains all reusable Kustomize components: - `vllm-decode/` — Decode pod with routing sidecar **Deployment overlays** — applied on top of the base components: -- `overlays/simulator/` — adds `--mode=${VLLM_SIM_MODE}`, vllm renderer sidecar, KV cache args, and `--zmq-endpoint` on Decode. Included by default in all dev scenario overlays. +- `overlays/simulator/` — adds `--mode=${VLLM_SIM_MODE}`, KV cache args, and `--zmq-endpoint` on Decode. Included by default in all dev scenario overlays. - `overlays/real-vllm/` — adds `--kv-events-config` on Decode, `--ec-transfer-config` on Encode, and a shared PVC for encoder embeddings. **Infrastructure components** — shared cluster infrastructure: @@ -523,7 +523,7 @@ a shared PVC (`ec-cache-pvc`) for encoder embeddings transfer. | Component | What it adds | When to use | |---|---|---| -| `overlays/simulator/` | `--mode=${VLLM_SIM_MODE}`, vLLM renderer, KV cache args, `--zmq-endpoint` on Decode | Dev/test with simulator image | +| `overlays/simulator/` | `--mode=${VLLM_SIM_MODE}`, KV cache args, `--zmq-endpoint` on Decode | Dev/test with simulator image | | `overlays/real-vllm/` | `--kv-events-config` on Decode (per-pod ZMQ publisher), `--ec-transfer-config` on Encode, ec-cache PVC | Production with real vLLM image | | Variable | Default | Description | diff --git a/Makefile b/Makefile index a4deaf313d..95977cb3e3 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,8 @@ export VLLM_IMAGE ?= $(VLLM_SIMULATOR_TAG_BASE):$(VLLM_SIMULATOR_TAG) # CPU-only vLLM image that exposes `vllm launch render` for the token-producer # plugin's HTTP backend. export VLLM_RENDER_IMAGE ?= vllm/vllm-openai-cpu:v0.21.0 +export VLLM_RENDER_PORT ?= 8082 +export VLLM_RENDER_URL ?= http://vllm-render:$(VLLM_RENDER_PORT) BUILDER_TAG ?= dev BUILDER_TAG_BASE ?= $(IMAGE_REGISTRY)/$(BUILDER_IMAGE_NAME) diff --git a/deploy/components/inference-gateway/deployment.yaml b/deploy/components/inference-gateway/deployment.yaml index 0e34342e4f..3117469d75 100644 --- a/deploy/components/inference-gateway/deployment.yaml +++ b/deploy/components/inference-gateway/deployment.yaml @@ -60,40 +60,9 @@ spec: mountPath: /etc/epp - name: cache mountPath: /cache - # vLLM render sidecar — exposes /v1/completions/render and - # /v1/chat/completions/render over loopback for the token-producer plugin. - - name: vllm-render - image: ${VLLM_RENDER_IMAGE} - imagePullPolicy: IfNotPresent - command: ["vllm", "launch", "render"] - args: ["${MODEL_NAME}", "--port=8000"] - ports: - - name: render-http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: 8000 - initialDelaySeconds: 2 - periodSeconds: 1 - timeoutSeconds: 1 - failureThreshold: 600 - readinessProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 5 - timeoutSeconds: 1 - failureThreshold: 1 - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface volumes: - name: epp-config configMap: name: epp-config - name: cache emptyDir: {} - # HuggingFace cache for the render sidecar's model weights. - - name: model-cache - emptyDir: {} diff --git a/deploy/components/overlays/simulator/kustomization.yaml b/deploy/components/overlays/simulator/kustomization.yaml index a250389271..b2cce3cb98 100644 --- a/deploy/components/overlays/simulator/kustomization.yaml +++ b/deploy/components/overlays/simulator/kustomization.yaml @@ -40,99 +40,3 @@ patches: - op: add path: /spec/template/spec/containers/0/args/4 value: "--zmq-endpoint=tcp://${EPP_NAME}.${NAMESPACE}.svc.cluster.local:5557" - - op: add - path: /spec/template/spec/initContainers/0 - value: - name: vllm-render - image: ${VLLM_RENDER_IMAGE} - imagePullPolicy: IfNotPresent - restartPolicy: Always - command: ["vllm"] - args: ["launch", "render", "${MODEL_NAME}", "--port=8082"] - env: - - name: HF_TOKEN - value: "${HF_TOKEN}" - ports: - - name: http - containerPort: 8082 - startupProbe: - httpGet: - path: /health - port: 8082 - initialDelaySeconds: 2 - periodSeconds: 1 - failureThreshold: 600 - timeoutSeconds: 1 - livenessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 1 - readinessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 5 - failureThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface - - op: add - path: /spec/template/spec/volumes/- - value: - name: model-cache - emptyDir: {} -# --- Prefill and Encode deployments (vllm-p, vllm-e) --- -# initContainers: [] in base, so append with /- -- target: - kind: Deployment - name: "vllm-[pe]" - patch: | - - op: add - path: /spec/template/spec/initContainers/- - value: - name: vllm-render - image: ${VLLM_RENDER_IMAGE} - imagePullPolicy: IfNotPresent - restartPolicy: Always - command: ["vllm"] - args: ["launch", "render", "${MODEL_NAME}", "--port=8082"] - env: - - name: HF_TOKEN - value: "${HF_TOKEN}" - ports: - - name: http - containerPort: 8082 - startupProbe: - httpGet: - path: /health - port: 8082 - initialDelaySeconds: 2 - periodSeconds: 1 - failureThreshold: 600 - timeoutSeconds: 1 - livenessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 1 - readinessProbe: - httpGet: - path: /health - port: 8082 - periodSeconds: 5 - failureThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface - - op: add - path: /spec/template/spec/volumes/- - value: - name: model-cache - emptyDir: {} diff --git a/deploy/components/vllm-decode/deployment.yaml b/deploy/components/vllm-decode/deployment.yaml index 69920f8e0e..dd6715fcf9 100644 --- a/deploy/components/vllm-decode/deployment.yaml +++ b/deploy/components/vllm-decode/deployment.yaml @@ -67,6 +67,7 @@ spec: - "--port=8200" - "--model=${MODEL_NAME}" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_D}" ports: - name: http diff --git a/deploy/components/vllm-encode/deployment.yaml b/deploy/components/vllm-encode/deployment.yaml index 927b13e5e5..6c287a684a 100644 --- a/deploy/components/vllm-encode/deployment.yaml +++ b/deploy/components/vllm-encode/deployment.yaml @@ -30,6 +30,7 @@ spec: - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - "--no-enable-prefix-caching" - "--mm-encoder-only" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_E}" ports: - name: encode-http diff --git a/deploy/components/vllm-prefill/deployment.yaml b/deploy/components/vllm-prefill/deployment.yaml index bf7d30dd4e..a97a7b6464 100644 --- a/deploy/components/vllm-prefill/deployment.yaml +++ b/deploy/components/vllm-prefill/deployment.yaml @@ -28,6 +28,7 @@ spec: - "--model=${MODEL_NAME}" - "--port=8000" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_P}" ports: - name: prefill-http diff --git a/deploy/environments/dev/README.md b/deploy/environments/dev/README.md index 7fe9ef9174..0f77ea988f 100644 --- a/deploy/environments/dev/README.md +++ b/deploy/environments/dev/README.md @@ -9,7 +9,7 @@ scenario-specific patches. The atomic components live in `deploy/components/`: | `vllm-decode/` | Decode pod — always deployed, includes routing sidecar (removed in EPD scenario) | | `vllm-prefill/` | Prefill pod — deployed when `DISAGG_P=true` | | `vllm-encode/` | Encoder pod — deployed when `DISAGG_E=true` | -| `overlays/simulator/` | Adds `--mode=${VLLM_SIM_MODE}`, vLLM render, KV cache and ZMQ args (included by all scenario overlays) | +| `overlays/simulator/` | Adds `--mode=${VLLM_SIM_MODE}`, KV cache and ZMQ args (included by all scenario overlays) | These overlays are used by both `scripts/kind-dev-env.sh` (for local KIND clusters) and e2e tests (via `kustomize build` + env var substitution). @@ -70,6 +70,8 @@ Variables substituted at deploy time via `envsubst` or Go test `substituteMany`: | `VLLM_IMAGE` | vLLM container image (simulator or real) | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` | | `SIDECAR_IMAGE` | Routing sidecar image | `ghcr.io/llm-d/llm-d-router-disagg-sidecar:dev` | | `VLLM_RENDER_IMAGE` | vLLM render sidecar image | `vllm/vllm-openai-cpu:v0.21.0` | +| `VLLM_RENDER_PORT` | Port the vllm-render Service listens on | `8082` | +| `VLLM_RENDER_URL` | URL of the vllm-render Service used by decode/encode/prefill pods | `http://vllm-render:${VLLM_RENDER_PORT}` | | `MODEL_NAME` | Model name passed to vLLM. Can be a real HuggingFace model (e.g. `TinyLlama/TinyLlama-1.1B-Chat-v1.0`, `Qwen/Qwen3-VL-2B-Instruct`) or an arbitrary name when using the simulator (e.g. `food-review`) | `food-review` | | `POOL_NAME` | InferencePool name | `food-review-inference-pool` | | `VLLM_REPLICA_COUNT_E` | Encode deployment replicas | `1` | diff --git a/deploy/environments/dev/base-kind-istio/kustomization.yaml b/deploy/environments/dev/base-kind-istio/kustomization.yaml index eda61c4f6d..9cffaff0bb 100644 --- a/deploy/environments/dev/base-kind-istio/kustomization.yaml +++ b/deploy/environments/dev/base-kind-istio/kustomization.yaml @@ -16,6 +16,7 @@ kind: Kustomization resources: - destination-rules.yaml - services.yaml +- vllm-render.yaml - ../../../components/istio-control-plane/ - ../../../components/inference-gateway/ diff --git a/deploy/environments/dev/base-kind-istio/vllm-render.yaml b/deploy/environments/dev/base-kind-istio/vllm-render.yaml new file mode 100644 index 0000000000..b09d71b204 --- /dev/null +++ b/deploy/environments/dev/base-kind-istio/vllm-render.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + replicas: 1 + selector: + matchLabels: + app: vllm-render + template: + metadata: + labels: + app: vllm-render + spec: + containers: + - name: vllm-render + image: ${VLLM_RENDER_IMAGE} + imagePullPolicy: IfNotPresent + command: ["vllm", "launch", "render"] + args: ["${MODEL_NAME}", "--port=${VLLM_RENDER_PORT}"] + # For gated models, inject HF_TOKEN via a Secret reference: + # env: + # - name: HF_TOKEN + # valueFrom: + # secretKeyRef: + # name: hf-token + # key: token + ports: + - name: render-http + containerPort: ${VLLM_RENDER_PORT} + startupProbe: + httpGet: + path: /health + port: ${VLLM_RENDER_PORT} + initialDelaySeconds: 2 + periodSeconds: 1 + failureThreshold: 600 + timeoutSeconds: 1 + livenessProbe: + httpGet: + path: /health + port: ${VLLM_RENDER_PORT} + periodSeconds: 10 + failureThreshold: 5 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /health + port: ${VLLM_RENDER_PORT} + periodSeconds: 5 + failureThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - name: model-cache + mountPath: /root/.cache/huggingface + volumes: + - name: model-cache + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + selector: + app: vllm-render + ports: + - name: http + protocol: TCP + port: ${VLLM_RENDER_PORT} + targetPort: ${VLLM_RENDER_PORT} + type: ClusterIP diff --git a/deploy/environments/dev/e2e-infra/vllm-render.yaml b/deploy/environments/dev/e2e-infra/vllm-render.yaml new file mode 100644 index 0000000000..f3b425f562 --- /dev/null +++ b/deploy/environments/dev/e2e-infra/vllm-render.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + replicas: 1 + selector: + matchLabels: + app: vllm-render + template: + metadata: + labels: + app: vllm-render + spec: + containers: + - name: vllm-render + image: ${VLLM_RENDER_IMAGE} + imagePullPolicy: IfNotPresent + args: + - "--model=${MODEL_NAME}" + - "--port=${VLLM_RENDER_PORT}" + - "--mode=echo" + - "--force-dummy-tokenizer" + ports: + - name: http + containerPort: ${VLLM_RENDER_PORT} + startupProbe: + httpGet: + path: /health + port: ${VLLM_RENDER_PORT} + initialDelaySeconds: 2 + periodSeconds: 1 + failureThreshold: 600 + timeoutSeconds: 1 + livenessProbe: + httpGet: + path: /health + port: ${VLLM_RENDER_PORT} + periodSeconds: 10 + failureThreshold: 5 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /health + port: ${VLLM_RENDER_PORT} + periodSeconds: 5 + failureThreshold: 1 + timeoutSeconds: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: vllm-render + labels: + app: vllm-render +spec: + selector: + app: vllm-render + ports: + - name: http + protocol: TCP + port: ${VLLM_RENDER_PORT} + targetPort: ${VLLM_RENDER_PORT} + type: ClusterIP diff --git a/deploy/environments/dev/epd/patch-decode.yaml b/deploy/environments/dev/epd/patch-decode.yaml index e10a96beeb..a48914ae15 100644 --- a/deploy/environments/dev/epd/patch-decode.yaml +++ b/deploy/environments/dev/epd/patch-decode.yaml @@ -22,6 +22,7 @@ spec: - "--block-size=16" - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}" - "--zmq-endpoint=tcp://${EPP_NAME}.${NAMESPACE}.svc.cluster.local:5557" + - "--render-url=${VLLM_RENDER_URL}" - "${VLLM_EXTRA_ARGS_D}" ports: - $patch: replace diff --git a/scripts/kind-dev-env.sh b/scripts/kind-dev-env.sh index 91edb0147d..611eee3e48 100755 --- a/scripts/kind-dev-env.sh +++ b/scripts/kind-dev-env.sh @@ -65,6 +65,8 @@ export SIDECAR_IMAGE # Set a default VLLM_RENDER_IMAGE if not provided (CPU-only vLLM image that # runs `vllm launch render` for the token-producer plugin's HTTP backend). export VLLM_RENDER_IMAGE="${VLLM_RENDER_IMAGE:-vllm/vllm-openai-cpu:v0.21.0}" +export VLLM_RENDER_PORT="${VLLM_RENDER_PORT:-8082}" +export VLLM_RENDER_URL="${VLLM_RENDER_URL:-http://vllm-render:${VLLM_RENDER_PORT}}" # Set the inference pool name for the deployment export POOL_NAME="${POOL_NAME:-${MODEL_NAME_SAFE}-inference-pool}" @@ -414,7 +416,7 @@ export ENABLE_LEADER_ELECTION=false # Deploy Istio base (shared infrastructure) kubectl kustomize --enable-helm deploy/environments/dev/base-kind-istio \ | envsubst '${POOL_NAME} ${MODEL_NAME} ${MODEL_NAME_SAFE} ${EPP_NAME} ${EPP_IMAGE} ${VLLM_IMAGE} \ - ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${TARGET_PORTS} ${NAMESPACE} ${METRICS_ENDPOINT_AUTH} \ + ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${VLLM_RENDER_PORT} ${VLLM_RENDER_URL} ${TARGET_PORTS} ${NAMESPACE} ${METRICS_ENDPOINT_AUTH} \ ${EPP_REPLICA_COUNT} ${VLLM_REPLICA_COUNT_E} ${VLLM_REPLICA_COUNT_P} ${VLLM_REPLICA_COUNT_D} \ ${VLLM_DATA_PARALLEL_SIZE} ${ENABLE_LEADER_ELECTION}' \ | kubectl --context ${KUBE_CONTEXT} apply -f - @@ -422,7 +424,7 @@ kubectl kustomize --enable-helm deploy/environments/dev/base-kind-istio \ # Deploy scenario-specific vLLM components kubectl kustomize --enable-helm ${KUSTOMIZE_DIR} \ | envsubst '${POOL_NAME} ${MODEL_NAME} ${MODEL_NAME_SAFE} ${EPP_NAME} ${EPP_IMAGE} ${VLLM_IMAGE} \ - ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${TARGET_PORTS} ${NAMESPACE} \ + ${SIDECAR_IMAGE} ${VLLM_RENDER_IMAGE} ${VLLM_RENDER_PORT} ${VLLM_RENDER_URL} ${TARGET_PORTS} ${NAMESPACE} \ ${VLLM_REPLICA_COUNT_E} ${VLLM_REPLICA_COUNT_P} ${VLLM_REPLICA_COUNT_D} ${VLLM_DATA_PARALLEL_SIZE} \ ${KV_CONNECTOR_TYPE} ${EC_CONNECTOR_TYPE} ${CONNECTOR_TYPE} ${KV_CACHE_ENABLED} ${HF_TOKEN} ${VLLM_SIM_MODE} \ ${DECODE_ROLE} ${VLLM_EXTRA_ARGS_E} ${VLLM_EXTRA_ARGS_P} ${VLLM_EXTRA_ARGS_D}' \ diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index 6760a66242..0afbb4abba 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -1,5 +1,7 @@ package e2e +import "fmt" + // Simple EPP configuration for running without P/D const simpleConfig = `apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -179,15 +181,17 @@ schedulingProfiles: weight: 2 ` -// EPP config for running with precise prefix scoring (i.e. KV events). -const kvConfig = `apiVersion: llm-d.ai/v1alpha1 +// kvConfig returns the EPP config for running with precise prefix scoring (i.e. KV events). +// The render URL is built from vllmRenderPort so VLLM_RENDER_PORT is respected. +func kvConfig() string { + return fmt.Sprintf(`apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig plugins: - type: token-producer parameters: modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: - url: http://localhost:8000 + url: http://vllm-render:%s - type: precise-prefix-cache-scorer parameters: tokenProcessorConfig: @@ -209,17 +213,20 @@ schedulingProfiles: - pluginRef: max-score-picker - pluginRef: precise-prefix-cache-scorer weight: 10 -` +`, vllmRenderPort) +} -// Alias of kvConfig retained for tests that reference the external-tokenizer name. -const kvExternalTokenizerConfig = `apiVersion: llm-d.ai/v1alpha1 +// kvExternalTokenizerConfig returns the EPP config for the external-tokenizer DataProducer variant. +// The render URL is built from vllmRenderPort so VLLM_RENDER_PORT is respected. +func kvExternalTokenizerConfig() string { + return fmt.Sprintf(`apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig plugins: - type: token-producer parameters: modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: - url: http://localhost:8000 + url: http://vllm-render:%s - type: precise-prefix-cache-scorer parameters: tokenProcessorConfig: @@ -240,7 +247,8 @@ schedulingProfiles: - pluginRef: max-score-picker - pluginRef: precise-prefix-cache-scorer weight: 10 -` +`, vllmRenderPort) +} // EPP configuration for running scale model server test const scaleConfig = `apiVersion: llm-d.ai/v1alpha1 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 71076fc8ab..9a1b05b9df 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -54,6 +54,8 @@ const ( serviceAccountManifest = "../../deploy/components/inference-gateway/service-accounts.yaml" // servicesManifest is the manifest for the EPP's service resources. servicesManifest = "../../deploy/environments/dev/e2e-infra/services.yaml" + // renderManifest is the manifest for the standalone vLLM render deployment and service. + renderManifest = "../../deploy/environments/dev/e2e-infra/vllm-render.yaml" // CI shards scheduler e2e specs with label filters. extendedTestLabel = "Extended" @@ -79,6 +81,7 @@ var ( vllmSimImage = env.GetEnvString("VLLM_IMAGE", "ghcr.io/llm-d/llm-d-inference-sim:v0.10.2", ginkgo.GinkgoLogr) sideCarImage = env.GetEnvString("SIDECAR_IMAGE", "ghcr.io/llm-d/llm-d-router-disagg-sidecar:dev", ginkgo.GinkgoLogr) vllmRenderImage = env.GetEnvString("VLLM_RENDER_IMAGE", "vllm/vllm-openai-cpu:v0.21.0", ginkgo.GinkgoLogr) + vllmRenderPort = env.GetEnvString("VLLM_RENDER_PORT", "8082", ginkgo.GinkgoLogr) loadRenderImage = env.GetEnvBool("LOAD_VLLM_RENDER_IMAGE", true, ginkgo.GinkgoLogr) numProcesses = env.GetEnvInt("E2E_NUM_PROCS", 1, ginkgo.GinkgoLogr) // baseNsName is the base of the namespace in which the K8S objects will be created @@ -95,6 +98,7 @@ var ( rbacObjects []string serviceAccountObjects []string serviceObjects []string + renderObjects []string infPoolObjects []string createdNameSpace bool @@ -133,6 +137,7 @@ var _ = ginkgo.BeforeSuite(func() { saYamls := substituteMany(testutils.ReadYaml(serviceAccountManifest), infraSubs) serviceAccountObjects = testutils.CreateObjsFromYaml(testConfig, saYamls, nsName) serviceObjects = testutils.ApplyYAMLFile(testConfig, servicesManifest, nsName) + renderObjects = createRender(nsName) // Prevent failure in tests due to InferencePool not existing before the test infPoolObjects = createInferencePool(1, false) @@ -190,6 +195,7 @@ var _ = ginkgo.ReportAfterSuite("cleanup", func(report ginkgo.Report) { nsName := getNamespace() ginkgo.By("Deleting created Kubernetes objects") testutils.DeleteObjects(testConfig, infPoolObjects, nsName) + testutils.DeleteObjects(testConfig, renderObjects, nsName) testutils.DeleteObjects(testConfig, serviceObjects, nsName) testutils.DeleteObjects(testConfig, serviceAccountObjects, nsName) testutils.DeleteObjects(testConfig, rbacObjects, nsName) @@ -242,7 +248,7 @@ func setupK8sCluster() { kindLoadImage(vllmSimImage) kindLoadImage(eppImage) kindLoadImage(sideCarImage) - if loadRenderImage { + if loadRenderImage && vllmRenderImage != vllmSimImage { kindLoadImage(vllmRenderImage) } } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 428a6821a6..5e406a0912 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -784,7 +784,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { infPoolObjects = createInferencePool(1, true) modelServers := createModelServersDecodeKV(1) - epp := createEndPointPicker(kvConfig) + epp := createEndPointPicker(kvConfig()) nsName := getNamespace() prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) @@ -807,7 +807,7 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() { infPoolObjects = createInferencePool(1, true) modelServers := createModelServersDecodeKV(1) - epp := createEndPointPicker(kvExternalTokenizerConfig) + epp := createEndPointPicker(kvExternalTokenizerConfig()) nsName := getNamespace() prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector) diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 578e938475..52e106a812 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -14,8 +14,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - configloader "github.com/llm-d/llm-d-router/pkg/epp/config/loader" - "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer" "github.com/llm-d/llm-d-router/pkg/sidecar/proxy" testutils "github.com/llm-d/llm-d-router/test/utils" ) @@ -26,7 +24,6 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin "${MODEL_NAME}": simModelName, "${POOL_NAME}": poolName, "${VLLM_IMAGE}": vllmSimImage, - "${VLLM_RENDER_IMAGE}": vllmRenderImage, "${SIDECAR_IMAGE}": sideCarImage, "${VLLM_DATA_PARALLEL_SIZE}": "1", "${VLLM_SIM_MODE}": "echo", @@ -35,9 +32,11 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin "${EPP_NAME}": "e2e-epp", "${NAMESPACE}": nsName, "${HF_TOKEN}": os.Getenv("HF_TOKEN"), - "${VLLM_EXTRA_ARGS_E}": "", - "${VLLM_EXTRA_ARGS_P}": "", - "${VLLM_EXTRA_ARGS_D}": "", + "${VLLM_EXTRA_ARGS_E}": "--force-dummy-tokenizer", + "${VLLM_EXTRA_ARGS_P}": "--force-dummy-tokenizer", + "${VLLM_EXTRA_ARGS_D}": "--force-dummy-tokenizer", + "${VLLM_RENDER_URL}": fmt.Sprintf("http://vllm-render.%s.svc.cluster.local:%s", nsName, vllmRenderPort), + "${VLLM_RENDER_PORT}": vllmRenderPort, } for k, v := range extra { subs[k] = v @@ -48,10 +47,6 @@ func createModelServersFromKustomize(kustomizeDir string, extra map[string]strin // Remove labels with empty values (produced when ${DECODE_ROLE} is empty) manifests = removeEmptyLabels(manifests) manifests = removeEmptyArgs(manifests) - // remove render sidecar if model is simulated - if !isModelReal(subs["${MODEL_NAME}"]) { - manifests = removeRenderSidecar(manifests) - } objects := testutils.CreateObjsFromYaml(testConfig, manifests, nsName) podsInDeploymentsReady(nsName, objects) return objects @@ -130,6 +125,18 @@ func createModelServersEPDUnified(replicas int) []string { }) } +func createRender(nsName string) []string { + renderYamls := substituteMany(testutils.ReadYaml(renderManifest), + map[string]string{ + "${MODEL_NAME}": kvModelName, + "${VLLM_RENDER_IMAGE}": vllmRenderImage, + "${VLLM_RENDER_PORT}": vllmRenderPort, + }) + objects := testutils.CreateObjsFromYaml(testConfig, renderYamls, nsName) + podsInDeploymentsReady(nsName, objects) + return objects +} + func createEndPointPicker(eppConfig string) []string { objects := createEndPointPickerHelper(eppConfig, 1, false, true) podsInDeploymentsReady(getNamespace(), objects) @@ -175,21 +182,14 @@ func createEndPointPickerHelper(eppConfig string, replicas int, isLeaderElection eppYamls := testutils.ReadYaml(eppManifest) eppYamls = substituteMany(eppYamls, map[string]string{ - "${EPP_NAME}": eppName, - "${EPP_IMAGE}": eppImage, - "${VLLM_RENDER_IMAGE}": vllmRenderImage, - // The render sidecar needs a real, fetchable model. Sim tests - // don't query it; the cost is paying weights-load on every EPP. - "${MODEL_NAME}": kvModelName, + "${EPP_NAME}": eppName, + "${EPP_IMAGE}": eppImage, "${NAMESPACE}": nsName, "${POOL_NAME}": simModelName + "-inference-pool", "${METRICS_ENDPOINT_AUTH}": "false", "${EPP_REPLICA_COUNT}": strconv.Itoa(replicas), "${ENABLE_LEADER_ELECTION}": strconv.FormatBool(isLeaderElectionEnabled), }) - if !usesTokenProducer(eppConfig) { - eppYamls = removeRenderSidecar(eppYamls) - } eppYamls = appendEppArgs(eppYamls, eppExtraArgs) if waitForReady { @@ -198,14 +198,3 @@ func createEndPointPickerHelper(eppConfig string, replicas int, isLeaderElection objs := testutils.CreateUnstructuredObjs(testConfig, eppYamls) return append(objects, testutils.CreateObjsWithVerifier(testConfig, objs, nsName, func(kind string, clientObj client.Object) {})...) } - -func usesTokenProducer(eppConfig string) bool { - cfg, _, err := configloader.LoadRawConfig([]byte(eppConfig), ginkgo.GinkgoLogr) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - for _, plugin := range cfg.Plugins { - if plugin.Type == tokenizer.PluginType { - return true - } - } - return false -} diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 4e3195e3d4..a71273600a 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/http" - "os" "os/exec" "strconv" "strings" @@ -15,6 +14,8 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" + "sigs.k8s.io/yaml" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,7 +23,6 @@ import ( apilabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" ) const ( @@ -212,81 +212,6 @@ func removeEmptyLabels(inputs []string) []string { return outputs } -func isModelReal(modelName string) bool { - req, err := http.NewRequest("GET", "https://huggingface.co/api/models/"+modelName, nil) - if err != nil { - return false - } - if token := os.Getenv("HF_TOKEN"); token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return false - } - defer resp.Body.Close() - - return resp.StatusCode == http.StatusOK -} - -// removeRenderSidecar takes a slice of YAML strings (each may contain multiple -// objects separated by "---") and returns the same slice with the vllm-render -// container and the model-cache volume stripped from any Deployment. -func removeRenderSidecar(inputs []string) []string { - outputs := make([]string, len(inputs)) - for idx, input := range inputs { - docs := strings.Split(input, "\n---") - rendered := make([]string, 0, len(docs)) - for _, doc := range docs { - if strings.TrimSpace(doc) == "" { - continue - } - rendered = append(rendered, filterDocument(doc)) - } - outputs[idx] = strings.Join(rendered, "\n---\n") - } - return outputs -} - -func filterDocument(doc string) string { - obj := &unstructured.Unstructured{} - err := yaml.Unmarshal([]byte(doc), &obj.Object) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - if len(obj.Object) == 0 { - return doc - } - if obj.GetKind() == kubernetesDeploymentKind { - removePodSpecListItem(obj, "containers", "vllm-render") - removePodSpecListItem(obj, "initContainers", "vllm-render") - removePodSpecListItem(obj, "volumes", "model-cache") - } - out, err := yaml.Marshal(obj.Object) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - return strings.TrimRight(string(out), "\n") -} - -func removePodSpecListItem(obj *unstructured.Unstructured, fieldName, itemName string) { - path := []string{"spec", "template", "spec", fieldName} - items, found, err := unstructured.NestedSlice(obj.Object, path...) - if err != nil || !found { - return - } - filtered := make([]any, 0, len(items)) - for _, item := range items { - if m, ok := item.(map[string]any); ok && m["name"] == itemName { - continue - } - filtered = append(filtered, item) - } - if len(filtered) == 0 { - unstructured.RemoveNestedField(obj.Object, path...) - return - } - err = unstructured.SetNestedSlice(obj.Object, filtered, path...) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) -} - // eppExtraArgs are appended to the "epp" container's args in the Deployment // created by createEndPointPickerHelper. // @@ -328,6 +253,7 @@ func appendArgsToEppContainer(doc string, args []string) string { } path := []string{"spec", "template", "spec", "containers"} containers, found, err := unstructured.NestedSlice(obj.Object, path...) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) if !found { return doc diff --git a/test/scripts/test-e2e-router.sh b/test/scripts/test-e2e-router.sh index 491cbdda3e..623f9a1ade 100755 --- a/test/scripts/test-e2e-router.sh +++ b/test/scripts/test-e2e-router.sh @@ -31,4 +31,6 @@ trap 'e2e_handle_interrupt "e2e-tests"' INT TERM echo "Running end to end tests" +export VLLM_RENDER_IMAGE=${VLLM_IMAGE} + run_ginkgo_suite "${DIR}/../e2e/" diff --git a/test/testdata/sim-deployment.yaml b/test/testdata/sim-deployment.yaml index 0292281317..c75cb9853a 100644 --- a/test/testdata/sim-deployment.yaml +++ b/test/testdata/sim-deployment.yaml @@ -19,7 +19,7 @@ spec: imagePullPolicy: IfNotPresent restartPolicy: Always command: ["vllm"] - args: ["launch", "render", "Qwen/Qwen3-32B", "--port=8082"] + args: ["launch", "render", "Qwen/Qwen2.5-1.5B-Instruct", "--port=8082"] ports: - name: http containerPort: 8082 @@ -57,7 +57,7 @@ spec: imagePullPolicy: IfNotPresent args: - --model - - Qwen/Qwen3-32B + - Qwen/Qwen2.5-1.5B-Instruct - --port - "8000" - --max-loras