Merge remote-tracking branch 'upstream/main' into flowcontrol/unify-on-priority-queue

Author: LukeAVanDrie

.github/actions/trivy-scan/action.yml

@@ -26,6 +26,11 @@ runs:
         output: 'trivy-results.sarif'
         severity: ${{ inputs.severity }}
         exit-code: '1'
+        # trivy-action drops the severity filter for SARIF output unless told
+        # otherwise, which fails the scan on findings outside `severity` (e.g.
+        # UNKNOWN-severity Go vulndb notes).
+        limit-severities-for-sarif: 'true'
+        version: 'v0.72.0'
 
     - name: Upload Trivy SARIF to Security tab
       uses: github/codeql-action/upload-sarif@v4.35.4 # v4.35.4 68bde559dea0fdcac2102bfdf6230c5f70eb485e

.github/workflows/ci-pr-checks.yaml

@@ -222,7 +222,7 @@ jobs:
             archive: llm-d-router-endpoint-picker
           - name: simulator
             source: pull
-            image: ghcr.io/llm-d/llm-d-inference-sim:v0.9.2
+            image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2
             archive: llm-d-inference-sim
     name: e2e-image-common (${{ matrix.image.name }})
     steps:

.github/workflows/nightly-router-perf-test-optimized-baseline-10k-1k.yaml

@@ -40,9 +40,14 @@ on:
         default: 'test/perf/config/shared_prefix_job1.yaml'
         required: true
         type: string
+      gcs_bucket:
+        description: 'GCS bucket name for storing performance test results'
+        default: 'llm-d-router-perf-results'
+        required: true
+        type: string
 
 permissions:
-  contents: write
+  contents: read
   actions: read
 
 jobs:
@@ -58,6 +63,7 @@ jobs:
       ROUTER_CONFIG: ${{ inputs.router_config || 'test/perf/config/router-configs/optimized-baseline.yaml' }}
       TEST_NAME: ${{ inputs.test_name || 'optimized-baseline-job1' }}
       PERF_JOB: ${{ inputs.perf_job || 'test/perf/config/shared_prefix_job1.yaml' }}
+      GCS_BUCKET: ${{ inputs.gcs_bucket || 'llm-d-router-perf-results' }}
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
     steps:
       - name: Checkout llm-d/llm-d-router
@@ -95,6 +101,10 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Install Graphviz
+        run: |
+          sudo apt-get update && sudo apt-get install -y graphviz
+
       - name: Install Python dependencies
         run: |
           python3 -m pip install --upgrade pip
@@ -110,21 +120,13 @@ jobs:
             --sim-replicas ${{ env.SIM_REPLICAS }} \
             --results-dir test/perf/results/optimized-baseline \
             --router-machine-family e2 \
-            --gcp-project ${{ env.GCP_PROJECT_ID }}
+            --gcp-project ${{ env.GCP_PROJECT_ID }} \
+            --collect-pprof-profiles \
+            --gcs-bucket ${{ env.GCS_BUCKET }}
 
-      - name: Upload Performance Results
+      - name: Upload Performance Results Artifact
         uses: actions/upload-artifact@v4
         if: always()
         with:
           name: epp-perf-results
           path: test/perf/results/optimized-baseline/
-
-      - name: Commit and push updated results
-        if: success() && github.ref == 'refs/heads/main'
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add test/perf/results/optimized-baseline/
-          git commit -m "Auto-update EPP nightly performance results [skip ci]" || exit 0
-          git pull --rebase origin main
-          git push

.gitignore

@@ -53,3 +53,7 @@ vendor
 # Helm dependency artifacts
 config/charts/*/charts/
 config/charts/*/Chart.lock
+
+# Performance benchmark local results directory
+test/perf/__pycache__/
+test/perf/results/

DEVELOPMENT.md

@@ -85,7 +85,7 @@ Creates a new `kind` cluster (or reuses an existing one) in the `default` namesp
 > [!NOTE]
 > You can pre-pull external images to avoid slow downloads:
 > ```
-> docker pull ghcr.io/llm-d/llm-d-inference-sim:v0.9.2
+> docker pull ghcr.io/llm-d/llm-d-inference-sim:v0.10.2
 > docker pull vllm/vllm-openai-cpu:v0.21.0
 > ```
 
@@ -528,7 +528,7 @@ a shared PVC (`ec-cache-pvc`) for encoder embeddings transfer.
 
 | Variable | Default | Description |
 |---|---|---|
-| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.9.2` | vLLM container image to deploy. Can be a simulator or a real vLLM image (e.g., `vllm/vllm-openai:v0.16.0`). Defaults to the simulator image. |
+| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` | vLLM container image to deploy. Can be a simulator or a real vLLM image (e.g., `vllm/vllm-openai:v0.16.0`). Defaults to the simulator image. |
 | `VLLM_SIM_MODE` | `echo` | Simulator response mode. `echo` returns the input prompt as the response (useful for routing validation). `random` returns random sentences from a pre-defined bank. Only applies when using the simulator overlay. |
 
 ### Cleanup
@@ -622,7 +622,7 @@ kubectl --context kind-e2e-tests get pods
 | `VLLM_EXTRA_ARGS_E` | _(empty)_ | Additional flags for the Encoder vLLM container (e.g. `--mm-processor-kwargs={}`) |
 | `VLLM_EXTRA_ARGS_P` | _(empty)_ | Additional flags for the Prefill vLLM container (e.g. `--gpu-memory-utilization=0.9`) |
 | `VLLM_EXTRA_ARGS_D` | _(empty)_ | Additional flags for the Decode vLLM container (e.g. `--tensor-parallel-size=2`) |
-| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.9.2` | vLLM container image to deploy. Can be a simulator or a real vLLM image (e.g., `vllm/vllm-openai:v0.16.0`) |
+| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` | vLLM container image to deploy. Can be a simulator or a real vLLM image (e.g., `vllm/vllm-openai:v0.16.0`) |
 | `VLLM_SIM_MODE` | `echo` | Simulator response mode. Supported values: `echo` (returns the input prompt as the response), `random` (returns a random sentence from a pre-defined bank) |
 | `SIDECAR_IMAGE` | `ghcr.io/llm-d/llm-d-router-disagg-sidecar:dev` | Routing sidecar image loaded into the Kind cluster |
 | `VLLM_RENDER_IMAGE` | `vllm/vllm-openai-cpu:v0.21.0` | vLLM renderer image loaded into the Kind cluster |

Makefile

@@ -33,7 +33,7 @@ SIDECAR_IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/$(SIDECAR_IMAGE_NAME)
 SIDECAR_TAG ?= dev
 export SIDECAR_IMAGE ?= $(SIDECAR_IMAGE_TAG_BASE):$(SIDECAR_TAG)
 
-VLLM_SIMULATOR_TAG ?= v0.9.2
+VLLM_SIMULATOR_TAG ?= v0.10.2
 VLLM_SIMULATOR_TAG_BASE ?= $(IMAGE_REGISTRY)/$(VLLM_SIMULATOR_IMAGE_NAME)
 export VLLM_IMAGE ?= $(VLLM_SIMULATOR_TAG_BASE):$(VLLM_SIMULATOR_TAG)
 

config/manifests/vllm/sim-deployment.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: vllm-sim
-        image: ghcr.io/llm-d/llm-d-inference-sim:v0.9.2
+        image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2
         imagePullPolicy: Always
         args:
         - --model

config/manifests/vllm/sim-grpc-deployment.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: vllm-sim
-        image: ghcr.io/llm-d/llm-d-inference-sim:v0.9.2
+        image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2
         imagePullPolicy: Always
         args:
         - --model

deploy/environments/dev/README.md

@@ -67,7 +67,7 @@ Variables substituted at deploy time via `envsubst` or Go test `substituteMany`:
 | `DISAGG_E` | Deploy a separate Encoder pod (`true`/`false`) | `false` |
 | `DISAGG_P` | Deploy a separate Prefill pod (`true`/`false`) | `false` |
 | `VLLM_SIM_MODE` | Simulator response mode: `echo` (returns input) or `random` (random sentences) | `echo` |
-| `VLLM_IMAGE` | vLLM container image (simulator or real) | `ghcr.io/llm-d/llm-d-inference-sim:v0.9.2` |
+| `VLLM_IMAGE` | vLLM container image (simulator or real) | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` |
 | `SIDECAR_IMAGE` | Routing sidecar image | `ghcr.io/llm-d/llm-d-router-disagg-sidecar:dev` |
 | `VLLM_RENDER_IMAGE` | vLLM render sidecar image | `vllm/vllm-openai-cpu:v0.21.0` |
 | `MODEL_NAME` | Model name passed to vLLM. Can be a real HuggingFace model (e.g. `TinyLlama/TinyLlama-1.1B-Chat-v1.0`, `Qwen/Qwen3-VL-2B-Instruct`) or an arbitrary name when using the simulator (e.g. `food-review`) | `food-review` |

pkg/coordinator/steps/prefill.go

@@ -137,6 +137,8 @@ func (s *PrefillStep) buildPrefillBody(ctx context.Context, reqCtx *pipeline.Req
 	switch format {
 	case gateway.FormatChatCompletions:
 		body := maps.Clone(reqCtx.Body)
+		body["stream"] = false
+		delete(body, "stream_options")
 		tokens := map[string]any{
 			"token_ids": reqCtx.TokenIDs,
 		}