Merge branch 'main' into fix/per-prompt-mm-features

Author: albertoperdomo2

.github/workflows/ci-coordinator.yaml

@@ -0,0 +1,106 @@
+name: CI - Coordinator Test
+
+# Runs coordinator unit tests and e2e tests. Path filter excludes paths not
+# exercised by the coordinator.
+on:
+  push:
+    branches:
+      - main
+      - 'release-*'
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      src: ${{ steps.filter.outputs.src }}
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+      - uses: dorny/paths-filter@v4
+        id: filter
+        with:
+          filters: |
+            src:
+              - '**/*.go'
+              - Dockerfile.*
+              - Makefile*
+              - go.mod
+              - go.sum
+              - scripts/**
+              - hack/**
+              - test/**
+              - .github/actions/e2e-runner-setup/action.yml
+              - .github/workflows/ci-coordinator.yaml
+              - '!Dockerfile.sidecar'
+              - '!cmd/pd-sidecar/**'
+              - '!pkg/sidecar/**'
+              - '!test/sidecar/**'
+              - '!pkg/generator/**'
+              - '!test/e2e/**'
+              - '!test/integration/**'
+              - '!test/profiling/**'
+              - '!Makefile'
+
+  unit-tests:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.src == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+
+      - name: Create Go cache dirs
+        id: go-cache
+        run: |
+          mkdir -p "$HOME/.cache/llm-d-gomodcache" "$HOME/.cache/llm-d-gobuildcache"
+          echo "mod=$HOME/.cache/llm-d-gomodcache" >> "$GITHUB_OUTPUT"
+          echo "build=$HOME/.cache/llm-d-gobuildcache" >> "$GITHUB_OUTPUT"
+
+      - name: Cache Go modules and build cache
+        uses: actions/cache@v6
+        with:
+          path: |
+            ${{ steps.go-cache.outputs.mod }}
+            ${{ steps.go-cache.outputs.build }}
+          key: go-cache-${{ runner.os }}-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            go-cache-${{ runner.os }}-
+
+      - name: Run coordinator unit tests
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
+          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
+        run: make -f Makefile.coord.mk test-unit
+
+  e2e-tests:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.src == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+
+      - name: Set up e2e runner
+        id: e2e-setup
+        uses: ./.github/actions/e2e-runner-setup
+
+      - name: Run coordinator e2e tests
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
+          GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
+        run: make -f Makefile.coord.mk test-e2e-coordinator

.github/workflows/ci-lint.yaml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
           cache: true

.github/workflows/ci-pr-checks.yaml

@@ -1,4 +1,4 @@
-# Matrix jobs (e2e-image-common, e2e-image-router, e2e-router) are each
+# Matrix jobs (e2e-image-common, e2e-image-router) are each
 # followed by a *-status job that aggregates their result, since GitHub
 # required status checks cannot reference individual matrix legs.
 name: CI - Test
@@ -45,6 +45,11 @@ jobs:
               - .github/actions/docker-build-and-push/action.yml
               - .github/actions/e2e-runner-setup/action.yml
               - .github/workflows/ci-pr-checks.yaml
+              - '!Dockerfile.coordinator'
+              - '!Makefile.coord.mk'
+              - '!cmd/coordinator/**'
+              - '!pkg/coordinator/**'
+              - '!test/coordinator/**'
             helm:
               - 'config/charts/**'
               - 'hack/verify-helm.sh'
@@ -59,7 +64,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
 
@@ -108,6 +113,16 @@ jobs:
           GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
         run: make test-unit
 
+      # Anti-rot: execute the benchmark bodies once so runtime breakage (panics,
+      # deadlocks) is caught. `go test` only compiles benchmarks; it never runs
+      # them without -bench.
+      - name: Run benchmark smoke
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
+          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
+        run: make bench-smoke
+
       - name: Run hermetic integration tests
         shell: bash
         env:
@@ -343,29 +358,6 @@ jobs:
     if: ${{ needs.check-changes.outputs.src == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        suite:
-          - name: pd
-            label-filter: "!Disruptive && !Extended && !SharedStorage && !Metrics"
-            needs-renderer: "false"
-          - name: pd-shared-storage-deprecated
-            label-filter: "SharedStorage && DeprecatedPD"
-            needs-renderer: "false"
-          - name: pd-shared-storage-disagg
-            label-filter: "SharedStorage && Disagg"
-            needs-renderer: "false"
-          - name: pd-metrics
-            label-filter: "Metrics"
-            needs-renderer: "false"
-          - name: extended
-            label-filter: "Extended"
-            needs-renderer: "true"
-          - name: disruption
-            label-filter: "Disruptive"
-            needs-renderer: "false"
-    name: e2e-router (${{ matrix.suite.name }})
     steps:
       - name: Checkout source
         uses: actions/checkout@v7
@@ -399,7 +391,6 @@ jobs:
           path: ${{ runner.temp }}/e2e-images
 
       - name: Download renderer image
-        if: ${{ matrix.suite.needs-renderer == 'true' }}
         uses: actions/download-artifact@v8
         with:
           name: e2e-image-renderer
@@ -412,26 +403,11 @@ jobs:
             docker load --input "$image"
           done
 
-      - name: Run e2e-router (${{ matrix.suite.name }})
+      - name: Run e2e-router
         shell: bash
         env:
           GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
           GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
-          E2E_LABEL_FILTER: ${{ matrix.suite.label-filter }}
-          LOAD_VLLM_RENDER_IMAGE: ${{ matrix.suite.needs-renderer }}
           PULL_VLLM_RENDER_IMAGE: "false"
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: make test-e2e-run
-
-  e2e-router-status:
-    needs: e2e-router
-    if: always()
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check e2e-router result
-        run: |
-          result="${{ needs.e2e-router.result }}"
-          if [[ "$result" != "success" && "$result" != "skipped" ]]; then
-            echo "::error::e2e-router failed for at least one matrix suite"
-            exit 1
-          fi

.github/workflows/ci-release.yaml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
 

.github/workflows/nightly-router-perf-test-optimized-baseline-10k-1k.yaml

@@ -67,21 +67,21 @@ jobs:
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
     steps:
       - name: Checkout llm-d/llm-d-router
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Checkout kubernetes-sigs/inference-perf
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: kubernetes-sigs/inference-perf
           path: inference-perf
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@v3
         with:
           credentials_json: ${{ secrets.GKE_SA_KEY }}
 
       - name: Set up gcloud CLI and kubectl
-        uses: google-github-actions/setup-gcloud@v2
+        uses: google-github-actions/setup-gcloud@v3
         with:
           project_id: ${{ env.GCP_PROJECT_ID }}
           install_components: 'kubectl,gke-gcloud-auth-plugin'
@@ -97,7 +97,7 @@ jobs:
           curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: "3.10"
 
@@ -125,7 +125,7 @@ jobs:
             --gcs-bucket ${{ env.GCS_BUCKET }}
 
       - name: Upload Performance Results Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: always()
         with:
           name: epp-perf-results

DEVELOPMENT.md

@@ -11,7 +11,6 @@ Documentation for developing the llm-d Router.
   - [Kind Development Environment](#kind-development-environment)
     - [Accessing the Gateway](#accessing-the-gateway)
     - [Prometheus Monitoring](#prometheus-monitoring)
-    - [Grafana Dashboard](#grafana-dashboard)
     - [Development Cycle](#development-cycle)
     - [Debugging](#debugging)
     - [Inference Disaggregation Modes](#inference-disaggregation-modes)
@@ -170,17 +169,10 @@ Prometheus will be accessible at `http://localhost:30090`. To use a different ho
 PROM_ENABLED=true KIND_PROM_HOST_PORT=30091 make env-dev-kind
 ```
 
-### Grafana Dashboard
-
-The bundled [Inference Gateway dashboard] covers EPP metrics across the inference
-pool, inference objective, and flow control layers.
-
-Add a Prometheus datasource at `http://localhost:30090`, then import the JSON via
-**Dashboards > New > Import**. See the
-[Grafana installation docs](https://grafana.com/docs/grafana/latest/setup-grafana/installation/)
-for setup.
-
-[Inference Gateway dashboard]:deploy/grafana/inference_gateway.json
+Grafana dashboards for the llm-d Router live in the
+[llm-d monorepo](https://github.com/llm-d/llm-d/tree/main/guides/recipes/observability/grafana/dashboards).
+See [Observability Setup](https://github.com/llm-d/llm-d/blob/main/docs/operations/observability/setup.md)
+for install and import instructions.
 
 > [!NOTE]
 > For significant customization beyond the standard deployment, use the `deploy/components`

Makefile

@@ -119,12 +119,14 @@ BUILDER_SOCK_FLAGS = $(DOCKER_GROUP_PARAM) \
 	-e CONTAINER_RUNTIME=docker
 endif
 
+E2E_NUM_PROCS ?= 5
+
 # Env vars forwarded into the e2e test container.
 # Add new image vars here so they are automatically passed through.
 # Should we pass ALL env vars here?
 E2E_ENV_VARS = EPP_IMAGE VLLM_IMAGE SIDECAR_IMAGE VLLM_RENDER_IMAGE \
                E2E_KEEP_CLUSTER_ON_FAILURE E2E_PORT E2E_METRICS_PORT K8S_CONTEXT READY_TIMEOUT \
-               E2E_LABEL_FILTER LOAD_VLLM_RENDER_IMAGE HF_TOKEN
+               E2E_NUM_PROCS LOAD_VLLM_RENDER_IMAGE HF_TOKEN
 BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
 ifneq ($(filter command line environment,$(origin NAMESPACE)),)
 BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)
@@ -159,7 +161,7 @@ LDFLAGS ?= -s -w
 BASE_IMAGE ?=
 
 # test packages
-epp_TEST_PACKAGES = $$(go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/ | tr '\n' ' ')
+epp_TEST_PACKAGES = $$(go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/ | grep -v ./pkg/coordinator/ | grep -v ./cmd/coordinator | tr '\n' ' ')
 sidecar_TEST_PACKAGES = ./pkg/sidecar/...
 
 # Internal variables for generic targets
@@ -313,6 +315,11 @@ bench-tokenizer: image-build-builder ## Run external tokenizer + scorer benchmar
 	@printf "Run 'EXTERNAL_TOKENIZER_ENABLED=true KV_CACHE_ENABLED=true make env-dev-kind' first.\n\n"
 	$(BUILDER_RUN_CLUSTER) 'go test -bench=. -benchmem -count=5 -timeout=5m ./test/profiling/tokenizerbench/'
 
+.PHONY: bench-smoke
+bench-smoke: image-build-builder ## Smoke-run the flowcontrol benchmarks once (-benchtime=1x) to catch runtime rot
+	@printf "\033[33;1m==== Running Flow Control Benchmark Smoke ====\033[0m\n"
+	$(BUILDER_RUN) 'go test -run=^$$ -bench=. -benchtime=1x -timeout=5m ./pkg/epp/flowcontrol/benchmark/...'
+
 .PHONY: post-deploy-test
 post-deploy-test: ## Run post deployment tests
 	@echo "Success!"