Merge branch 'main' into docs/flow-control-eviction-design

Author: LukeAVanDrie

.github/actions/docker-build-and-push/action.yml

@@ -33,6 +33,10 @@ inputs:
     required: false
     description: Git commit SHA to embed via COMMIT_SHA build arg
     default: ''
+  additional-tags:
+    required: false
+    description: Additional image tags to push
+    default: ''
 runs:
   using: "composite"
   steps:
@@ -58,6 +62,7 @@ runs:
           ${{ inputs.registry }}/${{ inputs.image-name }}:${{ inputs.tag }}
           ${{ inputs.push == 'true' && inputs.prerelease != 'true' && format('{0}/{1}:latest', inputs.registry, inputs.image-name) || '' }}
           ${{ inputs.commit-sha != '' && format('{0}/{1}:{2}', inputs.registry, inputs.image-name, inputs.commit-sha) || '' }}
+          ${{ inputs.additional-tags }}
         build-args: |
           LDFLAGS=-s -w
           COMMIT_SHA=${{ inputs.commit-sha || 'unknown' }}

.github/actions/helm-build-and-push/action.yml

@@ -12,7 +12,7 @@ inputs:
     required: false
     default: ''
   epp_image_repository:
-    description: 'EPP image repository name (e.g. llm-d-router-endpoint-picker or llm-d-router-endpoint-picker-dev)'
+    description: 'EPP image repository name (e.g. llm-d-router-endpoint-picker)'
     required: false
     default: 'llm-d-router-endpoint-picker'
 runs:

.github/workflows/ci-build-images.yaml

@@ -19,6 +19,14 @@ on:
         required: false
         type: string
         default: ''
+      epp-additional-tags:
+        required: false
+        type: string
+        default: ''
+      sidecar-additional-tags:
+        required: false
+        type: string
+        default: ''
 
 permissions:
   contents: read
@@ -60,6 +68,7 @@ jobs:
           prerelease: ${{ inputs.prerelease }}
           commit-sha: ${{ github.sha }}
           push: 'true'
+          additional-tags: ${{ inputs.epp-additional-tags }}
 
   build-sidecar:
     runs-on: ubuntu-latest
@@ -93,6 +102,7 @@ jobs:
           prerelease: ${{ inputs.prerelease }}
           commit-sha: ${{ github.sha }}
           push: 'true'
+          additional-tags: ${{ inputs.sidecar-additional-tags }}
 
   push-helm-charts:
     needs: [build-epp, build-sidecar]

.github/workflows/ci-coordinator.yaml

@@ -0,0 +1,106 @@
+name: CI - Coordinator Test
+
+# Runs coordinator unit tests and e2e tests. Path filter excludes paths not
+# exercised by the coordinator.
+on:
+  push:
+    branches:
+      - main
+      - 'release-*'
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      src: ${{ steps.filter.outputs.src }}
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+      - uses: dorny/paths-filter@v4
+        id: filter
+        with:
+          filters: |
+            src:
+              - '**/*.go'
+              - Dockerfile.*
+              - Makefile*
+              - go.mod
+              - go.sum
+              - scripts/**
+              - hack/**
+              - test/**
+              - .github/actions/e2e-runner-setup/action.yml
+              - .github/workflows/ci-coordinator.yaml
+              - '!Dockerfile.sidecar'
+              - '!cmd/pd-sidecar/**'
+              - '!pkg/sidecar/**'
+              - '!test/sidecar/**'
+              - '!pkg/generator/**'
+              - '!test/e2e/**'
+              - '!test/integration/**'
+              - '!test/profiling/**'
+              - '!Makefile'
+
+  unit-tests:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.src == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+
+      - name: Create Go cache dirs
+        id: go-cache
+        run: |
+          mkdir -p "$HOME/.cache/llm-d-gomodcache" "$HOME/.cache/llm-d-gobuildcache"
+          echo "mod=$HOME/.cache/llm-d-gomodcache" >> "$GITHUB_OUTPUT"
+          echo "build=$HOME/.cache/llm-d-gobuildcache" >> "$GITHUB_OUTPUT"
+
+      - name: Cache Go modules and build cache
+        uses: actions/cache@v6
+        with:
+          path: |
+            ${{ steps.go-cache.outputs.mod }}
+            ${{ steps.go-cache.outputs.build }}
+          key: go-cache-${{ runner.os }}-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            go-cache-${{ runner.os }}-
+
+      - name: Run coordinator unit tests
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
+          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
+        run: make -f Makefile.coord.mk test-unit
+
+  e2e-tests:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.src == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+
+      - name: Set up e2e runner
+        id: e2e-setup
+        uses: ./.github/actions/e2e-runner-setup
+
+      - name: Run coordinator e2e tests
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
+          GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
+        run: make -f Makefile.coord.mk test-e2e-coordinator

.github/workflows/ci-dev.yaml

@@ -19,6 +19,8 @@ jobs:
     outputs:
       epp_name: ${{ steps.version.outputs.epp_name }}
       sidecar_name: ${{ steps.version.outputs.sidecar_name }}
+      epp_base_name: ${{ steps.version.outputs.epp_base_name }}
+      sidecar_base_name: ${{ steps.version.outputs.sidecar_base_name }}
       tag: ${{ steps.tag.outputs.tag }}
     steps:
       - name: Set image names
@@ -27,6 +29,8 @@ jobs:
           repo="${GITHUB_REPOSITORY##*/}"
           echo "epp_name=${repo}-endpoint-picker-dev" >> "$GITHUB_OUTPUT"
           echo "sidecar_name=${repo}-disagg-sidecar-dev" >> "$GITHUB_OUTPUT"
+          echo "epp_base_name=${repo}-endpoint-picker" >> "$GITHUB_OUTPUT"
+          echo "sidecar_base_name=${repo}-disagg-sidecar" >> "$GITHUB_OUTPUT"
 
       - name: Set branch name as tag
         id: tag
@@ -42,5 +46,7 @@ jobs:
       sidecar-image-name: ${{ needs.set-params.outputs.sidecar_name }}
       tag: ${{ needs.set-params.outputs.tag }}
       prerelease: true
-      chart-suffix: "-dev"
+      chart-suffix: ""
+      epp-additional-tags: ghcr.io/llm-d/${{ needs.set-params.outputs.epp_base_name }}:main
+      sidecar-additional-tags: ghcr.io/llm-d/${{ needs.set-params.outputs.sidecar_base_name }}:main
 

.github/workflows/ci-lint.yaml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
           cache: true

.github/workflows/ci-pr-checks.yaml

@@ -1,4 +1,4 @@
-# Matrix jobs (e2e-image-common, e2e-image-router, e2e-router) are each
+# Matrix jobs (e2e-image-common, e2e-image-router) are each
 # followed by a *-status job that aggregates their result, since GitHub
 # required status checks cannot reference individual matrix legs.
 name: CI - Test
@@ -45,6 +45,11 @@ jobs:
               - .github/actions/docker-build-and-push/action.yml
               - .github/actions/e2e-runner-setup/action.yml
               - .github/workflows/ci-pr-checks.yaml
+              - '!Dockerfile.coordinator'
+              - '!Makefile.coord.mk'
+              - '!cmd/coordinator/**'
+              - '!pkg/coordinator/**'
+              - '!test/coordinator/**'
             helm:
               - 'config/charts/**'
               - 'hack/verify-helm.sh'
@@ -59,7 +64,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
 
@@ -108,6 +113,16 @@ jobs:
           GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
         run: make test-unit
 
+      # Anti-rot: execute the benchmark bodies once so runtime breakage (panics,
+      # deadlocks) is caught. `go test` only compiles benchmarks; it never runs
+      # them without -bench.
+      - name: Run benchmark smoke
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
+          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
+        run: make bench-smoke
+
       - name: Run hermetic integration tests
         shell: bash
         env:
@@ -343,29 +358,6 @@ jobs:
     if: ${{ needs.check-changes.outputs.src == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        suite:
-          - name: pd
-            label-filter: "!Disruptive && !Extended && !SharedStorage && !Metrics"
-            needs-renderer: "false"
-          - name: pd-shared-storage-deprecated
-            label-filter: "SharedStorage && DeprecatedPD"
-            needs-renderer: "false"
-          - name: pd-shared-storage-disagg
-            label-filter: "SharedStorage && Disagg"
-            needs-renderer: "false"
-          - name: pd-metrics
-            label-filter: "Metrics"
-            needs-renderer: "false"
-          - name: extended
-            label-filter: "Extended"
-            needs-renderer: "true"
-          - name: disruption
-            label-filter: "Disruptive"
-            needs-renderer: "false"
-    name: e2e-router (${{ matrix.suite.name }})
     steps:
       - name: Checkout source
         uses: actions/checkout@v7
@@ -399,7 +391,6 @@ jobs:
           path: ${{ runner.temp }}/e2e-images
 
       - name: Download renderer image
-        if: ${{ matrix.suite.needs-renderer == 'true' }}
         uses: actions/download-artifact@v8
         with:
           name: e2e-image-renderer
@@ -412,26 +403,11 @@ jobs:
             docker load --input "$image"
           done
 
-      - name: Run e2e-router (${{ matrix.suite.name }})
+      - name: Run e2e-router
         shell: bash
         env:
           GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
           GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
-          E2E_LABEL_FILTER: ${{ matrix.suite.label-filter }}
-          LOAD_VLLM_RENDER_IMAGE: ${{ matrix.suite.needs-renderer }}
           PULL_VLLM_RENDER_IMAGE: "false"
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: make test-e2e-run
-
-  e2e-router-status:
-    needs: e2e-router
-    if: always()
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check e2e-router result
-        run: |
-          result="${{ needs.e2e-router.result }}"
-          if [[ "$result" != "success" && "$result" != "skipped" ]]; then
-            echo "::error::e2e-router failed for at least one matrix suite"
-            exit 1
-          fi

.github/workflows/ci-release.yaml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
 

.github/workflows/nightly-router-perf-test-optimized-baseline-10k-1k.yaml

@@ -67,21 +67,21 @@ jobs:
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
     steps:
       - name: Checkout llm-d/llm-d-router
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Checkout kubernetes-sigs/inference-perf
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: kubernetes-sigs/inference-perf
           path: inference-perf
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@v3
         with:
           credentials_json: ${{ secrets.GKE_SA_KEY }}
 
       - name: Set up gcloud CLI and kubectl
-        uses: google-github-actions/setup-gcloud@v2
+        uses: google-github-actions/setup-gcloud@v3
         with:
           project_id: ${{ env.GCP_PROJECT_ID }}
           install_components: 'kubectl,gke-gcloud-auth-plugin'
@@ -97,7 +97,7 @@ jobs:
           curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: "3.10"
 
@@ -125,7 +125,7 @@ jobs:
             --gcs-bucket ${{ env.GCS_BUCKET }}
 
       - name: Upload Performance Results Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: always()
         with:
           name: epp-perf-results

DEVELOPMENT.md

@@ -11,7 +11,6 @@ Documentation for developing the llm-d Router.
   - [Kind Development Environment](#kind-development-environment)
     - [Accessing the Gateway](#accessing-the-gateway)
     - [Prometheus Monitoring](#prometheus-monitoring)
-    - [Grafana Dashboard](#grafana-dashboard)
     - [Development Cycle](#development-cycle)
     - [Debugging](#debugging)
     - [Inference Disaggregation Modes](#inference-disaggregation-modes)
@@ -170,17 +169,10 @@ Prometheus will be accessible at `http://localhost:30090`. To use a different ho
 PROM_ENABLED=true KIND_PROM_HOST_PORT=30091 make env-dev-kind
 ```
 
-### Grafana Dashboard
-
-The bundled [Inference Gateway dashboard] covers EPP metrics across the inference
-pool, inference objective, and flow control layers.
-
-Add a Prometheus datasource at `http://localhost:30090`, then import the JSON via
-**Dashboards > New > Import**. See the
-[Grafana installation docs](https://grafana.com/docs/grafana/latest/setup-grafana/installation/)
-for setup.
-
-[Inference Gateway dashboard]:deploy/grafana/inference_gateway.json
+Grafana dashboards for the llm-d Router live in the
+[llm-d monorepo](https://github.com/llm-d/llm-d/tree/main/guides/recipes/observability/grafana/dashboards).
+See [Observability Setup](https://github.com/llm-d/llm-d/blob/main/docs/operations/observability/setup.md)
+for install and import instructions.
 
 > [!NOTE]
 > For significant customization beyond the standard deployment, use the `deploy/components`