Merge branch 'main' into ashar/hybrid-concurrency-detector

Author: asharkhan3101

.github/workflows/ci-lint.yaml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
           cache: true

.github/workflows/ci-pr-checks.yaml

@@ -1,4 +1,4 @@
-# Matrix jobs (e2e-image-common, e2e-image-router, e2e-router) are each
+# Matrix jobs (e2e-image-common, e2e-image-router) are each
 # followed by a *-status job that aggregates their result, since GitHub
 # required status checks cannot reference individual matrix legs.
 name: CI - Test
@@ -64,7 +64,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
 
@@ -113,6 +113,16 @@ jobs:
           GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
         run: make test-unit
 
+      # Anti-rot: execute the benchmark bodies once so runtime breakage (panics,
+      # deadlocks) is caught. `go test` only compiles benchmarks; it never runs
+      # them without -bench.
+      - name: Run benchmark smoke
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
+          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
+        run: make bench-smoke
+
       - name: Run hermetic integration tests
         shell: bash
         env:
@@ -348,29 +358,6 @@ jobs:
     if: ${{ needs.check-changes.outputs.src == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        suite:
-          - name: pd
-            label-filter: "!Disruptive && !Extended && !SharedStorage && !Metrics"
-            needs-renderer: "false"
-          - name: pd-shared-storage-deprecated
-            label-filter: "SharedStorage && DeprecatedPD"
-            needs-renderer: "false"
-          - name: pd-shared-storage-disagg
-            label-filter: "SharedStorage && Disagg"
-            needs-renderer: "false"
-          - name: pd-metrics
-            label-filter: "Metrics"
-            needs-renderer: "false"
-          - name: extended
-            label-filter: "Extended"
-            needs-renderer: "true"
-          - name: disruption
-            label-filter: "Disruptive"
-            needs-renderer: "false"
-    name: e2e-router (${{ matrix.suite.name }})
     steps:
       - name: Checkout source
         uses: actions/checkout@v7
@@ -404,7 +391,6 @@ jobs:
           path: ${{ runner.temp }}/e2e-images
 
       - name: Download renderer image
-        if: ${{ matrix.suite.needs-renderer == 'true' }}
         uses: actions/download-artifact@v8
         with:
           name: e2e-image-renderer
@@ -417,26 +403,11 @@ jobs:
             docker load --input "$image"
           done
 
-      - name: Run e2e-router (${{ matrix.suite.name }})
+      - name: Run e2e-router
         shell: bash
         env:
           GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
           GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
-          E2E_LABEL_FILTER: ${{ matrix.suite.label-filter }}
-          LOAD_VLLM_RENDER_IMAGE: ${{ matrix.suite.needs-renderer }}
           PULL_VLLM_RENDER_IMAGE: "false"
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: make test-e2e-run
-
-  e2e-router-status:
-    needs: e2e-router
-    if: always()
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check e2e-router result
-        run: |
-          result="${{ needs.e2e-router.result }}"
-          if [[ "$result" != "success" && "$result" != "skipped" ]]; then
-            echo "::error::e2e-router failed for at least one matrix suite"
-            exit 1
-          fi

.github/workflows/ci-release.yaml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
 

.github/workflows/nightly-router-perf-test-optimized-baseline-10k-1k.yaml

@@ -67,21 +67,21 @@ jobs:
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
     steps:
       - name: Checkout llm-d/llm-d-router
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Checkout kubernetes-sigs/inference-perf
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: kubernetes-sigs/inference-perf
           path: inference-perf
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@v3
         with:
           credentials_json: ${{ secrets.GKE_SA_KEY }}
 
       - name: Set up gcloud CLI and kubectl
-        uses: google-github-actions/setup-gcloud@v2
+        uses: google-github-actions/setup-gcloud@v3
         with:
           project_id: ${{ env.GCP_PROJECT_ID }}
           install_components: 'kubectl,gke-gcloud-auth-plugin'
@@ -97,7 +97,7 @@ jobs:
           curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: "3.10"
 
@@ -125,7 +125,7 @@ jobs:
             --gcs-bucket ${{ env.GCS_BUCKET }}
 
       - name: Upload Performance Results Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: always()
         with:
           name: epp-perf-results

DEVELOPMENT.md

@@ -11,7 +11,6 @@ Documentation for developing the llm-d Router.
   - [Kind Development Environment](#kind-development-environment)
     - [Accessing the Gateway](#accessing-the-gateway)
     - [Prometheus Monitoring](#prometheus-monitoring)
-    - [Grafana Dashboard](#grafana-dashboard)
     - [Development Cycle](#development-cycle)
     - [Debugging](#debugging)
     - [Inference Disaggregation Modes](#inference-disaggregation-modes)
@@ -170,17 +169,10 @@ Prometheus will be accessible at `http://localhost:30090`. To use a different ho
 PROM_ENABLED=true KIND_PROM_HOST_PORT=30091 make env-dev-kind
 ```
 
-### Grafana Dashboard
-
-The bundled [Inference Gateway dashboard] covers EPP metrics across the inference
-pool, inference objective, and flow control layers.
-
-Add a Prometheus datasource at `http://localhost:30090`, then import the JSON via
-**Dashboards > New > Import**. See the
-[Grafana installation docs](https://grafana.com/docs/grafana/latest/setup-grafana/installation/)
-for setup.
-
-[Inference Gateway dashboard]:deploy/grafana/inference_gateway.json
+Grafana dashboards for the llm-d Router live in the
+[llm-d monorepo](https://github.com/llm-d/llm-d/tree/main/guides/recipes/observability/grafana/dashboards).
+See [Observability Setup](https://github.com/llm-d/llm-d/blob/main/docs/operations/observability/setup.md)
+for install and import instructions.
 
 > [!NOTE]
 > For significant customization beyond the standard deployment, use the `deploy/components`

Makefile

@@ -119,12 +119,14 @@ BUILDER_SOCK_FLAGS = $(DOCKER_GROUP_PARAM) \
 	-e CONTAINER_RUNTIME=docker
 endif
 
+E2E_NUM_PROCS ?= 5
+
 # Env vars forwarded into the e2e test container.
 # Add new image vars here so they are automatically passed through.
 # Should we pass ALL env vars here?
 E2E_ENV_VARS = EPP_IMAGE VLLM_IMAGE SIDECAR_IMAGE VLLM_RENDER_IMAGE \
                E2E_KEEP_CLUSTER_ON_FAILURE E2E_PORT E2E_METRICS_PORT K8S_CONTEXT READY_TIMEOUT \
-               E2E_LABEL_FILTER LOAD_VLLM_RENDER_IMAGE HF_TOKEN
+               E2E_NUM_PROCS LOAD_VLLM_RENDER_IMAGE HF_TOKEN
 BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
 ifneq ($(filter command line environment,$(origin NAMESPACE)),)
 BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)
@@ -313,6 +315,11 @@ bench-tokenizer: image-build-builder ## Run external tokenizer + scorer benchmar
 	@printf "Run 'EXTERNAL_TOKENIZER_ENABLED=true KV_CACHE_ENABLED=true make env-dev-kind' first.\n\n"
 	$(BUILDER_RUN_CLUSTER) 'go test -bench=. -benchmem -count=5 -timeout=5m ./test/profiling/tokenizerbench/'
 
+.PHONY: bench-smoke
+bench-smoke: image-build-builder ## Smoke-run the flowcontrol benchmarks once (-benchtime=1x) to catch runtime rot
+	@printf "\033[33;1m==== Running Flow Control Benchmark Smoke ====\033[0m\n"
+	$(BUILDER_RUN) 'go test -run=^$$ -bench=. -benchtime=1x -timeout=5m ./pkg/epp/flowcontrol/benchmark/...'
+
 .PHONY: post-deploy-test
 post-deploy-test: ## Run post deployment tests
 	@echo "Success!"

cmd/epp/runner/runner.go

@@ -60,13 +60,18 @@ import (
 	fwkdl "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/datalayer"
 	fwkplugin "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/plugin"
 	attrconcurrency "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/concurrency"
+	attrgpu "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/gpu"
 	attrlatency "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/latency"
 	attrmodels "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/models"
 	attrprefix "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/prefix"
 	attrsession "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/session"
+	attrtopology "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/topology"
 	discoveryfile "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/discovery/file"
+	extdcgm "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/dcgm"
 	extractormetrics "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/metrics"
 	extmodels "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/models"
+	exttopology "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/topology"
+	srcdcgm "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/dcgm"
 	sourcemetrics "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/metrics"
 	srcmodels "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/models"
 	sourcenotifications "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/notifications"
@@ -563,6 +568,11 @@ func (r *Runner) registerInTreePlugins() {
 	// data layer models source/extractor
 	fwkplugin.Register(srcmodels.ModelsDataSourceType, srcmodels.ModelDataSourceFactory)
 	fwkplugin.Register(attrmodels.ModelsExtractorType, extmodels.ModelServerExtractorFactory)
+	fwkplugin.Register(attrtopology.TopologyExtractorType, exttopology.Factory)
+
+	// data layer DCGM source/extractor
+	fwkplugin.Register(srcdcgm.DCGMDataSourceType, srcdcgm.DCGMDataSourceFactory)
+	fwkplugin.Register(attrgpu.DCGMExtractorType, extdcgm.DCGMExtractorFactory)
 
 	fwkplugin.Register(prefix.PrefixCacheScorerPluginType, prefix.PrefixCachePluginFactory)
 	fwkplugin.Register(maxscore.MaxScorePickerType, maxscore.MaxScorePickerFactory)

deploy/environments/dev/e2e-infra/envoy.yaml

@@ -299,7 +299,7 @@ spec:
     port: 8081
     protocol: TCP
     targetPort: 8081
-    nodePort: 30080
+    nodePort: ${ENVOY_NODE_PORT}
   selector:
     app: envoy
   type: NodePort

deploy/environments/dev/e2e-infra/services.yaml

@@ -30,5 +30,5 @@ spec:
       protocol: TCP
       port: 9090
       targetPort: 9090
-      nodePort: 32090
+      nodePort: ${METRICS_NODE_PORT}
   type: NodePort