Skip to content

Commit e172114

Browse files
authored
Merge branch 'llm-d:main' into main
2 parents cdab1b5 + 6181fd9 commit e172114

147 files changed

Lines changed: 7842 additions & 2789 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/ci-lint.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262
uses: actions/checkout@v7
6363

6464
- name: Set up Go
65-
uses: actions/setup-go@v6
65+
uses: actions/setup-go@v7
6666
with:
6767
go-version-file: go.mod
6868
cache: true

.github/workflows/ci-pr-checks.yaml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
uses: actions/checkout@v7
6565

6666
- name: Set up Go
67-
uses: actions/setup-go@v6
67+
uses: actions/setup-go@v7
6868
with:
6969
go-version-file: go.mod
7070

@@ -113,6 +113,16 @@ jobs:
113113
GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
114114
run: make test-unit
115115

116+
# Anti-rot: execute the benchmark bodies once so runtime breakage (panics,
117+
# deadlocks) is caught. `go test` only compiles benchmarks; it never runs
118+
# them without -bench.
119+
- name: Run benchmark smoke
120+
shell: bash
121+
env:
122+
GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
123+
GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
124+
run: make bench-smoke
125+
116126
- name: Run hermetic integration tests
117127
shell: bash
118128
env:

.github/workflows/ci-release.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262
uses: actions/checkout@v7
6363

6464
- name: Set up Go
65-
uses: actions/setup-go@v6
65+
uses: actions/setup-go@v7
6666
with:
6767
go-version-file: go.mod
6868

.github/workflows/nightly-router-perf-test-optimized-baseline-10k-1k.yaml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,21 +67,21 @@ jobs:
6767
HF_TOKEN: ${{ secrets.HF_TOKEN }}
6868
steps:
6969
- name: Checkout llm-d/llm-d-router
70-
uses: actions/checkout@v6
70+
uses: actions/checkout@v7
7171

7272
- name: Checkout kubernetes-sigs/inference-perf
73-
uses: actions/checkout@v6
73+
uses: actions/checkout@v7
7474
with:
7575
repository: kubernetes-sigs/inference-perf
7676
path: inference-perf
7777

7878
- name: Authenticate to Google Cloud
79-
uses: google-github-actions/auth@v2
79+
uses: google-github-actions/auth@v3
8080
with:
8181
credentials_json: ${{ secrets.GKE_SA_KEY }}
8282

8383
- name: Set up gcloud CLI and kubectl
84-
uses: google-github-actions/setup-gcloud@v2
84+
uses: google-github-actions/setup-gcloud@v3
8585
with:
8686
project_id: ${{ env.GCP_PROJECT_ID }}
8787
install_components: 'kubectl,gke-gcloud-auth-plugin'
@@ -97,7 +97,7 @@ jobs:
9797
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
9898
9999
- name: Set up Python
100-
uses: actions/setup-python@v5
100+
uses: actions/setup-python@v7
101101
with:
102102
python-version: "3.10"
103103

@@ -125,7 +125,7 @@ jobs:
125125
--gcs-bucket ${{ env.GCS_BUCKET }}
126126
127127
- name: Upload Performance Results Artifact
128-
uses: actions/upload-artifact@v4
128+
uses: actions/upload-artifact@v7
129129
if: always()
130130
with:
131131
name: epp-perf-results

.github/workflows/pr-kind-label.yaml

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Apply kind labels from PR body
1+
name: Apply kind/area labels and milestone from PR body
22

33
on:
44
pull_request_target:
@@ -23,14 +23,53 @@ jobs:
2323
gh pr view "$PR_NUMBER" --repo "$REPO" --json labels \
2424
-q '.labels[].name | select(startswith("kind/"))' | \
2525
while IFS= read -r label; do
26-
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label"
26+
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" || echo "::warning::Failed to remove label '$label'"
2727
done
2828
2929
# Extract /kind values from PR body and apply labels
30-
printf '%s' "$PR_BODY" | tr -d '\r' | grep '^/kind ' | \
31-
sed 's|^/kind ||' | sort -u | \
30+
printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/kind[[:space:]][[:space:]]*||p' | sed 's|[[:space:]]*$||' | sort -u | \
3231
while IFS= read -r kind; do
3332
[ -z "$kind" ] && continue
3433
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "kind/$kind" || \
3534
echo "::warning::Label 'kind/$kind' not found in repository"
3635
done
36+
37+
- name: Set area labels
38+
env:
39+
GH_TOKEN: ${{ github.token }}
40+
PR_BODY: ${{ github.event.pull_request.body }}
41+
PR_NUMBER: ${{ github.event.pull_request.number }}
42+
REPO: ${{ github.repository }}
43+
run: |
44+
# Absence of a directive is a no-op. Area labels are also set by the
45+
# /area comment command and clearing here would wipe those.
46+
areas=$(printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/area[[:space:]][[:space:]]*||p' | \
47+
sed 's|[[:space:]]*$||' | sed '/^$/d' | sort -u)
48+
[ -z "$areas" ] && exit 0
49+
50+
# Remove existing area/* labels
51+
gh pr view "$PR_NUMBER" --repo "$REPO" --json labels \
52+
-q '.labels[].name | select(startswith("area/"))' | \
53+
while IFS= read -r label; do
54+
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" || echo "::warning::Failed to remove label '$label'"
55+
done
56+
57+
printf '%s\n' "$areas" | \
58+
while IFS= read -r area; do
59+
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "area/$area" || \
60+
echo "::warning::Label 'area/$area' not found in repository"
61+
done
62+
63+
- name: Set milestone
64+
env:
65+
GH_TOKEN: ${{ github.token }}
66+
PR_BODY: ${{ github.event.pull_request.body }}
67+
PR_NUMBER: ${{ github.event.pull_request.number }}
68+
REPO: ${{ github.repository }}
69+
run: |
70+
# Use the last /milestone directive in the PR body
71+
milestone=$(printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/milestone[[:space:]][[:space:]]*||p' | \
72+
sed 's|[[:space:]]*$||' | sed '/^$/d' | tail -n 1)
73+
[ -z "$milestone" ] && exit 0
74+
gh pr edit "$PR_NUMBER" --repo "$REPO" --milestone "$milestone" || \
75+
echo "::warning::Milestone '$milestone' not found in repository"

CODEOWNERS

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@
1414
* @llm-d/router-maintainers
1515

1616
# Flow control framework + plugins
17-
/pkg/epp/flowcontrol/ @LukeAVanDrie @shmuelk @llm-d/router-maintainers
18-
/pkg/epp/framework/interface/flowcontrol/ @LukeAVanDrie @shmuelk @llm-d/router-maintainers
19-
/pkg/epp/framework/plugins/flowcontrol/ @LukeAVanDrie @shmuelk @llm-d/router-maintainers
17+
/pkg/epp/flowcontrol/ @LukeAVanDrie @RishabhSaini @shmuelk @llm-d/router-maintainers
18+
/pkg/epp/framework/interface/flowcontrol/ @LukeAVanDrie @RishabhSaini @shmuelk @llm-d/router-maintainers
19+
/pkg/epp/framework/plugins/flowcontrol/ @LukeAVanDrie @RishabhSaini @shmuelk @llm-d/router-maintainers
2020

2121
# Envoy ext_proc message layer
2222
/pkg/common/envoy/ @shmuelk @llm-d/router-maintainers

DEVELOPMENT.md

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ Documentation for developing the llm-d Router.
1111
- [Kind Development Environment](#kind-development-environment)
1212
- [Accessing the Gateway](#accessing-the-gateway)
1313
- [Prometheus Monitoring](#prometheus-monitoring)
14-
- [Grafana Dashboard](#grafana-dashboard)
1514
- [Development Cycle](#development-cycle)
1615
- [Debugging](#debugging)
1716
- [Inference Disaggregation Modes](#inference-disaggregation-modes)
@@ -170,17 +169,10 @@ Prometheus will be accessible at `http://localhost:30090`. To use a different ho
170169
PROM_ENABLED=true KIND_PROM_HOST_PORT=30091 make env-dev-kind
171170
```
172171

173-
### Grafana Dashboard
174-
175-
The bundled [Inference Gateway dashboard] covers EPP metrics across the inference
176-
pool, inference objective, and flow control layers.
177-
178-
Add a Prometheus datasource at `http://localhost:30090`, then import the JSON via
179-
**Dashboards > New > Import**. See the
180-
[Grafana installation docs](https://grafana.com/docs/grafana/latest/setup-grafana/installation/)
181-
for setup.
182-
183-
[Inference Gateway dashboard]:deploy/grafana/inference_gateway.json
172+
Grafana dashboards for the llm-d Router live in the
173+
[llm-d monorepo](https://github.com/llm-d/llm-d/tree/main/guides/recipes/observability/grafana/dashboards).
174+
See [Observability Setup](https://github.com/llm-d/llm-d/blob/main/docs/operations/observability/setup.md)
175+
for install and import instructions.
184176

185177
> [!NOTE]
186178
> For significant customization beyond the standard deployment, use the `deploy/components`

Makefile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,11 @@ bench-tokenizer: image-build-builder ## Run external tokenizer + scorer benchmar
315315
@printf "Run 'EXTERNAL_TOKENIZER_ENABLED=true KV_CACHE_ENABLED=true make env-dev-kind' first.\n\n"
316316
$(BUILDER_RUN_CLUSTER) 'go test -bench=. -benchmem -count=5 -timeout=5m ./test/profiling/tokenizerbench/'
317317

318+
.PHONY: bench-smoke
319+
bench-smoke: image-build-builder ## Smoke-run the flowcontrol benchmarks once (-benchtime=1x) to catch runtime rot
320+
@printf "\033[33;1m==== Running Flow Control Benchmark Smoke ====\033[0m\n"
321+
$(BUILDER_RUN) 'go test -run=^$$ -bench=. -benchtime=1x -timeout=5m ./pkg/epp/flowcontrol/benchmark/...'
322+
318323
.PHONY: post-deploy-test
319324
post-deploy-test: ## Run post deployment tests
320325
@echo "Success!"

Makefile.coord.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ endif
116116
# Env vars forwarded into the e2e test container.
117117
E2E_ENV_VARS = COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT \
118118
E2E_GATEWAY_PORT E2E_KEEP_CLUSTER_ON_FAILURE \
119-
E2E_PRINT_COORDINATOR_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
119+
E2E_PRINT_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
120120
BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
121121
ifneq ($(filter command line environment,$(origin NAMESPACE)),)
122122
BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)

README.coord.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ kubectl --context kind-e2e-coordinator-tests get pods
341341
|---|---|---|
342342
| `E2E_KEEP_CLUSTER_ON_FAILURE` | `false` | Preserve the Kind cluster when the suite fails |
343343
| `E2E_GATEWAY_PORT` | `30080` | Host port mapped to the gateway NodePort |
344-
| `E2E_PRINT_COORDINATOR_LOGS` | `false` | Print coordinator pod logs during the run |
344+
| `E2E_PRINT_LOGS` | `false` | Print all pod logs (coordinator, EPPs, Envoy, workers) for every spec, not just on failure |
345345
| `CONTAINER_RUNTIME` | `docker` | Container runtime used to load images into Kind (`docker` or `podman`) |
346346
| `EPP_IMAGE` | `ghcr.io/llm-d/llm-d-router-endpoint-picker:dev` | EPP image loaded into the Kind cluster |
347347
| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` | vLLM image loaded into the Kind cluster |

0 commit comments

Comments
 (0)