From 181aa8358916e19b8844ccc752b2d6153d4b2ad6 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Tue, 16 Jun 2026 13:10:29 -0700 Subject: [PATCH 1/6] release: set LATENCY_PREDICTOR_TAG to v0.8.0-rc.1 Signed-off-by: Cong Liu --- hack/push-chart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/push-chart.sh b/hack/push-chart.sh index 779dccf013..04c01ab013 100755 --- a/hack/push-chart.sh +++ b/hack/push-chart.sh @@ -26,7 +26,7 @@ IMAGE_REGISTRY=${IMAGE_REGISTRY:-ghcr.io/llm-d} AGENTGATEWAY_TAG=${AGENTGATEWAY_TAG:-${EXTRA_TAG}} CHART_SUFFIX=${CHART_SUFFIX:-""} EPP_RELEASE_IMAGE_REPOSITORY=${EPP_RELEASE_IMAGE_REPOSITORY:-llm-d-router-endpoint-picker} -LATENCY_PREDICTOR_TAG=${LATENCY_PREDICTOR_TAG:-latest} +LATENCY_PREDICTOR_TAG=${LATENCY_PREDICTOR_TAG:-"v0.8.0-rc.1"} export EXTRA_TAG AGENTGATEWAY_TAG IMAGE_REGISTRY EPP_RELEASE_IMAGE_REPOSITORY LATENCY_PREDICTOR_TAG CHART_SUFFIX HELM_CHART_REPO=${HELM_CHART_REPO:-${IMAGE_REGISTRY}/charts} From d528f188d028b7f38e850a7953f6ae12f1fc6c1e Mon Sep 17 00:00:00 2001 From: Maroon Ayoub Date: Fri, 19 Jun 2026 07:13:54 +0300 Subject: [PATCH 2/6] deps(go): bump llm-d-kv-cache to v0.9.0 (#1691) Update github.com/llm-d/llm-d-kv-cache from v0.9.0-rc.1 to the released v0.9.0. Signed-off-by: Maroon Ayoub Signed-off-by: Cong Liu --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6d21aa4811..bd29e9e972 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jellydator/ttlcache/v3 v3.4.0 - github.com/llm-d/llm-d-kv-cache v0.9.0-rc.1 + github.com/llm-d/llm-d-kv-cache v0.9.0 github.com/onsi/ginkgo/v2 v2.28.3 github.com/onsi/gomega v1.40.0 github.com/openai/openai-go v1.12.0 diff --git a/go.sum b/go.sum index 0fa9ca50d2..83ccb3af74 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/llm-d/llm-d-kv-cache v0.9.0-rc.1 h1:wg/lxAkdbIqVCuhY9gYoJJTXZtGGYTj73B5VAclbMFE= -github.com/llm-d/llm-d-kv-cache v0.9.0-rc.1/go.mod h1:dxtEGCGrKV7PwRhpSAOfJi3rf8NOhUeL4RNXW945HNU= +github.com/llm-d/llm-d-kv-cache v0.9.0 h1:6hLLJGrP91A28LYH2Gd5i81e5sreSSHizdlSnB9NXqc= +github.com/llm-d/llm-d-kv-cache v0.9.0/go.mod h1:dxtEGCGrKV7PwRhpSAOfJi3rf8NOhUeL4RNXW945HNU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= From f95e36b9989ac5b8b98bfebebcb455ce36e52ea6 Mon Sep 17 00:00:00 2001 From: weizhoublue <45163302+weizhoublue@users.noreply.github.com> Date: Sun, 21 Jun 2026 20:50:56 +0800 Subject: [PATCH 3/6] fix(chart): update latency predictor image path to match subchart nesting (#1683) Signed-off-by: weizhoublue Signed-off-by: Cong Liu --- hack/push-chart.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hack/push-chart.sh b/hack/push-chart.sh index 04c01ab013..16d9d42627 100755 --- a/hack/push-chart.sh +++ b/hack/push-chart.sh @@ -50,14 +50,14 @@ then exit 1 fi ${YQ} -i \ - '.latencyPredictor.trainingServer.image.registry=strenv(IMAGE_REGISTRY) | - .latencyPredictor.trainingServer.image.repository="llm-d-latency-predictor-training-server" | - .latencyPredictor.trainingServer.image.tag=strenv(LATENCY_PREDICTOR_TAG) | - .latencyPredictor.trainingServer.image.pullPolicy="IfNotPresent" | - .latencyPredictor.predictionServers.image.registry=strenv(IMAGE_REGISTRY) | - .latencyPredictor.predictionServers.image.repository="llm-d-latency-predictor-prediction-server" | - .latencyPredictor.predictionServers.image.tag=strenv(LATENCY_PREDICTOR_TAG) | - .latencyPredictor.predictionServers.image.pullPolicy="IfNotPresent"' \ + '.router.latencyPredictor.trainingServer.image.registry=strenv(IMAGE_REGISTRY) | + .router.latencyPredictor.trainingServer.image.repository="llm-d-latency-predictor-training-server" | + .router.latencyPredictor.trainingServer.image.tag=strenv(LATENCY_PREDICTOR_TAG) | + .router.latencyPredictor.trainingServer.image.pullPolicy="IfNotPresent" | + .router.latencyPredictor.predictionServers.image.registry=strenv(IMAGE_REGISTRY) | + .router.latencyPredictor.predictionServers.image.repository="llm-d-latency-predictor-prediction-server" | + .router.latencyPredictor.predictionServers.image.tag=strenv(LATENCY_PREDICTOR_TAG) | + .router.latencyPredictor.predictionServers.image.pullPolicy="IfNotPresent"' \ config/charts/${CHART}/values.yaml if [[ ${CHART} == "llm-d-router-standalone" ]]; then ${YQ} -i \ From 6322b6e13e3d0f6ac1741596b7595541834813f5 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Mon, 22 Jun 2026 15:10:58 -0700 Subject: [PATCH 4/6] ci: add artifacts target and upload-artifacts job for CRDs (#1713) Signed-off-by: Cong Liu --- .github/workflows/ci-release.yaml | 40 +++++++++++++++++++++++++++++++ Makefile | 13 ++++++++++ 2 files changed, 53 insertions(+) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index 3391c04824..fa914f946f 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -51,3 +51,43 @@ jobs: tag: ${{ needs.set-params.outputs.tag }} prerelease: ${{ fromJSON(needs.set-params.outputs.prerelease) }} chart-suffix: "" + + upload-artifacts: + needs: set-params + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout source + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Setup Kubectl + uses: azure/setup-kubectl@v4 + with: + version: 'latest' + + - name: Build Artifacts + run: | + make artifacts + + - name: Upload Release Assets + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + files=$(find artifacts -type f -size +0c | tr '\n' ' ') + if [ -z "$files" ]; then + echo "No artifacts to upload." + exit 0 + fi + if gh release view "${{ needs.set-params.outputs.tag }}" >/dev/null 2>&1; then + gh release upload "${{ needs.set-params.outputs.tag }}" $files --clobber + else + echo "Release ${{ needs.set-params.outputs.tag }} does not exist. Creating a draft release..." + gh release create "${{ needs.set-params.outputs.tag }}" $files --draft --title "${{ needs.set-params.outputs.tag }}" --notes "Release ${{ needs.set-params.outputs.tag }}" + fi + diff --git a/Makefile b/Makefile index 3fb12990e9..0c2f1b8d95 100644 --- a/Makefile +++ b/Makefile @@ -359,6 +359,19 @@ helm-push-standalone: ## Package and push the llm-d-router-standalone Helm chart $(MAKE) helm-push CHART=llm-d-router-standalone +##@ Release + +.PHONY: artifacts +artifacts: yq check-kustomize ## Generate release artifacts (CRD manifests) + if [ -d artifacts ]; then rm -rf artifacts; fi + mkdir -p artifacts + kubectl kustomize config/crd > artifacts/manifests_all.yaml + $(YQ) -P 'select(.spec.group == "llm-d.ai")' artifacts/manifests_all.yaml > artifacts/manifests.yaml + rm -f artifacts/manifests_all.yaml + $(YQ) -P 'select(.spec.versions | map(.name == "v1") | any)' artifacts/manifests.yaml > artifacts/v1-manifests.yaml + $(YQ) -P 'select(.spec.versions | map(.name != "v1") | all)' artifacts/manifests.yaml > artifacts/experimental-manifests.yaml + + ##@ Coverage COVERAGE_DIR ?= coverage From 573daaf5675909929be9af7e50fe9891bb38b06e Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Tue, 23 Jun 2026 01:14:58 -0700 Subject: [PATCH 5/6] fix: bump golang.org/x/crypto to v0.52.0 to resolve Trivy CVEs (#1720) Bumps golang.org/x/crypto from v0.51.0 to v0.52.0 to fix 8 HIGH severity CVEs (CVE-2026-39827 through CVE-2026-46598) that caused the sidecar image Trivy scan to fail in CI. Signed-off-by: Cong Liu --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bd29e9e972..94bb7af13b 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/zap v1.28.0 - golang.org/x/crypto v0.51.0 + golang.org/x/crypto v0.52.0 golang.org/x/sync v0.20.0 golang.org/x/time v0.15.0 google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa diff --git a/go.sum b/go.sum index 83ccb3af74..ff3e0cd31a 100644 --- a/go.sum +++ b/go.sum @@ -280,8 +280,8 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= From cde5e85b362475c6df1fb534bd97d5c5aaa291ef Mon Sep 17 00:00:00 2001 From: "llm-d-router-release-notes[bot]" <287676111+llm-d-router-release-notes[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:11:15 +0000 Subject: [PATCH 6/6] docs: assemble release notes for v0.9.2 Signed-off-by: llm-d-router-release-notes[bot] <287676111+llm-d-router-release-notes[bot]@users.noreply.github.com> --- RELEASE-NOTES.md | 42 ++++++++++++++++++++++++++++++ release-notes.d/unreleased/1030.md | 7 ----- release-notes.d/unreleased/1121.md | 7 ----- release-notes.d/unreleased/1160.md | 7 ----- release-notes.d/unreleased/1206.md | 7 ----- release-notes.d/unreleased/1218.md | 7 ----- release-notes.d/unreleased/1234.md | 7 ----- release-notes.d/unreleased/1244.md | 7 ----- release-notes.d/unreleased/1247.md | 7 ----- release-notes.d/unreleased/1248.md | 7 ----- release-notes.d/unreleased/1276.md | 7 ----- release-notes.d/unreleased/1288.md | 7 ----- release-notes.d/unreleased/1302.md | 7 ----- release-notes.d/unreleased/1372.md | 7 ----- release-notes.d/unreleased/1402.md | 7 ----- release-notes.d/unreleased/1418.md | 7 ----- release-notes.d/unreleased/1426.md | 7 ----- release-notes.d/unreleased/1429.md | 7 ----- release-notes.d/unreleased/1436.md | 7 ----- release-notes.d/unreleased/1444.md | 7 ----- release-notes.d/unreleased/1449.md | 7 ----- release-notes.d/unreleased/1475.md | 7 ----- release-notes.d/unreleased/1488.md | 7 ----- release-notes.d/unreleased/1493.md | 7 ----- release-notes.d/unreleased/1509.md | 7 ----- release-notes.d/unreleased/1513.md | 7 ----- release-notes.d/unreleased/1515.md | 7 ----- release-notes.d/unreleased/1536.md | 7 ----- release-notes.d/unreleased/1539.md | 7 ----- release-notes.d/unreleased/1548.md | 7 ----- release-notes.d/unreleased/1550.md | 7 ----- release-notes.d/unreleased/1554.md | 7 ----- release-notes.d/unreleased/1575.md | 7 ----- release-notes.d/unreleased/1603.md | 7 ----- release-notes.d/unreleased/1607.md | 7 ----- release-notes.d/unreleased/1608.md | 7 ----- release-notes.d/unreleased/1626.md | 7 ----- release-notes.d/unreleased/1631.md | 7 ----- release-notes.d/unreleased/1651.md | 7 ----- release-notes.d/unreleased/1653.md | 7 ----- release-notes.d/unreleased/1661.md | 7 ----- 41 files changed, 42 insertions(+), 280 deletions(-) delete mode 100644 release-notes.d/unreleased/1030.md delete mode 100644 release-notes.d/unreleased/1121.md delete mode 100644 release-notes.d/unreleased/1160.md delete mode 100644 release-notes.d/unreleased/1206.md delete mode 100644 release-notes.d/unreleased/1218.md delete mode 100644 release-notes.d/unreleased/1234.md delete mode 100644 release-notes.d/unreleased/1244.md delete mode 100644 release-notes.d/unreleased/1247.md delete mode 100644 release-notes.d/unreleased/1248.md delete mode 100644 release-notes.d/unreleased/1276.md delete mode 100644 release-notes.d/unreleased/1288.md delete mode 100644 release-notes.d/unreleased/1302.md delete mode 100644 release-notes.d/unreleased/1372.md delete mode 100644 release-notes.d/unreleased/1402.md delete mode 100644 release-notes.d/unreleased/1418.md delete mode 100644 release-notes.d/unreleased/1426.md delete mode 100644 release-notes.d/unreleased/1429.md delete mode 100644 release-notes.d/unreleased/1436.md delete mode 100644 release-notes.d/unreleased/1444.md delete mode 100644 release-notes.d/unreleased/1449.md delete mode 100644 release-notes.d/unreleased/1475.md delete mode 100644 release-notes.d/unreleased/1488.md delete mode 100644 release-notes.d/unreleased/1493.md delete mode 100644 release-notes.d/unreleased/1509.md delete mode 100644 release-notes.d/unreleased/1513.md delete mode 100644 release-notes.d/unreleased/1515.md delete mode 100644 release-notes.d/unreleased/1536.md delete mode 100644 release-notes.d/unreleased/1539.md delete mode 100644 release-notes.d/unreleased/1548.md delete mode 100644 release-notes.d/unreleased/1550.md delete mode 100644 release-notes.d/unreleased/1554.md delete mode 100644 release-notes.d/unreleased/1575.md delete mode 100644 release-notes.d/unreleased/1603.md delete mode 100644 release-notes.d/unreleased/1607.md delete mode 100644 release-notes.d/unreleased/1608.md delete mode 100644 release-notes.d/unreleased/1626.md delete mode 100644 release-notes.d/unreleased/1631.md delete mode 100644 release-notes.d/unreleased/1651.md delete mode 100644 release-notes.d/unreleased/1653.md delete mode 100644 release-notes.d/unreleased/1661.md diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 311f370948..6036fbcf48 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,45 @@ +RELEASE v0.9.2 2026-06-23 +2026-05-25 https://github.com/llm-d/llm-d-router/pull/1030 InFlightLoadProducer now reliably tracks global token and request counts in the presence of timeouts, disconnects, and long-lived streams. +2026-05-25 https://github.com/llm-d/llm-d-router/pull/1218 EPP can now run without a Kubernetes cluster. When `dataLayer.discovery.pluginRef` is set, the runner skips controller-manager setup and drives endpoint discovery through the file-discovery plugin ("file-discovery"). See docs/discovery.md and pkg/epp/framework/plugins/datalayer/discovery/file/README.md for more details. +2026-05-25 https://github.com/llm-d/llm-d-router/pull/1247 The EndpointPickerConfig API has been refactored to provide a more structured and logical grouping of configuration fields. These changes improve schema clarity and provide a cleaner foundation for future feature extensions. Existing YAML/JSON configuration files must be updated to reflect the new nested structure, old fields will continue to work for two releases: - Saturation Detector Migration: The SaturationDetector field has been moved from the top-level configuration into the FlowControl block. - Parser Encapsulation: A new requestHandler struct has been introduced to house request-handler component configurations. The parser field has been moved into this new block. +2026-05-25 https://github.com/llm-d/llm-d-router/pull/1276 Fix cached prompt-token usage extraction in the EPP OpenAI parser. `cached_tokens` is now read from `usage.prompt_tokens_details` (and from `usage.input_tokens_details` for the Responses API), so prompt cache-hit metrics are recorded instead of being silently dropped. +2026-05-26 https://github.com/llm-d/llm-d-router/pull/1244 Add `agent-identity` plugin that derives `FairnessID` from agent session headers (Claude Code, OpenCode, Codex). +2026-05-26 https://github.com/llm-d/llm-d-router/pull/1302 The helm charts are now released as part of the llm-d router project, they went through several structural changes and validation safeguards for users migrating from gateway-api-inference-extension ; see the "Migrating from gateway-api-inference-extension" section in https://github.com/llm-d/llm-d-router/blob/main/config/charts/README.md for the migration guide. +2026-05-27 https://github.com/llm-d/llm-d-router/pull/1234 Update the default vLLM and simulator images, and remove the UDS Tokenizer and the `UDS_TOKENIZER_IMAGE` environment variable. Use `VLLM_RENDER_IMAGE` environment variable to define the render image name. +2026-05-27 https://github.com/llm-d/llm-d-router/pull/1372 Added a `session-id-producer` DataProducer plugin (type: `session-id-producer`), which extracts a session identifier from a configured request header or cookie and publishes it as the `SessionID` attribute on the request attribute store for use by future affinity-aware scorers and filters. +2026-05-28 https://github.com/llm-d/llm-d-router/pull/1248 New `/inference/v1/generate` endpoint is added, that accepts pre-tokenized prompts (`token_ids`) and optional multimodal features (image/audio/video hashes and placeholder ranges). To enable, configure the new `vllmhttp-parser` (Helm value `router.epp.parser=vllmhttp-parser`, or set `parser: vllmhttp-parser` in EPP configuration). The parser handles `/inference/v1/generate` locally and delegates all other paths to the OpenAI parser, so a single instance covers both vLLM-specific and OpenAI-compatible HTTP traffic on the same endpoint. Existing `openai-parser` deployments are unaffected and need no changes unless `/inference/v1/generate` support is desired. +2026-05-30 https://github.com/llm-d/llm-d-router/pull/1402 Fix encode disaggregation not triggering for `audio_url` content type requests +2026-05-31 https://github.com/llm-d/llm-d-router/pull/1418 Remove the deprecated pkg/epp/backend/metrics package and the enableLegacyMetrics feature gate. All metrics collection now goes through the datalayer pipeline exclusively. Configurations referencing the enableLegacyMetrics feature gate should remove it. +2026-06-02 https://github.com/llm-d/llm-d-router/pull/1121 `precise-prefix-cache-scorer` is now a thin compatibility wrapper around `precise-prefix-cache-producer` and the `prefix-cache-scorer`. Existing configurations continue to work. Deployments without `endpoint-notification-source` wired must add it (or use global socket mode); the legacy in-Score subscriber discovery path is removed. The plugin is deprecated; configure `precise-prefix-cache-producer` + `prefix-cache-scorer` with `prefixMatchInfoProducerName: precise-prefix-cache-producer` directly for new deployments. +2026-06-02 https://github.com/llm-d/llm-d-router/pull/1160 The approximate prefix-cache plugin's autotune path now clamps blockSizeTokens at a minimum of 64 to bound EPP indexer memory. Manually configured values below 64 are still honored but log a deploy-time warning. This is a deliberate routing-precision / memory-stability tradeoff: the routing scorer measures prefix matches at coarser granularity than the model server's true block size. +2026-06-02 https://github.com/llm-d/llm-d-router/pull/1288 - When a request carries the standard HTTP `Prefer: if-available` header (RFC 7240), the EPP routes to a decode worker only if its KV cache already covers the prompt; otherwise it returns HTTP 412 Precondition Failed so the coordinator restarts the pipeline at encode/prefill/decode. - The cache check reads `PrefixCacheMatchInfo` from the chosen endpoint using the default-named approximate-prefix producer's key. Deployments using the auto-created `approx-prefix-cache-producer` (the canonical decode-EPP recipe) get the optimization. Deployments using a custom-named approx producer or `precise-prefix-cache-producer` write under a different key, so the gate misses and the coordinator receives 412 on every conditional-decode request — falling back to the full pipeline (correctness preserved, optimization effectively disabled) for those configurations. +2026-06-02 https://github.com/llm-d/llm-d-router/pull/1436 Metrics emitted by plugins will have `plugin_name` and `plugin_type` labels. +2026-06-02 https://github.com/llm-d/llm-d-router/pull/1449 If both legacy (inference.networking.x-k8s.io) and new (llm-d.ai) InferenceObjective/InferenceModelRewrite CRDs are installed. EPP will only reconcile the new group and IGNORE legacy resources. +2026-06-05 https://github.com/llm-d/llm-d-router/pull/1475 EPP now supports configuring multiple parsers under `requestHandler.parsers` in the `EndpointPickerConfig`. The router matches the request path suffix to select the appropriate parser (first match wins for duplicate parsers supporting suffix match). +2026-06-05 https://github.com/llm-d/llm-d-router/pull/1488 Enable openai, anthropic, and vllmhttp parsers by default in EPP. +2026-06-05 https://github.com/llm-d/llm-d-router/pull/1493 `inflight-load-producer`: a new `prefixMatchInfoProducerName` parameter selects which prefix-cache producer supplies the cached-prefix discount - the approximate-prefix producer by default, or a precise-prefix-cache producer when set. +2026-06-06 https://github.com/llm-d/llm-d-router/pull/1509 `anthropic-parser`: supports the `/v1/messages/count_tokens` endpoint; the body is forwarded unchanged as a raw payload. +2026-06-07 https://github.com/llm-d/llm-d-router/pull/1426 Requests that omit the model field can now be handled by generic InferenceModelRewrite rules instead of being rejected with BadRequest. +2026-06-08 https://github.com/llm-d/llm-d-router/pull/1513 Consolidated tracing initialization and tracer retrieval. Added a `--tracing` flag to `pd-sidecar` (defaulting to `false`) to allow conditionally enabling tracing and avoiding unwanted OTLP connection attempts by default. +2026-06-08 https://github.com/llm-d/llm-d-router/pull/1515 EPP: route /v1/chat/completions/render and /v1/completions/render through the OpenAI parser. +2026-06-09 https://github.com/llm-d/llm-d-router/pull/1444 New EC-NIXL encoder disaggregation connector: Use `--ec-connector=ec-nixl` to the sidecar options, to route multimodal encoder requests through NIXL prior to the prefill phase. OpenTelemetry (OTel) Span Attribute Renaming: Span attributes emitted by the encoder-disaggregation path have been updated from the llm_d.epd_proxy.* namespace to llm_d.ec_proxy.*. +2026-06-09 https://github.com/llm-d/llm-d-router/pull/1536 EPP: added `modality` label to encoder_cache_queries_total and encoder_cache_hits_total. +2026-06-09 https://github.com/llm-d/llm-d-router/pull/1539 EPP trace spans now consistently carry the build version and commit SHA across all instrumentation scopes, so a full request trace can be attributed to a single build. +2026-06-10 https://github.com/llm-d/llm-d-router/pull/1554 Approximate prefix cache affinity routing now considers tools +2026-06-11 https://github.com/llm-d/llm-d-router/pull/1548 The approximate prefix-cache producer (approx-prefix-cache-producer) now defaults maxPrefixTokensToMatch to 131072 (128K tokens), matching the context window of large production models such as gpt-oss 120b. This token-based cap takes precedence over maxPrefixBlocksToMatch, so by default up to 131072 / blockSizeTokens prefix blocks are matched per request instead of the previous 256-block cap. Set maxPrefixTokensToMatch: 0 to restore the block-based cap. +2026-06-11 https://github.com/llm-d/llm-d-router/pull/1575 The standalone Helm chart now supports `router.proxy.mode=service`, deploying the Envoy proxy as a separate horizontally scalable Service (instead of an EPP sidecar) that reaches EPP over the EPP Service with fail-open ext_proc for active/passive resiliency. Default remains `sidecar`. +2026-06-12 https://github.com/llm-d/llm-d-router/pull/1206 sidecar: replace fallback-to-decode on prefill failure with configurable retry logic (--prefill-max-retries, --prefill-retry-backoff). Prefill errors are now returned to the client instead of silently falling back to unaccelerated decode. +2026-06-12 https://github.com/llm-d/llm-d-router/pull/1603 Added an opt-in `--enable-grpc-stream-metrics` flag to the EPP exposing ext_proc gRPC stream metrics: in-flight stream count, hold duration, and completions by gRPC status code. +2026-06-12 https://github.com/llm-d/llm-d-router/pull/1607 The EPP gRPC health check port is configurable via `router.epp.grpcHealthPort` (default 9003). +2026-06-12 https://github.com/llm-d/llm-d-router/pull/1608 Fixed a regression where a priority band defined only via an InferenceObjective (not in the static EPP config) could be garbage-collected after a period of inactivity, causing subsequent requests at that priority to be rejected with "priority band not found". +2026-06-12 https://github.com/llm-d/llm-d-router/pull/1626 EPP now records an error status on the gateway.request and gateway.request_orchestration trace spans when a request fails, so failed requests can be filtered by error status in the trace backend. +2026-06-12 https://github.com/llm-d/llm-d-router/pull/1631 - Add a new `session-affinity-filter` scheduling plugin: pins a session to its previously selected pod as a hard constraint, falling back to all candidates when that pod is no longer available. Complements the existing `session-affinity-scorer` (soft preference). - `session-affinity-scorer` and `session-affinity-filter` now accept an optional `headerName` parameter to carry the session token on a custom request/response header instead of the default `x-session-token`. +2026-06-13 https://github.com/llm-d/llm-d-router/pull/1550 Add a bundled Grafana dashboard for EPP, inference pool, vLLM, and flow-control metrics. +2026-06-15 https://github.com/llm-d/llm-d-router/pull/1429 Added `llm_d_router_epp_encoder_cache_hit_ratio` histogram metric. +2026-06-15 https://github.com/llm-d/llm-d-router/pull/1651 `llm_d_router_epp_plugin_duration_seconds` was previously limited to scheduler plugins, now it's extended to record all plugins. +2026-06-15 https://github.com/llm-d/llm-d-router/pull/1653 Session affinity filter/scorer can optionally pick the scheduling profile to inject the routed endpoint from. This enables P/D disaggregation support. +2026-06-16 https://github.com/llm-d/llm-d-router/pull/1661 The metrics prefix changed to llm_d_epp + RELEASE pre-fragments 2026-05-24 2026-05-24 https://github.com/llm-d/llm-d-router/pull/1134 EPP now strictly parses plugin configurations — unknown fields cause plugin initialization to fail with a clear error rather than being silently ignored. Deprecated fields continue to be accepted with a warning until they are removed. 2026-05-24 https://github.com/llm-d/llm-d-router/pull/1079 deprecated UDS-backend in `token-producer` diff --git a/release-notes.d/unreleased/1030.md b/release-notes.d/unreleased/1030.md deleted file mode 100644 index 44108ed3e5..0000000000 --- a/release-notes.d/unreleased/1030.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1030 -url: https://github.com/llm-d/llm-d-router/pull/1030 -author: kaushikmitr -date: 2026-05-25 ---- -InFlightLoadProducer now reliably tracks global token and request counts in the presence of timeouts, disconnects, and long-lived streams. diff --git a/release-notes.d/unreleased/1121.md b/release-notes.d/unreleased/1121.md deleted file mode 100644 index 25e426037e..0000000000 --- a/release-notes.d/unreleased/1121.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1121 -url: https://github.com/llm-d/llm-d-router/pull/1121 -author: vMaroon -date: 2026-06-02 ---- -`precise-prefix-cache-scorer` is now a thin compatibility wrapper around `precise-prefix-cache-producer` and the `prefix-cache-scorer`. Existing configurations continue to work. Deployments without `endpoint-notification-source` wired must add it (or use global socket mode); the legacy in-Score subscriber discovery path is removed. The plugin is deprecated; configure `precise-prefix-cache-producer` + `prefix-cache-scorer` with `prefixMatchInfoProducerName: precise-prefix-cache-producer` directly for new deployments. diff --git a/release-notes.d/unreleased/1160.md b/release-notes.d/unreleased/1160.md deleted file mode 100644 index 89a83bbbe9..0000000000 --- a/release-notes.d/unreleased/1160.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1160 -url: https://github.com/llm-d/llm-d-router/pull/1160 -author: gkneighb -date: 2026-06-02 ---- -The approximate prefix-cache plugin's autotune path now clamps blockSizeTokens at a minimum of 64 to bound EPP indexer memory. Manually configured values below 64 are still honored but log a deploy-time warning. This is a deliberate routing-precision / memory-stability tradeoff: the routing scorer measures prefix matches at coarser granularity than the model server's true block size. diff --git a/release-notes.d/unreleased/1206.md b/release-notes.d/unreleased/1206.md deleted file mode 100644 index 90e4af75a0..0000000000 --- a/release-notes.d/unreleased/1206.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1206 -url: https://github.com/llm-d/llm-d-router/pull/1206 -author: ilmarkov -date: 2026-06-12 ---- -sidecar: replace fallback-to-decode on prefill failure with configurable retry logic (--prefill-max-retries, --prefill-retry-backoff). Prefill errors are now returned to the client instead of silently falling back to unaccelerated decode. diff --git a/release-notes.d/unreleased/1218.md b/release-notes.d/unreleased/1218.md deleted file mode 100644 index d5363c47ea..0000000000 --- a/release-notes.d/unreleased/1218.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1218 -url: https://github.com/llm-d/llm-d-router/pull/1218 -author: ezrasilvera -date: 2026-05-25 ---- -EPP can now run without a Kubernetes cluster. When `dataLayer.discovery.pluginRef` is set, the runner skips controller-manager setup and drives endpoint discovery through the file-discovery plugin ("file-discovery"). See docs/discovery.md and pkg/epp/framework/plugins/datalayer/discovery/file/README.md for more details. diff --git a/release-notes.d/unreleased/1234.md b/release-notes.d/unreleased/1234.md deleted file mode 100644 index e531ef82d6..0000000000 --- a/release-notes.d/unreleased/1234.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1234 -url: https://github.com/llm-d/llm-d-router/pull/1234 -author: mayabar -date: 2026-05-27 ---- -Update the default vLLM and simulator images, and remove the UDS Tokenizer and the `UDS_TOKENIZER_IMAGE` environment variable. Use `VLLM_RENDER_IMAGE` environment variable to define the render image name. diff --git a/release-notes.d/unreleased/1244.md b/release-notes.d/unreleased/1244.md deleted file mode 100644 index 78f62e10ae..0000000000 --- a/release-notes.d/unreleased/1244.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1244 -url: https://github.com/llm-d/llm-d-router/pull/1244 -author: D-Sai-Venkatesh -date: 2026-05-26 ---- -Add `agent-identity` plugin that derives `FairnessID` from agent session headers (Claude Code, OpenCode, Codex). diff --git a/release-notes.d/unreleased/1247.md b/release-notes.d/unreleased/1247.md deleted file mode 100644 index f1c7a4a39d..0000000000 --- a/release-notes.d/unreleased/1247.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1247 -url: https://github.com/llm-d/llm-d-router/pull/1247 -author: ahg-g -date: 2026-05-25 ---- -The EndpointPickerConfig API has been refactored to provide a more structured and logical grouping of configuration fields. These changes improve schema clarity and provide a cleaner foundation for future feature extensions. Existing YAML/JSON configuration files must be updated to reflect the new nested structure, old fields will continue to work for two releases: - Saturation Detector Migration: The SaturationDetector field has been moved from the top-level configuration into the FlowControl block. - Parser Encapsulation: A new requestHandler struct has been introduced to house request-handler component configurations. The parser field has been moved into this new block. diff --git a/release-notes.d/unreleased/1248.md b/release-notes.d/unreleased/1248.md deleted file mode 100644 index bdded42de3..0000000000 --- a/release-notes.d/unreleased/1248.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1248 -url: https://github.com/llm-d/llm-d-router/pull/1248 -author: dmitripikus -date: 2026-05-28 ---- -New `/inference/v1/generate` endpoint is added, that accepts pre-tokenized prompts (`token_ids`) and optional multimodal features (image/audio/video hashes and placeholder ranges). To enable, configure the new `vllmhttp-parser` (Helm value `router.epp.parser=vllmhttp-parser`, or set `parser: vllmhttp-parser` in EPP configuration). The parser handles `/inference/v1/generate` locally and delegates all other paths to the OpenAI parser, so a single instance covers both vLLM-specific and OpenAI-compatible HTTP traffic on the same endpoint. Existing `openai-parser` deployments are unaffected and need no changes unless `/inference/v1/generate` support is desired. diff --git a/release-notes.d/unreleased/1276.md b/release-notes.d/unreleased/1276.md deleted file mode 100644 index d2e9a27458..0000000000 --- a/release-notes.d/unreleased/1276.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1276 -url: https://github.com/llm-d/llm-d-router/pull/1276 -author: zhouyou9505 -date: 2026-05-25 ---- -Fix cached prompt-token usage extraction in the EPP OpenAI parser. `cached_tokens` is now read from `usage.prompt_tokens_details` (and from `usage.input_tokens_details` for the Responses API), so prompt cache-hit metrics are recorded instead of being silently dropped. diff --git a/release-notes.d/unreleased/1288.md b/release-notes.d/unreleased/1288.md deleted file mode 100644 index 54ee305deb..0000000000 --- a/release-notes.d/unreleased/1288.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1288 -url: https://github.com/llm-d/llm-d-router/pull/1288 -author: dmitripikus -date: 2026-06-02 ---- -- When a request carries the standard HTTP `Prefer: if-available` header (RFC 7240), the EPP routes to a decode worker only if its KV cache already covers the prompt; otherwise it returns HTTP 412 Precondition Failed so the coordinator restarts the pipeline at encode/prefill/decode. - The cache check reads `PrefixCacheMatchInfo` from the chosen endpoint using the default-named approximate-prefix producer's key. Deployments using the auto-created `approx-prefix-cache-producer` (the canonical decode-EPP recipe) get the optimization. Deployments using a custom-named approx producer or `precise-prefix-cache-producer` write under a different key, so the gate misses and the coordinator receives 412 on every conditional-decode request — falling back to the full pipeline (correctness preserved, optimization effectively disabled) for those configurations. diff --git a/release-notes.d/unreleased/1302.md b/release-notes.d/unreleased/1302.md deleted file mode 100644 index 459d27cfe4..0000000000 --- a/release-notes.d/unreleased/1302.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1302 -url: https://github.com/llm-d/llm-d-router/pull/1302 -author: ahg-g -date: 2026-05-26 ---- -The helm charts are now released as part of the llm-d router project, they went through several structural changes and validation safeguards for users migrating from gateway-api-inference-extension ; see the "Migrating from gateway-api-inference-extension" section in https://github.com/llm-d/llm-d-router/blob/main/config/charts/README.md for the migration guide. diff --git a/release-notes.d/unreleased/1372.md b/release-notes.d/unreleased/1372.md deleted file mode 100644 index c16d461b59..0000000000 --- a/release-notes.d/unreleased/1372.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1372 -url: https://github.com/llm-d/llm-d-router/pull/1372 -author: elevran -date: 2026-05-27 ---- -Added a `session-id-producer` DataProducer plugin (type: `session-id-producer`), which extracts a session identifier from a configured request header or cookie and publishes it as the `SessionID` attribute on the request attribute store for use by future affinity-aware scorers and filters. diff --git a/release-notes.d/unreleased/1402.md b/release-notes.d/unreleased/1402.md deleted file mode 100644 index b5ec27ff33..0000000000 --- a/release-notes.d/unreleased/1402.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1402 -url: https://github.com/llm-d/llm-d-router/pull/1402 -author: namgyu-youn -date: 2026-05-30 ---- -Fix encode disaggregation not triggering for `audio_url` content type requests diff --git a/release-notes.d/unreleased/1418.md b/release-notes.d/unreleased/1418.md deleted file mode 100644 index 4726b810a7..0000000000 --- a/release-notes.d/unreleased/1418.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1418 -url: https://github.com/llm-d/llm-d-router/pull/1418 -author: elevran -date: 2026-05-31 ---- -Remove the deprecated pkg/epp/backend/metrics package and the enableLegacyMetrics feature gate. All metrics collection now goes through the datalayer pipeline exclusively. Configurations referencing the enableLegacyMetrics feature gate should remove it. diff --git a/release-notes.d/unreleased/1426.md b/release-notes.d/unreleased/1426.md deleted file mode 100644 index d57f95f487..0000000000 --- a/release-notes.d/unreleased/1426.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1426 -url: https://github.com/llm-d/llm-d-router/pull/1426 -author: Jwrede -date: 2026-06-07 ---- -Requests that omit the model field can now be handled by generic InferenceModelRewrite rules instead of being rejected with BadRequest. diff --git a/release-notes.d/unreleased/1429.md b/release-notes.d/unreleased/1429.md deleted file mode 100644 index 0b003b8d97..0000000000 --- a/release-notes.d/unreleased/1429.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1429 -url: https://github.com/llm-d/llm-d-router/pull/1429 -author: Jwrede -date: 2026-06-15 ---- -Added `llm_d_router_epp_encoder_cache_hit_ratio` histogram metric. diff --git a/release-notes.d/unreleased/1436.md b/release-notes.d/unreleased/1436.md deleted file mode 100644 index 65e7f7bc02..0000000000 --- a/release-notes.d/unreleased/1436.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1436 -url: https://github.com/llm-d/llm-d-router/pull/1436 -author: liu-cong -date: 2026-06-02 ---- -Metrics emitted by plugins will have `plugin_name` and `plugin_type` labels. diff --git a/release-notes.d/unreleased/1444.md b/release-notes.d/unreleased/1444.md deleted file mode 100644 index e745e6b5d3..0000000000 --- a/release-notes.d/unreleased/1444.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1444 -url: https://github.com/llm-d/llm-d-router/pull/1444 -author: revit13 -date: 2026-06-09 ---- -New EC-NIXL encoder disaggregation connector: Use `--ec-connector=ec-nixl` to the sidecar options, to route multimodal encoder requests through NIXL prior to the prefill phase. OpenTelemetry (OTel) Span Attribute Renaming: Span attributes emitted by the encoder-disaggregation path have been updated from the llm_d.epd_proxy.* namespace to llm_d.ec_proxy.*. diff --git a/release-notes.d/unreleased/1449.md b/release-notes.d/unreleased/1449.md deleted file mode 100644 index f76fda384b..0000000000 --- a/release-notes.d/unreleased/1449.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1449 -url: https://github.com/llm-d/llm-d-router/pull/1449 -author: ahg-g -date: 2026-06-02 ---- -If both legacy (inference.networking.x-k8s.io) and new (llm-d.ai) InferenceObjective/InferenceModelRewrite CRDs are installed. EPP will only reconcile the new group and IGNORE legacy resources. diff --git a/release-notes.d/unreleased/1475.md b/release-notes.d/unreleased/1475.md deleted file mode 100644 index d1507574f4..0000000000 --- a/release-notes.d/unreleased/1475.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1475 -url: https://github.com/llm-d/llm-d-router/pull/1475 -author: zetxqx -date: 2026-06-05 ---- -EPP now supports configuring multiple parsers under `requestHandler.parsers` in the `EndpointPickerConfig`. The router matches the request path suffix to select the appropriate parser (first match wins for duplicate parsers supporting suffix match). diff --git a/release-notes.d/unreleased/1488.md b/release-notes.d/unreleased/1488.md deleted file mode 100644 index 3bbc2327b6..0000000000 --- a/release-notes.d/unreleased/1488.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1488 -url: https://github.com/llm-d/llm-d-router/pull/1488 -author: zetxqx -date: 2026-06-05 ---- -Enable openai, anthropic, and vllmhttp parsers by default in EPP. diff --git a/release-notes.d/unreleased/1493.md b/release-notes.d/unreleased/1493.md deleted file mode 100644 index 2279793198..0000000000 --- a/release-notes.d/unreleased/1493.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1493 -url: https://github.com/llm-d/llm-d-router/pull/1493 -author: vMaroon -date: 2026-06-05 ---- -`inflight-load-producer`: a new `prefixMatchInfoProducerName` parameter selects which prefix-cache producer supplies the cached-prefix discount - the approximate-prefix producer by default, or a precise-prefix-cache producer when set. diff --git a/release-notes.d/unreleased/1509.md b/release-notes.d/unreleased/1509.md deleted file mode 100644 index c1c950663f..0000000000 --- a/release-notes.d/unreleased/1509.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1509 -url: https://github.com/llm-d/llm-d-router/pull/1509 -author: zetxqx -date: 2026-06-06 ---- -`anthropic-parser`: supports the `/v1/messages/count_tokens` endpoint; the body is forwarded unchanged as a raw payload. diff --git a/release-notes.d/unreleased/1513.md b/release-notes.d/unreleased/1513.md deleted file mode 100644 index 687958961b..0000000000 --- a/release-notes.d/unreleased/1513.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1513 -url: https://github.com/llm-d/llm-d-router/pull/1513 -author: LukeAVanDrie -date: 2026-06-08 ---- -Consolidated tracing initialization and tracer retrieval. Added a `--tracing` flag to `pd-sidecar` (defaulting to `false`) to allow conditionally enabling tracing and avoiding unwanted OTLP connection attempts by default. diff --git a/release-notes.d/unreleased/1515.md b/release-notes.d/unreleased/1515.md deleted file mode 100644 index 365ed2a2bd..0000000000 --- a/release-notes.d/unreleased/1515.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1515 -url: https://github.com/llm-d/llm-d-router/pull/1515 -author: hexfusion -date: 2026-06-08 ---- -EPP: route /v1/chat/completions/render and /v1/completions/render through the OpenAI parser. diff --git a/release-notes.d/unreleased/1536.md b/release-notes.d/unreleased/1536.md deleted file mode 100644 index 7c8ea3a68b..0000000000 --- a/release-notes.d/unreleased/1536.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1536 -url: https://github.com/llm-d/llm-d-router/pull/1536 -author: hexfusion -date: 2026-06-09 ---- -EPP: added `modality` label to encoder_cache_queries_total and encoder_cache_hits_total. diff --git a/release-notes.d/unreleased/1539.md b/release-notes.d/unreleased/1539.md deleted file mode 100644 index 3d4fecad50..0000000000 --- a/release-notes.d/unreleased/1539.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1539 -url: https://github.com/llm-d/llm-d-router/pull/1539 -author: gyliu513 -date: 2026-06-09 ---- -EPP trace spans now consistently carry the build version and commit SHA across all instrumentation scopes, so a full request trace can be attributed to a single build. diff --git a/release-notes.d/unreleased/1548.md b/release-notes.d/unreleased/1548.md deleted file mode 100644 index 0777b92b8d..0000000000 --- a/release-notes.d/unreleased/1548.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1548 -url: https://github.com/llm-d/llm-d-router/pull/1548 -author: kaushikmitr -date: 2026-06-11 ---- -The approximate prefix-cache producer (approx-prefix-cache-producer) now defaults maxPrefixTokensToMatch to 131072 (128K tokens), matching the context window of large production models such as gpt-oss 120b. This token-based cap takes precedence over maxPrefixBlocksToMatch, so by default up to 131072 / blockSizeTokens prefix blocks are matched per request instead of the previous 256-block cap. Set maxPrefixTokensToMatch: 0 to restore the block-based cap. diff --git a/release-notes.d/unreleased/1550.md b/release-notes.d/unreleased/1550.md deleted file mode 100644 index 68ddfddf5b..0000000000 --- a/release-notes.d/unreleased/1550.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1550 -url: https://github.com/llm-d/llm-d-router/pull/1550 -author: LukeAVanDrie -date: 2026-06-13 ---- -Add a bundled Grafana dashboard for EPP, inference pool, vLLM, and flow-control metrics. diff --git a/release-notes.d/unreleased/1554.md b/release-notes.d/unreleased/1554.md deleted file mode 100644 index a2f544201c..0000000000 --- a/release-notes.d/unreleased/1554.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1554 -url: https://github.com/llm-d/llm-d-router/pull/1554 -author: zetxqx -date: 2026-06-10 ---- -Approximate prefix cache affinity routing now considers tools diff --git a/release-notes.d/unreleased/1575.md b/release-notes.d/unreleased/1575.md deleted file mode 100644 index 358ee12338..0000000000 --- a/release-notes.d/unreleased/1575.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1575 -url: https://github.com/llm-d/llm-d-router/pull/1575 -author: satyamg1620 -date: 2026-06-11 ---- -The standalone Helm chart now supports `router.proxy.mode=service`, deploying the Envoy proxy as a separate horizontally scalable Service (instead of an EPP sidecar) that reaches EPP over the EPP Service with fail-open ext_proc for active/passive resiliency. Default remains `sidecar`. diff --git a/release-notes.d/unreleased/1603.md b/release-notes.d/unreleased/1603.md deleted file mode 100644 index c533c27f03..0000000000 --- a/release-notes.d/unreleased/1603.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1603 -url: https://github.com/llm-d/llm-d-router/pull/1603 -author: hexfusion -date: 2026-06-12 ---- -Added an opt-in `--enable-grpc-stream-metrics` flag to the EPP exposing ext_proc gRPC stream metrics: in-flight stream count, hold duration, and completions by gRPC status code. diff --git a/release-notes.d/unreleased/1607.md b/release-notes.d/unreleased/1607.md deleted file mode 100644 index d303b5be47..0000000000 --- a/release-notes.d/unreleased/1607.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1607 -url: https://github.com/llm-d/llm-d-router/pull/1607 -author: zetxqx -date: 2026-06-12 ---- -The EPP gRPC health check port is configurable via `router.epp.grpcHealthPort` (default 9003). diff --git a/release-notes.d/unreleased/1608.md b/release-notes.d/unreleased/1608.md deleted file mode 100644 index a032e59388..0000000000 --- a/release-notes.d/unreleased/1608.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1608 -url: https://github.com/llm-d/llm-d-router/pull/1608 -author: LukeAVanDrie -date: 2026-06-12 ---- -Fixed a regression where a priority band defined only via an InferenceObjective (not in the static EPP config) could be garbage-collected after a period of inactivity, causing subsequent requests at that priority to be rejected with "priority band not found". diff --git a/release-notes.d/unreleased/1626.md b/release-notes.d/unreleased/1626.md deleted file mode 100644 index da6b9f41d3..0000000000 --- a/release-notes.d/unreleased/1626.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1626 -url: https://github.com/llm-d/llm-d-router/pull/1626 -author: gyliu513 -date: 2026-06-12 ---- -EPP now records an error status on the gateway.request and gateway.request_orchestration trace spans when a request fails, so failed requests can be filtered by error status in the trace backend. diff --git a/release-notes.d/unreleased/1631.md b/release-notes.d/unreleased/1631.md deleted file mode 100644 index fe6286ac6d..0000000000 --- a/release-notes.d/unreleased/1631.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1631 -url: https://github.com/llm-d/llm-d-router/pull/1631 -author: roytman -date: 2026-06-12 ---- -- Add a new `session-affinity-filter` scheduling plugin: pins a session to its previously selected pod as a hard constraint, falling back to all candidates when that pod is no longer available. Complements the existing `session-affinity-scorer` (soft preference). - `session-affinity-scorer` and `session-affinity-filter` now accept an optional `headerName` parameter to carry the session token on a custom request/response header instead of the default `x-session-token`. diff --git a/release-notes.d/unreleased/1651.md b/release-notes.d/unreleased/1651.md deleted file mode 100644 index bc061146c1..0000000000 --- a/release-notes.d/unreleased/1651.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1651 -url: https://github.com/llm-d/llm-d-router/pull/1651 -author: liu-cong -date: 2026-06-15 ---- -`llm_d_router_epp_plugin_duration_seconds` was previously limited to scheduler plugins, now it's extended to record all plugins. diff --git a/release-notes.d/unreleased/1653.md b/release-notes.d/unreleased/1653.md deleted file mode 100644 index fb8819c785..0000000000 --- a/release-notes.d/unreleased/1653.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1653 -url: https://github.com/llm-d/llm-d-router/pull/1653 -author: liu-cong -date: 2026-06-15 ---- -Session affinity filter/scorer can optionally pick the scheduling profile to inject the routed endpoint from. This enables P/D disaggregation support. diff --git a/release-notes.d/unreleased/1661.md b/release-notes.d/unreleased/1661.md deleted file mode 100644 index 33e60bcc65..0000000000 --- a/release-notes.d/unreleased/1661.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -pr: 1661 -url: https://github.com/llm-d/llm-d-router/pull/1661 -author: ahg-g -date: 2026-06-16 ---- -The metrics prefix changed to llm_d_epp