Skip to content

Commit e184d06

Browse files
LukeAVanDrieLukeAVanDrie
authored
Merge branch 'main' into feat/configurable-output-ratio
2 parents edbb2ca + 1055b45 commit e184d06

55 files changed

Lines changed: 546 additions & 356 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/ISSUE_TEMPLATE/new-release.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ This document defines the process for releasing llm-d-router.
7070
A release branch should already exist. In this case, check out the existing branch:
7171
7272
```shell
73-
git checkout release-${BRANCH_VERSION} ${REMOTE}/release-${BRANCH_VERSION}
73+
git fetch ${REMOTE}
74+
git reset --hard ${REMOTE}/release-${BRANCH_VERSION}
7475
```
7576
7677
1. By default, `LATENCY_PREDICTOR_TAG` in the `Makefile` resolves from the router release tag (via `BUILD_REF`). If the latency predictor tag does **not** align with the router version, update the default value of `LATENCY_PREDICTOR_TAG` in the `Makefile` to match your exported `${LATENCY_PREDICTOR_TAG}`.

‎RELEASE-NOTES.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,45 @@
1+
RELEASE v0.9.0 2026-06-23
2+
2026-05-25 https://github.com/llm-d/llm-d-router/pull/1030 InFlightLoadProducer now reliably tracks global token and request counts in the presence of timeouts, disconnects, and long-lived streams.
3+
2026-05-25 https://github.com/llm-d/llm-d-router/pull/1218 EPP can now run without a Kubernetes cluster. When `dataLayer.discovery.pluginRef` is set, the runner skips controller-manager setup and drives endpoint discovery through the file-discovery plugin ("file-discovery"). See docs/discovery.md and pkg/epp/framework/plugins/datalayer/discovery/file/README.md for more details.
4+
2026-05-25 https://github.com/llm-d/llm-d-router/pull/1247 The EndpointPickerConfig API has been refactored to provide a more structured and logical grouping of configuration fields. These changes improve schema clarity and provide a cleaner foundation for future feature extensions. Existing YAML/JSON configuration files must be updated to reflect the new nested structure, old fields will continue to work for two releases: - Saturation Detector Migration: The SaturationDetector field has been moved from the top-level configuration into the FlowControl block. - Parser Encapsulation: A new requestHandler struct has been introduced to house request-handler component configurations. The parser field has been moved into this new block.
5+
2026-05-25 https://github.com/llm-d/llm-d-router/pull/1276 Fix cached prompt-token usage extraction in the EPP OpenAI parser. `cached_tokens` is now read from `usage.prompt_tokens_details` (and from `usage.input_tokens_details` for the Responses API), so prompt cache-hit metrics are recorded instead of being silently dropped.
6+
2026-05-26 https://github.com/llm-d/llm-d-router/pull/1244 Add `agent-identity` plugin that derives `FairnessID` from agent session headers (Claude Code, OpenCode, Codex).
7+
2026-05-26 https://github.com/llm-d/llm-d-router/pull/1302 The helm charts are now released as part of the llm-d router project, they went through several structural changes and validation safeguards for users migrating from gateway-api-inference-extension ; see the "Migrating from gateway-api-inference-extension" section in https://github.com/llm-d/llm-d-router/blob/main/config/charts/README.md for the migration guide.
8+
2026-05-27 https://github.com/llm-d/llm-d-router/pull/1234 Update the default vLLM and simulator images, and remove the UDS Tokenizer and the `UDS_TOKENIZER_IMAGE` environment variable. Use `VLLM_RENDER_IMAGE` environment variable to define the render image name.
9+
2026-05-27 https://github.com/llm-d/llm-d-router/pull/1372 Added a `session-id-producer` DataProducer plugin (type: `session-id-producer`), which extracts a session identifier from a configured request header or cookie and publishes it as the `SessionID` attribute on the request attribute store for use by future affinity-aware scorers and filters.
10+
2026-05-28 https://github.com/llm-d/llm-d-router/pull/1248 New `/inference/v1/generate` endpoint is added, that accepts pre-tokenized prompts (`token_ids`) and optional multimodal features (image/audio/video hashes and placeholder ranges). To enable, configure the new `vllmhttp-parser` (Helm value `router.epp.parser=vllmhttp-parser`, or set `parser: vllmhttp-parser` in EPP configuration). The parser handles `/inference/v1/generate` locally and delegates all other paths to the OpenAI parser, so a single instance covers both vLLM-specific and OpenAI-compatible HTTP traffic on the same endpoint. Existing `openai-parser` deployments are unaffected and need no changes unless `/inference/v1/generate` support is desired.
11+
2026-05-30 https://github.com/llm-d/llm-d-router/pull/1402 Fix encode disaggregation not triggering for `audio_url` content type requests
12+
2026-05-31 https://github.com/llm-d/llm-d-router/pull/1418 Remove the deprecated pkg/epp/backend/metrics package and the enableLegacyMetrics feature gate. All metrics collection now goes through the datalayer pipeline exclusively. Configurations referencing the enableLegacyMetrics feature gate should remove it.
13+
2026-06-02 https://github.com/llm-d/llm-d-router/pull/1121 `precise-prefix-cache-scorer` is now a thin compatibility wrapper around `precise-prefix-cache-producer` and the `prefix-cache-scorer`. Existing configurations continue to work. Deployments without `endpoint-notification-source` wired must add it (or use global socket mode); the legacy in-Score subscriber discovery path is removed. The plugin is deprecated; configure `precise-prefix-cache-producer` + `prefix-cache-scorer` with `prefixMatchInfoProducerName: precise-prefix-cache-producer` directly for new deployments.
14+
2026-06-02 https://github.com/llm-d/llm-d-router/pull/1160 The approximate prefix-cache plugin's autotune path now clamps blockSizeTokens at a minimum of 64 to bound EPP indexer memory. Manually configured values below 64 are still honored but log a deploy-time warning. This is a deliberate routing-precision / memory-stability tradeoff: the routing scorer measures prefix matches at coarser granularity than the model server's true block size.
15+
2026-06-02 https://github.com/llm-d/llm-d-router/pull/1288 - When a request carries the standard HTTP `Prefer: if-available` header (RFC 7240), the EPP routes to a decode worker only if its KV cache already covers the prompt; otherwise it returns HTTP 412 Precondition Failed so the coordinator restarts the pipeline at encode/prefill/decode. - The cache check reads `PrefixCacheMatchInfo` from the chosen endpoint using the default-named approximate-prefix producer's key. Deployments using the auto-created `approx-prefix-cache-producer` (the canonical decode-EPP recipe) get the optimization. Deployments using a custom-named approx producer or `precise-prefix-cache-producer` write under a different key, so the gate misses and the coordinator receives 412 on every conditional-decode request — falling back to the full pipeline (correctness preserved, optimization effectively disabled) for those configurations.
16+
2026-06-02 https://github.com/llm-d/llm-d-router/pull/1436 Metrics emitted by plugins will have `plugin_name` and `plugin_type` labels.
17+
2026-06-02 https://github.com/llm-d/llm-d-router/pull/1449 If both legacy (inference.networking.x-k8s.io) and new (llm-d.ai) InferenceObjective/InferenceModelRewrite CRDs are installed. EPP will only reconcile the new group and IGNORE legacy resources.
18+
2026-06-05 https://github.com/llm-d/llm-d-router/pull/1475 EPP now supports configuring multiple parsers under `requestHandler.parsers` in the `EndpointPickerConfig`. The router matches the request path suffix to select the appropriate parser (first match wins for duplicate parsers supporting suffix match).
19+
2026-06-05 https://github.com/llm-d/llm-d-router/pull/1488 Enable openai, anthropic, and vllmhttp parsers by default in EPP.
20+
2026-06-05 https://github.com/llm-d/llm-d-router/pull/1493 `inflight-load-producer`: a new `prefixMatchInfoProducerName` parameter selects which prefix-cache producer supplies the cached-prefix discount - the approximate-prefix producer by default, or a precise-prefix-cache producer when set.
21+
2026-06-06 https://github.com/llm-d/llm-d-router/pull/1509 `anthropic-parser`: supports the `/v1/messages/count_tokens` endpoint; the body is forwarded unchanged as a raw payload.
22+
2026-06-07 https://github.com/llm-d/llm-d-router/pull/1426 Requests that omit the model field can now be handled by generic InferenceModelRewrite rules instead of being rejected with BadRequest.
23+
2026-06-08 https://github.com/llm-d/llm-d-router/pull/1513 Consolidated tracing initialization and tracer retrieval. Added a `--tracing` flag to `pd-sidecar` (defaulting to `false`) to allow conditionally enabling tracing and avoiding unwanted OTLP connection attempts by default.
24+
2026-06-08 https://github.com/llm-d/llm-d-router/pull/1515 EPP: route /v1/chat/completions/render and /v1/completions/render through the OpenAI parser.
25+
2026-06-09 https://github.com/llm-d/llm-d-router/pull/1444 New EC-NIXL encoder disaggregation connector: Use `--ec-connector=ec-nixl` to the sidecar options, to route multimodal encoder requests through NIXL prior to the prefill phase. OpenTelemetry (OTel) Span Attribute Renaming: Span attributes emitted by the encoder-disaggregation path have been updated from the llm_d.epd_proxy.* namespace to llm_d.ec_proxy.*.
26+
2026-06-09 https://github.com/llm-d/llm-d-router/pull/1536 EPP: added `modality` label to encoder_cache_queries_total and encoder_cache_hits_total.
27+
2026-06-09 https://github.com/llm-d/llm-d-router/pull/1539 EPP trace spans now consistently carry the build version and commit SHA across all instrumentation scopes, so a full request trace can be attributed to a single build.
28+
2026-06-10 https://github.com/llm-d/llm-d-router/pull/1554 Approximate prefix cache affinity routing now considers tools
29+
2026-06-11 https://github.com/llm-d/llm-d-router/pull/1548 The approximate prefix-cache producer (approx-prefix-cache-producer) now defaults maxPrefixTokensToMatch to 131072 (128K tokens), matching the context window of large production models such as gpt-oss 120b. This token-based cap takes precedence over maxPrefixBlocksToMatch, so by default up to 131072 / blockSizeTokens prefix blocks are matched per request instead of the previous 256-block cap. Set maxPrefixTokensToMatch: 0 to restore the block-based cap.
30+
2026-06-11 https://github.com/llm-d/llm-d-router/pull/1575 The standalone Helm chart now supports `router.proxy.mode=service`, deploying the Envoy proxy as a separate horizontally scalable Service (instead of an EPP sidecar) that reaches EPP over the EPP Service with fail-open ext_proc for active/passive resiliency. Default remains `sidecar`.
31+
2026-06-12 https://github.com/llm-d/llm-d-router/pull/1206 sidecar: replace fallback-to-decode on prefill failure with configurable retry logic (--prefill-max-retries, --prefill-retry-backoff). Prefill errors are now returned to the client instead of silently falling back to unaccelerated decode.
32+
2026-06-12 https://github.com/llm-d/llm-d-router/pull/1603 Added an opt-in `--enable-grpc-stream-metrics` flag to the EPP exposing ext_proc gRPC stream metrics: in-flight stream count, hold duration, and completions by gRPC status code.
33+
2026-06-12 https://github.com/llm-d/llm-d-router/pull/1607 The EPP gRPC health check port is configurable via `router.epp.grpcHealthPort` (default 9003).
34+
2026-06-12 https://github.com/llm-d/llm-d-router/pull/1608 Fixed a regression where a priority band defined only via an InferenceObjective (not in the static EPP config) could be garbage-collected after a period of inactivity, causing subsequent requests at that priority to be rejected with "priority band not found".
35+
2026-06-12 https://github.com/llm-d/llm-d-router/pull/1626 EPP now records an error status on the gateway.request and gateway.request_orchestration trace spans when a request fails, so failed requests can be filtered by error status in the trace backend.
36+
2026-06-12 https://github.com/llm-d/llm-d-router/pull/1631 - Add a new `session-affinity-filter` scheduling plugin: pins a session to its previously selected pod as a hard constraint, falling back to all candidates when that pod is no longer available. Complements the existing `session-affinity-scorer` (soft preference). - `session-affinity-scorer` and `session-affinity-filter` now accept an optional `headerName` parameter to carry the session token on a custom request/response header instead of the default `x-session-token`.
37+
2026-06-13 https://github.com/llm-d/llm-d-router/pull/1550 Add a bundled Grafana dashboard for EPP, inference pool, vLLM, and flow-control metrics.
38+
2026-06-15 https://github.com/llm-d/llm-d-router/pull/1429 Added `llm_d_router_epp_encoder_cache_hit_ratio` histogram metric.
39+
2026-06-15 https://github.com/llm-d/llm-d-router/pull/1651 `llm_d_router_epp_plugin_duration_seconds` was previously limited to scheduler plugins, now it's extended to record all plugins.
40+
2026-06-15 https://github.com/llm-d/llm-d-router/pull/1653 Session affinity filter/scorer can optionally pick the scheduling profile to inject the routed endpoint from. This enables P/D disaggregation support.
41+
2026-06-16 https://github.com/llm-d/llm-d-router/pull/1661 The metrics prefix changed to llm_d_epp
42+
143
RELEASE pre-fragments 2026-05-24
244
2026-05-24 https://github.com/llm-d/llm-d-router/pull/1134 EPP now strictly parses plugin configurations — unknown fields cause plugin initialization to fail with a clear error rather than being silently ignored. Deprecated fields continue to be accepted with a warning until they are removed.
345
2026-05-24 https://github.com/llm-d/llm-d-router/pull/1079 deprecated UDS-backend in `token-producer`

‎pkg/common/error/error.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ type RequestDroppedReason string
3535
const (
3636
// Rejected — request never reached an inference server.
3737
RequestDroppedReasonSaturated RequestDroppedReason = "rejected-saturated"
38+
RequestDroppedReasonNoEndpoints RequestDroppedReason = "rejected-no-endpoints"
3839
RequestDroppedReasonTTLExpired RequestDroppedReason = "rejected-ttl-expired"
3940
RequestDroppedReasonContextCancelled RequestDroppedReason = "rejected-context-cancelled"
4041
RequestDroppedReasonShuttingDown RequestDroppedReason = "rejected-shutting-down"

‎pkg/epp/flowcontrol/controller/internal/processor.go‎

Lines changed: 46 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,12 @@ type Processor struct {
8181
// enqueueChan is the entry point for new requests.
8282
enqueueChan chan *FlowItem
8383

84+
// poolEmpty caches whether the candidate pool had zero endpoints as of the most recent dispatchCycle. enqueue reads
85+
// it to distinguish a queue-capacity rejection caused by genuine unavailability (no backends, e.g. scale-from-zero)
86+
// from one caused by backpressure against a contended but non-empty pool. Only accessed from the Run goroutine, so
87+
// it needs no synchronization.
88+
poolEmpty bool
89+
8490
// wg is used to wait for background tasks (cleanup sweep) to complete on shutdown.
8591
wg sync.WaitGroup
8692
isShuttingDown atomic.Bool
@@ -280,6 +286,15 @@ func (sp *Processor) enqueue(item *FlowItem) {
280286
// --- Capacity Check ---
281287
// This check is safe because it is performed by the single-writer Run goroutine.
282288
if ok, stats := sp.hasCapacity(key.Priority, req.ByteSize()); !ok {
289+
// When the pool has no endpoints, the queue is acting as a scale-from-zero waiting room. A capacity rejection in
290+
// that state reflects genuine unavailability (surfaced as 503), not backpressure against a contended pool (429).
291+
if sp.poolEmpty {
292+
sp.logger.V(logutil.DEBUG).Info("Rejecting request, queue at capacity with no endpoints",
293+
"flowKey", key, "reqID", req.ID(), "reqByteSize", req.ByteSize())
294+
item.FinalizeWithOutcome(types.QueueOutcomeRejectedNoEndpoints, fmt.Errorf("%w: %w",
295+
types.ErrRejected, types.ErrNoEndpoints))
296+
return
297+
}
283298
sp.logger.V(logutil.DEBUG).Info("Rejecting request, queue at capacity",
284299
"flowKey", key, "requestID", req.ID(), "reqByteSize", req.ByteSize(),
285300
"totalLen", stats.TotalLen, "totalCapacityRequests", stats.TotalCapacityRequests,
@@ -346,6 +361,7 @@ func (sp *Processor) dispatchCycle(ctx context.Context) bool {
346361
}()
347362

348363
pool := sp.endpointCandidates.Locate(ctx, nil)
364+
sp.poolEmpty = len(pool) == 0
349365
saturation := sp.saturationDetector.Saturation(ctx, pool)
350366

351367
// Record pool saturation metric
@@ -536,55 +552,61 @@ func (sp *Processor) processAllQueuesConcurrently(
536552
) {
537553
logger := sp.logger.WithName(ctxName)
538554

539-
// Phase 1: Collect all queues to be processed into a single slice.
555+
type resolvedQueue struct {
556+
mq contracts.ManagedQueue
557+
logger logr.Logger
558+
}
559+
560+
// Phase 1: Collect all queues and resolve ManagedQueue handles in one pass.
540561
// This avoids holding locks on the shard while processing, and allows us to determine the optimal number of workers.
541-
var queuesToProcess []flowcontrol.FlowQueueAccessor
562+
var resolvedQueues []resolvedQueue
542563
for _, priority := range sp.registry.AllOrderedPriorityLevels() {
543564
band, err := sp.registry.PriorityBandAccessor(priority)
544565
if err != nil {
545566
logger.Error(err, "Failed to get PriorityBandAccessor", "priority", priority)
546567
continue
547568
}
548569
band.IterateQueues(func(queue flowcontrol.FlowQueueAccessor) bool {
549-
queuesToProcess = append(queuesToProcess, queue)
550-
return true // Continue iterating.
570+
key := queue.FlowKey()
571+
mq, err := sp.registry.ManagedQueue(key)
572+
if err != nil {
573+
logger.V(logutil.DEBUG).Info("Skipping queue; ManagedQueue no longer resolvable",
574+
"flowKey", key, "err", err)
575+
return true
576+
}
577+
resolvedQueues = append(resolvedQueues, resolvedQueue{
578+
mq: mq,
579+
logger: logger.WithValues(
580+
"flowKey", key,
581+
"flowID", key.ID,
582+
"flowPriority", key.Priority),
583+
})
584+
return true
551585
})
552586
}
553587

554-
if len(queuesToProcess) == 0 {
588+
if len(resolvedQueues) == 0 {
555589
return
556590
}
557591

558592
// Phase 2: Determine the optimal number of workers.
559-
// We cap the number of workers to a reasonable fixed number to avoid overwhelming the scheduler when many shards are
560-
// running. We also don't need more workers than there are queues.
561-
numWorkers := min(maxCleanupWorkers, len(queuesToProcess))
593+
numWorkers := min(maxCleanupWorkers, len(resolvedQueues))
562594

563-
// Phase 3: Create a worker pool to process the queues.
564-
tasks := make(chan flowcontrol.FlowQueueAccessor)
595+
// Phase 3: Create a worker pool to process the resolved queues.
596+
tasks := make(chan resolvedQueue)
565597

566598
var wg sync.WaitGroup
567599
for range numWorkers {
568600
wg.Go(func() {
569-
for q := range tasks {
570-
key := q.FlowKey()
571-
queueLogger := logger.WithValues(
572-
"flowKey", key,
573-
"flowID", key.ID,
574-
"flowPriority", key.Priority)
575-
managedQ, err := sp.registry.ManagedQueue(key)
576-
if err != nil {
577-
queueLogger.Error(err, "Failed to get ManagedQueue")
578-
continue
579-
}
580-
processFn(managedQ, queueLogger)
601+
for task := range tasks {
602+
processFn(task.mq, task.logger)
581603
}
582604
})
583605
}
584606

585607
// Feed the channel with all the queues to be processed.
586-
for _, q := range queuesToProcess {
587-
tasks <- q
608+
for _, task := range resolvedQueues {
609+
tasks <- task
588610
}
589611
close(tasks) // Close the channel to signal workers to exit.
590612
wg.Wait() // Wait for all workers to finish.

0 commit comments

Comments
 (0)