Merge remote-tracking branch 'upstream/main' into sync/upstream-ff5f8eab

Author: zdtsw

DEVELOPMENT.md

@@ -173,14 +173,15 @@ PROM_ENABLED=true KIND_PROM_HOST_PORT=30091 make env-dev-kind
 
 ### Grafana Dashboard
 
-The upstream [Inference Gateway dashboard] covers EPP, inference pool, and vLLM metrics.
+The bundled [Inference Gateway dashboard] covers EPP metrics across the inference
+pool, inference objective, and flow control layers.
 
 Add a Prometheus datasource at `http://localhost:30090`, then import the JSON via
 **Dashboards > New > Import**. See the
 [Grafana installation docs](https://grafana.com/docs/grafana/latest/setup-grafana/installation/)
 for setup.
 
-[Inference Gateway dashboard]:https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/tools/dashboards/inference_gateway.json
+[Inference Gateway dashboard]:deploy/grafana/inference_gateway.json
 
 > [!NOTE]
 > For significant customization beyond the standard deployment, use the `deploy/components`

cmd/epp/runner/runner.go

@@ -72,6 +72,7 @@ import (
 	srcmodels "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/models"
 	sourcenotifications "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/notifications"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/fairness/globalstrict"
+	programaware "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/fairness/program-aware"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/fairness/roundrobin"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/ordering/edf"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/ordering/fcfs"
@@ -525,6 +526,7 @@ func (r *Runner) registerInTreePlugins() {
 	// Flow Control plugins
 	fwkplugin.Register(globalstrict.GlobalStrictFairnessPolicyType, globalstrict.GlobalStrictFairnessPolicyFactory)
 	fwkplugin.Register(roundrobin.RoundRobinFairnessPolicyType, roundrobin.RoundRobinFairnessPolicyFactory)
+	fwkplugin.Register(programaware.ProgramAwarePluginType, programaware.ProgramAwarePluginFactory)
 	fwkplugin.Register(fcfs.FCFSOrderingPolicyType, fcfs.FCFSOrderingPolicyFactory)
 	fwkplugin.Register(edf.EDFOrderingPolicyType, edf.EDFOrderingPolicyFactory)
 	fwkplugin.Register(slodeadline.SLODeadlineOrderingPolicyType, slodeadline.SLODeadlineOrderingPolicyFactory)

config/charts/README.md

@@ -130,7 +130,7 @@ Core settings for the Endpoint Picker Proxy (EPP) container and pod, including s
 | `router.epp.flags` | Map of command-line flags passed directly to the EPP binary. | `{}` |
 | `router.epp.affinity` | Affinity rules for EPP pods. | `{}` |
 | `router.epp.tolerations` | Tolerations for EPP pods. | `[]` |
-| `router.epp.resources` | EPP container resource requests and limits. | `requests.cpu: "4"`, `requests.memory: 8Gi`, `limits.memory: 16Gi` |
+| `router.epp.resources` | EPP container resource requests and limits. | `requests.cpu: "8"`, `requests.memory: 8Gi`, `limits.memory: 16Gi` |
 | `router.epp.pluginsConfigFile` | EPP plugins configuration file name. | `default-plugins.yaml` |
 | `router.epp.pluginsCustomConfig` | Inline custom YAML configuration for EPP plugins. | `{}` |
 | `router.epp.volumes` | Extra volumes for EPP pod. | `[]` |

config/charts/routerlib/templates/_config.yaml

@@ -47,11 +47,13 @@ data:
       parameters:
         affinityThreshold: 0.99
         ttftSource: latencyPredictor
+        maxTTFTPenaltyMs: 5000
     - name: loose-affinity-filter
       type: prefix-cache-affinity-filter
       parameters:
         affinityThreshold: 0.80
         ttftSource: latencyPredictor
+        maxTTFTPenaltyMs: 5000
     - type: latency-scorer
     - type: weighted-random-picker
     - type: slo-headroom-tier-filter

config/charts/routerlib/values.yaml

@@ -59,7 +59,7 @@ epp:
   # EPP container resources: CPU limits unset for burst capacity, memory capped at 16Gi
   resources:
     requests:
-      cpu: "4"
+      cpu: "8"
       memory: 8Gi
     limits:
       memory: 16Gi
@@ -160,10 +160,9 @@ latencyPredictor:
   # Training Server Configuration
   trainingServer:
     image:
-      # TODO: Update this default once llm-d owns and publishes this image.
-      registry: us-central1-docker.pkg.dev/k8s-staging-images
-      repository: gateway-api-inference-extension/latency-training-server
-      tag: main
+      registry: ghcr.io/llm-d
+      repository: llm-d-latency-predictor-training-server-dev
+      tag: latest
       pullPolicy: Always
     port: 8000
     resources:
@@ -202,10 +201,9 @@ latencyPredictor:
     count: 1
     startPort: 8001
     image:
-      # TODO: Update this default once llm-d owns and publishes this image.
-      registry: us-central1-docker.pkg.dev/k8s-staging-images
-      repository: gateway-api-inference-extension/latency-prediction-server
-      tag: main
+      registry: ghcr.io/llm-d
+      repository: llm-d-latency-predictor-prediction-server-dev
+      tag: latest
       pullPolicy: Always
     resources:
       requests:

deploy/config/epp-mm-embeddings-cache-config.yaml

@@ -14,13 +14,16 @@ plugins:
     parameters:
       cacheSizeInMB: 2048
   - type: mm-embeddings-cache-scorer
-  - type: precise-prefix-cache-scorer
+  - type: precise-prefix-cache-producer
     parameters:
       tokenProcessorConfig:
         blockSize: 64
       indexerConfig:
         kvBlockIndexConfig:
           enableMetrics: true
+  - type: prefix-cache-scorer
+    parameters:
+      prefixMatchInfoProducerName: precise-prefix-cache-producer
   - type: queue-scorer
   - type: max-score-picker
   - type: single-profile-handler
@@ -33,12 +36,12 @@ dataLayer:
     - pluginRef: endpoint-notification-source
       extractors:
         - pluginRef: mm-embeddings-cache-producer
-        - pluginRef: precise-prefix-cache-scorer
+        - pluginRef: precise-prefix-cache-producer
 schedulingProfiles:
   - name: default
     plugins:
       - pluginRef: decode-filter
-      - pluginRef: precise-prefix-cache-scorer
+      - pluginRef: prefix-cache-scorer
         weight: 10
       - pluginRef: mm-embeddings-cache-scorer
         weight: 4

deploy/config/epp-precise-prefix-cache-config.yaml

@@ -1,4 +1,5 @@
-# Sample EPP configuration with the precise-prefix-cache scorer.
+# Sample EPP config: split precise-prefix-cache pipeline
+#   token-producer -> precise-prefix-cache-producer -> prefix-cache-scorer
 apiVersion: llm-d.ai/v1alpha1
 kind: EndpointPickerConfig
 plugins:
@@ -12,13 +13,16 @@ plugins:
   - type: core-metrics-extractor
   - type: single-profile-handler
   - type: decode-filter
-  - type: precise-prefix-cache-scorer
+  - type: precise-prefix-cache-producer
     parameters:
       tokenProcessorConfig:
         blockSize: 64
       indexerConfig:
         kvBlockIndexConfig:
-          enableMetrics: true         # enable kv-block index metrics (prometheus)
+          enableMetrics: true       # enable kv-block index metrics (prometheus)
+  - type: prefix-cache-scorer
+    parameters:
+      prefixMatchInfoProducerName: precise-prefix-cache-producer
   - type: kv-cache-utilization-scorer
   - type: queue-scorer
   - type: max-score-picker
@@ -29,12 +33,12 @@ dataLayer:
         - pluginRef: core-metrics-extractor
     - pluginRef: endpoint-notification-source
       extractors:
-        - pluginRef: precise-prefix-cache-scorer
+        - pluginRef: precise-prefix-cache-producer
 schedulingProfiles:
   - name: default
     plugins:
       - pluginRef: decode-filter
-      - pluginRef: precise-prefix-cache-scorer
+      - pluginRef: prefix-cache-scorer
         weight: 2.0
       - pluginRef: kv-cache-utilization-scorer
         weight: 1.0

deploy/config/epp-precise-prefix-cache-split-config.yaml

@@ -8,7 +8,7 @@ plugins:
     modelName: TinyLlama/TinyLlama-1.1B-Chat-v1.0  # replace to use a different model
     vllm:
       url: http://localhost:8000
-- type: precise-prefix-cache-scorer
+- type: precise-prefix-cache-producer
   parameters:
     tokenProcessorConfig:
       blockSize: 16
@@ -18,6 +18,9 @@ plugins:
       kvBlockIndexConfig:
         enableMetrics: false                # enable kv-block index metrics (prometheus)
         metricsLoggingInterval: 6000000000  # log kv-block metrics as well (1m in nanoseconds)
+- type: prefix-cache-scorer
+  parameters:
+    prefixMatchInfoProducerName: precise-prefix-cache-producer
 - type: decode-filter
 - type: max-score-picker
 - type: single-profile-handler
@@ -26,5 +29,5 @@ schedulingProfiles:
   plugins:
   - pluginRef: decode-filter
   - pluginRef: max-score-picker
-  - pluginRef: precise-prefix-cache-scorer
+  - pluginRef: prefix-cache-scorer
     weight: 10

deploy/config/sim-epp-kvcache-config.yaml

@@ -3,13 +3,16 @@
 apiVersion: llm-d.ai/v1alpha1
 kind: EndpointPickerConfig
 plugins:
-- type: precise-prefix-cache-scorer
+- type: precise-prefix-cache-producer
   parameters:
     indexerConfig:
       tokenProcessorConfig:
         blockSize: 5
       kvBlockIndexConfig:
         maxPrefixBlocksToMatch: 256
+- type: prefix-cache-scorer
+  parameters:
+    prefixMatchInfoProducerName: precise-prefix-cache-producer
 - type: no-hit-lru-scorer
   parameters:
     lruSize: 2048
@@ -21,7 +24,7 @@ schedulingProfiles:
   plugins:
   - pluginRef: decode-filter
   - pluginRef: max-score-picker
-  - pluginRef: precise-prefix-cache-scorer
+  - pluginRef: prefix-cache-scorer
     weight: 2
   - pluginRef: no-hit-lru-scorer
     weight: 1