Merge remote-tracking branch 'upstream/main' into sync/upstream-ff5f8eab

Author: zdtsw

deploy/grafana/inference_gateway.json

@@ -44,7 +44,7 @@ Metrics defined by llm-d Router are in addition to Inference Gateway metrics. Fo
 
 ## Opt-in ext_proc Stream Metrics
 
-Three metrics covering ext_proc gRPC stream lifecycle. Disabled by default; enable with `--enable-grpc-stream-metrics`. All carry the `llm_d_router_epp_` prefix.
+Three metrics covering ext_proc gRPC stream lifecycle. Disabled by default; enable with `--enable-grpc-stream-metrics`. These metrics are emitted under the `llm_d_epp_` prefix (separate from `llm_d_inference_scheduler_*`).
 
 ### `extproc_streams_inflight`
 

docs/metrics.md

@@ -1122,15 +1122,15 @@ func TestFlowControlMetricsEmitted(t *testing.T) {
 	var queueSizeWhileQueued float64
 	var foundQueueSize bool
 	for _, f := range families {
-		if f.GetName() == "llm_d_router_epp_flow_control_queue_size" {
+		if f.GetName() == "llm_d_epp_flow_control_queue_size" {
 			foundQueueSize = true
 			for _, m := range f.GetMetric() {
 				queueSizeWhileQueued += m.GetGauge().GetValue()
 			}
 		}
 	}
 	require.True(t, foundQueueSize,
-		"llm_d_router_epp_flow_control_queue_size metric should exist")
+		"llm_d_epp_flow_control_queue_size metric should exist")
 	require.Greater(t, queueSizeWhileQueued, 0.0,
 		"queue_size should be > 0 while a request is actively queued")
 

pkg/epp/flowcontrol/integration_test.go

@@ -63,7 +63,7 @@ A complete sample is shipped at [`deploy/config/sim-program-aware-config.yaml`](
 
 ## Observability
 
-The plugin exports two shared collectors and one strategy-owned collector under the `llm_d_router_epp` Prometheus subsystem:
+The plugin exports two shared collectors and one strategy-owned collector under the `llm_d_epp` Prometheus subsystem:
 
 | Metric | Type | Labels | Description |
 |---|---|---|---|

pkg/epp/framework/plugins/flowcontrol/fairness/program-aware/README.md

@@ -32,7 +32,7 @@ var (
 		prometheus.GaugeOpts{
 			Subsystem: eppmetrics.InferenceExtensionSubsystem,
 			Name:      "prefix_indexer_size",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_prefix_indexer_size] Size of the prefix indexer.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_prefix_indexer_size] Size of the prefix indexer.", compbasemetrics.ALPHA),
 		},
 		[]string{},
 	)
@@ -50,7 +50,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceExtensionSubsystem,
 			Name:      "prefix_indexer_hit_ratio",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_prefix_indexer_hit_ratio] Ratio of prefix length matched to total prefix length in the cache lookup.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_prefix_indexer_hit_ratio] Ratio of prefix length matched to total prefix length in the cache lookup.", compbasemetrics.ALPHA),
 			Buckets:   []float64{0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0},
 		},
 		[]string{},
@@ -70,7 +70,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceExtensionSubsystem,
 			Name:      "prefix_indexer_hit_bytes",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_prefix_indexer_hit_bytes] Length of the prefix match in number of bytes in the cache lookup.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_prefix_indexer_hit_bytes] Length of the prefix match in number of bytes in the cache lookup.", compbasemetrics.ALPHA),
 			Buckets:   []float64{0, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536},
 		},
 		[]string{},

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/metrics.go

@@ -68,7 +68,7 @@ var (
 		prometheus.GaugeOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "inference_request_metric",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_inference_request_metric] Consolidated gauge for various inference request metrics including TTFT, TPOT, SLOs, and prediction durations.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_inference_request_metric] Consolidated gauge for various inference request metrics including TTFT, TPOT, SLOs, and prediction durations.", compbasemetrics.ALPHA),
 		},
 		modelTypeLabels,
 	)
@@ -86,7 +86,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "request_ttft_seconds",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_request_ttft_seconds] Inference model TTFT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_request_ttft_seconds] Inference model TTFT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
 			Buckets:   generalLatencyBuckets,
 		},
 		modelLabels,
@@ -96,7 +96,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "request_predicted_ttft_seconds",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_request_predicted_ttft_seconds] Inference model Predicted TTFT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_request_predicted_ttft_seconds] Inference model Predicted TTFT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
 			Buckets:   generalLatencyBuckets,
 		},
 		modelLabels,
@@ -116,7 +116,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "request_ttft_prediction_duration_seconds",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_request_ttft_prediction_duration_seconds] Duration taken to generate TTFT predictions in seconds for each model and target model.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_request_ttft_prediction_duration_seconds] Duration taken to generate TTFT predictions in seconds for each model and target model.", compbasemetrics.ALPHA),
 			Buckets:   predictionLatencyBuckets,
 		},
 		modelLabels,
@@ -136,7 +136,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "request_tpot_seconds",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_request_tpot_seconds] Inference model TPOT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_request_tpot_seconds] Inference model TPOT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
 			Buckets:   tpotBuckets,
 		},
 		modelLabels,
@@ -146,7 +146,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "request_predicted_tpot_seconds",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_request_predicted_tpot_seconds] Inference model Predicted TPOT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_request_predicted_tpot_seconds] Inference model Predicted TPOT distribution in seconds for each model and target model.", compbasemetrics.ALPHA),
 			Buckets:   tpotBuckets,
 		},
 		modelLabels,
@@ -166,7 +166,7 @@ var (
 		prometheus.HistogramOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "request_tpot_prediction_duration_seconds",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_request_tpot_prediction_duration_seconds] Duration taken to generate TPOT predictions in seconds for each model and target model.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_request_tpot_prediction_duration_seconds] Duration taken to generate TPOT predictions in seconds for each model and target model.", compbasemetrics.ALPHA),
 			Buckets:   predictionLatencyBuckets,
 		},
 		modelLabels,
@@ -186,7 +186,7 @@ var (
 		prometheus.CounterOpts{
 			Subsystem: eppmetrics.InferenceObjectiveSubsystem,
 			Name:      "request_slo_violation_total",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_request_slo_violation_total] Counter of SLO violations for each model, target model, and violation type.", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_request_slo_violation_total] Counter of SLO violations for each model, target model, and violation type.", compbasemetrics.ALPHA),
 		},
 		modelTypeLabels,
 	)

pkg/epp/framework/plugins/requestcontrol/dataproducer/predictedlatency/metrics.go

@@ -47,7 +47,7 @@ var (
 		prometheus.CounterOpts{
 			Subsystem: eppmetrics.SchedulerSubsystem,
 			Name:      "pd_decision_total",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_pd_decision_total] Total number of P/D disaggregation decisions made", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_pd_decision_total] Total number of P/D disaggregation decisions made", compbasemetrics.ALPHA),
 		},
 		[]string{"model_name", "decision_type"}, // "decode-only" or "prefill-decode"
 	)
@@ -65,13 +65,13 @@ var (
 	// SchedulerDisaggDecisionCount records disaggregation routing decisions,
 	// covering all stages: decode-only, prefill-decode, encode-decode, encode-prefill-decode.
 	//
-	// Deprecated: Use llm_d_router_epp_disagg_decision_total instead.
+	// Deprecated: Use llm_d_epp_disagg_decision_total instead.
 	// Tracked in: https://github.com/llm-d/llm-d-inference-scheduler/issues/1070
 	SchedulerDisaggDecisionCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Subsystem: eppmetrics.SchedulerSubsystem,
 			Name:      "disagg_decision_total",
-			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_router_epp_disagg_decision_total] Total number of disaggregation routing decisions made", compbasemetrics.ALPHA),
+			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_disagg_decision_total] Total number of disaggregation routing decisions made", compbasemetrics.ALPHA),
 		},
 		[]string{"model_name", "decision_type"},
 	)

pkg/epp/framework/plugins/scheduling/profilehandler/disagg/metrics.go

@@ -34,7 +34,7 @@ func TestSchedulerPDDecisionCount(t *testing.T) {
 	RecordPDDecision("test-plugin", "test-type", model, DecisionTypePrefillDecode)
 
 	expected := `
-		# HELP llm_d_inference_scheduler_pd_decision_total [ALPHA] [Deprecated: Use llm_d_router_epp_pd_decision_total] Total number of P/D disaggregation decisions made
+		# HELP llm_d_inference_scheduler_pd_decision_total [ALPHA] [Deprecated: Use llm_d_epp_pd_decision_total] Total number of P/D disaggregation decisions made
 		# TYPE llm_d_inference_scheduler_pd_decision_total counter
 		llm_d_inference_scheduler_pd_decision_total{decision_type="decode-only",model_name="test-model"} 1
 		llm_d_inference_scheduler_pd_decision_total{decision_type="prefill-decode",model_name="test-model"} 2
@@ -46,14 +46,14 @@ func TestSchedulerPDDecisionCount(t *testing.T) {
 	}
 
 	expectedNew := `
-		# HELP llm_d_router_epp_pd_decision_total [ALPHA] Total number of P/D disaggregation decisions made
-		# TYPE llm_d_router_epp_pd_decision_total counter
-		llm_d_router_epp_pd_decision_total{decision_type="decode-only",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 1
-		llm_d_router_epp_pd_decision_total{decision_type="prefill-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 2
+		# HELP llm_d_epp_pd_decision_total [ALPHA] Total number of P/D disaggregation decisions made
+		# TYPE llm_d_epp_pd_decision_total counter
+		llm_d_epp_pd_decision_total{decision_type="decode-only",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 1
+		llm_d_epp_pd_decision_total{decision_type="prefill-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 2
 	`
 
 	if err := testutil.CollectAndCompare(LlmdPDDecisionCount, strings.NewReader(expectedNew),
-		"llm_d_router_epp_pd_decision_total"); err != nil {
+		"llm_d_epp_pd_decision_total"); err != nil {
 		t.Errorf("RecordPDDecision() new failed: %v", err)
 	}
 }
@@ -73,7 +73,7 @@ func TestRecordDisaggDecision(t *testing.T) {
 	RecordDisaggDecision("test-plugin", "test-type", model, DecisionTypeEncodePrefillDecode)
 
 	expected := `
-		# HELP llm_d_inference_scheduler_disagg_decision_total [ALPHA] [Deprecated: Use llm_d_router_epp_disagg_decision_total] Total number of disaggregation routing decisions made
+		# HELP llm_d_inference_scheduler_disagg_decision_total [ALPHA] [Deprecated: Use llm_d_epp_disagg_decision_total] Total number of disaggregation routing decisions made
 		# TYPE llm_d_inference_scheduler_disagg_decision_total counter
 		llm_d_inference_scheduler_disagg_decision_total{decision_type="decode-only",model_name="test-model"} 1
 		llm_d_inference_scheduler_disagg_decision_total{decision_type="encode-decode",model_name="test-model"} 1
@@ -87,16 +87,16 @@ func TestRecordDisaggDecision(t *testing.T) {
 	}
 
 	expectedNew := `
-		# HELP llm_d_router_epp_disagg_decision_total [ALPHA] Total number of disaggregation routing decisions made
-		# TYPE llm_d_router_epp_disagg_decision_total counter
-		llm_d_router_epp_disagg_decision_total{decision_type="decode-only",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 1
-		llm_d_router_epp_disagg_decision_total{decision_type="encode-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 1
-		llm_d_router_epp_disagg_decision_total{decision_type="encode-prefill-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 3
-		llm_d_router_epp_disagg_decision_total{decision_type="prefill-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 2
+		# HELP llm_d_epp_disagg_decision_total [ALPHA] Total number of disaggregation routing decisions made
+		# TYPE llm_d_epp_disagg_decision_total counter
+		llm_d_epp_disagg_decision_total{decision_type="decode-only",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 1
+		llm_d_epp_disagg_decision_total{decision_type="encode-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 1
+		llm_d_epp_disagg_decision_total{decision_type="encode-prefill-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 3
+		llm_d_epp_disagg_decision_total{decision_type="prefill-decode",model_name="test-model",plugin_name="test-plugin",plugin_type="test-type"} 2
 	`
 
 	if err := testutil.CollectAndCompare(LlmdDisaggDecisionCount, strings.NewReader(expectedNew),
-		"llm_d_router_epp_disagg_decision_total"); err != nil {
+		"llm_d_epp_disagg_decision_total"); err != nil {
 		t.Errorf("RecordDisaggDecision() new failed: %v", err)
 	}
 }
@@ -108,7 +108,7 @@ func TestRecordDisaggDecisionEmptyModel(t *testing.T) {
 	RecordDisaggDecision("test-plugin", "test-type", "", DecisionTypeDecodeOnly)
 
 	expected := `
-		# HELP llm_d_inference_scheduler_disagg_decision_total [ALPHA] [Deprecated: Use llm_d_router_epp_disagg_decision_total] Total number of disaggregation routing decisions made
+		# HELP llm_d_inference_scheduler_disagg_decision_total [ALPHA] [Deprecated: Use llm_d_epp_disagg_decision_total] Total number of disaggregation routing decisions made
 		# TYPE llm_d_inference_scheduler_disagg_decision_total counter
 		llm_d_inference_scheduler_disagg_decision_total{decision_type="decode-only",model_name="unknown"} 1
 	`
@@ -119,13 +119,13 @@ func TestRecordDisaggDecisionEmptyModel(t *testing.T) {
 	}
 
 	expectedNew := `
-		# HELP llm_d_router_epp_disagg_decision_total [ALPHA] Total number of disaggregation routing decisions made
-		# TYPE llm_d_router_epp_disagg_decision_total counter
-		llm_d_router_epp_disagg_decision_total{decision_type="decode-only",model_name="unknown",plugin_name="test-plugin",plugin_type="test-type"} 1
+		# HELP llm_d_epp_disagg_decision_total [ALPHA] Total number of disaggregation routing decisions made
+		# TYPE llm_d_epp_disagg_decision_total counter
+		llm_d_epp_disagg_decision_total{decision_type="decode-only",model_name="unknown",plugin_name="test-plugin",plugin_type="test-type"} 1
 	`
 
 	if err := testutil.CollectAndCompare(LlmdDisaggDecisionCount, strings.NewReader(expectedNew),
-		"llm_d_router_epp_disagg_decision_total"); err != nil {
+		"llm_d_epp_disagg_decision_total"); err != nil {
 		t.Errorf("RecordDisaggDecision() new empty model failed: %v", err)
 	}
 }

pkg/epp/framework/plugins/scheduling/profilehandler/disagg/metrics_test.go

@@ -300,7 +300,7 @@ func TestHandleResponseBodyWithoutSchedulingRequest(t *testing.T) {
 		server.HandleResponseBody(ctx, reqCtx, []byte(body), true)
 	})
 
-	histogram := findHistogramMetric(t, "llm_d_router_epp_request_ntpot_seconds", map[string]string{
+	histogram := findHistogramMetric(t, "llm_d_epp_request_ntpot_seconds", map[string]string{
 		"model_name":        "incoming-model",
 		"target_model_name": "target-model",
 		"fairness_id":       metadata.DefaultFairnessID,

pkg/epp/handlers/response_test.go

@@ -118,10 +118,10 @@ func TestMetricsCollected(t *testing.T) {
 		}
 
 		errNew := promtestutil.CollectAndCompare(collector, strings.NewReader(`
-		# HELP llm_d_router_epp_per_endpoint_queue_size [ALPHA] The total number of requests pending in the model server queue for each underlying endpoint.
-		# TYPE llm_d_router_epp_per_endpoint_queue_size gauge
-		llm_d_router_epp_per_endpoint_queue_size{model_server_endpoint="pod1-rank-0",name="test-pool"} 100
-`), "llm_d_router_epp_per_endpoint_queue_size")
+		# HELP llm_d_epp_per_endpoint_queue_size [ALPHA] The total number of requests pending in the model server queue for each underlying endpoint.
+		# TYPE llm_d_epp_per_endpoint_queue_size gauge
+		llm_d_epp_per_endpoint_queue_size{model_server_endpoint="pod1-rank-0",name="test-pool"} 100
+`), "llm_d_epp_per_endpoint_queue_size")
 		if errNew != nil {
 			t.Fatal(errNew)
 		}