Skip to content

Commit a9ed745

Browse files
satyamg1620satyamg1620
committed
fix subsystem prefix in processing latency metric tests
Signed-off-by: satyamg1620 <Satyam.Gupta.3@ibm.com>
1 parent 0682683 commit a9ed745

5 files changed

Lines changed: 53 additions & 33 deletions

File tree

‎docs/metrics.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,25 @@ Metrics defined by llm-d Router are in addition to Inference Gateway metrics. Fo
4242
> This metric is maintained for backward compatibility with the deprecated
4343
> `pd-profile-handler`. New deployments should use `disagg_decision_total`.
4444
45+
## EPP Processing Latency Metrics
46+
47+
Two metrics covering the EPP's own processing overhead, split across the request and response paths. Emitted under the `llm_d_epp_` prefix.
48+
49+
### `request_processing_duration_seconds`
50+
51+
* **Type:** Histogram
52+
* **Release Stage:** ALPHA
53+
* **Description:** EPP request orchestration latency in seconds, from request receipt through endpoint selection and request preparation. Admission-control time is excluded, since under flow control it is dominated by queue wait, which is driven by load rather than by the EPP and is tracked by `llm_d_epp_flow_control_request_queue_duration_seconds`.
54+
* **Usage:** The request-side EPP overhead. A rising distribution points at scheduling or plugin cost rather than model server latency.
55+
* **Actionability:** Break the increase down by plugin using `llm_d_epp_plugin_duration_seconds` to find which extension point is responsible.
56+
57+
### `response_processing_duration_seconds`
58+
59+
* **Type:** Histogram
60+
* **Release Stage:** ALPHA
61+
* **Description:** EPP response processing latency in seconds, accumulated across the response header and body handlers for a single request. Model server generation time is excluded.
62+
* **Usage:** The response-side EPP overhead. For streaming responses this is the sum over every chunk, so it scales with response length; read it alongside `llm_d_epp_output_tokens`.
63+
4564
## Opt-in ext_proc Stream Metrics
4665

4766
Three metrics covering ext_proc gRPC stream lifecycle. Disabled by default; enable with `--enable-grpc-stream-metrics`. These metrics are emitted under the `llm_d_epp_` prefix (separate from `llm_d_inference_scheduler_*`).

‎pkg/epp/metrics/metrics.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -720,7 +720,8 @@ func RecordSchedulerE2ELatency(duration time.Duration) {
720720
}
721721

722722
// RecordRequestProcessingLatency records the EPP request processing latency,
723-
// measured from request receipt until an endpoint is selected.
723+
// measured from request receipt through endpoint selection and request
724+
// preparation, excluding admission-control time.
724725
func RecordRequestProcessingLatency(duration time.Duration) {
725726
llmdRequestProcessingLatency.WithLabelValues().Observe(duration.Seconds())
726727
}

‎pkg/epp/metrics/metrics_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -867,7 +867,7 @@ func TestRequestProcessingLatency(t *testing.T) {
867867
t.Fatal(err)
868868
}
869869
defer want.Close()
870-
if err := promtestutil.GatherAndCompare(metrics.Registry, want, "llm_d_router_epp_request_processing_duration_seconds"); err != nil {
870+
if err := promtestutil.GatherAndCompare(metrics.Registry, want, "llm_d_epp_request_processing_duration_seconds"); err != nil {
871871
t.Error(err)
872872
}
873873
}
@@ -894,7 +894,7 @@ func TestResponseProcessingLatency(t *testing.T) {
894894
t.Fatal(err)
895895
}
896896
defer want.Close()
897-
if err := promtestutil.GatherAndCompare(metrics.Registry, want, "llm_d_router_epp_response_processing_duration_seconds"); err != nil {
897+
if err := promtestutil.GatherAndCompare(metrics.Registry, want, "llm_d_epp_response_processing_duration_seconds"); err != nil {
898898
t.Error(err)
899899
}
900900
}

‎pkg/epp/metrics/testdata/llm_d_request_processing_duration_seconds_metric‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
# HELP llm_d_router_epp_request_processing_duration_seconds [ALPHA] EPP request orchestration latency distribution in seconds, from request receipt through endpoint selection and request preparation, excluding admission-control time.
2-
# TYPE llm_d_router_epp_request_processing_duration_seconds histogram
3-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.0001"} 0
4-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.0002"} 1
5-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.0005"} 1
6-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.001"} 2
7-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.002"} 3
8-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.005"} 4
9-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.01"} 5
10-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.02"} 6
11-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.05"} 7
12-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="0.1"} 8
13-
llm_d_router_epp_request_processing_duration_seconds_bucket{le="+Inf"} 9
14-
llm_d_router_epp_request_processing_duration_seconds_sum 0.2835
15-
llm_d_router_epp_request_processing_duration_seconds_count 9
1+
# HELP llm_d_epp_request_processing_duration_seconds [ALPHA] EPP request orchestration latency distribution in seconds, from request receipt through endpoint selection and request preparation, excluding admission-control time.
2+
# TYPE llm_d_epp_request_processing_duration_seconds histogram
3+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.0001"} 0
4+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.0002"} 1
5+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.0005"} 1
6+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.001"} 2
7+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.002"} 3
8+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.005"} 4
9+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.01"} 5
10+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.02"} 6
11+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.05"} 7
12+
llm_d_epp_request_processing_duration_seconds_bucket{le="0.1"} 8
13+
llm_d_epp_request_processing_duration_seconds_bucket{le="+Inf"} 9
14+
llm_d_epp_request_processing_duration_seconds_sum 0.2835
15+
llm_d_epp_request_processing_duration_seconds_count 9

‎pkg/epp/metrics/testdata/llm_d_response_processing_duration_seconds_metric‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
# HELP llm_d_router_epp_response_processing_duration_seconds [ALPHA] EPP response processing latency distribution in seconds, accumulated across response handlers and excluding model server generation time.
2-
# TYPE llm_d_router_epp_response_processing_duration_seconds histogram
3-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.0001"} 0
4-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.0002"} 1
5-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.0005"} 1
6-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.001"} 2
7-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.002"} 3
8-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.005"} 4
9-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.01"} 5
10-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.02"} 6
11-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.05"} 7
12-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="0.1"} 8
13-
llm_d_router_epp_response_processing_duration_seconds_bucket{le="+Inf"} 9
14-
llm_d_router_epp_response_processing_duration_seconds_sum 0.2835
15-
llm_d_router_epp_response_processing_duration_seconds_count 9
1+
# HELP llm_d_epp_response_processing_duration_seconds [ALPHA] EPP response processing latency distribution in seconds, accumulated across response handlers and excluding model server generation time.
2+
# TYPE llm_d_epp_response_processing_duration_seconds histogram
3+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.0001"} 0
4+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.0002"} 1
5+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.0005"} 1
6+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.001"} 2
7+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.002"} 3
8+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.005"} 4
9+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.01"} 5
10+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.02"} 6
11+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.05"} 7
12+
llm_d_epp_response_processing_duration_seconds_bucket{le="0.1"} 8
13+
llm_d_epp_response_processing_duration_seconds_bucket{le="+Inf"} 9
14+
llm_d_epp_response_processing_duration_seconds_sum 0.2835
15+
llm_d_epp_response_processing_duration_seconds_count 9

0 commit comments

Comments
 (0)