fix(metrics): guard overflow series from flow GC pruning

A flow whose client-chosen fairness ID equals the overflow value must not
delete the shared series that aggregates every capped-out tenant, so
DeleteFlowControlFlowSeries now no-ops on the overflow value.

Also document the revive-during-GC gauge race as an accepted limitation and
clarify in docs/metrics.md that the model labels share one cap pool and that
caps apply over the process lifetime.

Signed-off-by: Luke Van Drie <lukevandrie@google.com>

Author: LukeAVanDrie

docs/metrics.md

@@ -64,10 +64,11 @@ and the release stage is ALPHA. Request and latency metrics share the label set
 `{model_name, target_model_name, fairness_id, priority}`.
 
 Client-derived label values are cardinality-bounded: `model_name` and `target_model_name` (from the
-request body) and `fairness_id` (from the `x-llm-d-inference-fairness-id` header) are each capped at
-1000 distinct values, after which new values are reported as `other`. Model names configured through
-InferenceModelRewrite rules never fold to `other`. Flow control series for a `fairness_id` are
-removed when its flow is garbage collected.
+request body) share a cap of 1000 distinct values, and `fairness_id` (from the
+`x-llm-d-inference-fairness-id` header) is capped at 1000 distinct values. Caps apply over the
+lifetime of the process; once a cap is reached, new values are reported as `other`. Model names
+configured through InferenceModelRewrite rules never fold to `other`. Flow control series for a
+`fairness_id` are removed when its flow is garbage collected.
 
 ### Request and latency
 

pkg/epp/metrics/cardinality_test.go

@@ -208,3 +208,30 @@ func TestQueueDurationBoundsModelLabels(t *testing.T) {
 	require.Equal(t, testCap+1, promtestutil.CollectAndCount(llmdFlowControlRequestQueueDuration),
 		"the llm_d_epp family must be bounded identically")
 }
+
+// A client can choose the overflow value itself as its fairness ID; GC of that flow must not
+// delete the shared overflow series that aggregates every capped-out tenant.
+func TestDeleteFlowControlFlowSeriesPreservesOverflowSeries(t *testing.T) {
+	old := fairnessLabelLimiter
+	fairnessLabelLimiter = newBoundedLabel(1)
+	flowControlRequestEnqueueDuration.Reset()
+	llmdFlowControlRequestEnqueueDuration.Reset()
+	t.Cleanup(func() {
+		fairnessLabelLimiter = old
+		flowControlRequestEnqueueDuration.Reset()
+		llmdFlowControlRequestEnqueueDuration.Reset()
+	})
+
+	// tenant-a fills the single cap slot; tenant-b folds to the overflow series.
+	RecordFlowControlRequestEnqueueDuration("tenant-a", "0", "Dispatched", time.Millisecond)
+	RecordFlowControlRequestEnqueueDuration("tenant-b", "0", "Dispatched", time.Millisecond)
+	require.Equal(t, 2, promtestutil.CollectAndCount(flowControlRequestEnqueueDuration),
+		"setup: expected the admitted series plus the overflow series")
+
+	DeleteFlowControlFlowSeries(overflowValue, "0")
+
+	require.Equal(t, 2, promtestutil.CollectAndCount(flowControlRequestEnqueueDuration),
+		"deleting the overflow value must be a no-op; the shared overflow series must survive")
+	require.Equal(t, 2, promtestutil.CollectAndCount(llmdFlowControlRequestEnqueueDuration),
+		"the llm_d_epp family must be preserved identically")
+}

pkg/epp/metrics/metrics.go

@@ -900,7 +900,16 @@ func IncFlowControlRequestsTotal(outcome, priority, inferencePool string) {
 // derived from client input, so its cardinality is unbounded; the flow registry calls this when it
 // garbage-collects an idle flow so that the metric vectors track live flows instead of growing
 // monotonically with every fairness ID ever observed.
+//
+// Pruning is not synchronized with recording: a request reviving the flow concurrently with GC can
+// have a queue gauge increment deleted here while its paired decrement lands afterwards, leaving
+// the queue size/bytes gauges negative until the flow's next collection deletes the series again.
 func DeleteFlowControlFlowSeries(fairnessID, priority string) {
+	// The overflow value aggregates every capped-out fairness ID, so a flow whose client-chosen ID
+	// equals it must not delete the shared series.
+	if fairnessID == overflowValue {
+		return
+	}
 	labels := prometheus.Labels{"fairness_id": fairnessID, "priority": priority}
 	flowControlRequestQueueDuration.DeletePartialMatch(labels)
 	flowControlRequestEnqueueDuration.DeletePartialMatch(labels)