test(metrics): cover fairness_id bounding paths; document label cardinality

- Verify the request-metric family bounds fairness_id like the flow
  control family, and that RecordFlowControlRequestQueueDuration bounds
  its model labels.
- Verify flow GC prunes the collected flow's metric series end to end.
- Document the client-derived label caps and GC pruning in
  docs/metrics.md.

Signed-off-by: Luke Van Drie <lukevandrie@google.com>

Author: LukeAVanDrie

docs/metrics.md

@@ -63,6 +63,12 @@ Names below omit the subsystem prefix. Unless a section states otherwise, the pr
 and the release stage is ALPHA. Request and latency metrics share the label set
 `{model_name, target_model_name, fairness_id, priority}`.
 
+Client-derived label values are cardinality-bounded: `model_name` and `target_model_name` (from the
+request body) and `fairness_id` (from the `x-llm-d-inference-fairness-id` header) are each capped at
+1000 distinct values, after which new values are reported as `other`. Model names configured through
+InferenceModelRewrite rules never fold to `other`. Flow control series for a `fairness_id` are
+removed when its flow is garbage collected.
+
 ### Request and latency
 
 | Name | Type | Notes |

pkg/epp/flowcontrol/registry/registry_test.go

@@ -19,6 +19,7 @@ package registry
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -28,10 +29,12 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	testclock "k8s.io/utils/clock/testing"
+	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol/contracts"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/interface/flowcontrol"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/interface/flowcontrol/mocks"
+	eppmetrics "github.com/llm-d/llm-d-router/pkg/epp/metrics"
 )
 
 // --- Test Harness ---
@@ -1349,3 +1352,45 @@ func TestFlowRegistry_FlowErrorScoping(t *testing.T) {
 	assert.Equal(t, int32(concurrency), errorCount.Load(), "All requests should fail flow provisioning")
 	assert.Equal(t, int32(0), successCount.Load(), "No request should succeed if flow provisioning failed")
 }
+
+// countSeriesWithFairnessID gathers the global metrics registry and counts series carrying the
+// given fairness_id label value, across all metric families.
+func countSeriesWithFairnessID(t *testing.T, fairnessID string) int {
+	t.Helper()
+	families, err := crmetrics.Registry.Gather()
+	require.NoError(t, err, "gathering the metrics registry must succeed")
+	n := 0
+	for _, mf := range families {
+		for _, m := range mf.GetMetric() {
+			for _, lp := range m.GetLabel() {
+				if lp.GetName() == "fairness_id" && lp.GetValue() == fairnessID {
+					n++
+				}
+			}
+		}
+	}
+	return n
+}
+
+// Metric series are labeled by the flow's client-derived fairness ID, so they must not outlive the
+// flow: gcFlows prunes them via metrics.DeleteFlowControlFlowSeries once the flow is collected.
+func TestFlowRegistry_GarbageCollection_PrunesMetricSeries(t *testing.T) {
+	// Not parallel: reads the process-global metrics registry. The unique fairness ID keeps the
+	// assertions isolated from series recorded by other tests.
+	eppmetrics.Register()
+	h := newRegistryTestHarness(t, harnessOptions{manualGC: true})
+	const flowID = "gc-metric-prune-flow"
+	key := flowcontrol.FlowKey{ID: flowID, Priority: highPriority}
+
+	h.openConnectionOnFlow(key)
+	eppmetrics.RecordFlowControlRequestEnqueueDuration(
+		flowID, strconv.Itoa(highPriority), "Dispatched", time.Millisecond)
+	require.Positive(t, countSeriesWithFairnessID(t, flowID), "Setup: series must exist before GC")
+
+	h.fakeClock.Step(h.config.FlowGCTimeout + time.Second)
+	h.fr.ExecuteGCCycle()
+
+	h.assertFlowDoesNotExist(key, "Setup: idle flow must have been collected")
+	assert.Zero(t, countSeriesWithFairnessID(t, flowID),
+		"GC must prune every metric series labeled with the collected flow's fairness ID")
+}

pkg/epp/metrics/cardinality_test.go

@@ -152,3 +152,59 @@ func TestDeleteFlowControlFlowSeries(t *testing.T) {
 	require.Equal(t, 1, promtestutil.CollectAndCount(llmdFlowControlRequestEnqueueDuration),
 		"the llm_d_epp family must be pruned identically")
 }
+
+// The bound is applied mechanically in every record function that takes a fairness ID; this guards
+// the request-metric family (distinct label ordering from the flow control family) against the
+// pattern regressing there.
+func TestFairnessLabelBoundOnRequestMetrics(t *testing.T) {
+	const testCap = 3
+	oldFairness := fairnessLabelLimiter
+	fairnessLabelLimiter = newBoundedLabel(testCap)
+	oldModels := modelLabelLimiter
+	modelLabelLimiter = newBoundedLabel(10)
+	requestCounter.Reset()
+	llmdRequestCounter.Reset()
+	t.Cleanup(func() {
+		fairnessLabelLimiter = oldFairness
+		modelLabelLimiter = oldModels
+		requestCounter.Reset()
+		llmdRequestCounter.Reset()
+	})
+
+	for i := 0; i < 100; i++ {
+		RecordRequestCounter("model-a", "model-a", fmt.Sprintf("tenant-%d", i), 0)
+	}
+
+	require.Equal(t, testCap+1, promtestutil.CollectAndCount(llmdRequestCounter),
+		"100 distinct fairness IDs must collapse to cap+overflow series on the request family")
+	require.Equal(t, 1, promtestutil.CollectAndCount(requestCounter),
+		"the deprecated family has no fairness_id label and must stay a single series")
+}
+
+// RecordFlowControlRequestQueueDuration takes request-body model names; they must flow through the
+// model limiter like every sibling record function.
+func TestQueueDurationBoundsModelLabels(t *testing.T) {
+	const testCap = 3
+	oldModels := modelLabelLimiter
+	modelLabelLimiter = newBoundedLabel(testCap)
+	oldFairness := fairnessLabelLimiter
+	fairnessLabelLimiter = newBoundedLabel(10)
+	flowControlRequestQueueDuration.Reset()
+	llmdFlowControlRequestQueueDuration.Reset()
+	t.Cleanup(func() {
+		modelLabelLimiter = oldModels
+		fairnessLabelLimiter = oldFairness
+		flowControlRequestQueueDuration.Reset()
+		llmdFlowControlRequestQueueDuration.Reset()
+	})
+
+	for i := 0; i < 100; i++ {
+		m := fmt.Sprintf("model-%d", i)
+		RecordFlowControlRequestQueueDuration("tenant", "0", "Dispatched", "pool", m, m, time.Millisecond)
+	}
+
+	require.Equal(t, testCap+1, promtestutil.CollectAndCount(flowControlRequestQueueDuration),
+		"100 distinct model names must collapse to cap+overflow series, not one series each")
+	require.Equal(t, testCap+1, promtestutil.CollectAndCount(llmdFlowControlRequestQueueDuration),
+		"the llm_d_epp family must be bounded identically")
+}