feature: add slo metrics to flowcontrol

apix/v1alpha2/inferenceobjective_types.go

@@ -76,6 +76,47 @@ type InferenceObjectiveSpec struct {
 	//
 	// +kubebuilder:validation:Required
 	PoolRef PoolObjectReference `json:"poolRef"`
+
+	// SLO defines the expected Service Level Objective ranges for requests targeting this objective.
+	// When set, the InferenceObjective name is used as the SLO class label in metrics.
+	// The ranges define the expected bounds for SLO header values (e.g. x-slo-ttft-ms),
+	// enabling mismatch detection between a request's SLO deadline and its declared objective.
+	//
+	// +optional
+	SLO *SLOSpec `json:"slo,omitempty"`
+}
+
+// SLOSpec defines the expected SLO ranges for this objective.
+type SLOSpec struct {
+	// TTFT defines the expected Time To First Token SLO range in milliseconds.
+	// The range bounds correspond to values expected in the x-slo-ttft-ms request header.
+	//
+	// +optional
+	TTFT *SLORangeMs `json:"ttft,omitempty"`
+
+	// TPOT defines the expected Time Per Output Token SLO range in milliseconds.
+	// The range bounds correspond to values expected in the x-slo-tpot-ms request header.
+	//
+	// +optional
+	TPOT *SLORangeMs `json:"tpot,omitempty"`
+}
+
+// SLORangeMs defines a millisecond range for an SLO metric.
+// MinMs is inclusive, MaxMs is exclusive: [MinMs, MaxMs).
+// Omitting MinMs implies 0 (no lower bound).
+// Omitting MaxMs implies no upper bound.
+type SLORangeMs struct {
+	// MinMs is the inclusive lower bound of the SLO range in milliseconds.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	MinMs *int64 `json:"minMs,omitempty"`
+
+	// MaxMs is the exclusive upper bound of the SLO range in milliseconds.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	MaxMs *int64 `json:"maxMs,omitempty"`
 }
 
 // InferenceObjectiveStatus defines the observed state of InferenceObjective

config/crd/bases/llm-d.ai_inferenceobjectives.yaml

@@ -99,6 +99,50 @@ spec:
                   Similarly requests with a Priority of -10 will always be served after requests with Priority of 0.
                 format: int32
                 type: integer
+              slo:
+                description: |-
+                  SLO defines the expected Service Level Objective ranges for requests targeting this objective.
+                  When set, the InferenceObjective name is used as the SLO class label in metrics.
+                  The ranges define the expected bounds for SLO header values (e.g. x-slo-ttft-ms),
+                  enabling mismatch detection between a request's SLO deadline and its declared objective.
+                properties:
+                  tpot:
+                    description: |-
+                      TPOT defines the expected Time Per Output Token SLO range in milliseconds.
+                      The range bounds correspond to values expected in the x-slo-tpot-ms request header.
+                    properties:
+                      maxMs:
+                        description: MaxMs is the exclusive upper bound of the SLO
+                          range in milliseconds.
+                        format: int64
+                        minimum: 0
+                        type: integer
+                      minMs:
+                        description: MinMs is the inclusive lower bound of the SLO
+                          range in milliseconds.
+                        format: int64
+                        minimum: 0
+                        type: integer
+                    type: object
+                  ttft:
+                    description: |-
+                      TTFT defines the expected Time To First Token SLO range in milliseconds.
+                      The range bounds correspond to values expected in the x-slo-ttft-ms request header.
+                    properties:
+                      maxMs:
+                        description: MaxMs is the exclusive upper bound of the SLO
+                          range in milliseconds.
+                        format: int64
+                        minimum: 0
+                        type: integer
+                      minMs:
+                        description: MinMs is the inclusive lower bound of the SLO
+                          range in milliseconds.
+                        format: int64
+                        minimum: 0
+                        type: integer
+                    type: object
+                type: object
             required:
             - poolRef
             type: object

config/manifests/inferenceobjective.yaml

@@ -7,6 +7,9 @@ spec:
   poolRef:
     group: inference.networking.k8s.io
     name: vllm-qwen3-32b
+  slo:
+    ttft:
+      minMs: 500
 ---
 apiVersion: llm-d.ai/v1alpha2
 kind: InferenceObjective
@@ -17,6 +20,9 @@ spec:
   poolRef:
     group: inference.networking.k8s.io
     name: vllm-qwen3-32b
+  slo:
+    ttft:
+      maxMs: 100
 ---
 apiVersion: llm-d.ai/v1alpha2
 kind: InferenceObjective
@@ -27,3 +33,7 @@ spec:
   poolRef:
     group: inference.networking.k8s.io
     name: vllm-qwen3-32b
+  slo:
+    ttft:
+      minMs: 100
+      maxMs: 500

pkg/epp/flowcontrol/controller/controller.go

@@ -31,7 +31,9 @@ import (
 	"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol/contracts"
 	"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol/controller/internal"
 	"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol/types"
+	fwkrequest "github.com/llm-d/llm-d-router/pkg/epp/framework/common/request"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/interface/flowcontrol"
+	"github.com/llm-d/llm-d-router/pkg/epp/metadata"
 	"github.com/llm-d/llm-d-router/pkg/epp/metrics"
 )
 
@@ -209,21 +211,36 @@ func (fc *FlowController) EnqueueAndWait(
 	flowKey := req.FlowKey()
 	priority := strconv.Itoa(flowKey.Priority)
 	reqBytes := req.ByteSize()
+	sloClass := metrics.SLOClassNone
+	if r := req.InferenceRequest(); r != nil {
+		sloClass = fwkrequest.GetHeader(r.Headers, metadata.ObjectiveKey)
+		if sloClass == "" {
+			sloClass = metrics.SLOClassNone
+		}
+	}
+	if sloClass == "" {
+		sloClass = metrics.SLOClassNone
+	}
+	metrics.RecordFlowControlSLOIncomingRequest(sloClass, req.InferencePoolName())
 	metrics.IncFlowControlQueueSize(
 		flowKey.ID, priority,
 		req.InferencePoolName(),
+		sloClass,
 		req.ModelName(), req.TargetModelName())
 	defer metrics.DecFlowControlQueueSize(
 		flowKey.ID, priority,
 		req.InferencePoolName(),
+		sloClass,
 		req.ModelName(), req.TargetModelName())
 	metrics.AddFlowControlQueueBytes(
 		flowKey.ID, priority,
 		req.InferencePoolName(),
+		sloClass,
 		req.ModelName(), req.TargetModelName(), reqBytes)
 	defer metrics.SubFlowControlQueueBytes(
 		flowKey.ID, priority,
 		req.InferencePoolName(),
+		sloClass,
 		req.ModelName(), req.TargetModelName(), reqBytes)
 
 	// 1. Create the derived context that governs this request's lifecycle (Parent Cancellation + TTL).

pkg/epp/flowcontrol/controller/internal/item.go

@@ -26,7 +26,9 @@ import (
 	"time"
 
 	"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol/types"
+	"github.com/llm-d/llm-d-router/pkg/epp/framework/common/request"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/interface/flowcontrol"
+	"github.com/llm-d/llm-d-router/pkg/epp/metadata"
 	"github.com/llm-d/llm-d-router/pkg/epp/metrics"
 )
 
@@ -159,12 +161,24 @@ func (fi *FlowItem) finalizeInternal(outcome types.QueueOutcome, err error) {
 
 	duration := time.Since(fi.enqueueTime)
 	flowKey := fi.originalRequest.FlowKey()
+	outcomeStr := outcome.String()
 	metrics.RecordFlowControlRequestQueueDuration(
-		flowKey.ID, strconv.Itoa(flowKey.Priority), outcome.String(),
+		flowKey.ID, strconv.Itoa(flowKey.Priority), outcomeStr,
 		fi.originalRequest.InferencePoolName(),
 		fi.OriginalRequest().ModelName(), fi.OriginalRequest().TargetModelName(),
 		duration)
 
+	sloClass := metrics.SLOClassNone
+	if req := fi.originalRequest.InferenceRequest(); req != nil {
+		sloClass = request.GetHeader(req.Headers, metadata.ObjectiveKey)
+		if sloClass == "" {
+			sloClass = metrics.SLOClassNone
+		}
+	}
+	metrics.RecordFlowControlSLORequestQueueDuration(
+		sloClass, outcomeStr, fi.originalRequest.InferencePoolName(),
+		duration)
+
 	fi.done <- finalState
 	close(fi.done)
 }