feat: add Prometheus metrics for request clamping and NaN/Inf data quality (#177)

Add two new Prometheus counter metrics:

- attune_request_clamped_total: incremented when a recommended request
  exceeds the container limit and is capped. Labels: namespace, policy,
  container, resource.

- attune_nan_inf_samples_total: incremented when all samples for a
  container metric type are non-finite (NaN or Inf), indicating a data
  quality issue with the metrics backend. Labels: namespace, policy,
  container, metric_type.

These metrics make clamping events and data quality issues visible in
Grafana dashboards without requiring debug-level logging.

Changes:
- internal/operatormetrics/metrics.go: register both counter vectors
- internal/controller/resize.go: buildResizeTarget returns clamped
  resource list; caller increments RequestClampedTotal
- internal/controller/prometheus.go: detect all-NaN/Inf profiles in
  computeRecommendations and increment NanInfSamplesTotal
- 4 new tests covering both metrics and partial clamping

Closes #174

Signed-off-by: Sebastien Tardif <sebtardif@ncf.ca>

Author: SebTardif

internal/controller/attunepolicy_controller_test.go

@@ -7030,11 +7030,12 @@ func TestBuildResizeTarget_OmitsLimitsWhenZero(t *testing.T) {
 			MemoryRequest: resource.MustParse("128Mi"),
 		},
 	}
-	target, _ := buildResizeTarget(rec)
+	target, clamped := buildResizeTarget(rec)
 	assert.Equal(t, int64(100), target.Requests.Cpu().MilliValue())
 	wantMem := resource.MustParse("128Mi")
 	assert.Equal(t, wantMem.Value(), target.Requests.Memory().Value())
 	assert.Nil(t, target.Limits, "Limits should be nil when recommendation limits are zero")
+	assert.Empty(t, clamped, "nothing should be clamped when no limits present")
 }
 
 func TestBuildResizeTarget_IncludesLimitsWhenNonZero(t *testing.T) {
@@ -7047,11 +7048,12 @@ func TestBuildResizeTarget_IncludesLimitsWhenNonZero(t *testing.T) {
 			MemoryLimit:   resource.MustParse("256Mi"),
 		},
 	}
-	target, _ := buildResizeTarget(rec)
+	target, clamped := buildResizeTarget(rec)
 	require.NotNil(t, target.Limits)
 	assert.Equal(t, int64(200), target.Limits.Cpu().MilliValue())
 	wantMemLim := resource.MustParse("256Mi")
 	assert.Equal(t, wantMemLim.Value(), target.Limits.Memory().Value())
+	assert.Empty(t, clamped, "nothing should be clamped when requests are below limits")
 }
 
 func TestBuildResizeTarget_PartialLimits(t *testing.T) {
@@ -7063,11 +7065,12 @@ func TestBuildResizeTarget_PartialLimits(t *testing.T) {
 			MemoryRequest: resource.MustParse("128Mi"),
 		},
 	}
-	target, _ := buildResizeTarget(rec)
+	target, clamped := buildResizeTarget(rec)
 	require.NotNil(t, target.Limits, "Limits should be non-nil when any limit is non-zero")
 	assert.Equal(t, int64(200), target.Limits.Cpu().MilliValue())
 	_, hasMemLimit := target.Limits[corev1.ResourceMemory]
 	assert.False(t, hasMemLimit, "Memory limit should not be set when zero in recommendation")
+	assert.Empty(t, clamped, "nothing should be clamped when requests are below limits")
 }
 
 func TestBuildResizeTarget_ClampsRequestsToLimits(t *testing.T) {
@@ -7127,6 +7130,78 @@ func TestBuildResizeTarget_PartialClamping(t *testing.T) {
 		"only CPU should be reported as clamped")
 }
 
+func TestComputeRecommendations_NanInfSamplesMetric(t *testing.T) {
+	policy := newTestPolicy("test-policy", "default")
+	deploy := newTestDeployment("api-server", "default", nil)
+	reconciler := newReconcilerWithClient()
+
+	// Return NaN/Inf samples so BuildProfile yields 0 data points.
+	mc := &mockCollector{
+		queryRangeGroupedFunc: func(_ context.Context, query string, _, _ time.Time, _ time.Duration) (map[string][]rsmetrics.Sample, error) {
+			return map[string][]rsmetrics.Sample{
+				"main": {
+					{Timestamp: time.Now().Add(-1 * time.Hour), Value: math.NaN()},
+					{Timestamp: time.Now().Add(-2 * time.Hour), Value: math.Inf(1)},
+					{Timestamp: time.Now().Add(-3 * time.Hour), Value: math.Inf(-1)},
+				},
+			}, nil
+		},
+	}
+
+	before := promtestutil.ToFloat64(operatormetrics.NanInfSamplesTotal.WithLabelValues("default", "test-policy", "main", "cpu"))
+	rec, _, _, _, err := reconciler.computeRecommendations(context.Background(), policy, deploy, mc, nil, nil, nil, nil)
+	assert.NoError(t, err)
+	assert.Nil(t, rec, "should produce no recommendation when all data is NaN/Inf")
+	after := promtestutil.ToFloat64(operatormetrics.NanInfSamplesTotal.WithLabelValues("default", "test-policy", "main", "cpu"))
+	assert.Equal(t, before+1, after, "NanInfSamplesTotal should increment for CPU")
+}
+
+func TestExecuteResizes_RequestClampedMetric(t *testing.T) {
+	pod := newResizePod("api-server", "600m", "1Gi", "500m", "512Mi")
+	deploy := newTestDeployment("api-server", "default", map[string]string{"app": "api-server"})
+	policy := newTestPolicy("test-policy", "default")
+	policy.Spec.UpdateStrategy.Type = attunev1alpha1.UpdateTypeAuto
+	clientset := kubefake.NewSimpleClientset(pod)
+
+	reconciler := newReconcilerWithClient(pod, deploy)
+	reconciler.Clientset = clientset
+
+	recommendations := []attunev1alpha1.WorkloadRecommendation{
+		{
+			Workload: "api-server",
+			Kind:     "Deployment",
+			Containers: []attunev1alpha1.ContainerRecommendation{
+				{
+					Name: "main",
+					Recommended: attunev1alpha1.ResourceValues{
+						CPURequest:    resource.MustParse("800m"), // Will be clamped to 500m limit
+						MemoryRequest: resource.MustParse("2Gi"),  // Will be clamped to 512Mi limit
+						CPULimit:      resource.MustParse("500m"),
+						MemoryLimit:   resource.MustParse("512Mi"),
+					},
+					Current: attunev1alpha1.ResourceValues{
+						CPURequest:    resource.MustParse("600m"),
+						MemoryRequest: resource.MustParse("1Gi"),
+						CPULimit:      resource.MustParse("500m"),
+						MemoryLimit:   resource.MustParse("512Mi"),
+					},
+				},
+			},
+		},
+	}
+
+	beforeCPU := promtestutil.ToFloat64(operatormetrics.RequestClampedTotal.WithLabelValues("default", "test-policy", "main", "cpu"))
+	beforeMem := promtestutil.ToFloat64(operatormetrics.RequestClampedTotal.WithLabelValues("default", "test-policy", "main", "memory"))
+
+	reconciler.executeResizes(context.Background(), policy, []client.Object{deploy},
+		recommendations, podMap("api-server", pod), nil, nil)
+
+	afterCPU := promtestutil.ToFloat64(operatormetrics.RequestClampedTotal.WithLabelValues("default", "test-policy", "main", "cpu"))
+	afterMem := promtestutil.ToFloat64(operatormetrics.RequestClampedTotal.WithLabelValues("default", "test-policy", "main", "memory"))
+	assert.Equal(t, beforeCPU+1, afterCPU, "RequestClampedTotal should increment for CPU")
+	assert.Equal(t, beforeMem+1, afterMem, "RequestClampedTotal should increment for memory")
+}
+
 func TestTryEvictionFallback_EvictsWhenMultipleReplicas(t *testing.T) {
 	pod1 := newTestPod("api-server-abc-1", "default", map[string]string{"app": "api-server"})
 	pod2 := newTestPod("api-server-abc-2", "default", map[string]string{"app": "api-server"})

internal/controller/prometheus.go

@@ -293,6 +293,21 @@ func (r *AttunePolicyReconciler) computeRecommendations(
 		cpuProfile := rsmetrics.BuildProfile(cpuSamples)
 		memProfile := rsmetrics.BuildProfile(memSamples)
 
+		// Detect when all samples were non-finite (NaN/Inf). This means
+		// Prometheus returned data but every value was unusable.
+		if len(cpuSamples) > 0 && cpuProfile.DataPoints == 0 {
+			operatormetrics.NanInfSamplesTotal.WithLabelValues(
+				policy.Namespace, policy.Name, containerName, "cpu").Inc()
+			logger.V(1).Info("All CPU samples are NaN/Inf, data quality issue",
+				"container", containerName, "rawSamples", len(cpuSamples))
+		}
+		if len(memSamples) > 0 && memProfile.DataPoints == 0 {
+			operatormetrics.NanInfSamplesTotal.WithLabelValues(
+				policy.Namespace, policy.Name, containerName, "memory").Inc()
+			logger.V(1).Info("All memory samples are NaN/Inf, data quality issue",
+				"container", containerName, "rawSamples", len(memSamples))
+		}
+
 		// Track maximum data points across all containers.
 		if pts := cpuProfile.DataPoints; pts > maxDataPoints {
 			maxDataPoints = pts

internal/controller/resize.go

@@ -258,6 +258,10 @@ func (r *AttunePolicyReconciler) executeResizes(
 						logger.V(1).Info("Requests clamped to limits",
 							"pod", pod.Name, "container", containerRec.Name,
 							"clampedResources", clamped)
+						for _, res := range clamped {
+							operatormetrics.RequestClampedTotal.WithLabelValues(
+								policy.Namespace, policy.Name, containerRec.Name, res).Inc()
+						}
 					}
 					cpuIncrease, memIncrease := budgetIncrease(&pod, containerRec.Name, target)
 
@@ -670,6 +674,7 @@ func (r *AttunePolicyReconciler) persistResizeAnnotations(
 // Limits are included when non-zero: for RequestsOnly they equal the current limits (no-op),
 // for RequestsAndLimits they are scaled proportionally. Pods that never had limits produce
 // zero-valued limit fields, which are omitted to avoid Kubernetes rejecting the resize.
+// Returns the target resources and the list of resource names that were clamped.
 func buildResizeTarget(rec attunev1alpha1.ContainerRecommendation) (corev1.ResourceRequirements, []string) {
 	target := corev1.ResourceRequirements{
 		Requests: corev1.ResourceList{

internal/operatormetrics/metrics.go

@@ -213,6 +213,22 @@ var (
 		},
 		[]string{"namespace", "policy"},
 	)
+
+	RequestClampedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "attune_request_clamped_total",
+			Help: "Total times a recommended request was capped at the container's limit value",
+		},
+		[]string{"namespace", "policy", "container", "resource"},
+	)
+
+	NanInfSamplesTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "attune_nan_inf_samples_total",
+			Help: "Total times all samples for a container metric were non-finite (NaN or Inf)",
+		},
+		[]string{"namespace", "policy", "container", "metric_type"},
+	)
 )
 
 // WebhookTimer tracks webhook operation duration and result.
@@ -265,5 +281,7 @@ func init() {
 		BurstFactor,
 		StartupBoostTotal,
 		StaleRecommendationsTotal,
+		RequestClampedTotal,
+		NanInfSamplesTotal,
 	)
 }