feat(validator): add Go-based CNCF AI conformance checks for aicr validate --phase conformance

Implement 11 conformance checks mapping to CNCF AI Conformance v1.35 MUST
requirements. Each check runs inside a K8s Job with in-cluster access,
replacing hundreds of lines of bash assertions with structured, testable Go.

Checks: platform-health, gpu-operator-health, dra-support,
secure-accelerator-access, accelerator-metrics, ai-service-metrics,
inference-gateway, gang-scheduling, robust-controller, pod-autoscaling,
cluster-autoscaling.

Infrastructure: wire buildTestPattern for conformance phase, add DynamicClient
to ValidationContext for CRD reads, add 10 read-only RBAC rules, register
conformance package in main.go, update 4 recipe overlays with per-intent
check lists.

Author: dims

cmd/aicr/main.go

@@ -19,6 +19,7 @@ import (
 
 	// Import check packages for side-effect registration.
 	// Each package's init() function registers its validators.
+	_ "github.com/NVIDIA/aicr/pkg/validator/checks/conformance"
 	_ "github.com/NVIDIA/aicr/pkg/validator/checks/deployment"
 	_ "github.com/NVIDIA/aicr/pkg/validator/checks/readiness"
 )

pkg/validator/agent/rbac.go

@@ -118,6 +118,65 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error {
 				Resources: []string{"pods", "services", "nodes"},
 				Verbs:     []string{"get", "list"},
 			},
+			// Conformance: cluster-wide core resources (platform-health, robust-controller)
+			{
+				APIGroups: []string{""},
+				Resources: []string{"namespaces", "endpoints"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: CRD discovery (inference-gateway, dra-support, gang-scheduling, robust-controller)
+			{
+				APIGroups: []string{"apiextensions.k8s.io"},
+				Resources: []string{"customresourcedefinitions"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: DRA support validation (resource.k8s.io/v1 — GA)
+			{
+				APIGroups: []string{"resource.k8s.io"},
+				Resources: []string{"resourceslices", "resourceclaims"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: GPU operator ClusterPolicy
+			{
+				APIGroups: []string{"nvidia.com"},
+				Resources: []string{"clusterpolicies"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: Gateway API validation
+			{
+				APIGroups: []string{"gateway.networking.k8s.io"},
+				Resources: []string{"gatewayclasses", "gateways"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: Gang scheduling (KAI scheduler)
+			{
+				APIGroups: []string{"scheduling.run.ai"},
+				Resources: []string{"queues", "podgroups"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: Cluster autoscaling (Karpenter)
+			{
+				APIGroups: []string{"karpenter.sh"},
+				Resources: []string{"nodepools"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: Aggregated metrics APIs (pod-autoscaling, ai-service-metrics)
+			{
+				APIGroups: []string{"custom.metrics.k8s.io"},
+				Resources: []string{"*"},
+				Verbs:     []string{"get", "list"},
+			},
+			{
+				APIGroups: []string{"external.metrics.k8s.io"},
+				Resources: []string{"*"},
+				Verbs:     []string{"get", "list"},
+			},
+			// Conformance: Robust controller — webhook configurations
+			{
+				APIGroups: []string{"admissionregistration.k8s.io"},
+				Resources: []string{"validatingwebhookconfigurations"},
+				Verbs:     []string{"get", "list"},
+			},
 		},
 	}
 

pkg/validator/checks/conformance/accelerator_metrics_check.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package conformance
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/NVIDIA/aicr/pkg/errors"
+	"github.com/NVIDIA/aicr/pkg/validator/checks"
+)
+
+// requiredDCGMMetrics are the DCGM metrics required by CNCF AI Conformance requirement #4.
+var requiredDCGMMetrics = []string{
+	"DCGM_FI_DEV_GPU_UTIL",
+	"DCGM_FI_DEV_FB_USED",
+	"DCGM_FI_DEV_GPU_TEMP",
+	"DCGM_FI_DEV_POWER_USAGE",
+}
+
+const dcgmExporterURL = "http://nvidia-dcgm-exporter.gpu-operator.svc:9400/metrics"
+
+func init() {
+	checks.RegisterCheck(&checks.Check{
+		Name:        "accelerator-metrics",
+		Description: "Verify DCGM exporter exposes required GPU metrics (utilization, memory, temperature, power)",
+		Phase:       "conformance",
+		Func:        CheckAcceleratorMetrics,
+		TestName:    "TestAcceleratorMetrics",
+	})
+}
+
+// CheckAcceleratorMetrics validates CNCF requirement #4: Accelerator Metrics.
+// Calls the DCGM exporter metrics endpoint directly via in-cluster DNS and verifies
+// that all required GPU metrics are present.
+func CheckAcceleratorMetrics(ctx *checks.ValidationContext) error {
+	return checkAcceleratorMetricsWithURL(ctx, dcgmExporterURL)
+}
+
+// checkAcceleratorMetricsWithURL is the testable implementation that accepts a configurable URL.
+func checkAcceleratorMetricsWithURL(ctx *checks.ValidationContext, url string) error {
+	body, err := httpGet(ctx.Context, url)
+	if err != nil {
+		return errors.Wrap(errors.ErrCodeUnavailable,
+			"DCGM exporter metrics endpoint unreachable", err)
+	}
+
+	missing := containsAllMetrics(string(body), requiredDCGMMetrics)
+	if len(missing) > 0 {
+		return errors.New(errors.ErrCodeNotFound,
+			fmt.Sprintf("DCGM metrics missing: %s", strings.Join(missing, ", ")))
+	}
+
+	return nil
+}

pkg/validator/checks/conformance/accelerator_metrics_check_test.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package conformance
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/aicr/pkg/validator/checks"
+)
+
+func TestAcceleratorMetrics(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	runner, err := checks.NewTestRunner(t)
+	if err != nil {
+		t.Skipf("Not in Job environment: %v", err)
+	}
+	defer runner.Cancel()
+
+	if !runner.HasCheck("conformance", "accelerator-metrics") {
+		t.Skip("Check accelerator-metrics not enabled in recipe")
+	}
+
+	t.Logf("Running check: accelerator-metrics")
+
+	ctx := runner.Context()
+	err = CheckAcceleratorMetrics(ctx)
+
+	if err != nil {
+		t.Errorf("Check failed: %v", err)
+	} else {
+		t.Logf("Check passed: accelerator-metrics")
+	}
+}

pkg/validator/checks/conformance/accelerator_metrics_check_unit_test.go

@@ -0,0 +1,184 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package conformance
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/NVIDIA/aicr/pkg/validator/checks"
+)
+
+func TestCheckAcceleratorMetrics(t *testing.T) {
+	allMetrics := `# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization
+# TYPE DCGM_FI_DEV_GPU_UTIL gauge
+DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-abc"} 42
+# HELP DCGM_FI_DEV_FB_USED Framebuffer memory used
+# TYPE DCGM_FI_DEV_FB_USED gauge
+DCGM_FI_DEV_FB_USED{gpu="0",UUID="GPU-abc"} 1024
+# HELP DCGM_FI_DEV_GPU_TEMP GPU temperature
+# TYPE DCGM_FI_DEV_GPU_TEMP gauge
+DCGM_FI_DEV_GPU_TEMP{gpu="0",UUID="GPU-abc"} 65
+# HELP DCGM_FI_DEV_POWER_USAGE Power draw
+# TYPE DCGM_FI_DEV_POWER_USAGE gauge
+DCGM_FI_DEV_POWER_USAGE{gpu="0",UUID="GPU-abc"} 200
+`
+
+	tests := []struct {
+		name        string
+		handler     http.HandlerFunc
+		wantErr     bool
+		errContains string
+	}{
+		{
+			name: "all metrics present",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprint(w, allMetrics)
+			},
+			wantErr: false,
+		},
+		{
+			name: "missing one metric",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				// Only 3 of 4 metrics
+				fmt.Fprint(w, `DCGM_FI_DEV_GPU_UTIL{gpu="0"} 42
+DCGM_FI_DEV_FB_USED{gpu="0"} 1024
+DCGM_FI_DEV_GPU_TEMP{gpu="0"} 65
+`)
+			},
+			wantErr:     true,
+			errContains: "DCGM_FI_DEV_POWER_USAGE",
+		},
+		{
+			name: "missing all metrics",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprint(w, "# no metrics here\n")
+			},
+			wantErr:     true,
+			errContains: "DCGM metrics missing",
+		},
+		{
+			name: "server returns 500",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusInternalServerError)
+			},
+			wantErr:     true,
+			errContains: "HTTP 500",
+		},
+		{
+			name:        "server unreachable",
+			handler:     nil, // No server started
+			wantErr:     true,
+			errContains: "DCGM exporter metrics endpoint unreachable",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var url string
+			if tt.handler != nil {
+				server := httptest.NewServer(tt.handler)
+				defer server.Close()
+				url = server.URL + "/metrics"
+			} else {
+				// Use an unreachable URL
+				url = "http://127.0.0.1:1/metrics"
+			}
+
+			ctx := &checks.ValidationContext{
+				Context: context.Background(),
+			}
+
+			err := checkAcceleratorMetricsWithURL(ctx, url)
+
+			if (err != nil) != tt.wantErr {
+				t.Errorf("checkAcceleratorMetricsWithURL() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if tt.wantErr && err != nil && tt.errContains != "" {
+				if !strings.Contains(err.Error(), tt.errContains) {
+					t.Errorf("checkAcceleratorMetricsWithURL() error = %v, should contain %q", err, tt.errContains)
+				}
+			}
+		})
+	}
+}
+
+func TestCheckAcceleratorMetricsRegistration(t *testing.T) {
+	check, ok := checks.GetCheck("accelerator-metrics")
+	if !ok {
+		t.Fatal("accelerator-metrics check not registered")
+	}
+	if check.Phase != "conformance" {
+		t.Errorf("Phase = %v, want conformance", check.Phase)
+	}
+	if check.Func == nil {
+		t.Fatal("Func is nil")
+	}
+}
+
+func TestContainsAllMetrics(t *testing.T) {
+	tests := []struct {
+		name     string
+		text     string
+		required []string
+		want     []string
+	}{
+		{
+			name:     "all present",
+			text:     "DCGM_FI_DEV_GPU_UTIL 42\nDCGM_FI_DEV_FB_USED 1024",
+			required: []string{"DCGM_FI_DEV_GPU_UTIL", "DCGM_FI_DEV_FB_USED"},
+			want:     nil,
+		},
+		{
+			name:     "one missing",
+			text:     "DCGM_FI_DEV_GPU_UTIL 42",
+			required: []string{"DCGM_FI_DEV_GPU_UTIL", "DCGM_FI_DEV_FB_USED"},
+			want:     []string{"DCGM_FI_DEV_FB_USED"},
+		},
+		{
+			name:     "all missing",
+			text:     "no metrics here",
+			required: []string{"DCGM_FI_DEV_GPU_UTIL", "DCGM_FI_DEV_FB_USED"},
+			want:     []string{"DCGM_FI_DEV_GPU_UTIL", "DCGM_FI_DEV_FB_USED"},
+		},
+		{
+			name:     "empty text",
+			text:     "",
+			required: []string{"DCGM_FI_DEV_GPU_UTIL"},
+			want:     []string{"DCGM_FI_DEV_GPU_UTIL"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := containsAllMetrics(tt.text, tt.required)
+			if len(got) != len(tt.want) {
+				t.Errorf("containsAllMetrics() = %v, want %v", got, tt.want)
+				return
+			}
+			for i := range got {
+				if got[i] != tt.want[i] {
+					t.Errorf("containsAllMetrics()[%d] = %v, want %v", i, got[i], tt.want[i])
+				}
+			}
+		})
+	}
+}