feat(validator): make secure-accelerator-access check self-contained

Rewrite CheckSecureAcceleratorAccess to programmatically create DRA test
resources instead of expecting pre-deployed pods. The check now creates a
namespace, ResourceClaim, and GPU test pod, waits for completion, validates
DRA access patterns, and cleans up.

- Create dra-test namespace, ResourceClaim, and Pod programmatically
- Poll for pod terminal state with 5-minute timeout (image pull)
- Validate: resourceClaims present, no device plugin, no hostPath,
  ResourceClaim exists, pod succeeded
- Cleanup: delete pod and claim (skip namespace to avoid finalizer hangs)
- Add DRATestPodTimeout constant to pkg/defaults
- Expand ClusterRole RBAC: create/delete for namespaces, pods, resourceclaims
- Update unit tests with fake client reactors for self-contained flow

Verified on live EKS cluster with H100 GPU: TestSecureAcceleratorAccess PASS (3.60s)

Author: dims

pkg/defaults/timeouts.go

@@ -137,6 +137,13 @@ const (
 	ComponentRenderTimeout = 60 * time.Second
 )
 
+// DRA test timeouts for conformance validation.
+const (
+	// DRATestPodTimeout is the timeout for the DRA test pod to complete.
+	// The pod runs a simple CUDA device check but may need time for image pull.
+	DRATestPodTimeout = 5 * time.Minute
+)
+
 // Pod operation timeouts for validation and agent operations.
 const (
 	// PodWaitTimeout is the maximum time to wait for pod operations to complete.

pkg/validator/agent/rbac.go

@@ -116,13 +116,13 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error {
 			{
 				APIGroups: []string{""},
 				Resources: []string{"pods", "services", "nodes"},
-				Verbs:     []string{"get", "list"},
+				Verbs:     []string{"get", "list", "create", "delete"},
 			},
 			// Conformance: cluster-wide core resources (platform-health, robust-controller)
 			{
 				APIGroups: []string{""},
 				Resources: []string{"namespaces", "endpoints"},
-				Verbs:     []string{"get", "list"},
+				Verbs:     []string{"get", "list", "create", "delete"},
 			},
 			// Conformance: CRD discovery (inference-gateway, dra-support, gang-scheduling, robust-controller)
 			{
@@ -134,7 +134,7 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error {
 			{
 				APIGroups: []string{"resource.k8s.io"},
 				Resources: []string{"resourceslices", "resourceclaims"},
-				Verbs:     []string{"get", "list"},
+				Verbs:     []string{"get", "list", "create", "delete"},
 			},
 			// Conformance: GPU operator ClusterPolicy
 			{

pkg/validator/checks/conformance/secure_access_check.go

@@ -15,14 +15,27 @@
 package conformance
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
+	"github.com/NVIDIA/aicr/pkg/defaults"
 	"github.com/NVIDIA/aicr/pkg/errors"
+	"github.com/NVIDIA/aicr/pkg/k8s"
 	"github.com/NVIDIA/aicr/pkg/validator/checks"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+)
+
+const (
+	draTestNamespace = "dra-test"
+	draTestPodName   = "dra-gpu-test"
+	draClaimName     = "gpu-claim"
 )
 
 func init() {
@@ -36,29 +49,113 @@ func init() {
 }
 
 // CheckSecureAcceleratorAccess validates CNCF requirement #3: Secure Accelerator Access.
-// Verifies that a DRA-based GPU workload uses proper access patterns:
+// Creates a DRA-based GPU test pod, waits for completion, and verifies proper access patterns:
 // resourceClaims instead of device plugin, no hostPath to GPU devices,
 // and ResourceClaim is allocated.
 func CheckSecureAcceleratorAccess(ctx *checks.ValidationContext) error {
 	if ctx.Clientset == nil {
 		return errors.New(errors.ErrCodeInvalidRequest, "kubernetes client is not available")
 	}
 
-	// 1. Get the DRA test pod (deployed by workflow before aicr validate runs)
-	pod, err := ctx.Clientset.CoreV1().Pods("dra-test").Get(
-		ctx.Context, "dra-gpu-test", metav1.GetOptions{})
+	dynClient, err := getDynamicClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Deploy DRA test resources and ensure cleanup.
+	if err = deployDRATestResources(ctx.Context, ctx.Clientset, dynClient); err != nil {
+		return err
+	}
+	defer cleanupDRATestResources(ctx.Context, ctx.Clientset, dynClient)
+
+	// Wait for test pod to reach terminal state.
+	pod, err := waitForDRATestPod(ctx.Context, ctx.Clientset)
 	if err != nil {
-		return errors.Wrap(errors.ErrCodeNotFound,
-			"DRA test pod not found (deploy dra-gpu-test.yaml first)", err)
+		return err
+	}
+
+	// Validate DRA access patterns on the completed pod.
+	return validateDRAPatterns(ctx.Context, dynClient, pod)
+}
+
+// deployDRATestResources creates the namespace, ResourceClaim, and Pod for the DRA test.
+func deployDRATestResources(ctx context.Context, clientset kubernetes.Interface, dynClient dynamic.Interface) error {
+	// 1. Create namespace (idempotent).
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: draTestNamespace},
+	}
+	if _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); k8s.IgnoreAlreadyExists(err) != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to create namespace", err)
+	}
+
+	// 2. Delete existing ResourceClaim if present, then create fresh.
+	claimGVR := schema.GroupVersionResource{
+		Group: "resource.k8s.io", Version: "v1", Resource: "resourceclaims",
+	}
+	if err := k8s.IgnoreNotFound(dynClient.Resource(claimGVR).Namespace(draTestNamespace).Delete(
+		ctx, draClaimName, metav1.DeleteOptions{})); err != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to delete existing ResourceClaim", err)
+	}
+
+	claim := buildResourceClaim()
+	if _, err := dynClient.Resource(claimGVR).Namespace(draTestNamespace).Create(
+		ctx, claim, metav1.CreateOptions{}); err != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to create ResourceClaim", err)
+	}
+
+	// 3. Delete existing Pod if present, then create fresh.
+	if err := k8s.IgnoreNotFound(clientset.CoreV1().Pods(draTestNamespace).Delete(
+		ctx, draTestPodName, metav1.DeleteOptions{})); err != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to delete existing pod", err)
 	}
 
-	// 2. Pod uses resourceClaims (DRA pattern)
+	pod := buildDRATestPod()
+	if _, err := clientset.CoreV1().Pods(draTestNamespace).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to create DRA test pod", err)
+	}
+
+	return nil
+}
+
+// waitForDRATestPod polls until the DRA test pod reaches a terminal state.
+func waitForDRATestPod(ctx context.Context, clientset kubernetes.Interface) (*corev1.Pod, error) {
+	var resultPod *corev1.Pod
+
+	waitCtx, cancel := context.WithTimeout(ctx, defaults.DRATestPodTimeout)
+	defer cancel()
+
+	err := wait.PollUntilContextCancel(waitCtx, defaults.PodPollInterval, true,
+		func(ctx context.Context) (bool, error) {
+			pod, err := clientset.CoreV1().Pods(draTestNamespace).Get(
+				ctx, draTestPodName, metav1.GetOptions{})
+			if err != nil {
+				return false, errors.Wrap(errors.ErrCodeInternal, "failed to get DRA test pod", err)
+			}
+			switch pod.Status.Phase { //nolint:exhaustive // only terminal states matter
+			case corev1.PodSucceeded, corev1.PodFailed:
+				resultPod = pod
+				return true, nil
+			default:
+				return false, nil
+			}
+		},
+	)
+	if err != nil {
+		return nil, errors.Wrap(errors.ErrCodeTimeout, "DRA test pod did not complete", err)
+	}
+
+	return resultPod, nil
+}
+
+// validateDRAPatterns verifies the completed pod uses proper DRA access patterns.
+func validateDRAPatterns(ctx context.Context, dynClient dynamic.Interface, pod *corev1.Pod) error {
+	// 1. Pod uses resourceClaims (DRA pattern).
 	if len(pod.Spec.ResourceClaims) == 0 {
 		return errors.New(errors.ErrCodeInternal,
 			"pod does not use DRA resourceClaims")
 	}
 
-	// 3. No nvidia.com/gpu in resources.limits (device plugin pattern)
+	// 2. No nvidia.com/gpu in resources.limits (device plugin pattern).
 	for _, c := range pod.Spec.Containers {
 		if c.Resources.Limits != nil {
 			if _, hasGPU := c.Resources.Limits["nvidia.com/gpu"]; hasGPU {
@@ -68,31 +165,25 @@ func CheckSecureAcceleratorAccess(ctx *checks.ValidationContext) error {
 		}
 	}
 
-	// 4. No hostPath volumes to /dev/nvidia*
+	// 3. No hostPath volumes to /dev/nvidia*.
 	for _, vol := range pod.Spec.Volumes {
 		if vol.HostPath != nil && strings.Contains(vol.HostPath.Path, "/dev/nvidia") {
 			return errors.New(errors.ErrCodeInternal,
 				fmt.Sprintf("pod has hostPath volume to %s", vol.HostPath.Path))
 		}
 	}
 
-	// 5. ResourceClaim exists
-	dynClient, err := getDynamicClient(ctx)
-	if err != nil {
-		return err
-	}
-	gvr := schema.GroupVersionResource{
+	// 4. ResourceClaim exists.
+	claimGVR := schema.GroupVersionResource{
 		Group: "resource.k8s.io", Version: "v1", Resource: "resourceclaims",
 	}
-	_, err = dynClient.Resource(gvr).Namespace("dra-test").Get(
-		ctx.Context, "gpu-claim", metav1.GetOptions{})
+	_, err := dynClient.Resource(claimGVR).Namespace(draTestNamespace).Get(
+		ctx, draClaimName, metav1.GetOptions{})
 	if err != nil {
 		return errors.Wrap(errors.ErrCodeNotFound, "ResourceClaim gpu-claim not found", err)
 	}
 
-	// 6. Pod completed successfully — proves DRA allocation worked.
-	// Note: status.allocation may be cleared after pod completion, so we verify
-	// success via the pod phase rather than the claim's allocation status.
+	// 5. Pod completed successfully — proves DRA allocation worked.
 	if pod.Status.Phase != corev1.PodSucceeded {
 		return errors.New(errors.ErrCodeInternal,
 			fmt.Sprintf("DRA test pod phase=%s (want Succeeded), GPU allocation may have failed",
@@ -101,3 +192,85 @@ func CheckSecureAcceleratorAccess(ctx *checks.ValidationContext) error {
 
 	return nil
 }
+
+// cleanupDRATestResources removes test resources. Errors are logged but not returned
+// since cleanup failures should not mask test results.
+// Note: the namespace is intentionally NOT deleted — it's harmless to leave and
+// namespace deletion can hang on DRA finalizers.
+func cleanupDRATestResources(ctx context.Context, clientset kubernetes.Interface, dynClient dynamic.Interface) {
+	claimGVR := schema.GroupVersionResource{
+		Group: "resource.k8s.io", Version: "v1", Resource: "resourceclaims",
+	}
+
+	// Delete pod first (releases claim reservation), then claim.
+	_ = k8s.IgnoreNotFound(clientset.CoreV1().Pods(draTestNamespace).Delete(
+		ctx, draTestPodName, metav1.DeleteOptions{}))
+	_ = k8s.IgnoreNotFound(dynClient.Resource(claimGVR).Namespace(draTestNamespace).Delete(
+		ctx, draClaimName, metav1.DeleteOptions{}))
+}
+
+// buildDRATestPod returns the Pod spec for the DRA GPU allocation test.
+func buildDRATestPod() *corev1.Pod {
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      draTestPodName,
+			Namespace: draTestNamespace,
+		},
+		Spec: corev1.PodSpec{
+			RestartPolicy: corev1.RestartPolicyNever,
+			Tolerations: []corev1.Toleration{
+				{Operator: corev1.TolerationOpExists},
+			},
+			ResourceClaims: []corev1.PodResourceClaim{
+				{
+					Name:              "gpu",
+					ResourceClaimName: strPtr(draClaimName),
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name:    "gpu-test",
+					Image:   "nvidia/cuda:12.9.0-base-ubuntu24.04",
+					Command: []string{"bash", "-c", "ls /dev/nvidia* && echo 'DRA GPU allocation successful'"},
+					Resources: corev1.ResourceRequirements{
+						Claims: []corev1.ResourceClaim{
+							{Name: "gpu"},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// buildResourceClaim returns the unstructured ResourceClaim for the DRA test.
+func buildResourceClaim() *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "resource.k8s.io/v1",
+			"kind":       "ResourceClaim",
+			"metadata": map[string]interface{}{
+				"name":      draClaimName,
+				"namespace": draTestNamespace,
+			},
+			"spec": map[string]interface{}{
+				"devices": map[string]interface{}{
+					"requests": []interface{}{
+						map[string]interface{}{
+							"name": "gpu",
+							"exactly": map[string]interface{}{
+								"deviceClassName": "gpu.nvidia.com",
+								"allocationMode":  "ExactCount",
+								"count":           int64(1),
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func strPtr(s string) *string {
+	return &s
+}