feat(conformance): enrich DRA, gang, and secure access evidence

Author: dims

pkg/validator/checks/conformance/dra_support_check.go

@@ -16,10 +16,13 @@ package conformance
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/NVIDIA/aicr/pkg/errors"
 	"github.com/NVIDIA/aicr/pkg/validator/checks"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
@@ -46,37 +49,61 @@ func CheckDRASupport(ctx *checks.ValidationContext) error {
 		return errors.New(errors.ErrCodeInvalidRequest, "kubernetes client is not available")
 	}
 
-	// 1. DRA driver controller Deployment available
+	// 1. DRA API resources are discoverable.
+	resources, err := ctx.Clientset.Discovery().ServerResourcesForGroupVersion("resource.k8s.io/v1")
+	if err != nil {
+		return errors.Wrap(errors.ErrCodeNotFound, "resource.k8s.io/v1 API resources not available", err)
+	}
+	var apiResources strings.Builder
+	for _, r := range resources.APIResources {
+		fmt.Fprintf(&apiResources, "%-26s %-22s namespaced=%t\n", r.Name, r.Kind, r.Namespaced)
+	}
+	recordRawTextArtifact(ctx, "DRA API resources",
+		"kubectl api-resources --api-group=resource.k8s.io", apiResources.String())
+
+	// 2. DRA driver pods inventory.
+	pods, err := ctx.Clientset.CoreV1().Pods("nvidia-dra-driver").List(ctx.Context, metav1.ListOptions{})
+	if err != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to list DRA driver pods", err)
+	}
+	var driverPods strings.Builder
+	for _, pod := range pods.Items {
+		fmt.Fprintf(&driverPods, "%-48s ready=%s phase=%s node=%s\n",
+			pod.Name, podReadyCount(pod), pod.Status.Phase, pod.Spec.NodeName)
+	}
+	recordRawTextArtifact(ctx, "DRA driver pods", "kubectl get pods -n nvidia-dra-driver -o wide", driverPods.String())
+
+	// 3. DRA driver controller Deployment available.
 	deploy, deployErr := getDeploymentIfAvailable(ctx, "nvidia-dra-driver", "nvidia-dra-driver-gpu-controller")
+	if deployErr != nil {
+		return errors.Wrap(errors.ErrCodeNotFound, "DRA driver controller check failed", deployErr)
+	}
 	if deploy != nil {
 		expected := int32(1)
 		if deploy.Spec.Replicas != nil {
 			expected = *deploy.Spec.Replicas
 		}
-		recordArtifact(ctx, "DRA Controller Deployment",
+		recordRawTextArtifact(ctx, "DRA Controller Deployment", "",
 			fmt.Sprintf("Name:      %s/%s\nReplicas:  %d/%d available\nImage:     %s",
 				deploy.Namespace, deploy.Name,
 				deploy.Status.AvailableReplicas, expected,
 				firstContainerImage(deploy.Spec.Template.Spec.Containers)))
 	}
-	if deployErr != nil {
-		return errors.Wrap(errors.ErrCodeNotFound, "DRA driver controller check failed", deployErr)
-	}
 
-	// 2. DRA kubelet plugin DaemonSet ready
+	// 4. DRA kubelet plugin DaemonSet ready.
 	ds, dsErr := getDaemonSetIfReady(ctx, "nvidia-dra-driver", "nvidia-dra-driver-gpu-kubelet-plugin")
+	if dsErr != nil {
+		return errors.Wrap(errors.ErrCodeNotFound, "DRA kubelet plugin check failed", dsErr)
+	}
 	if ds != nil {
-		recordArtifact(ctx, "DRA Kubelet Plugin DaemonSet",
+		recordRawTextArtifact(ctx, "DRA Kubelet Plugin DaemonSet", "",
 			fmt.Sprintf("Name:      %s/%s\nReady:     %d/%d pods\nImage:     %s",
 				ds.Namespace, ds.Name,
 				ds.Status.NumberReady, ds.Status.DesiredNumberScheduled,
 				firstContainerImage(ds.Spec.Template.Spec.Containers)))
 	}
-	if dsErr != nil {
-		return errors.Wrap(errors.ErrCodeNotFound, "DRA kubelet plugin check failed", dsErr)
-	}
 
-	// 3. ResourceSlices exist (GPU resources advertised via resource.k8s.io/v1 — GA)
+	// 5. ResourceSlices exist (GPU resources advertised via resource.k8s.io/v1 — GA).
 	dynClient, err := getDynamicClient(ctx)
 	if err != nil {
 		return err
@@ -88,11 +115,98 @@ func CheckDRASupport(ctx *checks.ValidationContext) error {
 	if err != nil {
 		return errors.Wrap(errors.ErrCodeInternal, "failed to list ResourceSlices", err)
 	}
-	recordArtifact(ctx, "ResourceSlices",
-		fmt.Sprintf("Total ResourceSlices: %d", len(slices.Items)))
+	var sliceSummary strings.Builder
+	fmt.Fprintf(&sliceSummary, "Total ResourceSlices: %d\n", len(slices.Items))
+	for _, item := range slices.Items {
+		driver, _, _ := unstructured.NestedString(item.Object, "spec", "driver")
+		nodeName, _, _ := unstructured.NestedString(item.Object, "spec", "nodeName")
+		poolName, _, _ := unstructured.NestedString(item.Object, "spec", "pool", "name")
+		fmt.Fprintf(&sliceSummary, "%-48s node=%s driver=%s pool=%s\n",
+			item.GetName(), nodeName, driver, poolName)
+	}
+	recordRawTextArtifact(ctx, "ResourceSlices", "kubectl get resourceslices", sliceSummary.String())
 	if len(slices.Items) == 0 {
 		return errors.New(errors.ErrCodeNotFound, "no ResourceSlices found (GPU resources not advertised)")
 	}
 
+	// 6. Behavioral DRA allocation validation (create claim+pod, wait, capture observed state).
+	run, err := newDRATestRun()
+	if err != nil {
+		return err
+	}
+	recordRawTextArtifact(ctx, "Apply test manifest",
+		"kubectl apply -f docs/conformance/cncf/manifests/dra-gpu-test.yaml",
+		fmt.Sprintf("Created Namespace=%s ResourceClaim=%s Pod=%s via Kubernetes API",
+			draTestNamespace, run.claimName, run.podName))
+
+	if err := deployDRATestResources(ctx.Context, ctx.Clientset, dynClient, run); err != nil {
+		return err
+	}
+	defer func() {
+		cleanupDRATestResources(ctx.Context, ctx.Clientset, dynClient, run)
+		recordRawTextArtifact(ctx, "Delete test namespace",
+			"kubectl delete namespace dra-test --ignore-not-found",
+			"Deleted DRA test pod and ResourceClaim; namespace retained intentionally to avoid DRA finalizer stalls.")
+	}()
+
+	pod, err := waitForDRATestPod(ctx.Context, ctx.Clientset, run)
+	if err != nil {
+		return err
+	}
+
+	claimObj, err := dynClient.Resource(claimGVR).Namespace(draTestNamespace).Get(
+		ctx.Context, run.claimName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to read DRA test ResourceClaim", err)
+	}
+	state, _, _ := unstructured.NestedString(claimObj.Object, "status", "state")
+	claimLines := []string{
+		fmt.Sprintf("Name:      %s/%s", draTestNamespace, run.claimName),
+		fmt.Sprintf("State:     %s", valueOrUnknown(state)),
+	}
+	recordRawTextArtifact(ctx, "ResourceClaim status",
+		"kubectl get resourceclaim -n dra-test -o wide", strings.Join(claimLines, "\n"))
+
+	podLines := []string{
+		fmt.Sprintf("Name:      %s/%s", pod.Namespace, pod.Name),
+		fmt.Sprintf("Phase:     %s", pod.Status.Phase),
+		fmt.Sprintf("Node:      %s", valueOrUnknown(pod.Spec.NodeName)),
+		fmt.Sprintf("PodIP:     %s", valueOrUnknown(pod.Status.PodIP)),
+		fmt.Sprintf("Claims:    %d", len(pod.Spec.ResourceClaims)),
+	}
+	recordRawTextArtifact(ctx, "Pod status",
+		"kubectl get pod dra-gpu-test -n dra-test -o wide", strings.Join(podLines, "\n"))
+
+	logBytes, logErr := ctx.Clientset.CoreV1().Pods(draTestNamespace).GetLogs(run.podName, &corev1.PodLogOptions{}).DoRaw(ctx.Context)
+	if logErr != nil {
+		recordRawTextArtifact(ctx, "Pod logs", "kubectl logs dra-gpu-test -n dra-test",
+			fmt.Sprintf("failed to read logs: %v", logErr))
+	} else {
+		recordChunkedTextArtifact(ctx, "Pod logs", "kubectl logs dra-gpu-test -n dra-test", string(logBytes))
+	}
+
+	if pod.Status.Phase != corev1.PodSucceeded {
+		return errors.New(errors.ErrCodeInternal,
+			fmt.Sprintf("DRA test pod phase=%s (want Succeeded), GPU allocation may have failed", pod.Status.Phase))
+	}
+
 	return nil
 }
+
+func valueOrUnknown(v string) string {
+	if strings.TrimSpace(v) == "" {
+		return "unknown"
+	}
+	return v
+}
+
+func podReadyCount(pod corev1.Pod) string {
+	var ready, total int
+	for _, cs := range pod.Status.ContainerStatuses {
+		total++
+		if cs.Ready {
+			ready++
+		}
+	}
+	return fmt.Sprintf("%d/%d", ready, total)
+}

pkg/validator/checks/conformance/dra_support_check_unit_test.go

@@ -20,11 +20,16 @@ import (
 	"testing"
 
 	"github.com/NVIDIA/aicr/pkg/validator/checks"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	discoveryfake "k8s.io/client-go/discovery/fake"
 	dynamicfake "k8s.io/client-go/dynamic/fake"
 	"k8s.io/client-go/kubernetes/fake"
+	k8stesting "k8s.io/client-go/testing"
 )
 
 func TestCheckDRASupport(t *testing.T) {
@@ -104,11 +109,65 @@ func TestCheckDRASupport(t *testing.T) {
 				//nolint:staticcheck // SA1019: fake.NewSimpleClientset is sufficient for tests
 				clientset := fake.NewSimpleClientset(tt.k8sObjects...)
 
+				// Discovery API resources for resource.k8s.io/v1.
+				if fd, ok := clientset.Discovery().(*discoveryfake.FakeDiscovery); ok {
+					fd.Resources = []*metav1.APIResourceList{
+						{
+							GroupVersion: "resource.k8s.io/v1",
+							APIResources: []metav1.APIResource{
+								{Name: "deviceclasses", Kind: "DeviceClass", Namespaced: false},
+								{Name: "resourceclaims", Kind: "ResourceClaim", Namespaced: true},
+								{Name: "resourceclaimtemplates", Kind: "ResourceClaimTemplate", Namespaced: true},
+								{Name: "resourceslices", Kind: "ResourceSlice", Namespaced: false},
+							},
+						},
+					}
+				}
+
+				// Behavioral test pod status stub: pods created with dra test prefix
+				// immediately appear completed to avoid poll timeouts in unit tests.
+				podDeleted := false
+				clientset.PrependReactor("get", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
+					ga, ok := action.(k8stesting.GetAction)
+					if !ok {
+						// Let non-standard pod gets (for example log subresource plumbing)
+						// fall back to the default fake behavior.
+						return false, nil, nil
+					}
+					if strings.HasPrefix(ga.GetName(), draTestPrefix) && ga.GetNamespace() == draTestNamespace {
+						if podDeleted {
+							return true, nil, k8serrors.NewNotFound(
+								schema.GroupResource{Resource: "pods"}, ga.GetName())
+						}
+						run := &draTestRun{podName: ga.GetName(), claimName: draClaimPrefix + ga.GetName()[len(draTestPrefix):]}
+						return true, &corev1.Pod{
+							ObjectMeta: metav1.ObjectMeta{
+								Name:      run.podName,
+								Namespace: draTestNamespace,
+							},
+							Spec: *buildDRATestPod(run).Spec.DeepCopy(),
+							Status: corev1.PodStatus{
+								Phase: corev1.PodSucceeded,
+							},
+						}, nil
+					}
+					return false, nil, nil
+				})
+				clientset.PrependReactor("delete", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
+					da := action.(k8stesting.DeleteAction)
+					if strings.HasPrefix(da.GetName(), draTestPrefix) && da.GetNamespace() == draTestNamespace {
+						podDeleted = true
+						return true, nil, nil
+					}
+					return false, nil, nil
+				})
+
 				scheme := runtime.NewScheme()
 				// Always register custom list kinds so List() works even with 0 objects.
 				dynClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme,
 					map[schema.GroupVersionResource]string{
 						{Group: "resource.k8s.io", Version: "v1", Resource: "resourceslices"}: "ResourceSliceList",
+						{Group: "resource.k8s.io", Version: "v1", Resource: "resourceclaims"}: "ResourceClaimList",
 					},
 					tt.dynamicObjects...)
 

pkg/validator/checks/conformance/gang_scheduling_check.go

@@ -115,7 +115,7 @@ func CheckGangScheduling(ctx *checks.ValidationContext) error {
 	}
 
 	// 1. All KAI scheduler deployments available.
-	var schedulerSummary strings.Builder
+	var deploymentsSummary strings.Builder
 	for _, name := range kaiSchedulerDeployments {
 		deploy, err := getDeploymentIfAvailable(ctx, "kai-scheduler", name)
 		if err != nil {
@@ -126,11 +126,24 @@ func CheckGangScheduling(ctx *checks.ValidationContext) error {
 		if deploy.Spec.Replicas != nil {
 			expected = *deploy.Spec.Replicas
 		}
-		fmt.Fprintf(&schedulerSummary, "  %-25s available=%d/%d image=%s\n",
+		fmt.Fprintf(&deploymentsSummary, "%-25s available=%d/%d image=%s\n",
 			name, deploy.Status.AvailableReplicas, expected,
 			firstContainerImage(deploy.Spec.Template.Spec.Containers))
 	}
-	recordArtifact(ctx, "KAI Scheduler Components", schedulerSummary.String())
+	recordRawTextArtifact(ctx, "KAI scheduler deployments",
+		"kubectl get deploy -n kai-scheduler", deploymentsSummary.String())
+
+	// KAI scheduler pods.
+	kaiPods, err := ctx.Clientset.CoreV1().Pods("kai-scheduler").List(ctx.Context, metav1.ListOptions{})
+	if err != nil {
+		return errors.Wrap(errors.ErrCodeInternal, "failed to list KAI scheduler pods", err)
+	}
+	var podsSummary strings.Builder
+	for _, p := range kaiPods.Items {
+		fmt.Fprintf(&podsSummary, "%-44s ready=%s phase=%s\n", p.Name, podReadyCount(p), p.Status.Phase)
+	}
+	recordRawTextArtifact(ctx, "KAI scheduler pods",
+		"kubectl get pods -n kai-scheduler", podsSummary.String())
 
 	// 2. Required CRDs for gang scheduling.
 	dynClient, err := getDynamicClient(ctx)
@@ -152,7 +165,9 @@ func CheckGangScheduling(ctx *checks.ValidationContext) error {
 		}
 		fmt.Fprintf(&crdSummary, "  %s: present\n", crd)
 	}
-	recordArtifact(ctx, "Gang Scheduling CRDs", crdSummary.String())
+	recordRawTextArtifact(ctx, "Gang Scheduling CRDs",
+		"kubectl get crd queues.scheduling.run.ai podgroups.scheduling.run.ai",
+		crdSummary.String())
 
 	// 3. Pre-flight: ensure enough free GPUs for the gang test.
 	total, free, gpuErr := countAvailableGPUs(ctx.Context, dynClient)
@@ -173,7 +188,18 @@ func CheckGangScheduling(ctx *checks.ValidationContext) error {
 		return err
 	}
 
-	defer cleanupGangTestResources(ctx.Context, ctx.Clientset, dynClient, run)
+	defer func() {
+		cleanupGangTestResources(ctx.Context, ctx.Clientset, dynClient, run)
+		recordRawTextArtifact(ctx, "Delete test namespace",
+			"kubectl delete namespace gang-scheduling-test --ignore-not-found",
+			"Deleted gang test pods, claims, and PodGroup; namespace retained intentionally to avoid DRA finalizer stalls.")
+	}()
+
+	recordRawTextArtifact(ctx, "Apply test manifest",
+		"kubectl apply -f docs/conformance/cncf/manifests/gang-scheduling-test.yaml",
+		fmt.Sprintf("Created PodGroup=%s ResourceClaims=%s,%s Pods=%s,%s in namespace=%s",
+			run.groupName, run.claims[0], run.claims[1], run.pods[0], run.pods[1], gangTestNamespace))
+
 	if err = deployGangTestResources(ctx.Context, ctx.Clientset, dynClient, run); err != nil {
 		return err
 	}
@@ -188,7 +214,24 @@ func CheckGangScheduling(ctx *checks.ValidationContext) error {
 		return err
 	}
 
-	// Record gang test results with scheduling timestamps.
+	// PodGroup status.
+	pgList, listErr := dynClient.Resource(podGroupGVR).Namespace(gangTestNamespace).List(
+		ctx.Context, metav1.ListOptions{})
+	if listErr != nil {
+		recordRawTextArtifact(ctx, "PodGroup status",
+			"kubectl get podgroups -n gang-scheduling-test -o wide",
+			fmt.Sprintf("failed to list PodGroups: %v", listErr))
+	} else {
+		var pgSummary strings.Builder
+		for _, item := range pgList.Items {
+			minMember, _, _ := unstructured.NestedInt64(item.Object, "spec", "minMember")
+			fmt.Fprintf(&pgSummary, "%-36s minMember=%d\n", item.GetName(), minMember)
+		}
+		recordRawTextArtifact(ctx, "PodGroup status",
+			"kubectl get podgroups -n gang-scheduling-test -o wide", pgSummary.String())
+	}
+
+	// Pod status and scheduling timestamps.
 	var gangResults strings.Builder
 	for i, pod := range pods {
 		if pod == nil {
@@ -209,7 +252,24 @@ func CheckGangScheduling(ctx *checks.ValidationContext) error {
 	fmt.Fprintf(&gangResults, "Earliest/Latest:  %s / %s\n",
 		gangReport.EarliestScheduled.Format(time.RFC3339),
 		gangReport.LatestScheduled.Format(time.RFC3339))
-	recordArtifact(ctx, "Gang Scheduling Test Results", gangResults.String())
+	recordRawTextArtifact(ctx, "Pod status",
+		"kubectl get pods -n gang-scheduling-test -o wide", gangResults.String())
+
+	// Worker logs.
+	for i := range gangMinMembers {
+		logBytes, logErr := ctx.Clientset.CoreV1().Pods(gangTestNamespace).GetLogs(
+			run.pods[i], &corev1.PodLogOptions{}).DoRaw(ctx.Context)
+		label := fmt.Sprintf("gang-worker-%d logs", i)
+		if logErr != nil {
+			recordRawTextArtifact(ctx, label,
+				fmt.Sprintf("kubectl logs gang-worker-%d -n gang-scheduling-test", i),
+				fmt.Sprintf("failed to read logs: %v", logErr))
+			continue
+		}
+		recordChunkedTextArtifact(ctx, label,
+			fmt.Sprintf("kubectl logs gang-worker-%d -n gang-scheduling-test", i),
+			string(logBytes))
+	}
 
 	return nil
 }