feat: add expected-resources deployment check for validating Kubernetes resources exist

Author: xdu31

docs/integrator/recipe-development.md

@@ -254,15 +254,23 @@ constraints:
 Optional multi-phase validation beyond basic constraints:
 
 ```yaml
+# expectedResources are declared on componentRefs, not under validation
+componentRefs:
+  - name: gpu-operator
+    type: Helm
+    expectedResources:
+      - kind: Deployment
+        name: gpu-operator
+        namespace: gpu-operator
+      - kind: DaemonSet
+        name: nvidia-driver-daemonset
+        namespace: gpu-operator
+
 validation:
   readiness:
     checks: [gpu-hardware-detection, kernel-parameters]
   deployment:
-    checks: [operator-health]
-    expectedResources:
-      - apiVersion: v1
-        kind: Pod
-        namespace: gpu-operator
+    checks: [operator-health, expected-resources]
   performance:
     infrastructure: nccl-doctor
     checks: [nccl-bandwidth-test]

examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml

@@ -97,6 +97,16 @@ componentRefs:
     source: https://helm.ngc.nvidia.com/nvidia
     version: v25.3.3
     valuesFile: components/gpu-operator/values-eks-training.yaml
+    expectedResources:
+      - kind: Deployment
+        name: gpu-operator
+        namespace: gpu-operator
+      - kind: DaemonSet
+        name: nvidia-driver-daemonset
+        namespace: gpu-operator
+      - kind: DaemonSet
+        name: nvidia-device-plugin-daemonset
+        namespace: gpu-operator
     overrides:
       cdi:
         default: false

pkg/recipe/metadata_store.go

@@ -241,6 +241,7 @@ func (s *MetadataStore) initBaseMergedSpec() (RecipeMetadataSpec, []string) {
 	mergedSpec := RecipeMetadataSpec{
 		Constraints:   make([]Constraint, len(s.Base.Spec.Constraints)),
 		ComponentRefs: make([]ComponentRef, len(s.Base.Spec.ComponentRefs)),
+		Validation:    s.Base.Spec.Validation,
 	}
 	copy(mergedSpec.Constraints, s.Base.Spec.Constraints)
 	copy(mergedSpec.ComponentRefs, s.Base.Spec.ComponentRefs)

pkg/validator/README.md

@@ -313,6 +313,15 @@ validation:
 
 **Checks** - Named validation tests:
 ```yaml
+# expected-resources check requires expectedResources on componentRefs
+componentRefs:
+  - name: gpu-operator
+    type: Helm
+    expectedResources:
+      - kind: Deployment
+        name: gpu-operator
+        namespace: gpu-operator
+
 validation:
   deployment:
     checks:
@@ -323,6 +332,18 @@ validation:
 ### Multi-Phase Recipe Example
 
 ```yaml
+# expectedResources are declared on componentRefs (used by expected-resources check)
+componentRefs:
+  - name: gpu-operator
+    type: Helm
+    expectedResources:
+      - kind: Deployment
+        name: gpu-operator
+        namespace: gpu-operator
+      - kind: DaemonSet
+        name: nvidia-driver-daemonset
+        namespace: gpu-operator
+
 validation:
   # Phase 1: Readiness (pre-deployment validation)
   readiness:

pkg/validator/checks/README.md

@@ -213,6 +213,18 @@ make generate-validator ARGS="--constraint Deployment.my-app.version --phase dep
 ### Example Recipe Usage
 
 ```yaml
+# expectedResources are declared on componentRefs (used by expected-resources check)
+componentRefs:
+  - name: gpu-operator
+    type: Helm
+    expectedResources:
+      - kind: Deployment
+        name: gpu-operator
+        namespace: gpu-operator
+      - kind: DaemonSet
+        name: nvidia-driver-daemonset
+        namespace: gpu-operator
+
 validation:
   deployment:
     constraints:
@@ -224,7 +236,7 @@ validation:
     checks:
       # These also run inside the Job
       - operator-health
-      - expected-resources
+      - expected-resources  # validates componentRefs[].expectedResources
 ```
 
 ## Registration Pattern

pkg/validator/checks/deployment/expected_resources_check.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deployment
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/NVIDIA/eidos/pkg/errors"
+	"github.com/NVIDIA/eidos/pkg/recipe"
+	"github.com/NVIDIA/eidos/pkg/validator/checks"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func init() {
+	// Register this check
+	checks.RegisterCheck(&checks.Check{
+		Name:        "expected-resources",
+		Description: "Verify expected Kubernetes resources exist and are healthy after component deployment",
+		Phase:       "deployment",
+		Func:        validateExpectedResources,
+		TestName:    "TestCheckExpectedResources",
+	})
+}
+
+// validateExpectedResources verifies that all expected Kubernetes resources declared
+// in the recipe's componentRefs exist and are healthy in the live cluster.
+func validateExpectedResources(ctx *checks.ValidationContext) error {
+	if ctx.Clientset == nil {
+		return errors.New(errors.ErrCodeInvalidRequest, "kubernetes client is not available")
+	}
+	if ctx.Recipe == nil {
+		return errors.New(errors.ErrCodeInvalidRequest, "recipe is not available")
+	}
+
+	var failures []string
+
+	for _, ref := range ctx.Recipe.ComponentRefs {
+		for _, er := range ref.ExpectedResources {
+			if err := verifyResource(ctx.Context, ctx.Clientset, er); err != nil {
+				failures = append(failures, fmt.Sprintf("%s %s/%s (%s): %s",
+					er.Kind, er.Namespace, er.Name, ref.Name, err.Error()))
+			}
+		}
+	}
+
+	if len(failures) > 0 {
+		return errors.New(errors.ErrCodeNotFound,
+			fmt.Sprintf("expected resource check failed:\n  %s", strings.Join(failures, "\n  ")))
+	}
+	return nil
+}
+
+// verifyResource checks that a single expected resource exists and is healthy.
+func verifyResource(ctx context.Context, clientset kubernetes.Interface, er recipe.ExpectedResource) error {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	switch er.Kind {
+	case "Deployment":
+		deploy, err := clientset.AppsV1().Deployments(er.Namespace).Get(ctx, er.Name, metav1.GetOptions{})
+		if err != nil {
+			return errors.Wrap(errors.ErrCodeNotFound, "not found", err)
+		}
+		expected := int32(1)
+		if deploy.Spec.Replicas != nil {
+			expected = *deploy.Spec.Replicas
+		}
+		if deploy.Status.AvailableReplicas < expected {
+			return errors.New(errors.ErrCodeInternal,
+				fmt.Sprintf("not healthy: %d/%d replicas available",
+					deploy.Status.AvailableReplicas, expected))
+		}
+
+	case "DaemonSet":
+		ds, err := clientset.AppsV1().DaemonSets(er.Namespace).Get(ctx, er.Name, metav1.GetOptions{})
+		if err != nil {
+			return errors.Wrap(errors.ErrCodeNotFound, "not found", err)
+		}
+		if ds.Status.NumberReady < ds.Status.DesiredNumberScheduled {
+			return errors.New(errors.ErrCodeInternal,
+				fmt.Sprintf("not healthy: %d/%d pods ready",
+					ds.Status.NumberReady, ds.Status.DesiredNumberScheduled))
+		}
+
+	case "StatefulSet":
+		ss, err := clientset.AppsV1().StatefulSets(er.Namespace).Get(ctx, er.Name, metav1.GetOptions{})
+		if err != nil {
+			return errors.Wrap(errors.ErrCodeNotFound, "not found", err)
+		}
+		expected := int32(1)
+		if ss.Spec.Replicas != nil {
+			expected = *ss.Spec.Replicas
+		}
+		if ss.Status.ReadyReplicas < expected {
+			return errors.New(errors.ErrCodeInternal,
+				fmt.Sprintf("not healthy: %d/%d replicas ready",
+					ss.Status.ReadyReplicas, expected))
+		}
+
+	case "Service":
+		_, err := clientset.CoreV1().Services(er.Namespace).Get(ctx, er.Name, metav1.GetOptions{})
+		if err != nil {
+			return errors.Wrap(errors.ErrCodeNotFound, "not found", err)
+		}
+
+	case "ConfigMap":
+		_, err := clientset.CoreV1().ConfigMaps(er.Namespace).Get(ctx, er.Name, metav1.GetOptions{})
+		if err != nil {
+			return errors.Wrap(errors.ErrCodeNotFound, "not found", err)
+		}
+
+	default:
+		return errors.New(errors.ErrCodeInvalidRequest,
+			fmt.Sprintf("unsupported resource kind %q", er.Kind))
+	}
+
+	return nil
+}

pkg/validator/checks/deployment/expected_resources_check_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deployment
+
+import (
+	"testing"
+
+	"github.com/NVIDIA/eidos/pkg/validator/checks"
+)
+
+// TestCheckExpectedResources is the integration test for expected-resources.
+// This runs inside validator Jobs and invokes the validator.
+func TestCheckExpectedResources(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	// Load Job environment
+	runner, err := checks.NewTestRunner(t)
+	if err != nil {
+		t.Skipf("Not in Job environment: %v", err)
+	}
+	defer runner.Cancel()
+
+	// Check if this check is enabled in recipe
+	if !runner.HasCheck("deployment", "expected-resources") {
+		t.Skip("Check expected-resources not enabled in recipe")
+	}
+
+	t.Logf("Running check: expected-resources")
+
+	// Run the validator
+	ctx := runner.Context()
+	err = validateExpectedResources(ctx)
+
+	if err != nil {
+		t.Errorf("Check failed: %v", err)
+	} else {
+		t.Logf("✓ Check passed: expected-resources")
+	}
+}