fix(bundler): honor bundlers query param; driver-managed cleanup (#1606)

Signed-off-by: Mark Chmarny <mark@chmarny.com>

Author: mchmarny

DEVELOPMENT.md

@@ -559,6 +559,26 @@ Verify a bundle with `aicr verify <dir>`. Update the trusted root cache with
 | Build failures | Run `make tidy` to update dependencies |
 | K8s connection fails | Check `~/.kube/config` or `KUBECONFIG` env |
 
+### Redeploying on Driver-Managed Clusters
+
+On clusters where gpu-operator installs the NVIDIA kernel driver
+(`driver.enabled: true` — e.g. EKS; GKE COS is host-managed and unaffected),
+running `make cleanup` (`tools/cleanup`) can leave the `nvidia_uvm` kernel
+module wedged mid-unload on GPU nodes. The symptom appears on the *next*
+install: the driver validation init container crashes
+(`Init:CrashLoopBackOff`, logs show `failed to create device node
+nvidia-uvm`). The cleanup script detects driver-managed mode (via the
+`nvidia-driver-daemonset` or `driver.enabled` in the release values) and
+prints this guidance; the fix is to reboot the GPU nodes before reinstalling:
+
+```bash
+kubectl cordon <gpu-node>            # each GPU node
+# reboot the instances, e.g. on EKS:
+aws ec2 reboot-instances --instance-ids <ids>
+# wait for the nodes to report Ready, then
+kubectl uncordon <gpu-node>
+```
+
 ### Debugging Tests
 
 ```bash

api/aicr/v1/server.yaml

@@ -742,8 +742,13 @@ paths:
           in: query
           required: false
           description: >
-            Comma-delimited list of bundler types to execute.
-            If not specified, all registered bundlers are executed.
+            Comma-delimited list of recipe component names to bundle.
+            If not specified, all enabled components are bundled.
+            Components not listed are skipped as if disabled (their
+            dependency edges are treated as satisfied externally).
+            A name the recipe does not declare, or one that is disabled
+            (by the recipe or by a "set" parameter enabled=false
+            override), is rejected with HTTP 400.
           schema:
             type: string
           examples:
@@ -1020,7 +1025,7 @@ paths:
                 type: string
                 format: binary
         "400":
-          description: Invalid request (invalid recipe or bundler type)
+          description: Invalid request (invalid recipe or bundlers filter)
           headers:
             X-Request-Id:
               $ref: "#/components/headers/RequestIdResponse"
@@ -1037,15 +1042,19 @@ paths:
                     requestId: "550e8400-e29b-41d4-a716-446655440000"
                     timestamp: "2025-01-15T10:30:00Z"
                     retryable: false
-                invalidBundler:
-                  summary: Invalid bundler type
+                unknownBundler:
+                  summary: Bundlers filter names an undeclared component
                   value:
                     code: INVALID_REQUEST
-                    message: "Invalid bundler type"
-                    details:
-                      bundler: "invalid-bundler"
-                      error: "unsupported bundle type: invalid-bundler"
-                      valid: ["gpu-operator", "network-operator", "nodewright-operator", "nvsentinel", "cert-manager", "nvidia-dra-driver-gpu"]
+                    message: 'unknown component "invalid-bundler" in bundlers filter; recipe declares: gpu-operator, network-operator, cert-manager'
+                    requestId: "550e8400-e29b-41d4-a716-446655440000"
+                    timestamp: "2025-01-15T10:30:00Z"
+                    retryable: false
+                disabledBundler:
+                  summary: Bundlers filter names a disabled component
+                  value:
+                    code: INVALID_REQUEST
+                    message: 'component "cert-manager" is disabled and cannot be selected via the bundlers filter'
                     requestId: "550e8400-e29b-41d4-a716-446655440000"
                     timestamp: "2025-01-15T10:30:00Z"
                     retryable: false
@@ -1554,8 +1563,10 @@ components:
           type: array
           items:
             type: string
-            enum: [gpu-operator, network-operator, nodewright-operator, nvsentinel, cert-manager, nvidia-dra-driver-gpu]
           description: >
-            Optional list of bundler types to execute.
-            If not specified, all registered bundlers are executed.
+            Optional list of recipe component names to bundle.
+            If not specified, all enabled components are bundled.
+            Superseded by the "bundlers" query parameter, which accepts
+            enabled component names the recipe declares; unknown or
+            disabled names are rejected with HTTP 400.
           example: [gpu-operator, network-operator]

docs/integrator/index.md

@@ -54,10 +54,10 @@ curl "http://aicrd.aicr.svc/v1/recipe?service=eks&accelerator=h100"
 - name: Generate bundles
   # recipe.json is the fully-hydrated RecipeResult from the GET above; the
   # bundle endpoint adopts it as-is and bundles all of its componentRefs.
-  # (The `bundlers` query param is not currently honored — see
-  # https://github.com/NVIDIA/aicr/issues/1531. There is no supported way to
-  # bundle a subset via the API today; hand-trimming componentRefs drops
-  # required dependencies, so bundle the full result.)
+  # To bundle a subset, add the `bundlers` query parameter (comma-delimited
+  # component names, e.g. "?bundlers=gpu-operator,network-operator") rather
+  # than hand-trimming componentRefs, which drops required dependencies.
+  # Unknown or disabled component names are rejected with HTTP 400.
   run: |
     curl -X POST "http://aicrd.aicr.svc/v1/bundle" \
       -H "Content-Type: application/json" \

docs/user/api-reference.md

@@ -367,7 +367,7 @@ Generate deployment bundles from a recipe.
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
-| `bundlers` | string | (all) | Comma-delimited list of bundler types to execute. **Not currently honored — all bundlers run regardless ([#1531](https://github.com/NVIDIA/aicr/issues/1531)).** |
+| `bundlers` | string | (all) | Comma-delimited list of recipe component names to bundle (e.g. `gpu-operator,network-operator`). Whitespace around names is trimmed. Components not listed are skipped as if disabled (their dependency edges are treated as satisfied externally). A name the recipe does not declare, or one that is disabled (by the recipe or a `set` `enabled=false` override), is rejected with HTTP 400. |
 | `set` | string[] | | Value overrides (format: `bundler:path.to.field=value`). Repeat for multiple. |
 | `dynamic` | string[] | | Declare value paths as install-time parameters (format: `component:path.to.field`). Repeat for multiple. Supported with `deployer=helm`, `deployer=argocd-helm`, `deployer=flux`, and `deployer=helmfile`. |
 | `system-node-selector` | string[] | | Node selectors for system components (format: `key=value`). Repeat for multiple. |
@@ -386,7 +386,7 @@ The request body is the recipe (RecipeResult) directly. No wrapper object needed
 
 #### Components
 
-These are the recipe **components** in [`recipes/registry.yaml`](https://github.com/NVIDIA/aicr/blob/main/recipes/registry.yaml). (The `bundlers` query parameter that would select a subset is currently ignored — all run regardless, #1531.) The registry is the authoritative source — see the [component catalog](component-catalog.md) for the full, current list with detailed descriptions. The table below is illustrative of commonly used components:
+These are the recipe **components** in [`recipes/registry.yaml`](https://github.com/NVIDIA/aicr/blob/main/recipes/registry.yaml) — the names the `bundlers` query parameter accepts (a request may only name components the recipe declares). The registry is the authoritative source — see the [component catalog](component-catalog.md) for the full, current list with detailed descriptions. The table below is illustrative of commonly used components:
 
 | Component | Description |
 |-----------|-------------|
@@ -436,12 +436,13 @@ These are the recipe **components** in [`recipes/registry.yaml`](https://github.
 > a few component fields shown) — use a generated `RecipeResult`, not these
 > literals.
 >
-> The `bundlers` query parameter is **not currently honored** — all bundlers run
-> regardless ([#1531](https://github.com/NVIDIA/aicr/issues/1531)). There is no
-> supported way to bundle a subset via the API today: hand-trimming
-> `componentRefs` is unsafe (it silently drops required dependencies and breaks
-> deployers like Helmfile on dangling `dependencyRefs`). Bundle the full
-> hydrated result.
+> To bundle a subset of the recipe's components, use the `bundlers` query
+> parameter (e.g. `?bundlers=gpu-operator,network-operator`) rather than
+> hand-trimming `componentRefs` — trimming the body silently drops required
+> dependencies and breaks deployers like Helmfile on dangling
+> `dependencyRefs`. The filter prunes those edges safely (a filtered-out
+> dependency is assumed satisfied externally) and rejects unknown or disabled
+> component names with HTTP 400.
 
 ```shell
 # Basic: pipe recipe to bundle

pkg/bundler/bundler.go

@@ -798,7 +798,8 @@ func (b *DefaultBundler) getTypedValueOverridesForComponent(componentName string
 }
 
 // filterEnabledComponents resolves the set of components to bundle by applying
-// recipe-level overrides.enabled and bundle-time --set enabled toggles, then
+// recipe-level overrides.enabled, bundle-time --set enabled toggles, and the
+// positive bundlers component-name filter (config.WithBundlers, #1531), then
 // returns the enabled refs (with dangling dependency edges pruned) alongside
 // the deployment order filtered to those refs.
 //
@@ -845,6 +846,46 @@ func (b *DefaultBundler) filterEnabledComponents(recipeResult *recipe.RecipeResu
 		enabledSet[ref.Name] = struct{}{}
 	}
 
+	// Apply the positive component-name filter (POST /v1/bundle ?bundlers=…,
+	// config.WithBundlers). Requested names must be declared AND enabled —
+	// an unknown name is a typo the operator needs to hear about, and a
+	// disabled one mirrors the --set re-enable rejection above (the recipe
+	// author disabled it because the platform provides it). Enabled
+	// components outside the requested set are skipped exactly like
+	// disabled ones, so the dependency-edge pruning below treats them as
+	// satisfied externally. See #1531.
+	if b.Config != nil {
+		if requested := b.Config.Bundlers(); len(requested) > 0 {
+			requestedSet := make(map[string]struct{}, len(requested))
+			for _, name := range requested {
+				if _, declared := declaredSet[name]; !declared {
+					declaredNames := make([]string, 0, len(recipeResult.ComponentRefs))
+					for _, ref := range recipeResult.ComponentRefs {
+						declaredNames = append(declaredNames, ref.Name)
+					}
+					return nil, nil, errors.New(errors.ErrCodeInvalidRequest, fmt.Sprintf(
+						"unknown component %q in bundlers filter; recipe declares: %s",
+						name, strings.Join(declaredNames, ", ")))
+				}
+				if _, enabled := enabledSet[name]; !enabled {
+					return nil, nil, errors.New(errors.ErrCodeInvalidRequest, fmt.Sprintf(
+						"component %q is disabled and cannot be selected via the bundlers filter", name))
+				}
+				requestedSet[name] = struct{}{}
+			}
+			kept := make([]recipe.ComponentRef, 0, len(requestedSet))
+			for _, ref := range enabledRefs {
+				if _, ok := requestedSet[ref.Name]; !ok {
+					slog.Info("skipping component excluded by bundlers filter", "component", ref.Name)
+					delete(enabledSet, ref.Name)
+					continue
+				}
+				kept = append(kept, ref)
+			}
+			enabledRefs = kept
+		}
+	}
+
 	if len(enabledRefs) == 0 {
 		return nil, nil, errors.New(errors.ErrCodeInvalidRequest,
 			"recipe has no enabled components after filtering")

pkg/bundler/bundler_test.go

@@ -426,6 +426,153 @@ func TestMake_UndeclaredDependencyErrors(t *testing.T) {
 	}
 }
 
+// TestMake_BundlersFilter pins the semantics of the `bundlers` positive
+// component-name filter (POST /v1/bundle ?bundlers=…, config.WithBundlers):
+// a subset selection bundles only the named components, an unknown or
+// disabled name fails with ErrCodeInvalidRequest, and an empty filter
+// preserves current behavior (all enabled components). See #1531.
+func TestMake_BundlersFilter(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		bundlers    []string
+		wantDirs    []string
+		wantAbsent  []string
+		wantErr     bool
+		wantErrText string
+	}{
+		{
+			name:       "empty filter bundles all enabled components",
+			bundlers:   nil,
+			wantDirs:   []string{"001-gpu-operator", "002-aws-ebs-csi-driver"},
+			wantAbsent: []string{"cert-manager", "001-cert-manager", "003-cert-manager"},
+		},
+		{
+			name:     "filter selects subset",
+			bundlers: []string{"gpu-operator"},
+			wantDirs: []string{"001-gpu-operator"},
+			wantAbsent: []string{
+				"aws-ebs-csi-driver", "001-aws-ebs-csi-driver", "002-aws-ebs-csi-driver",
+			},
+		},
+		{
+			name:        "unknown name errors",
+			bundlers:    []string{"gpu-operator", "no-such-component"},
+			wantErr:     true,
+			wantErrText: "unknown component",
+		},
+		{
+			name:        "disabled component request errors",
+			bundlers:    []string{"cert-manager"},
+			wantErr:     true,
+			wantErrText: "disabled",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			var bundlerOpts []Option
+			if tt.bundlers != nil {
+				bundlerOpts = append(bundlerOpts,
+					WithConfig(config.NewConfig(config.WithBundlers(tt.bundlers))))
+			}
+			bundler, err := New(bundlerOpts...)
+			if err != nil {
+				t.Fatalf("New() error = %v", err)
+			}
+
+			recipeResult := &recipe.RecipeResult{
+				APIVersion: "aicr.run/v1alpha2",
+				Kind:       "Recipe",
+				Criteria:   &recipe.Criteria{Service: "eks", Accelerator: "h100", Intent: "training"},
+				ComponentRefs: []recipe.ComponentRef{
+					{Name: "gpu-operator", Version: "v25.3.3", Type: "helm", Source: "https://helm.ngc.nvidia.com/nvidia"},
+					{Name: "aws-ebs-csi-driver", Version: "2.55.0", Type: "helm", Source: "https://kubernetes-sigs.github.io/aws-ebs-csi-driver"},
+					{Name: "cert-manager", Version: "v1.20.2", Type: "helm", Source: "https://charts.jetstack.io", Overrides: map[string]any{"enabled": false}},
+				},
+				DeploymentOrder: []string{"gpu-operator", "aws-ebs-csi-driver"},
+			}
+
+			ctx := context.Background()
+			tmpDir := t.TempDir()
+			_, makeErr := bundler.Make(ctx, recipeResult, tmpDir)
+			if tt.wantErr {
+				if makeErr == nil {
+					t.Fatal("Make() expected error, got nil")
+				}
+				if !stderrors.Is(makeErr, errors.New(errors.ErrCodeInvalidRequest, "")) {
+					t.Errorf("Make() error code = %v, want ErrCodeInvalidRequest", makeErr)
+				}
+				if !strings.Contains(makeErr.Error(), tt.wantErrText) {
+					t.Errorf("Make() error = %q, want substring %q", makeErr.Error(), tt.wantErrText)
+				}
+				return
+			}
+			if makeErr != nil {
+				t.Fatalf("Make() error = %v", makeErr)
+			}
+			for _, dir := range tt.wantDirs {
+				if _, statErr := os.Stat(filepath.Join(tmpDir, dir)); os.IsNotExist(statErr) {
+					t.Errorf("expected %s directory to be created", dir)
+				}
+			}
+			for _, dir := range tt.wantAbsent {
+				if _, statErr := os.Stat(filepath.Join(tmpDir, dir)); !os.IsNotExist(statErr) {
+					t.Errorf("expected %s directory to NOT be created", dir)
+				}
+			}
+		})
+	}
+}
+
+// TestMake_BundlersFilterDependencyPruned verifies that a dependency edge
+// pointing at an enabled-but-filtered-out component is pruned exactly like a
+// disabled one: the helmfile deployer (the only path that recomputes ordering
+// from dependency edges) must not fail with a false circular-dependency error
+// when the depended-upon component is excluded by the bundlers filter. See #1531.
+func TestMake_BundlersFilterDependencyPruned(t *testing.T) {
+	t.Parallel()
+
+	cfg := config.NewConfig(
+		config.WithDeployer(config.DeployerHelmfile),
+		config.WithBundlers([]string{"gpu-operator"}),
+	)
+	bundler, err := New(WithConfig(cfg))
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+
+	recipeResult := &recipe.RecipeResult{
+		APIVersion: "aicr.run/v1alpha2",
+		Kind:       "Recipe",
+		Criteria:   &recipe.Criteria{Service: "eks", Accelerator: "h100", Intent: "training"},
+		ComponentRefs: []recipe.ComponentRef{
+			// cert-manager is enabled but excluded by the bundlers filter;
+			// gpu-operator depends on it — assumed satisfied externally.
+			{Name: "cert-manager", Version: "v1.20.2", Type: "helm", Source: "https://charts.jetstack.io"},
+			{Name: "gpu-operator", Version: "v25.3.3", Type: "helm", Source: "https://helm.ngc.nvidia.com/nvidia", DependencyRefs: []string{"cert-manager"}},
+		},
+		DeploymentOrder: []string{"cert-manager", "gpu-operator"},
+	}
+
+	ctx := context.Background()
+	tmpDir := t.TempDir()
+	if _, makeErr := bundler.Make(ctx, recipeResult, tmpDir); makeErr != nil {
+		t.Fatalf("Make() with filtered-out depended-upon component error = %v", makeErr)
+	}
+	if _, statErr := os.Stat(filepath.Join(tmpDir, "001-gpu-operator")); os.IsNotExist(statErr) {
+		t.Error("expected 001-gpu-operator to be created")
+	}
+	for _, dir := range []string{"cert-manager", "001-cert-manager", "002-cert-manager"} {
+		if _, statErr := os.Stat(filepath.Join(tmpDir, dir)); !os.IsNotExist(statErr) {
+			t.Errorf("expected %s directory to NOT be created", dir)
+		}
+	}
+}
+
 func TestMake_SetEnabledOverridesPrecedence(t *testing.T) {
 	t.Parallel()