feat: add support for workload-gate and workload-selector

Mainly for use with Skyhook and skyhook-customizations to allow proper scheduling around workloads in day-2 operations

Author: ayuskauskas

docs/user/cli-reference.md

@@ -678,6 +678,8 @@ eidos bundle [flags]
 | `--system-node-toleration` | | string[] | Toleration for system components (format: key=value:effect, repeatable) |
 | `--accelerated-node-selector` | | string[] | Node selector for accelerated/GPU nodes (format: key=value, repeatable) |
 | `--accelerated-node-toleration` | | string[] | Toleration for accelerated/GPU nodes (format: key=value:effect, repeatable) |
+| `--workload-gate` | | string | Taint for skyhook-operator runtime required (format: key=value:effect or key:effect). This is a day 2 option for cluster scaling operations. |
+| `--workload-selector` | | string[] | Label selector for skyhook-customizations to prevent eviction of running training jobs (format: key=value, repeatable). Required when skyhook-customizations is enabled with training intent. |
 
 **Behavior:**
 - All components from the recipe are bundled automatically
@@ -768,6 +770,12 @@ eidos bundle -r recipe.yaml \
   --accelerated-node-toleration nvidia.com/gpu=present:NoSchedule \
   -o ./bundles
 
+# Day 2 options: workload-gate and workload-selector for skyhook
+eidos bundle -r recipe.yaml \
+  --workload-gate skyhook.io/runtime-required=true:NoSchedule \
+  --workload-selector workload-type=training \
+  -o ./bundles
+
 # Generate ArgoCD Application manifests for GitOps
 eidos bundle -r recipe.yaml --deployer argocd -o ./bundles
 
@@ -816,6 +824,48 @@ bundles/
 └── README.md                      # ArgoCD deployment guide
 ```
 
+**Day 2 Options:**
+
+The `--workload-gate` and `--workload-selector` flags are day 2 operational options for cluster scaling operations:
+
+- **`--workload-gate`**: Specifies a taint for skyhook-operator's runtime required feature. This ensures nodes are properly configured before workloads can schedule on them during cluster scaling. The taint is configured in the skyhook-operator Helm values file at `controllerManager.manager.env.runtimeRequiredTaint`. For more information about runtime required, see the [skyhook documentation](hhttps://github.com/NVIDIA/skyhook/blob/main/docs/runtime_required.md).
+
+- **`--workload-selector`**: Specifies a label selector for skyhook-customizations to prevent skyhook from evicting running training jobs. This is critical for training workloads where job eviction would cause significant disruption. The selector is set in the Skyhook CR manifest (tuning.yaml) in the `spec.workloadSelector.matchLabels` field.
+
+**Validation Warnings:**
+
+When generating bundles with skyhook-customizations enabled, validation warnings are displayed for missing configuration:
+
+1. **Workload Selector Warning**: When skyhook-customizations is enabled with training intent, if `--workload-selector` is not set, a warning will be displayed:
+
+```
+Warning: skyhook-customizations is enabled with training intent but --workload-selector is not set. 
+This may cause skyhook to evict running training jobs. Consider setting --workload-selector to prevent eviction.
+```
+
+2. **Accelerated Selector Warning**: When skyhook-customizations is enabled with training or inference intent, if `--accelerated-node-selector` is not set, a warning will be displayed:
+
+```
+Warning: skyhook-customizations is enabled with {training|inference} intent but --accelerated-node-selector is not set. 
+Without this selector, the customization will run on all nodes. Consider setting --accelerated-node-selector to target specific nodes.
+```
+
+**Examples:**
+```shell
+# Generate bundle with day 2 options for training workloads
+eidos bundle -r recipe.yaml \
+  --workload-gate skyhook.io/runtime-required=true:NoSchedule \
+  --workload-selector workload-type=training \
+  --workload-selector intent=training \
+  --accelerated-node-selector accelerator=nvidia-h100 \
+  -o ./bundles
+
+# Generate bundle for inference workloads with accelerated selector
+eidos bundle -r recipe.yaml \
+  --accelerated-node-selector accelerator=nvidia-h100 \
+  -o ./bundles
+```
+
 ArgoCD Applications use multi-source to:
 1. Pull Helm charts from upstream repositories
 2. Apply values.yaml from your GitOps repository

pkg/bundler/bundler.go

@@ -50,6 +50,9 @@ type DefaultBundler struct {
 	// AllowLists defines which criteria values are permitted for bundle requests.
 	// When set, the bundler validates that the recipe's criteria are within the allowed values.
 	AllowLists *recipe.AllowLists
+
+	// warnings stores warning messages to be added to deployment notes.
+	warnings []string
 }
 
 // Option defines a functional option for configuring DefaultBundler.
@@ -160,6 +163,12 @@ func (b *DefaultBundler) Make(ctx context.Context, input recipe.RecipeInput, dir
 			"failed to extract component values", err)
 	}
 
+	// Validate workload-selector for skyhook-customizations with training intent
+	b.validateWorkloadSelector(recipeResult)
+
+	// Validate accelerated selector for skyhook-customizations with training/inference intent
+	b.validateAcceleratedSelector(recipeResult)
+
 	// Route based on deployer
 	deployer := b.Config.Deployer()
 	if deployer == config.DeployerArgoCD {
@@ -226,9 +235,14 @@ func (b *DefaultBundler) makeHelmBundle(ctx context.Context, recipeResult *recip
 	resultOutput.Results = append(resultOutput.Results, helmResult)
 
 	// Populate deployment info from generator output
+	notes := make([]string, 0)
+	if len(b.warnings) > 0 {
+		notes = append(notes, b.warnings...)
+	}
 	resultOutput.Deployment = &result.DeploymentInfo{
 		Type:  "Helm per-component bundle",
 		Steps: output.DeploymentSteps,
+		Notes: notes,
 	}
 
 	slog.Debug("helm bundle generation complete",
@@ -284,10 +298,17 @@ func (b *DefaultBundler) makeArgoCD(ctx context.Context, recipeResult *recipe.Re
 	resultOutput.Results = append(resultOutput.Results, argocdResult)
 
 	// Populate deployment info from generator output
+	notes := make([]string, 0)
+	if len(output.DeploymentNotes) > 0 {
+		notes = append(notes, output.DeploymentNotes...)
+	}
+	if len(b.warnings) > 0 {
+		notes = append(notes, b.warnings...)
+	}
 	resultOutput.Deployment = &result.DeploymentInfo{
 		Type:  "ArgoCD applications",
 		Steps: output.DeploymentSteps,
-		Notes: output.DeploymentNotes,
+		Notes: notes,
 	}
 
 	slog.Debug("argocd applications generation complete",
@@ -329,7 +350,7 @@ func (b *DefaultBundler) extractComponentValues(ctx context.Context, recipeResul
 			}
 		}
 
-		// Apply node selectors and tolerations based on component type
+		// Apply node selectors, tolerations, workload selector, and taints based on component type
 		b.applyNodeSchedulingOverrides(ref.Name, values)
 
 		componentValues[ref.Name] = values
@@ -433,6 +454,109 @@ func (b *DefaultBundler) applyNodeSchedulingOverrides(componentName string, valu
 			component.ApplyTolerationsOverrides(values, tolerations, paths...)
 		}
 	}
+
+	// Apply workload selector (for components like skyhook-customizations)
+	if workloadSelector := b.Config.WorkloadSelector(); len(workloadSelector) > 0 {
+		if paths := comp.GetAcceleratedWorkloadSelectorPaths(); len(paths) > 0 {
+			component.ApplyNodeSelectorOverrides(values, workloadSelector, paths...)
+		}
+	}
+
+	// Apply workload-gate taint (as string format for skyhook-operator)
+	if taint := b.Config.WorkloadGateTaint(); taint != nil {
+		if paths := comp.GetAcceleratedTaintStrPaths(); len(paths) > 0 {
+			taintStr := taint.ToString()
+			overrides := make(map[string]string, len(paths))
+			for _, path := range paths {
+				overrides[path] = taintStr
+			}
+			if err := component.ApplyMapOverrides(values, overrides); err != nil {
+				slog.Warn("failed to apply workload-gate taint",
+					"component", componentName,
+					"error", err,
+				)
+			}
+		}
+	}
+}
+
+// validateWorkloadSelector validates that workload-selector is set when skyhook-customizations
+// is present with training intent.
+func (b *DefaultBundler) validateWorkloadSelector(recipeResult *recipe.RecipeResult) {
+	if b.Config == nil {
+		return
+	}
+
+	// Check if skyhook-customizations component exists
+	hasSkyhookCustomizations := false
+	for _, ref := range recipeResult.ComponentRefs {
+		if ref.Name == "skyhook-customizations" {
+			hasSkyhookCustomizations = true
+			break
+		}
+	}
+
+	if !hasSkyhookCustomizations {
+		return
+	}
+
+	// Check if intent is training
+	if recipeResult.Criteria == nil || recipeResult.Criteria.Intent != recipe.CriteriaIntentTraining {
+		return
+	}
+
+	// Check if workload-selector is not set
+	selector := b.Config.WorkloadSelector()
+	if len(selector) == 0 {
+		slog.Warn("skyhook-customizations is enabled with training intent but --workload-selector is not set",
+			"component", "skyhook-customizations",
+			"intent", "training",
+		)
+		// Store warning to be added to deployment notes
+		b.warnings = append(b.warnings, "Warning: skyhook-customizations is enabled with training intent but --workload-selector is not set. This may cause skyhook to evict running training jobs. Consider setting --workload-selector to prevent eviction.")
+	}
+}
+
+// validateAcceleratedSelector validates that accelerated-node-selector is set when skyhook-customizations
+// is present with training or inference intent.
+func (b *DefaultBundler) validateAcceleratedSelector(recipeResult *recipe.RecipeResult) {
+	if b.Config == nil {
+		return
+	}
+
+	// Check if skyhook-customizations component exists
+	hasSkyhookCustomizations := false
+	for _, ref := range recipeResult.ComponentRefs {
+		if ref.Name == "skyhook-customizations" {
+			hasSkyhookCustomizations = true
+			break
+		}
+	}
+
+	if !hasSkyhookCustomizations {
+		return
+	}
+
+	// Check if intent is training or inference
+	if recipeResult.Criteria == nil {
+		return
+	}
+	intent := recipeResult.Criteria.Intent
+	if intent != recipe.CriteriaIntentTraining && intent != recipe.CriteriaIntentInference {
+		return
+	}
+
+	// Check if accelerated-node-selector is not set
+	selector := b.Config.AcceleratedNodeSelector()
+	if len(selector) == 0 {
+		slog.Warn("skyhook-customizations is enabled with training/inference intent but --accelerated-node-selector is not set",
+			"component", "skyhook-customizations",
+			"intent", intent,
+		)
+		// Store warning to be added to deployment notes
+		warningMsg := fmt.Sprintf("Warning: skyhook-customizations is enabled with %s intent but --accelerated-node-selector is not set. Without this selector, the customization will run on all nodes. Consider setting --accelerated-node-selector to target specific nodes.", intent)
+		b.warnings = append(b.warnings, warningMsg)
+	}
 }
 
 // writeRecipeFile serializes the recipe to the bundle directory.