Auto discover accelerator name from deployment nodeSelector/nodeAffinity

shuynh2017 · shuynh2017 · commit 5ead7c522ddc · 2026-03-12T21:15:55.000-04:00
diff --git a/charts/workload-variant-autoscaler/values-dev.yaml b/charts/workload-variant-autoscaler/values-dev.yaml
@@ -61,6 +61,9 @@ llmd:
   modelID: "Qwen/Qwen3-0.6B"
 va:
   enabled: true
+  # accelerator: Optional. If not specified, it will be auto-discovered
+  # from target deployment. If specified, it will be used as fall-back value if it can't 
+  # be discovered.
   accelerator: H100
   sloTpot: 10
   sloTtft: 1000
diff --git a/charts/workload-variant-autoscaler/values.yaml b/charts/workload-variant-autoscaler/values.yaml
@@ -89,6 +89,9 @@ llmd:
 
 va:
   enabled: true
+  # accelerator: Otional. If not specified, it will be auto-discovered
+  # from target deployment. If specified, it will be used as fall-back value if it can't 
+  # be discovered.
   accelerator: H100
   # Cost per replica in arbitrary units (higher = more expensive to scale)
   # Used by saturation analysis to weight scaling decisions across variants
diff --git a/deploy/README.md b/deploy/README.md
@@ -358,6 +358,9 @@ llmd:
 # VariantAutoscaling Configuration
 va:
   enabled: true           # Create VariantAutoscaling CR
+  # accelerator: Optional. If not specified, it will be auto-discovered
+  # from target deployment. If specified, it will be used as fall-back value if it can't 
+  # be discovered.
   accelerator: H100       # GPU type: A100, H100, L40S, etc.
   sloTpot: 10            # Time per output token SLO (ms)
   sloTtft: 1000          # Time to first token SLO (ms)
@@ -475,6 +478,9 @@ llmd:
 # Create VariantAutoscaling
 va:
   enabled: true
+  # accelerator: Optional. If not specified, it will be auto-discovered
+  # from target deployment. If specified, it will be used as fall-back value if it can't 
+  # be discovered.
   accelerator: A100
   sloTpot: 10
   sloTtft: 1000
diff --git a/internal/actuator/actuator_test.go b/internal/actuator/actuator_test.go
@@ -205,7 +205,7 @@ var _ = Describe("Actuator", func() {
 					Name:      contextResourceName,
 					Namespace: namespace,
 					Labels: map[string]string{
-						"inference.optimization/acceleratorName": "A100",
+						ctrlutils.AcceleratorNameLabel: "A100",
 					},
 				},
 				Spec: llmdVariantAutoscalingV1alpha1.VariantAutoscalingSpec{
diff --git a/internal/collector/replica_metrics.go b/internal/collector/replica_metrics.go
@@ -440,13 +440,13 @@ func (c *ReplicaMetricsCollector) CollectReplicaMetrics(
 		}
 		variantKey := utils.GetNamespacedKey(namespace, vaName)
 
-		// Get accelerator name from VariantAutoscaling label
+		// Get accelerator name from Deployment nodeSelector or VariantAutoscaling label
 		acceleratorName := ""
 		if va, ok := variantAutoscalings[variantKey]; ok && va != nil {
-			if va.Labels != nil {
-				if accName, exists := va.Labels[utils.AcceleratorNameLabel]; exists {
-					acceleratorName = accName
-				}
+			// Find the deployment for this VA
+			deploymentKey := utils.GetNamespacedKey(va.Namespace, va.GetScaleTargetName())
+			if deployment, found := deployments[deploymentKey]; found {
+				acceleratorName = utils.GetAcceleratorNameFromDeployment(va, deployment)
 			}
 		}
 
diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go
@@ -7,8 +7,6 @@ import (
 	"time"
 
 	flag "github.com/spf13/pflag"
-
-
 )
 
 // writeTestConfigFile writes a YAML config file to a temp directory and returns its path.
diff --git a/internal/engines/saturation/engine.go b/internal/engines/saturation/engine.go
@@ -1112,9 +1112,9 @@ func (e *Engine) emitSafetyNetMetrics(
 			}
 		}
 		if accelerator == "" {
-			// Try to get from VA labels as last resort
-			if val, ok := va.Labels[utils.AcceleratorNameLabel]; ok && val != "" {
-				accelerator = val
+			// Try to get from deployment nodeSelector/nodeAffinity or VA labels
+			if acceleratorName, err := utils.GetAcceleratorNameFromVA(ctx, e.client, &va); err == nil && acceleratorName != "" {
+				accelerator = acceleratorName
 			}
 		}
 		if accelerator == "" {
diff --git a/internal/engines/saturation/engine_test.go b/internal/engines/saturation/engine_test.go
@@ -141,7 +141,7 @@ data:
 						Name:      name,
 						Namespace: "default",
 						Labels: map[string]string{
-							"inference.optimization/acceleratorName": "A100",
+							utils.AcceleratorNameLabel: "A100",
 						},
 					},
 					Spec: llmdVariantAutoscalingV1alpha1.VariantAutoscalingSpec{
@@ -377,7 +377,7 @@ data:
 						Name:      name,
 						Namespace: "default",
 						Labels: map[string]string{
-							"inference.optimization/acceleratorName": "A100",
+							utils.AcceleratorNameLabel: "A100",
 						},
 					},
 					Spec: llmdVariantAutoscalingV1alpha1.VariantAutoscalingSpec{
diff --git a/internal/engines/saturation/engine_v2.go b/internal/engines/saturation/engine_v2.go
@@ -38,7 +38,7 @@ func (e *Engine) runV2AnalysisOnly(
 				"variant", va.Name, "deployKey", deployKey)
 			continue
 		}
-		accelerator := utils.GetAcceleratorType(va)
+		accelerator := utils.GetAcceleratorNameFromDeployment(va, deploy)
 		gpuCount := getDeploymentGPUsPerReplica(deploy)
 		e.capacityStore.LoadFromDeployment(namespace, modelID, va.Name, accelerator, gpuCount, deploy)
 		logger.V(logging.DEBUG).Info("Pre-populated capacity store from deployment",
@@ -112,9 +112,9 @@ func (e *Engine) runAnalyzersAndScore(
 			continue
 		}
 		switch aw.Name {
+		// future: case "throughput", "slo"
 		case "saturation":
 			totalWeighted += baseResult.RequiredCapacity * aw.Score
-		// future: case "throughput", "slo"
 		}
 	}
 
diff --git a/internal/engines/scalefromzero/engine.go b/internal/engines/scalefromzero/engine.go
@@ -291,7 +291,7 @@ func (e *Engine) processInactiveVariant(ctx context.Context, va wvav1alpha1.Vari
 	accelerator = va.Status.DesiredOptimizedAlloc.Accelerator
 	if accelerator == "" {
 		// Try to get from VA labels as last resort
-		if val, ok := va.Labels["inference.optimization/acceleratorName"]; ok && val != "" {
+		if val, err := utils.GetAcceleratorNameFromVA(ctx, e.client, &va); err == nil {
 			accelerator = val
 		}
 	}
diff --git a/internal/utils/allocation.go b/internal/utils/allocation.go
@@ -26,13 +26,11 @@ func BuildAllocationFromMetrics(
 	// Number of replicas
 	numReplicas := int(*deployment.Spec.Replicas)
 
-	// Accelerator type - strict validation required
-	acc := ""
-	if val, ok := va.Labels["inference.optimization/acceleratorName"]; ok && val != "" {
-		acc = val
-	} else {
+	// Accelerator type - extract from deployment nodeSelector/nodeAffinity or VA labels
+	acc := GetAcceleratorNameFromDeployment(va, &deployment)
+	if acc == "" {
 		return interfaces.Allocation{},
-			fmt.Errorf("missing or empty acceleratorName label on VariantAutoscaling object: %s", va.Name)
+			fmt.Errorf("missing accelerator name from deployment or VA labels: %s", va.Name)
 	}
 
 	// Calculate variant cost
diff --git a/internal/utils/utils.go b/internal/utils/utils.go
@@ -415,3 +415,106 @@ func ValidatePrometheusAPIWithBackoff(ctx context.Context, promAPI promv1.API, b
 func ValidatePrometheusAPI(ctx context.Context, promAPI promv1.API) error {
 	return ValidatePrometheusAPIWithBackoff(ctx, promAPI, PrometheusValidationBackoff)
 }
+
+// GetAcceleratorNameFromDeployment extracts GPU product information from a Deployment's nodeSelector or nodeAffinity.
+// It checks for the following keys in order:
+// - nvidia.com/gpu.product
+// - amd.com/gpu.product-name
+// - cloud.google.com/gke-accelerator
+// If not found in nodeSelector or nodeAffinity, falls back to the AcceleratorNameLabel on the VariantAutoscaling.
+// Returns the first matching value found, or an empty string if none are found.
+func GetAcceleratorNameFromDeployment(va *llmdVariantAutoscalingV1alpha1.VariantAutoscaling, deployment *appsv1.Deployment) string {
+	if deployment == nil && va == nil {
+		return ""
+	}
+
+	gpuKeys := []string{
+		"nvidia.com/gpu.product",
+		"amd.com/gpu.product-name",
+		"cloud.google.com/gke-accelerator",
+	}
+
+	// Check nodeSelector first
+	if deployment != nil && deployment.Spec.Template.Spec.NodeSelector != nil {
+		for _, key := range gpuKeys {
+			if val, ok := deployment.Spec.Template.Spec.NodeSelector[key]; ok {
+				return val
+			}
+		}
+	}
+
+	// Check nodeAffinity
+	if deployment != nil && deployment.Spec.Template.Spec.Affinity != nil && deployment.Spec.Template.Spec.Affinity.NodeAffinity != nil {
+		if val := extractGPUFromNodeAffinity(deployment.Spec.Template.Spec.Affinity.NodeAffinity, gpuKeys); val != "" {
+			return val
+		}
+	}
+
+	// Fall back to VariantAutoscaling label
+	if va != nil && va.Labels != nil {
+		if accName, exists := va.Labels[AcceleratorNameLabel]; exists {
+			return accName
+		}
+	}
+
+	return ""
+}
+
+// GetAcceleratorNameFromVA is a convenience function that fetches the deployment for a VariantAutoscaling
+// and extracts GPU product/accelerator information using GetAcceleratorNameFromDeployment.
+// Returns the accelerator name string and any error from fetching the deployment.
+func GetAcceleratorNameFromVA(ctx context.Context, c client.Client, va *llmdVariantAutoscalingV1alpha1.VariantAutoscaling) (string, error) {
+	if va == nil {
+		return "", fmt.Errorf("VariantAutoscaling cannot be nil")
+	}
+
+	// Fetch the deployment using the VA's scale target name and namespace
+	deployment := &appsv1.Deployment{}
+	if err := GetDeploymentWithBackoff(ctx, c, va.GetScaleTargetName(), va.Namespace, deployment); err != nil {
+		return "", fmt.Errorf("failed to get deployment %s for VA %s/%s: %w", va.GetScaleTargetName(), va.Namespace, va.Name, err)
+	}
+
+	return GetAcceleratorNameFromDeployment(va, deployment), nil
+}
+
+// extractGPUFromNodeAffinity extracts GPU product information from NodeAffinity.
+// It checks both required and preferred node affinity terms for the given GPU keys.
+func extractGPUFromNodeAffinity(nodeAffinity *corev1.NodeAffinity, gpuKeys []string) string {
+	// Check required node affinity
+	if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
+		for _, term := range nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
+			if val := extractGPUFromNodeSelectorTerm(term, gpuKeys); val != "" {
+				return val
+			}
+		}
+	}
+
+	// Check preferred node affinity
+	if nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution != nil {
+		for _, preferred := range nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
+			if val := extractGPUFromNodeSelectorTerm(preferred.Preference, gpuKeys); val != "" {
+				return val
+			}
+		}
+	}
+
+	return ""
+}
+
+// extractGPUFromNodeSelectorTerm extracts GPU product from a NodeSelectorTerm.
+// It checks MatchExpressions for the given GPU keys with "In" or "Exists" operators.
+func extractGPUFromNodeSelectorTerm(term corev1.NodeSelectorTerm, gpuKeys []string) string {
+	for _, expr := range term.MatchExpressions {
+		for _, key := range gpuKeys {
+			if expr.Key == key {
+				// For "In" operator, return the first value
+				if expr.Operator == corev1.NodeSelectorOpIn && len(expr.Values) > 0 {
+					return expr.Values[0]
+				}
+				// For "Exists" operator, we found the key but no specific value
+				// Continue searching for other keys that might have values
+			}
+		}
+	}
+	return ""
+}
diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go
diff --git a/internal/utils/variant.go b/internal/utils/variant.go
diff --git a/internal/utils/variant_test.go b/internal/utils/variant_test.go
diff --git a/test/e2e/fixtures/va_builder.go b/test/e2e/fixtures/va_builder.go
diff --git a/test/e2e/limiter_test.go b/test/e2e/limiter_test.go

Original file line number	Diff line number	Diff line change
`@@ -440,13 +440,13 @@ func (c *ReplicaMetricsCollector) CollectReplicaMetrics(`
`440`	`440`	`}`
`441`	`441`	`variantKey := utils.GetNamespacedKey(namespace, vaName)`
`442`	`442`
`443`		`- // Get accelerator name from VariantAutoscaling label`
	`443`	`+ // Get accelerator name from Deployment nodeSelector or VariantAutoscaling label`
`444`	`444`	`acceleratorName := ""`
`445`	`445`	`if va, ok := variantAutoscalings[variantKey]; ok && va != nil {`
`446`		`- if va.Labels != nil {`
`447`		`- if accName, exists := va.Labels[utils.AcceleratorNameLabel]; exists {`
`448`		`- acceleratorName = accName`
`449`		`- }`
	`446`	`+ // Find the deployment for this VA`
	`447`	`+ deploymentKey := utils.GetNamespacedKey(va.Namespace, va.GetScaleTargetName())`
	`448`	`+ if deployment, found := deployments[deploymentKey]; found {`
	`449`	`+ acceleratorName = utils.GetAcceleratorNameFromDeployment(va, deployment)`
`450`	`450`	`}`
`451`	`451`	`}`
`452`	`452`
Original file line number	Diff line number	Diff line change
`@@ -7,8 +7,6 @@ import (`
`7`	`7`	`"time"`
`8`	`8`
`9`	`9`	`flag "github.com/spf13/pflag"`
`10`		`-`
`11`		`-`
`12`	`10`	`)`
`13`	`11`
`14`	`12`	`// writeTestConfigFile writes a YAML config file to a temp directory and returns its path.`
Original file line number	Diff line number	Diff line change
`@@ -1112,9 +1112,9 @@ func (e *Engine) emitSafetyNetMetrics(`
`1112`	`1112`	`}`
`1113`	`1113`	`}`
`1114`	`1114`	`if accelerator == "" {`
`1115`		`- // Try to get from VA labels as last resort`
`1116`		`- if val, ok := va.Labels[utils.AcceleratorNameLabel]; ok && val != "" {`
`1117`		`- accelerator = val`
	`1115`	`+ // Try to get from deployment nodeSelector/nodeAffinity or VA labels`
	`1116`	`+ if acceleratorName, err := utils.GetAcceleratorNameFromVA(ctx, e.client, &va); err == nil && acceleratorName != "" {`
	`1117`	`+ accelerator = acceleratorName`
`1118`	`1118`	`}`
`1119`	`1119`	`}`
`1120`	`1120`	`if accelerator == "" {`
Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,7 @@ func (e *Engine) runV2AnalysisOnly(`
`38`	`38`	`"variant", va.Name, "deployKey", deployKey)`
`39`	`39`	`continue`
`40`	`40`	`}`
`41`		`- accelerator := utils.GetAcceleratorType(va)`
	`41`	`+ accelerator := utils.GetAcceleratorNameFromDeployment(va, deploy)`
`42`	`42`	`gpuCount := getDeploymentGPUsPerReplica(deploy)`
`43`	`43`	`e.capacityStore.LoadFromDeployment(namespace, modelID, va.Name, accelerator, gpuCount, deploy)`
`44`	`44`	`logger.V(logging.DEBUG).Info("Pre-populated capacity store from deployment",`
`@@ -112,9 +112,9 @@ func (e *Engine) runAnalyzersAndScore(`
`112`	`112`	`continue`
`113`	`113`	`}`
`114`	`114`	`switch aw.Name {`
	`115`	`+ // future: case "throughput", "slo"`
`115`	`116`	`case "saturation":`
`116`	`117`	`totalWeighted += baseResult.RequiredCapacity * aw.Score`
`117`		`- // future: case "throughput", "slo"`
`118`	`118`	`}`
`119`	`119`	`}`
`120`	`120`
Original file line number	Diff line number	Diff line change
`@@ -291,7 +291,7 @@ func (e *Engine) processInactiveVariant(ctx context.Context, va wvav1alpha1.Vari`
`291`	`291`	`accelerator = va.Status.DesiredOptimizedAlloc.Accelerator`
`292`	`292`	`if accelerator == "" {`
`293`	`293`	`// Try to get from VA labels as last resort`
`294`		`- if val, ok := va.Labels["inference.optimization/acceleratorName"]; ok && val != "" {`
	`294`	`+ if val, err := utils.GetAcceleratorNameFromVA(ctx, e.client, &va); err == nil {`
`295`	`295`	`accelerator = val`
`296`	`296`	`}`
`297`	`297`	`}`