fix(llm): honor SLLM device override when building container spec (#24799)

Author: zexi

pkg/llm/drivers/llm_container/comfyui.go

@@ -88,8 +88,9 @@ func (c *comfyui) GetContainerSpec(ctx context.Context, llm *models.SLLM, image
 		},
 	}
 
-	if len(devices) == 0 && (sku.Devices != nil && len(*sku.Devices) > 0) {
-		for i := range *sku.Devices {
+	effDevs := models.GetEffectiveDevices(llm, sku)
+	if len(devices) == 0 && effDevs != nil && len(*effDevs) > 0 {
+		for i := range *effDevs {
 			index := i
 			spec.Devices = append(spec.Devices, &computeapi.ContainerDevice{
 				Type: commonapi.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,

pkg/llm/drivers/llm_container/ollama.go

@@ -81,8 +81,9 @@ func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *
 		},
 	}
 
-	if len(devices) == 0 && (sku.Devices != nil && len(*sku.Devices) > 0) {
-		for i := range *sku.Devices {
+	effDevs := models.GetEffectiveDevices(llm, sku)
+	if len(devices) == 0 && effDevs != nil && len(*effDevs) > 0 {
+		for i := range *effDevs {
 			index := i
 			spec.Devices = append(spec.Devices, &computeapi.ContainerDevice{
 				Type: commonapi.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,

pkg/llm/drivers/llm_container/vllm.go

@@ -372,8 +372,9 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo
 	}
 
 	// GPU Devices
-	if len(devices) == 0 && (sku.Devices != nil && len(*sku.Devices) > 0) {
-		for i := range *sku.Devices {
+	effDevs := models.GetEffectiveDevices(llm, sku)
+	if len(devices) == 0 && effDevs != nil && len(*effDevs) > 0 {
+		for i := range *effDevs {
 			index := i
 			spec.Devices = append(spec.Devices, &computeapi.ContainerDevice{
 				Type: commonapi.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,
@@ -498,8 +499,8 @@ func (v *vllm) StartLLM(ctx context.Context, userCred mcclient.TokenCredential,
 		return errors.Wrap(err, "get llm sku")
 	}
 	tensorParallelSize := 1
-	if sku.Devices != nil && len(*sku.Devices) > 0 {
-		tensorParallelSize = len(*sku.Devices)
+	if effDevs := models.GetEffectiveDevices(llm, sku); effDevs != nil && len(*effDevs) > 0 {
+		tensorParallelSize = len(*effDevs)
 	}
 	swapSpaceGiB := (sku.Memory * 1) / (2 * 1024)
 	if swapSpaceGiB < 1 {

pkg/llm/models/llm.go

@@ -80,8 +80,8 @@ func (llm *SLLM) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCre
 	return nil
 }
 
-// getEffectiveMountedModels returns the effective mounted model ids: llm's override takes priority over sku's when non-empty.
-func getEffectiveMountedModels(llm *SLLM, sku *SLLMSku) []string {
+// GetEffectiveMountedModels returns the effective mounted model ids: llm's override takes priority over sku's when non-empty.
+func GetEffectiveMountedModels(llm *SLLM, sku *SLLMSku) []string {
 	if llm != nil && len(llm.MountedModels) > 0 {
 		return llm.MountedModels
 	}

pkg/llm/models/llm_base.go

@@ -72,6 +72,21 @@ type SLLMBase struct {
 	Devices   *api.Devices   `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"`
 }
 
+// GetEffectiveDevices returns the devices to apply: llm's override takes priority over sku when non-empty.
+// Drivers should call this when materializing container device specs so that SLLM overrides on the
+// instance are honored (not just on server-level IsolatedDevices allocation).
+func GetEffectiveDevices(llm *SLLM, sku *SLLMSku) *api.Devices {
+	var llmBase *SLLMBase
+	var skuBase *SLLMSkuBase
+	if llm != nil {
+		llmBase = &llm.SLLMBase
+	}
+	if sku != nil {
+		skuBase = &sku.SLLMSkuBase
+	}
+	return getEffectiveDevices(llmBase, skuBase)
+}
+
 func getEffectiveDevices(llmBase *SLLMBase, skuBase *SLLMSkuBase) *api.Devices {
 	if llmBase != nil && llmBase.Devices != nil && !llmBase.Devices.IsZero() {
 		return llmBase.Devices
@@ -82,6 +97,19 @@ func getEffectiveDevices(llmBase *SLLMBase, skuBase *SLLMSkuBase) *api.Devices {
 	return nil
 }
 
+// GetEffectiveHostPaths returns the host_paths to apply with llm's override taking priority over sku.
+func GetEffectiveHostPaths(llm *SLLM, sku *SLLMSku) *api.HostPaths {
+	var llmBase *SLLMBase
+	var skuBase *SLLMSkuBase
+	if llm != nil {
+		llmBase = &llm.SLLMBase
+	}
+	if sku != nil {
+		skuBase = &sku.SLLMSkuBase
+	}
+	return getEffectiveHostPaths(llmBase, skuBase)
+}
+
 func getEffectiveHostPaths(llmBase *SLLMBase, skuBase *SLLMSkuBase) *api.HostPaths {
 	if llmBase != nil && llmBase.HostPaths != nil && !llmBase.HostPaths.IsZero() {
 		return llmBase.HostPaths

pkg/llm/models/llm_instant_model_sync.go

@@ -691,7 +691,7 @@ func (llm *SLLM) UpdateMountedModelFullNames(ctx context.Context, userCred mccli
 				return errors.Wrap(err, "getDeletedModelIds")
 			}
 		}
-		mountedModels := getEffectiveMountedModels(llm, sku)
+		mountedModels := GetEffectiveMountedModels(llm, sku)
 		for i := range mountedModels {
 			instMdl, err := GetInstantModelManager().FetchByIdOrName(ctx, userCred, mountedModels[i])
 			if err != nil {