fix(ci): fix Karpenter KWOK provider deployment failures

Three issues prevented the Karpenter Deployment from becoming Ready:

1. Image tag mismatch: ko outputs kind.local/<name>:<content-hash> but
   the script hardcoded --set controller.image.tag=latest. The :latest
   tag doesn't exist in kind's containerd. Fix: extract the actual tag
   from ko output and set imagePullPolicy=Never since the image is
   already side-loaded.

2. staticCapacity feature gate undefined: the deployment template
   references settings.featureGates.staticCapacity but it's missing
   from the chart's values.yaml, rendering as StaticCapacity=<no value>
   which crashes the controller. Fix: set staticCapacity=true.

3. JSON patch on missing arrays: configure_instance_types used JSON
   patch with /-  (append) on volumes/volumeMounts arrays that don't
   exist in the rendered deployment when extraVolumes is empty. Fix:
   pass extraVolumes, controller.extraVolumeMounts, and controller.env
   directly via Helm values, eliminating the patch entirely. The
   ConfigMap is now created before helm install so the volume mount
   is available immediately.

Also added diagnostic output (pod describe, logs) on Helm failure and
increased timeout to 300s to accommodate slower CI environments.

Signed-off-by: Davanum Srinivas <dsrinivas@nvidia.com>

Author: dims

kwok/scripts/install-karpenter-kwok.sh

@@ -19,8 +19,8 @@
 # GPU cluster autoscaling without real cloud infrastructure. It:
 #   1. Installs KWOK controller via Helm
 #   2. Clones Karpenter and builds the KWOK provider image via ko
-#   3. Deploys Karpenter via Helm with the locally-built image
-#   4. Configures GPU instance types (p5.48xlarge, g5.xlarge, etc.)
+#   3. Creates GPU instance types ConfigMap
+#   4. Deploys Karpenter via Helm with the locally-built image and instance types
 #
 # Prerequisites: Go, ko, Helm, kubectl, kind cluster running
 #
@@ -93,97 +93,101 @@ build_karpenter() {
 
     cd "${KARPENTER_CLONE_DIR}"
 
-    # ko build with kind.local side-loads the image directly into the kind cluster
+    # ko build with kind.local side-loads the image directly into the kind cluster.
+    # Output format: kind.local/<name>:<content-hash>
     CONTROLLER_IMG=$(KO_DOCKER_REPO=kind.local \
         KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME}" \
         ko build sigs.k8s.io/karpenter/kwok 2>&1 | tail -1)
 
-    IMG_REPOSITORY=$(echo "${CONTROLLER_IMG}" | cut -d ":" -f 1)
+    # Extract repository and tag from the ko output.
+    # ko outputs "kind.local/<name>:<hash>" — split on the first colon after the repo.
+    if [[ "${CONTROLLER_IMG}" == *":"* ]]; then
+        IMG_REPOSITORY="${CONTROLLER_IMG%%:*}"
+        IMG_TAG="${CONTROLLER_IMG#*:}"
+    else
+        IMG_REPOSITORY="${CONTROLLER_IMG}"
+        IMG_TAG=""
+    fi
 
     log_info "Built image: ${CONTROLLER_IMG}"
     log_info "Repository: ${IMG_REPOSITORY}"
+    log_info "Tag: ${IMG_TAG:-<none>}"
 
     # Export for use in deploy step
-    export CONTROLLER_IMG IMG_REPOSITORY
+    export CONTROLLER_IMG IMG_REPOSITORY IMG_TAG
 }
 
 # -------------------------------------------------------------------
 # Step 3: Deploy Karpenter via Helm
+# Creates the instance types ConfigMap first, then deploys Karpenter
+# with volume mounts and env vars configured via Helm values.
 # -------------------------------------------------------------------
 deploy_karpenter() {
     log_info "Deploying Karpenter to namespace ${KARPENTER_NAMESPACE}..."
 
     # Apply CRDs first
     kubectl apply -f "${KARPENTER_CLONE_DIR}/kwok/charts/crds"
 
-    # Deploy via Helm with the locally-built image
-    helm upgrade --install karpenter "${KARPENTER_CLONE_DIR}/kwok/charts" \
-        --namespace "${KARPENTER_NAMESPACE}" --create-namespace \
-        --set controller.image.repository="${IMG_REPOSITORY}" \
-        --set controller.image.tag=latest \
-        --set logLevel=info \
-        --set controller.resources.requests.cpu=500m \
-        --set controller.resources.requests.memory=512Mi \
-        --set controller.resources.limits.cpu=1 \
-        --set controller.resources.limits.memory=1Gi \
-        --wait --timeout 120s
-
-    log_info "Karpenter deployed"
-}
-
-# -------------------------------------------------------------------
-# Step 4: Configure GPU instance types
-# Patches the Karpenter deployment to mount the instance-types.json
-# ConfigMap, enabling Karpenter to provision GPU-capable KWOK nodes.
-# -------------------------------------------------------------------
-configure_instance_types() {
-    log_info "Configuring GPU instance types..."
+    # Create namespace and instance types ConfigMap before Helm install
+    # so the volume mount can reference it immediately.
+    kubectl create namespace "${KARPENTER_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
 
     local instance_types_file="${MANIFESTS_DIR}/instance-types.json"
     if [[ ! -f "${instance_types_file}" ]]; then
         log_error "Instance types file not found: ${instance_types_file}"
         exit 1
     fi
-
-    # Create ConfigMap from instance types JSON
     kubectl create configmap -n "${KARPENTER_NAMESPACE}" karpenter-instance-types \
         --from-file=instance-types.json="${instance_types_file}" \
         --dry-run=client -o yaml | kubectl apply -f -
 
-    # Patch deployment to mount the ConfigMap and set the env var
-    kubectl -n "${KARPENTER_NAMESPACE}" patch deployment karpenter --type=json -p='[
-        {
-            "op": "add",
-            "path": "/spec/template/spec/volumes/-",
-            "value": {
-                "name": "instance-types",
-                "configMap": {
-                    "name": "karpenter-instance-types"
-                }
-            }
-        },
-        {
-            "op": "add",
-            "path": "/spec/template/spec/containers/0/volumeMounts/-",
-            "value": {
-                "name": "instance-types",
-                "mountPath": "/etc/karpenter/instance-types",
-                "readOnly": true
-            }
-        },
-        {
-            "op": "add",
-            "path": "/spec/template/spec/containers/0/env/-",
-            "value": {
-                "name": "INSTANCE_TYPES_FILE_PATH",
-                "value": "/etc/karpenter/instance-types/instance-types.json"
-            }
-        }
-    ]'
+    # Build the image tag argument. If ko provided a tag, use it.
+    # If not, omit it and let the chart default to its AppVersion.
+    local tag_arg=""
+    if [[ -n "${IMG_TAG}" ]]; then
+        tag_arg="--set controller.image.tag=${IMG_TAG}"
+    fi
 
-    kubectl -n "${KARPENTER_NAMESPACE}" rollout status deployment/karpenter --timeout=60s
+    # Deploy via Helm with the locally-built image.
+    # - imagePullPolicy=Never: image is side-loaded into kind, no registry to pull from
+    # - staticCapacity=true: required by the deployment template but missing from chart defaults
+    # - extraVolumes + extraVolumeMounts: mount the instance types ConfigMap
+    # - controller.env: set INSTANCE_TYPES_FILE_PATH for the KWOK provider
+    # shellcheck disable=SC2086
+    helm upgrade --install karpenter "${KARPENTER_CLONE_DIR}/kwok/charts" \
+        --namespace "${KARPENTER_NAMESPACE}" --create-namespace \
+        --set controller.image.repository="${IMG_REPOSITORY}" \
+        ${tag_arg} \
+        --set imagePullPolicy=Never \
+        --set logLevel=info \
+        --set settings.featureGates.staticCapacity=true \
+        --set controller.resources.requests.cpu=500m \
+        --set controller.resources.requests.memory=512Mi \
+        --set controller.resources.limits.cpu=1 \
+        --set controller.resources.limits.memory=1Gi \
+        --set 'extraVolumes[0].name=instance-types' \
+        --set 'extraVolumes[0].configMap.name=karpenter-instance-types' \
+        --set 'controller.extraVolumeMounts[0].name=instance-types' \
+        --set 'controller.extraVolumeMounts[0].mountPath=/etc/karpenter/instance-types' \
+        --set 'controller.extraVolumeMounts[0].readOnly=true' \
+        --set 'controller.env[0].name=INSTANCE_TYPES_FILE_PATH' \
+        --set 'controller.env[0].value=/etc/karpenter/instance-types/instance-types.json' \
+        --wait --timeout 300s \
+        || {
+            log_error "Helm install failed. Diagnostics:"
+            kubectl -n "${KARPENTER_NAMESPACE}" get pods -o wide 2>/dev/null || true
+            kubectl -n "${KARPENTER_NAMESPACE}" describe deployment karpenter 2>/dev/null || true
+            local POD
+            POD=$(kubectl -n "${KARPENTER_NAMESPACE}" get pods -l app.kubernetes.io/name=karpenter \
+                -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+            if [[ -n "${POD}" ]]; then
+                kubectl -n "${KARPENTER_NAMESPACE}" describe pod "${POD}" 2>/dev/null || true
+                kubectl -n "${KARPENTER_NAMESPACE}" logs "${POD}" --tail=50 2>/dev/null || true
+            fi
+            exit 1
+        }
 
-    log_info "GPU instance types configured"
+    log_info "Karpenter deployed with GPU instance types configured"
 }
 
 # -------------------------------------------------------------------
@@ -198,7 +202,6 @@ main() {
     install_kwok
     build_karpenter
     deploy_karpenter
-    configure_instance_types
 
     log_info "=== Karpenter KWOK provider ready ==="
     log_info "Create a NodePool + KWOKNodeClass to start autoscaling"