Upgrade xpk from v0.10.1 to v0.13.0 (#1742)

- [x] Upgrade `xpk` from `v0.10.1` to `v0.13.0`

Author: aybchan

.github/actions/gke-xpk/action.yml

@@ -70,7 +70,7 @@ inputs:
   XPK_VERSION:
     description: 'XPK release tag'
     required: false
-    default: 'v0.10.1'
+    default: 'v0.13.0'
     type: string
   XPK_PYTHON:
     description: 'Python version for XPK'
@@ -118,7 +118,6 @@ runs:
   - name: Apply XPK workload create patch
     shell: bash -x -e -u {0}
     run: |
-      sed -i 's/{{ IMAGE_PULL_SECRET_NAME }}/${{ inputs.IMAGE_PULL_SECRET_NAME }}/g' .github/gke-workflow/xpk/${{ inputs.XPK_VERSION}}/workload.patch 
       PATCH_PATH=.github/gke-workflow/xpk/${{ inputs.XPK_VERSION}}
       ls ${PATCH_PATH}/*.patch | xargs -I {} git apply --unsafe-paths {} --directory ${WORKLOAD_NAME}/xpk
   
@@ -171,7 +170,14 @@ runs:
           --scheduler=gke.io/topology-aware-auto
       )
 
-      if [[ "${{ inputs.XPK_VERSION }}" == "v0.10.1" ]]; then
+      version_greater() {
+        if [[ $(echo -e "$1\n$2" | sort -V | head -n 1) != "$1" ]]; then
+          return 0
+        fi
+        return 1
+      }
+
+      if version_greater "${{ inputs.XPK_VERSION }}" "v0.10.0"; then
           args+=(
               --docker-image-pull-secret=${{ inputs.IMAGE_PULL_SECRET_NAME }}
               --env="JAX_COORDINATOR_PORT=3389"

.github/gke-workflow/xpk/v0.13.0/blueprint.patch

@@ -0,0 +1,19 @@
+diff --git a/src/xpk/core/blueprint/blueprint_generator.py b/src/xpk/core/blueprint/blueprint_generator.py
+index 3e432f7..c20e8e4 100644
+--- a/src/xpk/core/blueprint/blueprint_generator.py
++++ b/src/xpk/core/blueprint/blueprint_generator.py
+@@ -201,9 +201,13 @@ class BlueprintGenerator:
+                 "type": "nvidia-h100-mega-80gb",
+                 "count": 8,
+                 "gpu_driver_installation_config": {
+-                    "gpu_driver_version": "LATEST"
++                    # avoid using LATEST due to GPU driver forward compatibility support
++                    # not available when using a feature branch driver release
++                    "gpu_driver_version": "INSTALLATION_DISABLED"
+                 },
+             }],
++            # use private nodes in pool due to NVIDIA organization constraints
++            "enable_private_nodes": True,
+             "auto_upgrade": (
+                 True if capacity_type != CapacityType.FLEX_START else False
+             ),

.github/gke-workflow/xpk/v0.13.0/tcpxo_decorator.patch

@@ -0,0 +1,15 @@
+diff --git a/src/xpk/core/workload_decorators/tcpxo_decorator.py b/src/xpk/core/workload_decorators/tcpxo_decorator.py
+index 3734f87..dc3b24a 100644
+--- a/src/xpk/core/workload_decorators/tcpxo_decorator.py
++++ b/src/xpk/core/workload_decorators/tcpxo_decorator.py
+@@ -181,7 +181,9 @@ def update_gpu_containers(job_manifest):
+     if 'nvidia.com/gpu' in container.get('resources', {}).get('limits', {}):
+       container.setdefault('env', [])
+       container['env'].append(
+-          {'name': 'LD_LIBRARY_PATH', 'value': '/usr/local/nvidia/lib64'}
++          # set env. var. in user jobset container
++          # without prepend, host NCCL rather than container NCCL is used e.g. in NCCL tests
++          {'name': 'LD_LIBRARY_PATH', 'value': '/opt/nvidia/nccl/lib:/usr/local/nvidia/lib64'}
+       )
+       container['env'].append({
+           'name': 'NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY',