Merge pull request #4409 from RachaelSTamakloe/hotfix-resolve-a3u-a4h-nvidia-version-mismatch

Resolve a3u/a4h slurm nvidia version mismatch error

Author: RachaelSTamakloe

examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml

@@ -24,7 +24,7 @@ vars:
   # Image settings
   base_image:
     project: ubuntu-os-accelerator-images
-    family: ubuntu-accelerator-2204-amd64-with-nvidia-570
+    image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250712
   image_build_machine_type: n2-standard-16
   build_slurm_from_git_ref: 6.10.0
   # Cluster env settings
@@ -147,6 +147,24 @@ deployment_groups:
               ansible.builtin.apt:
                 deb: "{{ cuda_repo_filename }}"
                 state: present
+            # The following 2 tasks work around a temporary issue with Ubuntu
+            # packaging of NVIDIA 570 driver series for kernel 6.8.0-1032
+            # This command ensures that any holds are removed before attempting an upgrade.
+            # We ignore failures in case the packages were not held.
+            - name: Unhold NVIDIA driver packages
+              ansible.builtin.command:
+                cmd: apt-mark unhold linux-modules-nvidia-570-server-open-gcp linux-modules-nvidia-570-server-open-6.8.0-1032-gcp
+              become: true
+              changed_when: false
+              failed_when: false
+            - name: Install latest NVIDIA driver metapackage and kernel module
+              ansible.builtin.apt:
+                name:
+                  - linux-modules-nvidia-570-server-open-gcp
+                  - linux-modules-nvidia-570-server-open-6.8.0-1032-gcp
+                state: latest
+                update_cache: yes
+              become: true
             - name: Reduce NVIDIA repository priority
               ansible.builtin.copy:
                 dest: /etc/apt/preferences.d/cuda-repository-pin-600
@@ -236,7 +254,7 @@ deployment_groups:
     settings:
       disk_size: $(vars.disk_size_gb)
       machine_type: $(vars.image_build_machine_type)
-      source_image_family: $(vars.base_image.family)
+      source_image: $(vars.base_image.image)
       source_image_project_id: [$(vars.base_image.project)]
       image_family: $(vars.instance_image.family)
       omit_external_ip: false

examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml

@@ -24,7 +24,7 @@ vars:
   # Image settings
   base_image:
     project: ubuntu-os-accelerator-images
-    family: ubuntu-accelerator-2204-amd64-with-nvidia-570
+    image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250712
   image_build_machine_type: n2-standard-16
   build_slurm_from_git_ref: 6.10.0
   # Cluster env settings
@@ -148,6 +148,24 @@ deployment_groups:
               ansible.builtin.apt:
                 deb: "{{ cuda_repo_filename }}"
                 state: present
+            # The following 2 tasks work around a temporary issue with Ubuntu
+            # packaging of NVIDIA 570 driver series for kernel 6.8.0-1032
+            # This command ensures that any holds are removed before attempting an upgrade.
+            # We ignore failures in case the packages were not held.
+            - name: Unhold NVIDIA driver packages
+              ansible.builtin.command:
+                cmd: apt-mark unhold linux-modules-nvidia-570-server-open-gcp linux-modules-nvidia-570-server-open-6.8.0-1032-gcp
+              become: true
+              changed_when: false
+              failed_when: false
+            - name: Install latest NVIDIA driver metapackage and kernel module
+              ansible.builtin.apt:
+                name:
+                  - linux-modules-nvidia-570-server-open-gcp
+                  - linux-modules-nvidia-570-server-open-6.8.0-1032-gcp
+                state: latest
+                update_cache: yes
+              become: true
             - name: Reduce NVIDIA repository priority
               ansible.builtin.copy:
                 dest: /etc/apt/preferences.d/cuda-repository-pin-600
@@ -237,7 +255,7 @@ deployment_groups:
     settings:
       disk_size: $(vars.disk_size_gb)
       machine_type: $(vars.image_build_machine_type)
-      source_image_family: $(vars.base_image.family)
+      source_image: $(vars.base_image.image)
       source_image_project_id: [$(vars.base_image.project)]
       image_family: $(vars.instance_image.family)
       omit_external_ip: false