Merge pull request #4458 from samskillman/fix/resolve-version-mismatch

Hold all nvidia software to the same version

Author: samskillman

examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml

@@ -84,6 +84,36 @@ deployment_groups:
           Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1
           Pin: version 1.17.7-1
           Pin-Priority: 100
+
+      # The following holds NVIDIA software that was already installed on the
+      # accelerator base image to be the same driver version. This reduces the
+      # risk of a driver version mismatch.
+      # Additional packages are held by:
+      # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml
+      - type: ansible-local
+        destination: hold-nvidia-packages.yml
+        content: |
+          ---
+          - name: Hold nvidia packages
+            hosts: all
+            become: true
+            vars:
+              nvidia_packages_to_hold:
+              - libnvidia-cfg1-*-server
+              - libnvidia-compute-*-server
+              - libnvidia-nscq-*
+              - nvidia-compute-utils-*-server
+              - nvidia-fabricmanager-*
+              - nvidia-utils-*-server
+            tasks:
+            - name: Hold nvidia packages
+              ansible.builtin.command:
+                argv:
+                - apt-mark
+                - hold
+                - "{{ item }}"
+              loop: "{{ nvidia_packages_to_hold }}"
+
       # it is important that kernel upgrades do not occur before running the
       # solution for building Slurm (which doesn't handle them well on the fly)
       # if you follow this rule, any module which supports DKMS will be
@@ -181,7 +211,7 @@ deployment_groups:
         destination: configure_gpu_monitoring.yml
         content: |
           ---
-          - name: Install NVIDIA DCGM and Configure Ops Agent
+          - name: Install CUDA & DCGM & Configure Ops Agent
             hosts: all
             become: true
             vars:
@@ -193,9 +223,6 @@ deployment_groups:
               nvidia_packages:
               - cuda-toolkit-12-8
               - datacenter-gpu-manager-4-cuda12
-              - libnvidia-cfg1-570-server
-              - libnvidia-nscq-570
-              - nvidia-compute-utils-570-server
             tasks:
             - name: Download NVIDIA repository package
               ansible.builtin.get_url:
@@ -223,16 +250,11 @@ deployment_groups:
                   Package: *
                   Pin: release l=NVIDIA CUDA
                   Pin-Priority: 400
-            - name: Install NVIDIA fabric and CUDA
+            - name: Install CUDA & DCGM
               ansible.builtin.apt:
                 name: "{{ item }}"
                 update_cache: true
               loop: "{{ nvidia_packages }}"
-            - name: Freeze NVIDIA fabric and CUDA
-              ansible.builtin.dpkg_selections:
-                name: "{{ item }}"
-                selection: hold
-              loop: "{{ nvidia_packages }}"
             - name: Create nvidia-persistenced override directory
               ansible.builtin.file:
                 path: /etc/systemd/system/nvidia-persistenced.service.d
@@ -271,6 +293,7 @@ deployment_groups:
                 name: nvidia-persistenced.service
                 state: stopped
                 enabled: false
+
       - type: ansible-local
         destination: install_dmabuf.yml
         content: |

examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml

@@ -69,6 +69,36 @@ deployment_groups:
           Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1
           Pin: version 1.17.7-1
           Pin-Priority: 100
+
+      # The following holds NVIDIA software that was already installed on the
+      # accelerator base image to be the same driver version. This reduces the
+      # risk of a driver version mismatch.
+      # Additional packages are held by:
+      # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml
+      - type: ansible-local
+        destination: hold-nvidia-packages.yml
+        content: |
+          ---
+          - name: Hold nvidia packages
+            hosts: all
+            become: true
+            vars:
+              nvidia_packages_to_hold:
+              - libnvidia-cfg1-*-server
+              - libnvidia-compute-*-server
+              - libnvidia-nscq-*
+              - nvidia-compute-utils-*-server
+              - nvidia-fabricmanager-*
+              - nvidia-utils-*-server
+            tasks:
+            - name: Hold nvidia packages
+              ansible.builtin.command:
+                argv:
+                - apt-mark
+                - hold
+                - "{{ item }}"
+              loop: "{{ nvidia_packages_to_hold }}"
+
       - type: data
         destination: /var/tmp/slurm_vars.json
         content: |
@@ -93,7 +123,7 @@ deployment_groups:
               -i localhost, --limit localhost --connection=local \
               -e @/var/tmp/slurm_vars.json \
               ansible/playbook.yml
-            # this duplicates the ulimits configuration of the HPC VM Image
+      # this duplicates the ulimits configuration of the HPC VM Image
       - type: data
         destination: /etc/security/limits.d/99-unlimited.conf
         content: |
@@ -103,11 +133,12 @@ deployment_groups:
           * - nofile 1048576
           * - cpu unlimited
           * - rtprio unlimited
+
       - type: ansible-local
-        destination: configure_gpu.yml
+        destination: install_cuda_dcgm.yml
         content: |
           ---
-          - name: Install NVIDIA packages
+          - name: Install CUDA & DCGM
             hosts: all
             become: true
             vars:
@@ -118,9 +149,6 @@ deployment_groups:
               nvidia_packages:
               - cuda-toolkit-12-8
               - datacenter-gpu-manager-4-cuda12
-              - libnvidia-cfg1-570-server
-              - libnvidia-nscq-570
-              - nvidia-compute-utils-570-server
             tasks:
             - name: Download NVIDIA repository package
               ansible.builtin.get_url:
@@ -130,7 +158,6 @@ deployment_groups:
               ansible.builtin.apt:
                 deb: "{{ cuda_repo_filename }}"
                 state: present
-
             - name: Reduce NVIDIA repository priority
               ansible.builtin.copy:
                 dest: /etc/apt/preferences.d/cuda-repository-pin-600
@@ -149,16 +176,11 @@ deployment_groups:
                   Package: *
                   Pin: release l=NVIDIA CUDA
                   Pin-Priority: 400
-            - name: Install NVIDIA fabric and CUDA
+            - name: Install CUDA & DCGM
               ansible.builtin.apt:
                 name: "{{ item }}"
                 update_cache: true
               loop: "{{ nvidia_packages }}"
-            - name: Freeze NVIDIA fabric and CUDA
-              ansible.builtin.dpkg_selections:
-                name: "{{ item }}"
-                selection: hold
-              loop: "{{ nvidia_packages }}"
             - name: Create nvidia-persistenced override directory
               ansible.builtin.file:
                 path: /etc/systemd/system/nvidia-persistenced.service.d

examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml

@@ -70,6 +70,36 @@ deployment_groups:
           Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1
           Pin: version 1.17.7-1
           Pin-Priority: 100
+
+      # The following holds NVIDIA software that was already installed on the
+      # accelerator base image to be the same driver version. This reduces the
+      # risk of a driver version mismatch.
+      # Additional packages are held by:
+      # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml
+      - type: ansible-local
+        destination: hold-nvidia-packages.yml
+        content: |
+          ---
+          - name: Hold nvidia packages
+            hosts: all
+            become: true
+            vars:
+              nvidia_packages_to_hold:
+              - libnvidia-cfg1-*-server
+              - libnvidia-compute-*-server
+              - libnvidia-nscq-*
+              - nvidia-compute-utils-*-server
+              - nvidia-fabricmanager-*
+              - nvidia-utils-*-server
+            tasks:
+            - name: Hold nvidia packages
+              ansible.builtin.command:
+                argv:
+                - apt-mark
+                - hold
+                - "{{ item }}"
+              loop: "{{ nvidia_packages_to_hold }}"
+
       - type: data
         destination: /var/tmp/slurm_vars.json
         content: |
@@ -104,11 +134,12 @@ deployment_groups:
           * - nofile 1048576
           * - cpu unlimited
           * - rtprio unlimited
+
       - type: ansible-local
-        destination: configure_gpu.yml
+        destination: install_cuda_dcgm.yml
         content: |
           ---
-          - name: Install NVIDIA packages
+          - name: Install CUDA & DCGM
             hosts: all
             become: true
             vars:
@@ -119,9 +150,6 @@ deployment_groups:
               nvidia_packages:
               - cuda-toolkit-12-8
               - datacenter-gpu-manager-4-cuda12
-              - libnvidia-cfg1-570-server
-              - libnvidia-nscq-570
-              - nvidia-compute-utils-570-server
             tasks:
             - name: Download NVIDIA repository package
               ansible.builtin.get_url:
@@ -131,7 +159,6 @@ deployment_groups:
               ansible.builtin.apt:
                 deb: "{{ cuda_repo_filename }}"
                 state: present
-
             - name: Reduce NVIDIA repository priority
               ansible.builtin.copy:
                 dest: /etc/apt/preferences.d/cuda-repository-pin-600
@@ -150,16 +177,11 @@ deployment_groups:
                   Package: *
                   Pin: release l=NVIDIA CUDA
                   Pin-Priority: 400
-            - name: Install NVIDIA fabric and CUDA
+            - name: Install CUDA & DCGM
               ansible.builtin.apt:
                 name: "{{ item }}"
                 update_cache: true
               loop: "{{ nvidia_packages }}"
-            - name: Freeze NVIDIA fabric and CUDA
-              ansible.builtin.dpkg_selections:
-                name: "{{ item }}"
-                selection: hold
-              loop: "{{ nvidia_packages }}"
             - name: Create nvidia-persistenced override directory
               ansible.builtin.file:
                 path: /etc/systemd/system/nvidia-persistenced.service.d
@@ -225,7 +247,6 @@ deployment_groups:
       source_image_project_id: [$(vars.base_image.project)]
       image_family: $(vars.instance_image.family)
       omit_external_ip: false
-
       # Unattended upgrades are disabled in this blueprint so that software does not
       # get updated daily and lead to potential instability in the cluster environment.
       #

examples/machine-learning/build-service-images/a3m/blueprint.yaml

@@ -37,6 +37,36 @@ deployment_groups:
           Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1
           Pin: version 1.17.7-1
           Pin-Priority: 100
+
+      # The following holds NVIDIA software that was already installed on the
+      # accelerator base image to be the same driver version. This reduces the
+      # risk of a driver version mismatch.
+      # Additional packages are held by:
+      # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml
+      - type: ansible-local
+        destination: hold-nvidia-packages.yml
+        content: |
+          ---
+          - name: Hold nvidia packages
+            hosts: all
+            become: true
+            vars:
+              nvidia_packages_to_hold:
+              - libnvidia-cfg1-*-server
+              - libnvidia-compute-*-server
+              - libnvidia-nscq-*
+              - nvidia-compute-utils-*-server
+              - nvidia-fabricmanager-*
+              - nvidia-utils-*-server
+            tasks:
+            - name: Hold nvidia packages
+              ansible.builtin.command:
+                argv:
+                - apt-mark
+                - hold
+                - "{{ item }}"
+              loop: "{{ nvidia_packages_to_hold }}"
+
       - type: shell
         destination: hold_google_services.sh
         content: |
@@ -118,7 +148,7 @@ deployment_groups:
         destination: configure_gpu_monitoring.yml
         content: |
           ---
-          - name: Install NVIDIA DCGM and Configure Ops Agent
+          - name: Install CUDA & DCGM & Configure Ops Agent
             hosts: all
             become: true
             vars:
@@ -130,9 +160,6 @@ deployment_groups:
               nvidia_packages:
               - cuda-toolkit-12-8
               - datacenter-gpu-manager-4-cuda12
-              - libnvidia-cfg1-570-server
-              - libnvidia-nscq-570
-              - nvidia-compute-utils-570-server
             tasks:
             - name: Download NVIDIA repository package
               ansible.builtin.get_url:
@@ -160,16 +187,11 @@ deployment_groups:
                   Package: *
                   Pin: release l=NVIDIA CUDA
                   Pin-Priority: 400
-            - name: Install NVIDIA fabric and CUDA
+            - name: Install CUDA & DCGM
               ansible.builtin.apt:
                 name: "{{ item }}"
                 update_cache: true
               loop: "{{ nvidia_packages }}"
-            - name: Freeze NVIDIA fabric and CUDA
-              ansible.builtin.dpkg_selections:
-                name: "{{ item }}"
-                selection: hold
-              loop: "{{ nvidia_packages }}"
             - name: Create nvidia-persistenced override directory
               ansible.builtin.file:
                 path: /etc/systemd/system/nvidia-persistenced.service.d
@@ -208,6 +230,7 @@ deployment_groups:
                 name: nvidia-persistenced.service
                 state: stopped
                 enabled: false
+
       - type: ansible-local
         destination: install_dmabuf.yml
         content: |
@@ -300,7 +323,7 @@ deployment_groups:
       disk_size: 100
       machine_type: c2-standard-8
 
-      source_image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250712
+      source_image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250722
       source_image_project_id: [ubuntu-os-accelerator-images]
 
       image_family: $(vars.family)