Merge pull request #3729 from samskillman/fix/a4-nvidia-utils

nvidia compute server variants not yet available for 570

Author: harshthakkar01

examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml

@@ -113,9 +113,7 @@ deployment_groups:
               - nvidia-container-toolkit
               - cuda-toolkit-12-8
               - datacenter-gpu-manager
-              - libnvidia-cfg1-570-server
               - libnvidia-nscq-570
-              - nvidia-compute-utils-570-server
             tasks:
             - name: Download NVIDIA repository package
               ansible.builtin.get_url:
@@ -135,39 +133,12 @@ deployment_groups:
                 name: "{{ item }}"
                 selection: hold
               loop: "{{ nvidia_packages }}"
-            - name: Create nvidia-persistenced override directory
-              ansible.builtin.file:
-                path: /etc/systemd/system/nvidia-persistenced.service.d
-                state: directory
-                owner: root
-                group: root
-                mode: 0o755
-            - name: Configure nvidia-persistenced override
-              ansible.builtin.copy:
-                dest: /etc/systemd/system/nvidia-persistenced.service.d/persistence_mode.conf
-                owner: root
-                group: root
-                mode: 0o644
-                content: |
-                  [Service]
-                  ExecStart=
-                  ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose
-              notify: Reload SystemD
-            handlers:
-            - name: Reload SystemD
-              ansible.builtin.systemd:
-                daemon_reload: true
             post_tasks:
             - name: Disable NVIDIA DCGM by default (enable during boot on GPU nodes)
               ansible.builtin.service:
                 name: nvidia-dcgm.service
                 state: stopped
                 enabled: false
-            - name: Disable nvidia-persistenced SystemD unit (enable during boot on GPU nodes)
-              ansible.builtin.service:
-                name: nvidia-persistenced.service
-                state: stopped
-                enabled: false
 
       - type: ansible-local
         destination: install_ibverbs_utils.yml
@@ -351,7 +322,6 @@ deployment_groups:
             vars:
               enable_ops_agent: true
               enable_nvidia_dcgm: true
-              enable_nvidia_persistenced: true
             tasks:
             - name: Update Ops Agent configuration
               ansible.builtin.blockinfile:
@@ -386,11 +356,6 @@ deployment_groups:
                 name: nvidia-dcgm.service
                 state: "{{ 'started' if enable_nvidia_dcgm else 'stopped' }}"
                 enabled: "{{ enable_nvidia_dcgm }}"
-            - name: Enable NVIDIA Persistence Daemon
-              ansible.builtin.service:
-                name: nvidia-persistenced.service
-                state: "{{ 'started' if enable_nvidia_persistenced else 'stopped' }}"
-                enabled: "{{ enable_nvidia_persistenced }}"
 
   - id: a4high_nodeset
     source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
@@ -433,7 +398,7 @@ deployment_groups:
       is_default: true
       partition_conf:
         OverSubscribe: EXCLUSIVE
-        ResumeTimeout: 900
+        ResumeTimeout: 1200
         SuspendTimeout: 600
 
   - id: slurm_login