Merge pull request #3725 from samskillman/a4-nccl-tests

A4 nccl tests

Author: samskillman

examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml

@@ -41,6 +41,7 @@ vars:
     family: $(vars.deployment_name)-u22
   disk_size_gb: 200
   nccl_plugin_version: v1.0.4
+  benchmark_dir: $(ghpc_stage("system_benchmarks"))
 
 deployment_groups:
 - group: image-env
@@ -54,6 +55,7 @@ deployment_groups:
       install_ansible: true
       docker:
         enabled: true
+        world_writable: true
       runners:
       - type: shell
         destination: prep-for-slurm-build.sh
@@ -108,10 +110,12 @@ deployment_groups:
               nvidia_packages:
               - nvidia-open-570
               - nvidia-utils-570
-              - cuda-toolkit-12-8
               - nvidia-container-toolkit
+              - cuda-toolkit-12-8
               - datacenter-gpu-manager
+              - libnvidia-cfg1-570-server
               - libnvidia-nscq-570
+              - nvidia-compute-utils-570-server
             tasks:
             - name: Download NVIDIA repository package
               ansible.builtin.get_url:
@@ -131,12 +135,40 @@ deployment_groups:
                 name: "{{ item }}"
                 selection: hold
               loop: "{{ nvidia_packages }}"
+            - name: Create nvidia-persistenced override directory
+              ansible.builtin.file:
+                path: /etc/systemd/system/nvidia-persistenced.service.d
+                state: directory
+                owner: root
+                group: root
+                mode: 0o755
+            - name: Configure nvidia-persistenced override
+              ansible.builtin.copy:
+                dest: /etc/systemd/system/nvidia-persistenced.service.d/persistence_mode.conf
+                owner: root
+                group: root
+                mode: 0o644
+                content: |
+                  [Service]
+                  ExecStart=
+                  ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose
+              notify: Reload SystemD
+            handlers:
+            - name: Reload SystemD
+              ansible.builtin.systemd:
+                daemon_reload: true
             post_tasks:
             - name: Disable NVIDIA DCGM by default (enable during boot on GPU nodes)
               ansible.builtin.service:
                 name: nvidia-dcgm.service
                 state: stopped
                 enabled: false
+            - name: Disable nvidia-persistenced SystemD unit (enable during boot on GPU nodes)
+              ansible.builtin.service:
+                name: nvidia-persistenced.service
+                state: stopped
+                enabled: false
+
       - type: ansible-local
         destination: install_ibverbs_utils.yml
         content: |
@@ -150,6 +182,11 @@ deployment_groups:
                 name:
                 - ibverbs-utils
                 state: present
+      - type: data
+        destination: /etc/enroot/enroot.conf
+        content: |
+          ENROOT_CONFIG_PATH     ${HOME}/.enroot
+
 
 - group: image
   modules:
@@ -314,6 +351,7 @@ deployment_groups:
             vars:
               enable_ops_agent: true
               enable_nvidia_dcgm: true
+              enable_nvidia_persistenced: true
             tasks:
             - name: Update Ops Agent configuration
               ansible.builtin.blockinfile:
@@ -348,6 +386,11 @@ deployment_groups:
                 name: nvidia-dcgm.service
                 state: "{{ 'started' if enable_nvidia_dcgm else 'stopped' }}"
                 enabled: "{{ enable_nvidia_dcgm }}"
+            - name: Enable NVIDIA Persistence Daemon
+              ansible.builtin.service:
+                name: nvidia-persistenced.service
+                state: "{{ 'started' if enable_nvidia_persistenced else 'stopped' }}"
+                enabled: "{{ enable_nvidia_persistenced }}"
 
   - id: a4high_nodeset
     source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
@@ -415,6 +458,12 @@ deployment_groups:
           mkdir -m 0755 -p "${SLURM_ROOT}/scripts"
           mkdir -p "${SLURM_ROOT}/partition-${PARTITION_NAME}-epilog_slurmd.d"
           ln -s "/slurm/scripts/tools/gpu-test" "${SLURM_ROOT}/partition-${PARTITION_NAME}-epilog_slurmd.d/gpu-test.epilog_slurmd"
+      - type: data
+        destination: /opt/apps/system_benchmarks/run-nccl-tests-via-ramble.sh
+        source: $(vars.benchmark_dir)/run-nccl-tests-via-ramble.sh
+      - type: data
+        destination: /opt/apps/system_benchmarks/README.md
+        source: $(vars.benchmark_dir)/README.md
 
   - id: slurm_controller
     source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller

examples/machine-learning/a4-highgpu-8g/system_benchmarks/README.md

@@ -0,0 +1,58 @@
+Running System Benchmarks with Ramble
+=====================================
+
+[Ramble](https://github.com/GoogleCloudPlatform/ramble) is an open source
+multi-platform experimentation framework written in python. It can be used
+to easily reproduce benchmark results across systems, and here
+we will use it to run a series of system benchmarks.
+
+Currently the following benchmarks are supported:
+
+* NCCL tests (all-gather, all-reduce, reduce-scatter)
+
+The run scripts have all been staged into `/opt/apps/system_benchmarks`
+on the controller node (and available to all nodes). We recommend running
+them using `nohup` and redirecting the stdout/err to a logfile, as the tests can
+take 30-60 minutes or longer if other jobs are in the queue. The results can
+then be viewed by `tail`ing the log file.
+
+For NCCL tests, run:
+
+   ```bash
+   nohup bash /opt/apps/system_benchmarks/run-nccl-tests-via-ramble.sh >& nccl-$(date -Iseconds).log &
+   tail -f nccl-*.log
+   ```
+
+For each benchmark, multiple node scales will be submitted, up to your maximum
+node scale of your cluster.
+
+Viewing the Results
+-------------------
+
+For nccl, at the end of the nccl-$(date -Iseconds).log,
+you should see something like:
+
+   ```bash
+   ...
+   ---- SUMMARY for >1GB Message Sizes ----
+   workload        n_nodes msg_size        busbw
+   all-gather      2       1073741824      XXX.XX
+   all-gather      2       2147483648      XXX.XX
+   all-gather      2       4294967296      XXX.XX
+   all-gather      2       8589934592      XXX.XX
+   ...
+   all-reduce      2       1073741824      XXX.XX
+   ...
+   reduce-scatter  2       1073741824      XXX.XX
+   ...
+
+   -------- Benchmarking Complete -------
+   ```
+
+Cleaning Up
+-----------
+
+The ramble workspaces will be located in directories called `nccl-tests-*`.
+The ramble codebase was installed to
+`/opt/apps/ramble`.  Removing all of these directories will remove all of the
+files generated during these tests.