Fix transient aperture device error in A3M slurm yaml (#5114)

Author: agrawalkhushi18

examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml

@@ -459,8 +459,24 @@ deployment_groups:
           - name: Configure Slurm to depend upon aperture devices
             hosts: all
             become: true
-            vars: {}
             tasks:
+            - name: Create wait script for aperture devices
+              ansible.builtin.copy:
+                dest: /usr/local/bin/wait-for-aperture-devices.sh
+                owner: root
+                group: root
+                mode: 0o755
+                content: |
+                  #!/bin/bash
+                  # Wait up to 60 seconds for aperture devices to appear.
+                  for i in \$(seq 30); do
+                    # Check if the directory is not empty
+                    if [ -n "\$(ls -A /dev/aperture_devices 2>/dev/null)" ]; then
+                      exit 0
+                    fi
+                    sleep 2
+                  done
+                  exit 1
             - name: Ensure slurmd starts after aperture devices are ready
               ansible.builtin.copy:
                 dest: /etc/systemd/system/slurmd.service.d/aperture.conf
@@ -469,7 +485,7 @@ deployment_groups:
                 mode: 0o644
                 content: |
                   [Service]
-                  ExecCondition=/usr/bin/test -d /dev/aperture_devices/
+                  ExecCondition=/usr/local/bin/wait-for-aperture-devices.sh
               notify: Reload SystemD
             handlers:
             - name: Reload SystemD