Debugging nvidia smi

.github/workflows/cause-nvidia-smi-fail.yml

@@ -0,0 +1,41 @@
+name: Run Nvidia-SMI
+
+on:
+  pull_request:
+    branches:
+      - main
+defaults:
+  run:
+    # The default `sh` shell doesn't have the ERR trap status
+    shell: bash
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+jobs:
+  smi:
+    continue-on-error: true
+    strategy:
+      matrix:
+        runner: ["mike-x86-g2-16-l4-1gpu"]
+        sleep-time: ["1","2","3","4"]
+    runs-on: ${{ matrix.runner }}
+    container:
+      image: "us-central1-docker.pkg.dev/tensorflow-sigs/tensorflow/ml-build:latest"
+    steps:
+      - uses: actions/checkout@v3
+      # Halt for testing
+      - name: Wait For Connection
+        uses: google-ml-infra/actions/ci_connection@output-halt-location
+        id: wait-for-connection
+      - name: Nvidia smi
+        run: |
+            set +e  # revert the default `set -e` 
+            x=1
+            while [ $x -le 15 ]
+            do
+                ls /usr/bin | grep nvidia || true
+                x=$(( $x + 1 ))
+                echo "Sleeping"
+                sleep 5
+            done
+            nvidia-smi