CI: show Docker diagnostics on failure

In order to help confirm/diagnose issues such as OOM
kills.

Author: stsnel

.github/workflows/api-and-integration-tests.yml

@@ -71,6 +71,29 @@ jobs:
         until $(curl -k --output /dev/null --silent --head --fail https://eus.yoda:8444 ); do printf '.' ; sleep 1; done
         docker exec provider.yoda sh -c 'while ! pgrep irodsServer > /dev/null  ; do echo Waiting for iRODS to start ...  ; sleep 1; done'
 
+    # To diagnose failures
+    - name: Docker diagnostics
+      if: always()
+      run: |
+        cd yoda/docker/compose/
+        echo "=== docker ps ==="
+        docker ps -a
+
+        echo "=== compose logs ==="
+        docker compose logs --no-color || true
+
+        echo "=== memory ==="
+        free -h || true
+
+        echo "=== disk ==="
+        df -h || true
+
+        echo "=== inspect ==="
+        for c in $(docker ps -aq); do
+          docker inspect $c \
+            --format='{{.Name}} Exit={{.State.ExitCode}} OOM={{.State.OOMKilled}}'
+        done
+
     - name: Pull and install latest version of ruleset
       shell: bash
       run: |
@@ -148,6 +171,29 @@ jobs:
         docker exec provider.yoda sh -c 'cd /tmp; sudo -u irods python3 -m coverage combine -q /tmp/coverage.dat.*'
         docker exec provider.yoda sh -c 'cd /tmp; sudo -u irods python3 -m coverage report'
 
+    # This is useful for diagnosing/confirming OOM issues
+    - name: Docker diagnostics
+      if: failure()
+      run: |
+        cd yoda/docker/compose/
+        echo "=== docker ps ==="
+        docker ps -a
+
+        echo "=== compose logs ==="
+        docker compose logs --no-color || true
+
+        echo "=== memory ==="
+        free -h || true
+
+        echo "=== disk ==="
+        df -h || true
+
+        echo "=== inspect ==="
+        for c in $(docker ps -aq); do
+          docker inspect $c \
+            --format='{{.Name}} Exit={{.State.ExitCode}} OOM={{.State.OOMKilled}}'
+        done
+
     - name: Output rodsLogs
       if: failure()
       run: |
@@ -163,6 +209,7 @@ jobs:
       run: |
         docker exec eus.yoda sh -c 'set -x ; for log in error.log access.log ; do echo "${log}:" ; cat "/var/log/apache2/$log" ; echo; done'
 
+
 # Uncomment section below when needed for debugging.
 #
 #    - name: Setup tmate session for debugging