i#7281: Move failure notification config to reusable workflow (#7282)

Refactors the Github Action configs to move logic to send notification email on master merge failure into a reusable workflow which is now used by all workflows instead of duplicating the logic 24 times. Also, each workflow file now has a separate job for the new reusable workflow that is invoked after all other jobs (that actually run the test suite) are done. Status of all the test suite jobs are sent as part of the single failure notification email, if there's at least one failure; we skip filtering of status message by test suite status mostly for simplicity of logic that must be duplicated in each workflow file. Note that the reusable workflow cannot be invoked as part of some step of a job; it has to be a separate job on its own. The failure notification job requires a separate worker. Note that since it is invoked only when all other jobs in the same workflow are done, this doesn't make our concurrent job quota usage for a single master merge event any worse. However, as we're still adding a new job, it indeed uses some quota that may have been used by other concurrent triggers (PRs or master merges); but note that since it is gated by the failure condition and is very short anyway, this is not a concern. Test run: Email notification: https://groups.google.com/g/dynamorio-devs/c/EWkaTPBrjww Notification job in failing workflow: https://github.com/DynamoRIO/dynamorio/actions/runs/13351946734/job/37289912108?pr=7282 (other notification jobs in non-failing workflows were skipped) Issue: #7281
DynamoRIO · Feb 19, 2025 · df62cf4 · df62cf4
1 parent 18a77d5
commit df62cf4
Show file tree

Hide file tree

Showing 10 changed files with 241 additions and 560 deletions.
diff --git a/.github/workflows/ci-aarchxx-cross.yml b/.github/workflows/ci-aarchxx-cross.yml
@@ -29,6 +29,10 @@
 # DAMAGE.
 
 # Github Actions workflow for AArchXX Continuous Integration testing.
+#
+# Each test suite is a separate job. When adding a new test suite job,
+# remember to add it to the send-failure-notification job to ensure failure
+# notifications are set up properly.
 
 name: ci-aarchxx-cross
 on:
@@ -98,30 +102,6 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/aarchxx-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # ARM cross-compile with gcc, with some tests run under QEMU.
   # We use a more recent Ubuntu for a more recent QEMU.
   arm-cross-compile:
@@ -171,30 +151,6 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/aarchxx-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # Android ARM cross-compile with gcc, no tests:
   android-arm-cross-compile:
     runs-on: ubuntu-20.04
@@ -243,30 +199,6 @@ jobs:
         CI_BRANCH: ${{ github.ref }}
       run: ./suite/runsuite_wrapper.pl automated_ci 32_only
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/android-arm-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # Android AArch64 cross-compile with LLVM, no tests:
   android-aarch64-cross-compile:
     runs-on: ubuntu-22.04
@@ -309,30 +241,6 @@ jobs:
         CI_BRANCH: ${{ github.ref }}
       run: ./suite/runsuite_wrapper.pl automated_ci 64_only
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/android-aarch64-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # AArch64 drdecode and drmemtrace on x86:
   a64-on-x86:
     runs-on: ubuntu-20.04
@@ -369,26 +277,19 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [aarch64-cross-compile, arm-cross-compile, android-arm-cross-compile, android-aarch64-cross-compile, a64-on-x86]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/a64-on-x86
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
+        test_suite_status: ${{ format('{0} {1} | {2} {3} | {4} {5} | {6} {7} | {8} {9}',
+                                      'aarch64-cross-compile', needs.aarch64-cross-compile.result,
+                                      'arm-cross-compile', needs.arm-cross-compile.result,
+                                      'android-arm-cross-compile', needs.android-arm-cross-compile.result,
+                                      'android-aarch64-cross-compile', needs.android-aarch64-cross-compile.result,
+                                      'a64-on-x86', needs.a64-on-x86.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit
diff --git a/.github/workflows/ci-aarchxx.yml b/.github/workflows/ci-aarchxx.yml
@@ -30,6 +30,10 @@
 # DAMAGE.
 
 # Github Actions workflow for aarch64 Continuous Integration testing.
+#
+# Each test suite is a separate job. When adding a new test suite job,
+# remember to add it to the send-failure-notification job to ensure failure
+# notifications are set up properly.
 
 name: ci-aarchxx
 on:
@@ -128,26 +132,15 @@ jobs:
           CI_BRANCH: ${{ github.ref }}
         if: ${{ matrix.sve == true && matrix.sve_length != ''}}
 
-      - name: Send failure mail to dynamorio-devs
-        if: failure() && github.ref == 'refs/heads/master'
-        uses: dawidd6/action-send-mail@v2
-        with:
-          server_address: smtp.gmail.com
-          server_port: 465
-          username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-          password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-          subject: |
-            [${{github.repository}}] ${{github.workflow}} FAILED
-            on ${{github.event_name}} at ${{github.ref}}
-          body: |
-            Github Actions CI workflow run FAILED!
-            Workflow: ${{github.workflow}}/x86-32
-            Repository: ${{github.repository}}
-            Branch ref: ${{github.ref}}
-            SHA: ${{github.sha}}
-            Triggering actor: ${{github.actor}}
-            Triggering event: ${{github.event_name}}
-            Run Id: ${{github.run_id}}
-            See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-          to: [email protected]
-          from: Github Action CI
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [aarch64-precommit]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
+      with:
+        test_suite_status: ${{ format('{0} {1}',
+                                      'aarch64-precommit', needs.aarch64-precommit.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit
diff --git a/.github/workflows/ci-clang-format.yml b/.github/workflows/ci-clang-format.yml
@@ -29,6 +29,9 @@
 # DAMAGE.
 
 # Github Actions workflow for clang-format Continuous Integration testing.
+#
+# When adding a new job, remember to add it to the send-failure-notification
+# job to ensure failure notifications are set up properly.
 
 name: ci-clang-format
 on:
@@ -87,26 +90,16 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [clang-format]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/clang-format
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
+        test_suite_status: ${{ format('{0} {1}',
+                                      'clang-format', needs.clang-format.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit
diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml
@@ -29,6 +29,9 @@
 # DAMAGE.
 
 # Github Actions workflow for building doxygen docs for the web site.
+#
+# When adding a new job, remember to add it to the send-failure-notification
+# job to ensure failure notifications are set up properly.
 
 name: ci-docs
 on:
@@ -130,26 +133,15 @@ jobs:
         # We need a personal access token for write access to another repo.
         GITHUB_TOKEN: ${{ secrets.DOCS_TOKEN }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [docs]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/docs
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action package jobs
+        test_suite_status: ${{ format('{0} {1}',
+                                      'docs', needs.docs.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit
diff --git a/.github/workflows/ci-osx.yml b/.github/workflows/ci-osx.yml
@@ -29,6 +29,10 @@
 # DAMAGE.
 
 # Github Actions workflow for Mac OSX Continuous Integration testing.
+#
+# Each test suite is a separate job. When adding a new test suite job,
+# remember to add it to the send-failure-notification job to ensure failure
+# notifications are set up properly.
 
 name: ci-osx
 on:
@@ -86,26 +90,15 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [osx-x86-64]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/osx-x86-64
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
+        test_suite_status: ${{ format('{0} {1}',
+                                      'osx-x86-64', needs.osx-x86-64.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit