i#7281: Move failure notification config to reusable workflow (#7282)

Refactors the Github Action configs to move logic to send notification
email on master merge failure into a reusable workflow which is now used
by all workflows instead of duplicating the logic 24 times.

Also, each workflow file now has a separate job for the new reusable
workflow that is invoked after all other jobs (that actually run the
test suite) are done. Status of all the test suite jobs are sent as part
of the single failure notification email, if there's at least one failure;
we skip filtering of status message by test suite status mostly for
simplicity of logic that must be duplicated in each workflow file.

Note that the reusable workflow cannot be invoked as part of some step
of a job; it has to be a separate job on its own.

The failure notification job requires a separate worker. Note that since
it is invoked only when all other jobs in the same workflow are done,
this doesn't make our concurrent job quota usage for a single master
merge event any worse. However, as we're still adding a new job, it
indeed uses some quota that may have been used by other concurrent
triggers (PRs or master merges); but note that since it is gated by the
failure condition and is very short anyway, this is not a concern.

Test run:

Email notification:
https://groups.google.com/g/dynamorio-devs/c/EWkaTPBrjww
Notification job in failing workflow:
https://github.com/DynamoRIO/dynamorio/actions/runs/13351946734/job/37289912108?pr=7282
(other notification jobs in non-failing workflows were skipped)

Issue: #7281

Author: abhinav92003

.github/workflows/ci-aarchxx-cross.yml

@@ -29,6 +29,10 @@
 # DAMAGE.
 
 # Github Actions workflow for AArchXX Continuous Integration testing.
+#
+# Each test suite is a separate job. When adding a new test suite job,
+# remember to add it to the send-failure-notification job to ensure failure
+# notifications are set up properly.
 
 name: ci-aarchxx-cross
 on:
@@ -98,30 +102,6 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/aarchxx-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # ARM cross-compile with gcc, with some tests run under QEMU.
   # We use a more recent Ubuntu for a more recent QEMU.
   arm-cross-compile:
@@ -171,30 +151,6 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/aarchxx-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # Android ARM cross-compile with gcc, no tests:
   android-arm-cross-compile:
     runs-on: ubuntu-20.04
@@ -243,30 +199,6 @@ jobs:
         CI_BRANCH: ${{ github.ref }}
       run: ./suite/runsuite_wrapper.pl automated_ci 32_only
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/android-arm-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # Android AArch64 cross-compile with LLVM, no tests:
   android-aarch64-cross-compile:
     runs-on: ubuntu-22.04
@@ -309,30 +241,6 @@ jobs:
         CI_BRANCH: ${{ github.ref }}
       run: ./suite/runsuite_wrapper.pl automated_ci 64_only
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/android-aarch64-cross-compile
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
-
   # AArch64 drdecode and drmemtrace on x86:
   a64-on-x86:
     runs-on: ubuntu-20.04
@@ -369,26 +277,19 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [aarch64-cross-compile, arm-cross-compile, android-arm-cross-compile, android-aarch64-cross-compile, a64-on-x86]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/a64-on-x86
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
+        test_suite_status: ${{ format('{0} {1} | {2} {3} | {4} {5} | {6} {7} | {8} {9}',
+                                      'aarch64-cross-compile', needs.aarch64-cross-compile.result,
+                                      'arm-cross-compile', needs.arm-cross-compile.result,
+                                      'android-arm-cross-compile', needs.android-arm-cross-compile.result,
+                                      'android-aarch64-cross-compile', needs.android-aarch64-cross-compile.result,
+                                      'a64-on-x86', needs.a64-on-x86.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit

.github/workflows/ci-aarchxx.yml

@@ -30,6 +30,10 @@
 # DAMAGE.
 
 # Github Actions workflow for aarch64 Continuous Integration testing.
+#
+# Each test suite is a separate job. When adding a new test suite job,
+# remember to add it to the send-failure-notification job to ensure failure
+# notifications are set up properly.
 
 name: ci-aarchxx
 on:
@@ -128,26 +132,15 @@ jobs:
           CI_BRANCH: ${{ github.ref }}
         if: ${{ matrix.sve == true && matrix.sve_length != ''}}
 
-      - name: Send failure mail to dynamorio-devs
-        if: failure() && github.ref == 'refs/heads/master'
-        uses: dawidd6/action-send-mail@v2
-        with:
-          server_address: smtp.gmail.com
-          server_port: 465
-          username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-          password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-          subject: |
-            [${{github.repository}}] ${{github.workflow}} FAILED
-            on ${{github.event_name}} at ${{github.ref}}
-          body: |
-            Github Actions CI workflow run FAILED!
-            Workflow: ${{github.workflow}}/x86-32
-            Repository: ${{github.repository}}
-            Branch ref: ${{github.ref}}
-            SHA: ${{github.sha}}
-            Triggering actor: ${{github.actor}}
-            Triggering event: ${{github.event_name}}
-            Run Id: ${{github.run_id}}
-            See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-          to: [email protected]
-          from: Github Action CI
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [aarch64-precommit]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
+      with:
+        test_suite_status: ${{ format('{0} {1}',
+                                      'aarch64-precommit', needs.aarch64-precommit.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit

.github/workflows/ci-clang-format.yml

@@ -29,6 +29,9 @@
 # DAMAGE.
 
 # Github Actions workflow for clang-format Continuous Integration testing.
+#
+# When adding a new job, remember to add it to the send-failure-notification
+# job to ensure failure notifications are set up properly.
 
 name: ci-clang-format
 on:
@@ -87,26 +90,16 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [clang-format]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/clang-format
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
+        test_suite_status: ${{ format('{0} {1}',
+                                      'clang-format', needs.clang-format.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit

.github/workflows/ci-docs.yml

@@ -29,6 +29,9 @@
 # DAMAGE.
 
 # Github Actions workflow for building doxygen docs for the web site.
+#
+# When adding a new job, remember to add it to the send-failure-notification
+# job to ensure failure notifications are set up properly.
 
 name: ci-docs
 on:
@@ -130,26 +133,15 @@ jobs:
         # We need a personal access token for write access to another repo.
         GITHUB_TOKEN: ${{ secrets.DOCS_TOKEN }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [docs]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/docs
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action package jobs
+        test_suite_status: ${{ format('{0} {1}',
+                                      'docs', needs.docs.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit

.github/workflows/ci-osx.yml

@@ -29,6 +29,10 @@
 # DAMAGE.
 
 # Github Actions workflow for Mac OSX Continuous Integration testing.
+#
+# Each test suite is a separate job. When adding a new test suite job,
+# remember to add it to the send-failure-notification job to ensure failure
+# notifications are set up properly.
 
 name: ci-osx
 on:
@@ -86,26 +90,15 @@ jobs:
         CI_TRIGGER: ${{ github.event_name }}
         CI_BRANCH: ${{ github.ref }}
 
-    - name: Send failure mail to dynamorio-devs
-      if: failure() && github.ref == 'refs/heads/master'
-      uses: dawidd6/action-send-mail@v2
+  send-failure-notification:
+      uses: ./.github/workflows/failure-notification.yml
+      needs: [osx-x86-64]
+      # By default, a failure in a job skips the jobs that need it. The
+      # following expression ensures that failure-notification.yml is
+      # always invoked.
+      if: ${{ always() }}
       with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}}
-        password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}}
-        subject: |
-          [${{github.repository}}] ${{github.workflow}} FAILED
-          on ${{github.event_name}} at ${{github.ref}}
-        body: |
-          Github Actions CI workflow run FAILED!
-          Workflow: ${{github.workflow}}/osx-x86-64
-          Repository: ${{github.repository}}
-          Branch ref: ${{github.ref}}
-          SHA: ${{github.sha}}
-          Triggering actor: ${{github.actor}}
-          Triggering event: ${{github.event_name}}
-          Run Id: ${{github.run_id}}
-          See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}}
-        to: [email protected]
-        from: Github Action CI
+        test_suite_status: ${{ format('{0} {1}',
+                                      'osx-x86-64', needs.osx-x86-64.result) }}
+        test_suite_results_only: ${{ join(needs.*.result, ',') }}
+      secrets: inherit