chore(ci): add failure notification for release pipeline (camunda#354)

* chore(ci): add failure notification for release pipeline

* fix(ci): use OIDC/JWT for Vault auth and slack-github-action for notifications

Switch the notify-on-failure jobs from Vault AppRole to the JWT/OIDC auth
method (VAULT_JWT_ROLE/PATH/AUDIENCE, already provisioned for this repo),
and replace the ad-hoc curl-based Slack notification with the maintained
slackapi/slack-github-action, matching the pattern used across the org.

* test(ci): add temporary workflow to validate Vault JWT + Slack notify

Sanity-checks the Vault JWT auth + slack-github-action wiring used in
build-and-test-ee.yml / build-and-test-qa.yml before relying on it.
To be removed once confirmed.

* chore(ci): remove temporary Vault/Slack validation workflow

Confirmed working, no longer needed.

---------

Co-authored-by: Patrick Poetz <patrick.poetz@camunda.com>

Author: igpetrov

.github/workflows/build-and-test-ee.yml

@@ -98,3 +98,46 @@ jobs:
           SNAPSHOT: ${{ github.event.inputs.snapshot }}
           PUSH_REGISTRY: registry.camunda.cloud/cambpm-ee
 
+  # Runs whenever any of the jobs above failed, so failures don't pass by unnoticed.
+  # `if: failure()` deliberately overrides the default "skip if a dependency didn't
+  # succeed" behavior, so this still fires even if e.g. create-manifests-ee was
+  # skipped (PR runs) while build-and-test-ee failed.
+  notify-on-failure:
+    if: failure()
+    needs: [build-and-test-ee, create-manifests-ee]
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      id-token: write  # mint GitHub OIDC token for Vault JWT auth
+    steps:
+      - name: Import Secrets
+        id: secrets
+        uses: hashicorp/vault-action@v3.4.0
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          method: jwt
+          path: ${{ secrets.VAULT_JWT_PATH }}
+          role: ${{ secrets.VAULT_JWT_ROLE }}
+          jwtGithubAudience: ${{ secrets.VAULT_JWT_AUDIENCE }}
+          exportEnv: false
+          secrets: |
+            secret/data/products/cambpm/ci/slack NOTIFY_INTERNAL_WEBHOOK | NOTIFY_INTERNAL_WEBHOOK;
+
+      - name: Notify Slack
+        uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3
+        with:
+          webhook: ${{ steps.secrets.outputs.NOTIFY_INTERNAL_WEBHOOK }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":x: *Build and Test EE* failed${{ github.event.inputs.version && format(' for version `{0}`', github.event.inputs.version) || '' }} on `${{ github.ref_name }}`. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
+                  }
+                }
+              ]
+            }
+

.github/workflows/build-and-test-qa.yml

@@ -249,3 +249,46 @@ jobs:
           VERSION_TAG: ${{ needs.prepare.outputs.version_tag }}
           LATEST_TAG: ${{ needs.prepare.outputs.latest_tag }}
 
+  # Runs whenever any of the jobs above failed, so failures don't pass by unnoticed.
+  # `if: failure()` deliberately overrides the default "skip if a dependency didn't
+  # succeed" behavior, so this still fires even if e.g. `prepare` failed and the
+  # remaining jobs were consequently skipped rather than failed themselves.
+  notify-on-failure:
+    if: failure()
+    needs: [prepare, build-and-test-qa, create-manifests-qa]
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      id-token: write  # mint GitHub OIDC token for Vault JWT auth
+    steps:
+      - name: Import Secrets
+        id: secrets
+        uses: hashicorp/vault-action@v3.4.0
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          method: jwt
+          path: ${{ secrets.VAULT_JWT_PATH }}
+          role: ${{ secrets.VAULT_JWT_ROLE }}
+          jwtGithubAudience: ${{ secrets.VAULT_JWT_AUDIENCE }}
+          exportEnv: false
+          secrets: |
+            secret/data/products/cambpm/ci/slack NOTIFY_INTERNAL_WEBHOOK | NOTIFY_INTERNAL_WEBHOOK;
+
+      - name: Notify Slack
+        uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3
+        with:
+          webhook: ${{ steps.secrets.outputs.NOTIFY_INTERNAL_WEBHOOK }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":x: *Build and Test QA* failed for version `${{ github.event.inputs.version }}` on `${{ github.ref_name }}`. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
+                  }
+                }
+              ]
+            }
+