ci: embed immediate Slack failure notification in release pipelines (alt to #6337)

.github/actions/notify-slack-failure/action.yml

@@ -0,0 +1,54 @@
+name: 'Notify Slack Failure'
+description: 'Post a failed auto-release run to a Slack Workflow Builder webhook.'
+
+inputs:
+  webhook-url:
+    description: 'Slack Workflow Builder webhook URL (secret).'
+    required: true
+  workflow-name:
+    description: 'Workflow display name (github.workflow). "Auto Release - " prefix is stripped.'
+    required: true
+  config-file:
+    description: 'Path to the image config YAML; its basename identifies which config failed.'
+    required: true
+  run-url:
+    description: 'URL of the failed workflow run.'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Send Slack notification
+      shell: bash
+      env:
+        SLACK_WEBHOOK_URL: ${{ inputs.webhook-url }}
+        WORKFLOW_NAME: ${{ inputs.workflow-name }}
+        CONFIG_FILE: ${{ inputs.config-file }}
+        RUN_URL: ${{ inputs.run-url }}
+      run: |
+        if [[ -z "${SLACK_WEBHOOK_URL}" ]]; then
+          echo "::error::webhook-url is empty — set the SLACK_RELEASE_FAILURE_WEBHOOK_URL secret."
+          exit 1
+        fi
+
+        # Include the config basename: one workflow releases several configs via matrix.
+        workflow="${WORKFLOW_NAME#Auto Release - } ($(basename "${CONFIG_FILE}"))"
+
+        payload=$(jq -n \
+          --arg workflow "${workflow}" \
+          --arg url "${RUN_URL}" \
+          '{
+            workflow: $workflow,
+            url: $url
+          }')
+
+        http_code=$(curl -s -o /dev/null -w "%{http_code}" \
+          --max-time 10 \
+          -X POST "${SLACK_WEBHOOK_URL}" \
+          -H 'Content-Type: application/json' \
+          --data "${payload}")
+
+        echo "Slack webhook responded with HTTP ${http_code}"
+        if [[ "${http_code}" != "200" ]]; then
+          echo "::warning::Slack notification failed (HTTP ${http_code})"
+        fi

.github/workflows/base.pipeline.yml

@@ -119,3 +119,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [build, sanity-test, security-test, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/huggingface-vllm.pipeline.yml

@@ -151,3 +151,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [build, sanity-test, security-test, telemetry-test, sagemaker-tests, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/lambda.pipeline.yml

@@ -202,3 +202,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [build, sanity-test, security-test, telemetry-test, validation-test, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/openfold3.pipeline.yml

@@ -145,3 +145,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [ci-config, build, sanity-test, security-test, sagemaker-tests, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/pytorch.pipeline.yml

@@ -249,3 +249,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [ci-config, build, sanity-test, security-test, telemetry-test, unit-test, single-gpu-test, multi-gpu-test, multi-node-gpu-test, efa-test, sagemaker-test, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/ray.pipeline.yml

@@ -195,3 +195,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [ci-config, build, sanity-test, security-test, telemetry-test, ffmpeg-test, serve-test, sagemaker-test, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/sglang.pipeline.yml

@@ -217,3 +217,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [ci-config, build, sanity-test, security-test, telemetry-test, upstream-tests, model-tests, sagemaker-tests, efa-test, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/vllm-omni.pipeline.yml

@@ -180,3 +180,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [ci-config, build, sanity-test, security-test, telemetry-test, model-tests, sagemaker-tests, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/vllm.pipeline.yml

@@ -216,3 +216,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [ci-config, build, sanity-test, security-test, telemetry-test, upstream-tests, model-tests, sagemaker-tests, efa-test, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/xgboost.pipeline.yml

@@ -182,3 +182,23 @@ jobs:
       environment: ${{ needs.release-gate.outputs.environment }}
       aws-region: ${{ vars.AWS_REGION }}
     secrets: inherit
+
+  notify-failure:
+    # Notify Slack immediately when an auto-release/dispatch-release run fails.
+    # Guarded on release context so PR runs never notify.
+    if: >-
+      failure() &&
+      github.ref == 'refs/heads/main' &&
+      (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
+      !contains(github.workflow_ref, '.pr') &&
+      inputs.release
+    needs: [build, sanity-test, security-test, unit-test, integ-test, integ-test-full, release-gate, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/notify-slack-failure
+        with:
+          webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
+          workflow-name: ${{ github.workflow }}
+          config-file: ${{ inputs.config-file }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}