Skip to content

Commit 0cfb8a4

Browse files
Eren-Jeager123Eren-Jeager123
authored
ci: embed immediate Slack failure notification in release pipelines (#6354)
Alternative to the scheduled failure digest (#6337): notify Slack the moment an auto-release run fails — no scan timing, no dedup logic, and a failed rerun re-posts. Shared logic lives in a composite action (.github/actions/ notify-slack-failure) so the 10 *.pipeline.yml orchestrators only add a thin notify-failure job that needs every pipeline job, runs on failure(), and calls the action. Guarded on the same release context as release-gate (main + autorelease/dispatch-release ref + inputs.release), so PR runs never notify. Payload is {workflow, url}; the label strips the 'Auto Release - ' prefix and adds the config basename (one workflow releases several configs via matrix). Webhook secret: SLACK_RELEASE_FAILURE_WEBHOOK_URL.
1 parent 78b8339 commit 0cfb8a4

11 files changed

Lines changed: 254 additions & 0 deletions

‎.github/actions/notify-slack-failure/action.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: 'Notify Slack Failure'
2+
description: 'Post a failed auto-release run to a Slack Workflow Builder webhook.'
3+
4+
inputs:
5+
webhook-url:
6+
description: 'Slack Workflow Builder webhook URL (secret).'
7+
required: true
8+
workflow-name:
9+
description: 'Workflow display name (github.workflow). "Auto Release - " prefix is stripped.'
10+
required: true
11+
config-file:
12+
description: 'Path to the image config YAML; its basename identifies which config failed.'
13+
required: true
14+
run-url:
15+
description: 'URL of the failed workflow run.'
16+
required: true
17+
18+
runs:
19+
using: 'composite'
20+
steps:
21+
- name: Send Slack notification
22+
shell: bash
23+
env:
24+
SLACK_WEBHOOK_URL: ${{ inputs.webhook-url }}
25+
WORKFLOW_NAME: ${{ inputs.workflow-name }}
26+
CONFIG_FILE: ${{ inputs.config-file }}
27+
RUN_URL: ${{ inputs.run-url }}
28+
run: |
29+
if [[ -z "${SLACK_WEBHOOK_URL}" ]]; then
30+
echo "::error::webhook-url is empty — set the SLACK_RELEASE_FAILURE_WEBHOOK_URL secret."
31+
exit 1
32+
fi
33+
34+
# Include the config basename: one workflow releases several configs via matrix.
35+
workflow="${WORKFLOW_NAME#Auto Release - } ($(basename "${CONFIG_FILE}"))"
36+
37+
payload=$(jq -n \
38+
--arg workflow "${workflow}" \
39+
--arg url "${RUN_URL}" \
40+
'{
41+
workflow: $workflow,
42+
url: $url
43+
}')
44+
45+
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
46+
--max-time 10 \
47+
-X POST "${SLACK_WEBHOOK_URL}" \
48+
-H 'Content-Type: application/json' \
49+
--data "${payload}")
50+
51+
echo "Slack webhook responded with HTTP ${http_code}"
52+
if [[ "${http_code}" != "200" ]]; then
53+
echo "::warning::Slack notification failed (HTTP ${http_code})"
54+
fi

‎.github/workflows/base.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,3 +119,23 @@ jobs:
119119
environment: ${{ needs.release-gate.outputs.environment }}
120120
aws-region: ${{ vars.AWS_REGION }}
121121
secrets: inherit
122+
123+
notify-failure:
124+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
125+
# Guarded on release context so PR runs never notify.
126+
if: >-
127+
failure() &&
128+
github.ref == 'refs/heads/main' &&
129+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
130+
!contains(github.workflow_ref, '.pr') &&
131+
inputs.release
132+
needs: [build, sanity-test, security-test, release-gate, release]
133+
runs-on: ubuntu-latest
134+
steps:
135+
- uses: actions/checkout@v6
136+
- uses: ./.github/actions/notify-slack-failure
137+
with:
138+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
139+
workflow-name: ${{ github.workflow }}
140+
config-file: ${{ inputs.config-file }}
141+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/huggingface-vllm.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,3 +151,23 @@ jobs:
151151
environment: ${{ needs.release-gate.outputs.environment }}
152152
aws-region: ${{ vars.AWS_REGION }}
153153
secrets: inherit
154+
155+
notify-failure:
156+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
157+
# Guarded on release context so PR runs never notify.
158+
if: >-
159+
failure() &&
160+
github.ref == 'refs/heads/main' &&
161+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
162+
!contains(github.workflow_ref, '.pr') &&
163+
inputs.release
164+
needs: [build, sanity-test, security-test, telemetry-test, sagemaker-tests, release-gate, release]
165+
runs-on: ubuntu-latest
166+
steps:
167+
- uses: actions/checkout@v6
168+
- uses: ./.github/actions/notify-slack-failure
169+
with:
170+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
171+
workflow-name: ${{ github.workflow }}
172+
config-file: ${{ inputs.config-file }}
173+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/lambda.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,3 +202,23 @@ jobs:
202202
environment: ${{ needs.release-gate.outputs.environment }}
203203
aws-region: ${{ vars.AWS_REGION }}
204204
secrets: inherit
205+
206+
notify-failure:
207+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
208+
# Guarded on release context so PR runs never notify.
209+
if: >-
210+
failure() &&
211+
github.ref == 'refs/heads/main' &&
212+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
213+
!contains(github.workflow_ref, '.pr') &&
214+
inputs.release
215+
needs: [build, sanity-test, security-test, telemetry-test, validation-test, release-gate, release]
216+
runs-on: ubuntu-latest
217+
steps:
218+
- uses: actions/checkout@v6
219+
- uses: ./.github/actions/notify-slack-failure
220+
with:
221+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
222+
workflow-name: ${{ github.workflow }}
223+
config-file: ${{ inputs.config-file }}
224+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/openfold3.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,3 +145,23 @@ jobs:
145145
environment: ${{ needs.release-gate.outputs.environment }}
146146
aws-region: ${{ vars.AWS_REGION }}
147147
secrets: inherit
148+
149+
notify-failure:
150+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
151+
# Guarded on release context so PR runs never notify.
152+
if: >-
153+
failure() &&
154+
github.ref == 'refs/heads/main' &&
155+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
156+
!contains(github.workflow_ref, '.pr') &&
157+
inputs.release
158+
needs: [ci-config, build, sanity-test, security-test, sagemaker-tests, release-gate, release]
159+
runs-on: ubuntu-latest
160+
steps:
161+
- uses: actions/checkout@v6
162+
- uses: ./.github/actions/notify-slack-failure
163+
with:
164+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
165+
workflow-name: ${{ github.workflow }}
166+
config-file: ${{ inputs.config-file }}
167+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/pytorch.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,3 +249,23 @@ jobs:
249249
environment: ${{ needs.release-gate.outputs.environment }}
250250
aws-region: ${{ vars.AWS_REGION }}
251251
secrets: inherit
252+
253+
notify-failure:
254+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
255+
# Guarded on release context so PR runs never notify.
256+
if: >-
257+
failure() &&
258+
github.ref == 'refs/heads/main' &&
259+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
260+
!contains(github.workflow_ref, '.pr') &&
261+
inputs.release
262+
needs: [ci-config, build, sanity-test, security-test, telemetry-test, unit-test, single-gpu-test, multi-gpu-test, multi-node-gpu-test, efa-test, sagemaker-test, release-gate, release]
263+
runs-on: ubuntu-latest
264+
steps:
265+
- uses: actions/checkout@v6
266+
- uses: ./.github/actions/notify-slack-failure
267+
with:
268+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
269+
workflow-name: ${{ github.workflow }}
270+
config-file: ${{ inputs.config-file }}
271+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/ray.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,3 +195,23 @@ jobs:
195195
environment: ${{ needs.release-gate.outputs.environment }}
196196
aws-region: ${{ vars.AWS_REGION }}
197197
secrets: inherit
198+
199+
notify-failure:
200+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
201+
# Guarded on release context so PR runs never notify.
202+
if: >-
203+
failure() &&
204+
github.ref == 'refs/heads/main' &&
205+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
206+
!contains(github.workflow_ref, '.pr') &&
207+
inputs.release
208+
needs: [ci-config, build, sanity-test, security-test, telemetry-test, ffmpeg-test, serve-test, sagemaker-test, release-gate, release]
209+
runs-on: ubuntu-latest
210+
steps:
211+
- uses: actions/checkout@v6
212+
- uses: ./.github/actions/notify-slack-failure
213+
with:
214+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
215+
workflow-name: ${{ github.workflow }}
216+
config-file: ${{ inputs.config-file }}
217+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/sglang.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,3 +217,23 @@ jobs:
217217
environment: ${{ needs.release-gate.outputs.environment }}
218218
aws-region: ${{ vars.AWS_REGION }}
219219
secrets: inherit
220+
221+
notify-failure:
222+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
223+
# Guarded on release context so PR runs never notify.
224+
if: >-
225+
failure() &&
226+
github.ref == 'refs/heads/main' &&
227+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
228+
!contains(github.workflow_ref, '.pr') &&
229+
inputs.release
230+
needs: [ci-config, build, sanity-test, security-test, telemetry-test, upstream-tests, model-tests, sagemaker-tests, efa-test, release-gate, release]
231+
runs-on: ubuntu-latest
232+
steps:
233+
- uses: actions/checkout@v6
234+
- uses: ./.github/actions/notify-slack-failure
235+
with:
236+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
237+
workflow-name: ${{ github.workflow }}
238+
config-file: ${{ inputs.config-file }}
239+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/vllm-omni.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,3 +180,23 @@ jobs:
180180
environment: ${{ needs.release-gate.outputs.environment }}
181181
aws-region: ${{ vars.AWS_REGION }}
182182
secrets: inherit
183+
184+
notify-failure:
185+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
186+
# Guarded on release context so PR runs never notify.
187+
if: >-
188+
failure() &&
189+
github.ref == 'refs/heads/main' &&
190+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
191+
!contains(github.workflow_ref, '.pr') &&
192+
inputs.release
193+
needs: [ci-config, build, sanity-test, security-test, telemetry-test, model-tests, sagemaker-tests, release-gate, release]
194+
runs-on: ubuntu-latest
195+
steps:
196+
- uses: actions/checkout@v6
197+
- uses: ./.github/actions/notify-slack-failure
198+
with:
199+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
200+
workflow-name: ${{ github.workflow }}
201+
config-file: ${{ inputs.config-file }}
202+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/vllm.pipeline.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,3 +216,23 @@ jobs:
216216
environment: ${{ needs.release-gate.outputs.environment }}
217217
aws-region: ${{ vars.AWS_REGION }}
218218
secrets: inherit
219+
220+
notify-failure:
221+
# Notify Slack immediately when an auto-release/dispatch-release run fails.
222+
# Guarded on release context so PR runs never notify.
223+
if: >-
224+
failure() &&
225+
github.ref == 'refs/heads/main' &&
226+
(contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) &&
227+
!contains(github.workflow_ref, '.pr') &&
228+
inputs.release
229+
needs: [ci-config, build, sanity-test, security-test, telemetry-test, upstream-tests, model-tests, sagemaker-tests, efa-test, release-gate, release]
230+
runs-on: ubuntu-latest
231+
steps:
232+
- uses: actions/checkout@v6
233+
- uses: ./.github/actions/notify-slack-failure
234+
with:
235+
webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }}
236+
workflow-name: ${{ github.workflow }}
237+
config-file: ${{ inputs.config-file }}
238+
run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

0 commit comments

Comments
 (0)