From 5d6714d134467b6618d46fb868fe16de397ff1de Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 1 Jul 2026 21:12:45 +0000 Subject: [PATCH 1/3] ci: add compact Slack digest of auto-release failures Auto-release workflows (discover -> pipeline) fail silently today: a scheduled release breaks with no PR involved and nobody watching the Actions tab. Add a digest that runs twice daily (16:00 and 20:00 UTC), sweeps the GitHub API for auto-release runs that failed since the previous sweep, and posts ONE compact Slack message listing them. Stays silent when nothing failed, so no notification noise; each failure is reported exactly once via complementary lookback windows that tile the day (20:00 sweep looks back 4h, 16:00 sweep looks back 20h, chosen by the sweep hour). Runs are matched on failure time (updated_at) not start time, so heavy GPU builds that start before a sweep but fail within the window are still caught. Coverage is by naming convention (name starts with 'Auto Release'), so new autorelease variants (e.g. PyTorch 2.12) are picked up automatically with no list to maintain. Manual 'Dispatch -' release runs are excluded. Flat key/value payload (count/failures) to a Slack Workflow Builder webhook. Webhook stored as secret SLACK_RELEASE_FAILURE_WEBHOOK_URL. --- .../_scheduled.notify-release-failure.yml | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 .github/workflows/_scheduled.notify-release-failure.yml diff --git a/.github/workflows/_scheduled.notify-release-failure.yml b/.github/workflows/_scheduled.notify-release-failure.yml new file mode 100644 index 000000000000..2181b6f43e4e --- /dev/null +++ b/.github/workflows/_scheduled.notify-release-failure.yml @@ -0,0 +1,106 @@ +# notify-release-failure.yml — Posts a compact Slack digest of auto-release failures. +# +# Auto-release workflows (discover -> pipeline) fail silently: a scheduled release +# breaks, no PR is involved, and nobody is watching the Actions tab. +# +# This runs twice a day (16:00 and 20:00 UTC), sweeps the GitHub API for +# auto-release runs that FAILED since the previous sweep, and posts ONE compact +# Slack message listing them. If nothing failed, it stays silent (no noise). Each +# failure is reported exactly once: the two sweeps use complementary lookback +# windows that tile the day (20:00 sweep looks back 4h; 16:00 sweep looks back 20h). +# +# Runs are matched on FAILURE time (updated_at), not start time, because heavy GPU +# releases can build for hours — a run started before a sweep may only fail after. +# +# Coverage is by naming convention: any workflow whose name starts with "Auto +# Release" is included, so new autorelease variants are picked up automatically +# with no list to maintain. Manual "Dispatch -" release runs are excluded (a human +# is already watching those). +# +# Payload is a flat key/value body posted to a Slack Workflow Builder webhook +# (same style as scripts/ci/autocurrency/utils.sh). Slack owns formatting/routing: +# count — number of failed auto-release runs in the window +# failures — pre-formatted multi-line list ("• → " per line) +# Webhook is the GitHub Actions secret SLACK_RELEASE_FAILURE_WEBHOOK_URL. + +name: Notify Release Failures + +on: + schedule: + - cron: "0 16 * * *" + - cron: "0 20 * * *" + workflow_dispatch: + +permissions: + actions: read + +jobs: + digest: + runs-on: ubuntu-latest + steps: + - name: Collect failures and post digest + env: + GH_TOKEN: ${{ github.token }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + run: | + set -euo pipefail + + if [[ -z "${SLACK_WEBHOOK_URL}" ]]; then + echo "Slack notifications: webhook URL not configured. Skipping." + exit 0 + fi + + # The two daily sweeps tile the day: the 20:00 sweep covers the 16:00-20:00 + # gap (4h); the 16:00 sweep covers the prior 20:00 -> today 16:00 gap (20h). + if [[ "$(date -u +%H)" -lt 18 ]]; then + lookback=20 + else + lookback=4 + fi + echo "Looking back ${lookback}h for failed auto-release runs." + + since_epoch=$(date -u -d "${lookback} hours ago" +%s) + # Cap pagination volume: only pull failure runs created in the last 3 days + # (long enough to cover any heavy build still running at window open). + created_since=$(date -u -d "3 days ago" +%Y-%m-%d) + + # Filter on updated_at (~= failure time), not created_at, so long builds + # that started before this sweep but failed within the window are caught. + lines=$(gh api --paginate \ + "/repos/${GITHUB_REPOSITORY}/actions/runs?status=failure&created=>=${created_since}&per_page=100" \ + --jq '.workflow_runs[] + | select(.conclusion == "failure") + | select(.name | startswith("Auto Release")) + | [(.updated_at | fromdateiso8601 | tostring), .name, .html_url] + | @tsv' \ + | awk -F'\t' -v s="${since_epoch}" '$1 >= s {print}') + + count=$(printf '%s' "${lines}" | grep -c . || true) + + if [[ "${count}" -eq 0 ]]; then + echo "No auto-release failures in the last ${lookback}h. Staying silent." + exit 0 + fi + + failures=$(printf '%s\n' "${lines}" | awk -F'\t' 'NF {print "• " $2 " → " $3}') + echo "Found ${count} failed auto-release run(s):" + echo "${failures}" + + payload=$(jq -n \ + --arg count "${count}" \ + --arg failures "${failures}" \ + '{ + count: $count, + failures: $failures + }') + + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + --max-time 10 \ + -X POST "${SLACK_WEBHOOK_URL}" \ + -H 'Content-Type: application/json' \ + --data "${payload}") + + echo "Slack webhook responded with HTTP ${http_code}" + if [[ "${http_code}" != "200" ]]; then + echo "::warning::Slack notification failed (HTTP ${http_code})" + fi From 7b1a2a02bae62b484825f6e45e747c8ed2c92749 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 1 Jul 2026 21:35:15 +0000 Subject: [PATCH 2/3] ci: shorten failure labels and use Slack hyperlinks Strip the redundant 'Auto Release - ' prefix from each failure label (the channel already implies auto-release), and render each bullet as a Slack hyperlink '' so the full run URL isn't shown inline. --- .../workflows/_scheduled.notify-release-failure.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_scheduled.notify-release-failure.yml b/.github/workflows/_scheduled.notify-release-failure.yml index 2181b6f43e4e..e1fe5785197c 100644 --- a/.github/workflows/_scheduled.notify-release-failure.yml +++ b/.github/workflows/_scheduled.notify-release-failure.yml @@ -20,7 +20,8 @@ # Payload is a flat key/value body posted to a Slack Workflow Builder webhook # (same style as scripts/ci/autocurrency/utils.sh). Slack owns formatting/routing: # count — number of failed auto-release runs in the window -# failures — pre-formatted multi-line list ("• → " per line) +# failures — pre-formatted multi-line list, one Slack hyperlink per line +# ("• ", "Auto Release - " prefix stripped) # Webhook is the GitHub Actions secret SLACK_RELEASE_FAILURE_WEBHOOK_URL. name: Notify Release Failures @@ -82,7 +83,13 @@ jobs: exit 0 fi - failures=$(printf '%s\n' "${lines}" | awk -F'\t' 'NF {print "• " $2 " → " $3}') + # One Slack-hyperlink bullet per failure: "• ". The redundant + # "Auto Release - " prefix is stripped from the label; the channel already + # implies these are auto-release failures. + failures=$(printf '%s\n' "${lines}" | awk -F'\t' 'NF { + label = $2; sub(/^Auto Release - /, "", label) + print "• <" $3 "|" label ">" + }') echo "Found ${count} failed auto-release run(s):" echo "${failures}" From 0029360b8f34ead093e9da51eb9f7f5b3989ca29 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 1 Jul 2026 21:39:21 +0000 Subject: [PATCH 3/3] ci: use bare URLs in failure digest (Slack Workflow Builder plain-text) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slack Workflow Builder inserts variable values as plain text and does not render mrkdwn hyperlinks () from a variable, so the links showed up literally. Switch each bullet to '•