diff --git a/.github/actions/notify-slack-failure/action.yml b/.github/actions/notify-slack-failure/action.yml new file mode 100644 index 000000000000..876bf21743b0 --- /dev/null +++ b/.github/actions/notify-slack-failure/action.yml @@ -0,0 +1,54 @@ +name: 'Notify Slack Failure' +description: 'Post a failed auto-release run to a Slack Workflow Builder webhook.' + +inputs: + webhook-url: + description: 'Slack Workflow Builder webhook URL (secret).' + required: true + workflow-name: + description: 'Workflow display name (github.workflow). "Auto Release - " prefix is stripped.' + required: true + config-file: + description: 'Path to the image config YAML; its basename identifies which config failed.' + required: true + run-url: + description: 'URL of the failed workflow run.' + required: true + +runs: + using: 'composite' + steps: + - name: Send Slack notification + shell: bash + env: + SLACK_WEBHOOK_URL: ${{ inputs.webhook-url }} + WORKFLOW_NAME: ${{ inputs.workflow-name }} + CONFIG_FILE: ${{ inputs.config-file }} + RUN_URL: ${{ inputs.run-url }} + run: | + if [[ -z "${SLACK_WEBHOOK_URL}" ]]; then + echo "::error::webhook-url is empty — set the SLACK_RELEASE_FAILURE_WEBHOOK_URL secret." + exit 1 + fi + + # Include the config basename: one workflow releases several configs via matrix. + workflow="${WORKFLOW_NAME#Auto Release - } ($(basename "${CONFIG_FILE}"))" + + payload=$(jq -n \ + --arg workflow "${workflow}" \ + --arg url "${RUN_URL}" \ + '{ + workflow: $workflow, + url: $url + }') + + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + --max-time 10 \ + -X POST "${SLACK_WEBHOOK_URL}" \ + -H 'Content-Type: application/json' \ + --data "${payload}") + + echo "Slack webhook responded with HTTP ${http_code}" + if [[ "${http_code}" != "200" ]]; then + echo "::warning::Slack notification failed (HTTP ${http_code})" + fi diff --git a/.github/workflows/base.pipeline.yml b/.github/workflows/base.pipeline.yml index 45423b432331..a7a1c614b7f5 100644 --- a/.github/workflows/base.pipeline.yml +++ b/.github/workflows/base.pipeline.yml @@ -119,3 +119,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [build, sanity-test, security-test, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/huggingface-vllm.pipeline.yml b/.github/workflows/huggingface-vllm.pipeline.yml index 813ec0235756..9bce56beb3fa 100644 --- a/.github/workflows/huggingface-vllm.pipeline.yml +++ b/.github/workflows/huggingface-vllm.pipeline.yml @@ -151,3 +151,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [build, sanity-test, security-test, telemetry-test, sagemaker-tests, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/lambda.pipeline.yml b/.github/workflows/lambda.pipeline.yml index 7a7aa0024a43..d92b889c482d 100644 --- a/.github/workflows/lambda.pipeline.yml +++ b/.github/workflows/lambda.pipeline.yml @@ -202,3 +202,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [build, sanity-test, security-test, telemetry-test, validation-test, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/openfold3.pipeline.yml b/.github/workflows/openfold3.pipeline.yml index e97c33f2bd85..d551ede6cc98 100644 --- a/.github/workflows/openfold3.pipeline.yml +++ b/.github/workflows/openfold3.pipeline.yml @@ -145,3 +145,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [ci-config, build, sanity-test, security-test, sagemaker-tests, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/pytorch.pipeline.yml b/.github/workflows/pytorch.pipeline.yml index 909bebf317ee..3990c42cb1da 100644 --- a/.github/workflows/pytorch.pipeline.yml +++ b/.github/workflows/pytorch.pipeline.yml @@ -249,3 +249,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [ci-config, build, sanity-test, security-test, telemetry-test, unit-test, single-gpu-test, multi-gpu-test, multi-node-gpu-test, efa-test, sagemaker-test, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/ray.pipeline.yml b/.github/workflows/ray.pipeline.yml index 14efc7ed422f..a9da176a9cd2 100644 --- a/.github/workflows/ray.pipeline.yml +++ b/.github/workflows/ray.pipeline.yml @@ -195,3 +195,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [ci-config, build, sanity-test, security-test, telemetry-test, ffmpeg-test, serve-test, sagemaker-test, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/sglang.pipeline.yml b/.github/workflows/sglang.pipeline.yml index 65edf6155320..a632a89b49e8 100644 --- a/.github/workflows/sglang.pipeline.yml +++ b/.github/workflows/sglang.pipeline.yml @@ -217,3 +217,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [ci-config, build, sanity-test, security-test, telemetry-test, upstream-tests, model-tests, sagemaker-tests, efa-test, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/vllm-omni.pipeline.yml b/.github/workflows/vllm-omni.pipeline.yml index b7ee17582e5e..c8caadabb085 100644 --- a/.github/workflows/vllm-omni.pipeline.yml +++ b/.github/workflows/vllm-omni.pipeline.yml @@ -180,3 +180,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [ci-config, build, sanity-test, security-test, telemetry-test, model-tests, sagemaker-tests, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/vllm.pipeline.yml b/.github/workflows/vllm.pipeline.yml index e666f56ef805..f07bebcaab65 100644 --- a/.github/workflows/vllm.pipeline.yml +++ b/.github/workflows/vllm.pipeline.yml @@ -216,3 +216,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [ci-config, build, sanity-test, security-test, telemetry-test, upstream-tests, model-tests, sagemaker-tests, efa-test, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/xgboost.pipeline.yml b/.github/workflows/xgboost.pipeline.yml index e95b0d2fd345..695009236b50 100644 --- a/.github/workflows/xgboost.pipeline.yml +++ b/.github/workflows/xgboost.pipeline.yml @@ -182,3 +182,23 @@ jobs: environment: ${{ needs.release-gate.outputs.environment }} aws-region: ${{ vars.AWS_REGION }} secrets: inherit + + notify-failure: + # Notify Slack immediately when an auto-release/dispatch-release run fails. + # Guarded on release context so PR runs never notify. + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [build, sanity-test, security-test, unit-test, integ-test, integ-test-full, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}