test-infra/.github/workflows/vllm-torch-nightly-triage.yml at 3547814d75908a248bcbe3de0fef036cf2c1b1b9 · atalman/test-infra · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
name: vLLM torch-nightly triage

# Detects vLLM CI regressions attributable to PyTorch nightly.
#
# The vLLM Buildkite pipeline runs "Full CI run torch nightly" (TORCH_NIGHTLY=1) and
# "Full CI run - nightly" from the same HEAD in the same cron slot, so the pair is a
# controlled A/B. This job reports jobs that fail in the former and pass in the latter.
#
# ClickHouse stores job metadata only -- log contents are not ingested -- so this
# identifies *which* jobs regressed, not *why*. The issue body ends with @claude so a
# maintainer gets on-demand root-cause analysis (which reads the linked Buildkite logs
# via .claude/skills/vllm-pytorch-ci-triage) without this job needing a Buildkite token.

on:
  workflow_dispatch:
    inputs:
      lookback_days:
        description: "How far back to look for a torch-nightly build"
        required: false
        default: "14"
        type: string
  schedule:
    # torch nightly fires 23:00 America/Los_Angeles Mon/Tue/Thu (06:00-07:00 UTC the
    # next day depending on DST). Run well after the build has had time to finish.
    - cron: 0 12 * * 2,3,5

jobs:
  triage:
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"
          cache: pip

      - name: Generate report
        id: report
        working-directory: tools
        env:
          CLICKHOUSE_ENDPOINT: ${{ secrets.CLICKHOUSE_HUD_USER_URL }}
          CLICKHOUSE_USERNAME: ${{ secrets.CLICKHOUSE_HUD_USER_USERNAME }}
          CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_HUD_USER_PASSWORD }}
        run: |
          set -euxo pipefail
          python3 -m pip install clickhouse-connect==0.8.14
          PYTHONPATH=. python3 -m torchci.vllm_torch_nightly_triage \
            --lookback-days "${{ inputs.lookback_days || '14' }}" \
            --output "${RUNNER_TEMP}/report.md" \
            --json-output "${RUNNER_TEMP}/report.json"
          # No pair found => no report written => nothing to post.
          if [[ -s "${RUNNER_TEMP}/report.json" ]]; then
            echo "has_report=true" >> "${GITHUB_OUTPUT}"
            jq -r '"build=\(.torch_nightly_build)"' "${RUNNER_TEMP}/report.json" >> "${GITHUB_OUTPUT}"
            jq -r '"regressed=\(.regressed | length)"' "${RUNNER_TEMP}/report.json" >> "${GITHUB_OUTPUT}"
          else
            echo "has_report=false" >> "${GITHUB_OUTPUT}"
          fi

      - name: Open triage issue
        if: steps.report.outputs.has_report == 'true' && steps.report.outputs.regressed != '0'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BUILD: ${{ steps.report.outputs.build }}
          COUNT: ${{ steps.report.outputs.regressed }}
        run: |
          set -euo pipefail
          TITLE="[vllm] torch-nightly regressions in Buildkite #${BUILD} (${COUNT} jobs)"
          {
            cat "${RUNNER_TEMP}/report.md"
            echo
            echo "---"
            echo
            echo "@claude please root-cause these regressions using the"
            echo "\`vllm-pytorch-ci-triage\` skill. Fetch the Buildkite logs for one"
            echo "representative job per cluster above, group the failures by root cause,"
            echo "and draft (do not file) one upstream issue per distinct cause."
          } > "${RUNNER_TEMP}/issue.md"
          gh issue create \
            --repo "${GITHUB_REPOSITORY}" \
            --title "${TITLE}" \
            --body-file "${RUNNER_TEMP}/issue.md" \
            --label "vllm" --label "ci: torch-nightly"

      - name: Upload report
        if: steps.report.outputs.has_report == 'true'
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: vllm-torch-nightly-triage
          path: ${{ runner.temp }}/report.*
          if-no-files-found: ignore