Parity auto-trigger · pytorch/pytorch main #429
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Parity Auto Trigger | |
| run-name: "Parity auto-trigger · pytorch/pytorch main" | |
| # Every 10 min, dispatch parity.yml once per completed upstream trunk.yml push | |
| # whose parity inputs have all finished. Scope is trunk only: the mi350 ROCm | |
| # shards that ride along in trunk.yml vs that run's CUDA shards. Other arches | |
| # have their own periodic workflows - run parity.yml manually for those. | |
| # | |
| # Readiness is gated per check-run, not on workflow_run conclusion: one failed | |
| # shard flips the parent run to failure while siblings are still going, so we | |
| # wait for every ROCm + CUDA test check-run to complete before dispatching. | |
| on: | |
| schedule: | |
| - cron: '*/10 * * * *' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/parity-auto.yml' | |
| - '.github/workflows/parity.yml' | |
| - '.automation_scripts/pytorch-unit-test-scripts/parity_job_config.json' | |
| workflow_dispatch: | |
| inputs: | |
| max_commits: | |
| description: 'How many of the most recent completed upstream trunk.yml pushes on main to scan.' | |
| required: false | |
| default: '200' | |
| type: string | |
| max_dispatches: | |
| description: 'Maximum number of ready upstream commits to dispatch in one scan.' | |
| required: false | |
| default: '50' | |
| type: string | |
| max_age_hours: | |
| description: 'Skip commits older than this (avoid back-filling ancient SHAs).' | |
| required: false | |
| default: '72' | |
| type: string | |
| arch_jobname_regex_map: | |
| description: 'Optional override: JSON map of arch -> PCRE regex for that arch''s ROCm test check-run names. Blank = use parity_job_config.json (rocm.<arch>.checkrun_regex).' | |
| required: false | |
| default: '' | |
| type: string | |
| arch_workflow_regex_map: | |
| description: 'Optional override: JSON map of arch -> PCRE regex for upstream workflow paths meaning the arch ran. Blank = derive from parity_job_config.json (union of each arch''s workflow values).' | |
| required: false | |
| default: '' | |
| type: string | |
| target_ref: | |
| description: 'Ref of this repo to dispatch parity.yml against. Leave blank to use this workflow run''s ref.' | |
| required: false | |
| default: '' | |
| type: string | |
| dry_run: | |
| description: 'Scan and log, but do not actually dispatch parity.yml.' | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: read | |
| actions: write | |
| concurrency: | |
| group: parity-auto-trigger | |
| cancel-in-progress: false | |
| jobs: | |
| scan-and-dispatch: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Find ready arches per upstream commit and dispatch parity.yml | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| UPSTREAM: pytorch/pytorch | |
| BRANCH: main | |
| MAX_COMMITS: ${{ github.event_name == 'pull_request' && '20' || inputs.max_commits || '200' }} | |
| MAX_DISPATCHES: ${{ github.event_name == 'pull_request' && '5' || inputs.max_dispatches || '50' }} | |
| MAX_AGE_HOURS: ${{ inputs.max_age_hours || '72' }} | |
| # Auto-parity is trunk-scoped: mi350 is the only ROCm arch that rides | |
| # along in trunk.yml. Other arches have their own periodic workflows. | |
| ARCHS_IN: mi350 | |
| # Optional manual overrides; blank means "derive from parity_job_config.json". | |
| ARCH_JOBNAME_REGEX_OVERRIDE: ${{ inputs.arch_jobname_regex_map || '' }} | |
| ARCH_WORKFLOW_REGEX_OVERRIDE: ${{ inputs.arch_workflow_regex_map || '' }} | |
| CONFIG_PATH: .automation_scripts/pytorch-unit-test-scripts/parity_job_config.json | |
| TARGET_REF_IN: ${{ inputs.target_ref || '' }} | |
| DRY_RUN: ${{ github.event_name == 'pull_request' && 'true' || inputs.dry_run || 'false' }} | |
| run: | | |
| # GitHub runs this step as `bash -e`, which exits on the first non-zero | |
| # status. Our paginated-API calls return non-zero in normal cases and | |
| # were silently killing the scan loop, so disable -e. | |
| # Keep -u (catch unset vars) and pipefail (catch errors mid-pipe). | |
| set +e | |
| set -uo pipefail | |
| # Check-runs from rerun/flaky tooling are not parity test shards. | |
| NON_TEST_CHECKRUNS='mem_leak_check|rerun_disabled_tests' | |
| # Read parity_job_config.json over the API at GITHUB_SHA (this job never | |
| # checks out the fork) and build the regex maps used for matching. | |
| # Manual override inputs win when set. Sets ARCH_JOBNAME_REGEX_MAP, | |
| # ARCH_WORKFLOW_REGEX_MAP, and CUDA_JOBNAME_REGEX. | |
| load_matching_config() { | |
| local config_json | |
| config_json=$(gh api "repos/$GITHUB_REPOSITORY/contents/$CONFIG_PATH?ref=$GITHUB_SHA" --jq '.content' | base64 -d) | |
| if [ -z "$config_json" ] || ! echo "$config_json" | jq -e . >/dev/null 2>&1; then | |
| echo "::error::Could not read $CONFIG_PATH at $GITHUB_SHA" | |
| exit 1 | |
| fi | |
| ARCH_JOBNAME_REGEX_MAP=${ARCH_JOBNAME_REGEX_OVERRIDE:-$(echo "$config_json" | jq -c '.rocm | map_values(.checkrun_regex)')} | |
| # workflow_regex is not stored: derive it per arch from the union of | |
| # the workflow values across that arch's default/distributed/inductor | |
| # source lists (primary + fallback entries). | |
| ARCH_WORKFLOW_REGEX_MAP=${ARCH_WORKFLOW_REGEX_OVERRIDE:-$(echo "$config_json" | jq -c ' | |
| .rocm | map_values( | |
| [ (.default[]?, .distributed[]?, .inductor[]?).workflow ] | unique | |
| | "(^|/)(" + join("|") + ")[.]yml$" | |
| )')} | |
| CUDA_JOBNAME_REGEX=$(echo "$config_json" | jq -r '.cuda.checkrun_regex') | |
| } | |
| print_run_config() { | |
| printf '%s\n' \ | |
| "Upstream: $UPSTREAM@$BRANCH" \ | |
| "Target ref: $TARGET_REF" \ | |
| "Scope archs: $ARCHS" \ | |
| "Max trunk runs: $MAX_COMMITS" \ | |
| "Max dispatches: $MAX_DISPATCHES" \ | |
| "Max age: ${MAX_AGE_HOURS}h" \ | |
| "Dry run: $DRY_RUN" \ | |
| "Arch->jobs: $ARCH_JOBNAME_REGEX_MAP" \ | |
| "Arch->workflows: $ARCH_WORKFLOW_REGEX_MAP" \ | |
| "CUDA jobs: $CUDA_JOBNAME_REGEX" \ | |
| "" | |
| } | |
| # Echo "<sha> <created_at>" lines for the most recent completed | |
| # trunk.yml pushes, deduped and newest-first, up to MAX_COMMITS. | |
| # Candidate SHAs come from completed trunk pushes (not raw main | |
| # commits): the report consumes trunk's jobs, so a completed trunk run | |
| # is the earliest a SHA can be ready. | |
| fetch_trunk_commits() { | |
| local commits_json='[]' page=1 page_runs | |
| while [ "$(echo "$commits_json" | jq 'length')" -lt "$MAX_COMMITS" ]; do | |
| page_runs=$(gh api \ | |
| "repos/$UPSTREAM/actions/workflows/trunk.yml/runs?branch=$BRANCH&event=push&status=completed&per_page=100&page=$page" \ | |
| --jq '.workflow_runs | map({head_sha, created_at})' 2>/dev/null) | |
| # On a gh api failure (rate limit / transient 5xx) page_runs is | |
| # empty or non-JSON; stop paginating and use what we have so far. | |
| if ! echo "$page_runs" | jq -e . >/dev/null 2>&1; then | |
| echo "::warning::trunk.yml runs page $page returned no/invalid JSON - stopping pagination" >&2 | |
| break | |
| fi | |
| [ "$(echo "$page_runs" | jq 'length')" -eq 0 ] && break | |
| commits_json=$(jq -s --arg max "$MAX_COMMITS" ' | |
| (.[0] + .[1]) as $runs | |
| | reduce $runs[] as $run ({seen:{}, rows:[]}; | |
| if .seen[$run.head_sha] then . | |
| else .seen[$run.head_sha] = true | .rows += [$run] | |
| end | |
| ) | |
| | .rows[:($max | tonumber)] | |
| ' <(echo "$commits_json") <(echo "$page_runs")) | |
| page=$((page + 1)) | |
| done | |
| echo "$commits_json" | jq -r '.[] | "\(.head_sha) \(.created_at)"' | |
| } | |
| # Echo a JSON array of recent auto-parity workflow_dispatch runs in our | |
| # repo, used to skip SHAs we already dispatched. Auto runs come from | |
| # github-actions[bot] and carry an "autoparity-" run-name prefix; we | |
| # match either signal so manual parity.yml runs don't suppress us. | |
| fetch_existing_dispatches() { | |
| gh api --paginate \ | |
| "repos/$GITHUB_REPOSITORY/actions/workflows/parity.yml/runs?event=workflow_dispatch&created=%3E%3D$(date -u -d "@$MAX_AGE_EPOCH" '+%Y-%m-%dT%H:%M:%SZ')&per_page=100" \ | |
| --jq '.workflow_runs[] | {display_title, actor: .actor.login}' | | |
| jq -s '.' | |
| } | |
| # True if EXISTING already has an auto-parity run for this SHA. | |
| sha_already_dispatched() { | |
| local sha="$1" | |
| echo "$EXISTING" | jq -e --arg sha "$sha" \ | |
| 'any(.[]; ((.display_title // "") | contains($sha)) and (((.display_title // "") | startswith("autoparity-")) or (.actor == "github-actions[bot]")))' >/dev/null | |
| } | |
| # Filter a check-run JSON array down to the parity test shards whose | |
| # name matches $2 (PCRE), dropping non-test check-runs. Reads the array | |
| # from stdin, echoes the filtered array. | |
| select_test_check_runs() { | |
| local regex="$1" | |
| jq --arg rx "$regex" --arg skip "$NON_TEST_CHECKRUNS" \ | |
| '[.[] | select((.name | test($rx)) and (.name | test($skip) | not))]' | |
| } | |
| # Determine which in-scope archs had their upstream workflow run on $1. | |
| # Sets RUN_ARCHS (space separated) and NOT_RUN_NOTES (per-arch reasons | |
| # for the ones that did not, for logging). Sets globals rather than | |
| # echoing so both outputs survive - command substitution would run this | |
| # in a subshell and drop NOT_RUN_NOTES. | |
| archs_that_ran() { | |
| local sha="$1" all_runs arch wf_regex wf_total run_archs="" notes="" | |
| all_runs=$(gh api --paginate \ | |
| "repos/$UPSTREAM/actions/runs?head_sha=$sha&per_page=100" \ | |
| --jq '.workflow_runs[] | {name,path,status,conclusion}' \ | |
| 2>/dev/null | jq -s '.' || echo '[]') | |
| for arch in $ARCHS; do | |
| wf_regex=$(echo "$ARCH_WORKFLOW_REGEX_MAP" | jq -r --arg a "$arch" '.[$a] // ""') | |
| if [ -z "$wf_regex" ]; then | |
| notes="$notes $arch:no-workflow-regex" | |
| continue | |
| fi | |
| wf_total=$(echo "$all_runs" | jq --arg rx "$wf_regex" \ | |
| 'map(select((.path // "") | test($rx))) | length') | |
| if [ "$wf_total" -eq 0 ]; then | |
| notes="$notes $arch:no-workflow" | |
| else | |
| run_archs="$run_archs $arch" | |
| fi | |
| done | |
| RUN_ARCHS=$(echo "$run_archs" | xargs) | |
| NOT_RUN_NOTES=$(echo "$notes" | xargs) | |
| } | |
| # Determine which archs from $1 have at least one ROCm test check-run | |
| # present in ROCM_CHECK_RUNS. Sets READY (space separated) and | |
| # NOT_READY_NOTES (per-arch reasons for the ones still missing shards). | |
| # Sets globals for the same subshell reason as archs_that_ran. | |
| archs_with_shards() { | |
| local run_archs="$1" arch regex total ready="" notes="" | |
| for arch in $run_archs; do | |
| regex=$(echo "$ARCH_JOBNAME_REGEX_MAP" | jq -r --arg a "$arch" '.[$a] // ""') | |
| if [ -z "$regex" ]; then | |
| notes="$notes $arch:no-regex" | |
| continue | |
| fi | |
| total=$(echo "$ROCM_CHECK_RUNS" | jq --arg rx "$regex" \ | |
| 'map(select(.name | test($rx))) | length') | |
| if [ "$total" -eq 0 ]; then | |
| notes="$notes $arch:workflow-run-no-shards-yet" | |
| else | |
| ready="$ready $arch" | |
| fi | |
| done | |
| READY=$(echo "$ready" | xargs) | |
| NOT_READY_NOTES=$(echo "$notes" | xargs) | |
| } | |
| # Evaluate one trunk SHA and dispatch parity.yml when it is ready and | |
| # not already processed. Returns 0 to keep scanning, 1 to stop the scan | |
| # (commit too old, or per-scan dispatch cap reached). | |
| process_commit() { | |
| local sha="$1" date="$2" | |
| local short commit_epoch all_check_runs cuda_check_runs | |
| local total_cr pending_cr pending_sample arch_dispatch | |
| short=$(echo "$sha" | cut -c1-8) | |
| commit_epoch=$(date -u -d "$date" +%s 2>/dev/null || echo 0) | |
| if [ "$commit_epoch" -ne 0 ] && [ "$commit_epoch" -lt "$MAX_AGE_EPOCH" ]; then | |
| echo "[$short] $date too old (>${MAX_AGE_HOURS}h) - stopping scan" | |
| return 1 | |
| fi | |
| if sha_already_dispatched "$sha"; then | |
| echo "[$short] parity report already exists for this SHA - skip" | |
| return 0 | |
| fi | |
| # Sets RUN_ARCHS and NOT_RUN_NOTES. | |
| archs_that_ran "$sha" | |
| if [ -z "$RUN_ARCHS" ]; then | |
| echo "[$short] $date no in-scope ROCm workflows ran on upstream (${NOT_RUN_NOTES:-none}) - skip" | |
| return 0 | |
| fi | |
| # Per-shard check-run state for this SHA (workflow_run conclusion can | |
| # flip to failure before sibling shards finish, so we look per shard). | |
| all_check_runs=$(gh api --paginate \ | |
| "repos/$UPSTREAM/commits/$sha/check-runs?per_page=100" \ | |
| --jq '.check_runs[] | {name,status,conclusion}' \ | |
| 2>/dev/null | jq -s '.' || echo '[]') | |
| ROCM_CHECK_RUNS='[]' | |
| local arch regex arch_check_runs | |
| for arch in $RUN_ARCHS; do | |
| regex=$(echo "$ARCH_JOBNAME_REGEX_MAP" | jq -r --arg a "$arch" '.[$a] // ""') | |
| [ -z "$regex" ] && continue | |
| arch_check_runs=$(echo "$all_check_runs" | select_test_check_runs "$regex") | |
| ROCM_CHECK_RUNS=$(jq -s 'add | unique_by(.name)' \ | |
| <(echo "$ROCM_CHECK_RUNS") <(echo "$arch_check_runs")) | |
| done | |
| cuda_check_runs=$(echo "$all_check_runs" | select_test_check_runs "$CUDA_JOBNAME_REGEX") | |
| if [ "$(echo "$ROCM_CHECK_RUNS" | jq 'length')" -eq 0 ]; then | |
| echo "[$short] $date ROCm workflows ran ($RUN_ARCHS) but no parity check-runs yet - skip" | |
| return 0 | |
| fi | |
| if [ "$(echo "$cuda_check_runs" | jq 'length')" -eq 0 ]; then | |
| echo "[$short] $date no CUDA parity check-runs yet on upstream - skip" | |
| return 0 | |
| fi | |
| # Gate 1: every check-run the report consumes (ROCm shards for arches | |
| # that ran + CUDA tests) must be status=completed - we author the | |
| # SHA's report on dispatch, so dispatching early = partial data. | |
| local gate_check_runs | |
| gate_check_runs=$(jq -s 'add' <(echo "$ROCM_CHECK_RUNS") <(echo "$cuda_check_runs")) | |
| total_cr=$(echo "$gate_check_runs" | jq 'length') | |
| pending_cr=$(echo "$gate_check_runs" | jq 'map(select(.status != "completed")) | length') | |
| if [ "$pending_cr" -ne 0 ]; then | |
| pending_sample=$(echo "$gate_check_runs" | jq -r \ | |
| 'map(select(.status != "completed")) | .[0:3] | map(.name) | join(", ")') | |
| echo "[$short] $date ${pending_cr}/${total_cr} parity check-runs still pending - skip (e.g. $pending_sample)" | |
| return 0 | |
| fi | |
| # Gate 2: every arch workflow that ran must actually have its test | |
| # shards present before we author the SHA's one report. | |
| # Sets READY and NOT_READY_NOTES. | |
| archs_with_shards "$RUN_ARCHS" | |
| if [ -n "$NOT_READY_NOTES" ]; then | |
| echo "[$short] $date ROCm workflows ran ($RUN_ARCHS) but some test shards are missing - skip (${NOT_READY_NOTES})" | |
| return 0 | |
| fi | |
| if [ -z "$READY" ]; then | |
| echo "[$short] $date ROCm workflows ran ($RUN_ARCHS) but no in-scope arches are ready" | |
| return 0 | |
| fi | |
| arch_dispatch=$(echo "$READY" | sed 's/ /, /g') | |
| echo "[$short] READY archs: '$(echo "$READY" | tr ' ' ',')' (committed $date; not-run: ${NOT_RUN_NOTES:-none})" | |
| echo "[$short] dispatching for: '$(echo "$READY" | tr ' ' ',')'" | |
| if [ "$DRY_RUN" = "true" ]; then | |
| echo "[$short] DRY_RUN=true - not dispatching" | |
| else | |
| gh workflow run parity.yml \ | |
| --repo "$GITHUB_REPOSITORY" \ | |
| --ref "$TARGET_REF" \ | |
| -f sha="$sha" \ | |
| -f arch="$arch_dispatch" \ | |
| -f auto_triggered=true | |
| fi | |
| DISPATCHED_COUNT=$((DISPATCHED_COUNT + 1)) | |
| DISPATCHED_SUMMARY="${DISPATCHED_SUMMARY}${short}:${arch_dispatch}"$'\n' | |
| if [ "$DISPATCHED_COUNT" -ge "$MAX_DISPATCHES" ]; then | |
| echo "Reached max dispatches for this scan ($MAX_DISPATCHES); stopping" | |
| return 1 | |
| fi | |
| return 0 | |
| } | |
| write_step_summary() { | |
| local result_line dispatched_list | |
| if [ "$DISPATCHED_COUNT" -gt 0 ]; then | |
| result_line=$([ "$DRY_RUN" = "true" ] \ | |
| && echo "would dispatch $DISPATCHED_COUNT parity run(s) (dry-run)" \ | |
| || echo "dispatched $DISPATCHED_COUNT parity run(s)") | |
| dispatched_list=$(echo "$DISPATCHED_SUMMARY" | sed '/^$/d; s/^/- /') | |
| else | |
| result_line="no ready unprocessed SHAs found" | |
| dispatched_list="" | |
| fi | |
| { | |
| printf '%s\n' \ | |
| "### Parity auto-trigger" \ | |
| "" \ | |
| "- Upstream: \`$UPSTREAM@$BRANCH\`" \ | |
| "- Scope archs: \`$ARCHS\`" \ | |
| "- Max commits: $MAX_COMMITS" \ | |
| "- Max dispatches: $MAX_DISPATCHES" \ | |
| "- Max age: ${MAX_AGE_HOURS}h" \ | |
| "- Target ref: \`$TARGET_REF\`" \ | |
| "- Result: $result_line" | |
| # if (not &&) so a 0-dispatch scan leaves a 0 exit status, not the | |
| # 1 from the empty-string test - which would fail the whole step. | |
| if [ -n "$dispatched_list" ]; then printf '\n%s\n' "$dispatched_list"; fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| } | |
| main() { | |
| NOW_EPOCH=$(date -u +%s) | |
| MAX_AGE_EPOCH=$((NOW_EPOCH - MAX_AGE_HOURS * 3600)) | |
| TARGET_REF="${TARGET_REF_IN:-$GITHUB_REF_NAME}" | |
| ARCHS=$(echo "$ARCHS_IN" | tr ',' ' ' | xargs) | |
| load_matching_config | |
| print_run_config | |
| local commits | |
| commits=$(fetch_trunk_commits) | |
| if [ -z "$commits" ]; then | |
| echo "::warning::No completed trunk.yml push runs returned from $UPSTREAM@$BRANCH" | |
| exit 0 | |
| fi | |
| EXISTING=$(fetch_existing_dispatches) | |
| DISPATCHED_COUNT=0 | |
| DISPATCHED_SUMMARY="" | |
| local sha date | |
| while IFS=' ' read -r sha date; do | |
| [ -z "$sha" ] && continue | |
| # process_commit returns non-zero to stop the scan (too old / cap). | |
| process_commit "$sha" "$date" || break | |
| done <<< "$commits" | |
| write_step_summary | |
| } | |
| main |