Auto Rerun Approved PRs #1146
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto Rerun Approved PRs | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '*/20 * * * *' | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| actions: write | |
| jobs: | |
| scan-open-prs: | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| MAX_RUN_ATTEMPTS: 5 | |
| MAX_FAILED_TASKS: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Scan open PRs and trigger reruns | |
| run: | | |
| set -euo pipefail | |
| source .github/scripts/common.sh | |
| GH_RETRY_CONTEXT="ci-auto-trigger checkpoint lookup" | |
| if ! checkpoint_runs=$(gh_retry run list \ | |
| --workflow .github/workflows/ci-auto-trigger.yaml \ | |
| --status completed \ | |
| --limit 20 \ | |
| --json createdAt,conclusion \ | |
| --jq '[.[] | select(.conclusion == "success")][0:2]'); then | |
| echo "Warning: failed to fetch checkpoint after retries; processing all open PRs" | |
| checkpoint_runs='[]' | |
| fi | |
| GH_RETRY_CONTEXT="" | |
| # Use the checkpoint from the most recent successful ci-auto-trigger run, | |
| # or if there is only one successful run, use its checkpoint. This creates | |
| # a rolling window of PR updates to consider for rerunning, while allowing | |
| # for some flexibility in timing and avoiding reliance on a single run's completion | |
| # time as the cutoff. | |
| latest_checkpoint=$(echo "$checkpoint_runs" | jq -r '.[0].createdAt // empty') | |
| previous_checkpoint=$(echo "$checkpoint_runs" | jq -r '.[1].createdAt // empty') | |
| checkpoint_time="$previous_checkpoint" | |
| if [[ -z "$checkpoint_time" ]]; then | |
| checkpoint_time="$latest_checkpoint" | |
| fi | |
| if [[ -n "$checkpoint_time" ]]; then | |
| if [[ -n "$previous_checkpoint" ]]; then | |
| echo "Using checkpoint from previous successful ci-auto-trigger run (2-run window): $checkpoint_time" | |
| else | |
| echo "Only one successful ci-auto-trigger run found; using its checkpoint: $checkpoint_time" | |
| fi | |
| else | |
| echo "No previous successful ci-auto-trigger run found; processing all open PRs" | |
| fi | |
| # Fetch all open PRs and inspect them for rerun eligibility | |
| prs=$(gh_retry pr list --state open --limit 200 --json number,isDraft,labels,headRefOid,reviews,baseRefName,updatedAt) | |
| echo "Inspecting $(echo "$prs" | jq 'length') open PRs for rerun..." | |
| echo "$prs" | jq -c '.[]' | while read -r pr; do | |
| pr_number=$(echo "$pr" | jq -r '.number') | |
| head_sha=$(echo "$pr" | jq -r '.headRefOid') | |
| pr_updated_at=$(echo "$pr" | jq -r '.updatedAt // empty') | |
| echo | |
| echo "===== PR #$pr_number =====" | |
| echo "PR #$pr_number | Step: checkpoint freshness" | |
| # Skip PRs that haven't been updated since the checkpoint time | |
| if [[ -n "$checkpoint_time" && -n "$pr_updated_at" ]]; then | |
| if [[ "$pr_updated_at" < "$checkpoint_time" || "$pr_updated_at" == "$checkpoint_time" ]]; then | |
| echo "PR #$pr_number | Skip: not updated since checkpoint ($pr_updated_at <= $checkpoint_time)" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| fi | |
| # Check for general PR eligibility (draft status, required labels, reviews, etc.) before doing any more expensive operations | |
| echo "PR #$pr_number | Step: eligibility checks" | |
| if ! pr_is_rerun_eligible "$pr" 2 false; then | |
| echo "PR #$pr_number | Skip: $NOT_RERUN_REASON" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| # Ensure the PR has the auto rerun label, to provide visibility and allow | |
| # manual exclusion from future runs if needed | |
| echo "PR #$pr_number | Step: ensure auto-rerun label" | |
| if ! ensure_auto_rerun_label "$pr_number" "$pr"; then | |
| echo "PR #$pr_number | Skip: $NOT_RERUN_REASON" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| # Find the most recent workflow run for this PR's head SHA, and check if | |
| # it's eligible for rerunning (completed, not successful, not already rerun | |
| # too many times) | |
| if [[ -z "$head_sha" || "$head_sha" == "null" ]]; then | |
| echo "PR #$pr_number | Skip: missing head SHA" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| # To avoid hitting GitHub API rate limits, we need to be careful about how we | |
| # query for workflow runs and jobs. We want to minimize the number of API | |
| # calls, while still reliably determining whether the latest run for this | |
| # PR is eligible for rerunning. | |
| GH_RETRY_CONTEXT="PR #$pr_number run listing for head SHA $head_sha" | |
| if ! run_list_json=$(gh_retry run list \ | |
| --workflow .github/workflows/ci-test.yaml \ | |
| --limit 100 \ | |
| --json databaseId,attempt,status,conclusion,headSha); then | |
| echo "PR #$pr_number | Skip: could not fetch workflow runs for head SHA $head_sha after retries" | |
| GH_RETRY_CONTEXT="" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| GH_RETRY_CONTEXT="" | |
| # Find the most recent run with a matching head SHA | |
| last_run_json=$(echo "$run_list_json" | jq -c --arg head_sha "$head_sha" 'map(select(.headSha == $head_sha)) | first // empty') | |
| if [[ -z "$last_run_json" || "$last_run_json" == "null" ]]; then | |
| echo "PR #$pr_number | Skip: no pull_request run found for head SHA $head_sha" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| # Extract the run ID and attempt number for the most recent run with a matching head SHA | |
| last_run=$(echo "$last_run_json" | jq -r '.databaseId // empty') | |
| run_attempt=$(echo "$last_run_json" | jq -r '.attempt // 0') | |
| if [[ -z "$last_run" ]]; then | |
| echo "PR #$pr_number | Skip: no pull_request run found for head SHA $head_sha" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| # Check if the most recent run is eligible for rerunning | |
| echo "PR #$pr_number | Step: run completion/conclusion checks (run_id=$last_run, attempt=$run_attempt)" | |
| if ! run_is_completed "$last_run_json" "$last_run"; then | |
| echo "PR #$pr_number | Skip: $NOT_RERUN_REASON" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| if [[ "$run_attempt" -ge "$MAX_RUN_ATTEMPTS" ]]; then | |
| echo "PR #$pr_number | Skip: run_id=$last_run already at attempt $run_attempt" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| # Check the jobs for the most recent run to see if there were any completed spread jobs, | |
| # and if so, whether the number of failed tasks on any required spread job exceeds our | |
| # threshold for rerunning | |
| GH_RETRY_CONTEXT="PR #$pr_number job listing for run_id=$last_run" | |
| if ! run_jobs=$(gh_retry run view "$last_run" --json jobs --jq '.jobs'); then | |
| echo "PR #$pr_number | Skip: could not fetch jobs for run_id=$last_run after retries" | |
| GH_RETRY_CONTEXT="" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| GH_RETRY_CONTEXT="" | |
| # If there are no completed spread jobs, the run likely failed before reaching the spread stage. | |
| # In that case, we skip auto-rerun here to avoid rerunning failures outside the spread stage. | |
| # If this policy changes, update the checks below accordingly. | |
| spread_jobs_ran=$(echo "$run_jobs" | jq '[.[] | select(.name | test("^spread ")) | select(.status == "completed")] | length') | |
| if [[ "$spread_jobs_ran" -lt 1 ]]; then | |
| echo "PR #$pr_number | Skip: no completed spread jobs found in run_id=$last_run" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| # Check if any required spread jobs have a number of failed tasks that exceeds our threshold for rerunning | |
| pr_base=$(echo "$pr" | jq -r '.baseRefName // empty') | |
| echo "PR #$pr_number | Step: required spread failure threshold checks" | |
| if ! required_spread_failure_threshold_allows_rerun "$last_run" "$pr_base" "$GH_REPO" "$MAX_FAILED_TASKS"; then | |
| echo "PR #$pr_number | Skip: $NOT_RERUN_REASON" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| echo "PR #$pr_number | Action: triggering rerun workflow using run_id=$last_run (attempt $run_attempt)" | |
| GH_RETRY_CONTEXT="PR #$pr_number trigger rerun workflow for run_id=$last_run" | |
| if ! gh_retry workflow run rerun.yaml -f run_id="$last_run"; then | |
| echo "PR #$pr_number | Skip: failed to trigger rerun workflow after retries" | |
| GH_RETRY_CONTEXT="" | |
| echo "----- End PR #$pr_number -----" | |
| continue | |
| fi | |
| GH_RETRY_CONTEXT="" | |
| echo "----- End PR #$pr_number -----" | |
| done |