ci-auto-trigger.yaml

name: Auto Rerun Approved PRs

on:
  workflow_dispatch:
  schedule:
    - cron: '*/20 * * * *'

permissions:
  contents: read
  pull-requests: write
  actions: write

jobs:
  scan-open-prs:
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ github.token }}
      GH_REPO: ${{ github.repository }}
      MAX_RUN_ATTEMPTS: 5
      MAX_FAILED_TASKS: 30
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Scan open PRs and trigger reruns
        run: |
          set -euo pipefail

          source .github/scripts/common.sh

          GH_RETRY_CONTEXT="ci-auto-trigger checkpoint lookup"
          if ! checkpoint_runs=$(gh_retry run list \
            --workflow .github/workflows/ci-auto-trigger.yaml \
            --status completed \
            --limit 20 \
            --json createdAt,conclusion \
            --jq '[.[] | select(.conclusion == "success")][0:2]'); then
            echo "Warning: failed to fetch checkpoint after retries; processing all open PRs"
            checkpoint_runs='[]'
          fi
          GH_RETRY_CONTEXT=""

          # Use the checkpoint from the most recent successful ci-auto-trigger run,
          # or if there is only one successful run, use its checkpoint. This creates
          # a rolling window of PR updates to consider for rerunning, while allowing
          # for some flexibility in timing and avoiding reliance on a single run's completion
          # time as the cutoff. 
          latest_checkpoint=$(echo "$checkpoint_runs" | jq -r '.[0].createdAt // empty')
          previous_checkpoint=$(echo "$checkpoint_runs" | jq -r '.[1].createdAt // empty')

          checkpoint_time="$previous_checkpoint"
          if [[ -z "$checkpoint_time" ]]; then
            checkpoint_time="$latest_checkpoint"
          fi

          if [[ -n "$checkpoint_time" ]]; then
            if [[ -n "$previous_checkpoint" ]]; then
              echo "Using checkpoint from previous successful ci-auto-trigger run (2-run window): $checkpoint_time"
            else
              echo "Only one successful ci-auto-trigger run found; using its checkpoint: $checkpoint_time"
            fi
          else
            echo "No previous successful ci-auto-trigger run found; processing all open PRs"
          fi

          # Fetch all open PRs and inspect them for rerun eligibility
          prs=$(gh_retry pr list --state open --limit 200 --json number,isDraft,labels,headRefOid,reviews,baseRefName,updatedAt)
          echo "Inspecting $(echo "$prs" | jq 'length') open PRs for rerun..."

          echo "$prs" | jq -c '.[]' | while read -r pr; do
            pr_number=$(echo "$pr" | jq -r '.number')
            head_sha=$(echo "$pr" | jq -r '.headRefOid')
            pr_updated_at=$(echo "$pr" | jq -r '.updatedAt // empty')

            echo
            echo "===== PR #$pr_number ====="
            echo "PR #$pr_number | Step: checkpoint freshness"

            # Skip PRs that haven't been updated since the checkpoint time
            if [[ -n "$checkpoint_time" && -n "$pr_updated_at" ]]; then
              if [[ "$pr_updated_at" < "$checkpoint_time" || "$pr_updated_at" == "$checkpoint_time" ]]; then
                echo "PR #$pr_number | Skip: not updated since checkpoint ($pr_updated_at <= $checkpoint_time)"
                echo "----- End PR #$pr_number -----"
                continue
              fi
            fi

            # Check for general PR eligibility (draft status, required labels, reviews, etc.) before doing any more expensive operations
            echo "PR #$pr_number | Step: eligibility checks"
            if ! pr_is_rerun_eligible "$pr" 2 false; then
              echo "PR #$pr_number | Skip: $NOT_RERUN_REASON"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            # Ensure the PR has the auto rerun label, to provide visibility and allow
            # manual exclusion from future runs if needed
            echo "PR #$pr_number | Step: ensure auto-rerun label"
            if ! ensure_auto_rerun_label "$pr_number" "$pr"; then
               echo "PR #$pr_number | Skip: $NOT_RERUN_REASON"
               echo "----- End PR #$pr_number -----"
               continue
             fi

            # Find the most recent workflow run for this PR's head SHA, and check if
            # it's eligible for rerunning (completed, not successful, not already rerun
            # too many times)
            if [[ -z "$head_sha" || "$head_sha" == "null" ]]; then
              echo "PR #$pr_number | Skip: missing head SHA"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            # To avoid hitting GitHub API rate limits, we need to be careful about how we
            # query for workflow runs and jobs. We want to minimize the number of API
            # calls, while still reliably determining whether the latest run for this
            # PR is eligible for rerunning.
            GH_RETRY_CONTEXT="PR #$pr_number run listing for head SHA $head_sha"
            if ! run_list_json=$(gh_retry run list \
              --workflow .github/workflows/ci-test.yaml \
              --limit 100 \
              --json databaseId,attempt,status,conclusion,headSha); then
              echo "PR #$pr_number | Skip: could not fetch workflow runs for head SHA $head_sha after retries"
              GH_RETRY_CONTEXT=""
              echo "----- End PR #$pr_number -----"
              continue
            fi
            GH_RETRY_CONTEXT=""

            # Find the most recent run with a matching head SHA
            last_run_json=$(echo "$run_list_json" | jq -c --arg head_sha "$head_sha" 'map(select(.headSha == $head_sha)) | first // empty')

            if [[ -z "$last_run_json" || "$last_run_json" == "null" ]]; then
              echo "PR #$pr_number | Skip: no pull_request run found for head SHA $head_sha"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            # Extract the run ID and attempt number for the most recent run with a matching head SHA
            last_run=$(echo "$last_run_json" | jq -r '.databaseId // empty')
            run_attempt=$(echo "$last_run_json" | jq -r '.attempt // 0')

            if [[ -z "$last_run" ]]; then
              echo "PR #$pr_number | Skip: no pull_request run found for head SHA $head_sha"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            # Check if the most recent run is eligible for rerunning
            echo "PR #$pr_number | Step: run completion/conclusion checks (run_id=$last_run, attempt=$run_attempt)"
            if ! run_is_completed "$last_run_json" "$last_run"; then
              echo "PR #$pr_number | Skip: $NOT_RERUN_REASON"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            if [[ "$run_attempt" -ge "$MAX_RUN_ATTEMPTS" ]]; then
              echo "PR #$pr_number | Skip: run_id=$last_run already at attempt $run_attempt"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            # Check the jobs for the most recent run to see if there were any completed spread jobs,
            # and if so, whether the number of failed tasks on any required spread job exceeds our
            # threshold for rerunning
            GH_RETRY_CONTEXT="PR #$pr_number job listing for run_id=$last_run"
            if ! run_jobs=$(gh_retry run view "$last_run" --json jobs --jq '.jobs'); then
              echo "PR #$pr_number | Skip: could not fetch jobs for run_id=$last_run after retries"
              GH_RETRY_CONTEXT=""
              echo "----- End PR #$pr_number -----"
              continue
            fi
            GH_RETRY_CONTEXT=""

            # If there are no completed spread jobs, the run likely failed before reaching the spread stage.
            # In that case, we skip auto-rerun here to avoid rerunning failures outside the spread stage.
            # If this policy changes, update the checks below accordingly.
            spread_jobs_ran=$(echo "$run_jobs" | jq '[.[] | select(.name | test("^spread ")) | select(.status == "completed")] | length')
            if [[ "$spread_jobs_ran" -lt 1 ]]; then
              echo "PR #$pr_number | Skip: no completed spread jobs found in run_id=$last_run"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            # Check if any required spread jobs have a number of failed tasks that exceeds our threshold for rerunning
            pr_base=$(echo "$pr" | jq -r '.baseRefName // empty')
            echo "PR #$pr_number | Step: required spread failure threshold checks"
            if ! required_spread_failure_threshold_allows_rerun "$last_run" "$pr_base" "$GH_REPO" "$MAX_FAILED_TASKS"; then
              echo "PR #$pr_number | Skip: $NOT_RERUN_REASON"
              echo "----- End PR #$pr_number -----"
              continue
            fi

            echo "PR #$pr_number | Action: triggering rerun workflow using run_id=$last_run (attempt $run_attempt)"
            GH_RETRY_CONTEXT="PR #$pr_number trigger rerun workflow for run_id=$last_run"
            if ! gh_retry workflow run rerun.yaml -f run_id="$last_run"; then
              echo "PR #$pr_number | Skip: failed to trigger rerun workflow after retries"
              GH_RETRY_CONTEXT=""
              echo "----- End PR #$pr_number -----"
              continue
            fi
            GH_RETRY_CONTEXT=""
            echo "----- End PR #$pr_number -----"
          done