diff --git a/.github/workflows/calibration-sweep.yml b/.github/workflows/calibration-sweep.yml new file mode 100644 index 0000000..87257b6 --- /dev/null +++ b/.github/workflows/calibration-sweep.yml @@ -0,0 +1,166 @@ +name: Calibration sweep (weekly) + +# Weekly calibration-feedback runner — reads the week's triage verdicts +# from HarperFast/ai-review-log and opens ONE calibration PR against this +# repo per week (CALIBRATION.md entry + conservative layer edits). The +# agent's instructions are rubric-as-code in CALIBRATION-SWEEP.md at the +# repo root — edit THAT file to change sweep behavior, not the prompt +# below. +# +# Successor to the claude.ai "weekly calibration sweep" routine (disabled +# 2026-07 — same platform breakage that killed the nightly triage +# routine; ai-review-log's triage-assist.yml is the sibling migration). +# +# Design notes: +# * Data is PRE-FETCHED in a deterministic bash step using +# AI_REVIEW_LOG_TOKEN (this repo's checkout token can't read the +# private log repo). The agent itself holds only `github.token`, +# scoped to this repo by the job's permissions block — it can't +# touch ai-review-log at all. +# * The agent creates the branch + PR with `github.token`, so the +# calibration PR does NOT retrigger this repo's own PR workflows +# (GitHub suppresses workflow-triggering for github.token events). +# The gemini dogfood review therefore doesn't fire on these PRs — +# apply the `gemini-review` label manually if a bot pass is wanted +# before merge. +# * Cron fires Monday 01:00 UTC = Sunday evening PT, matching the old +# routine. The WEEK computed is the (PT) week just ending. A manual +# dispatch can override `week` to back-fill a missed week. +# +# Prerequisites: +# - ANTHROPIC_API_KEY (repo secret) +# - AI_REVIEW_LOG_TOKEN (repo secret — issues:read on ai-review-log) + +on: + schedule: + - cron: '23 1 * * 1' + workflow_dispatch: + inputs: + week: + description: >- + ISO Monday (YYYY-MM-DD) of the week to sweep. Leave empty to + compute the current PT week — set explicitly to back-fill a + missed week. + required: false + type: string + +concurrency: + group: calibration-sweep + cancel-in-progress: false + +jobs: + sweep: + runs-on: ubuntu-latest + timeout-minutes: 30 + # Deliberately NO `id-token: write`: with an explicit `github_token` + # input, claude-code-action skips the OIDC/App token exchange + # entirely, so the OIDC mint permission is dead weight. Proven in + # production by ai-review-log's triage-assist.yml — the identical + # shape (api-key auth + explicit github_token, no id-token) runs + # green on its nightly schedule. Reviewers keep flagging this from + # v1.0 priors; see the resolved threads on #75 before re-adding. + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout ai-review-prompts (main) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: main + # Full history so the agent can branch/rebase against origin + # state; checkout persists github.token credentials, which is + # what authenticates the agent's `git push`. + fetch-depth: 0 + + - name: Pre-fetch the week's verdicts from ai-review-log + env: + GH_TOKEN: ${{ secrets.AI_REVIEW_LOG_TOKEN }} + WEEK_OVERRIDE: ${{ inputs.week }} + run: | + set -euo pipefail + if [ -n "${WEEK_OVERRIDE}" ]; then + WEEK="${WEEK_OVERRIDE}" + else + # Monday of the current week in America/Los_Angeles. At the + # scheduled fire time (Sun evening PT) this is the Monday six + # days back — the week just ending. + WEEK=$(TZ=America/Los_Angeles date -d "-$(( ($(TZ=America/Los_Angeles date +%u) + 6) % 7 )) days" +%F) + fi + # Exclusive upper bound (next Monday). Without it a back-fill + # run would sweep in every verdict closed AFTER the target + # week too, double-counting when the following week runs. + WEEK_END=$(date -d "${WEEK} + 7 days" +%F) + echo "WEEK=$WEEK" >> "$GITHUB_ENV" + D="$RUNNER_TEMP/calib" + mkdir -p "$D" + echo "CALIB_DATA=$D" >> "$GITHUB_ENV" + + # Verdicts applied this week. `since` filters on updated_at + # (server-side, cheap); the jq filter narrows to closed_at + # within the week so re-touched older issues don't leak in. + # `--paginate` emits one JSON array PER PAGE, so slurp (-s) + # and flatten to keep each output file a single array. + for v in useful noise partial; do + gh api --paginate \ + "repos/HarperFast/ai-review-log/issues?state=closed&labels=verdict:$v&since=${WEEK}T00:00:00Z&per_page=100" \ + | jq -s --arg w "${WEEK}T00:00:00Z" --arg w_end "${WEEK_END}T00:00:00Z" \ + '[.[][] | select(.closed_at >= $w and .closed_at < $w_end) | {number, title, closed_at, body, labels: [.labels[].name]}]' \ + > "$D/$v.json" + done + + # Triage rationale for the interesting classes. + for v in noise partial; do + for n in $(jq -r '.[].number' "$D/$v.json"); do + gh api "repos/HarperFast/ai-review-log/issues/$n/comments" \ + --jq '[.[] | {user: .user.login, body}]' \ + > "$D/comments-$n.json" \ + || { echo "::warning::comments fetch failed for ai-review-log#$n — rationale gap"; echo '[]' > "$D/comments-$n.json"; } + done + done + + # Curated weekly issues (legacy supplements) — best-effort. + for pair in "calibration:calibration-log" "false-negatives:false-negative-log"; do + label="${pair%%:*}"; out="${pair##*:}" + gh api --paginate \ + "repos/HarperFast/ai-review-log/issues?state=all&labels=$label&per_page=100" \ + | jq -s --arg w "$WEEK" '[.[][] | select(.title | contains($w)) | {number, title, body}]' \ + > "$D/$out.json" \ + || { echo "::warning::curated '$label' fetch failed — continuing without it"; echo '[]' > "$D/$out.json"; } + done + + # How much of the week is still untriaged (signal completeness). + gh api --paginate \ + "repos/HarperFast/ai-review-log/issues?state=open&labels=verdict:pending&per_page=100" \ + | jq -s 'map(length) | add // 0' > "$D/pending-count.txt" \ + || { echo "::warning::pending-count fetch failed — recording 0"; echo 0 > "$D/pending-count.txt"; } + + for f in "$D"/useful.json "$D"/noise.json "$D"/partial.json; do + [ "$(jq 'length' "$f")" -gt 0 ] || echo "::warning::$(basename "$f") is empty for week $WEEK" + done + + echo "Pre-fetched for week $WEEK:" + for f in "$D"/*.json; do printf ' %s: %s items\n' "$(basename "$f")" "$(jq 'length' "$f")"; done + + - name: Run weekly sweep per CALIBRATION-SWEEP.md + uses: anthropics/claude-code-action@ef50f123a3a9be95b60040d042717517407c7256 # v1.0.110 + env: + GH_TOKEN: ${{ github.token }} + WEEK: ${{ env.WEEK }} + CALIB_DATA: ${{ env.CALIB_DATA }} + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + # Explicit token skips the GitHub-App token exchange (the + # Claude Code app isn't a prerequisite this way) — same + # pattern as ai-review-log's triage-assist.yml. + github_token: ${{ github.token }} + prompt: | + Read CALIBRATION-SWEEP.md at the root of this checkout and + execute it exactly. The checkout is HarperFast/ai-review-prompts + on main. Pre-fetched data for week $WEEK is in $CALIB_DATA + (layout documented in CALIBRATION-SWEEP.md). Configure git + author as "calibration-sweep[bot] " before + committing. + claude_args: | + --model claude-opus-4-8 + --max-turns 200 + --allowedTools "Read,Grep,Glob,Edit,Write,Bash(date:*),Bash(jq:*),Bash(ls:*),Bash(git status:*),Bash(git log:*),Bash(git diff:*),Bash(git config:*),Bash(git checkout:*),Bash(git add:*),Bash(git commit:*),Bash(git push:*),Bash(git branch:*),Bash(git fetch:*),Bash(git ls-remote:*),Bash(git rev-parse:*),Bash(git show:*),Bash(gh pr list:*),Bash(gh pr view:*),Bash(gh pr create:*),Bash(gh pr edit:*)" diff --git a/CALIBRATION-SWEEP.md b/CALIBRATION-SWEEP.md new file mode 100644 index 0000000..8f7ae67 --- /dev/null +++ b/CALIBRATION-SWEEP.md @@ -0,0 +1,131 @@ +# Weekly calibration sweep — runner instructions + +You are the weekly calibration-feedback routine for Harper's AI code review. +Once a week you read the accumulated calibration signal from +HarperFast/ai-review-log and open a PULL REQUEST against this repo +(HarperFast/ai-review-prompts) proposing review-prompt refinements. + +This runs UNATTENDED in GitHub Actions (`calibration-sweep.yml`) — there is +no human in the loop and no follow-up. NEVER ask questions; if a step is +blocked, degrade gracefully and still open the PR with whatever you have. +You ALWAYS end with exactly ONE open PR for the week (step 4). NEVER merge +and NEVER enable auto-merge — a human reviews and merges. + +Successor to the claude.ai "weekly calibration sweep" routine (disabled +2026-07; see ai-review-log's TRIAGE.md header for the same migration story +on the nightly triage side). Two deliberate changes from that routine: + +* **Raw labels are the primary source.** The old routine read curated + daily-summary comments maintained by two other routines (rolling + calibration log, false-negative scan); both have stalled. The workflow + now pre-fetches the week's verdict-labeled issues directly (layout + below). The curated `Calibration log — week of ` / + `False-negative log — week of ` issues are SUPPLEMENTS: read them + if the pre-fetch captured them, but never block on their absence. +* **Slice by model and prompt ref.** Every log entry records `**Model:**` + and `**Prompt ref:**`. The synthesis must break the verdict mix down by + both — this is what powers model canaries (e.g. the harper + claude-sonnet-5 canary vs the claude-sonnet-4-6 fleet) and + before/after reads on prompt changes. + +## Inputs + +The workflow pre-fetches everything from ai-review-log before you start +(you have NO token for ai-review-log — do not try to read it directly). +The env var `CALIB_DATA` points at a directory containing: + +* `useful.json`, `noise.json`, `partial.json` — arrays of the issues whose + verdict was applied this week (`{number, title, closed_at, body, + labels}`). Bodies carry the `**Repo:**` / `**PR:**` / `**Model:**` / + `**Prompt ref:**` fields. +* `comments-.json` — all comments for each noise/partial issue (the + triage rationale lives here). +* `calibration-log.json`, `false-negative-log.json` — the curated weekly + issues (body + comments) when they exist; may be empty arrays. +* `pending-count.txt` — how many `verdict:pending` issues remain open + (context for how complete the week's signal is). + +The env var `WEEK` is the ISO Monday (America/Los_Angeles) of the week +being summarized. + +## 1. Synthesize the week + +Aggregate from the pre-fetched data: + +* Verdict mix (useful / noise / partial counts) — overall, **per model**, + and **per prompt ref** (short SHA). Small per-model cells are fine; + report the counts and resist conclusions below ~15 entries per cell. +* Recurring NOISE patterns — what is the review over-flagging that triage + repeatedly marks noise? (→ candidate prompt text to suppress) +* Recurring PARTIAL / FALSE-NEGATIVE patterns — severity inflation, + severity deflation, and classes of real issues the review missed. + (→ candidate prompt text to add) +* Weight HUMAN-corroborated / author-fixed findings higher than bot-only + (e.g. gemini-only) signal; call out which is which. + +## 2. Decide prompt changes (CONSERVATIVE) + +The review layers live in this checkout: `universal.md`, `harper/*.md`, +`repo-type/*.md` (glob them — the set grows over time). Read the relevant +ones. Propose edits ONLY where a pattern is clear and recurring (≥ 2 +independent data points, ideally human-corroborated). Prefer small, +targeted additions. If the week's signal is thin or ambiguous, propose NO +prompt-file edits — a log-only week is expected and fine. + +Check open PRs first (`gh pr list`): if a prior calibration PR is still +open and unmerged, do not duplicate or contradict its edits — note the +overlap in your entry instead. + +## 3. Update CALIBRATION.md + +Prepend a dated `## Week of ` section to `CALIBRATION.md` at the +repo root (create the file if missing): verdict mix including the +per-model / per-prompt-ref table, noise + partial/false-negative patterns +(with links to the source ai-review-log issues), and either the prompt +edits made or an explicit `No prompt changes this week — `. +If a `## Week of ` section already exists (a re-run or back-fill +of the same week), REPLACE that section in place — never leave two +entries for the same week. + +## 4. ALWAYS end with exactly one open PR for (idempotent) + +First check whether an OPEN PR for this week already exists: head branch +`calibration/week-of-`, or an open PR whose body contains the marker +`` and title `calibration: week of `. +If it exists, REUSE its branch (push further commits) instead of opening a +second PR — re-runs must not create duplicates. + +a. Create or resume the branch — never force-reset it. Probe with + `git ls-remote --heads origin calibration/week-of-` (empty + output = branch doesn't exist; unlike `git fetch` of a missing ref, + this doesn't exit non-zero). If it exists: + `git fetch origin calibration/week-of-` then + `git checkout -b calibration/week-of- + origin/calibration/week-of-` (resume the existing work); + else `git checkout -b calibration/week-of-` off the current + checkout (which the workflow puts on latest `main`). Do not use + `-B` — it would reset an existing branch to `main` and discard the + prior run's commits. +b. Commit the `CALIBRATION.md` update ALWAYS, plus any prompt-file edits + from step 2. Commit message: `calibration: week of `. +c. `git push -u origin calibration/week-of-`. +d. If no PR exists yet: `gh pr create` into `main` titled + `calibration: week of `. The PR DESCRIPTION must contain the full + step-1 synthesis (verdict mix incl. per-model/per-ref, patterns, links + to source issues) and a clear changelist (prompt edits, or `log-only — + no prompt changes this week`). End the description with + `` followed by + `🤖 Generated with [Claude Code](https://claude.com/claude-code)`. +e. Do NOT merge and do NOT enable auto-merge. + +## Constraints + +* Exactly one open PR per week; never merge it, never auto-merge. +* Conservative on prompt edits — log-only weeks are expected and fine. +* Never ask questions; if any read/write fails, proceed with what you + have and STILL open (or update) the PR. +* Only modify `CALIBRATION.md` and the layer files (`universal.md`, + `harper/*.md`, `repo-type/*.md`). Never touch `.github/`, scripts, or + any other files, and never write outside this repo. +* If `CALIB_DATA` is missing or empty, still open the PR with a + `no calibration data captured this week` entry.