eval.yml

name: eval

# EVAL-1..6: the eval loop as a PR check.
#
# Split from the build workflow on purpose. `build` gates hard: validation
# errors and uncommitted write-back fail the PR. This one is REPORT-ONLY to
# start (continue-on-error), because the thing under test is a model and a
# freshly-wired suite that blocks merges on a stochastic run would be turned off
# within a week. Promote to blocking once the pass rate is green and stable —
# that is a one-line change: drop `continue-on-error` from the eval step.
#
# The deterministic checks that used to run here as their own job — every case
# parses, every gold_query still runs, no committed gold value has drifted —
# now run inside `npm run build` (SIMP-4), which already blocks the PR. They
# are validation, and validation belongs in the build rather than in a second
# gate that has to be remembered. What remains below is a pre-flight of the
# same check in this job: it costs seconds and stops a broken case from
# spending a whole sweep before the build workflow has gone red.
#
# Data: the parquet files under ParquetFiles/ are committed, so CI gets
# byte-identical fixtures on every run and gold values are stable (EVAL-4).
# Live-data drift runs are scheduled separately and never run on a PR.

on:
  pull_request:
    # Docs-only PRs don't touch anything a case fingerprint depends on, so the
    # sweep would spin up only to skip every case. Skip the whole job instead.
    # NOTE: keep this to TRUE docs. Concept files, CLAUDE.md and
    # kp/agent/examples.md are .md too and DO move verdicts — never add `**.md`.
    # A PR that touches any non-ignored path still runs the full sweep.
    paths-ignore:
      - 'ROADMAP.md'
      - 'README.md'
      - 'LICENSE'
      - 'docs/**'
  push:
    branches: [main]
  schedule:
    # drift run: same cases, live data, Mondays 06:00 UTC
    - cron: '0 6 * * 1'
  workflow_dispatch:

jobs:
  eval:
    name: agent eval sweep (report-only)
    runs-on: ubuntu-latest
    # The agent under test needs credentials. Skip rather than fail red on forks
    # and on any repo where the secret is not configured.
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0   # semantic identity records git tree shas
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: npm
      - run: npm ci

      # Pre-flight: cheap, deterministic, and it fails before any spend.
      - name: Verify every case + committed gold value
        run: npm run eval:check

      - name: Install the agent under test
        run: npm install -g @anthropic-ai/claude-code

      # EVAL-12b: impact selection skips a case whose fingerprint already has a
      # clean measurement, and a fresh checkout has no measurements at all. This
      # pulls the results of the last successful run on main so a PR has a
      # ledger to consult. Best effort on purpose: if the artifact is missing,
      # expired, or the download fails, the ledger is empty and `--select`
      # degrades to a full sweep — the safe direction, and the same thing that
      # happens today.
      - name: Fetch the results ledger from main
        if: github.event_name == 'pull_request'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          run_id=$(gh run list --workflow eval.yml --branch main --status success \
                     --limit 1 --json databaseId --jq '.[0].databaseId')
          [ -n "$run_id" ] || exit 0
          gh run download "$run_id" -n "eval-results-$run_id" -D evals/results

      # --select on PRs only. A push to main, a schedule and a manual dispatch
      # all run the full sweep: something has to keep re-measuring the cases a
      # PR was allowed to skip, and the fingerprints are only as good as the
      # sweep that last confirmed them.
      - name: Run evals against fixtures
        id: sweep
        continue-on-error: true          # <- remove to make the suite blocking
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          extra=''
          if [ "${{ github.event_name }}" = "pull_request" ]; then extra='--select'; fi
          npm run eval -- --runs 3 --concurrency 4 $extra

      # EVAL-12's invariant: a cheap tier is only trustworthy while it keeps
      # agreeing with the expensive tier it replaces. This runs each tier-1 case
      # in BOTH lanes and compares the verdicts, so a cheap lane that has
      # drifted into passing what the real agent fails is caught here rather
      # than by nobody. It costs 2x the runs of the cases it covers, so it rides
      # the schedule and manual dispatch, never a PR.
      - name: Correlation check — does the cheap lane still agree?
        id: correlate
        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
        continue-on-error: true          # <- remove together with the sweep's
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: npm run eval:correlate -- --runs 3

      - name: Report pass rate and flips vs the previous run
        if: always()
        run: npm run eval:report || true

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results-${{ github.run_id }}
          path: |
            evals/results/*.jsonl
            evals/results/correlation/**
          retention-days: 90

      - name: Summarise
        if: always()
        run: |
          {
            echo "## Eval sweep"
            echo ''
            if [ "${{ steps.sweep.outcome }}" = "success" ]; then
              echo 'All cases passed.'
            else
              echo 'Some cases failed — report-only, not blocking this PR.'
            fi
            echo ''
            case "${{ steps.correlate.outcome }}" in
              success) echo 'Correlation: ESTABLISHED — every tier-1 case was confirmed by a tier-2 run.' ;;
              failure) echo 'Correlation: NOT established — the cheap lane disagrees with the expensive one. See the correlate step.' ;;
              *)       echo 'Correlation check not run (schedule / manual dispatch only).' ;;
            esac
            echo ''
            echo '```'
            npm run eval:report 2>&1 | tail -40 || true
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"