benchmarks-report.yml

name: Report Benchmark Results

on:
  workflow_run:
    workflows: [Benchmarks]
    branches: ['**']
    types:
      - completed

permissions:
  contents: read

jobs:
  # PR path: render comment, post / edit in place. Runs on all successful
  # bench runs whose triggering event was `pull_request`.
  comment:
    name: Post PR bench comment
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    if: >
      github.event.workflow_run.conclusion == 'success'
      && github.event.workflow_run.event == 'pull_request'
    steps:
      # Generate an installation access token for the Semantic Performance
      # Bot GitHub App. Comments post under the bot's identity + avatar
      # rather than the generic `github-actions[bot]`. Same permissions
      # surface as GITHUB_TOKEN; branded, not widened.
      - name: Generate bot token
        id: bot-token
        uses: actions/create-github-app-token@v3
        with:
          client-id: ${{ secrets.SEMANTIC_PERF_BOT_CLIENT_ID }}
          private-key: ${{ secrets.SEMANTIC_PERF_BOT_PRIVATE_KEY }}

      # check out main: tooling comes from main and the reported sha comes from the event
      - uses: actions/checkout@v7
        with:
          ref: main

      - uses: actions/setup-node@v7
        with:
          node-version-file: '.node-version'

      # Pin reporter to main. The PR cannot author the tool that renders
      # its own bench comment — classification thresholds, output format,
      # and peak attribution must be author-neutral. Overlaid before any
      # downstream step that reads tools/ci/bench/reporter/.
      - name: Overlay reporter from main
        run: |
          git fetch origin main --depth=1
          git checkout origin/main -- tools/ci/bench/reporter/ 2>/dev/null || true

      - name: Download bench artifacts
        uses: dawidd6/action-download-artifact@v21
        with:
          github_token: ${{ steps.bot-token.outputs.token }}
          workflow: ${{ github.event.workflow.id }}
          run_id: ${{ github.event.workflow_run.id }}
          name_is_regexp: true
          name: results-.*
          # download to a scratch dir, not the workspace, since these artifacts are PR-built
          path: ${{ runner.temp }}/bench-results
          allow_forks: true

      - name: Resolve PR number
        id: pr
        env:
          GH_TOKEN: ${{ steps.bot-token.outputs.token }}
          HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
        run: |
          NUMBER='${{ github.event.workflow_run.pull_requests[0].number }}'
          if [ -z "$NUMBER" ] || [ "$NUMBER" = "null" ]; then
            NUMBER=$(gh pr list --repo '${{ github.repository }}' \
              --head "$HEAD_BRANCH" --state open --json number --jq '.[0].number // ""')
          fi
          echo "number=$NUMBER" >> "$GITHUB_OUTPUT"
          echo "Resolved PR number: $NUMBER"

      # Walk prior successful bench runs on this PR's branch and build a
      # per-iteration history. The reporter merges this with bench-history.json
      # (main commits) so peak attribution spans BOTH main AND this PR's
      # iterations — an agent sees "iteration 3 was the best on update-10th;
      # your current iteration regressed from that."
      - name: Fetch PR iteration history
        env:
          GH_TOKEN: ${{ steps.bot-token.outputs.token }}
          HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
        run: |
          node tools/ci/bench/reporter/fetch-pr-history.js \
            --branch "$HEAD_BRANCH" \
            --repo '${{ github.repository }}' \
            --current-run-id '${{ github.event.workflow_run.id }}' \
            --out pr-history.json

      - name: Generate report
        env:
          STARTED: ${{ github.event.workflow_run.created_at }}
          ENDED: ${{ github.event.workflow_run.updated_at }}
          DISPLAY_TITLE: ${{ github.event.workflow_run.display_title }}
        run: |
          WALL_CLOCK=$(( $(date -d "$ENDED" +%s) - $(date -d "$STARTED" +%s) ))
          # --scope pr: peak attribution uses PR-iteration history only.
          # main-history is still loaded for drift quantification but excluded
          # from the comparison set.
          node tools/ci/bench/reporter/reporter.js \
            --results ${{ runner.temp }}/bench-results \
            --sha '${{ github.event.workflow_run.head_sha }}' \
            --msg "$DISPLAY_TITLE" \
            --run-url '${{ github.event.workflow_run.html_url }}' \
            --run-id '${{ github.event.workflow_run.id }}' \
            --base-ref 'main' \
            --repo '${{ github.repository }}' \
            --pr-history pr-history.json \
            --scope pr \
            --wall-clock "$WALL_CLOCK" \
            --out bench-report

      - name: Upload bench-report.json adjunct
        uses: actions/upload-artifact@v7
        with:
          name: bench-report
          path: bench-report/bench-report.json

      - name: Post or update PR comment
        if: steps.pr.outputs.number != ''
        uses: ./.github/actions/sticky-bot-comment
        with:
          token: ${{ steps.bot-token.outputs.token }}
          repo: ${{ github.repository }}
          pr-number: ${{ steps.pr.outputs.number }}
          bot-login: semantic-performance-bot[bot]
          body-file: bench-report/comment.md

  # History path: append this commit's absolute CIs to bench-history.json
  # and commit back to main. Runs on successful bench runs triggered by a
  # push to main. Serialized by concurrency group so two near-simultaneous
  # merges don't race on the file.
  history:
    name: Append bench history
    runs-on: ubuntu-latest
    permissions:
      contents: write
    if: >
      github.event.workflow_run.conclusion == 'success'
      && github.event.workflow_run.event == 'push'
    concurrency:
      group: bench-history-append
      cancel-in-progress: false
    steps:
      # Bot token for the archival commit — shows up on main as authored
      # by Semantic Performance Bot rather than github-actions[bot].
      - name: Generate bot token
        id: bot-token
        uses: actions/create-github-app-token@v3
        with:
          client-id: ${{ secrets.SEMANTIC_PERF_BOT_CLIENT_ID }}
          private-key: ${{ secrets.SEMANTIC_PERF_BOT_PRIVATE_KEY }}

      # Check out main's tip for the commit target. We push back to main,
      # so we need the most recent state — not the bench's head_sha (which
      # may have been superseded by a later merge during the ~10-min bench).
      - uses: actions/checkout@v7
        with:
          ref: main
          fetch-depth: 2  # need HEAD~1 for parent_sha lookup
          token: ${{ steps.bot-token.outputs.token }}

      - uses: actions/setup-node@v7
        with:
          node-version-file: '.node-version'

      - name: Download bench artifacts
        uses: dawidd6/action-download-artifact@v21
        with:
          github_token: ${{ steps.bot-token.outputs.token }}
          workflow: ${{ github.event.workflow.id }}
          run_id: ${{ github.event.workflow_run.id }}
          name_is_regexp: true
          name: results-.*
          # download to a scratch dir, not the workspace, since these artifacts are PR-built
          path: ${{ runner.temp }}/bench-results
          allow_forks: true

      - name: Append history entry
        env:
          # squash subjects come from PR titles, so this is PR input even on push
          DISPLAY_TITLE: ${{ github.event.workflow_run.display_title }}
        run: |
          # Benched commit is the workflow_run's head_sha; parent comes from
          # the depth-2 fetch above. Timestamp is the bench run's completion
          # so history is ordered by measurement, not commit-land time.
          # baseline-sha.txt is the sidecar uploaded next to each matrix
          # cell's tachometer JSON. Any cell's value works — all cells in
          # one workflow_run benched against the same baseline.
          BENCHED_SHA='${{ github.event.workflow_run.head_sha }}'
          PARENT_SHA=$(git rev-parse "$BENCHED_SHA^" 2>/dev/null || echo '')
          BASELINE_SHA=$(find "${{ runner.temp }}/bench-results" -name baseline-sha.txt -type f -exec cat {} \; -quit)
          node tools/ci/bench/reporter/append-history.js \
            --results ${{ runner.temp }}/bench-results \
            --sha "$BENCHED_SHA" \
            --msg "$DISPLAY_TITLE" \
            --parent-sha "$PARENT_SHA" \
            --baseline-sha "$BASELINE_SHA" \
            --timestamp '${{ github.event.workflow_run.updated_at }}' \
            --history tools/ci/bench/reporter/bench-history.json

      - name: Commit + push
        env:
          # The checkout step's token persists as the git remote's credential
          # helper, so the push below authenticates as the bot automatically.
          BOT_APP_SLUG: semantic-performance-bot
        run: |
          # Use the bot's noreply email format. The exact local-part is the
          # app's numeric user id; GitHub renders the commit as authored by
          # the app regardless, as long as the name matches the app slug.
          git config user.name 'semantic-performance-bot[bot]'
          git config user.email '${{ secrets.SEMANTIC_PERF_BOT_APP_ID }}+semantic-performance-bot[bot]@users.noreply.github.com'
          # If nothing changed (e.g. a re-run for a SHA already archived with
          # identical numbers), skip rather than making an empty commit.
          if git diff --quiet tools/ci/bench/reporter/bench-history.json; then
            echo 'No change to bench-history.json; skipping commit.'
            exit 0
          fi
          BENCHED_SHA='${{ github.event.workflow_run.head_sha }}'
          SHORT_SHA=${BENCHED_SHA:0:7}
          git add tools/ci/bench/reporter/bench-history.json
          git commit -m "Chore: Archive bench history for $SHORT_SHA [skip ci]"
          # Narrow race: another merge may have landed on main while we were
          # running benches. Try push; if it's rejected as non-fast-forward,
          # rebase on latest main and retry once. Concurrency group on this
          # job prevents overlap with another history-append; only peer race
          # is a human merge via GitHub UI.
          if ! git push origin main; then
            echo 'Push rejected (likely non-fast-forward); rebasing and retrying.'
            git fetch origin main
            git rebase origin/main
            git push origin main
          fi