Report Benchmark Results #703
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Report Benchmark Results | |
| on: | |
| workflow_run: | |
| workflows: [Benchmarks] | |
| branches: ['**'] | |
| types: | |
| - completed | |
| permissions: | |
| contents: read | |
| jobs: | |
| # PR path: render comment, post / edit in place. Runs on all successful | |
| # bench runs whose triggering event was `pull_request`. | |
| comment: | |
| name: Post PR bench comment | |
| runs-on: ubuntu-latest | |
| permissions: | |
| issues: write | |
| pull-requests: write | |
| if: > | |
| github.event.workflow_run.conclusion == 'success' | |
| && github.event.workflow_run.event == 'pull_request' | |
| steps: | |
| # Generate an installation access token for the Semantic Performance | |
| # Bot GitHub App. Comments post under the bot's identity + avatar | |
| # rather than the generic `github-actions[bot]`. Same permissions | |
| # surface as GITHUB_TOKEN; branded, not widened. | |
| - name: Generate bot token | |
| id: bot-token | |
| uses: actions/create-github-app-token@v3 | |
| with: | |
| client-id: ${{ secrets.SEMANTIC_PERF_BOT_CLIENT_ID }} | |
| private-key: ${{ secrets.SEMANTIC_PERF_BOT_PRIVATE_KEY }} | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version-file: '.node-version' | |
| # Pin reporter to main. The PR cannot author the tool that renders | |
| # its own bench comment — classification thresholds, output format, | |
| # and peak attribution must be author-neutral. Overlaid before any | |
| # downstream step that reads tools/ci/bench/reporter/. | |
| - name: Overlay reporter from main | |
| run: | | |
| git fetch origin main --depth=1 | |
| git checkout origin/main -- tools/ci/bench/reporter/ 2>/dev/null || true | |
| - name: Download bench artifacts | |
| uses: dawidd6/action-download-artifact@v21 | |
| with: | |
| github_token: ${{ steps.bot-token.outputs.token }} | |
| workflow: ${{ github.event.workflow.id }} | |
| run_id: ${{ github.event.workflow_run.id }} | |
| name_is_regexp: true | |
| name: results-.* | |
| path: results | |
| allow_forks: true | |
| - name: Resolve PR number | |
| id: pr | |
| env: | |
| GH_TOKEN: ${{ steps.bot-token.outputs.token }} | |
| run: | | |
| NUMBER='${{ github.event.workflow_run.pull_requests[0].number }}' | |
| if [ -z "$NUMBER" ] || [ "$NUMBER" = "null" ]; then | |
| HEAD='${{ github.event.workflow_run.head_branch }}' | |
| NUMBER=$(gh pr list --repo '${{ github.repository }}' \ | |
| --head "$HEAD" --state open --json number --jq '.[0].number // ""') | |
| fi | |
| echo "number=$NUMBER" >> "$GITHUB_OUTPUT" | |
| echo "Resolved PR number: $NUMBER" | |
| # Walk prior successful bench runs on this PR's branch and build a | |
| # per-iteration history. The reporter merges this with bench-history.json | |
| # (main commits) so peak attribution spans BOTH main AND this PR's | |
| # iterations — an agent sees "iteration 3 was the best on update-10th; | |
| # your current iteration regressed from that." | |
| - name: Fetch PR iteration history | |
| env: | |
| GH_TOKEN: ${{ steps.bot-token.outputs.token }} | |
| run: | | |
| node tools/ci/bench/reporter/fetch-pr-history.js \ | |
| --branch '${{ github.event.workflow_run.head_branch }}' \ | |
| --repo '${{ github.repository }}' \ | |
| --current-run-id '${{ github.event.workflow_run.id }}' \ | |
| --out pr-history.json | |
| - name: Generate report | |
| env: | |
| STARTED: ${{ github.event.workflow_run.created_at }} | |
| ENDED: ${{ github.event.workflow_run.updated_at }} | |
| run: | | |
| WALL_CLOCK=$(( $(date -d "$ENDED" +%s) - $(date -d "$STARTED" +%s) )) | |
| # --scope pr: peak attribution uses PR-iteration history only. | |
| # main-history is still loaded for drift quantification but excluded | |
| # from the comparison set. | |
| node tools/ci/bench/reporter/reporter.js \ | |
| --results results \ | |
| --sha '${{ github.event.workflow_run.head_sha }}' \ | |
| --msg '${{ github.event.workflow_run.display_title }}' \ | |
| --run-url '${{ github.event.workflow_run.html_url }}' \ | |
| --run-id '${{ github.event.workflow_run.id }}' \ | |
| --base-ref 'main' \ | |
| --repo '${{ github.repository }}' \ | |
| --pr-history pr-history.json \ | |
| --scope pr \ | |
| --wall-clock "$WALL_CLOCK" \ | |
| --out bench-report | |
| - name: Upload bench-report.json adjunct | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: bench-report | |
| path: bench-report/bench-report.json | |
| - name: Post or update PR comment | |
| if: steps.pr.outputs.number != '' | |
| env: | |
| GH_TOKEN: ${{ steps.bot-token.outputs.token }} | |
| PR: ${{ steps.pr.outputs.number }} | |
| REPO: ${{ github.repository }} | |
| BOT_LOGIN: semantic-performance-bot[bot] | |
| run: | | |
| # Explicit author-login lookup (not `gh pr comment --edit-last`): | |
| # --edit-last queries comments by viewer identity, which can drift | |
| # across short-lived installation-token rotations and silently | |
| # create a duplicate comment instead of editing the prior one. | |
| # Filtering REST results by `user.login` is stable across tokens. | |
| EXISTING_ID=$(gh api "repos/$REPO/issues/$PR/comments" \ | |
| --jq "[.[] | select(.user.login == \"$BOT_LOGIN\")] | last | .id // empty") | |
| if [ -n "$EXISTING_ID" ]; then | |
| jq -nR --rawfile body bench-report/comment.md '{body: $body}' \ | |
| | gh api "repos/$REPO/issues/comments/$EXISTING_ID" -X PATCH --input - | |
| else | |
| gh pr comment "$PR" --repo "$REPO" --body-file bench-report/comment.md | |
| fi | |
| # History path: append this commit's absolute CIs to bench-history.json | |
| # and commit back to main. Runs on successful bench runs triggered by a | |
| # push to main. Serialized by concurrency group so two near-simultaneous | |
| # merges don't race on the file. | |
| history: | |
| name: Append bench history | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| if: > | |
| github.event.workflow_run.conclusion == 'success' | |
| && github.event.workflow_run.event == 'push' | |
| concurrency: | |
| group: bench-history-append | |
| cancel-in-progress: false | |
| steps: | |
| # Bot token for the archival commit — shows up on main as authored | |
| # by Semantic Performance Bot rather than github-actions[bot]. | |
| - name: Generate bot token | |
| id: bot-token | |
| uses: actions/create-github-app-token@v3 | |
| with: | |
| client-id: ${{ secrets.SEMANTIC_PERF_BOT_CLIENT_ID }} | |
| private-key: ${{ secrets.SEMANTIC_PERF_BOT_PRIVATE_KEY }} | |
| # Check out main's tip for the commit target. We push back to main, | |
| # so we need the most recent state — not the bench's head_sha (which | |
| # may have been superseded by a later merge during the ~10-min bench). | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: main | |
| fetch-depth: 2 # need HEAD~1 for parent_sha lookup | |
| token: ${{ steps.bot-token.outputs.token }} | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version-file: '.node-version' | |
| - name: Download bench artifacts | |
| uses: dawidd6/action-download-artifact@v21 | |
| with: | |
| github_token: ${{ steps.bot-token.outputs.token }} | |
| workflow: ${{ github.event.workflow.id }} | |
| run_id: ${{ github.event.workflow_run.id }} | |
| name_is_regexp: true | |
| name: results-.* | |
| path: results | |
| allow_forks: true | |
| - name: Append history entry | |
| run: | | |
| # Benched commit is the workflow_run's head_sha; parent comes from | |
| # the depth-2 fetch above. Timestamp is the bench run's completion | |
| # so history is ordered by measurement, not commit-land time. | |
| # baseline-sha.txt is the sidecar uploaded next to each matrix | |
| # cell's tachometer JSON. Any cell's value works — all cells in | |
| # one workflow_run benched against the same baseline. | |
| BENCHED_SHA='${{ github.event.workflow_run.head_sha }}' | |
| PARENT_SHA=$(git rev-parse "$BENCHED_SHA^" 2>/dev/null || echo '') | |
| BASELINE_SHA=$(find results -name baseline-sha.txt -type f -exec cat {} \; -quit) | |
| node tools/ci/bench/reporter/append-history.js \ | |
| --results results \ | |
| --sha "$BENCHED_SHA" \ | |
| --msg '${{ github.event.workflow_run.display_title }}' \ | |
| --parent-sha "$PARENT_SHA" \ | |
| --baseline-sha "$BASELINE_SHA" \ | |
| --timestamp '${{ github.event.workflow_run.updated_at }}' \ | |
| --history tools/ci/bench/reporter/bench-history.json | |
| - name: Commit + push | |
| env: | |
| # The checkout step's token persists as the git remote's credential | |
| # helper, so the push below authenticates as the bot automatically. | |
| BOT_APP_SLUG: semantic-performance-bot | |
| run: | | |
| # Use the bot's noreply email format. The exact local-part is the | |
| # app's numeric user id; GitHub renders the commit as authored by | |
| # the app regardless, as long as the name matches the app slug. | |
| git config user.name 'semantic-performance-bot[bot]' | |
| git config user.email '${{ secrets.SEMANTIC_PERF_BOT_APP_ID }}+semantic-performance-bot[bot]@users.noreply.github.com' | |
| # If nothing changed (e.g. a re-run for a SHA already archived with | |
| # identical numbers), skip rather than making an empty commit. | |
| if git diff --quiet tools/ci/bench/reporter/bench-history.json; then | |
| echo 'No change to bench-history.json; skipping commit.' | |
| exit 0 | |
| fi | |
| BENCHED_SHA='${{ github.event.workflow_run.head_sha }}' | |
| SHORT_SHA=${BENCHED_SHA:0:7} | |
| git add tools/ci/bench/reporter/bench-history.json | |
| git commit -m "Chore: Archive bench history for $SHORT_SHA [skip ci]" | |
| # Narrow race: another merge may have landed on main while we were | |
| # running benches. Try push; if it's rejected as non-fast-forward, | |
| # rebase on latest main and retry once. Concurrency group on this | |
| # job prevents overlap with another history-append; only peer race | |
| # is a human merge via GitHub UI. | |
| if ! git push origin main; then | |
| echo 'Push rejected (likely non-fast-forward); rebasing and retrying.' | |
| git fetch origin main | |
| git rebase origin/main | |
| git push origin main | |
| fi |