benchmarks.yml

name: Benchmarks

on:
  push:
    branches: [ main ]
    paths:
      - 'src/**'
      - '.github/workflows/benchmarks.yml'
  pull_request:
    branches: [ main ]
    paths:
      - 'src/**'
      - '.github/workflows/benchmarks.yml'

# The core suite is ~300 cases and, at full BenchmarkDotNet accuracy (Job.Default x2
# launches, see CiConfig.cs), one pass takes ~3h. The PR path runs it TWICE on one
# runner (PR head + main base, same-runner A/B, so hardware variance cancels), which
# blew past GitHub's 6h job ceiling and was cancelled on every PR (#217). Rather than
# trade away measurement accuracy, we shard the suite across a parallel matrix: each
# shard measures only its slice (head + base) on its own runner, so wall time is
# ~3h / SHARD_TOTAL at full accuracy, and an aggregate job stitches the per-shard JSON
# back together for the regression comment / gh-pages publish.
#
# The same-runner A/B invariant is preserved because sharding is by benchmark *class*
# (Program.cs `--shard`): a class's PR-head and main-base measurements always run
# together on the one shard runner, so the per-benchmark delta still cancels hardware.
env:
  # Keep in sync with the matrix.shard list below.
  #
  # Raised 6 -> 8: at 6 the heaviest shard measured ~65 min per slice, so head + base
  # came to ~130 min and overran the 120 min timeout, while the lightest finished in
  # ~86 min. CiConfig.cs is explicit that the job schedule stays as-is and the matrix
  # is what scales when the suite grows, so this widens the matrix rather than trading
  # away measurement accuracy.
  SHARD_TOTAL: '8'

jobs:
  benchmark-shard:
    name: benchmark (shard ${{ matrix.shard }})
    runs-on: ubuntu-latest
    # Measured at SHARD_TOTAL=6: slices ran 43-65 min, so head + base came to 86-130 min
    # and the heaviest shard overran this timeout. At 8 the same work divides further,
    # putting the heaviest shard back comfortably inside the limit. Re-measure and widen
    # the matrix again if shard durations creep back toward it.
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        # Must enumerate 0 .. SHARD_TOTAL-1.
        shard: [0, 1, 2, 3, 4, 5, 6, 7]

    permissions:
      contents: read

    defaults:
      run:
        working-directory: src

    steps:
      # NOTE: do NOT add `filter: tree:0` here. A blobless partial clone makes the repo
      # advertise every commit as "have" while missing the underlying objects, which
      # breaks the base `git worktree add <base-sha>` below (and, in the aggregate job,
      # github-action-benchmark's own gh-pages fetch) with "missing blob object … did
      # not send all necessary objects" (#217). fetch-depth: 0 is required for the
      # base worktree.
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: 8.0.x

      - name: Run head benchmarks (shard ${{ matrix.shard }})
        # working-directory must be the project folder, not src/. BenchmarkDotNet writes
        # its artifacts to ./BenchmarkDotNet.Artifacts relative to the process CWD.
        working-directory: src/Celerity.Benchmarks
        run: dotnet run --configuration Release -- --ci --shard "${SHARD_TOTAL}" "${{ matrix.shard }}"

      - name: Stage head report
        run: |
          set -euo pipefail
          mkdir -p /tmp/reports
          report=$(ls Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results/*-report-full.json | head -n 1)
          cp "$report" "/tmp/reports/head-shard-${{ matrix.shard }}.json"
          echo "Staged head shard ${{ matrix.shard }}: $report"

      - name: Run base (main) benchmarks for this shard on the same runner
        # Same-runner A/B: build and benchmark the main tip back-to-back with the PR head
        # on THIS runner so hardware variance cancels (hosted runners vary 20-50%
        # run-to-run, so a stored cross-runner baseline would be noise-dominated).
        #
        # `--shard` is on main now, so the base tip honours it and measures only this
        # shard's slice — the transitional full-suite base run that this comment used to
        # warn about no longer happens. A cancellation here is therefore a real signal (the
        # slice genuinely exceeded the job timeout) rather than expected behaviour that will
        # self-heal, and should be investigated instead of dismissed.
        if: github.event_name == 'pull_request'
        working-directory: ${{ github.workspace }}
        run: |
          set -euo pipefail
          git fetch origin main
          BASE_SHA=$(git rev-parse origin/main)
          echo "Benchmarking base (main) at $BASE_SHA, shard ${{ matrix.shard }}"
          git worktree add /tmp/base-tree "$BASE_SHA"
          cd /tmp/base-tree/src/Celerity.Benchmarks
          dotnet run --configuration Release -- --ci --shard "${SHARD_TOTAL}" "${{ matrix.shard }}"
          report=$(ls BenchmarkDotNet.Artifacts/results/*-report-full.json | head -n 1)
          cp "$report" "/tmp/reports/base-shard-${{ matrix.shard }}.json"
          echo "Staged base shard ${{ matrix.shard }}: $report"

      - name: Upload shard reports
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-shard-${{ matrix.shard }}
          path: /tmp/reports/**
          if-no-files-found: error
          retention-days: 30

  aggregate:
    name: aggregate & report
    needs: benchmark-shard
    # Run even if some shards were cancelled/failed (e.g. the transitional base run
    # above) so partial results are still reported where possible.
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 20

    permissions:
      contents: write
      pull-requests: write

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Download all shard reports
        uses: actions/download-artifact@v4
        with:
          pattern: benchmark-shard-*
          merge-multiple: true
          path: /tmp/reports

      - name: Merge shard reports
        id: merge
        run: |
          set -euo pipefail
          shopt -s nullglob
          head_files=(/tmp/reports/head-shard-*.json)
          if [ ${#head_files[@]} -eq 0 ]; then
            echo "No head shard reports found (all shards cancelled/failed) — nothing to report."
            echo "have_head=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          echo "have_head=true" >> "$GITHUB_OUTPUT"

          # Stitch every shard's joined report back into one: keep shard 0's metadata,
          # concatenate all shards' Benchmarks arrays.
          jq -s '.[0] * {Benchmarks: ([.[].Benchmarks] | add)}' "${head_files[@]}" > /tmp/pr-report-full.json
          echo "Merged $(jq '.Benchmarks | length' /tmp/pr-report-full.json) head benchmarks from ${#head_files[@]} shard(s)."

          base_files=(/tmp/reports/base-shard-*.json)
          if [ ${#base_files[@]} -gt 0 ]; then
            jq -s '.[0] * {Benchmarks: ([.[].Benchmarks] | add)}' "${base_files[@]}" > /tmp/base-report-full.json
            echo "have_base=true" >> "$GITHUB_OUTPUT"
            echo "Merged $(jq '.Benchmarks | length' /tmp/base-report-full.json) base benchmarks from ${#base_files[@]} shard(s)."
          else
            echo "have_base=false" >> "$GITHUB_OUTPUT"
            echo "No base shard reports (main push, or base run cancelled)."
          fi

      - name: Resolve base SHA (for the comment footer)
        if: github.event_name == 'pull_request' && steps.merge.outputs.have_head == 'true'
        run: |
          set -euo pipefail
          git fetch origin main
          echo "BASE_SHA=$(git rev-parse origin/main)" >> "$GITHUB_ENV"

      - name: Upload merged reports
        if: steps.merge.outputs.have_head == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: /tmp/*-report-full.json
          if-no-files-found: warn
          retention-days: 30

      # ----- PR path: post the same-runner A/B comment -----
      - name: Post benchmark comment
        if: github.event_name == 'pull_request' && steps.merge.outputs.have_head == 'true' && steps.merge.outputs.have_base == 'true'
        uses: actions/github-script@v7
        env:
          ALERT_THRESHOLD_RATIO: '1.10'
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const fs = require('fs');

            const pr = JSON.parse(fs.readFileSync('/tmp/pr-report-full.json', 'utf8'));
            const base = JSON.parse(fs.readFileSync('/tmp/base-report-full.json', 'utf8'));

            // Index base results by full benchmark name.
            const baseMap = new Map();
            for (const b of base.Benchmarks) {
              baseMap.set(b.FullName, b.Statistics);
            }

            const formatNs = (ns) => {
              if (ns == null) return 'n/a';
              if (ns < 1000) return `${ns.toFixed(1)} ns`;
              if (ns < 1_000_000) return `${(ns/1000).toFixed(2)} μs`;
              if (ns < 1_000_000_000) return `${(ns/1_000_000).toFixed(2)} ms`;
              return `${(ns/1_000_000_000).toFixed(2)} s`;
            };

            const threshold = parseFloat(process.env.ALERT_THRESHOLD_RATIO);
            const thresholdPct = ((threshold - 1) * 100).toFixed(0);
            // Each entry: { name, isHasher, prCell, stdCell, baseCell, deltaCell, flag }
            // where flag is 'regression' | 'improvement' | null.
            const entries = [];
            let regressions = 0;
            let improvements = 0;
            let errored = 0;

            for (const b of pr.Benchmarks) {
              const name = b.FullName;
              // BenchmarkDotNet emits a benchmark with null Statistics and empty
              // Measurements when that case errored or was not run (a throwing
              // [GlobalSetup], a cancelled shard, an OOM, ...). Surface it as a row
              // rather than dereferencing null — one errored case must never crash
              // the aggregate job and red-X the whole PR gate.
              if (b.Statistics == null) {
                errored++;
                const baseStats = baseMap.get(name);
                entries.push({
                  name,
                  isHasher: /Hasher/.test(name),
                  prCell: '⚠️ errored',
                  stdCell: 'n/a',
                  baseCell: baseStats ? formatNs(baseStats.Mean) : 'n/a',
                  deltaCell: 'no measurements',
                  flag: null,
                });
                continue;
              }
              const prMean = b.Statistics.Mean;
              const prStdDev = b.Statistics.StandardDeviation;
              const baseStats = baseMap.get(name);
              let deltaCell = baseMap.has(name) ? '⚠️ base errored' : '🆕 new';
              let flag = null;
              if (baseStats) {
                const baseMean = baseStats.Mean;
                const baseStdDev = baseStats.StandardDeviation;
                const ratio = prMean / baseMean;
                const pct = (ratio - 1) * 100;
                const sign = pct >= 0 ? '+' : '';
                deltaCell = `${sign}${pct.toFixed(1)}%`;
                // Flag only when the change clears the threshold AND exceeds the
                // combined run-to-run noise of both measurements.
                const beyondNoise = Math.abs(prMean - baseMean) > (prStdDev + baseStdDev);
                if (ratio >= threshold && beyondNoise) {
                  deltaCell += ' ⚠️';
                  flag = 'regression';
                  regressions++;
                } else if (ratio <= (1 / threshold) && beyondNoise) {
                  deltaCell += ' ✅';
                  flag = 'improvement';
                  improvements++;
                }
              }
              entries.push({
                name,
                // Hasher throughput benchmarks (StringHasherBenchmark,
                // IntegerHasherBenchmark) get their own section; everything else
                // is a collection benchmark.
                isHasher: /Hasher/.test(name),
                prCell: formatNs(prMean),
                stdCell: formatNs(prStdDev),
                baseCell: baseStats ? formatNs(baseStats.Mean) : 'n/a',
                deltaCell,
                flag,
              });
            }

            let subtitle;
            if (regressions > 0) {
              subtitle = `${regressions} regression${regressions === 1 ? '' : 's'} ⚠️ vs main` +
                (improvements > 0 ? `, ${improvements} improvement${improvements === 1 ? '' : 's'} ✅` : '') +
                ` (rows past ±${thresholdPct}% beyond noise).`;
            } else if (improvements > 0) {
              subtitle = `No regressions — ${improvements} improvement${improvements === 1 ? '' : 's'} ✅ vs main.`;
            } else {
              subtitle = `No significant change vs main (all within ±${thresholdPct}% or noise).`;
            }
            if (errored > 0) {
              subtitle += ` ⚠️ ${errored} benchmark${errored === 1 ? '' : 's'} errored (no measurements) — see the table.`;
            }

            const baseSha = (process.env.BASE_SHA || '').slice(0, 7);
            const footer = `<sub>Same-runner A/B (sharded ${process.env.SHARD_TOTAL || ''}-way): main (\`${baseSha}\`) and this PR were built and benchmarked back-to-back on the same runner per shard, so hardware variance cancels out. ⚠️ = PR mean ≥ +${thresholdPct}% slower than main and beyond combined std-dev; ✅ = correspondingly faster.</sub>`;

            const header = ['| Benchmark | This PR | StdDev | main | Δ |', '|---|---:|---:|---:|---:|'];
            const toRow = (e) => `| \`${e.name}\` | ${e.prCell} | ${e.stdCell} | ${e.baseCell} | ${e.deltaCell} |`;

            // A collapsible section per benchmark family, collapsed by default so
            // the (often large) tables stay below the fold.
            const section = (title, list) => {
              if (list.length === 0) return [];
              return [
                '<details>',
                `<summary><b>${title}</b> (${list.length})</summary>`,
                '',
                ...header,
                ...list.map(toRow),
                '</details>',
                '',
              ];
            };

            // Highlights: only the flagged rows stay above the fold so reviewers
            // see what actually moved without expanding either table.
            const flagged = entries.filter(e => e.flag);
            const highlights = flagged.length === 0 ? [] : [
              '**Highlights**',
              '',
              ...header,
              ...flagged.map(toRow),
              '',
            ];

            const collections = entries.filter(e => !e.isHasher);
            const hashers = entries.filter(e => e.isHasher);

            const marker = '<!-- celerity-benchmarks-comment -->';
            const body = [
              marker,
              '## Benchmarks',
              '',
              subtitle,
              '',
              ...highlights,
              ...section('Collections', collections),
              ...section('Hashers', hashers),
              footer,
            ].join('\n');

            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c => c.body && c.body.includes(marker));

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body,
              });
              core.info(`Updated comment ${existing.id}`);
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body,
              });
              core.info('Created new benchmarks comment');
            }

      # ----- Main path: publish merged report to gh-pages -----
      - name: Stage merged report for publish
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.merge.outputs.have_head == 'true'
        run: |
          set -euo pipefail
          mkdir -p src/Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results
          cp /tmp/pr-report-full.json src/Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results/ci-report-full.json

      - name: Publish to gh-pages
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.merge.outputs.have_head == 'true'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Celerity Benchmarks
          tool: 'benchmarkdotnet'
          output-file-path: src/Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results/ci-report-full.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          comment-on-alert: false
          comment-always: false
          summary-always: true
          fail-on-alert: false
          gh-pages-branch: gh-pages
          benchmark-data-dir-path: dev/bench

      - name: Sync custom dashboard to gh-pages
        # Runs only on main push (same gate as the action's auto-push).
        # Copies web/index.html + web/dev/bench/*.html into gh-pages,
        # leaving the action-managed data.js untouched. Idempotent.
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.merge.outputs.have_head == 'true'
        run: |
          set -euo pipefail
          git fetch origin gh-pages
          git worktree add /tmp/gh-pages gh-pages
          cp web/index.html /tmp/gh-pages/index.html
          mkdir -p /tmp/gh-pages/dev/bench
          cp web/dev/bench/index.html /tmp/gh-pages/dev/bench/index.html
          cp web/dev/bench/detail.html /tmp/gh-pages/dev/bench/detail.html
          cd /tmp/gh-pages
          if git diff --quiet && git diff --cached --quiet; then
            echo "No dashboard changes to sync"
            exit 0
          fi
          git -c user.name="github-actions" -c user.email="github-actions@github.com" \
            add index.html dev/bench/index.html dev/bench/detail.html
          git -c user.name="github-actions" -c user.email="github-actions@github.com" \
            commit -m "Sync custom dashboard from ${GITHUB_SHA:0:7}"
          git push origin gh-pages