benchmark.yml

name: Benchmark Service

on:
  pull_request:
    paths:
      - 'computation/**'
      - 'analytics/**'
      - '.github/workflows/benchmark.yml'
  workflow_dispatch:

permissions:
  contents: read
  pull-requests: write

concurrency:
  group: benchmark-${{ github.ref }}
  cancel-in-progress: true

jobs:
  benchmark:
    name: Performance Benchmark Regression
    runs-on: ubuntu-latest
    steps:
      - name: Resolve base and head refs
        id: refs
        run: |
          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
            base="${{ github.event.pull_request.base.sha }}"
            head="${{ github.event.pull_request.head.sha }}"
            is_pr=true
          else
            base="$GITHUB_SHA"
            head="$GITHUB_SHA"
            is_pr=false
          fi
          {
            echo "base=$base"
            echo "head=$head"
            echo "base_short=${base:0:7}"
            echo "head_short=${head:0:7}"
            echo "is_pr=$is_pr"
          } >> "$GITHUB_OUTPUT"

      - name: Checkout head
        uses: actions/checkout@v4
        with:
          ref: ${{ steps.refs.outputs.head }}
          path: head

      - name: Checkout base
        if: steps.refs.outputs.is_pr == 'true'
        uses: actions/checkout@v4
        with:
          ref: ${{ steps.refs.outputs.base }}
          path: base

      - name: Set up Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: '3.12'
          cache: pip
          cache-dependency-path: head/analytics/requirements.txt

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r head/analytics/requirements.txt

      - name: Run head benchmark
        working-directory: head/analytics
        run: |
          python -m pytest tests \
            --benchmark-only \
            --benchmark-storage=.benchmarks \
            --benchmark-save=head

      - name: Run base benchmark
        if: steps.refs.outputs.is_pr == 'true'
        working-directory: base/analytics
        run: |
          python -m pytest tests \
            --benchmark-only \
            --benchmark-storage=.benchmarks \
            --benchmark-save=base

      - name: Compare base vs head (same-runner A/B)
        if: steps.refs.outputs.is_pr == 'true'
        continue-on-error: true
        working-directory: head
        run: |
          set +e
          base_json="$(find ../base/analytics/.benchmarks -name '*_base.json' | sort | tail -n 1)"
          head_json="$(find analytics/.benchmarks -name '*_head.json' | sort | tail -n 1)"
          if [ -z "$base_json" ] || [ -z "$head_json" ]; then
            echo "Could not locate benchmark JSONs (base=$base_json head=$head_json). Comparison skipped." \
              | tee analytics/benchmark-comparison.md
            exit 0
          fi
          # Advisory: compare_benchmarks.py exits 1 on a >threshold regression, but the
          # verdict is surfaced in the PR comment and must not fail the job.
          python analytics/compare_benchmarks.py "$base_json" "$head_json" \
            --threshold 10 --metric mean --format markdown \
            | tee analytics/benchmark-comparison.md
          exit 0

      - name: Publish benchmark report
        if: always()
        run: |
          {
            echo "<!-- benchmark-report -->"
            echo "## Benchmark Regression Report"
            echo ""
            echo "Engine: same-runner A/B (base \`${{ steps.refs.outputs.base_short }}\` vs head \`${{ steps.refs.outputs.head_short }}\` on a single job), gated by \`compare_benchmarks.py\` at >10% mean regression. See [ADR-030](docs/adr/ADR-030-same-runner-benchmark-ab.md)."
            echo ""
            if [ "${{ steps.refs.outputs.is_pr }}" = "true" ] && [ -f head/analytics/benchmark-comparison.md ]; then
              cat head/analytics/benchmark-comparison.md
            else
              echo "_No comparison produced (non-PR trigger or benchmark error). Head benchmark was recorded._"
            fi
          } > benchmark-report.md
          cat benchmark-report.md

      - name: Comment on PR
        if: github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_REPO: ${{ github.repository }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          # Find this bot's prior benchmark report (matched by the hidden marker in the
          # report body) and update it in place, so repeated runs don't pile up comments.
          comment_id=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
            --paginate \
            --jq '.[] | select(.user.login == "github-actions[bot]") | select(.body | contains("<!-- benchmark-report -->")) | .id' \
            | tail -n 1)

          if [ -n "$comment_id" ]; then
            jq -n --rawfile body benchmark-report.md '{body: $body}' > "$RUNNER_TEMP/benchmark-comment.json"
            gh api -X PATCH "repos/${{ github.repository }}/issues/comments/${comment_id}" \
              --input "$RUNNER_TEMP/benchmark-comment.json"
          else
            gh pr comment "$PR_NUMBER" --body-file benchmark-report.md
          fi

      - name: Deferred benchmark capabilities
        run: |
          echo "Java JMH benchmark tasks (backtestBenchmark, regulatoryScenarioTest) and the"
          echo "rds_check.py helper script are deferred to Tracks 5/8."
          echo "The chaos-benchmark.yml scheduled job is out of scope for same-runner A/B"
          echo "(it is not PR-triggered); see docs/adr/ADR-030-same-runner-benchmark-ab.md."