benchmark.yml

name: Benchmark

on:
  pull_request:
    branches: [main]

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  OX_CONTENT_BENCHMARK_RUNNER: blacksmith-32vcpu-ubuntu-2404
  OX_CONTENT_BENCHMARK_RUNS: "5"

jobs:
  benchmark:
    name: Benchmark
    runs-on: blacksmith-32vcpu-ubuntu-2404
    env:
      CARGO_TARGET_DIR: /tmp/ox-content-cargo-target
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Checkout base
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ${{ github.repository }}
          ref: ${{ github.event.pull_request.base.sha }}
          path: .benchmark/base
          persist-credentials: false

      - name: Checkout head
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.sha }}
          path: .benchmark/head
          persist-credentials: false

      - name: Setup Vite+
        uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1
        with:
          node-version-file: ".node-version"
          cache: true
          run-install: false

      # The parse/render benchmark compares against `Bun.markdown.html`, which
      # only exists in Bun >= 1.3.8. Without an explicit Bun the runner has no
      # (or too old a) bun, so `loadBunMarkdownBenchmarks` silently drops the
      # Bun.markdown row. Pin a version that ships the API so the comparison
      # always includes Bun.
      - name: Setup Bun
        # oven-sh/setup-bun is outside this repo's allowed-actions policy, so
        # every workflow that referenced it died with a startup_failure before
        # any job ran. Install the same pinned Bun through npm (Node is set up
        # by the preceding Setup Vite+ step) instead of widening the policy.
        # Bun is a one-off global CLI tool pinned to an exact version, so there
        # is no project lockfile to install it from.
        run: npm install --global bun@1.3.14 # zizmor: ignore[adhoc-packages]

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

      - name: Mount sticky cache
        uses: ./.github/actions/mount-sticky-cache
        with:
          key-prefix: ${{ github.repository }}-benchmark-${{ github.job }}
          cargo: "true"
          target: /tmp/ox-content-cargo-target

      - name: Benchmark base
        working-directory: .benchmark/base
        run: |
          node "$GITHUB_WORKSPACE/.github/scripts/run-pr-benchmark.mjs" \
            --source "$GITHUB_WORKSPACE" \
            --runs "$OX_CONTENT_BENCHMARK_RUNS" \
            --runtime-json "$RUNNER_TEMP/benchmark-base.json" \
            --bundle-json "$RUNNER_TEMP/bundle-base.json"

      - name: Benchmark head
        working-directory: .benchmark/head
        run: |
          node "$GITHUB_WORKSPACE/.github/scripts/run-pr-benchmark.mjs" \
            --source "$GITHUB_WORKSPACE" \
            --runs "$OX_CONTENT_BENCHMARK_RUNS" \
            --runtime-json "$RUNNER_TEMP/benchmark-head.json" \
            --bundle-json "$RUNNER_TEMP/bundle-head.json"

      - name: Compare benchmarks
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          OX_CONTENT_BENCHMARK_ALLOW_REGRESSION: ${{ contains(github.event.pull_request.labels.*.name, 'benchmark-regression-accepted') && '1' || '0' }}
        working-directory: .benchmark/head
        run: node "$GITHUB_WORKSPACE/.github/scripts/compare-pr-benchmark.mjs"

      - name: Upload benchmark comment
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-comment
          path: ${{ runner.temp }}/benchmark-comment.md
          if-no-files-found: error
          retention-days: 1

  comment:
    name: Comment
    runs-on: blacksmith-32vcpu-ubuntu-2404
    needs: benchmark
    if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
    permissions:
      contents: read
      issues: write
      pull-requests: write
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          persist-credentials: false

      - name: Download benchmark comment
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: benchmark-comment
          path: ${{ runner.temp }}

      - name: Comment on pull request
        env:
          GITHUB_TOKEN: ${{ github.token }}
          COMMENT_PATH: ${{ runner.temp }}/benchmark-comment.md
        run: node "$GITHUB_WORKSPACE/.github/scripts/comment-pr-benchmark.mjs"