bench.yml

name: cargo bench
on:
  workflow_call:
  pull_request:
  workflow_dispatch:
env:
  CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
  CARGO_PROFILE_RELEASE_DEBUG: true
  CARGO_PROFILE_RELEASE_LTO: true
  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
  RUSTUP_TOOLCHAIN: stable
  PERF_OPT: record -F2999 --call-graph fp -g
  SCCACHE_CACHE_SIZE: 128G
  SCCACHE_DIRECT: true
  MTU: 1504 # https://github.com/microsoft/msquic/issues/4618
  CFLAGS: -fno-omit-frame-pointer
  CXXFLAGS: -fno-omit-frame-pointer
  WORKSPACE: ${{ github.workspace }}
  BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  build:
    name: Build bench binaries
    runs-on: ubuntu-24.04
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout mozilla/neqo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: mozilla/neqo
          path: neqo
          submodules: "recursive"
          persist-credentials: false

      - id: nss-version
        uses: ./neqo/.github/actions/minimum-version
      - uses: ./neqo/.github/actions/build-neqo
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          artifact-name: bench-build
          benches: "true"
          baseline-ref: origin/${{ env.BASE_REF }}
          nss-minimum-version: ${{ steps.nss-version.outputs.minimum }}

  bench:
    name: cargo bench
    runs-on: "self-hosted" # zizmor: ignore[self-hosted-runner]
    needs: [build]
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout mozilla/neqo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: mozilla/neqo
          path: neqo
          persist-credentials: false
          clean: false
      - run: |
          cd neqo
          git fetch --no-tags --depth=1 origin "$BASE_REF"

      - name: Set PATH and environment
        run: |
          echo "$HOME/.cargo/bin" >> "${GITHUB_PATH}"

      - name: Install Rust
        uses: mozilla/actions/rust@27cbe8fb5d338c2861b787e5de10410559065db1 # v1.1.3
        with:
          version: ${{ env.RUSTUP_TOOLCHAIN }}
          tools: flamegraph, samply
          token: ${{ secrets.GITHUB_TOKEN }}
          sccache: true

      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: bench-build
          path: dist

      - name: Set up build artifacts
        run: |
          chmod +x dist/neqo/* dist/neqo-baseline/*
          mkdir -p binaries/neqo binaries/neqo-baseline
          cp dist/neqo/* binaries/neqo/
          cp dist/neqo-baseline/* binaries/neqo-baseline/
          {
            echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/dist/lib:$LD_LIBRARY_PATH"
            echo "TEST_FIXTURE_DB=$GITHUB_WORKSPACE/dist/test-fixture/db"
            echo "NSS_DB_PATH=$GITHUB_WORKSPACE/dist/test-fixture/db" # TODO: Remove once baseline uses nss-rs >= 0.11.0
          } >> "$GITHUB_ENV"

      # Disable turboboost, hyperthreading, use performance governor, and isolate CPUs with cset.
      - name: Prepare machine
        id: cpu-tuning
        uses: ./neqo/.github/actions/cpu-tuning
        with:
          mtu: ${{ env.MTU }}

      - name: Run cargo bench
        env:
          BENCH_SET: ${{ steps.cpu-tuning.outputs.bench-set }}
        run: |
          # TODO: Remove NSS_DB_PATH once baseline uses nss-rs >= 0.11.0
          bench_exec() { sudo nice -n -20 env "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" "TEST_FIXTURE_DB=$TEST_FIXTURE_DB" "NSS_DB_PATH=$NSS_DB_PATH" setarch --addr-no-randomize cset proc --set="$BENCH_SET" --exec "$@"; }
          filter_cset() { grep -v '^cset' || test $? = 1; }
          shopt -s extglob nullglob

          # Kill any stale processes from previous runs that may be holding port 4433.
          sudo fuser -k 4433/udp 4433/tcp 2>/dev/null || true

          cd neqo
          # Fix any root-owned files left by a previous run before removing them
          sudo chown -R "$(id -u)" target/criterion 2>/dev/null || true
          rm -rf target/criterion
          # Run the baseline builds first, to establish a baseline.
          for BENCH in ../dist/neqo-baseline/!(neqo-client|neqo-server); do
            bench_exec "$BENCH" -- --bench --save-baseline baseline --noplot | filter_cset | tee -a ../results-baseline.txt
          done
          # Run pull request builds twice, once without perf for baseline comparison, and once with perf for profiling.
          # (Perf seems to introduce some variability in the results.)
          for BENCH in ../dist/neqo/!(neqo-client|neqo-server); do
            # --baseline-lenient allows us to add new benchmarks in pull requests.
            bench_exec "$BENCH" -- --bench --baseline-lenient baseline --noplot | filter_cset | tee -a ../results.txt
          done
          sanitize_name() { echo "$1" | tr '/ ()' '___-' | tr -s '_'; }
          for BENCH in ../dist/neqo/!(neqo-client|neqo-server); do
            BIN_NAME=$(basename "$BENCH")
            VARIANTS=$("$BENCH" --bench --list 2>/dev/null | sed -n 's/: benchmark$//p' || true)
            if [ -n "$VARIANTS" ]; then
              while IFS= read -r VARIANT; do
                SAFE_NAME="${BIN_NAME}_$(sanitize_name "$VARIANT")"
                # shellcheck disable=SC2086
                bench_exec perf -- $PERF_OPT -o "../$SAFE_NAME.perf" "$BENCH" --bench --noplot --discard-baseline --exact "$VARIANT" | filter_cset
              done <<< "$VARIANTS"
            else
              # shellcheck disable=SC2086
              bench_exec perf -- $PERF_OPT -o "../$BIN_NAME.perf" "$BENCH" --bench --noplot --discard-baseline | filter_cset
            fi
          done
          # bench_exec runs as root via sudo nice, so fix ownership of all files it wrote
          sudo chown -R "$(id -u)" target/criterion ../*.perf

      # Re-enable turboboost, hyperthreading, use powersave governor, remove all CPU sets, restore MTU.
      - name: Restore machine
        if: always()
        uses: ./neqo/.github/actions/cpu-tuning
        with:
          mode: restore

      - name: Post-process perf data
        run: |
          for f in *.perf; do
            # Convert for profiler.firefox.com
            samply import "$f" -o "$f.samply.json.gz" --save-only --presymbolicate
            # Generate flamegraphs
            flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}"
          done

      - name: Format results as Markdown
        id: results
        env:
          EVENT_PATH: ${{ github.event_path }}
          TESTBED: "On-prem"
        run: |
          SHA=$(cd neqo && git log origin/"$BASE_REF" -1 --format=%H)
          echo "$SHA" > baseline-sha.txt
          python3 neqo/.github/scripts/format-bench-results.py results.txt
          # Build final markdown output
          {
            echo "### Benchmark results"
            echo
            if [ -s significant-results.md ]; then
              echo "Significant performance differences relative to $SHA."
              cat significant-results.md
            else
              echo "No significant performance differences relative to $SHA."
            fi
            echo
            echo "<details><summary>All results</summary>"
            echo
            cat all-bench-results.md
            echo
            echo "</details>"
          } > results.md
          rm -f all-bench-results.md significant-results.md
          cat results.md > "$GITHUB_STEP_SUMMARY"
          echo "$TESTBED" > testbed.txt
          cp "$EVENT_PATH" event.json

      - name: Export profiler.firefox.com data
        id: export_samply
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-samply
          path: |
            *.samply.json.gz
            *.syms.json
            binaries
          compression-level: 9

      - name: Export performance data
        id: export_perf
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-perf
          path: |
            *.svg
            *.txt
            *.md
            event.json
            results.*
            results-baseline.*
          compression-level: 9

      - name: Export PR comment data
        uses: ./neqo/.github/actions/pr-comment-data-export
        with:
          name: ${{ github.workflow }}
          contents: results.md
          log-md: ${{ format('[Download data for `profiler.firefox.com`]({0}) or [download performance comparison data]({1}).', steps.export_samply.outputs.artifact-url, steps.export_perf.outputs.artifact-url) }}

      - name: Fail on regression
        # Don't check for regressions when running on main.
        if: github.ref != 'refs/heads/main' && github.event.merge_group.base_ref != 'refs/heads/main'
        run: |
          if grep -q "Performance has regressed." results.txt; then
            echo "Performance regression detected."
            exit 1
          else
            echo "No performance regression detected."
          fi

      - name: Remove benchmark artifacts
        if: always()
        run: |
          rm -- * || true
          rm -r -- binaries dist comment-data || true