bench.yml

name: cargo bench
on:
  workflow_call:
  pull_request:
    branches: ["main"]
  workflow_dispatch:
    inputs:
      bencher:
        type: choice
        description: Which benchmarking testbed to run on.
        required: true
        default: codspeed
        options:
          - codspeed
          - on-prem
          - gcp
env:
  CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
  CARGO_PROFILE_RELEASE_DEBUG: true
  RUSTUP_TOOLCHAIN: stable
  PERF_OPT: record -F2999 --call-graph fp -g
  SCCACHE_CACHE_SIZE: 128G
  SCCACHE_DIRECT: true
  MTU: 1504 # https://github.com/microsoft/msquic/issues/4618
  CFLAGS: -fno-omit-frame-pointer
  CXXFLAGS: -fno-omit-frame-pointer
  WORKSPACE: ${{ github.workspace}}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  bench:
    name: cargo bench
    runs-on: ${{ (inputs.bencher || 'codspeed') == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || (inputs.bencher || 'codspeed') == 'codspeed' && 'Neqo' || 'self-hosted' }}
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout mozilla/neqo
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          repository: mozilla/neqo
          path: neqo
          submodules: 'recursive'
          persist-credentials: false
          clean: false
      - run: |
          cd neqo
          git fetch --no-tags --depth=1 origin main

      - name: Set PATH and environment
        run: |
          echo "$HOME/.cargo/bin" >> "${GITHUB_PATH}"

      - name: Install Rust
        uses: ./neqo/.github/actions/rust
        with:
          version: ${{ env.RUSTUP_TOOLCHAIN }}
          tools: flamegraph, samply
          token: ${{ secrets.GITHUB_TOKEN }}
          workspaces: |
            neqo

      - name: Get minimum NSS version
        id: nss-version
        run: |
          cd neqo
          cat neqo-crypto/min_version.txt > versions.txt
          git show origin/main:neqo-crypto/min_version.txt >> versions.txt
          # Use the maximum version from both branches.
          echo "minimum=$(sort -u versions.txt | tail -n1)" >> "$GITHUB_OUTPUT"

      - name: Install NSS
        id: nss
        uses: ./neqo/.github/actions/nss
        with:
          minimum-version: ${{ steps.nss-version.outputs.minimum }}

      - name: Build neqo
        run: |
          # See https://github.com/flamegraph-rs/flamegraph for why we append to RUSTFLAGS here.
          export RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment -C force-frame-pointers=yes $RUSTFLAGS"
          echo "RUSTFLAGS=$RUSTFLAGS" >> "$GITHUB_ENV"
          mkdir -p binaries/neqo-main
          mkdir -p binaries/neqo
          cd neqo
          git checkout origin/main
          cargo build --locked --release --bin neqo-client --bin neqo-server
          cp target/release/neqo-client ../binaries/neqo-main/
          cp target/release/neqo-server ../binaries/neqo-main/
          git checkout -
          cargo build --locked --release --bin neqo-client --bin neqo-server
          cp target/release/neqo-client ../binaries/neqo/
          cp target/release/neqo-server ../binaries/neqo/

      # Disable turboboost, hyperthreading and use performance governor.
      # Also creates "cpu23", "cpu2" and "cpu3" CPU sets for use with cset.
      # On the bencher, logical cores 2 and 3 have been isolated for use by the benchmarks.
      - name: Prepare machine
        run: sudo /root/bin/prep.sh

      - name: Run cargo bench
        env:
          NSS_DB_PATH: ${{ github.workspace }}/neqo/test-fixture/db
        run: |
          sudo ip link set dev lo mtu "$MTU"
          cd neqo
          rm -rf target/criterion
          git checkout origin/main
          cargo bench --locked --workspace --features bench --no-run 2>&1 | tee benches-main.txt
          BENCHES_MAIN=$(grep Executable benches-main.txt | cut -d\( -f2 | cut -d\) -f1 | tr -s '\n' ' ')
          # Run the builds from the main branch first, to establish a baseline.
          for BENCH in $BENCHES_MAIN; do
            cp "$BENCH" ../binaries/neqo-main/
            # shellcheck disable=SC2086
            nice -n -20 setarch --addr-no-randomize cset proc --set=cpu23 --exec \
              $BENCH -- --bench --save-baseline main --noplot | { grep -v '^cset' || test $? = 1; } | tee -a ../results-main.txt
          done
          # Copy the main branch results to a separate directory for bencher.dev.
          cp -r target/criterion ../criterion-main
          git checkout -
          cargo bench --locked --workspace --features bench --no-run 2>&1 | tee benches.txt
          BENCHES=$(grep Executable benches.txt | cut -d\( -f2 | cut -d\) -f1 | tr -s '\n' ' ')
          # Run pull request builds twice, once without perf for baseline comparison, and once with perf for profiling.
          # (Perf seems to introduce some variability in the results.)
          for BENCH in $BENCHES; do
            cp "$BENCH" ../binaries/neqo/
            # shellcheck disable=SC2086
            # --baseline-lenient allows us to add new benchmarks in pull requests.
            nice -n -20 setarch --addr-no-randomize cset proc --set=cpu23 --exec \
              $BENCH -- --bench --baseline-lenient main --noplot | { grep -v '^cset' || test $? = 1; } | tee -a ../results.txt
          done
          for BENCH in $BENCHES; do
            NAME=$(basename "$BENCH" | cut -d- -f1)
            # shellcheck disable=SC2086
            nice -n -20 setarch --addr-no-randomize cset proc --set=cpu23 --exec \
              perf -- $PERF_OPT -o "../$NAME.perf" $BENCH --bench --noplot --discard-baseline | { grep -v '^cset' || test $? = 1; }
          done
          cp -r target/criterion ../criterion

      # Re-enable turboboost, hyperthreading and use powersave governor. Remove all CPU sets.
      - name: Restore machine
        if: success() || failure() || cancelled()
        run: |
          sudo /root/bin/unprep.sh
          # In case the previous test failed:
          sudo ip link set dev lo mtu 65536

      - name: Post-process perf data
        run: |
          for f in *.perf; do
            # Convert for profiler.firefox.com
            samply import "$f" -o "$f.samply.json.gz" --save-only --presymbolicate
            # Generate flamegraphs
            flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}"
          done

      - name: Format results as Markdown
        id: results
        env:
          EVENT_PATH: ${{ github.event_path }}
          TESTBED: ${{ (inputs.bencher || 'codspeed') == 'gcp' && 'GCP' || (inputs.bencher || 'codspeed') == 'codspeed' && 'CodSpeed' || 'On-prem' }}
        run: |
          SHA=$(cd neqo && git log origin/main -1 --format=%H)
          echo "$SHA" > main-sha.txt
          python3 neqo/.github/scripts/format-bench-results.py results.txt
          # Build final markdown output
          {
            echo "### Benchmark results"
            echo
            if [ -s significant-results.md ]; then
              echo "Significant performance differences relative to $SHA."
              cat significant-results.md
            else
              echo "No significant performance differences relative to $SHA."
            fi
            echo
            echo "<details><summary>All results</summary>"
            echo
            cat all-bench-results.md
            echo
            echo "</details>"
          } > results.md
          rm -f all-bench-results.md significant-results.md
          cat results.md > "$GITHUB_STEP_SUMMARY"
          echo "$TESTBED" > testbed.txt
          cp "$EVENT_PATH" event.json

      - name: Export profiler.firefox.com data
        id: export_samply
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-samply
          path: |
            *.samply.json.gz
            *.syms.json
            binaries
          compression-level: 9

      - name: Export performance data
        id: export_perf
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-perf
          path: |
            *.svg
            *.txt
            *.md
            event.json
            results.*
            results-main.*
            criterion
            criterion-main
          compression-level: 9

      - name: Export PR comment data
        uses: ./neqo/.github/actions/pr-comment-data-export
        with:
          name: ${{ github.workflow }}
          contents: results.md
          log-md: ${{ format('[Download data for `profiler.firefox.com`]({0}) or [download performance comparison data]({1}).', steps.export_samply.outputs.artifact-url, steps.export_perf.outputs.artifact-url) }}

      - name: Fail on regression
        # Don't check for regressions when running on main.
        if: github.ref != 'refs/heads/main' && github.event.merge_group.base_ref != 'refs/heads/main'
        run: |
          if grep -q "Performance has regressed." results.txt; then
            echo "Performance regression detected."
            exit 1
          else
            echo "No performance regression detected."
          fi

      - name: Remove benchmark artifacts
        if: always()
        run: |
          rm -- * || true
          rm -r -- binaries comment-data criterion criterion-main "$COMPARE_TMP" || true