perfcompare.yml

name: Performance comparison
on:
  workflow_call:
  pull_request:
  workflow_dispatch:
env:
  CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
  CARGO_PROFILE_RELEASE_DEBUG: true
  CARGO_PROFILE_RELEASE_LTO: true
  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
  RUSTUP_TOOLCHAIN: stable
  PERF_OPT: record -F2999 --call-graph fp -g
  SCCACHE_CACHE_SIZE: 128G
  SCCACHE_DIRECT: true
  MTU: 1504 # https://github.com/microsoft/msquic/issues/4618
  CFLAGS: -fno-omit-frame-pointer
  CXXFLAGS: -fno-omit-frame-pointer
  BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  build-neqo:
    name: Build neqo
    runs-on: ubuntu-24.04
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout mozilla/neqo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: mozilla/neqo
          path: neqo
          submodules: "recursive"
          persist-credentials: false
          clean: false

      - id: nss-version
        uses: ./neqo/.github/actions/minimum-version
      - uses: ./neqo/.github/actions/build-neqo
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          artifact-name: build-neqo
          baseline-ref: origin/${{ env.BASE_REF }}
          nss-minimum-version: ${{ steps.nss-version.outputs.minimum }}

  build-msquic:
    name: Build msquic
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: .github/actions
          persist-credentials: false

      - uses: ./.github/actions/quic-build
        with:
          impl: msquic
          token: ${{ secrets.GITHUB_TOKEN }}
          artifact-name: build-msquic

  build-google:
    name: Build google/quiche
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: .github/actions
          persist-credentials: false

      - uses: ./.github/actions/quic-build
        with:
          impl: google
          token: ${{ secrets.GITHUB_TOKEN }}
          artifact-name: build-google

  build-quiche:
    name: Build cloudflare/quiche
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: .github/actions
          persist-credentials: false

      - uses: ./.github/actions/quic-build
        with:
          impl: quiche
          token: ${{ secrets.GITHUB_TOKEN }}
          artifact-name: build-quiche

  build-s2n:
    name: Build s2n-quic
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: .github/actions
          persist-credentials: false

      - uses: ./.github/actions/quic-build
        with:
          impl: s2n
          token: ${{ secrets.GITHUB_TOKEN }}
          artifact-name: build-s2n

  perfcompare:
    name: Performance comparison
    runs-on: "self-hosted" # zizmor: ignore[self-hosted-runner]
    needs: [build-neqo, build-msquic, build-google, build-quiche, build-s2n]
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout mozilla/neqo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: mozilla/neqo
          path: neqo
          persist-credentials: false
          clean: false
      - run: |
          cd neqo
          git fetch --no-tags --depth=1 origin "$BASE_REF"

      - name: Set PATH and environment
        run: |
          echo "$HOME/.cargo/bin" >> "${GITHUB_PATH}"

      - name: Install Rust
        uses: mozilla/actions/rust@27cbe8fb5d338c2861b787e5de10410559065db1 # v1.1.3
        with:
          version: ${{ env.RUSTUP_TOOLCHAIN }}
          tools: hyperfine, flamegraph, samply
          token: ${{ secrets.GITHUB_TOKEN }}
          sccache: true

      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: build-neqo
          path: build-neqo
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: build-msquic
          path: build-msquic
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: build-google
          path: build-google
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: build-quiche
          path: build-quiche
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: build-s2n
          path: build-s2n

      - name: Set up build artifacts
        run: |
          # Clean stale binaries dir from previous workflow runs.
          # perfcompare.py recreates and populates binaries/ during the run.
          rm -rf binaries
          chmod +x build-neqo/neqo/* build-neqo/neqo-baseline/* \
            build-msquic/* build-google/* build-quiche/* build-s2n/*
          {
            echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build-neqo/lib:$LD_LIBRARY_PATH"
            echo "TEST_FIXTURE_DB=$GITHUB_WORKSPACE/build-neqo/test-fixture/db"
            echo "NSS_DB_PATH=$GITHUB_WORKSPACE/build-neqo/test-fixture/db" # TODO: Remove once baseline uses nss-rs >= 0.11.0
          } >> "$GITHUB_ENV"

      # Disable turboboost, hyperthreading, use performance governor, and isolate CPUs with cset.
      # See https://github.com/microsoft/msquic/issues/4618#issuecomment-2422611592
      - name: Prepare machine
        id: cpu-tuning
        uses: ./neqo/.github/actions/cpu-tuning
        with:
          mtu: ${{ env.MTU }}

      # Compare various configurations of neqo against msquic and google/quiche, and gather perf data
      # during the hyperfine runs.
      - name: Compare QUIC implementations
        env:
          WORKSPACE: ${{ github.workspace }}
          SERVER_SET: ${{ steps.cpu-tuning.outputs.server-set }}
          CLIENT_SET: ${{ steps.cpu-tuning.outputs.client-set }}
        run: |
          python3 neqo/.github/scripts/perfcompare.py \
            --host 127.0.0.1 \
            --port 4433 \
            --size 33554432 \
            --runs 100 \
            --workspace "$WORKSPACE" \
            --perf-opt "$PERF_OPT" \
            --server-set "$SERVER_SET" \
            --client-set "$CLIENT_SET"
          # perfcompare.py runs under sudo nice, so output files are root-owned; fix for post-processing
          sudo chown -R "$(id -u)" ./*.perf hyperfine hyperfine-baseline 2>/dev/null || true

      # Re-enable turboboost, hyperthreading, use powersave governor, remove all CPU sets, restore MTU.
      - name: Restore machine
        if: always()
        uses: ./neqo/.github/actions/cpu-tuning
        with:
          mode: restore

      - name: Post-process perf data
        run: |
          for f in *.perf; do
            # Convert for profiler.firefox.com
            samply import "$f" -o "$f.samply.json.gz" --save-only --presymbolicate
            # Generate flamegraphs
            flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}"
          done

      - name: Format results as Markdown
        id: results
        env:
          EVENT_PATH: ${{ github.event_path }}
          TESTBED: "On-prem"
        run: |
          SHA=$(cd neqo && git log origin/"$BASE_REF" -1 --format=%H)
          echo "$SHA" > baseline-sha.txt
          {
            echo "### Client/server transfer results"
            echo
            echo "Performance differences relative to $SHA."
            echo
            # Extract rows with significant changes (red or green heart).
            if grep -q ':broken_heart:\|:green_heart:' comparison.md; then
              # Print all lines up to and including the first separator line (header)
              awk 'BEGIN{found=0} /^[|][ -|:]+[|]$/ {found=1} {print} found{exit}' comparison.md
              grep ':broken_heart:\|:green_heart:' comparison.md
              echo
            else
              echo "No significant performance differences."
              echo
            fi
            echo "Table above only shows statistically significant changes. See all results below."
            echo "<details><summary>All results</summary>"
            echo
            cat comparison.md
            echo
            echo "</details>"
          } >> results.md
          cat results.md > "$GITHUB_STEP_SUMMARY"
          echo "$TESTBED" > testbed.txt
          cp "$EVENT_PATH" event.json

      - name: Export profiler.firefox.com data
        id: export_samply
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-perfcompare-samply
          path: |
            *.samply.json.gz
            binaries
          compression-level: 9

      - name: Export performance data
        id: export_perf
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-perfcompare-perf
          path: |
            *.svg
            *.txt
            *.md
            event.json
            results.*
            results-baseline.*
            hyperfine
            hyperfine-baseline
          compression-level: 9

      - name: Export PR comment data
        uses: ./neqo/.github/actions/pr-comment-data-export
        with:
          name: ${{ github.workflow }}
          contents: results.md
          log-md: ${{ format('[Download data for `profiler.firefox.com`]({0}) or [download performance comparison data]({1}).', steps.export_samply.outputs.artifact-url, steps.export_perf.outputs.artifact-url) }}

      - name: Fail on regression
        # Don't check for regressions when running on main.
        if: github.ref != 'refs/heads/main' && github.event.merge_group.base_ref != 'refs/heads/main'
        run: |
          if grep -q "Performance has regressed." results.txt; then
            echo "Performance regression detected."
            exit 1
          else
            echo "No performance regression detected."
          fi

      - name: Remove benchmark artifacts
        if: always()
        run: |
          # Kill any server processes that survived the run (e.g. after workflow cancellation).
          # Tags are _tag() values from IMPLS in perfcompare.py: basename of server binary, truncated to 15 chars.
          for tag in neqo-server quicinteropserv quic_server quiche-server s2n-quic-qns; do
            sudo pkill -9 "$tag" 2>/dev/null || true
          done
          sudo fuser -k 4433/udp 4433/tcp 2>/dev/null || true
          rm -- * || true
          rm -r -- binaries build-neqo build-msquic build-google build-quiche build-s2n \
            comment-data hyperfine hyperfine-baseline || true