cargo bench #1973
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: cargo bench | |
| on: | |
| workflow_call: | |
| pull_request: | |
| branches: ["main"] | |
| merge_group: | |
| workflow_dispatch: | |
| inputs: | |
| bencher: | |
| type: choice | |
| description: Which benchmarking testbed to run on. | |
| required: true | |
| default: on-prem | |
| options: | |
| - on-prem | |
| - gcp | |
| env: | |
| CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true | |
| CARGO_PROFILE_RELEASE_DEBUG: true | |
| RUSTUP_TOOLCHAIN: stable | |
| PERF_OPT: record -F2999 --call-graph fp -g | |
| SCCACHE_CACHE_SIZE: 128G | |
| SCCACHE_DIRECT: true | |
| MTU: 1504 # https://github.com/microsoft/msquic/issues/4618 | |
| CFLAGS: -fno-omit-frame-pointer | |
| CXXFLAGS: -fno-omit-frame-pointer | |
| WORKSPACE: ${{ github.workspace}} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| bench: | |
| name: cargo bench | |
| runs-on: ${{ inputs.bencher == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || 'self-hosted' }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout mozilla/neqo | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: mozilla/neqo | |
| path: neqo | |
| submodules: 'recursive' | |
| persist-credentials: false | |
| clean: false | |
| - run: | | |
| cd neqo | |
| git fetch --no-tags --depth=1 origin main | |
| - name: Set PATH and environment | |
| run: | | |
| echo "$HOME/.cargo/bin" >> "${GITHUB_PATH}" | |
| - name: Install Rust | |
| uses: ./neqo/.github/actions/rust | |
| with: | |
| version: ${{ env.RUSTUP_TOOLCHAIN }} | |
| tools: flamegraph, samply | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| workspaces: | | |
| neqo | |
| - name: Get minimum NSS version | |
| id: nss-version | |
| run: | | |
| cd neqo | |
| cat neqo-crypto/min_version.txt > versions.txt | |
| git show origin/main:neqo-crypto/min_version.txt >> versions.txt | |
| # Use the maximum version from both branches. | |
| echo "minimum=$(sort -u versions.txt | tail -n1)" >> "$GITHUB_OUTPUT" | |
| - name: Install NSS | |
| id: nss | |
| uses: ./neqo/.github/actions/nss | |
| with: | |
| minimum-version: ${{ steps.nss-version.outputs.minimum }} | |
| - name: Build neqo | |
| run: | | |
| # See https://github.com/flamegraph-rs/flamegraph for why we append to RUSTFLAGS here. | |
| export RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment -C force-frame-pointers=yes $RUSTFLAGS" | |
| echo "RUSTFLAGS=$RUSTFLAGS" >> "$GITHUB_ENV" | |
| mkdir -p binaries/neqo-main | |
| mkdir -p binaries/neqo | |
| cd neqo | |
| git checkout origin/main | |
| cargo build --locked --release --bin neqo-client --bin neqo-server | |
| cp target/release/neqo-client ../binaries/neqo-main/ | |
| cp target/release/neqo-server ../binaries/neqo-main/ | |
| git checkout - | |
| cargo build --locked --release --bin neqo-client --bin neqo-server | |
| cp target/release/neqo-client ../binaries/neqo/ | |
| cp target/release/neqo-server ../binaries/neqo/ | |
| # Disable turboboost, hyperthreading and use performance governor. | |
| # Also creates "cpu23", "cpu2" and "cpu3" CPU sets for use with cset. | |
| # On the bencher, logical cores 2 and 3 have been isolated for use by the benchmarks. | |
| - name: Prepare machine | |
| run: sudo /root/bin/prep.sh | |
| - name: Run cargo bench | |
| env: | |
| NSS_DB_PATH: ${{ github.workspace }}/neqo/test-fixture/db | |
| run: | | |
| sudo ip link set dev lo mtu "$MTU" | |
| cd neqo | |
| rm -rf target/criterion | |
| git checkout origin/main | |
| cargo bench --locked --workspace --features bench --no-run 2>&1 | tee benches-main.txt | |
| BENCHES_MAIN=$(grep Executable benches-main.txt | cut -d\( -f2 | cut -d\) -f1 | tr -s '\n' ' ') | |
| # Run the builds from the main branch first, to establish a baseline. | |
| for BENCH in $BENCHES_MAIN; do | |
| cp "$BENCH" ../binaries/neqo-main/ | |
| # shellcheck disable=SC2086 | |
| nice -n -20 setarch --addr-no-randomize cset proc --set=cpu23 --exec \ | |
| $BENCH -- --bench --save-baseline main --noplot | { grep -v '^cset' || test $? = 1; } | tee -a ../results-main.txt | |
| done | |
| # Copy the main branch results to a separate directory for bencher.dev. | |
| cp -r target/criterion ../criterion-main | |
| git checkout - | |
| cargo bench --locked --workspace --features bench --no-run 2>&1 | tee benches.txt | |
| BENCHES=$(grep Executable benches.txt | cut -d\( -f2 | cut -d\) -f1 | tr -s '\n' ' ') | |
| # Run pull request builds twice, once without perf for baseline comparison, and once with perf for profiling. | |
| # (Perf seems to introduce some variability in the results.) | |
| for BENCH in $BENCHES; do | |
| cp "$BENCH" ../binaries/neqo/ | |
| # shellcheck disable=SC2086 | |
| # --baseline-lenient allows us to add new benchmarks in pull requests. | |
| nice -n -20 setarch --addr-no-randomize cset proc --set=cpu23 --exec \ | |
| $BENCH -- --bench --baseline-lenient main --noplot | { grep -v '^cset' || test $? = 1; } | tee -a ../results.txt | |
| done | |
| for BENCH in $BENCHES; do | |
| NAME=$(basename "$BENCH" | cut -d- -f1) | |
| # shellcheck disable=SC2086 | |
| nice -n -20 setarch --addr-no-randomize cset proc --set=cpu23 --exec \ | |
| perf -- $PERF_OPT -o "../$NAME.perf" $BENCH --bench --noplot --discard-baseline | { grep -v '^cset' || test $? = 1; } | |
| done | |
| cp -r target/criterion ../criterion | |
| # Re-enable turboboost, hyperthreading and use powersave governor. Remove all CPU sets. | |
| - name: Restore machine | |
| if: success() || failure() || cancelled() | |
| run: | | |
| sudo /root/bin/unprep.sh | |
| # In case the previous test failed: | |
| sudo ip link set dev lo mtu 65536 | |
| - name: Post-process perf data | |
| run: | | |
| for f in *.perf; do | |
| # Convert for profiler.firefox.com | |
| samply import "$f" -o "$f.samply.json.gz" --save-only --presymbolicate | |
| # Generate flamegraphs | |
| flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}" | |
| done | |
| - name: Format results as Markdown | |
| id: results | |
| env: | |
| EVENT_PATH: ${{ github.event_path }} | |
| TESTBED: ${{ inputs.bencher == 'gcp' && 'GCP' || 'On-prem' }} | |
| run: | | |
| SHA=$(cd neqo && git log origin/main -1 --format=%H) | |
| echo "$SHA" > main-sha.txt | |
| { | |
| echo "### Benchmark results" | |
| echo | |
| echo "Performance differences relative to $SHA." | |
| echo | |
| sed -E -e 's/^ //gi' \ | |
| -e '/cset:.*last message/d' \ | |
| -e '/Criterion.rs ERROR: Error in Gnuplot:.*$/d' \ | |
| -e 's/((change|time|thrpt):[^%]*% )([^%]*%)(.*)/\1<b>\3<\/b>\4/gi' results.txt |\ | |
| perl -p0e 's/(Change within noise threshold\.|No change in performance detected\.|Performance has regressed\.|Performance has improved\.)\n([a-zA-Z0-9][^F][^\s]*[\-\/])/$1\n\n$2/g' |\ | |
| perl -p -0777 -e 's/(.*?)\n(.*?)(((No change|Change within|Performance has).*?)(\nFound .*?)?)?\n\n/<details><summary>$1: $4<\/summary><pre>\n$2$6<\/pre><\/details>\n/gs' |\ | |
| sed -E -e 's/(Performance has regressed.)/:broken_heart: <b>\1<\/b>/gi' \ | |
| -e 's/(Performance has improved.)/:green_heart: <b>\1<\/b>/gi' \ | |
| -e 's/^ +((<\/pre>|Found).*)/\1/gi' \ | |
| -e 's/^<details>(.*Performance has.*)/<details open>\1/gi' | |
| } >> results.md | |
| cat results.md > "$GITHUB_STEP_SUMMARY" | |
| echo "$TESTBED" > testbed.txt | |
| cp "$EVENT_PATH" event.json | |
| - name: Export profiler.firefox.com data | |
| id: export_samply | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-samply | |
| path: | | |
| *.samply.json.gz | |
| *.syms.json | |
| binaries | |
| compression-level: 9 | |
| - name: Export performance data | |
| id: export_perf | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-perf | |
| path: | | |
| *.svg | |
| *.txt | |
| *.md | |
| event.json | |
| results.* | |
| results-main.* | |
| criterion | |
| criterion-main | |
| compression-level: 9 | |
| - name: Export PR comment data | |
| uses: ./neqo/.github/actions/pr-comment-data-export | |
| with: | |
| name: ${{ github.workflow }} | |
| contents: results.md | |
| log-md: ${{ format('[Download data for `profiler.firefox.com`]({0}) or [download performance comparison data]({1}).', steps.export_samply.outputs.artifact-url, steps.export_perf.outputs.artifact-url) }} | |
| - name: Fail on regression | |
| # Don't check for regressions when running on main. | |
| if: github.ref != 'refs/heads/main' && github.event.merge_group.base_ref != 'refs/heads/main' | |
| run: | | |
| if grep -q "Performance has regressed." results.txt; then | |
| echo "Performance regression detected." | |
| exit 1 | |
| else | |
| echo "No performance regression detected." | |
| fi | |
| - name: Remove benchmark artifacts | |
| if: always() | |
| run: | | |
| rm -- * || true | |
| rm -r -- binaries comment-data criterion criterion-main "$COMPARE_TMP" || true |