feat: Define a path migration event #3857

Workflow file for this run

	name: cargo bench
	on:
	workflow_call:
	pull_request:
	workflow_dispatch:
	env:
	CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
	CARGO_PROFILE_RELEASE_DEBUG: true
	CARGO_PROFILE_RELEASE_LTO: true
	CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
	RUSTUP_TOOLCHAIN: stable
	PERF_OPT: record -F2999 --call-graph fp -g
	SCCACHE_CACHE_SIZE: 128G
	SCCACHE_DIRECT: true
	MTU: 1504 # https://github.com/microsoft/msquic/issues/4618
	CFLAGS: -fno-omit-frame-pointer
	CXXFLAGS: -fno-omit-frame-pointer
	WORKSPACE: ${{ github.workspace }}
	BASE_REF: ${{ github.event.pull_request.base.ref \|\| 'main' }}

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	permissions:
	contents: read

	jobs:
	build:
	name: Build bench binaries
	runs-on: ubuntu-24.04
	defaults:
	run:
	shell: bash

	steps:
	- name: Checkout mozilla/neqo
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	repository: mozilla/neqo
	path: neqo
	submodules: "recursive"
	persist-credentials: false

	- id: nss-version
	uses: ./neqo/.github/actions/minimum-version
	- uses: ./neqo/.github/actions/build-neqo
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	artifact-name: bench-build
	benches: "true"
	baseline-ref: origin/${{ env.BASE_REF }}
	nss-minimum-version: ${{ steps.nss-version.outputs.minimum }}

	bench:
	name: cargo bench
	runs-on: "self-hosted" # zizmor: ignore[self-hosted-runner]
	needs: [build]
	defaults:
	run:
	shell: bash

	steps:
	- name: Checkout mozilla/neqo
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	repository: mozilla/neqo
	path: neqo
	persist-credentials: false
	clean: false
	- run: \|
	cd neqo
	git fetch --no-tags --depth=1 origin "$BASE_REF"

	- name: Set PATH and environment
	run: \|
	echo "$HOME/.cargo/bin" >> "${GITHUB_PATH}"

	- name: Install Rust
	uses: mozilla/actions/rust@25cb84d060946c0ad6d2c3f79da479b16d180d71 # v1.1.0
	with:
	version: ${{ env.RUSTUP_TOOLCHAIN }}
	tools: flamegraph, samply
	token: ${{ secrets.GITHUB_TOKEN }}
	sccache: true

	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: bench-build
	path: dist

	- name: Set up build artifacts
	run: \|
	chmod +x dist/neqo/* dist/neqo-baseline/*
	mkdir -p binaries/neqo binaries/neqo-baseline
	cp dist/neqo/* binaries/neqo/
	cp dist/neqo-baseline/* binaries/neqo-baseline/
	{
	echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/dist/lib:$LD_LIBRARY_PATH"
	echo "NSS_DB_PATH=$GITHUB_WORKSPACE/dist/test-fixture/db"
	} >> "$GITHUB_ENV"

	# Disable turboboost, hyperthreading, use performance governor, and isolate CPUs with cset.
	- name: Prepare machine
	id: cpu-tuning
	uses: ./neqo/.github/actions/cpu-tuning
	with:
	mtu: ${{ env.MTU }}

	- name: Run cargo bench
	env:
	BENCH_SET: ${{ steps.cpu-tuning.outputs.bench-set }}
	run: \|
	bench_exec() { sudo nice -n -20 env "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" "NSS_DB_PATH=$NSS_DB_PATH" setarch --addr-no-randomize cset proc --set="$BENCH_SET" --exec "$@"; }
	filter_cset() { grep -v '^cset' \|\| test $? = 1; }
	shopt -s extglob nullglob

	# Kill any stale processes from previous runs that may be holding port 4433.
	sudo fuser -k 4433/udp 4433/tcp 2>/dev/null \|\| true

	cd neqo
	# Fix any root-owned files left by a previous run before removing them
	sudo chown -R "$(id -u)" target/criterion 2>/dev/null \|\| true
	rm -rf target/criterion
	# Run the baseline builds first, to establish a baseline.
	for BENCH in ../dist/neqo-baseline/!(neqo-client\|neqo-server); do
	bench_exec "$BENCH" -- --bench --save-baseline baseline --noplot \| filter_cset \| tee -a ../results-baseline.txt
	done
	# Run pull request builds twice, once without perf for baseline comparison, and once with perf for profiling.
	# (Perf seems to introduce some variability in the results.)
	for BENCH in ../dist/neqo/!(neqo-client\|neqo-server); do
	# --baseline-lenient allows us to add new benchmarks in pull requests.
	bench_exec "$BENCH" -- --bench --baseline-lenient baseline --noplot \| filter_cset \| tee -a ../results.txt
	done
	sanitize_name() { echo "$1" \| tr '/ ()' '___-' \| tr -s '_'; }
	for BENCH in ../dist/neqo/!(neqo-client\|neqo-server); do
	BIN_NAME=$(basename "$BENCH")
	VARIANTS=$("$BENCH" --bench --list 2>/dev/null \| sed -n 's/: benchmark$//p' \|\| true)
	if [ -n "$VARIANTS" ]; then
	while IFS= read -r VARIANT; do
	SAFE_NAME="${BIN_NAME}_$(sanitize_name "$VARIANT")"
	# shellcheck disable=SC2086
	bench_exec perf -- $PERF_OPT -o "../$SAFE_NAME.perf" "$BENCH" --bench --noplot --discard-baseline --exact "$VARIANT" \| filter_cset
	done <<< "$VARIANTS"
	else
	# shellcheck disable=SC2086
	bench_exec perf -- $PERF_OPT -o "../$BIN_NAME.perf" "$BENCH" --bench --noplot --discard-baseline \| filter_cset
	fi
	done
	# bench_exec runs as root via sudo nice, so fix ownership of all files it wrote
	sudo chown -R "$(id -u)" target/criterion ../*.perf

	# Re-enable turboboost, hyperthreading, use powersave governor, remove all CPU sets, restore MTU.
	- name: Restore machine
	if: always()
	uses: ./neqo/.github/actions/cpu-tuning
	with:
	mode: restore

	- name: Post-process perf data
	run: \|
	for f in *.perf; do
	# Convert for profiler.firefox.com
	samply import "$f" -o "$f.samply.json.gz" --save-only --presymbolicate
	# Generate flamegraphs
	flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}"
	done

	- name: Format results as Markdown
	id: results
	env:
	EVENT_PATH: ${{ github.event_path }}
	TESTBED: "On-prem"
	run: \|
	SHA=$(cd neqo && git log origin/"$BASE_REF" -1 --format=%H)
	echo "$SHA" > baseline-sha.txt
	python3 neqo/.github/scripts/format-bench-results.py results.txt
	# Build final markdown output
	{
	echo "### Benchmark results"
	echo
	if [ -s significant-results.md ]; then
	echo "Significant performance differences relative to $SHA."
	cat significant-results.md
	else
	echo "No significant performance differences relative to $SHA."
	fi
	echo
	echo "<details><summary>All results</summary>"
	echo
	cat all-bench-results.md
	echo
	echo "</details>"
	} > results.md
	rm -f all-bench-results.md significant-results.md
	cat results.md > "$GITHUB_STEP_SUMMARY"
	echo "$TESTBED" > testbed.txt
	cp "$EVENT_PATH" event.json

	- name: Export profiler.firefox.com data
	id: export_samply
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-samply
	path: \|
	*.samply.json.gz
	*.syms.json
	binaries
	compression-level: 9

	- name: Export performance data
	id: export_perf
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-bench-perf
	path: \|
	*.svg
	*.txt
	*.md
	event.json
	results.*
	results-baseline.*
	compression-level: 9

	- name: Export PR comment data
	uses: ./neqo/.github/actions/pr-comment-data-export
	with:
	name: ${{ github.workflow }}
	contents: results.md
	log-md: ${{ format('[Download data for `profiler.firefox.com`]({0}) or [download performance comparison data]({1}).', steps.export_samply.outputs.artifact-url, steps.export_perf.outputs.artifact-url) }}

	- name: Fail on regression
	# Don't check for regressions when running on main.
	if: github.ref != 'refs/heads/main' && github.event.merge_group.base_ref != 'refs/heads/main'
	run: \|
	if grep -q "Performance has regressed." results.txt; then
	echo "Performance regression detected."
	exit 1
	else
	echo "No performance regression detected."
	fi

	- name: Remove benchmark artifacts
	if: always()
	run: \|
	rm -- * \|\| true
	rm -r -- binaries dist comment-data \|\| true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: Define a path migration event #3857

Workflow file

feat: Define a path migration event #3857

Uh oh!

Workflow file for this run