build(deps): bump CodSpeedHQ/action from 4.15.0 to 4.15.1 #3109

Workflow file for this run

.github/workflows/perfcompare.yml at 3485c8f

	name: Performance comparison
	on:
	workflow_call:
	pull_request:
	workflow_dispatch:
	env:
	CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
	CARGO_PROFILE_RELEASE_DEBUG: true
	CARGO_PROFILE_RELEASE_LTO: true
	CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
	RUSTUP_TOOLCHAIN: stable
	PERF_OPT: record -F2999 --call-graph fp -g
	SCCACHE_CACHE_SIZE: 128G
	SCCACHE_DIRECT: true
	MTU: 1504 # https://github.com/microsoft/msquic/issues/4618
	CFLAGS: -fno-omit-frame-pointer
	CXXFLAGS: -fno-omit-frame-pointer
	BASE_REF: ${{ github.event.pull_request.base.ref \|\| 'main' }}

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	permissions:
	contents: read

	jobs:
	build-neqo:
	name: Build neqo
	runs-on: ubuntu-24.04
	defaults:
	run:
	shell: bash

	steps:
	- name: Checkout mozilla/neqo
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	repository: mozilla/neqo
	path: neqo
	submodules: "recursive"
	persist-credentials: false
	clean: false

	- id: nss-version
	uses: ./neqo/.github/actions/minimum-version
	- uses: ./neqo/.github/actions/build-neqo
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	artifact-name: build-neqo
	baseline-ref: origin/${{ env.BASE_REF }}
	nss-minimum-version: ${{ steps.nss-version.outputs.minimum }}

	build-msquic:
	name: Build msquic
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	sparse-checkout: .github/actions
	persist-credentials: false

	- uses: ./.github/actions/quic-build
	with:
	impl: msquic
	token: ${{ secrets.GITHUB_TOKEN }}
	artifact-name: build-msquic

	build-google:
	name: Build google/quiche
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	sparse-checkout: .github/actions
	persist-credentials: false

	- uses: ./.github/actions/quic-build
	with:
	impl: google
	token: ${{ secrets.GITHUB_TOKEN }}
	artifact-name: build-google

	build-quiche:
	name: Build cloudflare/quiche
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	sparse-checkout: .github/actions
	persist-credentials: false

	- uses: ./.github/actions/quic-build
	with:
	impl: quiche
	token: ${{ secrets.GITHUB_TOKEN }}
	artifact-name: build-quiche

	build-s2n:
	name: Build s2n-quic
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	sparse-checkout: .github/actions
	persist-credentials: false

	- uses: ./.github/actions/quic-build
	with:
	impl: s2n
	token: ${{ secrets.GITHUB_TOKEN }}
	artifact-name: build-s2n

	perfcompare:
	name: Performance comparison
	runs-on: "self-hosted" # zizmor: ignore[self-hosted-runner]
	needs: [build-neqo, build-msquic, build-google, build-quiche, build-s2n]
	defaults:
	run:
	shell: bash

	steps:
	- name: Checkout mozilla/neqo
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	repository: mozilla/neqo
	path: neqo
	persist-credentials: false
	clean: false
	- run: \|
	cd neqo
	git fetch --no-tags --depth=1 origin "$BASE_REF"

	- name: Set PATH and environment
	run: \|
	echo "$HOME/.cargo/bin" >> "${GITHUB_PATH}"

	- name: Install Rust
	uses: mozilla/actions/rust@27cbe8fb5d338c2861b787e5de10410559065db1 # v1.1.3
	with:
	version: ${{ env.RUSTUP_TOOLCHAIN }}
	tools: hyperfine, flamegraph, samply
	token: ${{ secrets.GITHUB_TOKEN }}
	sccache: true

	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: build-neqo
	path: build-neqo
	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: build-msquic
	path: build-msquic
	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: build-google
	path: build-google
	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: build-quiche
	path: build-quiche
	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: build-s2n
	path: build-s2n

	- name: Set up build artifacts
	run: \|
	# Clean stale binaries dir from previous workflow runs.
	# perfcompare.py recreates and populates binaries/ during the run.
	rm -rf binaries
	chmod +x build-neqo/neqo/* build-neqo/neqo-baseline/* \
	build-msquic/* build-google/* build-quiche/* build-s2n/*
	{
	echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build-neqo/lib:$LD_LIBRARY_PATH"
	echo "TEST_FIXTURE_DB=$GITHUB_WORKSPACE/build-neqo/test-fixture/db"
	echo "NSS_DB_PATH=$GITHUB_WORKSPACE/build-neqo/test-fixture/db" # TODO: Remove once baseline uses nss-rs >= 0.11.0
	} >> "$GITHUB_ENV"

	# Disable turboboost, hyperthreading, use performance governor, and isolate CPUs with cset.
	# See https://github.com/microsoft/msquic/issues/4618#issuecomment-2422611592
	- name: Prepare machine
	id: cpu-tuning
	uses: ./neqo/.github/actions/cpu-tuning
	with:
	mtu: ${{ env.MTU }}

	# Compare various configurations of neqo against msquic and google/quiche, and gather perf data
	# during the hyperfine runs.
	- name: Compare QUIC implementations
	env:
	WORKSPACE: ${{ github.workspace }}
	SERVER_SET: ${{ steps.cpu-tuning.outputs.server-set }}
	CLIENT_SET: ${{ steps.cpu-tuning.outputs.client-set }}
	run: \|
	python3 neqo/.github/scripts/perfcompare.py \
	--host 127.0.0.1 \
	--port 4433 \
	--size 33554432 \
	--runs 100 \
	--workspace "$WORKSPACE" \
	--perf-opt "$PERF_OPT" \
	--server-set "$SERVER_SET" \
	--client-set "$CLIENT_SET"
	# perfcompare.py runs under sudo nice, so output files are root-owned; fix for post-processing
	sudo chown -R "$(id -u)" ./*.perf hyperfine hyperfine-baseline 2>/dev/null \|\| true

	# Re-enable turboboost, hyperthreading, use powersave governor, remove all CPU sets, restore MTU.
	- name: Restore machine
	if: always()
	uses: ./neqo/.github/actions/cpu-tuning
	with:
	mode: restore

	- name: Post-process perf data
	run: \|
	for f in *.perf; do
	# Convert for profiler.firefox.com
	samply import "$f" -o "$f.samply.json.gz" --save-only --presymbolicate
	# Generate flamegraphs
	flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}"
	done

	- name: Format results as Markdown
	id: results
	env:
	EVENT_PATH: ${{ github.event_path }}
	TESTBED: "On-prem"
	run: \|
	SHA=$(cd neqo && git log origin/"$BASE_REF" -1 --format=%H)
	echo "$SHA" > baseline-sha.txt
	{
	echo "### Client/server transfer results"
	echo
	echo "Performance differences relative to $SHA."
	echo
	# Extract rows with significant changes (red or green heart).
	if grep -q ':broken_heart:\\|:green_heart:' comparison.md; then
	# Print all lines up to and including the first separator line (header)
	awk 'BEGIN{found=0} /^[\|][ -\|:]+[\|]$/ {found=1} {print} found{exit}' comparison.md
	grep ':broken_heart:\\|:green_heart:' comparison.md
	echo
	else
	echo "No significant performance differences."
	echo
	fi
	echo "Table above only shows statistically significant changes. See all results below."
	echo "<details><summary>All results</summary>"
	echo
	cat comparison.md
	echo
	echo "</details>"
	} >> results.md
	cat results.md > "$GITHUB_STEP_SUMMARY"
	echo "$TESTBED" > testbed.txt
	cp "$EVENT_PATH" event.json

	- name: Export profiler.firefox.com data
	id: export_samply
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-perfcompare-samply
	path: \|
	*.samply.json.gz
	binaries
	compression-level: 9

	- name: Export performance data
	id: export_perf
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-perfcompare-perf
	path: \|
	*.svg
	*.txt
	*.md
	event.json
	results.*
	results-baseline.*
	hyperfine
	hyperfine-baseline
	compression-level: 9

	- name: Export PR comment data
	uses: ./neqo/.github/actions/pr-comment-data-export
	with:
	name: ${{ github.workflow }}
	contents: results.md
	log-md: ${{ format('[Download data for `profiler.firefox.com`]({0}) or [download performance comparison data]({1}).', steps.export_samply.outputs.artifact-url, steps.export_perf.outputs.artifact-url) }}

	- name: Fail on regression
	# Don't check for regressions when running on main.
	if: github.ref != 'refs/heads/main' && github.event.merge_group.base_ref != 'refs/heads/main'
	run: \|
	if grep -q "Performance has regressed." results.txt; then
	echo "Performance regression detected."
	exit 1
	else
	echo "No performance regression detected."
	fi

	- name: Remove benchmark artifacts
	if: always()
	run: \|
	# Kill any server processes that survived the run (e.g. after workflow cancellation).
	# Tags are _tag() values from IMPLS in perfcompare.py: basename of server binary, truncated to 15 chars.
	for tag in neqo-server quicinteropserv quic_server quiche-server s2n-quic-qns; do
	sudo pkill -9 "$tag" 2>/dev/null \|\| true
	done
	sudo fuser -k 4433/udp 4433/tcp 2>/dev/null \|\| true
	rm -- * \|\| true
	rm -r -- binaries build-neqo build-msquic build-google build-quiche build-s2n \
	comment-data hyperfine hyperfine-baseline \|\| true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

build(deps): bump CodSpeedHQ/action from 4.15.0 to 4.15.1 #3109

Workflow file

build(deps): bump CodSpeedHQ/action from 4.15.0 to 4.15.1 #3109

Uh oh!

Workflow file for this run