docs(CuckooFilter): correct when IsFull clears #284

Workflow file for this run

.github/workflows/benchmarks.yml at 843b75a

	name: Benchmarks

	on:
	push:
	branches: [ main ]
	paths:
	- 'src/**'
	- '.github/workflows/benchmarks.yml'
	pull_request:
	branches: [ main ]
	paths:
	- 'src/**'
	- '.github/workflows/benchmarks.yml'

	# The core suite is ~300 cases and, at full BenchmarkDotNet accuracy (Job.Default x2
	# launches, see CiConfig.cs), one pass takes ~3h. The PR path runs it TWICE on one
	# runner (PR head + main base, same-runner A/B, so hardware variance cancels), which
	# blew past GitHub's 6h job ceiling and was cancelled on every PR (#217). Rather than
	# trade away measurement accuracy, we shard the suite across a parallel matrix: each
	# shard measures only its slice (head + base) on its own runner, so wall time is
	# ~3h / SHARD_TOTAL at full accuracy, and an aggregate job stitches the per-shard JSON
	# back together for the regression comment / gh-pages publish.
	#
	# The same-runner A/B invariant is preserved because sharding is by benchmark class
	# (Program.cs `--shard`): a class's PR-head and main-base measurements always run
	# together on the one shard runner, so the per-benchmark delta still cancels hardware.
	env:
	# Keep in sync with the matrix.shard list below.
	SHARD_TOTAL: '6'

	jobs:
	benchmark-shard:
	name: benchmark (shard ${{ matrix.shard }})
	runs-on: ubuntu-latest
	# Steady state: head slice (~30 min) + base slice (~30 min) ~= 60 min. The one
	# transitional run of THIS PR is slower (see the base step) and may be cancelled
	# here — that is no worse than the status quo (the gate is cancelled on every PR
	# today) and self-heals the moment `--shard` lands on main.
	timeout-minutes: 120
	strategy:
	fail-fast: false
	matrix:
	# Must enumerate 0 .. SHARD_TOTAL-1.
	shard: [0, 1, 2, 3, 4, 5]

	permissions:
	contents: read

	defaults:
	run:
	working-directory: src

	steps:
	# NOTE: do NOT add `filter: tree:0` here. A blobless partial clone makes the repo
	# advertise every commit as "have" while missing the underlying objects, which
	# breaks the base `git worktree add <base-sha>` below (and, in the aggregate job,
	# github-action-benchmark's own gh-pages fetch) with "missing blob object … did
	# not send all necessary objects" (#217). fetch-depth: 0 is required for the
	# base worktree.
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Setup .NET
	uses: actions/setup-dotnet@v4
	with:
	dotnet-version: 8.0.x

	- name: Run head benchmarks (shard ${{ matrix.shard }})
	# working-directory must be the project folder, not src/. BenchmarkDotNet writes
	# its artifacts to ./BenchmarkDotNet.Artifacts relative to the process CWD.
	working-directory: src/Celerity.Benchmarks
	run: dotnet run --configuration Release -- --ci --shard "${SHARD_TOTAL}" "${{ matrix.shard }}"

	- name: Stage head report
	run: \|
	set -euo pipefail
	mkdir -p /tmp/reports
	report=$(ls Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results/*-report-full.json \| head -n 1)
	cp "$report" "/tmp/reports/head-shard-${{ matrix.shard }}.json"
	echo "Staged head shard ${{ matrix.shard }}: $report"

	- name: Run base (main) benchmarks for this shard on the same runner
	# Same-runner A/B: build and benchmark the main tip back-to-back with the PR head
	# on THIS runner so hardware variance cancels (hosted runners vary 20-50%
	# run-to-run, so a stored cross-runner baseline would be noise-dominated).
	#
	# TRANSITIONAL: until this PR's `--shard` support lands on main, the base tip's
	# `--ci` does not understand `--shard` and runs the FULL ~3h suite here, so these
	# base steps will exceed the job timeout and be cancelled on THIS PR only. That
	# matches today's behaviour (the gate is already cancelled on every PR) and the
	# workflow self-heals once merged — every later PR's base honours `--shard`.
	if: github.event_name == 'pull_request'
	working-directory: ${{ github.workspace }}
	run: \|
	set -euo pipefail
	git fetch origin main
	BASE_SHA=$(git rev-parse origin/main)
	echo "Benchmarking base (main) at $BASE_SHA, shard ${{ matrix.shard }}"
	git worktree add /tmp/base-tree "$BASE_SHA"
	cd /tmp/base-tree/src/Celerity.Benchmarks
	dotnet run --configuration Release -- --ci --shard "${SHARD_TOTAL}" "${{ matrix.shard }}"
	report=$(ls BenchmarkDotNet.Artifacts/results/*-report-full.json \| head -n 1)
	cp "$report" "/tmp/reports/base-shard-${{ matrix.shard }}.json"
	echo "Staged base shard ${{ matrix.shard }}: $report"

	- name: Upload shard reports
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-shard-${{ matrix.shard }}
	path: /tmp/reports/**
	if-no-files-found: error
	retention-days: 30

	aggregate:
	name: aggregate & report
	needs: benchmark-shard
	# Run even if some shards were cancelled/failed (e.g. the transitional base run
	# above) so partial results are still reported where possible.
	if: always()
	runs-on: ubuntu-latest
	timeout-minutes: 20

	permissions:
	contents: write
	pull-requests: write

	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Download all shard reports
	uses: actions/download-artifact@v4
	with:
	pattern: benchmark-shard-*
	merge-multiple: true
	path: /tmp/reports

	- name: Merge shard reports
	id: merge
	run: \|
	set -euo pipefail
	shopt -s nullglob
	head_files=(/tmp/reports/head-shard-*.json)
	if [ ${#head_files[@]} -eq 0 ]; then
	echo "No head shard reports found (all shards cancelled/failed) — nothing to report."
	echo "have_head=false" >> "$GITHUB_OUTPUT"
	exit 0
	fi
	echo "have_head=true" >> "$GITHUB_OUTPUT"

	# Stitch every shard's joined report back into one: keep shard 0's metadata,
	# concatenate all shards' Benchmarks arrays.
	jq -s '.[0] * {Benchmarks: ([.[].Benchmarks] \| add)}' "${head_files[@]}" > /tmp/pr-report-full.json
	echo "Merged $(jq '.Benchmarks \| length' /tmp/pr-report-full.json) head benchmarks from ${#head_files[@]} shard(s)."

	base_files=(/tmp/reports/base-shard-*.json)
	if [ ${#base_files[@]} -gt 0 ]; then
	jq -s '.[0] * {Benchmarks: ([.[].Benchmarks] \| add)}' "${base_files[@]}" > /tmp/base-report-full.json
	echo "have_base=true" >> "$GITHUB_OUTPUT"
	echo "Merged $(jq '.Benchmarks \| length' /tmp/base-report-full.json) base benchmarks from ${#base_files[@]} shard(s)."
	else
	echo "have_base=false" >> "$GITHUB_OUTPUT"
	echo "No base shard reports (main push, or base run cancelled)."
	fi

	- name: Resolve base SHA (for the comment footer)
	if: github.event_name == 'pull_request' && steps.merge.outputs.have_head == 'true'
	run: \|
	set -euo pipefail
	git fetch origin main
	echo "BASE_SHA=$(git rev-parse origin/main)" >> "$GITHUB_ENV"

	- name: Upload merged reports
	if: steps.merge.outputs.have_head == 'true'
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-results
	path: /tmp/*-report-full.json
	if-no-files-found: warn
	retention-days: 30

	# ----- PR path: post the same-runner A/B comment -----
	- name: Post benchmark comment
	if: github.event_name == 'pull_request' && steps.merge.outputs.have_head == 'true' && steps.merge.outputs.have_base == 'true'
	uses: actions/github-script@v7
	env:
	ALERT_THRESHOLD_RATIO: '1.10'
	with:
	github-token: ${{ secrets.GITHUB_TOKEN }}
	script: \|
	const fs = require('fs');

	const pr = JSON.parse(fs.readFileSync('/tmp/pr-report-full.json', 'utf8'));
	const base = JSON.parse(fs.readFileSync('/tmp/base-report-full.json', 'utf8'));

	// Index base results by full benchmark name.
	const baseMap = new Map();
	for (const b of base.Benchmarks) {
	baseMap.set(b.FullName, b.Statistics);
	}

	const formatNs = (ns) => {
	if (ns == null) return 'n/a';
	if (ns < 1000) return `${ns.toFixed(1)} ns`;
	if (ns < 1_000_000) return `${(ns/1000).toFixed(2)} μs`;
	if (ns < 1_000_000_000) return `${(ns/1_000_000).toFixed(2)} ms`;
	return `${(ns/1_000_000_000).toFixed(2)} s`;
	};

	const threshold = parseFloat(process.env.ALERT_THRESHOLD_RATIO);
	const thresholdPct = ((threshold - 1) * 100).toFixed(0);
	// Each entry: { name, isHasher, prCell, stdCell, baseCell, deltaCell, flag }
	// where flag is 'regression' \| 'improvement' \| null.
	const entries = [];
	let regressions = 0;
	let improvements = 0;
	let errored = 0;

	for (const b of pr.Benchmarks) {
	const name = b.FullName;
	// BenchmarkDotNet emits a benchmark with null Statistics and empty
	// Measurements when that case errored or was not run (a throwing
	// [GlobalSetup], a cancelled shard, an OOM, ...). Surface it as a row
	// rather than dereferencing null — one errored case must never crash
	// the aggregate job and red-X the whole PR gate.
	if (b.Statistics == null) {
	errored++;
	const baseStats = baseMap.get(name);
	entries.push({
	name,
	isHasher: /Hasher/.test(name),
	prCell: '⚠️ errored',
	stdCell: 'n/a',
	baseCell: baseStats ? formatNs(baseStats.Mean) : 'n/a',
	deltaCell: 'no measurements',
	flag: null,
	});
	continue;
	}
	const prMean = b.Statistics.Mean;
	const prStdDev = b.Statistics.StandardDeviation;
	const baseStats = baseMap.get(name);
	let deltaCell = baseMap.has(name) ? '⚠️ base errored' : '🆕 new';
	let flag = null;
	if (baseStats) {
	const baseMean = baseStats.Mean;
	const baseStdDev = baseStats.StandardDeviation;
	const ratio = prMean / baseMean;
	const pct = (ratio - 1) * 100;
	const sign = pct >= 0 ? '+' : '';
	deltaCell = `${sign}${pct.toFixed(1)}%`;
	// Flag only when the change clears the threshold AND exceeds the
	// combined run-to-run noise of both measurements.
	const beyondNoise = Math.abs(prMean - baseMean) > (prStdDev + baseStdDev);
	if (ratio >= threshold && beyondNoise) {
	deltaCell += ' ⚠️';
	flag = 'regression';
	regressions++;
	} else if (ratio <= (1 / threshold) && beyondNoise) {
	deltaCell += ' ✅';
	flag = 'improvement';
	improvements++;
	}
	}
	entries.push({
	name,
	// Hasher throughput benchmarks (StringHasherBenchmark,
	// IntegerHasherBenchmark) get their own section; everything else
	// is a collection benchmark.
	isHasher: /Hasher/.test(name),
	prCell: formatNs(prMean),
	stdCell: formatNs(prStdDev),
	baseCell: baseStats ? formatNs(baseStats.Mean) : 'n/a',
	deltaCell,
	flag,
	});
	}

	let subtitle;
	if (regressions > 0) {
	subtitle = `${regressions} regression${regressions === 1 ? '' : 's'} ⚠️ vs main` +
	(improvements > 0 ? `, ${improvements} improvement${improvements === 1 ? '' : 's'} ✅` : '') +
	` (rows past ±${thresholdPct}% beyond noise).`;
	} else if (improvements > 0) {
	subtitle = `No regressions — ${improvements} improvement${improvements === 1 ? '' : 's'} ✅ vs main.`;
	} else {
	subtitle = `No significant change vs main (all within ±${thresholdPct}% or noise).`;
	}
	if (errored > 0) {
	subtitle += ` ⚠️ ${errored} benchmark${errored === 1 ? '' : 's'} errored (no measurements) — see the table.`;
	}

	const baseSha = (process.env.BASE_SHA \|\| '').slice(0, 7);
	const footer = `<sub>Same-runner A/B (sharded ${process.env.SHARD_TOTAL \|\| ''}-way): main (\`${baseSha}\`) and this PR were built and benchmarked back-to-back on the same runner per shard, so hardware variance cancels out. ⚠️ = PR mean ≥ +${thresholdPct}% slower than main and beyond combined std-dev; ✅ = correspondingly faster.</sub>`;

	const header = ['\| Benchmark \| This PR \| StdDev \| main \| Δ \|', '\|---\|---:\|---:\|---:\|---:\|'];
	const toRow = (e) => `\| \`${e.name}\` \| ${e.prCell} \| ${e.stdCell} \| ${e.baseCell} \| ${e.deltaCell} \|`;

	// A collapsible section per benchmark family, collapsed by default so
	// the (often large) tables stay below the fold.
	const section = (title, list) => {
	if (list.length === 0) return [];
	return [
	'<details>',
	`<summary><b>${title}</b> (${list.length})</summary>`,
	'',
	...header,
	...list.map(toRow),
	'</details>',
	'',
	];
	};

	// Highlights: only the flagged rows stay above the fold so reviewers
	// see what actually moved without expanding either table.
	const flagged = entries.filter(e => e.flag);
	const highlights = flagged.length === 0 ? [] : [
	'Highlights',
	'',
	...header,
	...flagged.map(toRow),
	'',
	];

	const collections = entries.filter(e => !e.isHasher);
	const hashers = entries.filter(e => e.isHasher);

	const marker = '<!-- celerity-benchmarks-comment -->';
	const body = [
	marker,
	'## Benchmarks',
	'',
	subtitle,
	'',
	...highlights,
	...section('Collections', collections),
	...section('Hashers', hashers),
	footer,
	].join('\n');

	const { data: comments } = await github.rest.issues.listComments({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	});
	const existing = comments.find(c => c.body && c.body.includes(marker));

	if (existing) {
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: existing.id,
	body,
	});
	core.info(`Updated comment ${existing.id}`);
	} else {
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	body,
	});
	core.info('Created new benchmarks comment');
	}

	# ----- Main path: publish merged report to gh-pages -----
	- name: Stage merged report for publish
	if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.merge.outputs.have_head == 'true'
	run: \|
	set -euo pipefail
	mkdir -p src/Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results
	cp /tmp/pr-report-full.json src/Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results/ci-report-full.json

	- name: Publish to gh-pages
	if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.merge.outputs.have_head == 'true'
	uses: benchmark-action/github-action-benchmark@v1
	with:
	name: Celerity Benchmarks
	tool: 'benchmarkdotnet'
	output-file-path: src/Celerity.Benchmarks/BenchmarkDotNet.Artifacts/results/ci-report-full.json
	github-token: ${{ secrets.GITHUB_TOKEN }}
	auto-push: true
	comment-on-alert: false
	comment-always: false
	summary-always: true
	fail-on-alert: false
	gh-pages-branch: gh-pages
	benchmark-data-dir-path: dev/bench

	- name: Sync custom dashboard to gh-pages
	# Runs only on main push (same gate as the action's auto-push).
	# Copies web/index.html + web/dev/bench/*.html into gh-pages,
	# leaving the action-managed data.js untouched. Idempotent.
	if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.merge.outputs.have_head == 'true'
	run: \|
	set -euo pipefail
	git fetch origin gh-pages
	git worktree add /tmp/gh-pages gh-pages
	cp web/index.html /tmp/gh-pages/index.html
	mkdir -p /tmp/gh-pages/dev/bench
	cp web/dev/bench/index.html /tmp/gh-pages/dev/bench/index.html
	cp web/dev/bench/detail.html /tmp/gh-pages/dev/bench/detail.html
	cd /tmp/gh-pages
	if git diff --quiet && git diff --cached --quiet; then
	echo "No dashboard changes to sync"
	exit 0
	fi
	git -c user.name="github-actions" -c user.email="github-actions@github.com" \
	add index.html dev/bench/index.html dev/bench/detail.html
	git -c user.name="github-actions" -c user.email="github-actions@github.com" \
	commit -m "Sync custom dashboard from ${GITHUB_SHA:0:7}"
	git push origin gh-pages

Navigation Menu

Uh oh!

docs(CuckooFilter): correct when IsFull clears #284

Workflow file

docs(CuckooFilter): correct when IsFull clears #284

Uh oh!

Workflow file for this run