feat(storage): replace the LMDB chunk store with one file per chunk, and migrate onto it #1099

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main, "rc-*"]
	pull_request:
	branches: [main, "rc-*"]

	env:
	CARGO_TERM_COLOR: always
	RUSTFLAGS: "-D warnings"
	FOUNDRY_VERSION: v1.7.1

	jobs:
	fmt:
	name: Format Check
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	with:
	components: rustfmt
	- run: cargo fmt --all -- --check

	clippy:
	name: Clippy
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	with:
	components: clippy
	- uses: Swatinem/rust-cache@v2
	- run: cargo clippy --all-targets --all-features -- -D warnings

	test:
	name: Test (${{ matrix.os }})
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-latest]
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	- uses: Swatinem/rust-cache@v2
	- name: Install Foundry
	uses: foundry-rs/foundry-toolchain@v1
	with:
	version: ${{ env.FOUNDRY_VERSION }}
	- name: Run unit tests
	run: cargo test --lib --features test-utils
	# Before the e2e suite, deliberately. These are fast and deterministic, and the e2e
	# suite flakes on hosted runners for transport reasons that have nothing to do with
	# storage. A failing step aborts the job, so anything sequenced after a flaky one
	# never reports, which is how these ran on no platform at all for a whole run.
	- name: Prove the migration returns disk to the filesystem
	shell: bash
	run: \|
	set -euo pipefail
	cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 \| tee /tmp/reclaims_disk.log
	# A target whose required features are not passed is skipped with a
	# warning and a zero exit, so a harness can stop running without anyone
	# noticing. This is what makes that loud.
	grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \
	\|\| { echo 'reclaims_disk ran no tests'; exit 1; }
	- name: Kill a node mid-migration and check what survived
	shell: bash
	run: \|
	set -euo pipefail
	cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 \| tee /tmp/crash_safety.log
	# A target whose required features are not passed is skipped with a
	# warning and a zero exit, so a harness can stop running without anyone
	# noticing. This is what makes that loud.
	grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \
	\|\| { echo 'crash_safety ran no tests'; exit 1; }
	- name: Several nodes migrating on one disk
	shell: bash
	run: \|
	set -euo pipefail
	cargo test --test migration_shared_volume --features test-utils 2>&1 \| tee /tmp/shared_volume.log
	# A target whose required features are not passed is skipped with a
	# warning and a zero exit, so a harness can stop running without anyone
	# noticing. This is what makes that loud.
	grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \
	\|\| { echo 'shared_volume ran no tests'; exit 1; }
	# Linux only. This one plants a hundred thousand files to measure what a restart
	# costs, and the answer it is after is a fleet answer, where every node is Linux.
	# The scan itself reads names and nothing else, which is not a platform-specific
	# path, and opening a store is covered on all three by the unit tests. Planting that
	# many files on the Windows runner would cost minutes of every run to re-measure
	# something no node will ever do there.
	- name: Startup scan, index memory and inode cost at scale
	if: runner.os == 'Linux'
	shell: bash
	run: \|
	set -euo pipefail
	cargo test --test storage_scale --features test-utils -- --nocapture --test-threads=1 2>&1 \| tee /tmp/scale.log
	# A target whose required features are not passed is skipped with a
	# warning and a zero exit, so a harness can stop running without anyone
	# noticing. This is what makes that loud.
	grep -qE 'test result: ok\. [1-9]' /tmp/scale.log \
	\|\| { echo 'scale ran no tests'; exit 1; }
	- name: Run e2e tests
	run: cargo test --test e2e --features test-utils -- --test-threads=1
	- name: Run v12 storage-bound audit attack PoCs
	run: cargo test --test poc_commitment_audit_attacks --features test-utils
	- name: Run v12 live audit-handler tests
	run: cargo test --test poc_audit_handler_live --features test-utils
	- name: Run bootstrap-stall PoC regression marker
	run: cargo test --test poc_bootstrap_stall --features test-utils
	- name: Shutdown waits for writes whose caller has gone
	run: cargo test --test poc_shutdown_lmdb_drain --features test-utils

	# Runs the storage tests against real ext4, XFS and btrfs rather than whatever the
	# runner provides. Deliberately NOT named durability: killing a process and reopening
	# the same mounted filesystem keeps the page cache, so this exercises each filesystem's
	# syscall, locking, rename and delete behaviour, not its behaviour under power loss.
	# That still needs block-device fault injection or a real machine, and remains a fleet
	# gate.
	filesystems:
	name: Storage on ${{ matrix.fs }}
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	fs: [ext4, xfs, btrfs]
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	- uses: Swatinem/rust-cache@v2
	- name: Install the filesystem tools
	run: sudo apt-get update && sudo apt-get install -y xfsprogs btrfs-progs
	- name: Make a ${{ matrix.fs }} volume and mount it
	shell: bash
	run: \|
	set -euo pipefail
	# A loopback image, so these run on a filesystem of the right kind rather than
	# on whatever the runner happens to give us. ext4 is what most of the fleet is
	# on; XFS and btrfs are the two the design reasons about separately, btrfs
	# because it has been observed reordering writes around a rename.
	# 3 GiB is ample: these tests use tens of MiB. The scale harness, which is
	# the one that needs room, is not in this job.
	truncate -s 3G /tmp/${{ matrix.fs }}.img
	mkfs.${{ matrix.fs }} -q /tmp/${{ matrix.fs }}.img
	sudo mkdir -p /mnt/antfs
	sudo mount -o loop /tmp/${{ matrix.fs }}.img /mnt/antfs
	sudo chown "$USER" /mnt/antfs
	df -hT /mnt/antfs
	# TMPDIR is what `TempDir::new` uses, so this is what puts the test data on the
	# mounted filesystem rather than on the runner's root.
	- name: The migration returns disk on ${{ matrix.fs }}
	env:
	TMPDIR: /mnt/antfs
	shell: bash
	run: \|
	set -euo pipefail
	cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 \| tee /tmp/reclaims_disk.log
	# A target whose required features are not passed is skipped with a
	# warning and a zero exit, so a harness can stop running without anyone
	# noticing. This is what makes that loud.
	grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \
	\|\| { echo 'reclaims_disk ran no tests'; exit 1; }
	- name: A node killed mid-write on ${{ matrix.fs }} loses nothing
	env:
	TMPDIR: /mnt/antfs
	shell: bash
	run: \|
	set -euo pipefail
	cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 \| tee /tmp/crash_safety.log
	# A target whose required features are not passed is skipped with a
	# warning and a zero exit, so a harness can stop running without anyone
	# noticing. This is what makes that loud.
	grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \
	\|\| { echo 'crash_safety ran no tests'; exit 1; }
	- name: Several nodes on one ${{ matrix.fs }} volume
	env:
	TMPDIR: /mnt/antfs
	shell: bash
	run: \|
	set -euo pipefail
	cargo test --test migration_shared_volume --features test-utils 2>&1 \| tee /tmp/shared_volume.log
	# A target whose required features are not passed is skipped with a
	# warning and a zero exit, so a harness can stop running without anyone
	# noticing. This is what makes that loud.
	grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \
	\|\| { echo 'shared_volume ran no tests'; exit 1; }

	doc:
	name: Documentation
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	- uses: Swatinem/rust-cache@v2
	- name: Build documentation
	run: cargo doc --all-features --no-deps
	env:
	RUSTDOCFLAGS: "-D warnings"

	build:
	name: Build (${{ matrix.os }})
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-latest]
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	- uses: Swatinem/rust-cache@v2
	- name: Build release (no logging)
	run: cargo build --release --no-default-features
	# The crash harness drives the store through a failpoint that parks the process
	# forever on an environment variable. It is compiled only under `test-utils`, which
	# is not a default feature and is not passed by the release workflow, so a shipped
	# binary does not contain it. This proves that rather than trusting it: the variable
	# name is a string literal, so it survives into the binary whenever the code that
	# reads it is compiled, and its absence is the absence of the failpoint.
	- name: A shipped binary carries no failpoint
	if: runner.os == 'Linux'
	shell: bash
	run: \|
	set -euo pipefail
	cargo build --bin ant-node
	found=$(strings -a target/debug/ant-node \| grep -c 'ANT_HALT_' \|\| true)
	# With --features test-utils this count is not zero, which is what makes a zero
	# here evidence rather than an accident of how the binary was stripped. The exact
	# number is one per failpoint and is deliberately not asserted, so that adding a
	# failpoint does not fail this check.
	if [ "$found" != "0" ]; then
	echo "the publish failpoint is compiled into a default-feature build"
	exit 1
	fi
	echo "no failpoint in a default-feature build"

	test-no-logging:
	name: Test (no logging)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	- uses: Swatinem/rust-cache@v2
	- name: Install Foundry
	uses: foundry-rs/foundry-toolchain@v1
	with:
	version: ${{ env.FOUNDRY_VERSION }}
	- name: Run unit tests without logging
	run: cargo test --lib --no-default-features

	security:
	name: Security Audit
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@stable
	- name: Install cargo-audit
	run: cargo install cargo-audit
	- name: Run security audit
	run: cargo audit

Navigation Menu

Uh oh!

feat(storage): replace the LMDB chunk store with one file per chunk, and migrate onto it #1099

Workflow file

feat(storage): replace the LMDB chunk store with one file per chunk, and migrate onto it #1099

Uh oh!

Workflow file for this run