diff --git a/Cargo.lock b/Cargo.lock index 2335abe1b..3068fad1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,7 @@ dependencies = [ "linkme", "once_cell", "rand 0.8.5", + "rand_core 0.10.0", "rustc_version_runtime", "serde", "serde_json", @@ -1364,6 +1365,20 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "harness" +version = "0.1.0" +dependencies = [ + "antithesis_sdk", + "anyhow", + "byte-unit", + "lading", + "lading-antithesis", + "lading-payload", + "rand 0.10.1", + "serde_yaml", +] + [[package]] name = "hashbrown" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index 8b5c9a15a..419e90495 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "lading_fuzz", "lading_payload", "lading_throttle", + "test/antithesis/harness", "test/antithesis/sink", ] diff --git a/docs/adr/009-antithesis-test-harness.md b/docs/adr/009-antithesis-test-harness.md index 1e0230cb6..b2d69b558 100644 --- a/docs/adr/009-antithesis-test-harness.md +++ b/docs/adr/009-antithesis-test-harness.md @@ -46,13 +46,15 @@ Datadog/saluki's layout, with a "general" MVP scenario of three containers: sancov coverage instrumentation. - **lading** (system under test): the real lading binary, built `--features antithesis` with sancov coverage instrumentation and - `panic="abort"`, run with `--no-target --experiment-duration-infinite` against - a hard-wired `lading.yaml`, a tcp generator pointed at the sink, with telemetry - exposed via `--prometheus-addr` (lading requires telemetry to be configured). - Faulted. -- **workload** (driver): emits the Antithesis `setup_complete` signal, then - idles. lading drives the load, so this container is the seam where - config-variation test commands will land later. + `panic="abort"`. lading reads its config once at startup and cannot be + reconfigured, so its entrypoint blocks on a `ready` sentinel and then boots + under the per-timeline config the harness sampled to the shared volume, run + `--no-target --experiment-duration-infinite --prometheus-addr ` + (`--prometheus-addr` satisfies lading's telemetry requirement). A tcp generator + points at the sink. Faulted. +- **workload** (driver): emits the Antithesis `setup_complete` signal, then idles + to host the test commands. lading drives the load itself, so the only command + today is `first_sample_config`, which samples this timeline's lading config. In a like manner to saluki we introduce a generic `test/antithesis/bin/launch.sh`, driven by per-scenario `launch.env`, tags @@ -67,6 +69,25 @@ is off, is the single path both the sink and lading's bootstrap use to reach `lading/src/antithesis_hooks.rs`, referenced from `lading/src/bin/lading.rs`. It does SDK init plus a panic-reporting hook. +Config variation is a shared mechanism, not per-scenario code. The `harness` +crate (`test/antithesis/harness/`) samples a config by building lading's own +`generator::tcp::Config` from a value menu and serializing it, so the menu cannot +drift from the real schema. Its `first_sample_config` command draws the +structured choices from `AntithesisRng` (the SDK RNG, so each draw is a branch +point Antithesis explores; `thread_rng` under Antithesis is seeded once and does +not branch richly), writes the config plus a `ready` sentinel to a volume shared +with the lading container, and tags the sample with `reachable!` so triage can +count distinct variants. Scenarios reuse `harness` and differ only in wiring. The +MVP menu varies the free axes the TCP sink already catches -- payload variant, +throughput, and parallel connections -- over a fixed TCP transport; transport +variation waits on a multi-protocol sink. Block size is derived from the sampled +rate and connection count (not varied independently) so it stays within the +divided per-connection throttle capacity. The payload `seed` is drawn from system +entropy, not `AntithesisRng`: lading seeds its own PRNG from it and the docs +forbid seeding a userspace RNG from SDK randomness, so payload content is opaque +and effectively fixed across timelines -- acceptable because the sink asserts on +bytes received, not content. + Key sub-decisions: - **Standalone sink, not lading's blackhole, as the oracle, per constraint 3.** @@ -82,8 +103,13 @@ Key sub-decisions: - **Three containers.** `setup_complete` and future config-variation live in a dedicated workload container rather than being owned by the faulted system under test. -- **Config hard-wired for the MVP.** Config variation and test commands are - deferred to the workload seam. +- **Config varied per timeline, sampled from lading's own types.** The shared + `harness` builds `tcp::Config` and serializes it, drawing the structured + choices from `AntithesisRng`. Sampling is a post-`setup_complete` `first_` + command so Antithesis branches each choice per timeline. Unit tests assert + every sampled config re-deserializes as a valid lading config and that the + block size never exceeds the divided per-connection throttle capacity, so the + menu cannot silently drift from the schema or produce a discard-spin config. ## Alternatives Considered @@ -113,10 +139,22 @@ Rejected: the faulted system under test would own the setup signal, and config-variation test commands would have no home. A dedicated workload container keeps those concerns separate. +### Sample the config in lading's entrypoint at boot + +Rejected. `snouty validate` does not execute `first_` commands, so with the +sentinel approach lading stays blocked and does not boot under validate -- +validate still passes on `setup_complete`, exactly as saluki behaves. Sampling in +lading's own entrypoint would make validate boot lading and push, but a boot-time +draw is pre-`setup_complete`, which Antithesis branches less richly than a +post-setup `first_` command. We chose the richer exploration; validate passing +without booting the SUT is acceptable and is what saluki lives with. + ## References - `lading_antithesis/` - SDK facade over `antithesis_sdk` -- `test/antithesis/` - harness (to be created) +- `test/antithesis/sink/` - the sink oracle crate +- `test/antithesis/harness/` - shared config-variation crate (`first_sample_config`) +- `test/antithesis/scenarios/general/` - the general scenario (Dockerfile, compose, launcher inputs) - `integration/sheepdog/`, `integration/ducks/` - the mechanism this replaces - saluki `test/antithesis/` - pattern source - ADR-001: Generator-Target-Blackhole Architecture (the sink is an diff --git a/test/antithesis/harness/Cargo.toml b/test/antithesis/harness/Cargo.toml new file mode 100644 index 000000000..2da18465d --- /dev/null +++ b/test/antithesis/harness/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "harness" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false +description = "Shared Antithesis harness for lading scenarios: per-timeline config sampling." + +[lib] +doctest = false + +[[bin]] +name = "first_sample_config" +path = "src/bin/first_sample_config.rs" + +[lints] +workspace = true + +[dependencies] +lading = { path = "../../../lading" } +lading-payload = { path = "../../../lading_payload" } +lading-antithesis = { workspace = true, features = ["antithesis"] } +antithesis_sdk = { workspace = true, features = ["full", "rand_v0_10"] } +byte-unit = { workspace = true, features = ["std"] } +rand = { workspace = true, features = ["thread_rng", "std_rng"] } +serde_yaml = { workspace = true } +anyhow = { workspace = true } diff --git a/test/antithesis/harness/src/bin/first_sample_config.rs b/test/antithesis/harness/src/bin/first_sample_config.rs new file mode 100644 index 000000000..bd7fa4594 --- /dev/null +++ b/test/antithesis/harness/src/bin/first_sample_config.rs @@ -0,0 +1,43 @@ +//! Antithesis `first_` command: sample this timeline's lading config and release +//! the blocked system-under-test. +//! +//! Runs once per timeline after `setup_complete`, so the `AntithesisRng` draws +//! are post-snapshot decisions Antithesis branches: each timeline boots lading +//! under its own sampled config. Writes the config to the shared volume, tags +//! the sample for triage, then writes the `ready` sentinel last so the config is +//! always present before the SUT unblocks. + +use std::path::PathBuf; + +use anyhow::Context; + +fn main() -> anyhow::Result<()> { + lading_antithesis::init(); + + let dir: PathBuf = + std::env::var_os("CONFIG_DIR").map_or_else(|| PathBuf::from("/shared"), PathBuf::from); + std::fs::create_dir_all(&dir) + .with_context(|| format!("create config dir {}", dir.display()))?; + + // Draw structured choices from AntithesisRng so Antithesis branches each pick + // and explores the config menu across timelines. UnwrapErr adapts the SDK's + // fallible RNG to rand's infallible RngCore. + let mut rng = rand::rand_core::UnwrapErr(antithesis_sdk::random::AntithesisRng); + let cfg = harness::config::sample(&mut rng); + let variant = harness::config::variant_label(&cfg.variant); + let yaml = harness::config::to_yaml(&cfg).context("serialize sampled config")?; + + let config_path = dir.join("lading.yaml"); + std::fs::write(&config_path, yaml.as_bytes()) + .with_context(|| format!("write {}", config_path.display()))?; + + // Per-timeline anchor: counting these in triage shows how many distinct + // variants the run explored. + lading_antithesis::reachable!("first_sample_config sampled a config", { "variant": variant }); + + let ready = dir.join("ready"); + std::fs::write(&ready, b"ready\n") + .with_context(|| format!("write sentinel {}", ready.display()))?; + + Ok(()) +} diff --git a/test/antithesis/harness/src/config.rs b/test/antithesis/harness/src/config.rs new file mode 100644 index 000000000..607034318 --- /dev/null +++ b/test/antithesis/harness/src/config.rs @@ -0,0 +1,159 @@ +//! Per-timeline lading config sampling. +//! +//! The sampler builds lading's real [`tcp::Config`] so the value menu is exactly +//! the config schema, then serializes it under the `generator: [{ tcp: … }]` +//! shape lading parses. The transport is fixed to TCP against the sink oracle, +//! which counts bytes and so catches any payload variant; only the free axes +//! (payload variant, rate, connections, block sizes, seed) vary. Transport +//! variation waits on a multi-protocol sink. + +use lading::generator::tcp; +use rand::seq::IndexedRandom; +use rand::{Rng, RngExt}; + +/// Fixed address of the sink oracle every sampled config targets. +const SINK_ADDR: &str = "sink:9000"; + +/// Payload variants the TCP sink catches with no decoding. All are fieldless, so +/// they need no extra configuration to sample. +fn variant_menu() -> [lading_payload::Config; 6] { + [ + lading_payload::Config::Ascii, + lading_payload::Config::Syslog5424, + lading_payload::Config::Json, + lading_payload::Config::Fluent, + lading_payload::Config::ApacheCommon, + lading_payload::Config::DatadogLog, + ] +} + +/// Sample a lading TCP generator config for one timeline. +/// +/// Draws the payload variant, throughput, parallel connections, and seed from +/// `rng`; the transport and target stay fixed to the sink. +#[must_use] +pub fn sample(rng: &mut R) -> tcp::Config { + // Structured choices come from the caller's rng -- AntithesisRng in + // production -- so Antithesis branches each pick and sweeps the menu. + let variant = variant_menu() + .choose(rng) + .cloned() + .unwrap_or(lading_payload::Config::Ascii); + + let bps_mib = [1_u64, 5, 10, 50, 100].choose(rng).copied().unwrap_or(10); + let bytes_per_second_bytes = bps_mib * 1024 * 1024; + let bytes_per_second = Some(byte_unit::Byte::from_u64(bytes_per_second_bytes)); + + let parallel_connections = rng.random_range(1..=8_u16); + + // Cap the block size at the smallest per-connection throttle capacity. lading + // divides `bytes_per_second` evenly across `parallel_connections`; a block + // larger than a worker's divided capacity is rejected by the throttle, and + // the TCP worker then busy-spins discarding it with no backoff. Keeping + // `maximum_block_size <= bytes_per_second / parallel_connections` guarantees + // every block fits, so no timeline degrades into a discard spin. + let per_connection_capacity = bytes_per_second_bytes / u64::from(parallel_connections); + let maximum_block_size = + byte_unit::Byte::from_u64(per_connection_capacity.clamp(1, 1024 * 1024)); + + // lading seeds its own payload PRNG from `seed`. The Antithesis docs warn + // against seeding your own RNG from SDK randomness, so draw the seed from + // system entropy rather than the (Antithesis) `rng`. This makes payload byte + // content opaque to Antithesis and effectively fixed across timelines, which + // is fine: the sink asserts on bytes received, not on content. + let mut seed = [0u8; 32]; + rand::rng().fill_bytes(&mut seed); + + tcp::Config { + seed, + addr: SINK_ADDR.to_string(), + variant, + bytes_per_second, + maximum_block_size, + maximum_prebuild_cache_size_bytes: byte_unit::Byte::from_u64(8 * 1024 * 1024), + parallel_connections, + throttle: None, + } +} + +/// Serialize a sampled `tcp::Config` into the top-level `generator: [{ tcp: … }]` +/// YAML that lading consumes. +/// +/// # Errors +/// +/// Returns an error if serialization fails. +pub fn to_yaml(cfg: &tcp::Config) -> Result { + let mut tcp_item = serde_yaml::Mapping::new(); + tcp_item.insert(serde_yaml::Value::from("tcp"), serde_yaml::to_value(cfg)?); + let generators = serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping(tcp_item)]); + let mut top = serde_yaml::Mapping::new(); + top.insert(serde_yaml::Value::from("generator"), generators); + serde_yaml::to_string(&serde_yaml::Value::Mapping(top)) +} + +/// Short, stable label for a payload variant, for tagging the Antithesis sample. +#[must_use] +pub fn variant_label(variant: &lading_payload::Config) -> &'static str { + match variant { + lading_payload::Config::Ascii => "ascii", + lading_payload::Config::Syslog5424 => "syslog5424", + lading_payload::Config::Json => "json", + lading_payload::Config::Fluent => "fluent", + lading_payload::Config::ApacheCommon => "apache_common", + lading_payload::Config::DatadogLog => "datadog_log", + _ => "other", + } +} + +#[cfg(test)] +mod tests { + use super::{sample, to_yaml}; + use rand::SeedableRng; + use rand::rngs::StdRng; + + #[test] + fn sampled_config_deserializes_as_valid_lading_config() { + // The load-bearing invariant: whatever we sample must be a config lading + // actually accepts. Sweep many seeds so the whole menu is exercised. + for s in 0..256_u64 { + let mut rng = StdRng::seed_from_u64(s); + let cfg = sample(&mut rng); + let yaml = to_yaml(&cfg).expect("serialize sampled config"); + let parsed: Result = serde_yaml::from_str(&yaml); + assert!( + parsed.is_ok(), + "seed {s} produced a config lading rejects: {err:?}\n{yaml}", + err = parsed.err() + ); + } + } + + #[test] + fn sampled_config_holds_invariants() { + for s in 0..256_u64 { + let mut rng = StdRng::seed_from_u64(s); + let cfg = sample(&mut rng); + assert_eq!(cfg.addr, "sink:9000"); + assert!((1..=8).contains(&cfg.parallel_connections)); + assert!(cfg.bytes_per_second.is_some()); + } + } + + #[test] + fn block_size_never_exceeds_per_connection_capacity() { + // Regression: lading divides bytes_per_second across parallel_connections, + // and a block larger than a worker's divided capacity busy-spins on + // discard. maximum_block_size must fit the smallest per-connection share. + for s in 0..256_u64 { + let mut rng = StdRng::seed_from_u64(s); + let cfg = sample(&mut rng); + let bps = cfg.bytes_per_second.expect("bytes_per_second set").as_u64(); + let per_connection = bps / u64::from(cfg.parallel_connections); + assert!( + cfg.maximum_block_size.as_u64() <= per_connection, + "seed {s}: block {block} exceeds per-connection capacity {per_connection}", + block = cfg.maximum_block_size.as_u64() + ); + } + } +} diff --git a/test/antithesis/harness/src/lib.rs b/test/antithesis/harness/src/lib.rs new file mode 100644 index 000000000..383d0fc81 --- /dev/null +++ b/test/antithesis/harness/src/lib.rs @@ -0,0 +1,9 @@ +//! Shared Antithesis harness for lading scenarios. +//! +//! Holds the per-timeline config-variation mechanism every scenario reuses: +//! [`config::sample`] draws a lading generator config from a value menu, and the +//! `first_sample_config` command serializes it to the shared volume the +//! system-under-test boots from. The menu is built from lading's own +//! `tcp::Config`, so it cannot drift from the real config schema. + +pub mod config; diff --git a/test/antithesis/scenarios/general/Dockerfile b/test/antithesis/scenarios/general/Dockerfile index 790b5f10e..c867ccfea 100644 --- a/test/antithesis/scenarios/general/Dockerfile +++ b/test/antithesis/scenarios/general/Dockerfile @@ -72,7 +72,9 @@ RUN --mount=type=cache,target=/tools/target,id=antithesis-tools-target \ --mount=type=cache,target=/root/.cargo/registry,id=cargo-registry \ --mount=type=cache,target=/root/.cargo/git,id=cargo-git \ cargo build --release --package sink --bin sink && \ - cp /tools/target/release/sink /usr/local/bin/sink + cargo build --release --package harness --bin first_sample_config && \ + cp /tools/target/release/sink /usr/local/bin/sink && \ + cp /tools/target/release/first_sample_config /usr/local/bin/first_sample_config # --------------------------------------------------------------------------- # Runtime: instrumented lading (system under test). @@ -83,12 +85,13 @@ RUN apt-get update && \ apt-get install --no-install-recommends -y ca-certificates libssl3 && \ rm -rf /var/lib/apt/lists/* COPY --from=lading-builder /usr/local/bin/lading /usr/local/bin/lading -COPY test/antithesis/scenarios/general/lading.yaml /etc/lading/lading.yaml +COPY --chmod=755 test/antithesis/scenarios/general/lading-entrypoint.sh /usr/local/bin/lading-entrypoint.sh # Antithesis scans /symbols in every pushed image and matches by build-id. RUN mkdir -p /symbols && ln -s /usr/local/bin/lading /symbols/lading -# lading requires telemetry to be configured; --prometheus-addr satisfies it -# without the unbounded disk growth a capture file would incur over an infinite run. -ENTRYPOINT ["/usr/local/bin/lading", "--no-target", "--experiment-duration-infinite", "--prometheus-addr", "0.0.0.0:9102", "--config-path", "/etc/lading/lading.yaml"] +# lading reads its config once at startup, so the entrypoint blocks on the sampled +# config (first_sample_config -> shared volume) before exec'ing lading. Telemetry +# (--prometheus-addr) is set inside the entrypoint. +ENTRYPOINT ["/usr/local/bin/lading-entrypoint.sh"] # --------------------------------------------------------------------------- # Runtime: sink oracle (SDK-linked, uninstrumented). @@ -107,6 +110,15 @@ ENTRYPOINT ["/usr/local/bin/sink"] # --------------------------------------------------------------------------- FROM debian:bookworm-slim AS workload ENV NO_COLOR=1 +# first_sample_config links lading, which carries a dynamic libssl dependency. +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates libssl3 && \ + rm -rf /var/lib/apt/lists/* COPY --chmod=755 test/antithesis/scenarios/general/workload/setup-complete.sh /opt/antithesis/setup-complete.sh COPY --chmod=755 test/antithesis/scenarios/general/workload/entrypoint.sh /entrypoint.sh +# Antithesis test commands live here; the platform discovers and runs them from +# /opt/antithesis/test/v1/