From e7d7349ecfec6bacccbba53225dc480ee4eeabc8 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 02:11:23 +0800 Subject: [PATCH 01/21] docs(e2e): docker build test matrix from a real-world Dockerfile corpus --- .../plans/docker-build-e2e-matrix.md | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 internal-docs/plans/docker-build-e2e-matrix.md diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md new file mode 100644 index 000000000..a3e5033dc --- /dev/null +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -0,0 +1,140 @@ +# docker build E2E matrix — derived from a real-world Dockerfile corpus + +Companion to `network-workload-e2e.md`. W14 there proved the *network +datapath* survives one `docker build`; this plan derives the **build +compatibility and performance matrix** from what flagship open-source +projects actually put in their Dockerfiles, so the future `docker_build` +e2e suite tests the build surface developers really hit — not features we +imagine they use. + +Method: 25 Dockerfiles fetched 2026-07-22 from each repo's default branch +(`raw.githubusercontent.com//HEAD/`), feature vectors extracted +programmatically (stage counts, `--mount` types, platform ARGs, heredocs, +syntax directives, network calls). Re-fetch with the same repo/path table +below to refresh the census. + +## Corpus + +| Source (repo — path) | Ecosystem | Shape highlights | +|---|---|---| +| moby/moby — `Dockerfile` | Go | 56 stages, 51 cache mounts, 35 `COPY --link`, heredocs, `FROM scratch` exports | +| moby/buildkit — `Dockerfile` | Go | 44 stages, 19 bind mounts, 51 `COPY --link`, 8 `FROM scratch`, cross-platform (`--platform=$BUILDPLATFORM`) | +| docker/buildx — `Dockerfile` | Go | 29 stages, same family as buildkit | +| grafana/grafana — `Dockerfile` | Go+Node hybrid | 14 stages, `syntax=docker/dockerfile:1.7-labs`, `COPY --parents` | +| apache/airflow — `Dockerfile` | Python | 1.9k lines, 94 ARGs, ARG-parameterized `FROM ${BASE_IMAGE}`, `FROM scratch as scripts` heredoc script-carrier stage, 15 heredocs | +| mastodon/mastodon — `Dockerfile` | Ruby+Node | 9 stages, 11 cache mounts with `sharing=locked` + per-`TARGETPLATFORM` cache IDs | +| goauthentik/authentik — `lifecycle/container/Dockerfile` | Go+Python+Node | 34 bind mounts, **2 `--mount=type=secret`** (only secret user in corpus), `ADD` from URL | +| immich-app/immich — `server/Dockerfile` | Node native-deps | 16 bind mounts for pnpm lockfiles, 6 cache mounts | +| vercel/next.js — `examples/with-docker/Dockerfile` | Node | canonical 3-stage standalone build; npm/yarn/pnpm autodetect via corepack; cache mounts | +| vercel/turborepo — `examples/with-docker/apps/web/Dockerfile` | Node monorepo | `turbo prune --docker` context-slimming pattern | +| n8n-io/n8n — `docker/images/n8n/Dockerfile` | Node | modest 2-FROM prod image | +| go-gitea/gitea — `Dockerfile` | Go | 2-stage + cross-platform FROM | +| traefik/traefik — `Dockerfile` | Go | minimal `syntax=1.2` + TARGETPLATFORM COPY | +| caddyserver/caddy-docker — `2.11/builder/Dockerfile` | Go | xcaddy builder: toolchain download at build time | +| prometheus/prometheus — `Dockerfile` | Go | pure COPY-binary, zero network | +| dani-garcia/vaultwarden — `docker/Dockerfile.debian` | Rust | 4 FROMs, TARGETPLATFORM cross-compile | +| meilisearch/meilisearch — `Dockerfile` | Rust | alpine cargo build | +| home-assistant/core — `Dockerfile` | Python | `syntax=` **pinned by sha256 digest**, uv installs | +| paperless-ngx/paperless-ngx — `Dockerfile` | Python+Node | 2-stage, TARGETPLATFORM, HEALTHCHECK | +| keycloak/keycloak — `quarkus/container/Dockerfile` | JVM | 1-stage + chown COPY | +| mastodon-class official images: docker-library/postgres `17/bookworm`, docker-library/redis `7.4/debian`, nginxinc/docker-nginx `mainline/debian`, docker-library/ghost `6/bookworm` | C/misc | classic single-stage: apt + `wget` release binaries + **gpg keyserver verify**, tiny build context | +| traefik/traefik — `webui/buildx.Dockerfile` | Node | trivial single-stage yarn | + +## Feature census (what production Dockerfiles actually use) + +Ranked by prevalence in the corpus (n=25): + +| Feature | Prevalence | Exemplars | +|---|---|---| +| Multi-stage + `COPY --from` | 17/25; extremes 56/44/29 stages | moby, buildkit, buildx, grafana | +| Build-time network (apt/apk/npm/pip/cargo/bundle) | 24/25 | everything except prometheus | +| External `# syntax=` frontend (adds a frontend **image pull** to every build) | 12/25 — incl. version-pinned (`1.18`, `1.2`), digest-pinned (home-assistant), labs channel (grafana), master (buildkit) | see syntax list | +| `RUN --mount=type=cache` | 10/25 | moby(51), mastodon(11, `sharing=locked`, per-platform IDs), authentik, immich, next.js | +| `TARGETPLATFORM`/`TARGETARCH` + `--platform=$BUILDPLATFORM` cross-compile | 9/25 | moby, buildkit, mastodon, vaultwarden, gitea, paperless | +| `RUN --mount=type=bind` (lockfile mounts, no COPY layer) | 5/25, heavy | authentik(34), immich(16), buildkit(19) | +| `COPY --link` | 4/25 | buildkit(51), moby(35), buildx, grafana | +| Heredocs (`RUN < Date: Wed, 22 Jul 2026 02:23:10 +0800 Subject: [PATCH 02/21] test(e2e): docker build suite harness + D2 stage-graph scenario --- tests/e2e/tests/docker_build.rs | 281 ++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 tests/e2e/tests/docker_build.rs diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs new file mode 100644 index 000000000..14eebd4be --- /dev/null +++ b/tests/e2e/tests/docker_build.rs @@ -0,0 +1,281 @@ +//! docker build e2e — Phase 1 (D1–D3) of +//! internal-docs/plans/docker-build-e2e-matrix.md. +//! +//! Where `network_workload` W14 drives *one* build to prove the datapath, +//! this suite tests the build surface itself, with scenarios modeled on the +//! real-world Dockerfile corpus in the plan: +//! +//! - **D2 stage graph** (grafana / buildkit shape): a 12-stage diamond — +//! `FROM scratch AS scripts` carrier (airflow pattern), four independent +//! branches, `COPY --from` joins, a heredoc RUN, `COPY --link`, and a +//! `FROM scratch AS export` tail, under `# syntax=docker/dockerfile:1` +//! (the external frontend 12 of 25 corpus files declare). Asserts the +//! final image carries exactly the export-stage artifact and none of the +//! builder-stage residue. +//! +//! All scenarios share one booted daemon; failures aggregate. The workload +//! suite's quiet-log rule applies: builds must leave no proxy-layer ERROR. + +use std::path::Path; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use arcbox_e2e::docker::{docker_ignore, docker_output, ensure_image}; +use arcbox_e2e::metrics::RunMetrics; +use arcbox_e2e::net_fixtures::daemon_log_cursor; +use arcbox_e2e::scenario::run_vz_scenario_with_log; + +const READY_TIMEOUT: Duration = Duration::from_secs(180); + +/// External dockerfile frontend used by D2's `# syntax=` directive, +/// pre-loaded so the build does not depend on registry reachability. +const FRONTEND_IMAGE: &str = "docker/dockerfile:1"; + +/// D2: stage-graph build deadline. +const STAGE_GRAPH_DEADLINE: Duration = Duration::from_secs(120); +const STAGE_GRAPH_TAG: &str = "arcbox-e2e-build:stage-graph"; +const STAGE_GRAPH_PROBE: &str = "arcbox-e2e-build-d2-probe"; + +#[test] +#[ignore = "boots a VZ System VM through a real daemon; run on the e2e runner"] +fn docker_build_suite() -> Result<()> { + run_vz_scenario_with_log("docker_build", "info", |daemon, data_dir, metrics| { + metrics.time("daemon_ready", || daemon.wait_ready_blocking(READY_TIMEOUT))?; + let image = + std::env::var("ARCBOX_E2E_IMAGE").unwrap_or_else(|_| "alpine:latest".to_owned()); + metrics.time("docker_pull", || ensure_image(data_dir, &image))?; + metrics.time("frontend_pull", || ensure_image(data_dir, FRONTEND_IMAGE))?; + + // Taken after setup so image-pull noise is out of scope; covers + // every build below. + let log = daemon_log_cursor(data_dir); + + let scenarios: [(&str, ScenarioFn); 1] = [("stage_graph", stage_graph)]; + // Diagnostic filter: run only the named scenario, e.g. + // ARCBOX_E2E_BUILD_ONLY=stage_graph. + let only = std::env::var("ARCBOX_E2E_BUILD_ONLY").ok(); + let mut failures = Vec::new(); + for (name, scenario) in scenarios { + if let Some(ref only) = only + && only != name + { + continue; + } + tracing::info!(scenario = name, "starting"); + match scenario(data_dir, metrics, &image) { + Ok(()) => tracing::info!(scenario = name, "passed"), + Err(error) => { + tracing::warn!(scenario = name, "failed: {error:#}"); + failures.push(format!("{name}: {error:#}")); + } + } + } + + match log.proxy_errors() { + Ok(errors) if !errors.is_empty() => failures.push(format!( + "quiet-log: {} proxy-layer ERROR line(s) during builds:\n{}", + errors.len(), + errors.join("\n") + )), + Ok(_) => {} + Err(error) => failures.push(format!("quiet-log: unreadable: {error:#}")), + } + + if failures.is_empty() { + Ok(()) + } else { + bail!( + "{} of {} build checks failed:\n{}", + failures.len(), + scenarios.len() + 1, + failures.join("\n---\n") + ) + } + }) +} + +type ScenarioFn = fn(&Path, &mut RunMetrics, &str) -> Result<()>; + +/// Runs `docker build --progress=plain -t tag ctx`, recording the wall +/// under `key` (also for failed builds — a slow failure is still trend +/// data) and failing past `deadline`. +fn timed_build( + data_dir: &Path, + metrics: &mut RunMetrics, + key: &str, + ctx: &Path, + tag: &str, + deadline: Duration, +) -> Result<(String, Duration)> { + let ctx_arg = ctx.display().to_string(); + let started = Instant::now(); + let build = docker_output( + data_dir, + &["build", "--progress=plain", "-t", tag, &ctx_arg], + deadline + Duration::from_secs(60), + ); + let elapsed = started.elapsed(); + metrics.record(key, elapsed.as_secs_f64()); + let output = build.with_context(|| format!("{key}: docker build"))?; + if elapsed >= deadline { + bail!("{key}: build took {elapsed:?} (>= {deadline:?})"); + } + Ok((output, elapsed)) +} + +/// D2: 12-stage diamond under the external `docker/dockerfile:1` frontend. +/// Four independent branches (one fed from a `FROM scratch` script-carrier +/// stage, one built through a heredoc RUN) join into an aggregate, and a +/// `FROM scratch AS export` tail carries exactly one artifact. The final +/// image must contain that artifact byte-exact and none of the builder +/// residue (`/sentinel.txt`, parts, intermediates). +fn stage_graph(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d2-ctx"); + std::fs::create_dir_all(&ctx).context("creating d2 context dir")?; + std::fs::write(ctx.join("make-part.sh"), "echo \"part-$1\"\n")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"# syntax=docker/dockerfile:1 +FROM scratch AS scripts +COPY make-part.sh /make-part.sh + +FROM {image} AS base +RUN echo shared > /shared.txt + +FROM base AS branch-a +COPY --from=scripts /make-part.sh /make-part.sh +RUN sh /make-part.sh a > /part.txt && echo builder-only > /sentinel.txt + +FROM base AS branch-a-out +COPY --from=branch-a /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS branch-b +RUN < /part.txt +echo b-extra >> /part.txt +EOT + +FROM base AS branch-b-out +COPY --from=branch-b /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS branch-c +RUN echo part-c > /part.txt + +FROM base AS branch-c-out +COPY --from=branch-c /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS branch-d +RUN echo part-d > /part.txt + +FROM base AS branch-d-out +COPY --from=branch-d /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS join +COPY --from=branch-a-out /out.txt /join/out-a.txt +COPY --from=branch-b-out /out.txt /join/out-b.txt +COPY --from=branch-c-out /out.txt /join/out-c.txt +COPY --from=branch-d-out /out.txt /join/out-d.txt +RUN cat /join/out-a.txt /join/out-b.txt /join/out-c.txt /join/out-d.txt > /artifact.txt + +FROM scratch AS export +COPY --link --from=join /artifact.txt /artifact.txt +" + ), + ) + .context("writing d2 Dockerfile")?; + let expected_artifact = + "shared\npart-a\nshared\npart-b\nb-extra\nshared\npart-c\nshared\npart-d\n"; + + let result = (|| { + timed_build( + data_dir, + metrics, + "stage_graph_build_wall", + &ctx, + STAGE_GRAPH_TAG, + STAGE_GRAPH_DEADLINE, + )?; + + // A scratch image cannot run; probe it via a created (never + // started) container. + docker_output( + data_dir, + &[ + "create", + "--name", + STAGE_GRAPH_PROBE, + STAGE_GRAPH_TAG, + "/noop", + ], + Duration::from_secs(60), + ) + .context("creating probe container")?; + + let artifact_host = data_dir.join("d2-artifact.txt"); + let artifact_arg = artifact_host.display().to_string(); + docker_output( + data_dir, + &[ + "cp", + &format!("{STAGE_GRAPH_PROBE}:/artifact.txt"), + &artifact_arg, + ], + Duration::from_secs(60), + ) + .context("copying artifact out of the built image")?; + let artifact = std::fs::read_to_string(&artifact_host)?; + if artifact != expected_artifact { + bail!( + "stage_graph: artifact mismatch:\n{artifact:?}\nexpected:\n{expected_artifact:?}" + ); + } + + // Full-rootfs sweep for builder residue: the export listing may + // carry init-layer entries (/dev, /etc/hosts, …) but must not + // carry anything from the builder stages. + let export_tar = data_dir.join("d2-export.tar"); + let export_arg = export_tar.display().to_string(); + docker_output( + data_dir, + &["export", "-o", &export_arg, STAGE_GRAPH_PROBE], + Duration::from_secs(120), + ) + .context("exporting probe container")?; + let listing = std::process::Command::new("tar") + .args(["-tf", &export_arg]) + .output() + .context("listing export tar")?; + if !listing.status.success() { + bail!( + "stage_graph: tar -tf failed: {}", + String::from_utf8_lossy(&listing.stderr) + ); + } + let names = String::from_utf8_lossy(&listing.stdout); + if !names.lines().any(|l| l.ends_with("artifact.txt")) { + bail!("stage_graph: artifact.txt missing from export listing:\n{names}"); + } + for residue in ["sentinel", "part", "shared", "out-", "make-"] { + if let Some(hit) = names.lines().find(|l| l.contains(residue)) { + bail!("stage_graph: builder-stage residue {hit:?} leaked into the final image"); + } + } + Ok(()) + })(); + + docker_ignore( + data_dir, + &["rm".into(), "-f".into(), STAGE_GRAPH_PROBE.into()], + ); + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), STAGE_GRAPH_TAG.into()], + ); + result +} From 2865fe8b17e5c2d2e8ddf8e2e6ef803cfe0e062f Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 02:24:01 +0800 Subject: [PATCH 03/21] test(e2e): D3 cache-semantics scenario for the docker build suite --- tests/e2e/tests/docker_build.rs | 169 +++++++++++++++++++++++++++++++- 1 file changed, 167 insertions(+), 2 deletions(-) diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index 14eebd4be..247ddbfe4 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -12,6 +12,12 @@ //! (the external frontend 12 of 25 corpus files declare). Asserts the //! final image carries exactly the export-stage artifact and none of the //! builder-stage residue. +//! - **D3 cache semantics** (mastodon / next.js shape): a +//! `RUN --mount=type=cache,sharing=locked` step plus a leaf step, then +//! three rebuilds — unchanged (full cache hit: identical image ID, warm +//! wall bounded), leaf-only change (upstream stamp stable, downstream +//! re-runs), base change (both re-run, and the cache mount's content +//! demonstrably survives across builds). //! //! All scenarios share one booted daemon; failures aggregate. The workload //! suite's quiet-log rule applies: builds must leave no proxy-layer ERROR. @@ -36,6 +42,14 @@ const STAGE_GRAPH_DEADLINE: Duration = Duration::from_secs(120); const STAGE_GRAPH_TAG: &str = "arcbox-e2e-build:stage-graph"; const STAGE_GRAPH_PROBE: &str = "arcbox-e2e-build-d2-probe"; +/// D3: per-build deadline, the sleep that gives the cold build measurable +/// duration (two sleeping steps ⇒ cold ≥ 2× this), and the warm bound — +/// `max(cold / 10, WARM_FLOOR)`, the suite's usual CI-slack floor so a +/// fast cold build isn't judged on noise. +const CACHE_BUILD_DEADLINE: Duration = Duration::from_secs(120); +const CACHE_STEP_SLEEP_SECS: u64 = 4; +const CACHE_WARM_FLOOR: Duration = Duration::from_secs(3); + #[test] #[ignore = "boots a VZ System VM through a real daemon; run on the e2e runner"] fn docker_build_suite() -> Result<()> { @@ -50,9 +64,12 @@ fn docker_build_suite() -> Result<()> { // every build below. let log = daemon_log_cursor(data_dir); - let scenarios: [(&str, ScenarioFn); 1] = [("stage_graph", stage_graph)]; + let scenarios: [(&str, ScenarioFn); 2] = [ + ("stage_graph", stage_graph), + ("cache_semantics", cache_semantics), + ]; // Diagnostic filter: run only the named scenario, e.g. - // ARCBOX_E2E_BUILD_ONLY=stage_graph. + // ARCBOX_E2E_BUILD_ONLY=cache_semantics. let only = std::env::var("ARCBOX_E2E_BUILD_ONLY").ok(); let mut failures = Vec::new(); for (name, scenario) in scenarios { @@ -123,6 +140,17 @@ fn timed_build( Ok((output, elapsed)) } +/// `cat`s one file out of a built image via a throwaway container. +fn image_file(data_dir: &Path, tag: &str, path: &str) -> Result { + docker_output( + data_dir, + &["run", "--rm", tag, "cat", path], + Duration::from_secs(60), + ) + .map(|out| out.trim().to_owned()) + .with_context(|| format!("reading {path} from {tag}")) +} + /// D2: 12-stage diamond under the external `docker/dockerfile:1` frontend. /// Four independent branches (one fed from a `FROM scratch` script-carrier /// stage, one built through a heredoc RUN) join into an aggregate, and a @@ -279,3 +307,140 @@ COPY --link --from=join /artifact.txt /artifact.txt ); result } + +/// D3: cache semantics across four builds of one Dockerfile. Each RUN step +/// writes a per-execution random stamp, so "was this step re-executed?" is +/// read from the image itself rather than parsed out of progress output. +/// The cached step's `--mount=type=cache` also round-trips a token, proving +/// the cache mount's content survives across builds. +fn cache_semantics(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d3-ctx"); + std::fs::create_dir_all(&ctx).context("creating d3 context dir")?; + std::fs::write(ctx.join("base.txt"), "base-v1\n")?; + std::fs::write(ctx.join("leaf.txt"), "leaf-v1\n")?; + let sleep = CACHE_STEP_SLEEP_SECS; + let stamp = "head -c 16 /dev/urandom | sha256sum | cut -d' ' -f1"; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} +COPY base.txt /base.txt +RUN --mount=type=cache,target=/build-cache,sharing=locked (cp /build-cache/token /cache-prev 2>/dev/null || : > /cache-prev) && cp /base.txt /build-cache/token && sleep {sleep} && {stamp} > /stamp-base +COPY leaf.txt /leaf.txt +RUN sleep {sleep} && {stamp} > /stamp-leaf +" + ), + ) + .context("writing d3 Dockerfile")?; + + let tags = ["cold", "warm", "leaf", "base"].map(|t| format!("arcbox-e2e-build:cache-{t}")); + let [cold_tag, warm_tag, leaf_tag, base_tag] = &tags; + + let result = (|| { + // Build 1 — cold: everything executes, the cache mount is seeded. + let (_, cold_wall) = timed_build( + data_dir, + metrics, + "cache_cold_wall", + &ctx, + cold_tag, + CACHE_BUILD_DEADLINE, + )?; + let cold_stamp_base = image_file(data_dir, cold_tag, "/stamp-base")?; + let cold_stamp_leaf = image_file(data_dir, cold_tag, "/stamp-leaf")?; + let cold_cache_prev = image_file(data_dir, cold_tag, "/cache-prev")?; + if cold_stamp_base.is_empty() || cold_stamp_leaf.is_empty() { + bail!("cache: cold build produced empty stamps"); + } + if !cold_cache_prev.is_empty() { + bail!( + "cache: cold build saw a pre-existing cache token {cold_cache_prev:?} — \ + the fresh daemon's build cache was not empty" + ); + } + + // Build 2 — unchanged: a full cache hit, so the same image ID and a + // wall bounded far below cold (with the usual CI-slack floor). + let (_, warm_wall) = timed_build( + data_dir, + metrics, + "cache_warm_wall", + &ctx, + warm_tag, + CACHE_BUILD_DEADLINE, + )?; + let id = |tag: &str| { + docker_output( + data_dir, + &["image", "inspect", "-f", "{{.Id}}", tag], + Duration::from_secs(30), + ) + .map(|out| out.trim().to_owned()) + }; + let (cold_id, warm_id) = (id(cold_tag)?, id(warm_tag)?); + if warm_id != cold_id { + bail!("cache: unchanged rebuild produced a different image ({cold_id} vs {warm_id})"); + } + let warm_bound = cold_wall.div_f64(10.0).max(CACHE_WARM_FLOOR); + if warm_wall > warm_bound { + bail!( + "cache: unchanged rebuild took {warm_wall:?} (bound {warm_bound:?}, \ + cold {cold_wall:?}) — cache hit not engaging" + ); + } + + // Build 3 — leaf change: the step above the change stays cached, + // the steps at and below it re-execute. + std::fs::write(ctx.join("leaf.txt"), "leaf-v2\n")?; + timed_build( + data_dir, + metrics, + "cache_leaf_wall", + &ctx, + leaf_tag, + CACHE_BUILD_DEADLINE, + )?; + let leaf_stamp_base = image_file(data_dir, leaf_tag, "/stamp-base")?; + let leaf_stamp_leaf = image_file(data_dir, leaf_tag, "/stamp-leaf")?; + if leaf_stamp_base != cold_stamp_base { + bail!("cache: leaf-only change re-executed the upstream cached step"); + } + if leaf_stamp_leaf == cold_stamp_leaf { + bail!("cache: leaf change did not re-execute the downstream step"); + } + + // Build 4 — base change: both steps re-execute, and the re-run + // cached step reads back the token build 1 wrote — the cache mount + // persisted across builds. + std::fs::write(ctx.join("base.txt"), "base-v2\n")?; + timed_build( + data_dir, + metrics, + "cache_base_wall", + &ctx, + base_tag, + CACHE_BUILD_DEADLINE, + )?; + let base_stamp_base = image_file(data_dir, base_tag, "/stamp-base")?; + let base_stamp_leaf = image_file(data_dir, base_tag, "/stamp-leaf")?; + let base_cache_prev = image_file(data_dir, base_tag, "/cache-prev")?; + if base_stamp_base == cold_stamp_base { + bail!("cache: base change did not re-execute the cached step"); + } + if base_stamp_leaf == leaf_stamp_leaf { + bail!("cache: base change did not invalidate the downstream step"); + } + if base_cache_prev != "base-v1" { + bail!( + "cache: cache-mount token did not survive across builds \ + (read {base_cache_prev:?}, expected \"base-v1\")" + ); + } + Ok(()) + })(); + + for tag in &tags { + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.clone()]); + } + result +} From 918cfbbbafa8d55c5c7d5f567cb898c7cca805b5 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 02:24:12 +0800 Subject: [PATCH 04/21] test(e2e): D1 large-context scenario; wire docker_build into xtask prebuild --- .../plans/docker-build-e2e-matrix.md | 6 +- tests/e2e/tests/docker_build.rs | 121 +++++++++++++++++- xtask/src/commands/e2e.rs | 5 +- 3 files changed, 126 insertions(+), 6 deletions(-) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index a3e5033dc..cc3c98884 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -92,9 +92,9 @@ assertions are deadlines and behavior, never absolute throughput. | ID | Pattern (modeled on) | Mechanism | Assertions | |---|---|---|---| -| D1 | large context upload (immich/next.js node_modules shape) | 512 MiB incompressible + ~100k small files; `.dockerignore` excluding half | build ok ≤ deadline; in-build `wc -c` byte-exact; ignored file absent from context (`COPY` of it fails); `context_upload_wall` trend | -| D2 | deep multi-stage graph (grafana/buildkit) | 12-stage diamond: parallel-schedulable branches, `COPY --from` joins, `FROM scratch AS export` tail, `COPY --link`, heredoc RUN | final image contains only export-stage content; builder-stage files absent; build ok ≤ deadline | -| D3 | cache semantics (mastodon/next.js) | `RUN --mount=type=cache,sharing=locked`; then 3 rebuilds: unchanged, one-file leaf change, base-layer change | unchanged rebuild ≤ 10% of cold wall; leaf change re-runs only downstream steps (assert via `--progress=plain` CACHED lines); cache mount contents survive across builds | +| D1 | large context upload (immich/next.js node_modules shape) | 512 MiB incompressible + 100k small files; `.dockerignore` excluding a marker dir | build ok ≤ deadline; in-build payload sha + whole-tree sha byte-exact; ignored dir absent in-guest; `large_context_transfer` trend *(implemented)* | +| D2 | deep multi-stage graph (grafana/buildkit) | 12-stage diamond: `FROM scratch AS scripts` carrier, 4 parallel-schedulable branches, `COPY --from` joins, `FROM scratch AS export` tail, `COPY --link`, heredoc RUN, `# syntax=docker/dockerfile:1` (frontend pre-loaded) | export-stage artifact byte-exact via `docker cp`; full `docker export` listing free of builder residue; build ok ≤ deadline *(implemented)* | +| D3 | cache semantics (mastodon/next.js) | `RUN --mount=type=cache,sharing=locked`; then 3 rebuilds: unchanged, one-file leaf change, base-layer change | unchanged rebuild reuses the image (identical ID) with wall ≤ max(cold/10, 3 s floor); per-execution stamps baked into the image prove leaf change re-runs only downstream steps and base change re-runs both; cache-mount token survives across builds *(implemented)* | | D4 | BuildKit session (authentik secrets; ssh has no OSS exemplar but same channel) | `--secret id=x,src=file` consumed by `RUN --mount=type=secret`; `--ssh default` against a throwaway host `ssh-agent` | secret readable in RUN, **absent from every layer** (`docker history`/`save` sweep); ssh sock present in RUN; `/session` upgrade leaves no proxy ERROR | | D5 | bind-mount lockfiles (authentik/immich) | `RUN --mount=type=bind,source=...` consuming context files without COPY layers | build ok; layer count unchanged by bind mounts | | D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) correct arch strings baked; (b) FEX provisioned ⇒ `x86_64` output, else **fail-closed** with the actionable error (both directions asserted) | diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index 247ddbfe4..cf2375a1c 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -5,6 +5,11 @@ //! this suite tests the build surface itself, with scenarios modeled on the //! real-world Dockerfile corpus in the plan: //! +//! - **D1 large context** (immich / node_modules shape): 512 MiB +//! incompressible payload + 100k small files through the context upload, +//! with `.dockerignore` exclusion. Byte-exactness asserted *inside* the +//! build (payload sha + whole-tree sha), so truncation or corruption +//! anywhere in the transfer fails the build. //! - **D2 stage graph** (grafana / buildkit shape): a 12-stage diamond — //! `FROM scratch AS scripts` carrier (airflow pattern), four independent //! branches, `COPY --from` joins, a heredoc RUN, `COPY --link`, and a @@ -30,6 +35,7 @@ use arcbox_e2e::docker::{docker_ignore, docker_output, ensure_image}; use arcbox_e2e::metrics::RunMetrics; use arcbox_e2e::net_fixtures::daemon_log_cursor; use arcbox_e2e::scenario::run_vz_scenario_with_log; +use sha2::Digest; const READY_TIMEOUT: Duration = Duration::from_secs(180); @@ -37,6 +43,13 @@ const READY_TIMEOUT: Duration = Duration::from_secs(180); /// pre-loaded so the build does not depend on registry reachability. const FRONTEND_IMAGE: &str = "docker/dockerfile:1"; +/// D1: payload size and small-file tree shape (100 dirs × 1000 files). +const LARGE_CTX_PAYLOAD: usize = 512 * 1024 * 1024; +const LARGE_CTX_TREE_DIRS: usize = 100; +const LARGE_CTX_FILES_PER_DIR: usize = 1000; +const LARGE_CTX_DEADLINE: Duration = Duration::from_secs(300); +const LARGE_CTX_TAG: &str = "arcbox-e2e-build:large-ctx"; + /// D2: stage-graph build deadline. const STAGE_GRAPH_DEADLINE: Duration = Duration::from_secs(120); const STAGE_GRAPH_TAG: &str = "arcbox-e2e-build:stage-graph"; @@ -64,9 +77,10 @@ fn docker_build_suite() -> Result<()> { // every build below. let log = daemon_log_cursor(data_dir); - let scenarios: [(&str, ScenarioFn); 2] = [ + let scenarios: [(&str, ScenarioFn); 3] = [ ("stage_graph", stage_graph), ("cache_semantics", cache_semantics), + ("large_context", large_context), ]; // Diagnostic filter: run only the named scenario, e.g. // ARCBOX_E2E_BUILD_ONLY=cache_semantics. @@ -151,6 +165,33 @@ fn image_file(data_dir: &Path, tag: &str, path: &str) -> Result { .with_context(|| format!("reading {path} from {tag}")) } +/// Extracts BuildKit's own context-transfer timing from `--progress=plain` +/// output (`#N transferring context: s done`). Best-effort: +/// the line format is BuildKit's, not ours, so a miss records nothing. +fn context_transfer_seconds(output: &str) -> Option { + output + .lines() + .rfind(|line| line.contains("transferring context:") && line.trim_end().ends_with("done")) + .and_then(|line| { + line.split_whitespace() + .rev() + .filter(|token| token.len() > 1 && token.ends_with('s')) + .find_map(|token| token.trim_end_matches('s').parse::().ok()) + }) +} + +/// Fills `buf` with xorshift output — incompressible, so context transfer +/// and layer commits move real bytes. +fn fill_incompressible(buf: &mut [u8]) { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + for chunk in buf.chunks_mut(8) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + chunk.copy_from_slice(&state.to_le_bytes()[..chunk.len()]); + } +} + /// D2: 12-stage diamond under the external `docker/dockerfile:1` frontend. /// Four independent branches (one fed from a `FROM scratch` script-carrier /// stage, one built through a heredoc RUN) join into an aggregate, and a @@ -444,3 +485,81 @@ RUN sleep {sleep} && {stamp} > /stamp-leaf } result } + +/// D1: 512 MiB incompressible payload plus a 100k-file tree through the +/// context upload, `.dockerignore` honored. All integrity assertions run +/// *inside* the build: the payload sha, the file count, and a whole-tree +/// sha computed in deterministic path order, so a single lost, truncated, +/// or corrupted entry anywhere in the transfer fails the build. +fn large_context(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d1-ctx"); + std::fs::create_dir_all(&ctx).context("creating d1 context dir")?; + + let generated = Instant::now(); + let mut payload = vec![0u8; LARGE_CTX_PAYLOAD]; + fill_incompressible(&mut payload); + std::fs::write(ctx.join("payload.bin"), &payload).context("writing payload")?; + let payload_sha = format!("{:x}", sha2::Sha256::digest(&payload)); + drop(payload); + + // Zero-padded names make guest-side `find . | sort` order equal this + // generation order, so one incremental host hash matches the in-build + // whole-tree hash. + let mut tree_hasher = sha2::Sha256::new(); + for dir in 0..LARGE_CTX_TREE_DIRS { + let dir_name = format!("d{dir:03}"); + let dir_path = ctx.join("tree").join(&dir_name); + std::fs::create_dir_all(&dir_path)?; + for file in 0..LARGE_CTX_FILES_PER_DIR { + let content = format!("{dir_name}/f{file:03}\n"); + std::fs::write(dir_path.join(format!("f{file:03}")), &content)?; + tree_hasher.update(content.as_bytes()); + } + } + let tree_sha = format!("{:x}", tree_hasher.finalize()); + let file_count = LARGE_CTX_TREE_DIRS * LARGE_CTX_FILES_PER_DIR; + + std::fs::write(ctx.join(".dockerignore"), "excluded/\n")?; + std::fs::create_dir_all(ctx.join("excluded"))?; + std::fs::write(ctx.join("excluded/marker.txt"), "must-not-transfer\n")?; + tracing::info!( + elapsed = ?generated.elapsed(), + payload_mib = LARGE_CTX_PAYLOAD / (1024 * 1024), + file_count, + "large_context: context generated" + ); + + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +COPY . /ctx +RUN echo "{payload_sha} /ctx/payload.bin" | sha256sum -c - +RUN test "$(find /ctx/tree -type f | wc -l)" -eq {file_count} +RUN cd /ctx/tree && test "$(find . -type f | sort | xargs cat | sha256sum | cut -d' ' -f1)" = "{tree_sha}" +RUN test ! -e /ctx/excluded +"# + ), + ) + .context("writing d1 Dockerfile")?; + + let result = (|| { + let (output, elapsed) = timed_build( + data_dir, + metrics, + "large_context_build_wall", + &ctx, + LARGE_CTX_TAG, + LARGE_CTX_DEADLINE, + )?; + match context_transfer_seconds(&output) { + Some(seconds) => metrics.record("large_context_transfer", seconds), + None => tracing::info!("large_context: no transfer timing in build output"), + } + tracing::info!(?elapsed, "large_context: build done"); + Ok(()) + })(); + + docker_ignore(data_dir, &["rmi".into(), "-f".into(), LARGE_CTX_TAG.into()]); + result +} diff --git a/xtask/src/commands/e2e.rs b/xtask/src/commands/e2e.rs index 518a4a606..cbf631fc3 100644 --- a/xtask/src/commands/e2e.rs +++ b/xtask/src/commands/e2e.rs @@ -153,8 +153,9 @@ fn prebuild(root: &std::path::Path, test: &str) -> Result { .run()?; Ok(true) } - // Must match tests/e2e/tests/egress_throughput.rs's own build. - "egress_throughput" => { + // Must match tests/e2e/src/scenario.rs's self-build (the shared + // daemon-boot scaffolding both of these targets run through). + "egress_throughput" | "docker_build" => { xshell::cmd!(shell, "cargo build --release -p arcbox-daemon").run()?; Ok(true) } From 0a9915121e9479a9b90bf7d256427fb59f7ee07a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 03:24:01 +0800 Subject: [PATCH 05/21] =?UTF-8?q?docs(e2e):=20record=20docker=5Fbuild=20su?= =?UTF-8?q?ite=20status=20=E2=80=94=20red=20on=20ABX-494=20(FEX=20wedges?= =?UTF-8?q?=20BuildKit)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal-docs/plans/docker-build-e2e-matrix.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index cc3c98884..eea9d3053 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -129,6 +129,15 @@ workload suite. Cross-runtime comparison (OrbStack / Docker Desktop) is a manual `xtask` run of the same fixtures against another engine's `DOCKER_HOST`, not an automated gate. +## Status + +D1–D3 are implemented (`tests/e2e/tests/docker_build.rs`) and currently +**red for a real product reason**: every buildx-driven build hangs because +guest FEX spins forever on BuildKit's amd64 arch-probe ELF, permanently +wedging the BuildKit Control API (ABX-494 — first caught by this suite's +first run; `network_workload` W14 reproduces it too). Do not weaken the +suite; it goes green when ABX-494 is fixed. + ## Phasing 1. D1–D3 (context, stages, cache) — pure fixture work, no new capability From 0360c5a7a691a9435c798fd63504550eb1c5be3b Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 14:02:32 +0800 Subject: [PATCH 06/21] docs(xtask): add egress_throughput and docker_build rows to the prebuild recipe table --- xtask/AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xtask/AGENTS.md b/xtask/AGENTS.md index c9da3e5ba..a189f8597 100644 --- a/xtask/AGENTS.md +++ b/xtask/AGENTS.md @@ -34,6 +34,8 @@ Current recipe ↔ self-build mapping (keep both columns identical): | `hv_vmm` | `cargo build --release -p arcbox-e2e --bin hv_e2e`| `hv_vmm.rs` (same) | | `stats_watch` | release daemon + musl `arcbox-agent` | `stats_watch.rs` (same) | | `sandbox` | release cli+daemon + musl `arcbox-agent`/`arcbox-vm` bins | `sandbox.rs::build_binaries` (same) | +| `egress_throughput` | `cargo build --release -p arcbox-daemon` | `scenario.rs::run_vz_scenario_with_log` (same) | +| `docker_build` | same as above | `scenario.rs::run_vz_scenario_with_log` (same) | | `virtio_debug` | none (self-builds) | `cargo build --release -p arcbox-daemon` (RELEASE only) | | `daemon_failure` | none (self-builds) | `cargo build -p arcbox-daemon` (**DEBUG**) | | anything else | none — fallback notice, no `SKIP_BUILD` | its own build | From 62abf37dcaefd617dd4b490f8a6b9122d6c93725 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 15:23:23 +0800 Subject: [PATCH 07/21] test(e2e): D4 session (secret/ssh) + D5 bind-mount scenarios for docker build --- tests/e2e/tests/docker_build.rs | 270 ++++++++++++++++++++++++++++++-- 1 file changed, 260 insertions(+), 10 deletions(-) diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index cf2375a1c..d9280cb7b 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -1,4 +1,4 @@ -//! docker build e2e — Phase 1 (D1–D3) of +//! docker build e2e — Phases 1–2 (D1–D5, D9–D10) of //! internal-docs/plans/docker-build-e2e-matrix.md. //! //! Where `network_workload` W14 drives *one* build to prove the datapath, @@ -23,6 +23,16 @@ //! wall bounded), leaf-only change (upstream stamp stable, downstream //! re-runs), base change (both re-run, and the cache mount's content //! demonstrably survives across builds). +//! - **D4 session channel** (authentik shape): `--secret` readable inside +//! its RUN, gone from the next layer, and absent from every byte of the +//! saved image; `--ssh` forwards a live host agent socket. Both ride the +//! `/session` upgrade proxy, previously covered by unit tests only. +//! - **D5 bind-mount lockfiles** (authentik / immich pnpm shape): +//! `RUN --mount=type=bind` consumes a context file without a COPY layer. +//! - **D9 output streaming**: a chatty RUN streamed un-clipped through the +//! proxy, and `-q` mode reduced to the bare image ID. +//! - **D10 exporters**: `--output type=local` and `type=tar` — build +//! artifacts streaming host-ward through the session (reverse direction). //! //! All scenarios share one booted daemon; failures aggregate. The workload //! suite's quiet-log rule applies: builds must leave no proxy-layer ERROR. @@ -63,6 +73,17 @@ const CACHE_BUILD_DEADLINE: Duration = Duration::from_secs(120); const CACHE_STEP_SLEEP_SECS: u64 = 4; const CACHE_WARM_FLOOR: Duration = Duration::from_secs(3); +/// D4: the secret value asserted inside the build and swept for outside it. +/// Alphanumeric+dash so it can be inlined into shell fragments verbatim. +const SECRET_VALUE: &str = "s3cret-build-token-abx494"; +const SESSION_DEADLINE: Duration = Duration::from_secs(120); +const SECRET_TAG: &str = "arcbox-e2e-build:secret"; +const SSH_TAG: &str = "arcbox-e2e-build:ssh"; + +/// D5: bind-mount lockfile scenario. +const BIND_DEADLINE: Duration = Duration::from_secs(120); +const BIND_TAG: &str = "arcbox-e2e-build:bind"; + #[test] #[ignore = "boots a VZ System VM through a real daemon; run on the e2e runner"] fn docker_build_suite() -> Result<()> { @@ -77,9 +98,11 @@ fn docker_build_suite() -> Result<()> { // every build below. let log = daemon_log_cursor(data_dir); - let scenarios: [(&str, ScenarioFn); 3] = [ + let scenarios: [(&str, ScenarioFn); 5] = [ ("stage_graph", stage_graph), ("cache_semantics", cache_semantics), + ("session_secret_ssh", session_secret_ssh), + ("bind_mounts", bind_mounts), ("large_context", large_context), ]; // Diagnostic filter: run only the named scenario, e.g. @@ -127,24 +150,24 @@ fn docker_build_suite() -> Result<()> { type ScenarioFn = fn(&Path, &mut RunMetrics, &str) -> Result<()>; -/// Runs `docker build --progress=plain -t tag ctx`, recording the wall -/// under `key` (also for failed builds — a slow failure is still trend -/// data) and failing past `deadline`. +/// Runs `docker build --progress=plain -t tag ctx`, recording +/// the wall under `key` (also for failed builds — a slow failure is still +/// trend data) and failing past `deadline`. fn timed_build( data_dir: &Path, metrics: &mut RunMetrics, key: &str, ctx: &Path, tag: &str, + extra_args: &[&str], deadline: Duration, ) -> Result<(String, Duration)> { let ctx_arg = ctx.display().to_string(); + let mut args = vec!["build", "--progress=plain", "-t", tag]; + args.extend_from_slice(extra_args); + args.push(&ctx_arg); let started = Instant::now(); - let build = docker_output( - data_dir, - &["build", "--progress=plain", "-t", tag, &ctx_arg], - deadline + Duration::from_secs(60), - ); + let build = docker_output(data_dir, &args, deadline + Duration::from_secs(60)); let elapsed = started.elapsed(); metrics.record(key, elapsed.as_secs_f64()); let output = build.with_context(|| format!("{key}: docker build"))?; @@ -268,6 +291,7 @@ COPY --link --from=join /artifact.txt /artifact.txt "stage_graph_build_wall", &ctx, STAGE_GRAPH_TAG, + &[], STAGE_GRAPH_DEADLINE, )?; @@ -385,6 +409,7 @@ RUN sleep {sleep} && {stamp} > /stamp-leaf "cache_cold_wall", &ctx, cold_tag, + &[], CACHE_BUILD_DEADLINE, )?; let cold_stamp_base = image_file(data_dir, cold_tag, "/stamp-base")?; @@ -408,6 +433,7 @@ RUN sleep {sleep} && {stamp} > /stamp-leaf "cache_warm_wall", &ctx, warm_tag, + &[], CACHE_BUILD_DEADLINE, )?; let id = |tag: &str| { @@ -439,6 +465,7 @@ RUN sleep {sleep} && {stamp} > /stamp-leaf "cache_leaf_wall", &ctx, leaf_tag, + &[], CACHE_BUILD_DEADLINE, )?; let leaf_stamp_base = image_file(data_dir, leaf_tag, "/stamp-base")?; @@ -460,6 +487,7 @@ RUN sleep {sleep} && {stamp} > /stamp-leaf "cache_base_wall", &ctx, base_tag, + &[], CACHE_BUILD_DEADLINE, )?; let base_stamp_base = image_file(data_dir, base_tag, "/stamp-base")?; @@ -550,6 +578,7 @@ RUN test ! -e /ctx/excluded "large_context_build_wall", &ctx, LARGE_CTX_TAG, + &[], LARGE_CTX_DEADLINE, )?; match context_transfer_seconds(&output) { @@ -563,3 +592,224 @@ RUN test ! -e /ctx/excluded docker_ignore(data_dir, &["rmi".into(), "-f".into(), LARGE_CTX_TAG.into()]); result } + +/// A throwaway host `ssh-agent` on a private socket, killed on drop. The +/// agent carries one fresh ed25519 key so the forwarded socket is a +/// realistic one, not an empty stub. +struct ThrowawaySshAgent { + pid: String, + sock: std::path::PathBuf, +} + +impl ThrowawaySshAgent { + fn spawn(dir: &Path) -> Result { + let sock = dir.join("d4-agent.sock"); + let output = std::process::Command::new("ssh-agent") + .args(["-a", &sock.display().to_string()]) + .output() + .context("spawning ssh-agent")?; + if !output.status.success() { + bail!( + "ssh-agent failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let pid = stdout + .split("SSH_AGENT_PID=") + .nth(1) + .and_then(|rest| rest.split(';').next()) + .map(str::to_owned) + .ok_or_else(|| anyhow::anyhow!("no SSH_AGENT_PID in ssh-agent output: {stdout}"))?; + let agent = Self { pid, sock }; + + let key = dir.join("d4-ssh-key"); + let keygen = std::process::Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&key) + .output() + .context("generating throwaway ssh key")?; + if !keygen.status.success() { + bail!( + "ssh-keygen failed: {}", + String::from_utf8_lossy(&keygen.stderr) + ); + } + let add = std::process::Command::new("ssh-add") + .arg(&key) + .env("SSH_AUTH_SOCK", &agent.sock) + .output() + .context("loading key into ssh-agent")?; + if !add.status.success() { + bail!("ssh-add failed: {}", String::from_utf8_lossy(&add.stderr)); + } + Ok(agent) + } +} + +impl Drop for ThrowawaySshAgent { + fn drop(&mut self) { + let _ = std::process::Command::new("kill").arg(&self.pid).status(); + } +} + +/// Sweeps every byte of `docker save ` for `needle` — including inside +/// gzip'd layer blobs (the containerd image store compresses them). A hit +/// anywhere fails: build secrets must never persist into image content. +fn assert_absent_from_saved_image( + data_dir: &Path, + tag: &str, + needle: &str, + label: &str, +) -> Result<()> { + let tar = data_dir.join(format!("{label}-save.tar")); + let tar_arg = tar.display().to_string(); + docker_output( + data_dir, + &["save", "-o", &tar_arg, tag], + Duration::from_secs(120), + ) + .context("docker save for secret sweep")?; + + let script = format!( + r#"set -e +d=$(mktemp -d) +trap 'rm -rf "$d"' EXIT +tar -xf "{tar_arg}" -C "$d" +find "$d" -type f | while read -r f; do + if [ "$(head -c 2 "$f" | od -An -tx1 | tr -d ' \n')" = "1f8b" ]; then + gunzip -c "$f" 2>/dev/null | grep -aq -- "{needle}" && echo "LEAK:$f" + else + grep -aq -- "{needle}" "$f" && echo "LEAK:$f" + fi +done +echo SWEEP-DONE"# + ); + let sweep = std::process::Command::new("sh") + .args(["-c", &script]) + .output() + .context("running secret sweep")?; + let stdout = String::from_utf8_lossy(&sweep.stdout); + if !stdout.contains("SWEEP-DONE") { + bail!( + "{label}: secret sweep did not complete: {stdout}{}", + String::from_utf8_lossy(&sweep.stderr) + ); + } + if stdout.contains("LEAK:") { + bail!("{label}: secret bytes leaked into the saved image:\n{stdout}"); + } + Ok(()) +} + +/// D4: the BuildKit session channel — `--secret` and `--ssh` both ride the +/// `/session` HTTP upgrade through the proxy. The secret must be readable +/// inside its mounting RUN, gone from the next layer, and absent from every +/// byte of the exported image; the ssh mount must expose a live forwarded +/// agent socket. +fn session_secret_ssh(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d4-secret-ctx"); + std::fs::create_dir_all(&ctx).context("creating d4 secret context dir")?; + std::fs::write(ctx.join("token.txt"), SECRET_VALUE)?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +RUN --mount=type=secret,id=build_token test "$(cat /run/secrets/build_token)" = "{SECRET_VALUE}" && echo secret-visible > /probe +RUN test ! -e /run/secrets/build_token +"# + ), + ) + .context("writing d4 secret Dockerfile")?; + + let secret_result = (|| { + timed_build( + data_dir, + metrics, + "session_secret_build_wall", + &ctx, + SECRET_TAG, + &["--secret", "id=build_token,src=token.txt"], + SESSION_DEADLINE, + )?; + if image_file(data_dir, SECRET_TAG, "/probe")? != "secret-visible" { + bail!("secret: probe layer missing"); + } + assert_absent_from_saved_image(data_dir, SECRET_TAG, SECRET_VALUE, "session_secret") + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), SECRET_TAG.into()]); + secret_result?; + + let agent = ThrowawaySshAgent::spawn(data_dir)?; + let ctx = data_dir.join("d4-ssh-ctx"); + std::fs::create_dir_all(&ctx).context("creating d4 ssh context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +RUN --mount=type=ssh test -S "$SSH_AUTH_SOCK" && echo ssh-forwarded > /probe +"# + ), + ) + .context("writing d4 ssh Dockerfile")?; + + let ssh_arg = format!("default={}", agent.sock.display()); + let ssh_result = (|| { + timed_build( + data_dir, + metrics, + "session_ssh_build_wall", + &ctx, + SSH_TAG, + &["--ssh", &ssh_arg], + SESSION_DEADLINE, + )?; + if image_file(data_dir, SSH_TAG, "/probe")? != "ssh-forwarded" { + bail!("ssh: probe layer missing"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), SSH_TAG.into()]); + ssh_result +} + +/// D5: `RUN --mount=type=bind` consumes a context file with no COPY layer — +/// the pnpm-lockfile shape. The mounted file must be readable in its RUN, +/// derived content must land in the image, and the mount itself must leave +/// no trace in the final filesystem. +fn bind_mounts(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d5-ctx"); + std::fs::create_dir_all(&ctx).context("creating d5 context dir")?; + let lockfile = "lockfile-v1\n"; + std::fs::write(ctx.join("lockfile.txt"), lockfile)?; + let lockfile_sha = format!("{:x}", sha2::Sha256::digest(lockfile.as_bytes())); + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} +RUN --mount=type=bind,source=lockfile.txt,target=/lockfile.txt sha256sum /lockfile.txt | cut -d' ' -f1 > /derived +RUN test ! -e /lockfile.txt +" + ), + ) + .context("writing d5 Dockerfile")?; + + let result = (|| { + timed_build( + data_dir, + metrics, + "bind_mounts_build_wall", + &ctx, + BIND_TAG, + &[], + BIND_DEADLINE, + )?; + let derived = image_file(data_dir, BIND_TAG, "/derived")?; + if derived != lockfile_sha { + bail!("bind_mounts: derived hash {derived} != host hash {lockfile_sha}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), BIND_TAG.into()]); + result +} From 95eefc69c62f16a0e39e0431c18c27af01f98730 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 15:24:57 +0800 Subject: [PATCH 08/21] test(e2e): D9 output-streaming + D10 exporter scenarios for docker build --- .../plans/docker-build-e2e-matrix.md | 8 +- tests/e2e/tests/docker_build.rs | 142 +++++++++++++++++- 2 files changed, 145 insertions(+), 5 deletions(-) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index eea9d3053..c6bed17bc 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -95,13 +95,13 @@ assertions are deadlines and behavior, never absolute throughput. | D1 | large context upload (immich/next.js node_modules shape) | 512 MiB incompressible + 100k small files; `.dockerignore` excluding a marker dir | build ok ≤ deadline; in-build payload sha + whole-tree sha byte-exact; ignored dir absent in-guest; `large_context_transfer` trend *(implemented)* | | D2 | deep multi-stage graph (grafana/buildkit) | 12-stage diamond: `FROM scratch AS scripts` carrier, 4 parallel-schedulable branches, `COPY --from` joins, `FROM scratch AS export` tail, `COPY --link`, heredoc RUN, `# syntax=docker/dockerfile:1` (frontend pre-loaded) | export-stage artifact byte-exact via `docker cp`; full `docker export` listing free of builder residue; build ok ≤ deadline *(implemented)* | | D3 | cache semantics (mastodon/next.js) | `RUN --mount=type=cache,sharing=locked`; then 3 rebuilds: unchanged, one-file leaf change, base-layer change | unchanged rebuild reuses the image (identical ID) with wall ≤ max(cold/10, 3 s floor); per-execution stamps baked into the image prove leaf change re-runs only downstream steps and base change re-runs both; cache-mount token survives across builds *(implemented)* | -| D4 | BuildKit session (authentik secrets; ssh has no OSS exemplar but same channel) | `--secret id=x,src=file` consumed by `RUN --mount=type=secret`; `--ssh default` against a throwaway host `ssh-agent` | secret readable in RUN, **absent from every layer** (`docker history`/`save` sweep); ssh sock present in RUN; `/session` upgrade leaves no proxy ERROR | -| D5 | bind-mount lockfiles (authentik/immich) | `RUN --mount=type=bind,source=...` consuming context files without COPY layers | build ok; layer count unchanged by bind mounts | +| D4 | BuildKit session (authentik secrets; ssh has no OSS exemplar but same channel) | `--secret id=x,src=file` consumed by `RUN --mount=type=secret`; `--ssh default=` against a throwaway host `ssh-agent` (one loaded key) | secret readable in RUN, gone next layer, **absent from every byte of `docker save`** (gzip-aware sweep); forwarded ssh sock present in RUN; quiet-log covers the `/session` upgrades *(implemented)* | +| D5 | bind-mount lockfiles (authentik/immich) | `RUN --mount=type=bind,source=...` consuming context files without COPY layers | in-build read + derived hash lands in image; mount leaves no trace in the final fs *(implemented)* | | D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) correct arch strings baked; (b) FEX provisioned ⇒ `x86_64` output, else **fail-closed** with the actionable error (both directions asserted) | | D7 | concurrent builds (CI shape) | 4 parallel distinct builds + 2 same-context duplicates | all complete ≤ deadline; no cross-talk (unique markers per image); zombie sweep clean | | D8 | cancellation (production reality) | kill client mid-context-upload and mid-RUN; rebuild after | daemon log free of proxy ERROR; follow-up build succeeds; no leaked build processes guest-side | -| D9 | output streaming | `--progress=plain` with a RUN emitting 10 MB stdout; `docker build -q` | full log arrives uncorrupted; quiet mode prints only the image ID; no proxy wedge | -| D10 | exporters | `docker build -o type=local,dest=…` and `-o type=tar` | exported artifact byte-exact vs in-image content — exercises reverse session streaming | +| D9 | output streaming | `--progress=plain` with a RUN emitting ~1 MiB of ordered lines (below BuildKit's per-step clip, so "no clip marker" is a fair assertion); `docker build -q` | tail lines present, no clip marker, deadline catches a wedge; quiet mode prints exactly one image ID *(implemented)* | +| D10 | exporters | `docker build -o type=local,dest=…` and `-o type=tar` | exported artifact byte-exact vs in-image content — exercises reverse session streaming *(implemented)* | External syntax frontends (`# syntax=docker/dockerfile:1`) are exercised in D2 by pre-pulling the frontend image in the fixture; if the guest BuildKit diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index d9280cb7b..762d9a8cd 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -84,6 +84,18 @@ const SSH_TAG: &str = "arcbox-e2e-build:ssh"; const BIND_DEADLINE: Duration = Duration::from_secs(120); const BIND_TAG: &str = "arcbox-e2e-build:bind"; +/// D9: chatty-RUN line count (~1 MiB of step output — deliberately below +/// BuildKit's default per-step log clip, so "no clip marker" is a fair +/// streaming-integrity assertion rather than a tunable). +const STREAM_LINES: usize = 150_000; +const STREAM_DEADLINE: Duration = Duration::from_secs(120); +const STREAM_TAG: &str = "arcbox-e2e-build:stream"; +const QUIET_TAG: &str = "arcbox-e2e-build:quiet"; + +/// D10: exporter scenario — artifacts stream host-ward through the session. +const EXPORT_DEADLINE: Duration = Duration::from_secs(120); +const EXPORT_PAYLOAD: &str = "exporter-proof-payload\n"; + #[test] #[ignore = "boots a VZ System VM through a real daemon; run on the e2e runner"] fn docker_build_suite() -> Result<()> { @@ -98,11 +110,13 @@ fn docker_build_suite() -> Result<()> { // every build below. let log = daemon_log_cursor(data_dir); - let scenarios: [(&str, ScenarioFn); 5] = [ + let scenarios: [(&str, ScenarioFn); 7] = [ ("stage_graph", stage_graph), ("cache_semantics", cache_semantics), ("session_secret_ssh", session_secret_ssh), ("bind_mounts", bind_mounts), + ("output_streaming", output_streaming), + ("exporters", exporters), ("large_context", large_context), ]; // Diagnostic filter: run only the named scenario, e.g. @@ -813,3 +827,129 @@ RUN test ! -e /lockfile.txt docker_ignore(data_dir, &["rmi".into(), "-f".into(), BIND_TAG.into()]); result } + +/// D9: build-output streaming. A RUN step emitting ~1 MiB of ordered lines +/// must arrive un-clipped through the proxy (a wedge shows up as the +/// deadline, truncation as the clip marker or a missing tail), and `-q` +/// mode must reduce to exactly one image-ID line. +fn output_streaming(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d9-ctx"); + std::fs::create_dir_all(&ctx).context("creating d9 context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} +RUN seq 1 {STREAM_LINES} +RUN echo streamed > /marker +" + ), + ) + .context("writing d9 Dockerfile")?; + + let result = (|| { + let (output, _) = timed_build( + data_dir, + metrics, + "streaming_build_wall", + &ctx, + STREAM_TAG, + &["--no-cache"], + STREAM_DEADLINE, + )?; + if output.contains("output clipped") { + bail!("streaming: BuildKit clipped the step log (proxy delivered a truncated stream)"); + } + let tail_marker = STREAM_LINES.to_string(); + let near_tail_marker = (STREAM_LINES - 1).to_string(); + if !output.contains(&tail_marker) || !output.contains(&near_tail_marker) { + bail!("streaming: tail of the step log missing ({near_tail_marker}/{tail_marker})"); + } + + // Quiet mode: exactly one image-ID line on stdout. + let ctx_arg = ctx.display().to_string(); + let quiet = docker_output( + data_dir, + &["build", "-q", "-t", QUIET_TAG, &ctx_arg], + STREAM_DEADLINE, + ) + .context("quiet build")?; + let quiet = quiet.trim(); + if !quiet.starts_with("sha256:") || quiet.lines().count() != 1 { + bail!("streaming: -q output is not a single image ID: {quiet:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), STREAM_TAG.into()]); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), QUIET_TAG.into()]); + result +} + +/// D10: exporters — `--output type=local` and `type=tar` stream the build +/// artifact back to the client through the session, the reverse of the +/// context-upload direction. Content must arrive byte-exact. +fn exporters(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d10-ctx"); + std::fs::create_dir_all(&ctx).context("creating d10 context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} AS build +RUN mkdir /out && printf '{}' > /out/artifact.txt +FROM scratch AS export +COPY --from=build /out/ / +", + EXPORT_PAYLOAD.trim_end_matches('\n'), + ), + ) + .context("writing d10 Dockerfile")?; + // printf without a trailing newline directive: the payload constant's + // newline is added back by comparing against the trimmed form. + let expected = EXPORT_PAYLOAD.trim_end_matches('\n'); + let ctx_arg = ctx.display().to_string(); + + let local_dest = data_dir.join("d10-local-out"); + let local_arg = format!("type=local,dest={}", local_dest.display()); + let started = Instant::now(); + let local = docker_output( + data_dir, + &["build", "--progress=plain", "-o", &local_arg, &ctx_arg], + EXPORT_DEADLINE, + ); + metrics.record("exporter_local_wall", started.elapsed().as_secs_f64()); + local.context("local exporter build")?; + let exported = std::fs::read_to_string(local_dest.join("artifact.txt")) + .context("reading locally-exported artifact")?; + if exported != expected { + bail!("exporters: local export mismatch: {exported:?} != {expected:?}"); + } + + let tar_dest = data_dir.join("d10-export.tar"); + let tar_arg = format!("type=tar,dest={}", tar_dest.display()); + let started = Instant::now(); + let tar_build = docker_output( + data_dir, + &["build", "--progress=plain", "-o", &tar_arg, &ctx_arg], + EXPORT_DEADLINE, + ); + metrics.record("exporter_tar_wall", started.elapsed().as_secs_f64()); + tar_build.context("tar exporter build")?; + let extract_dir = data_dir.join("d10-tar-out"); + std::fs::create_dir_all(&extract_dir)?; + let extract = std::process::Command::new("tar") + .args(["-xf", &tar_dest.display().to_string(), "-C"]) + .arg(&extract_dir) + .output() + .context("extracting tar export")?; + if !extract.status.success() { + bail!( + "exporters: tar extract failed: {}", + String::from_utf8_lossy(&extract.stderr) + ); + } + let exported = std::fs::read_to_string(extract_dir.join("artifact.txt")) + .context("reading tar-exported artifact")?; + if exported != expected { + bail!("exporters: tar export mismatch: {exported:?} != {expected:?}"); + } + Ok(()) +} From da91d80d6b435f9a40f0e2b01d314e75586ddfca Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 15:32:44 +0800 Subject: [PATCH 09/21] test(e2e): hash-compare the D4 secret; pace D9 under BuildKit's log rate clip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-validation against a fixed-FEX guest caught both: RUN lines persist into image-config history, so inlining the secret value planted the very leak the save-sweep hunts; and BuildKit clips step logs at 200KiB/s, so a bulk emitter is clipped by design — the paced emitter keeps the no-clip assertion honest. Also un-wedge the sweep script's set -e/grep interaction. --- .../plans/docker-build-e2e-matrix.md | 2 +- tests/e2e/tests/docker_build.rs | 46 +++++++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index c6bed17bc..5a9eaad2b 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -100,7 +100,7 @@ assertions are deadlines and behavior, never absolute throughput. | D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) correct arch strings baked; (b) FEX provisioned ⇒ `x86_64` output, else **fail-closed** with the actionable error (both directions asserted) | | D7 | concurrent builds (CI shape) | 4 parallel distinct builds + 2 same-context duplicates | all complete ≤ deadline; no cross-talk (unique markers per image); zombie sweep clean | | D8 | cancellation (production reality) | kill client mid-context-upload and mid-RUN; rebuild after | daemon log free of proxy ERROR; follow-up build succeeds; no leaked build processes guest-side | -| D9 | output streaming | `--progress=plain` with a RUN emitting ~1 MiB of ordered lines (below BuildKit's per-step clip, so "no clip marker" is a fair assertion); `docker build -q` | tail lines present, no clip marker, deadline catches a wedge; quiet mode prints exactly one image ID *(implemented)* | +| D9 | output streaming | `--progress=plain` with a RUN emitting 50k ordered lines PACED under BuildKit's 200 KiB/s step-log rate clip (bulk output is clipped by design and proves nothing); `docker build -q` | tail lines present, no clip marker, deadline catches a wedge; quiet mode prints exactly one image ID *(implemented)* | | D10 | exporters | `docker build -o type=local,dest=…` and `-o type=tar` | exported artifact byte-exact vs in-image content — exercises reverse session streaming *(implemented)* | External syntax frontends (`# syntax=docker/dockerfile:1`) are exercised in diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index 762d9a8cd..7900180fa 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -84,10 +84,14 @@ const SSH_TAG: &str = "arcbox-e2e-build:ssh"; const BIND_DEADLINE: Duration = Duration::from_secs(120); const BIND_TAG: &str = "arcbox-e2e-build:bind"; -/// D9: chatty-RUN line count (~1 MiB of step output — deliberately below -/// BuildKit's default per-step log clip, so "no clip marker" is a fair -/// streaming-integrity assertion rather than a tunable). -const STREAM_LINES: usize = 150_000; +/// D9: chatty-RUN shape. BuildKit rate-limits step logs (the embedded +/// builder clips at 200 KiB/s — measured, the marker names the limit), so +/// the emitter is PACED: `STREAM_CHUNKS` bursts of `STREAM_CHUNK_LINES` +/// lines with a 1 s sleep between (~68 KiB/s). Under the limit, "no clip +/// marker + tail lines present" is a fair streaming-integrity assertion; +/// a bulk `seq` would be clipped by design and prove nothing. +const STREAM_CHUNKS: usize = 5; +const STREAM_CHUNK_LINES: usize = 10_000; const STREAM_DEADLINE: Duration = Duration::from_secs(120); const STREAM_TAG: &str = "arcbox-e2e-build:stream"; const QUIET_TAG: &str = "arcbox-e2e-build:quiet"; @@ -685,6 +689,9 @@ fn assert_absent_from_saved_image( ) .context("docker save for secret sweep")?; + // No `X && echo LEAK` shorthand under `set -e`: a non-matching grep + // returns 1, fails the AND-list, and aborts the sweep before + // SWEEP-DONE — every clean image would read as an incomplete sweep. let script = format!( r#"set -e d=$(mktemp -d) @@ -692,9 +699,9 @@ trap 'rm -rf "$d"' EXIT tar -xf "{tar_arg}" -C "$d" find "$d" -type f | while read -r f; do if [ "$(head -c 2 "$f" | od -An -tx1 | tr -d ' \n')" = "1f8b" ]; then - gunzip -c "$f" 2>/dev/null | grep -aq -- "{needle}" && echo "LEAK:$f" - else - grep -aq -- "{needle}" "$f" && echo "LEAK:$f" + if gunzip -c "$f" 2>/dev/null | grep -aq -- "{needle}"; then echo "LEAK:$f"; fi + elif grep -aq -- "{needle}" "$f"; then + echo "LEAK:$f" fi done echo SWEEP-DONE"# @@ -725,11 +732,15 @@ fn session_secret_ssh(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> let ctx = data_dir.join("d4-secret-ctx"); std::fs::create_dir_all(&ctx).context("creating d4 secret context dir")?; std::fs::write(ctx.join("token.txt"), SECRET_VALUE)?; + // The Dockerfile must carry only the HASH of the secret: RUN command + // lines persist verbatim into the image config's history, so inlining + // the value would plant exactly the leak the sweep below hunts for. + let secret_sha = format!("{:x}", sha2::Sha256::digest(SECRET_VALUE.as_bytes())); std::fs::write( ctx.join("Dockerfile"), format!( r#"FROM {image} -RUN --mount=type=secret,id=build_token test "$(cat /run/secrets/build_token)" = "{SECRET_VALUE}" && echo secret-visible > /probe +RUN --mount=type=secret,id=build_token echo "{secret_sha} /run/secrets/build_token" | sha256sum -c - && echo secret-visible > /probe RUN test ! -e /run/secrets/build_token "# ), @@ -828,18 +839,23 @@ RUN test ! -e /lockfile.txt result } -/// D9: build-output streaming. A RUN step emitting ~1 MiB of ordered lines -/// must arrive un-clipped through the proxy (a wedge shows up as the -/// deadline, truncation as the clip marker or a missing tail), and `-q` -/// mode must reduce to exactly one image-ID line. +/// D9: build-output streaming. A RUN step emitting ordered lines, paced +/// under BuildKit's step-log rate clip, must arrive complete through the +/// proxy (a wedge shows up as the deadline, truncation as the clip marker +/// or a missing tail), and `-q` mode must reduce to one image-ID line. fn output_streaming(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { let ctx = data_dir.join("d9-ctx"); std::fs::create_dir_all(&ctx).context("creating d9 context dir")?; + let total_lines = STREAM_CHUNKS * STREAM_CHUNK_LINES; + let chunk_indices = (1..=STREAM_CHUNKS) + .map(|i| i.to_string()) + .collect::>() + .join(" "); std::fs::write( ctx.join("Dockerfile"), format!( r"FROM {image} -RUN seq 1 {STREAM_LINES} +RUN for i in {chunk_indices}; do seq $(( (i-1)*{STREAM_CHUNK_LINES} + 1 )) $(( i*{STREAM_CHUNK_LINES} )); sleep 1; done RUN echo streamed > /marker " ), @@ -859,8 +875,8 @@ RUN echo streamed > /marker if output.contains("output clipped") { bail!("streaming: BuildKit clipped the step log (proxy delivered a truncated stream)"); } - let tail_marker = STREAM_LINES.to_string(); - let near_tail_marker = (STREAM_LINES - 1).to_string(); + let tail_marker = total_lines.to_string(); + let near_tail_marker = (total_lines - 1).to_string(); if !output.contains(&tail_marker) || !output.contains(&near_tail_marker) { bail!("streaming: tail of the step log missing ({near_tail_marker}/{tail_marker})"); } From 3abcee820fd67dd16f7289ef5f3be67ffd492663 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 16:29:54 +0800 Subject: [PATCH 10/21] test(e2e): fix D3/D4 assertions found by first green-guest run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BuildKit re-stamps created per build, so identical-image-ID is not a valid full-cache-hit assertion — the per-execution stamps are; and a relative --secret src resolves against the CLI CWD, not the context. --- .../plans/docker-build-e2e-matrix.md | 2 +- tests/e2e/tests/docker_build.rs | 28 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index 5a9eaad2b..9a195b624 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -94,7 +94,7 @@ assertions are deadlines and behavior, never absolute throughput. |---|---|---|---| | D1 | large context upload (immich/next.js node_modules shape) | 512 MiB incompressible + 100k small files; `.dockerignore` excluding a marker dir | build ok ≤ deadline; in-build payload sha + whole-tree sha byte-exact; ignored dir absent in-guest; `large_context_transfer` trend *(implemented)* | | D2 | deep multi-stage graph (grafana/buildkit) | 12-stage diamond: `FROM scratch AS scripts` carrier, 4 parallel-schedulable branches, `COPY --from` joins, `FROM scratch AS export` tail, `COPY --link`, heredoc RUN, `# syntax=docker/dockerfile:1` (frontend pre-loaded) | export-stage artifact byte-exact via `docker cp`; full `docker export` listing free of builder residue; build ok ≤ deadline *(implemented)* | -| D3 | cache semantics (mastodon/next.js) | `RUN --mount=type=cache,sharing=locked`; then 3 rebuilds: unchanged, one-file leaf change, base-layer change | unchanged rebuild reuses the image (identical ID) with wall ≤ max(cold/10, 3 s floor); per-execution stamps baked into the image prove leaf change re-runs only downstream steps and base change re-runs both; cache-mount token survives across builds *(implemented)* | +| D3 | cache semantics (mastodon/next.js) | `RUN --mount=type=cache,sharing=locked`; then 3 rebuilds: unchanged, one-file leaf change, base-layer change | per-execution stamps baked into the image prove the unchanged rebuild re-runs nothing (image IDs are NOT comparable — BuildKit re-stamps `created` per build), leaf change re-runs only downstream steps, base change re-runs both; warm wall ≤ max(cold/10, 3 s floor); cache-mount token survives across builds *(implemented)* | | D4 | BuildKit session (authentik secrets; ssh has no OSS exemplar but same channel) | `--secret id=x,src=file` consumed by `RUN --mount=type=secret`; `--ssh default=` against a throwaway host `ssh-agent` (one loaded key) | secret readable in RUN, gone next layer, **absent from every byte of `docker save`** (gzip-aware sweep); forwarded ssh sock present in RUN; quiet-log covers the `/session` upgrades *(implemented)* | | D5 | bind-mount lockfiles (authentik/immich) | `RUN --mount=type=bind,source=...` consuming context files without COPY layers | in-build read + derived hash lands in image; mount leaves no trace in the final fs *(implemented)* | | D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) correct arch strings baked; (b) FEX provisioned ⇒ `x86_64` output, else **fail-closed** with the actionable error (both directions asserted) | diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index 7900180fa..c0293d584 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -443,8 +443,12 @@ RUN sleep {sleep} && {stamp} > /stamp-leaf ); } - // Build 2 — unchanged: a full cache hit, so the same image ID and a - // wall bounded far below cold (with the usual CI-slack floor). + // Build 2 — unchanged: a full cache hit. Image IDs are NOT compared: + // BuildKit stamps a fresh `created` timestamp into the config on + // every build, so even a fully-cached rebuild gets a new ID. The + // per-execution stamps are the execution truth — both must match + // the cold build's — plus a wall bound far below cold (with the + // usual CI-slack floor). let (_, warm_wall) = timed_build( data_dir, metrics, @@ -454,17 +458,10 @@ RUN sleep {sleep} && {stamp} > /stamp-leaf &[], CACHE_BUILD_DEADLINE, )?; - let id = |tag: &str| { - docker_output( - data_dir, - &["image", "inspect", "-f", "{{.Id}}", tag], - Duration::from_secs(30), - ) - .map(|out| out.trim().to_owned()) - }; - let (cold_id, warm_id) = (id(cold_tag)?, id(warm_tag)?); - if warm_id != cold_id { - bail!("cache: unchanged rebuild produced a different image ({cold_id} vs {warm_id})"); + let warm_stamp_base = image_file(data_dir, warm_tag, "/stamp-base")?; + let warm_stamp_leaf = image_file(data_dir, warm_tag, "/stamp-leaf")?; + if warm_stamp_base != cold_stamp_base || warm_stamp_leaf != cold_stamp_leaf { + bail!("cache: unchanged rebuild re-executed a step (stamps changed)"); } let warm_bound = cold_wall.div_f64(10.0).max(CACHE_WARM_FLOOR); if warm_wall > warm_bound { @@ -747,6 +744,9 @@ RUN test ! -e /run/secrets/build_token ) .context("writing d4 secret Dockerfile")?; + // Absolute path: BuildKit resolves a relative `src` against the CLI's + // working directory, not the build context. + let secret_src = format!("id=build_token,src={}", ctx.join("token.txt").display()); let secret_result = (|| { timed_build( data_dir, @@ -754,7 +754,7 @@ RUN test ! -e /run/secrets/build_token "session_secret_build_wall", &ctx, SECRET_TAG, - &["--secret", "id=build_token,src=token.txt"], + &["--secret", &secret_src], SESSION_DEADLINE, )?; if image_file(data_dir, SECRET_TAG, "/probe")? != "secret-visible" { From 5fa8850ae7fc26aa54ed7f1070a442faa405ae90 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 16:40:28 +0800 Subject: [PATCH 11/21] fix(e2e): drain command pipes during run_with_timeout; carry output tail on timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old loop only read the pipes after exit, so any command with more than ~64 KiB of output (a --progress=plain build, a chatty pull) blocked on the full pipe and read as a bogus timeout — first exposed by the D9 streaming scenario, which emits ~350 KiB of plain progress. --- tests/e2e/src/docker.rs | 93 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/tests/e2e/src/docker.rs b/tests/e2e/src/docker.rs index 177e1b8f9..b9995ae66 100644 --- a/tests/e2e/src/docker.rs +++ b/tests/e2e/src/docker.rs @@ -5,6 +5,7 @@ //! via `DOCKER_HOST`, so the developer's Docker context and any host //! daemon stay untouched (see tests/e2e/README.md on isolation). +use std::io::Read as _; use std::path::Path; use std::process::{Command, Stdio}; use std::thread; @@ -141,22 +142,104 @@ pub fn ensure_image(data_dir: &Path, image: &str) -> Result<()> { } /// Runs a command, killing it once `timeout` passes. +/// +/// Both pipes are drained on background threads for the whole run: an +/// undrained pipe fills at ~64 KiB and blocks the child, which turns any +/// chatty command (a `--progress=plain` build, a large pull) into a bogus +/// timeout. On a real timeout the error carries the output tail, so a +/// killed command still leaves forensics. pub fn run_with_timeout(command: &mut Command, timeout: Duration) -> Result { let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; + let drain = |pipe: Option>| { + thread::spawn(move || { + let mut buf = Vec::new(); + if let Some(mut pipe) = pipe { + let _ = pipe.read_to_end(&mut buf); + } + buf + }) + }; + let stdout_thread = drain( + child + .stdout + .take() + .map(|p| Box::new(p) as Box), + ); + let stderr_thread = drain( + child + .stderr + .take() + .map(|p| Box::new(p) as Box), + ); + let start = Instant::now(); while start.elapsed() < timeout { - if child.try_wait()?.is_some() { - return child - .wait_with_output() - .context("collecting command output"); + if let Some(status) = child.try_wait()? { + let stdout = stdout_thread.join().unwrap_or_default(); + let stderr = stderr_thread.join().unwrap_or_default(); + return Ok(std::process::Output { + status, + stdout, + stderr, + }); } thread::sleep(Duration::from_millis(100)); } let _ = child.kill(); let _ = child.wait(); - Err(anyhow!("command timed out after {}s", timeout.as_secs())) + // Killing the child EOFs the pipes, so the drain threads finish. + let stdout = stdout_thread.join().unwrap_or_default(); + let stderr = stderr_thread.join().unwrap_or_default(); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + let mut tail_start = combined.len().saturating_sub(2000); + while !combined.is_char_boundary(tail_start) { + tail_start += 1; + } + Err(anyhow!( + "command timed out after {}s; output tail:\n{}", + timeout.as_secs(), + &combined[tail_start..] + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A command whose output exceeds the OS pipe buffer must complete: an + /// undrained pipe blocks the child at ~64 KiB and the old implementation + /// turned every chatty command into a bogus timeout (caught by the + /// docker_build D9 streaming scenario). + #[test] + fn chatty_command_is_drained_not_deadlocked() { + let output = run_with_timeout( + Command::new("sh").args([ + "-c", + "dd if=/dev/zero bs=1024 count=256 2>/dev/null | base64", + ]), + Duration::from_secs(20), + ) + .expect("chatty command must not time out"); + assert!(output.status.success()); + assert!(output.stdout.len() > 64 * 1024); + } + + /// A genuine timeout must surface the output tail for forensics. + #[test] + fn timeout_error_carries_output_tail() { + let error = run_with_timeout( + Command::new("sh").args(["-c", "echo tail-marker; sleep 30"]), + Duration::from_secs(1), + ) + .expect_err("command must time out"); + assert!(error.to_string().contains("tail-marker")); + } } From c80c639303a1867a801eece6e46b33e1f29a3855 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 16:47:03 +0800 Subject: [PATCH 12/21] docs(e2e): docker_build suite green on boot 0.6.10; record BuildKit realities --- internal-docs/plans/docker-build-e2e-matrix.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index 9a195b624..43d84d1af 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -131,12 +131,17 @@ manual `xtask` run of the same fixtures against another engine's ## Status -D1–D3 are implemented (`tests/e2e/tests/docker_build.rs`) and currently -**red for a real product reason**: every buildx-driven build hangs because -guest FEX spins forever on BuildKit's amd64 arch-probe ELF, permanently -wedging the BuildKit Control API (ABX-494 — first caught by this suite's -first run; `network_workload` W14 reproduces it too). Do not weaken the -suite; it goes green when ABX-494 is fixed. +D1–D5 and D9–D10 are implemented (`tests/e2e/tests/docker_build.rs`) and +**fully green** against boot bundle ≥ 0.6.10. The suite's first-ever run +caught ABX-494 (guest FEX spun forever on BuildKit's amd64 arch-probe, +wedging the whole BuildKit Control API; fixed by boot-assets#44), and its +first green-guest run caught a latent harness bug (`run_with_timeout` +didn't drain pipes, turning any >64 KiB-output command into a bogus +timeout). On bundles < 0.6.10 the suite is red by design — do not weaken +it. Two BuildKit realities encoded in the assertions: image IDs are not +comparable across builds (`created` is re-stamped even on full cache +hits), and step logs are rate-clipped at 200 KiB/s (chatty RUNs must be +paced). ## Phasing From 75168f6485d3853c64ba4f16a7dbd291a0e4c2dc Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 17:40:33 +0800 Subject: [PATCH 13/21] test(e2e): D6 cross-platform, D7 concurrent, D8 cancellation scenarios --- tests/e2e/tests/docker_build.rs | 349 +++++++++++++++++++++++++++++++- 1 file changed, 345 insertions(+), 4 deletions(-) diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index c0293d584..70c3a350f 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -1,4 +1,4 @@ -//! docker build e2e — Phases 1–2 (D1–D5, D9–D10) of +//! docker build e2e — Phases 1–3 (D1–D10) of //! internal-docs/plans/docker-build-e2e-matrix.md. //! //! Where `network_workload` W14 drives *one* build to prove the datapath, @@ -29,6 +29,17 @@ //! `/session` upgrade proxy, previously covered by unit tests only. //! - **D5 bind-mount lockfiles** (authentik / immich pnpm shape): //! `RUN --mount=type=bind` consumes a context file without a COPY layer. +//! - **D6 cross-platform** (vaultwarden / moby shape): `BUILDPLATFORM` / +//! `TARGETARCH` expansion on a native build, and a `--platform +//! linux/amd64` build whose RUN executes through the binfmt translator +//! (asserted in-build via `uname -m`). +//! - **D7 concurrent builds** (CI shape): four distinct builds plus two +//! racing on one shared context, all in flight at once; unique markers +//! prove no cross-talk. +//! - **D8 cancellation** (production reality): the client is SIGKILLed +//! mid-RUN and mid-context-upload; the guest must reap the build +//! container, the daemon must stay quiet, and the next build must +//! succeed promptly. //! - **D9 output streaming**: a chatty RUN streamed un-clipped through the //! proxy, and `-q` mode reduced to the bare image ID. //! - **D10 exporters**: `--output type=local` and `type=tar` — build @@ -40,8 +51,8 @@ use std::path::Path; use std::time::{Duration, Instant}; -use anyhow::{Context, Result, bail}; -use arcbox_e2e::docker::{docker_ignore, docker_output, ensure_image}; +use anyhow::{Context, Result, anyhow, bail}; +use arcbox_e2e::docker::{docker_host, docker_ignore, docker_output, ensure_image}; use arcbox_e2e::metrics::RunMetrics; use arcbox_e2e::net_fixtures::daemon_log_cursor; use arcbox_e2e::scenario::run_vz_scenario_with_log; @@ -100,6 +111,25 @@ const QUIET_TAG: &str = "arcbox-e2e-build:quiet"; const EXPORT_DEADLINE: Duration = Duration::from_secs(120); const EXPORT_PAYLOAD: &str = "exporter-proof-payload\n"; +/// D6: cross-platform args + the amd64 binfmt translator. FEX ships in +/// every bundle ≥ 0.6.6, so only the provisioned direction is asserted; +/// the fail-closed unprovisioned direction awaits an e2e knob (see the +/// plan's phasing note). +const XPLAT_DEADLINE: Duration = Duration::from_secs(180); +const XPLAT_ARGS_TAG: &str = "arcbox-e2e-build:xplat-args"; +const XPLAT_AMD64_TAG: &str = "arcbox-e2e-build:xplat-amd64"; + +/// D7: concurrent builds — distinct contexts plus two racing on one shared +/// context. +const CONCURRENT_DISTINCT: usize = 4; +const CONCURRENT_SHARED: usize = 2; +const CONCURRENT_DEADLINE: Duration = Duration::from_secs(180); + +/// D8: cancellation probes — the mid-upload kill needs a payload big +/// enough that the transfer is still in flight when the client dies. +const CANCEL_UPLOAD_PAYLOAD: usize = 256 * 1024 * 1024; +const CANCEL_DEADLINE: Duration = Duration::from_secs(120); + #[test] #[ignore = "boots a VZ System VM through a real daemon; run on the e2e runner"] fn docker_build_suite() -> Result<()> { @@ -114,13 +144,16 @@ fn docker_build_suite() -> Result<()> { // every build below. let log = daemon_log_cursor(data_dir); - let scenarios: [(&str, ScenarioFn); 7] = [ + let scenarios: [(&str, ScenarioFn); 10] = [ ("stage_graph", stage_graph), ("cache_semantics", cache_semantics), ("session_secret_ssh", session_secret_ssh), ("bind_mounts", bind_mounts), ("output_streaming", output_streaming), ("exporters", exporters), + ("cross_platform", cross_platform), + ("concurrent_builds", concurrent_builds), + ("cancellation", cancellation), ("large_context", large_context), ]; // Diagnostic filter: run only the named scenario, e.g. @@ -839,6 +872,314 @@ RUN test ! -e /lockfile.txt result } +/// D6: cross-platform builds. (a) A native build expands +/// `--platform=$BUILDPLATFORM` and `TARGETARCH` correctly. (b) A +/// `--platform linux/amd64` build executes its RUN step through the binfmt +/// translator — asserted in-build via `uname -m`, plus the image's +/// recorded architecture. This is the surface ABX-494 wedged: the arch +/// probe behind it now runs on every ListWorkers. +fn cross_platform(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d6-args-ctx"); + std::fs::create_dir_all(&ctx).context("creating d6 args context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM --platform=$BUILDPLATFORM {image} AS build +ARG BUILDPLATFORM +ARG TARGETARCH +RUN echo "$BUILDPLATFORM:$TARGETARCH" > /args.txt +FROM {image} +COPY --from=build /args.txt /args.txt +"# + ), + ) + .context("writing d6 args Dockerfile")?; + + let args_result = (|| { + timed_build( + data_dir, + metrics, + "xplat_args_build_wall", + &ctx, + XPLAT_ARGS_TAG, + &[], + XPLAT_DEADLINE, + )?; + let args = image_file(data_dir, XPLAT_ARGS_TAG, "/args.txt")?; + if args != "linux/arm64:arm64" { + bail!("cross_platform: platform args expanded to {args:?}, expected linux/arm64:arm64"); + } + Ok(()) + })(); + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), XPLAT_ARGS_TAG.into()], + ); + args_result?; + + // The amd64 base variant is not in the arm64 tar cache; pull it with + // the same retry discipline as `ensure_image`. + let mut last_err = None; + for attempt in 1..=3 { + match docker_output( + data_dir, + &["pull", "--platform", "linux/amd64", image], + Duration::from_secs(90), + ) { + Ok(_) => { + last_err = None; + break; + } + Err(e) => { + tracing::warn!(attempt, "amd64 variant pull failed: {e:#}"); + last_err = Some(e); + } + } + } + if let Some(e) = last_err { + return Err(e).context("pulling amd64 base variant (3 attempts)"); + } + + let ctx = data_dir.join("d6-amd64-ctx"); + std::fs::create_dir_all(&ctx).context("creating d6 amd64 context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +RUN uname -m > /arch && test "$(cat /arch)" = "x86_64" +"# + ), + ) + .context("writing d6 amd64 Dockerfile")?; + + let amd64_result = (|| { + timed_build( + data_dir, + metrics, + "xplat_amd64_build_wall", + &ctx, + XPLAT_AMD64_TAG, + &["--platform", "linux/amd64"], + XPLAT_DEADLINE, + )?; + let arch = docker_output( + data_dir, + &[ + "image", + "inspect", + "-f", + "{{.Architecture}}", + XPLAT_AMD64_TAG, + ], + Duration::from_secs(30), + )?; + if arch.trim() != "amd64" { + bail!( + "cross_platform: built image records architecture {:?}, expected amd64", + arch.trim() + ); + } + Ok(()) + })(); + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), XPLAT_AMD64_TAG.into()], + ); + amd64_result +} + +/// D7: concurrent builds — the CI shape. Four builds with distinct +/// contexts plus two racing on one shared context all run at once; each +/// image must carry exactly its own marker (no cross-talk between build +/// sessions sharing the daemon and builder). +fn concurrent_builds(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let mut jobs: Vec<(String, std::path::PathBuf, String)> = Vec::new(); + for i in 0..CONCURRENT_DISTINCT { + let ctx = data_dir.join(format!("d7-ctx-{i}")); + std::fs::create_dir_all(&ctx)?; + let marker = format!("marker-{i}"); + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nRUN sleep 2 && echo {marker} > /marker\n"), + )?; + jobs.push((format!("arcbox-e2e-build:conc-{i}"), ctx, marker)); + } + let shared = data_dir.join("d7-ctx-shared"); + std::fs::create_dir_all(&shared)?; + std::fs::write( + shared.join("Dockerfile"), + format!("FROM {image}\nRUN sleep 2 && echo marker-shared > /marker\n"), + )?; + for j in 0..CONCURRENT_SHARED { + jobs.push(( + format!("arcbox-e2e-build:conc-shared-{j}"), + shared.clone(), + "marker-shared".to_owned(), + )); + } + + let started = Instant::now(); + let results: Vec<(String, Result)> = std::thread::scope(|scope| { + #[expect( + clippy::needless_collect, + reason = "collect forces every spawn before the first join; a lazy chain would run the builds serially" + )] + let handles: Vec<_> = jobs + .iter() + .map(|(tag, ctx, _)| { + scope.spawn(move || { + let ctx_arg = ctx.display().to_string(); + docker_output( + data_dir, + &["build", "-t", tag, &ctx_arg], + CONCURRENT_DEADLINE, + ) + }) + }) + .collect(); + handles + .into_iter() + .zip(&jobs) + .map(|(handle, (tag, _, _))| { + ( + tag.clone(), + handle + .join() + .unwrap_or_else(|_| Err(anyhow!("build thread panicked"))), + ) + }) + .collect() + }); + let elapsed = started.elapsed(); + metrics.record("concurrent_builds_wall", elapsed.as_secs_f64()); + + let result = (|| { + for (tag, result) in &results { + if let Err(error) = result { + bail!("concurrent: {tag} failed: {error:#}"); + } + } + if elapsed >= CONCURRENT_DEADLINE { + bail!("concurrent: builds took {elapsed:?} (>= {CONCURRENT_DEADLINE:?})"); + } + for (tag, _, marker) in &jobs { + let got = image_file(data_dir, tag, "/marker")?; + if got != *marker { + bail!("concurrent: {tag} carries marker {got:?}, expected {marker:?} — cross-talk"); + } + } + Ok(()) + })(); + for (tag, _, _) in &jobs { + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.clone()]); + } + result +} + +/// Spawns a `docker build` that the caller intends to kill — output goes +/// to /dev/null since nothing will ever collect it. +fn spawn_build(data_dir: &Path, ctx: &Path, tag: &str) -> Result { + std::process::Command::new("docker") + .env("DOCKER_HOST", docker_host(data_dir)) + .args(["build", "-t", tag]) + .arg(ctx) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .context("spawning cancellable docker build") +} + +/// D8: cancellation — the client is SIGKILLed mid-RUN and mid-context- +/// upload. The guest must reap the cancelled RUN's process, and the next +/// build must succeed promptly. The suite-level quiet-log check asserts +/// the daemon rode both kills without proxy ERRORs. +fn cancellation(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + // (1) Kill mid-RUN: the sleep marker is uniquely greppable guest-side. + let ctx = data_dir.join("d8-run-ctx"); + std::fs::create_dir_all(&ctx).context("creating d8 run context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nRUN sleep 6543\n"), + )?; + let mut child = spawn_build(data_dir, &ctx, "arcbox-e2e-build:cancel-run")?; + std::thread::sleep(Duration::from_secs(8)); + let _ = child.kill(); + let _ = child.wait(); + + let mut reaped = false; + for _ in 0..15 { + let count = docker_output( + data_dir, + &[ + "run", + "--rm", + "--pid=host", + image, + "sh", + "-c", + "ps | grep 'sleep [6]543' | wc -l", + ], + Duration::from_secs(60), + ) + .context("guest process sweep")?; + if count.trim() == "0" { + reaped = true; + break; + } + std::thread::sleep(Duration::from_secs(2)); + } + if !reaped { + bail!("cancellation: the cancelled RUN's process is still alive in the guest"); + } + + // (2) Kill mid-context-upload: 256 MiB is several seconds of transfer. + let ctx = data_dir.join("d8-upload-ctx"); + std::fs::create_dir_all(&ctx).context("creating d8 upload context dir")?; + let mut payload = vec![0u8; CANCEL_UPLOAD_PAYLOAD]; + fill_incompressible(&mut payload); + std::fs::write(ctx.join("payload.bin"), &payload).context("writing d8 payload")?; + drop(payload); + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nCOPY payload.bin /payload.bin\n"), + )?; + let mut child = spawn_build(data_dir, &ctx, "arcbox-e2e-build:cancel-upload")?; + std::thread::sleep(Duration::from_millis(1500)); + let _ = child.kill(); + let _ = child.wait(); + + // (3) The daemon must build normally right after both kills. + let ctx = data_dir.join("d8-followup-ctx"); + std::fs::create_dir_all(&ctx).context("creating d8 followup context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nRUN echo recovered > /marker\n"), + )?; + let result = (|| { + timed_build( + data_dir, + metrics, + "cancel_followup_wall", + &ctx, + "arcbox-e2e-build:cancel-followup", + &[], + CANCEL_DEADLINE, + )?; + if image_file(data_dir, "arcbox-e2e-build:cancel-followup", "/marker")? != "recovered" { + bail!("cancellation: follow-up build produced a wrong image"); + } + Ok(()) + })(); + for tag in ["cancel-run", "cancel-upload", "cancel-followup"] { + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), format!("arcbox-e2e-build:{tag}")], + ); + } + result +} + /// D9: build-output streaming. A RUN step emitting ordered lines, paced /// under BuildKit's step-log rate clip, must arrive complete through the /// proxy (a wedge shows up as the deadline, truncation as the clip marker From 0cd5ad1de1a78c7342521b0b44112e115b58a96a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 17:42:55 +0800 Subject: [PATCH 14/21] =?UTF-8?q?test(e2e):=20Tier=20X=20external=20suite?= =?UTF-8?q?=20=E2=80=94=20pinned=20postgres/next.js/caddy=20real=20builds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plans/docker-build-e2e-matrix.md | 20 +- tests/e2e/tests/docker_build_external.rs | 239 ++++++++++++++++++ xtask/AGENTS.md | 1 + xtask/src/commands/e2e.rs | 4 +- 4 files changed, 254 insertions(+), 10 deletions(-) create mode 100644 tests/e2e/tests/docker_build_external.rs diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index 43d84d1af..b9dc34f89 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -97,9 +97,9 @@ assertions are deadlines and behavior, never absolute throughput. | D3 | cache semantics (mastodon/next.js) | `RUN --mount=type=cache,sharing=locked`; then 3 rebuilds: unchanged, one-file leaf change, base-layer change | per-execution stamps baked into the image prove the unchanged rebuild re-runs nothing (image IDs are NOT comparable — BuildKit re-stamps `created` per build), leaf change re-runs only downstream steps, base change re-runs both; warm wall ≤ max(cold/10, 3 s floor); cache-mount token survives across builds *(implemented)* | | D4 | BuildKit session (authentik secrets; ssh has no OSS exemplar but same channel) | `--secret id=x,src=file` consumed by `RUN --mount=type=secret`; `--ssh default=` against a throwaway host `ssh-agent` (one loaded key) | secret readable in RUN, gone next layer, **absent from every byte of `docker save`** (gzip-aware sweep); forwarded ssh sock present in RUN; quiet-log covers the `/session` upgrades *(implemented)* | | D5 | bind-mount lockfiles (authentik/immich) | `RUN --mount=type=bind,source=...` consuming context files without COPY layers | in-build read + derived hash lands in image; mount leaves no trace in the final fs *(implemented)* | -| D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) correct arch strings baked; (b) FEX provisioned ⇒ `x86_64` output, else **fail-closed** with the actionable error (both directions asserted) | -| D7 | concurrent builds (CI shape) | 4 parallel distinct builds + 2 same-context duplicates | all complete ≤ deadline; no cross-talk (unique markers per image); zombie sweep clean | -| D8 | cancellation (production reality) | kill client mid-context-upload and mid-RUN; rebuild after | daemon log free of proxy ERROR; follow-up build succeeds; no leaked build processes guest-side | +| D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) `linux/arm64:arm64` baked; (b) `x86_64` asserted in-build + image records `amd64` — the surface ABX-494 wedged. Fail-closed unprovisioned direction still awaits an e2e knob (FEX ships in every bundle ≥ 0.6.6) *(implemented, positive paths)* | +| D7 | concurrent builds (CI shape) | 4 parallel distinct builds + 2 same-context duplicates, all in flight at once | all complete ≤ deadline; no cross-talk (unique markers per image); suite quiet-log covers the daemon side *(implemented)* | +| D8 | cancellation (production reality) | SIGKILL the client mid-RUN and mid-context-upload (256 MiB); rebuild after | guest reaps the cancelled RUN's process (polled via `--pid=host`); follow-up build succeeds promptly; suite quiet-log asserts the daemon rode both kills without proxy ERRORs *(implemented)* | | D9 | output streaming | `--progress=plain` with a RUN emitting 50k ordered lines PACED under BuildKit's 200 KiB/s step-log rate clip (bulk output is clipped by design and proves nothing); `docker build -q` | tail lines present, no clip marker, deadline catches a wedge; quiet mode prints exactly one image ID *(implemented)* | | D10 | exporters | `docker build -o type=local,dest=…` and `-o type=tar` | exported artifact byte-exact vs in-image content — exercises reverse session streaming *(implemented)* | @@ -114,9 +114,9 @@ upstream commits: | ID | Build | Why this one | |---|---|---| -| X1 | docker-library/postgres `17/bookworm`, redis `7.4/debian`, nginx `mainline/debian` | tiny contexts, real apt + wget + **gpg keyserver** traffic — the classic official-image shape, near-zero context cost | -| X2 | vercel/next.js `examples/with-docker` | real `npm ci` through the datapath; the single most common user Dockerfile shape | -| X3 | caddyserver/caddy-docker builder | Go toolchain + module downloads inside the build | +| X1 | docker-library/postgres `17/bookworm` | tiny context, real apt + wget + **gpg keyserver** traffic — the classic official-image shape; smoke: `postgres --version` *(implemented; redis/nginx are the same shape and stay out to keep the run bounded)* | +| X2 | vercel/next.js `examples/with-docker` | real `pnpm install --frozen-lockfile` + `next build` through the datapath; the single most common user Dockerfile shape *(implemented)* | +| X3 | caddyserver/caddy-docker `2.11/builder` | apk + checksum-verified xcaddy release fetch (the builder image installs the toolchain; the full compile happens when it is used) *(implemented)* | | X4 | mastodon or immich at a pinned tag | hour-scale polyglot heavyweight; manual smoke when touching the proxy or datapath, never CI | ## Bench methodology @@ -131,8 +131,12 @@ manual `xtask` run of the same fixtures against another engine's ## Status -D1–D5 and D9–D10 are implemented (`tests/e2e/tests/docker_build.rs`) and -**fully green** against boot bundle ≥ 0.6.10. The suite's first-ever run +The full Tier-D matrix D1–D10 is implemented +(`tests/e2e/tests/docker_build.rs`), plus Tier X X1–X3 +(`tests/e2e/tests/docker_build_external.rs`, gated on +`ARCBOX_E2E_EXTERNAL=1`, pinned upstream commits), and Tier D is +**fully green** against boot bundle ≥ 0.6.10. The one deliberate gap: +D6's fail-closed unprovisioned-FEX direction awaits an e2e knob. The suite's first-ever run caught ABX-494 (guest FEX spun forever on BuildKit's amd64 arch-probe, wedging the whole BuildKit Control API; fixed by boot-assets#44), and its first green-guest run caught a latent harness bug (`run_with_timeout` diff --git a/tests/e2e/tests/docker_build_external.rs b/tests/e2e/tests/docker_build_external.rs new file mode 100644 index 000000000..c3cec54c3 --- /dev/null +++ b/tests/e2e/tests/docker_build_external.rs @@ -0,0 +1,239 @@ +//! docker build e2e — Tier X of +//! internal-docs/plans/docker-build-e2e-matrix.md: real-project builds at +//! pinned upstream commits, with real registry/package-manager/keyserver +//! traffic through the datapath. +//! +//! Env-gated behind `ARCBOX_E2E_EXTERNAL=1` (the network-workload plan's +//! external-phase convention): results depend on upstream availability by +//! nature, so this suite is run manually when touching the proxy or +//! datapath — it gates nothing. +//! +//! - **X1 postgres** (`docker-library/postgres` `17/bookworm`): the classic +//! official-image shape — tiny context, real `apt` + `wget` + **gpg +//! keyserver** traffic. +//! - **X2 next.js** (`vercel/next.js` `examples/with-docker`): the single +//! most common user Dockerfile shape — multi-stage standalone build with +//! a real `pnpm install --frozen-lockfile`. +//! - **X3 caddy builder** (`caddyserver/caddy-docker` `2.11/builder`): +//! apk + checksum-verified release-binary fetch (xcaddy). +//! +//! X4 (mastodon/immich-scale heavyweights) stays manual-only — see the +//! plan. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use arcbox_e2e::docker::{docker_ignore, docker_output, run_with_timeout}; +use arcbox_e2e::metrics::RunMetrics; +use arcbox_e2e::scenario::run_vz_scenario_with_log; + +const READY_TIMEOUT: Duration = Duration::from_secs(180); +/// Real builds pull base images and run package managers; give each one a +/// generous ceiling — the assertion is "completes", not "fast". +const BUILD_DEADLINE: Duration = Duration::from_secs(900); +const FETCH_TIMEOUT: Duration = Duration::from_secs(180); + +/// Pinned upstream commits (2026-07-22). Bump deliberately: a pin move can +/// change what the build exercises. +const POSTGRES_PIN: (&str, &str, &str) = ( + "https://github.com/docker-library/postgres.git", + "62a714f93cc32220de46fd12235c9d509e3b1ad6", + "17/bookworm", +); +const NEXTJS_PIN: (&str, &str, &str) = ( + "https://github.com/vercel/next.js.git", + "1d3bf10cde7b19093222305c4ded5f5948928419", + "examples/with-docker", +); +const CADDY_PIN: (&str, &str, &str) = ( + "https://github.com/caddyserver/caddy-docker.git", + "70350320b11d6cb04586bb869b798273180aa6d1", + "2.11/builder", +); + +#[test] +#[ignore = "real upstream builds over the internet; set ARCBOX_E2E_EXTERNAL=1 and run manually"] +fn docker_build_external_suite() -> Result<()> { + if !arcbox_e2e::env_flag("ARCBOX_E2E_EXTERNAL") { + eprintln!("skipping: ARCBOX_E2E_EXTERNAL is not set"); + return Ok(()); + } + run_vz_scenario_with_log( + "docker_build_external", + "info", + |daemon, data_dir, metrics| { + metrics.time("daemon_ready", || daemon.wait_ready_blocking(READY_TIMEOUT))?; + + let scenarios: [(&str, ScenarioFn); 3] = [ + ("x3_caddy_builder", x3_caddy_builder), + ("x1_postgres", x1_postgres), + ("x2_nextjs", x2_nextjs), + ]; + let only = std::env::var("ARCBOX_E2E_BUILD_ONLY").ok(); + let mut failures = Vec::new(); + for (name, scenario) in scenarios { + if let Some(ref only) = only + && only != name + { + continue; + } + tracing::info!(scenario = name, "starting"); + match scenario(data_dir, metrics) { + Ok(()) => tracing::info!(scenario = name, "passed"), + Err(error) => { + tracing::warn!(scenario = name, "failed: {error:#}"); + failures.push(format!("{name}: {error:#}")); + } + } + } + if failures.is_empty() { + Ok(()) + } else { + bail!( + "{} of {} external builds failed:\n{}", + failures.len(), + scenarios.len(), + failures.join("\n---\n") + ) + } + }, + ) +} + +type ScenarioFn = fn(&Path, &mut RunMetrics) -> Result<()>; + +/// Sparse-fetches `subdir` of `repo` at the pinned `sha` (GitHub allows +/// arbitrary-SHA fetches) and returns the materialized subdir path. +fn fetch_pinned_subdir( + data_dir: &Path, + label: &str, + (repo, sha, subdir): (&str, &str, &str), +) -> Result { + let dest = data_dir.join(format!("src-{label}")); + std::fs::create_dir_all(&dest)?; + let git = |args: &[&str]| -> Result<()> { + let output = run_with_timeout( + Command::new("git").arg("-C").arg(&dest).args(args), + FETCH_TIMEOUT, + ) + .with_context(|| format!("git {args:?}"))?; + if !output.status.success() { + bail!( + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) + }; + git(&["init", "-q"])?; + git(&["remote", "add", "origin", repo])?; + git(&["sparse-checkout", "init", "--cone"])?; + git(&["sparse-checkout", "set", subdir])?; + git(&[ + "fetch", + "-q", + "--depth", + "1", + "--filter=blob:none", + "origin", + sha, + ])?; + git(&[ + "-c", + "advice.detachedHead=false", + "checkout", + "-q", + "FETCH_HEAD", + ])?; + let ctx = dest.join(subdir); + if !ctx.join("Dockerfile").is_file() { + bail!("{label}: no Dockerfile at {}", ctx.display()); + } + Ok(ctx) +} + +fn build_and_time( + data_dir: &Path, + metrics: &mut RunMetrics, + key: &str, + ctx: &Path, + tag: &str, +) -> Result<()> { + let ctx_arg = ctx.display().to_string(); + let started = std::time::Instant::now(); + let result = docker_output( + data_dir, + &["build", "--progress=plain", "-t", tag, &ctx_arg], + BUILD_DEADLINE, + ); + metrics.record(key, started.elapsed().as_secs_f64()); + result.with_context(|| format!("{key}: docker build"))?; + Ok(()) +} + +/// X1: the official postgres image — apt, wget'd gosu, gpg keyserver +/// verification, locale generation. The smoke check runs the built binary. +fn x1_postgres(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { + let ctx = fetch_pinned_subdir(data_dir, "postgres", POSTGRES_PIN)?; + let tag = "arcbox-e2e-external:postgres"; + let result = (|| { + build_and_time(data_dir, metrics, "x1_postgres_build_wall", &ctx, tag)?; + let version = docker_output( + data_dir, + &["run", "--rm", tag, "postgres", "--version"], + Duration::from_secs(60), + )?; + if !version.contains("PostgreSQL") { + bail!("postgres smoke: unexpected version output {version:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.into()]); + result +} + +/// X2: the canonical Next.js standalone build — multi-stage, corepack + +/// `pnpm install --frozen-lockfile`, `next build`. Smoke: the runtime +/// stage carries a working node. +fn x2_nextjs(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { + let ctx = fetch_pinned_subdir(data_dir, "nextjs", NEXTJS_PIN)?; + let tag = "arcbox-e2e-external:nextjs"; + let result = (|| { + build_and_time(data_dir, metrics, "x2_nextjs_build_wall", &ctx, tag)?; + let version = docker_output( + data_dir, + &["run", "--rm", "--entrypoint", "node", tag, "--version"], + Duration::from_secs(60), + )?; + if !version.trim().starts_with('v') { + bail!("nextjs smoke: unexpected node version {version:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.into()]); + result +} + +/// X3: the caddy builder image — apk + checksum-verified xcaddy release +/// fetch. Cheapest of the three; runs first to fail fast on a broken +/// external environment. +fn x3_caddy_builder(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { + let ctx = fetch_pinned_subdir(data_dir, "caddy", CADDY_PIN)?; + let tag = "arcbox-e2e-external:caddy-builder"; + let result = (|| { + build_and_time(data_dir, metrics, "x3_caddy_build_wall", &ctx, tag)?; + let version = docker_output( + data_dir, + &["run", "--rm", "--entrypoint", "xcaddy", tag, "version"], + Duration::from_secs(60), + )?; + if !version.contains("v0.4") { + bail!("caddy smoke: unexpected xcaddy version {version:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.into()]); + result +} diff --git a/xtask/AGENTS.md b/xtask/AGENTS.md index a189f8597..7d5c7d89e 100644 --- a/xtask/AGENTS.md +++ b/xtask/AGENTS.md @@ -36,6 +36,7 @@ Current recipe ↔ self-build mapping (keep both columns identical): | `sandbox` | release cli+daemon + musl `arcbox-agent`/`arcbox-vm` bins | `sandbox.rs::build_binaries` (same) | | `egress_throughput` | `cargo build --release -p arcbox-daemon` | `scenario.rs::run_vz_scenario_with_log` (same) | | `docker_build` | same as above | `scenario.rs::run_vz_scenario_with_log` (same) | +| `docker_build_external` | same as above | `scenario.rs::run_vz_scenario_with_log` (same) | | `virtio_debug` | none (self-builds) | `cargo build --release -p arcbox-daemon` (RELEASE only) | | `daemon_failure` | none (self-builds) | `cargo build -p arcbox-daemon` (**DEBUG**) | | anything else | none — fallback notice, no `SKIP_BUILD` | its own build | diff --git a/xtask/src/commands/e2e.rs b/xtask/src/commands/e2e.rs index cbf631fc3..7d983687c 100644 --- a/xtask/src/commands/e2e.rs +++ b/xtask/src/commands/e2e.rs @@ -154,8 +154,8 @@ fn prebuild(root: &std::path::Path, test: &str) -> Result { Ok(true) } // Must match tests/e2e/src/scenario.rs's self-build (the shared - // daemon-boot scaffolding both of these targets run through). - "egress_throughput" | "docker_build" => { + // daemon-boot scaffolding these targets run through). + "egress_throughput" | "docker_build" | "docker_build_external" => { xshell::cmd!(shell, "cargo build --release -p arcbox-daemon").run()?; Ok(true) } From 1ace168a2bf53579a38e2b0ce3aa55f96e1b8460 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 17:57:36 +0800 Subject: [PATCH 15/21] test(e2e): allow dependency build scripts in the pinned next.js build Upstream pins no packageManager, so corepack pulls latest pnpm, whose supply-chain default refuses sharp's postinstall (ERR_PNPM_IGNORED_BUILDS) on any engine. Apply pnpm's documented escape hatch; bail loudly if a pin bump ever makes the accommodation stale. --- tests/e2e/tests/docker_build_external.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/e2e/tests/docker_build_external.rs b/tests/e2e/tests/docker_build_external.rs index c3cec54c3..a39f8a011 100644 --- a/tests/e2e/tests/docker_build_external.rs +++ b/tests/e2e/tests/docker_build_external.rs @@ -199,6 +199,21 @@ fn x1_postgres(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { /// stage carries a working node. fn x2_nextjs(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { let ctx = fetch_pinned_subdir(data_dir, "nextjs", NEXTJS_PIN)?; + // Upstream ships no `packageManager` pin, so corepack pulls the latest + // pnpm at build time, whose supply-chain default refuses sharp's build + // scripts (ERR_PNPM_IGNORED_BUILDS) — the verbatim build fails on ANY + // engine. Apply pnpm's documented escape hatch and keep everything + // else verbatim; bail loudly if a pin bump makes the patch stale. + let dockerfile = ctx.join("Dockerfile"); + let content = std::fs::read_to_string(&dockerfile)?; + let patched = content.replace( + "pnpm install --frozen-lockfile", + "pnpm install --frozen-lockfile --dangerously-allow-all-builds", + ); + if patched == content { + bail!("nextjs: pnpm install line not found — re-check the accommodation against the pin"); + } + std::fs::write(&dockerfile, patched)?; let tag = "arcbox-e2e-external:nextjs"; let result = (|| { build_and_time(data_dir, metrics, "x2_nextjs_build_wall", &ctx, tag)?; From bc5055413f67f2b872463dedfa062f2bcdfc7089 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 18:15:25 +0800 Subject: [PATCH 16/21] docs(e2e): record 2026-07-22 docker build bench baseline --- .../plans/docker-build-e2e-matrix.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index b9dc34f89..5830fd98f 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -119,6 +119,31 @@ upstream commits: | X3 | caddyserver/caddy-docker `2.11/builder` | apk + checksum-verified xcaddy release fetch (the builder image installs the toolchain; the full compile happens when it is used) *(implemented)* | | X4 | mastodon or immich at a pinned tag | hour-scale polyglot heavyweight; manual smoke when touching the proxy or datapath, never CI | +## Baseline (2026-07-22, M-series host, VZ guest, boot 0.6.10, dockerd 29.6.1 / BuildKit v0.31.1) + +Measured with an engine-agnostic harness (plain `docker build` timed +host-side via `DOCKER_HOST`; cold-ness via a per-build nonce file in the +context so no engine-global cache prune is needed; base images pre-pulled +outside the timed region; real-project contexts fetched at the Tier X +pins, with X2's pnpm accommodation). Single-shot numbers — trend anchors, +not gates. + +| shape | wall | +|---|---| +| simple 3-RUN alpine build, cold (includes first-build builder bootstrap + arch probes) | 8.1 s | +| same build, full cache hit | 1.8 s | +| 512 MiB + 100k-file context, cold | 26.4 s | +| 12-stage diamond, cold | 12.7 s | +| `--platform linux/amd64` (FEX): probe + 300k-iteration shell loop | 6.4 s | +| postgres `17/bookworm` (real apt + gpg + wget) | 75.6 s | +| next.js `with-docker` (pnpm frozen install + `next build`) | 37.8 s | +| caddy `2.11/builder` (apk + xcaddy fetch) | 25.2 s | + +Cross-engine comparison was not possible on the baseline host — OrbStack +is uninstalled there (dangling `~/.orbstack`); the harness takes any +engine via `DOCKER_HOST` when one is available. Pre-ABX-494 every row +below the cache-hit line was ∞ (all buildx builds hung). + ## Bench methodology No criterion micro-bench: build performance is dominated by the guest and From e6cce2d832c2b6077bbce2e0a115898664fb88fe Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 18:53:46 +0800 Subject: [PATCH 17/21] docs(e2e): add Colima comparison to the build bench baseline (ABX-496) --- .../plans/docker-build-e2e-matrix.md | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index 5830fd98f..5ba2a8eaf 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -128,21 +128,33 @@ outside the timed region; real-project contexts fetched at the Tier X pins, with X2's pnpm accommodation). Single-shot numbers — trend anchors, not gates. -| shape | wall | -|---|---| -| simple 3-RUN alpine build, cold (includes first-build builder bootstrap + arch probes) | 8.1 s | -| same build, full cache hit | 1.8 s | -| 512 MiB + 100k-file context, cold | 26.4 s | -| 12-stage diamond, cold | 12.7 s | -| `--platform linux/amd64` (FEX): probe + 300k-iteration shell loop | 6.4 s | -| postgres `17/bookworm` (real apt + gpg + wget) | 75.6 s | -| next.js `with-docker` (pnpm frozen install + `next build`) | 37.8 s | -| caddy `2.11/builder` (apk + xcaddy fetch) | 25.2 s | - -Cross-engine comparison was not possible on the baseline host — OrbStack -is uninstalled there (dangling `~/.orbstack`); the harness takes any -engine via `DOCKER_HOST` when one is available. Pre-ABX-494 every row -below the cache-hit line was ∞ (all buildx builds hung). +Comparison engine: Colima 0.10.3 on the same host, matched shape (18 +vCPU / 16 GiB, VZ, virtiofs, `--vz-rosetta`; its dockerd 29.5.2 — one +minor behind ArcBox's 29.6.1, same BuildKit generation). + +| shape | ArcBox | Colima | ratio | +|---|---|---|---| +| simple 3-RUN alpine build, cold | 8.1 s | 0.60 s | **13.5×** | +| same build, full cache hit | 1.8 s | 0.16 s | **11×** | +| 512 MiB + 100k-file context, cold | 26.4 s | 20.3 s | 1.3× | +| 12-stage diamond, cold | 12.7 s | 1.5 s | **8.4×** | +| `--platform linux/amd64` (FEX vs Rosetta): probe + 300k-iter shell loop | 6.4 s | 0.87 s | 7.3× | +| postgres `17/bookworm` (real apt + gpg + wget) | 75.6 s | 40.7 s | 1.9× | +| next.js `with-docker` (pnpm frozen install + `next build`) | 37.8 s | 27.1 s | 1.4× | +| caddy `2.11/builder` (apk + xcaddy fetch) | 25.2 s | 16.0 s | 1.6× | + +Reading: throughput-bound shapes (large context, real network builds) +sit at 1.3–1.9× — the datapath holds up. The 8–13× rows are all +**per-build / per-step overhead**: ~1 s per stage on ArcBox vs ~0.1 s on +Colima (multistage), and a 1.8 s floor for a fully-cached build vs +0.16 s. The bottleneck is round-trip/latency-shaped, not +bandwidth-shaped — suspects are the per-request vsock connect in the +docker proxy, session/gRPC round trips through the two-hop relay, and +per-layer snapshot-commit cost on the guest disk; profiling needed +(tracked in Linear). The amd64 row is dominated by the same overhead, +so it does NOT cleanly measure FEX-vs-Rosetta CPU speed. Pre-ABX-494 +every ArcBox row below the cache-hit line was ∞ (all buildx builds +hung). ## Bench methodology From c725c1ac4b89d694d42ccc9e0f7f77a18955a92e Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 05:49:00 +0800 Subject: [PATCH 18/21] =?UTF-8?q?docs(plans):=20add=20post-ABX-496=20bench?= =?UTF-8?q?=20re-run=20=E2=80=94=20parity=20on=20real=20builds,=20residual?= =?UTF-8?q?=20gap=20in=20per-step=20overhead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plans/docker-build-e2e-matrix.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index 5ba2a8eaf..2881e04dd 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -156,6 +156,35 @@ so it does NOT cleanly measure FEX-vs-Rosetta CPU speed. Pre-ABX-494 every ArcBox row below the cache-hit line was ∞ (all buildx builds hung). +### 2026-07-23 re-run — after the ABX-496 fixes + +Same script, same fixtures, same host, both engines cold and measured in +the same session. ArcBox = master `e4aca033` (rcu_expedited + ext4 +metadata volume + VZ `.fsync`, boot bundle 0.6.11). Absolute numbers for +the network rows are not comparable across days (registry weather); +ratios within a session are. + +| shape | ArcBox | Colima | ratio (was) | +|---|---|---|---| +| simple 3-RUN alpine build, cold | 3.0 s | 0.95 s | 3.2× (13.5×) | +| same build, full cache hit | 0.52 s | 0.30 s | 1.7× (11×) | +| 512 MiB + 100k-file context, cold | 23.8–31.6 s | 32.9 s | **≤1× (1.3×)** | +| 12-stage diamond, cold | 5.0 s | 1.6 s | 3.0× (8.4×) | +| `--platform linux/amd64` (FEX vs Rosetta) | 2.13 s | 2.14 s | **1.0× (7.3×)** | +| postgres `17/bookworm` | 83.7 s | 67.5 s | 1.24× (1.9×) | +| next.js `with-docker` | 54.3 s | 50.3 s | 1.08× (1.4×) | +| caddy `2.11/builder` | 23.7 s | 23.8 s | **1.0× (1.6×)** | + +Reading: real-project builds, the amd64 row, and large-context are at +parity (1.0–1.24×). The residual gap is confined to the pure-overhead +shapes (3.0–3.2×, ~2 s absolute) and matches the still-open `docker +start` network-endpoint/iptables segment (~330 ms per container-backed +step, measured 416 ms vs Colima 82 ms) — the next lever, tracked +separately from ABX-496. Bench-hygiene note: Colima retains BuildKit +cache across sessions and the real-project contexts carry no nonce, so +re-runs must `docker builder prune -af` on Colima first or its +real-project rows are cache hits. + ## Bench methodology No criterion micro-bench: build performance is dominated by the guest and From 51200f403b6e01eb4967ad11e775f78fb1053659 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 06:09:39 +0800 Subject: [PATCH 19/21] fix(e2e): kill the whole process group on timeout; harden D4 and the backend flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_with_timeout killed only the direct child, so a descendant that inherited the pipes kept the write end open and the drain-thread joins blocked past the deadline — 30s for the shell repro, unbounded when the survivor is a wedged docker-buildx, which is the shape docker build takes. Spawn into a fresh process group and killpg it. The regression test now bounds elapsed time and uses `sleep 30 & wait`: with a plain `sleep 30` the shell execs it, leaving no grandchild, and the bug hides (verified A/B — 30.06s before, 1.05s after). D4's ssh probe asserted `test -S $SSH_AUTH_SOCK`, which only stats the socket inode BuildKit created and passes even when no agent traffic crosses. Query the agent instead and require the throwaway key's own fingerprint back. xtask e2e now rejects --backend hv/both for targets built on scenario::run_vz_scenario*, which hardcode vz: the run was VZ regardless while the label, metrics, and archive said HV, silently corrupting any oracle comparison read from the artifacts. --- .../plans/docker-build-e2e-matrix.md | 2 +- tests/e2e/src/docker.rs | 50 ++++++++++++++-- tests/e2e/tests/docker_build.rs | 57 +++++++++++++++++-- xtask/AGENTS.md | 14 +++++ xtask/src/commands/e2e.rs | 23 ++++++++ 5 files changed, 134 insertions(+), 12 deletions(-) diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md index 2881e04dd..522a7dc3b 100644 --- a/internal-docs/plans/docker-build-e2e-matrix.md +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -95,7 +95,7 @@ assertions are deadlines and behavior, never absolute throughput. | D1 | large context upload (immich/next.js node_modules shape) | 512 MiB incompressible + 100k small files; `.dockerignore` excluding a marker dir | build ok ≤ deadline; in-build payload sha + whole-tree sha byte-exact; ignored dir absent in-guest; `large_context_transfer` trend *(implemented)* | | D2 | deep multi-stage graph (grafana/buildkit) | 12-stage diamond: `FROM scratch AS scripts` carrier, 4 parallel-schedulable branches, `COPY --from` joins, `FROM scratch AS export` tail, `COPY --link`, heredoc RUN, `# syntax=docker/dockerfile:1` (frontend pre-loaded) | export-stage artifact byte-exact via `docker cp`; full `docker export` listing free of builder residue; build ok ≤ deadline *(implemented)* | | D3 | cache semantics (mastodon/next.js) | `RUN --mount=type=cache,sharing=locked`; then 3 rebuilds: unchanged, one-file leaf change, base-layer change | per-execution stamps baked into the image prove the unchanged rebuild re-runs nothing (image IDs are NOT comparable — BuildKit re-stamps `created` per build), leaf change re-runs only downstream steps, base change re-runs both; warm wall ≤ max(cold/10, 3 s floor); cache-mount token survives across builds *(implemented)* | -| D4 | BuildKit session (authentik secrets; ssh has no OSS exemplar but same channel) | `--secret id=x,src=file` consumed by `RUN --mount=type=secret`; `--ssh default=` against a throwaway host `ssh-agent` (one loaded key) | secret readable in RUN, gone next layer, **absent from every byte of `docker save`** (gzip-aware sweep); forwarded ssh sock present in RUN; quiet-log covers the `/session` upgrades *(implemented)* | +| D4 | BuildKit session (authentik secrets; ssh has no OSS exemplar but same channel) | `--secret id=x,src=file` consumed by `RUN --mount=type=secret`; `--ssh default=` against a throwaway host `ssh-agent` (one loaded key) | secret readable in RUN, gone next layer, **absent from every byte of `docker save`** (gzip-aware sweep); the forwarded agent answers `ssh-add -l` inside the RUN with the throwaway key's own `SHA256:` fingerprint — a `test -S` on the socket inode would pass even when no agent traffic survives the crossing, which is the half that actually breaks; quiet-log covers the `/session` upgrades *(implemented)* | | D5 | bind-mount lockfiles (authentik/immich) | `RUN --mount=type=bind,source=...` consuming context files without COPY layers | in-build read + derived hash lands in image; mount leaves no trace in the final fs *(implemented)* | | D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) `linux/arm64:arm64` baked; (b) `x86_64` asserted in-build + image records `amd64` — the surface ABX-494 wedged. Fail-closed unprovisioned direction still awaits an e2e knob (FEX ships in every bundle ≥ 0.6.6) *(implemented, positive paths)* | | D7 | concurrent builds (CI shape) | 4 parallel distinct builds + 2 same-context duplicates, all in flight at once | all complete ≤ deadline; no cross-talk (unique markers per image); suite quiet-log covers the daemon side *(implemented)* | diff --git a/tests/e2e/src/docker.rs b/tests/e2e/src/docker.rs index b9995ae66..ec988fa45 100644 --- a/tests/e2e/src/docker.rs +++ b/tests/e2e/src/docker.rs @@ -6,6 +6,7 @@ //! daemon stay untouched (see tests/e2e/README.md on isolation). use std::io::Read as _; +use std::os::unix::process::CommandExt as _; use std::path::Path; use std::process::{Command, Stdio}; use std::thread; @@ -141,6 +142,19 @@ pub fn ensure_image(data_dir: &Path, image: &str) -> Result<()> { Err(last_err.expect("loop ran at least once")).context("docker pull (3 attempts)") } +/// SIGKILLs the child's whole process group, falling back to the direct +/// child if the group is already gone. +fn kill_process_group(child: &mut std::process::Child) { + let pgid = i32::try_from(child.id()).expect("pid fits in i32"); + // SAFETY: `killpg` is async-signal-safe and takes no pointers. `pgid` is + // this child's pid, which is also its group id (`process_group(0)`), and + // the child has not been reaped yet, so the id cannot have been recycled. + let killed = unsafe { libc::killpg(pgid, libc::SIGKILL) } == 0; + if !killed { + let _ = child.kill(); + } +} + /// Runs a command, killing it once `timeout` passes. /// /// Both pipes are drained on background threads for the whole run: an @@ -148,10 +162,18 @@ pub fn ensure_image(data_dir: &Path, image: &str) -> Result<()> { /// chatty command (a `--progress=plain` build, a large pull) into a bogus /// timeout. On a real timeout the error carries the output tail, so a /// killed command still leaves forensics. +/// +/// The child leads its own process group and the timeout path signals the +/// whole group. Killing just the direct child is not enough: `docker build` +/// runs the build in a `docker-buildx` grandchild that inherits these pipes, +/// so the write end stays open and the drain-thread joins below block past +/// the deadline — indefinitely if that descendant is itself wedged, which is +/// exactly what this suite exists to catch. pub fn run_with_timeout(command: &mut Command, timeout: Duration) -> Result { let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) + .process_group(0) .spawn()?; let drain = |pipe: Option>| { thread::spawn(move || { @@ -189,9 +211,10 @@ pub fn run_with_timeout(command: &mut Command, timeout: Duration) -> Result 64 * 1024); } - /// A genuine timeout must surface the output tail for forensics. + /// A genuine timeout must surface the output tail for forensics, and + /// must return at the deadline even though the shell leaves a `sleep` + /// descendant holding the inherited pipes. Killing only the direct child + /// leaves that write end open, so the drain-thread joins block until the + /// descendant exits — ~30s here, unbounded when the survivor is a wedged + /// `docker-buildx`, which is the shape this suite hits for real. + /// + /// `sleep 30 & wait` is deliberate: with a plain `sleep 30` the shell + /// `exec`s it as the last command, so there is no grandchild and the bug + /// hides. Backgrounding forces the shell to stay alive as a real parent. + /// The elapsed bound is the regression; the tail is the original contract. #[test] - fn timeout_error_carries_output_tail() { + fn timeout_returns_at_deadline_despite_surviving_descendant() { + let start = Instant::now(); let error = run_with_timeout( - Command::new("sh").args(["-c", "echo tail-marker; sleep 30"]), + Command::new("sh").args(["-c", "echo tail-marker; sleep 30 & wait"]), Duration::from_secs(1), ) .expect_err("command must time out"); + let elapsed = start.elapsed(); assert!(error.to_string().contains("tail-marker")); + assert!( + elapsed < Duration::from_secs(10), + "timeout took {elapsed:?}; the `sleep` descendant outlived the \ + kill and held the pipes open" + ); } } diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs index 70c3a350f..820095bbb 100644 --- a/tests/e2e/tests/docker_build.rs +++ b/tests/e2e/tests/docker_build.rs @@ -647,6 +647,10 @@ RUN test ! -e /ctx/excluded struct ThrowawaySshAgent { pid: String, sock: std::path::PathBuf, + /// `SHA256:…` fingerprint of the loaded key. The build asserts the + /// forwarded agent reports exactly this, which only a working + /// round-trip can produce. + fingerprint: String, } impl ThrowawaySshAgent { @@ -669,8 +673,6 @@ impl ThrowawaySshAgent { .and_then(|rest| rest.split(';').next()) .map(str::to_owned) .ok_or_else(|| anyhow::anyhow!("no SSH_AGENT_PID in ssh-agent output: {stdout}"))?; - let agent = Self { pid, sock }; - let key = dir.join("d4-ssh-key"); let keygen = std::process::Command::new("ssh-keygen") .args(["-q", "-t", "ed25519", "-N", "", "-f"]) @@ -685,13 +687,40 @@ impl ThrowawaySshAgent { } let add = std::process::Command::new("ssh-add") .arg(&key) - .env("SSH_AUTH_SOCK", &agent.sock) + .env("SSH_AUTH_SOCK", &sock) .output() .context("loading key into ssh-agent")?; if !add.status.success() { bail!("ssh-add failed: {}", String::from_utf8_lossy(&add.stderr)); } - Ok(agent) + + // `ssh-keygen -lf` prints " SHA256: (ED25519)"; + // the fingerprint token is what `ssh-add -l` echoes back inside the + // build, so the two are directly comparable. + let pub_key = key.with_extension("pub"); + let show = std::process::Command::new("ssh-keygen") + .arg("-lf") + .arg(&pub_key) + .output() + .context("fingerprinting throwaway ssh key")?; + if !show.status.success() { + bail!( + "ssh-keygen -lf failed: {}", + String::from_utf8_lossy(&show.stderr) + ); + } + let listing = String::from_utf8_lossy(&show.stdout); + let fingerprint = listing + .split_whitespace() + .find(|token| token.starts_with("SHA256:")) + .map(str::to_owned) + .ok_or_else(|| anyhow::anyhow!("no SHA256 fingerprint in: {listing}"))?; + + Ok(Self { + pid, + sock, + fingerprint, + }) } } @@ -801,12 +830,28 @@ RUN test ! -e /run/secrets/build_token let agent = ThrowawaySshAgent::spawn(data_dir)?; let ctx = data_dir.join("d4-ssh-ctx"); std::fs::create_dir_all(&ctx).context("creating d4 ssh context dir")?; + // `test -S` only stats the socket inode BuildKit created — it passes even + // when no agent traffic survives the crossing, which is the half that can + // actually break. Query the agent instead and require the throwaway key's + // own fingerprint back: that answer can only come from a completed + // request/response round-trip over the forwarded socket. + // + // `apk` is the one image assumption in this suite; it holds for the + // default `alpine:latest` and any alpine mirror `ARCBOX_E2E_IMAGE` points + // at. A non-alpine override fails here loudly rather than silently + // weakening the check. std::fs::write( ctx.join("Dockerfile"), format!( r#"FROM {image} -RUN --mount=type=ssh test -S "$SSH_AUTH_SOCK" && echo ssh-forwarded > /probe -"# +RUN --mount=type=ssh set -e; \ + test -S "$SSH_AUTH_SOCK"; \ + apk add --no-cache openssh-client >/dev/null; \ + ssh-add -l | tee /probe-identities; \ + grep -q '{fingerprint}' /probe-identities; \ + echo ssh-forwarded > /probe +"#, + fingerprint = agent.fingerprint, ), ) .context("writing d4 ssh Dockerfile")?; diff --git a/xtask/AGENTS.md b/xtask/AGENTS.md index 7d5c7d89e..575018bcc 100644 --- a/xtask/AGENTS.md +++ b/xtask/AGENTS.md @@ -45,6 +45,18 @@ Note `virtio_debug` and `daemon_failure` deliberately have NO recipe and their profiles differ (release vs debug daemon). If you ever add a recipe for either, it must reproduce that exact profile. +## `xtask e2e` — `--backend` cannot move a VZ-pinned target + +`scenario::run_vz_scenario*` hardcodes `ARCBOX_VM_BACKEND=vz` and stamps +`"vz"` into the run's metrics, so the runner's `--backend` env is ignored by +every target built on it. `VZ_PINNED_TESTS` (`commands/e2e.rs`) lists them +and the runner **errors** on `--backend hv`/`both` for those targets. WHY: +the alternative is a VZ run archived under an HV label — the same +ghost-debugging class as a mismatched `SKIP_BUILD` recipe, and it silently +corrupts any HV↔VZ oracle comparison read from the artifacts. Moving a +target off `run_vz_scenario` means removing it from that list in the same +change. + ### Extending — adding a new e2e target (lockstep set) 1. The test must gate its build on `arcbox_e2e::env_flag("SKIP_BUILD")`, or `xtask e2e` cannot prebuild it and it rebuilds every repeat. @@ -52,6 +64,8 @@ for either, it must reproduce that exact profile. or accept the self-build fallback — never a half-match. 3. If you add a recipe, verify the packages AND profile match the test's own `cargo build` line character-for-character. +4. If the target runs through `scenario::run_vz_scenario*`, add it to + `VZ_PINNED_TESTS` so `--backend hv` fails loudly instead of mislabeling. ## `xtask e2e` — forensics linkage (fragile string/env coupling) diff --git a/xtask/src/commands/e2e.rs b/xtask/src/commands/e2e.rs index 7d983687c..0ecbd9293 100644 --- a/xtask/src/commands/e2e.rs +++ b/xtask/src/commands/e2e.rs @@ -16,6 +16,20 @@ use xtask_kit::repo; use crate::{E2eArgs, E2eBackend}; +/// Targets that drive their daemon through `scenario::run_vz_scenario*`, +/// which hardcodes `ARCBOX_VM_BACKEND=vz` (`tests/e2e/src/scenario.rs`) and +/// stamps `"vz"` into their metrics. The runner's `--backend` cannot move +/// them, so honoring an HV request would archive a VZ run under an HV label +/// and corrupt any backend comparison read from it. +const VZ_PINNED_TESTS: &[&str] = &[ + "docker_build", + "docker_build_external", + "egress_throughput", + "network_fault", + "network_workload", + "reconciler_teardown", +]; + /// One test-run outcome for the final summary. struct RunOutcome { label: String, @@ -42,6 +56,15 @@ pub fn run(args: E2eArgs) -> Result<()> { E2eBackend::Hv => &[Some("hv")], E2eBackend::Both => &[Some("vz"), Some("hv")], }; + if !matches!(args.backend, E2eBackend::Vz) && VZ_PINNED_TESTS.contains(&args.test.as_str()) { + bail!( + "test `{}` pins ARCBOX_VM_BACKEND=vz in its scenario harness, so the \ + requested backend cannot take effect — the run would be VZ while the \ + label, metrics, and archive said HV. Re-run with `--backend vz`, or \ + move the target off scenario::run_vz_scenario first.", + args.test, + ); + } // Build what the selected test would build, once, so every repeat // runs with SKIP_BUILD=1 instead of re-invoking cargo. From 580fb4c244f346b4f6bf7dd8bdc8f037fc0fc8b5 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 06:50:16 +0800 Subject: [PATCH 20/21] fix(e2e): kill the process group on the success path too; derive the VZ pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The success branch joined the drain threads without touching the process group, so a command that exits promptly while leaving a descendant on the inherited pipes blocked the join with no deadline left to enforce — `sh -c 'echo done; sleep 30 & exit 0'` took 30.07s. Both exit paths now kill the group before joining (verified A/B: 30.07s -> 1.04s). kill_process_group takes the pgid captured at spawn rather than a live Child, since the success path has already reaped the leader. That is sound: POSIX forbids reusing a pid while it is still the group id of an existing group, so the signal reaches that group or nothing (ESRCH). The VZ pin is now read from the target's own source instead of a hardcoded list. The list was already stale at this branch's merged head — network_iperf and bench_virtiofs both use run_vz_scenario_with_log and were missing — which is the failure mode a list has: it goes stale exactly when a new target needs the guard. --- tests/e2e/src/docker.rs | 62 +++++++++++++++++++++++++++------------ xtask/AGENTS.md | 23 +++++++++------ xtask/src/commands/e2e.rs | 32 ++++++++++++-------- 3 files changed, 77 insertions(+), 40 deletions(-) diff --git a/tests/e2e/src/docker.rs b/tests/e2e/src/docker.rs index ec988fa45..edda30083 100644 --- a/tests/e2e/src/docker.rs +++ b/tests/e2e/src/docker.rs @@ -142,17 +142,15 @@ pub fn ensure_image(data_dir: &Path, image: &str) -> Result<()> { Err(last_err.expect("loop ran at least once")).context("docker pull (3 attempts)") } -/// SIGKILLs the child's whole process group, falling back to the direct -/// child if the group is already gone. -fn kill_process_group(child: &mut std::process::Child) { - let pgid = i32::try_from(child.id()).expect("pid fits in i32"); - // SAFETY: `killpg` is async-signal-safe and takes no pointers. `pgid` is - // this child's pid, which is also its group id (`process_group(0)`), and - // the child has not been reaped yet, so the id cannot have been recycled. - let killed = unsafe { libc::killpg(pgid, libc::SIGKILL) } == 0; - if !killed { - let _ = child.kill(); - } +/// SIGKILLs the process group led by `pgid`, which `process_group(0)` made +/// equal to the spawned child's pid. Safe to call after the leader has been +/// reaped: POSIX forbids reusing a pid while it is still the group id of an +/// existing group, so this either reaches that group or nothing at all +/// (`ESRCH`), never an unrelated one. +fn kill_process_group(pgid: i32) { + // SAFETY: `killpg` takes no pointers and cannot fail unsoundly; a stale + // or empty group yields ESRCH, which we ignore. + unsafe { libc::killpg(pgid, libc::SIGKILL) }; } /// Runs a command, killing it once `timeout` passes. @@ -163,18 +161,23 @@ fn kill_process_group(child: &mut std::process::Child) { /// timeout. On a real timeout the error carries the output tail, so a /// killed command still leaves forensics. /// -/// The child leads its own process group and the timeout path signals the -/// whole group. Killing just the direct child is not enough: `docker build` -/// runs the build in a `docker-buildx` grandchild that inherits these pipes, -/// so the write end stays open and the drain-thread joins below block past -/// the deadline — indefinitely if that descendant is itself wedged, which is -/// exactly what this suite exists to catch. +/// The child leads its own process group and **both** exit paths signal the +/// whole group before joining. Killing just the direct child is not enough: +/// `docker build` runs the build in a `docker-buildx` grandchild that +/// inherits these pipes, so the write end stays open and the drain-thread +/// joins block past the deadline — indefinitely if that descendant is itself +/// wedged, which is exactly what this suite exists to catch. The same holds +/// when the command *succeeds* while leaving a descendant behind, so the +/// group kill is not conditional on timing out: once the direct child is +/// gone, nothing else may hold pipes this function is about to join on. pub fn run_with_timeout(command: &mut Command, timeout: Duration) -> Result { let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .process_group(0) .spawn()?; + // Captured before any reap: `Child::id` is not meaningful afterwards. + let pgid = i32::try_from(child.id()).expect("pid fits in i32"); let drain = |pipe: Option>| { thread::spawn(move || { let mut buf = Vec::new(); @@ -200,6 +203,7 @@ pub fn run_with_timeout(command: &mut Command, timeout: Duration) -> Result Result 64 * 1024); } + /// The success path has the same hazard as the timeout path: a command + /// can exit promptly while leaving a descendant holding the inherited + /// pipes, and the drain-thread joins would then block on that descendant + /// with no deadline left to enforce. Returns in ~0s once the group is + /// killed, ~30s if the success branch stops doing so. + #[test] + fn success_returns_promptly_despite_surviving_descendant() { + let start = Instant::now(); + let output = run_with_timeout( + Command::new("sh").args(["-c", "echo done; sleep 30 & exit 0"]), + Duration::from_secs(60), + ) + .expect("command must succeed"); + let elapsed = start.elapsed(); + assert!(output.status.success()); + assert!( + elapsed < Duration::from_secs(10), + "success took {elapsed:?}; a descendant outlived the command and \ + held the pipes open" + ); + } + /// A genuine timeout must surface the output tail for forensics, and /// must return at the deadline even though the shell leaves a `sleep` /// descendant holding the inherited pipes. Killing only the direct child diff --git a/xtask/AGENTS.md b/xtask/AGENTS.md index 575018bcc..6f1c9acda 100644 --- a/xtask/AGENTS.md +++ b/xtask/AGENTS.md @@ -49,13 +49,18 @@ for either, it must reproduce that exact profile. `scenario::run_vz_scenario*` hardcodes `ARCBOX_VM_BACKEND=vz` and stamps `"vz"` into the run's metrics, so the runner's `--backend` env is ignored by -every target built on it. `VZ_PINNED_TESTS` (`commands/e2e.rs`) lists them -and the runner **errors** on `--backend hv`/`both` for those targets. WHY: -the alternative is a VZ run archived under an HV label — the same -ghost-debugging class as a mismatched `SKIP_BUILD` recipe, and it silently -corrupts any HV↔VZ oracle comparison read from the artifacts. Moving a -target off `run_vz_scenario` means removing it from that list in the same -change. +every target built on it. The runner **errors** on `--backend hv`/`both` for +those targets. WHY: the alternative is a VZ run archived under an HV label — +the same ghost-debugging class as a mismatched `SKIP_BUILD` recipe, and it +silently corrupts any HV↔VZ oracle comparison read from the artifacts. + +`is_vz_pinned` (`commands/e2e.rs`) decides this by reading +`tests/e2e/tests/.rs` for `run_vz_scenario`, deliberately NOT from a +list here or in the code: the set grows whenever someone writes a +scenario-based test, and a hardcoded list goes stale exactly when a new +target needs the guard most. Nothing to keep in lockstep — but if you move a +target off `run_vz_scenario`, the guard stops applying on its own, so make +sure the target really honors `ARCBOX_VM_BACKEND` before relying on that. ### Extending — adding a new e2e target (lockstep set) 1. The test must gate its build on `arcbox_e2e::env_flag("SKIP_BUILD")`, @@ -64,8 +69,8 @@ change. or accept the self-build fallback — never a half-match. 3. If you add a recipe, verify the packages AND profile match the test's own `cargo build` line character-for-character. -4. If the target runs through `scenario::run_vz_scenario*`, add it to - `VZ_PINNED_TESTS` so `--backend hv` fails loudly instead of mislabeling. +4. Nothing to do for the VZ pin — `is_vz_pinned` reads it off the target's + source, so a scenario-based target gets the `--backend hv` guard for free. ## `xtask e2e` — forensics linkage (fragile string/env coupling) diff --git a/xtask/src/commands/e2e.rs b/xtask/src/commands/e2e.rs index 0ecbd9293..9563e9aba 100644 --- a/xtask/src/commands/e2e.rs +++ b/xtask/src/commands/e2e.rs @@ -16,19 +16,25 @@ use xtask_kit::repo; use crate::{E2eArgs, E2eBackend}; -/// Targets that drive their daemon through `scenario::run_vz_scenario*`, +/// Whether `test` drives its daemon through `scenario::run_vz_scenario*`, /// which hardcodes `ARCBOX_VM_BACKEND=vz` (`tests/e2e/src/scenario.rs`) and -/// stamps `"vz"` into their metrics. The runner's `--backend` cannot move -/// them, so honoring an HV request would archive a VZ run under an HV label -/// and corrupt any backend comparison read from it. -const VZ_PINNED_TESTS: &[&str] = &[ - "docker_build", - "docker_build_external", - "egress_throughput", - "network_fault", - "network_workload", - "reconciler_teardown", -]; +/// stamps `"vz"` into the metrics. The runner's `--backend` cannot move such +/// a target, so honoring an HV request would archive a VZ run under an HV +/// label and corrupt any backend comparison read from it. +/// +/// Read from the target's source rather than kept as a list here: the set +/// grows whenever someone writes a scenario-based test, and a list would +/// silently go stale exactly when a new target needs the guard most. A +/// target with no source file (an unknown name) is not pinned — cargo +/// reports that better than we can. +fn is_vz_pinned(root: &std::path::Path, test: &str) -> Result { + let path = root.join("tests/e2e/tests").join(format!("{test}.rs")); + match fs::read_to_string(&path) { + Ok(source) => Ok(source.contains("run_vz_scenario")), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e).with_context(|| format!("reading {}", path.display())), + } +} /// One test-run outcome for the final summary. struct RunOutcome { @@ -56,7 +62,7 @@ pub fn run(args: E2eArgs) -> Result<()> { E2eBackend::Hv => &[Some("hv")], E2eBackend::Both => &[Some("vz"), Some("hv")], }; - if !matches!(args.backend, E2eBackend::Vz) && VZ_PINNED_TESTS.contains(&args.test.as_str()) { + if !matches!(args.backend, E2eBackend::Vz) && is_vz_pinned(&root, &args.test)? { bail!( "test `{}` pins ARCBOX_VM_BACKEND=vz in its scenario harness, so the \ requested backend cannot take effect — the run would be VZ while the \ From 92b1cde9b09f894e4fd585d6fb7604df09d36f28 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 11:14:47 +0800 Subject: [PATCH 21/21] fix(xtask): guard every backend-pinned e2e target, not just scenario ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous derivation only recognised scenario::run_vz_scenario, so five targets that hardcode ARCBOX_VM_BACKEND in their own source stayed unguarded — idle_balloon, machine, stats_watch, nfs_restart_probe pin vz, and virtio_debug and hv_reboot pin hv. The hv-pinned pair is why broadening cannot be a simple 'is it pinned to vz' check: --backend vz on virtio_debug mislabels exactly as badly as --backend hv on a vz-pinned target. pinned_backend now reports WHICH backend a target pins, and the runner errors only when the request conflicts with it. Derived from the sources, not a list: scan the target for a line carrying both ARCBOX_VM_BACKEND and a "vz"/"hv" literal, then the arcbox_e2e modules it imports — one level of indirection, where run_vz_scenario and the sandbox harness keep theirs. Lines that merely read the variable carry no literal, so boot_assets and backend_matrix stay correctly unpinned. Verified against all 22 targets in the tree; unit tests pin the three pin shapes actually used, the read-the-env non-pin, and the import scan. --- xtask/AGENTS.md | 47 ++++++++----- xtask/src/commands/e2e.rs | 143 ++++++++++++++++++++++++++++++++------ 2 files changed, 152 insertions(+), 38 deletions(-) diff --git a/xtask/AGENTS.md b/xtask/AGENTS.md index 6f1c9acda..cb3e357c2 100644 --- a/xtask/AGENTS.md +++ b/xtask/AGENTS.md @@ -45,22 +45,30 @@ Note `virtio_debug` and `daemon_failure` deliberately have NO recipe and their profiles differ (release vs debug daemon). If you ever add a recipe for either, it must reproduce that exact profile. -## `xtask e2e` — `--backend` cannot move a VZ-pinned target - -`scenario::run_vz_scenario*` hardcodes `ARCBOX_VM_BACKEND=vz` and stamps -`"vz"` into the run's metrics, so the runner's `--backend` env is ignored by -every target built on it. The runner **errors** on `--backend hv`/`both` for -those targets. WHY: the alternative is a VZ run archived under an HV label — -the same ghost-debugging class as a mismatched `SKIP_BUILD` recipe, and it -silently corrupts any HV↔VZ oracle comparison read from the artifacts. - -`is_vz_pinned` (`commands/e2e.rs`) decides this by reading -`tests/e2e/tests/.rs` for `run_vz_scenario`, deliberately NOT from a -list here or in the code: the set grows whenever someone writes a -scenario-based test, and a hardcoded list goes stale exactly when a new -target needs the guard most. Nothing to keep in lockstep — but if you move a -target off `run_vz_scenario`, the guard stops applying on its own, so make -sure the target really honors `ARCBOX_VM_BACKEND` before relying on that. +## `xtask e2e` — `--backend` cannot move a pinned target + +Most e2e targets hardcode `ARCBOX_VM_BACKEND` and stamp that backend into +their metrics, so the runner's `--backend` env has no effect on them. Only a +few (`boot_assets`, `backend_matrix`) actually read it. The runner therefore +**errors** when the requested backend conflicts with a pinned one. WHY: the +alternative is a run archived under the wrong backend's label — the same +ghost-debugging class as a mismatched `SKIP_BUILD` recipe, and it silently +corrupts any HV↔VZ oracle comparison read from the artifacts. + +This cuts both ways: `virtio_debug` and `hv_reboot` pin **hv**, so +`--backend vz` on them is exactly as wrong as `--backend hv` on a vz-pinned +target. `--backend both` conflicts with any pin. + +`pinned_backend` (`commands/e2e.rs`) derives this from the sources rather +than from a list, deliberately: the set changes whenever someone adds a +target, and a list goes stale exactly when a new target needs the guard +most. It scans the target's own source for a line carrying both +`ARCBOX_VM_BACKEND` and a `"vz"`/`"hv"` literal, then the `arcbox_e2e` +modules it imports — one level of indirection, which is where +`scenario::run_vz_scenario*` and the `sandbox` harness keep their pin. A +line that merely reads the variable carries no literal and is correctly +ignored. Nothing to keep in lockstep; the unit tests in that file pin the +shapes. ### Extending — adding a new e2e target (lockstep set) 1. The test must gate its build on `arcbox_e2e::env_flag("SKIP_BUILD")`, @@ -69,8 +77,11 @@ sure the target really honors `ARCBOX_VM_BACKEND` before relying on that. or accept the self-build fallback — never a half-match. 3. If you add a recipe, verify the packages AND profile match the test's own `cargo build` line character-for-character. -4. Nothing to do for the VZ pin — `is_vz_pinned` reads it off the target's - source, so a scenario-based target gets the `--backend hv` guard for free. +4. Nothing to do for the backend pin — `pinned_backend` reads it off the + sources, so a new target gets the `--backend` guard for free as long as + it pins the way every existing one does (an `ARCBOX_VM_BACKEND` line with + a `"vz"`/`"hv"` literal, in the target or in an `arcbox_e2e` module it + imports). ## `xtask e2e` — forensics linkage (fragile string/env coupling) diff --git a/xtask/src/commands/e2e.rs b/xtask/src/commands/e2e.rs index 9563e9aba..f21ab6de1 100644 --- a/xtask/src/commands/e2e.rs +++ b/xtask/src/commands/e2e.rs @@ -16,24 +16,82 @@ use xtask_kit::repo; use crate::{E2eArgs, E2eBackend}; -/// Whether `test` drives its daemon through `scenario::run_vz_scenario*`, -/// which hardcodes `ARCBOX_VM_BACKEND=vz` (`tests/e2e/src/scenario.rs`) and -/// stamps `"vz"` into the metrics. The runner's `--backend` cannot move such -/// a target, so honoring an HV request would archive a VZ run under an HV -/// label and corrupt any backend comparison read from it. +/// The backend `test` hardcodes, if any — `--backend` cannot move it. /// -/// Read from the target's source rather than kept as a list here: the set -/// grows whenever someone writes a scenario-based test, and a list would -/// silently go stale exactly when a new target needs the guard most. A -/// target with no source file (an unknown name) is not pinned — cargo -/// reports that better than we can. -fn is_vz_pinned(root: &std::path::Path, test: &str) -> Result { +/// Targets fall into three groups. A few read `ARCBOX_VM_BACKEND` from the +/// environment and genuinely honor `--backend` (`boot_assets`, +/// `backend_matrix`). Most pin one backend outright, either in their own +/// source (`virtio_debug` and `hv_reboot` pin **hv**; `machine`, +/// `idle_balloon`, `stats_watch`, `nfs_restart_probe` pin vz) or through a +/// shared harness in `tests/e2e/src` (`scenario::run_vz_scenario*`, +/// `sandbox`). For a pinned target the runner's env has no effect, so a +/// mismatched request archives a run under the wrong backend's label and +/// corrupts any comparison read from it — the same ghost-debugging class as +/// a mismatched `SKIP_BUILD` recipe. Note this cuts both ways: `--backend +/// vz` on an hv-pinned target is just as wrong as `--backend hv` on a +/// vz-pinned one. +/// +/// Derived from the sources rather than kept as a list here: the set changes +/// whenever someone adds a target, and a list goes stale exactly when a new +/// target needs the guard most. An unreadable or absent target is treated as +/// unpinned — cargo reports an unknown test better than we can. +fn pinned_backend(root: &std::path::Path, test: &str) -> Result> { let path = root.join("tests/e2e/tests").join(format!("{test}.rs")); - match fs::read_to_string(&path) { - Ok(source) => Ok(source.contains("run_vz_scenario")), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(e) => Err(e).with_context(|| format!("reading {}", path.display())), + let source = match fs::read_to_string(&path) { + Ok(source) => source, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())), + }; + + // The target's own source first, then the `arcbox_e2e` modules it pulls + // in — that one level of indirection is where `run_vz_scenario` and the + // sandbox harness hide their pin. + if let Some(backend) = literal_backend_pin(&source) { + return Ok(Some(backend)); } + for module in imported_e2e_modules(&source) { + let path = root.join("tests/e2e/src").join(format!("{module}.rs")); + if let Ok(helper) = fs::read_to_string(&path) + && let Some(backend) = literal_backend_pin(&helper) + { + return Ok(Some(backend)); + } + } + Ok(None) +} + +/// Finds a hardcoded `ARCBOX_VM_BACKEND` value in `source`. +/// +/// A pin is a line that both names the variable and carries a `"vz"` or +/// `"hv"` literal, which is how every pin in the tree is written. Lines that +/// merely *read* the variable (`env::var("ARCBOX_VM_BACKEND")`) or forward +/// an already-computed value carry no literal and are correctly ignored. +fn literal_backend_pin(source: &str) -> Option { + source + .lines() + .filter(|line| line.contains("ARCBOX_VM_BACKEND")) + .find_map(|line| { + ["vz", "hv"] + .into_iter() + .find(|backend| line.contains(&format!("\"{backend}\""))) + .map(str::to_owned) + }) +} + +/// Module names from every `arcbox_e2e::` path in `source`. +fn imported_e2e_modules(source: &str) -> Vec { + const PREFIX: &str = "arcbox_e2e::"; + let mut modules = Vec::new(); + for (index, _) in source.match_indices(PREFIX) { + let module: String = source[index + PREFIX.len()..] + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect(); + if !module.is_empty() && !modules.contains(&module) { + modules.push(module); + } + } + modules } /// One test-run outcome for the final summary. @@ -62,12 +120,14 @@ pub fn run(args: E2eArgs) -> Result<()> { E2eBackend::Hv => &[Some("hv")], E2eBackend::Both => &[Some("vz"), Some("hv")], }; - if !matches!(args.backend, E2eBackend::Vz) && is_vz_pinned(&root, &args.test)? { + if let Some(pinned) = pinned_backend(&root, &args.test)? + && backends.iter().flatten().any(|wanted| *wanted != pinned) + { bail!( - "test `{}` pins ARCBOX_VM_BACKEND=vz in its scenario harness, so the \ - requested backend cannot take effect — the run would be VZ while the \ - label, metrics, and archive said HV. Re-run with `--backend vz`, or \ - move the target off scenario::run_vz_scenario first.", + "test `{}` hardcodes ARCBOX_VM_BACKEND={pinned}, so the requested \ + backend cannot take effect — the run would be {pinned} while the \ + label, metrics, and archive said otherwise. Re-run with \ + `--backend {pinned}`, or make the target read the env first.", args.test, ); } @@ -251,3 +311,46 @@ fn summarize(outcomes: &[RunOutcome], artifacts_dir: &std::path::Path) -> Result } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The shapes every pin in `tests/e2e` is actually written in — a + /// tuple entry, `.into()` instead of `.to_owned()`, and an hv pin. + #[test] + fn literal_pins_are_recognized_in_the_shapes_used() { + let vz = r#"("ARCBOX_VM_BACKEND".to_owned(), "vz".to_owned()),"#; + let vz_into = r#"("ARCBOX_VM_BACKEND".into(), "vz".into()),"#; + let hv = r#"("ARCBOX_VM_BACKEND".to_owned(), "hv".to_owned()),"#; + assert_eq!(literal_backend_pin(vz).as_deref(), Some("vz")); + assert_eq!(literal_backend_pin(vz_into).as_deref(), Some("vz")); + assert_eq!(literal_backend_pin(hv).as_deref(), Some("hv")); + } + + /// A target that *reads* the env honors `--backend` and must not be + /// mistaken for a pinned one — `boot_assets` and `backend_matrix` are + /// the whole reason the runner has a `--backend` flag. + #[test] + fn reading_the_env_is_not_a_pin() { + let reads = r#"let backend = match env::var("ARCBOX_VM_BACKEND") {"#; + let forwards = + r#"env.push(("ARCBOX_VM_BACKEND".to_owned(), backend.as_str().to_owned()));"#; + assert!(literal_backend_pin(reads).is_none()); + assert!(literal_backend_pin(forwards).is_none()); + } + + /// The pin a scenario-based target inherits lives one level away, in + /// the `arcbox_e2e` module it imports — that indirection is what a + /// naive scan of the target's own source misses. + #[test] + fn imported_modules_are_collected_for_the_indirect_pin() { + let source = "use arcbox_e2e::scenario::run_vz_scenario_with_log;\n\ + use arcbox_e2e::metrics::RunMetrics;\n\ + let root = arcbox_e2e::repo_root();\n"; + let modules = imported_e2e_modules(source); + assert!(modules.contains(&"scenario".to_owned())); + assert!(modules.contains(&"metrics".to_owned())); + assert!(modules.contains(&"repo_root".to_owned())); + } +}