diff --git a/internal-docs/plans/docker-build-e2e-matrix.md b/internal-docs/plans/docker-build-e2e-matrix.md new file mode 100644 index 000000000..522a7dc3b --- /dev/null +++ b/internal-docs/plans/docker-build-e2e-matrix.md @@ -0,0 +1,224 @@ +# docker build E2E matrix — derived from a real-world Dockerfile corpus + +Companion to `network-workload-e2e.md`. W14 there proved the *network +datapath* survives one `docker build`; this plan derives the **build +compatibility and performance matrix** from what flagship open-source +projects actually put in their Dockerfiles, so the future `docker_build` +e2e suite tests the build surface developers really hit — not features we +imagine they use. + +Method: 25 Dockerfiles fetched 2026-07-22 from each repo's default branch +(`raw.githubusercontent.com//HEAD/`), feature vectors extracted +programmatically (stage counts, `--mount` types, platform ARGs, heredocs, +syntax directives, network calls). Re-fetch with the same repo/path table +below to refresh the census. + +## Corpus + +| Source (repo — path) | Ecosystem | Shape highlights | +|---|---|---| +| moby/moby — `Dockerfile` | Go | 56 stages, 51 cache mounts, 35 `COPY --link`, heredocs, `FROM scratch` exports | +| moby/buildkit — `Dockerfile` | Go | 44 stages, 19 bind mounts, 51 `COPY --link`, 8 `FROM scratch`, cross-platform (`--platform=$BUILDPLATFORM`) | +| docker/buildx — `Dockerfile` | Go | 29 stages, same family as buildkit | +| grafana/grafana — `Dockerfile` | Go+Node hybrid | 14 stages, `syntax=docker/dockerfile:1.7-labs`, `COPY --parents` | +| apache/airflow — `Dockerfile` | Python | 1.9k lines, 94 ARGs, ARG-parameterized `FROM ${BASE_IMAGE}`, `FROM scratch as scripts` heredoc script-carrier stage, 15 heredocs | +| mastodon/mastodon — `Dockerfile` | Ruby+Node | 9 stages, 11 cache mounts with `sharing=locked` + per-`TARGETPLATFORM` cache IDs | +| goauthentik/authentik — `lifecycle/container/Dockerfile` | Go+Python+Node | 34 bind mounts, **2 `--mount=type=secret`** (only secret user in corpus), `ADD` from URL | +| immich-app/immich — `server/Dockerfile` | Node native-deps | 16 bind mounts for pnpm lockfiles, 6 cache mounts | +| vercel/next.js — `examples/with-docker/Dockerfile` | Node | canonical 3-stage standalone build; npm/yarn/pnpm autodetect via corepack; cache mounts | +| vercel/turborepo — `examples/with-docker/apps/web/Dockerfile` | Node monorepo | `turbo prune --docker` context-slimming pattern | +| n8n-io/n8n — `docker/images/n8n/Dockerfile` | Node | modest 2-FROM prod image | +| go-gitea/gitea — `Dockerfile` | Go | 2-stage + cross-platform FROM | +| traefik/traefik — `Dockerfile` | Go | minimal `syntax=1.2` + TARGETPLATFORM COPY | +| caddyserver/caddy-docker — `2.11/builder/Dockerfile` | Go | xcaddy builder: toolchain download at build time | +| prometheus/prometheus — `Dockerfile` | Go | pure COPY-binary, zero network | +| dani-garcia/vaultwarden — `docker/Dockerfile.debian` | Rust | 4 FROMs, TARGETPLATFORM cross-compile | +| meilisearch/meilisearch — `Dockerfile` | Rust | alpine cargo build | +| home-assistant/core — `Dockerfile` | Python | `syntax=` **pinned by sha256 digest**, uv installs | +| paperless-ngx/paperless-ngx — `Dockerfile` | Python+Node | 2-stage, TARGETPLATFORM, HEALTHCHECK | +| keycloak/keycloak — `quarkus/container/Dockerfile` | JVM | 1-stage + chown COPY | +| mastodon-class official images: docker-library/postgres `17/bookworm`, docker-library/redis `7.4/debian`, nginxinc/docker-nginx `mainline/debian`, docker-library/ghost `6/bookworm` | C/misc | classic single-stage: apt + `wget` release binaries + **gpg keyserver verify**, tiny build context | +| traefik/traefik — `webui/buildx.Dockerfile` | Node | trivial single-stage yarn | + +## Feature census (what production Dockerfiles actually use) + +Ranked by prevalence in the corpus (n=25): + +| Feature | Prevalence | Exemplars | +|---|---|---| +| Multi-stage + `COPY --from` | 17/25; extremes 56/44/29 stages | moby, buildkit, buildx, grafana | +| Build-time network (apt/apk/npm/pip/cargo/bundle) | 24/25 | everything except prometheus | +| External `# syntax=` frontend (adds a frontend **image pull** to every build) | 12/25 — incl. version-pinned (`1.18`, `1.2`), digest-pinned (home-assistant), labs channel (grafana), master (buildkit) | see syntax list | +| `RUN --mount=type=cache` | 10/25 | moby(51), mastodon(11, `sharing=locked`, per-platform IDs), authentik, immich, next.js | +| `TARGETPLATFORM`/`TARGETARCH` + `--platform=$BUILDPLATFORM` cross-compile | 9/25 | moby, buildkit, mastodon, vaultwarden, gitea, paperless | +| `RUN --mount=type=bind` (lockfile mounts, no COPY layer) | 5/25, heavy | authentik(34), immich(16), buildkit(19) | +| `COPY --link` | 4/25 | buildkit(51), moby(35), buildx, grafana | +| Heredocs (`RUN <` against a throwaway host `ssh-agent` (one loaded key) | secret readable in RUN, gone next layer, **absent from every byte of `docker save`** (gzip-aware sweep); the forwarded agent answers `ssh-add -l` inside the RUN with the throwaway key's own `SHA256:` fingerprint — a `test -S` on the socket inode would pass even when no agent traffic survives the crossing, which is the half that actually breaks; quiet-log covers the `/session` upgrades *(implemented)* | +| D5 | bind-mount lockfiles (authentik/immich) | `RUN --mount=type=bind,source=...` consuming context files without COPY layers | in-build read + derived hash lands in image; mount leaves no trace in the final fs *(implemented)* | +| D6 | cross-platform args + FEX (vaultwarden/moby; ABX-375) | (a) `--platform=$BUILDPLATFORM` FROM + `TARGETARCH` expansion, native; (b) `--platform linux/amd64` with a `RUN uname -m` step | (a) `linux/arm64:arm64` baked; (b) `x86_64` asserted in-build + image records `amd64` — the surface ABX-494 wedged. Fail-closed unprovisioned direction still awaits an e2e knob (FEX ships in every bundle ≥ 0.6.6) *(implemented, positive paths)* | +| D7 | concurrent builds (CI shape) | 4 parallel distinct builds + 2 same-context duplicates, all in flight at once | all complete ≤ deadline; no cross-talk (unique markers per image); suite quiet-log covers the daemon side *(implemented)* | +| D8 | cancellation (production reality) | SIGKILL the client mid-RUN and mid-context-upload (256 MiB); rebuild after | guest reaps the cancelled RUN's process (polled via `--pid=host`); follow-up build succeeds promptly; suite quiet-log asserts the daemon rode both kills without proxy ERRORs *(implemented)* | +| D9 | output streaming | `--progress=plain` with a RUN emitting 50k ordered lines PACED under BuildKit's 200 KiB/s step-log rate clip (bulk output is clipped by design and proves nothing); `docker build -q` | tail lines present, no clip marker, deadline catches a wedge; quiet mode prints exactly one image ID *(implemented)* | +| D10 | exporters | `docker build -o type=local,dest=…` and `-o type=tar` | exported artifact byte-exact vs in-image content — exercises reverse session streaming *(implemented)* | + +External syntax frontends (`# syntax=docker/dockerfile:1`) are exercised in +D2 by pre-pulling the frontend image in the fixture; if the guest BuildKit +still dials out for it, the scenario moves to Tier X rather than weakening +the no-network rule. + +Tier X (`ARCBOX_E2E_EXTERNAL=1`, manual, real internet — the +network-workload plan's external-phase convention), pinned to specific +upstream commits: + +| ID | Build | Why this one | +|---|---|---| +| X1 | docker-library/postgres `17/bookworm` | tiny context, real apt + wget + **gpg keyserver** traffic — the classic official-image shape; smoke: `postgres --version` *(implemented; redis/nginx are the same shape and stay out to keep the run bounded)* | +| X2 | vercel/next.js `examples/with-docker` | real `pnpm install --frozen-lockfile` + `next build` through the datapath; the single most common user Dockerfile shape *(implemented)* | +| X3 | caddyserver/caddy-docker `2.11/builder` | apk + checksum-verified xcaddy release fetch (the builder image installs the toolchain; the full compile happens when it is used) *(implemented)* | +| X4 | mastodon or immich at a pinned tag | hour-scale polyglot heavyweight; manual smoke when touching the proxy or datapath, never CI | + +## Baseline (2026-07-22, M-series host, VZ guest, boot 0.6.10, dockerd 29.6.1 / BuildKit v0.31.1) + +Measured with an engine-agnostic harness (plain `docker build` timed +host-side via `DOCKER_HOST`; cold-ness via a per-build nonce file in the +context so no engine-global cache prune is needed; base images pre-pulled +outside the timed region; real-project contexts fetched at the Tier X +pins, with X2's pnpm accommodation). Single-shot numbers — trend anchors, +not gates. + +Comparison engine: Colima 0.10.3 on the same host, matched shape (18 +vCPU / 16 GiB, VZ, virtiofs, `--vz-rosetta`; its dockerd 29.5.2 — one +minor behind ArcBox's 29.6.1, same BuildKit generation). + +| shape | ArcBox | Colima | ratio | +|---|---|---|---| +| simple 3-RUN alpine build, cold | 8.1 s | 0.60 s | **13.5×** | +| same build, full cache hit | 1.8 s | 0.16 s | **11×** | +| 512 MiB + 100k-file context, cold | 26.4 s | 20.3 s | 1.3× | +| 12-stage diamond, cold | 12.7 s | 1.5 s | **8.4×** | +| `--platform linux/amd64` (FEX vs Rosetta): probe + 300k-iter shell loop | 6.4 s | 0.87 s | 7.3× | +| postgres `17/bookworm` (real apt + gpg + wget) | 75.6 s | 40.7 s | 1.9× | +| next.js `with-docker` (pnpm frozen install + `next build`) | 37.8 s | 27.1 s | 1.4× | +| caddy `2.11/builder` (apk + xcaddy fetch) | 25.2 s | 16.0 s | 1.6× | + +Reading: throughput-bound shapes (large context, real network builds) +sit at 1.3–1.9× — the datapath holds up. The 8–13× rows are all +**per-build / per-step overhead**: ~1 s per stage on ArcBox vs ~0.1 s on +Colima (multistage), and a 1.8 s floor for a fully-cached build vs +0.16 s. The bottleneck is round-trip/latency-shaped, not +bandwidth-shaped — suspects are the per-request vsock connect in the +docker proxy, session/gRPC round trips through the two-hop relay, and +per-layer snapshot-commit cost on the guest disk; profiling needed +(tracked in Linear). The amd64 row is dominated by the same overhead, +so it does NOT cleanly measure FEX-vs-Rosetta CPU speed. Pre-ABX-494 +every ArcBox row below the cache-hit line was ∞ (all buildx builds +hung). + +### 2026-07-23 re-run — after the ABX-496 fixes + +Same script, same fixtures, same host, both engines cold and measured in +the same session. ArcBox = master `e4aca033` (rcu_expedited + ext4 +metadata volume + VZ `.fsync`, boot bundle 0.6.11). Absolute numbers for +the network rows are not comparable across days (registry weather); +ratios within a session are. + +| shape | ArcBox | Colima | ratio (was) | +|---|---|---|---| +| simple 3-RUN alpine build, cold | 3.0 s | 0.95 s | 3.2× (13.5×) | +| same build, full cache hit | 0.52 s | 0.30 s | 1.7× (11×) | +| 512 MiB + 100k-file context, cold | 23.8–31.6 s | 32.9 s | **≤1× (1.3×)** | +| 12-stage diamond, cold | 5.0 s | 1.6 s | 3.0× (8.4×) | +| `--platform linux/amd64` (FEX vs Rosetta) | 2.13 s | 2.14 s | **1.0× (7.3×)** | +| postgres `17/bookworm` | 83.7 s | 67.5 s | 1.24× (1.9×) | +| next.js `with-docker` | 54.3 s | 50.3 s | 1.08× (1.4×) | +| caddy `2.11/builder` | 23.7 s | 23.8 s | **1.0× (1.6×)** | + +Reading: real-project builds, the amd64 row, and large-context are at +parity (1.0–1.24×). The residual gap is confined to the pure-overhead +shapes (3.0–3.2×, ~2 s absolute) and matches the still-open `docker +start` network-endpoint/iptables segment (~330 ms per container-backed +step, measured 416 ms vs Colima 82 ms) — the next lever, tracked +separately from ABX-496. Bench-hygiene note: Colima retains BuildKit +cache across sessions and the real-project contexts carry no nonce, so +re-runs must `docker builder prune -af` on Colima first or its +real-project rows are cache hits. + +## Bench methodology + +No criterion micro-bench: build performance is dominated by the guest and +the datapath, so the signal is the Tier-D wall-time trend lines +(`docker_build_wall`, `context_upload_wall`, cold-vs-warm ratios in D3) +recorded per run by `RunMetrics` — same trend-not-gate policy as the +workload suite. Cross-runtime comparison (OrbStack / Docker Desktop) is a +manual `xtask` run of the same fixtures against another engine's +`DOCKER_HOST`, not an automated gate. + +## Status + +The full Tier-D matrix D1–D10 is implemented +(`tests/e2e/tests/docker_build.rs`), plus Tier X X1–X3 +(`tests/e2e/tests/docker_build_external.rs`, gated on +`ARCBOX_E2E_EXTERNAL=1`, pinned upstream commits), and Tier D is +**fully green** against boot bundle ≥ 0.6.10. The one deliberate gap: +D6's fail-closed unprovisioned-FEX direction awaits an e2e knob. The suite's first-ever run +caught ABX-494 (guest FEX spun forever on BuildKit's amd64 arch-probe, +wedging the whole BuildKit Control API; fixed by boot-assets#44), and its +first green-guest run caught a latent harness bug (`run_with_timeout` +didn't drain pipes, turning any >64 KiB-output command into a bogus +timeout). On bundles < 0.6.10 the suite is red by design — do not weaken +it. Two BuildKit realities encoded in the assertions: image IDs are not +comparable across builds (`created` is re-stamped even on full cache +hits), and step logs are rate-clipped at 200 KiB/s (chatty RUNs must be +paced). + +## Phasing + +1. D1–D3 (context, stages, cache) — pure fixture work, no new capability + questions, covers the three highest-prevalence census rows. +2. D4–D5 + D9–D10 (session, streaming, exporters) — the proxy-layer + surfaces with today’s thinnest coverage. +3. D6 (FEX) once an e2e knob to assert the unprovisioned path exists; + D7–D8 alongside, sharing the concurrency fixtures. +4. Tier X wiring (pinned clones, env gate) — manual-run documentation. diff --git a/tests/e2e/src/docker.rs b/tests/e2e/src/docker.rs index 177e1b8f9..edda30083 100644 --- a/tests/e2e/src/docker.rs +++ b/tests/e2e/src/docker.rs @@ -5,6 +5,8 @@ //! via `DOCKER_HOST`, so the developer's Docker context and any host //! daemon stay untouched (see tests/e2e/README.md on isolation). +use std::io::Read as _; +use std::os::unix::process::CommandExt as _; use std::path::Path; use std::process::{Command, Stdio}; use std::thread; @@ -140,23 +142,170 @@ pub fn ensure_image(data_dir: &Path, image: &str) -> Result<()> { Err(last_err.expect("loop ran at least once")).context("docker pull (3 attempts)") } +/// SIGKILLs the process group led by `pgid`, which `process_group(0)` made +/// equal to the spawned child's pid. Safe to call after the leader has been +/// reaped: POSIX forbids reusing a pid while it is still the group id of an +/// existing group, so this either reaches that group or nothing at all +/// (`ESRCH`), never an unrelated one. +fn kill_process_group(pgid: i32) { + // SAFETY: `killpg` takes no pointers and cannot fail unsoundly; a stale + // or empty group yields ESRCH, which we ignore. + unsafe { libc::killpg(pgid, libc::SIGKILL) }; +} + /// Runs a command, killing it once `timeout` passes. +/// +/// Both pipes are drained on background threads for the whole run: an +/// undrained pipe fills at ~64 KiB and blocks the child, which turns any +/// chatty command (a `--progress=plain` build, a large pull) into a bogus +/// timeout. On a real timeout the error carries the output tail, so a +/// killed command still leaves forensics. +/// +/// The child leads its own process group and **both** exit paths signal the +/// whole group before joining. Killing just the direct child is not enough: +/// `docker build` runs the build in a `docker-buildx` grandchild that +/// inherits these pipes, so the write end stays open and the drain-thread +/// joins block past the deadline — indefinitely if that descendant is itself +/// wedged, which is exactly what this suite exists to catch. The same holds +/// when the command *succeeds* while leaving a descendant behind, so the +/// group kill is not conditional on timing out: once the direct child is +/// gone, nothing else may hold pipes this function is about to join on. pub fn run_with_timeout(command: &mut Command, timeout: Duration) -> Result { let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) + .process_group(0) .spawn()?; + // Captured before any reap: `Child::id` is not meaningful afterwards. + let pgid = i32::try_from(child.id()).expect("pid fits in i32"); + let drain = |pipe: Option>| { + thread::spawn(move || { + let mut buf = Vec::new(); + if let Some(mut pipe) = pipe { + let _ = pipe.read_to_end(&mut buf); + } + buf + }) + }; + let stdout_thread = drain( + child + .stdout + .take() + .map(|p| Box::new(p) as Box), + ); + let stderr_thread = drain( + child + .stderr + .take() + .map(|p| Box::new(p) as Box), + ); + let start = Instant::now(); while start.elapsed() < timeout { - if child.try_wait()?.is_some() { - return child - .wait_with_output() - .context("collecting command output"); + if let Some(status) = child.try_wait()? { + kill_process_group(pgid); + let stdout = stdout_thread.join().unwrap_or_default(); + let stderr = stderr_thread.join().unwrap_or_default(); + return Ok(std::process::Output { + status, + stdout, + stderr, + }); } thread::sleep(Duration::from_millis(100)); } - let _ = child.kill(); + kill_process_group(pgid); let _ = child.wait(); - Err(anyhow!("command timed out after {}s", timeout.as_secs())) + // The whole group is gone, so every inherited write end is closed and the + // drain threads see EOF. + let stdout = stdout_thread.join().unwrap_or_default(); + let stderr = stderr_thread.join().unwrap_or_default(); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + let mut tail_start = combined.len().saturating_sub(2000); + while !combined.is_char_boundary(tail_start) { + tail_start += 1; + } + Err(anyhow!( + "command timed out after {}s; output tail:\n{}", + timeout.as_secs(), + &combined[tail_start..] + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A command whose output exceeds the OS pipe buffer must complete: an + /// undrained pipe blocks the child at ~64 KiB and the old implementation + /// turned every chatty command into a bogus timeout (caught by the + /// docker_build D9 streaming scenario). + #[test] + fn chatty_command_is_drained_not_deadlocked() { + let output = run_with_timeout( + Command::new("sh").args([ + "-c", + "dd if=/dev/zero bs=1024 count=256 2>/dev/null | base64", + ]), + Duration::from_secs(20), + ) + .expect("chatty command must not time out"); + assert!(output.status.success()); + assert!(output.stdout.len() > 64 * 1024); + } + + /// The success path has the same hazard as the timeout path: a command + /// can exit promptly while leaving a descendant holding the inherited + /// pipes, and the drain-thread joins would then block on that descendant + /// with no deadline left to enforce. Returns in ~0s once the group is + /// killed, ~30s if the success branch stops doing so. + #[test] + fn success_returns_promptly_despite_surviving_descendant() { + let start = Instant::now(); + let output = run_with_timeout( + Command::new("sh").args(["-c", "echo done; sleep 30 & exit 0"]), + Duration::from_secs(60), + ) + .expect("command must succeed"); + let elapsed = start.elapsed(); + assert!(output.status.success()); + assert!( + elapsed < Duration::from_secs(10), + "success took {elapsed:?}; a descendant outlived the command and \ + held the pipes open" + ); + } + + /// A genuine timeout must surface the output tail for forensics, and + /// must return at the deadline even though the shell leaves a `sleep` + /// descendant holding the inherited pipes. Killing only the direct child + /// leaves that write end open, so the drain-thread joins block until the + /// descendant exits — ~30s here, unbounded when the survivor is a wedged + /// `docker-buildx`, which is the shape this suite hits for real. + /// + /// `sleep 30 & wait` is deliberate: with a plain `sleep 30` the shell + /// `exec`s it as the last command, so there is no grandchild and the bug + /// hides. Backgrounding forces the shell to stay alive as a real parent. + /// The elapsed bound is the regression; the tail is the original contract. + #[test] + fn timeout_returns_at_deadline_despite_surviving_descendant() { + let start = Instant::now(); + let error = run_with_timeout( + Command::new("sh").args(["-c", "echo tail-marker; sleep 30 & wait"]), + Duration::from_secs(1), + ) + .expect_err("command must time out"); + let elapsed = start.elapsed(); + assert!(error.to_string().contains("tail-marker")); + assert!( + elapsed < Duration::from_secs(10), + "timeout took {elapsed:?}; the `sleep` descendant outlived the \ + kill and held the pipes open" + ); + } } diff --git a/tests/e2e/tests/docker_build.rs b/tests/e2e/tests/docker_build.rs new file mode 100644 index 000000000..820095bbb --- /dev/null +++ b/tests/e2e/tests/docker_build.rs @@ -0,0 +1,1357 @@ +//! docker build e2e — Phases 1–3 (D1–D10) of +//! internal-docs/plans/docker-build-e2e-matrix.md. +//! +//! Where `network_workload` W14 drives *one* build to prove the datapath, +//! this suite tests the build surface itself, with scenarios modeled on the +//! real-world Dockerfile corpus in the plan: +//! +//! - **D1 large context** (immich / node_modules shape): 512 MiB +//! incompressible payload + 100k small files through the context upload, +//! with `.dockerignore` exclusion. Byte-exactness asserted *inside* the +//! build (payload sha + whole-tree sha), so truncation or corruption +//! anywhere in the transfer fails the build. +//! - **D2 stage graph** (grafana / buildkit shape): a 12-stage diamond — +//! `FROM scratch AS scripts` carrier (airflow pattern), four independent +//! branches, `COPY --from` joins, a heredoc RUN, `COPY --link`, and a +//! `FROM scratch AS export` tail, under `# syntax=docker/dockerfile:1` +//! (the external frontend 12 of 25 corpus files declare). Asserts the +//! final image carries exactly the export-stage artifact and none of the +//! builder-stage residue. +//! - **D3 cache semantics** (mastodon / next.js shape): a +//! `RUN --mount=type=cache,sharing=locked` step plus a leaf step, then +//! three rebuilds — unchanged (full cache hit: identical image ID, warm +//! wall bounded), leaf-only change (upstream stamp stable, downstream +//! re-runs), base change (both re-run, and the cache mount's content +//! demonstrably survives across builds). +//! - **D4 session channel** (authentik shape): `--secret` readable inside +//! its RUN, gone from the next layer, and absent from every byte of the +//! saved image; `--ssh` forwards a live host agent socket. Both ride the +//! `/session` upgrade proxy, previously covered by unit tests only. +//! - **D5 bind-mount lockfiles** (authentik / immich pnpm shape): +//! `RUN --mount=type=bind` consumes a context file without a COPY layer. +//! - **D6 cross-platform** (vaultwarden / moby shape): `BUILDPLATFORM` / +//! `TARGETARCH` expansion on a native build, and a `--platform +//! linux/amd64` build whose RUN executes through the binfmt translator +//! (asserted in-build via `uname -m`). +//! - **D7 concurrent builds** (CI shape): four distinct builds plus two +//! racing on one shared context, all in flight at once; unique markers +//! prove no cross-talk. +//! - **D8 cancellation** (production reality): the client is SIGKILLed +//! mid-RUN and mid-context-upload; the guest must reap the build +//! container, the daemon must stay quiet, and the next build must +//! succeed promptly. +//! - **D9 output streaming**: a chatty RUN streamed un-clipped through the +//! proxy, and `-q` mode reduced to the bare image ID. +//! - **D10 exporters**: `--output type=local` and `type=tar` — build +//! artifacts streaming host-ward through the session (reverse direction). +//! +//! All scenarios share one booted daemon; failures aggregate. The workload +//! suite's quiet-log rule applies: builds must leave no proxy-layer ERROR. + +use std::path::Path; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail}; +use arcbox_e2e::docker::{docker_host, docker_ignore, docker_output, ensure_image}; +use arcbox_e2e::metrics::RunMetrics; +use arcbox_e2e::net_fixtures::daemon_log_cursor; +use arcbox_e2e::scenario::run_vz_scenario_with_log; +use sha2::Digest; + +const READY_TIMEOUT: Duration = Duration::from_secs(180); + +/// External dockerfile frontend used by D2's `# syntax=` directive, +/// pre-loaded so the build does not depend on registry reachability. +const FRONTEND_IMAGE: &str = "docker/dockerfile:1"; + +/// D1: payload size and small-file tree shape (100 dirs × 1000 files). +const LARGE_CTX_PAYLOAD: usize = 512 * 1024 * 1024; +const LARGE_CTX_TREE_DIRS: usize = 100; +const LARGE_CTX_FILES_PER_DIR: usize = 1000; +const LARGE_CTX_DEADLINE: Duration = Duration::from_secs(300); +const LARGE_CTX_TAG: &str = "arcbox-e2e-build:large-ctx"; + +/// D2: stage-graph build deadline. +const STAGE_GRAPH_DEADLINE: Duration = Duration::from_secs(120); +const STAGE_GRAPH_TAG: &str = "arcbox-e2e-build:stage-graph"; +const STAGE_GRAPH_PROBE: &str = "arcbox-e2e-build-d2-probe"; + +/// D3: per-build deadline, the sleep that gives the cold build measurable +/// duration (two sleeping steps ⇒ cold ≥ 2× this), and the warm bound — +/// `max(cold / 10, WARM_FLOOR)`, the suite's usual CI-slack floor so a +/// fast cold build isn't judged on noise. +const CACHE_BUILD_DEADLINE: Duration = Duration::from_secs(120); +const CACHE_STEP_SLEEP_SECS: u64 = 4; +const CACHE_WARM_FLOOR: Duration = Duration::from_secs(3); + +/// D4: the secret value asserted inside the build and swept for outside it. +/// Alphanumeric+dash so it can be inlined into shell fragments verbatim. +const SECRET_VALUE: &str = "s3cret-build-token-abx494"; +const SESSION_DEADLINE: Duration = Duration::from_secs(120); +const SECRET_TAG: &str = "arcbox-e2e-build:secret"; +const SSH_TAG: &str = "arcbox-e2e-build:ssh"; + +/// D5: bind-mount lockfile scenario. +const BIND_DEADLINE: Duration = Duration::from_secs(120); +const BIND_TAG: &str = "arcbox-e2e-build:bind"; + +/// D9: chatty-RUN shape. BuildKit rate-limits step logs (the embedded +/// builder clips at 200 KiB/s — measured, the marker names the limit), so +/// the emitter is PACED: `STREAM_CHUNKS` bursts of `STREAM_CHUNK_LINES` +/// lines with a 1 s sleep between (~68 KiB/s). Under the limit, "no clip +/// marker + tail lines present" is a fair streaming-integrity assertion; +/// a bulk `seq` would be clipped by design and prove nothing. +const STREAM_CHUNKS: usize = 5; +const STREAM_CHUNK_LINES: usize = 10_000; +const STREAM_DEADLINE: Duration = Duration::from_secs(120); +const STREAM_TAG: &str = "arcbox-e2e-build:stream"; +const QUIET_TAG: &str = "arcbox-e2e-build:quiet"; + +/// D10: exporter scenario — artifacts stream host-ward through the session. +const EXPORT_DEADLINE: Duration = Duration::from_secs(120); +const EXPORT_PAYLOAD: &str = "exporter-proof-payload\n"; + +/// D6: cross-platform args + the amd64 binfmt translator. FEX ships in +/// every bundle ≥ 0.6.6, so only the provisioned direction is asserted; +/// the fail-closed unprovisioned direction awaits an e2e knob (see the +/// plan's phasing note). +const XPLAT_DEADLINE: Duration = Duration::from_secs(180); +const XPLAT_ARGS_TAG: &str = "arcbox-e2e-build:xplat-args"; +const XPLAT_AMD64_TAG: &str = "arcbox-e2e-build:xplat-amd64"; + +/// D7: concurrent builds — distinct contexts plus two racing on one shared +/// context. +const CONCURRENT_DISTINCT: usize = 4; +const CONCURRENT_SHARED: usize = 2; +const CONCURRENT_DEADLINE: Duration = Duration::from_secs(180); + +/// D8: cancellation probes — the mid-upload kill needs a payload big +/// enough that the transfer is still in flight when the client dies. +const CANCEL_UPLOAD_PAYLOAD: usize = 256 * 1024 * 1024; +const CANCEL_DEADLINE: Duration = Duration::from_secs(120); + +#[test] +#[ignore = "boots a VZ System VM through a real daemon; run on the e2e runner"] +fn docker_build_suite() -> Result<()> { + run_vz_scenario_with_log("docker_build", "info", |daemon, data_dir, metrics| { + metrics.time("daemon_ready", || daemon.wait_ready_blocking(READY_TIMEOUT))?; + let image = + std::env::var("ARCBOX_E2E_IMAGE").unwrap_or_else(|_| "alpine:latest".to_owned()); + metrics.time("docker_pull", || ensure_image(data_dir, &image))?; + metrics.time("frontend_pull", || ensure_image(data_dir, FRONTEND_IMAGE))?; + + // Taken after setup so image-pull noise is out of scope; covers + // every build below. + let log = daemon_log_cursor(data_dir); + + let scenarios: [(&str, ScenarioFn); 10] = [ + ("stage_graph", stage_graph), + ("cache_semantics", cache_semantics), + ("session_secret_ssh", session_secret_ssh), + ("bind_mounts", bind_mounts), + ("output_streaming", output_streaming), + ("exporters", exporters), + ("cross_platform", cross_platform), + ("concurrent_builds", concurrent_builds), + ("cancellation", cancellation), + ("large_context", large_context), + ]; + // Diagnostic filter: run only the named scenario, e.g. + // ARCBOX_E2E_BUILD_ONLY=cache_semantics. + let only = std::env::var("ARCBOX_E2E_BUILD_ONLY").ok(); + let mut failures = Vec::new(); + for (name, scenario) in scenarios { + if let Some(ref only) = only + && only != name + { + continue; + } + tracing::info!(scenario = name, "starting"); + match scenario(data_dir, metrics, &image) { + Ok(()) => tracing::info!(scenario = name, "passed"), + Err(error) => { + tracing::warn!(scenario = name, "failed: {error:#}"); + failures.push(format!("{name}: {error:#}")); + } + } + } + + match log.proxy_errors() { + Ok(errors) if !errors.is_empty() => failures.push(format!( + "quiet-log: {} proxy-layer ERROR line(s) during builds:\n{}", + errors.len(), + errors.join("\n") + )), + Ok(_) => {} + Err(error) => failures.push(format!("quiet-log: unreadable: {error:#}")), + } + + if failures.is_empty() { + Ok(()) + } else { + bail!( + "{} of {} build checks failed:\n{}", + failures.len(), + scenarios.len() + 1, + failures.join("\n---\n") + ) + } + }) +} + +type ScenarioFn = fn(&Path, &mut RunMetrics, &str) -> Result<()>; + +/// Runs `docker build --progress=plain -t tag ctx`, recording +/// the wall under `key` (also for failed builds — a slow failure is still +/// trend data) and failing past `deadline`. +fn timed_build( + data_dir: &Path, + metrics: &mut RunMetrics, + key: &str, + ctx: &Path, + tag: &str, + extra_args: &[&str], + deadline: Duration, +) -> Result<(String, Duration)> { + let ctx_arg = ctx.display().to_string(); + let mut args = vec!["build", "--progress=plain", "-t", tag]; + args.extend_from_slice(extra_args); + args.push(&ctx_arg); + let started = Instant::now(); + let build = docker_output(data_dir, &args, deadline + Duration::from_secs(60)); + let elapsed = started.elapsed(); + metrics.record(key, elapsed.as_secs_f64()); + let output = build.with_context(|| format!("{key}: docker build"))?; + if elapsed >= deadline { + bail!("{key}: build took {elapsed:?} (>= {deadline:?})"); + } + Ok((output, elapsed)) +} + +/// `cat`s one file out of a built image via a throwaway container. +fn image_file(data_dir: &Path, tag: &str, path: &str) -> Result { + docker_output( + data_dir, + &["run", "--rm", tag, "cat", path], + Duration::from_secs(60), + ) + .map(|out| out.trim().to_owned()) + .with_context(|| format!("reading {path} from {tag}")) +} + +/// Extracts BuildKit's own context-transfer timing from `--progress=plain` +/// output (`#N transferring context: s done`). Best-effort: +/// the line format is BuildKit's, not ours, so a miss records nothing. +fn context_transfer_seconds(output: &str) -> Option { + output + .lines() + .rfind(|line| line.contains("transferring context:") && line.trim_end().ends_with("done")) + .and_then(|line| { + line.split_whitespace() + .rev() + .filter(|token| token.len() > 1 && token.ends_with('s')) + .find_map(|token| token.trim_end_matches('s').parse::().ok()) + }) +} + +/// Fills `buf` with xorshift output — incompressible, so context transfer +/// and layer commits move real bytes. +fn fill_incompressible(buf: &mut [u8]) { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + for chunk in buf.chunks_mut(8) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + chunk.copy_from_slice(&state.to_le_bytes()[..chunk.len()]); + } +} + +/// D2: 12-stage diamond under the external `docker/dockerfile:1` frontend. +/// Four independent branches (one fed from a `FROM scratch` script-carrier +/// stage, one built through a heredoc RUN) join into an aggregate, and a +/// `FROM scratch AS export` tail carries exactly one artifact. The final +/// image must contain that artifact byte-exact and none of the builder +/// residue (`/sentinel.txt`, parts, intermediates). +fn stage_graph(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d2-ctx"); + std::fs::create_dir_all(&ctx).context("creating d2 context dir")?; + std::fs::write(ctx.join("make-part.sh"), "echo \"part-$1\"\n")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"# syntax=docker/dockerfile:1 +FROM scratch AS scripts +COPY make-part.sh /make-part.sh + +FROM {image} AS base +RUN echo shared > /shared.txt + +FROM base AS branch-a +COPY --from=scripts /make-part.sh /make-part.sh +RUN sh /make-part.sh a > /part.txt && echo builder-only > /sentinel.txt + +FROM base AS branch-a-out +COPY --from=branch-a /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS branch-b +RUN < /part.txt +echo b-extra >> /part.txt +EOT + +FROM base AS branch-b-out +COPY --from=branch-b /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS branch-c +RUN echo part-c > /part.txt + +FROM base AS branch-c-out +COPY --from=branch-c /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS branch-d +RUN echo part-d > /part.txt + +FROM base AS branch-d-out +COPY --from=branch-d /part.txt /part.txt +RUN cat /shared.txt /part.txt > /out.txt + +FROM base AS join +COPY --from=branch-a-out /out.txt /join/out-a.txt +COPY --from=branch-b-out /out.txt /join/out-b.txt +COPY --from=branch-c-out /out.txt /join/out-c.txt +COPY --from=branch-d-out /out.txt /join/out-d.txt +RUN cat /join/out-a.txt /join/out-b.txt /join/out-c.txt /join/out-d.txt > /artifact.txt + +FROM scratch AS export +COPY --link --from=join /artifact.txt /artifact.txt +" + ), + ) + .context("writing d2 Dockerfile")?; + let expected_artifact = + "shared\npart-a\nshared\npart-b\nb-extra\nshared\npart-c\nshared\npart-d\n"; + + let result = (|| { + timed_build( + data_dir, + metrics, + "stage_graph_build_wall", + &ctx, + STAGE_GRAPH_TAG, + &[], + STAGE_GRAPH_DEADLINE, + )?; + + // A scratch image cannot run; probe it via a created (never + // started) container. + docker_output( + data_dir, + &[ + "create", + "--name", + STAGE_GRAPH_PROBE, + STAGE_GRAPH_TAG, + "/noop", + ], + Duration::from_secs(60), + ) + .context("creating probe container")?; + + let artifact_host = data_dir.join("d2-artifact.txt"); + let artifact_arg = artifact_host.display().to_string(); + docker_output( + data_dir, + &[ + "cp", + &format!("{STAGE_GRAPH_PROBE}:/artifact.txt"), + &artifact_arg, + ], + Duration::from_secs(60), + ) + .context("copying artifact out of the built image")?; + let artifact = std::fs::read_to_string(&artifact_host)?; + if artifact != expected_artifact { + bail!( + "stage_graph: artifact mismatch:\n{artifact:?}\nexpected:\n{expected_artifact:?}" + ); + } + + // Full-rootfs sweep for builder residue: the export listing may + // carry init-layer entries (/dev, /etc/hosts, …) but must not + // carry anything from the builder stages. + let export_tar = data_dir.join("d2-export.tar"); + let export_arg = export_tar.display().to_string(); + docker_output( + data_dir, + &["export", "-o", &export_arg, STAGE_GRAPH_PROBE], + Duration::from_secs(120), + ) + .context("exporting probe container")?; + let listing = std::process::Command::new("tar") + .args(["-tf", &export_arg]) + .output() + .context("listing export tar")?; + if !listing.status.success() { + bail!( + "stage_graph: tar -tf failed: {}", + String::from_utf8_lossy(&listing.stderr) + ); + } + let names = String::from_utf8_lossy(&listing.stdout); + if !names.lines().any(|l| l.ends_with("artifact.txt")) { + bail!("stage_graph: artifact.txt missing from export listing:\n{names}"); + } + for residue in ["sentinel", "part", "shared", "out-", "make-"] { + if let Some(hit) = names.lines().find(|l| l.contains(residue)) { + bail!("stage_graph: builder-stage residue {hit:?} leaked into the final image"); + } + } + Ok(()) + })(); + + docker_ignore( + data_dir, + &["rm".into(), "-f".into(), STAGE_GRAPH_PROBE.into()], + ); + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), STAGE_GRAPH_TAG.into()], + ); + result +} + +/// D3: cache semantics across four builds of one Dockerfile. Each RUN step +/// writes a per-execution random stamp, so "was this step re-executed?" is +/// read from the image itself rather than parsed out of progress output. +/// The cached step's `--mount=type=cache` also round-trips a token, proving +/// the cache mount's content survives across builds. +fn cache_semantics(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d3-ctx"); + std::fs::create_dir_all(&ctx).context("creating d3 context dir")?; + std::fs::write(ctx.join("base.txt"), "base-v1\n")?; + std::fs::write(ctx.join("leaf.txt"), "leaf-v1\n")?; + let sleep = CACHE_STEP_SLEEP_SECS; + let stamp = "head -c 16 /dev/urandom | sha256sum | cut -d' ' -f1"; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} +COPY base.txt /base.txt +RUN --mount=type=cache,target=/build-cache,sharing=locked (cp /build-cache/token /cache-prev 2>/dev/null || : > /cache-prev) && cp /base.txt /build-cache/token && sleep {sleep} && {stamp} > /stamp-base +COPY leaf.txt /leaf.txt +RUN sleep {sleep} && {stamp} > /stamp-leaf +" + ), + ) + .context("writing d3 Dockerfile")?; + + let tags = ["cold", "warm", "leaf", "base"].map(|t| format!("arcbox-e2e-build:cache-{t}")); + let [cold_tag, warm_tag, leaf_tag, base_tag] = &tags; + + let result = (|| { + // Build 1 — cold: everything executes, the cache mount is seeded. + let (_, cold_wall) = timed_build( + data_dir, + metrics, + "cache_cold_wall", + &ctx, + cold_tag, + &[], + CACHE_BUILD_DEADLINE, + )?; + let cold_stamp_base = image_file(data_dir, cold_tag, "/stamp-base")?; + let cold_stamp_leaf = image_file(data_dir, cold_tag, "/stamp-leaf")?; + let cold_cache_prev = image_file(data_dir, cold_tag, "/cache-prev")?; + if cold_stamp_base.is_empty() || cold_stamp_leaf.is_empty() { + bail!("cache: cold build produced empty stamps"); + } + if !cold_cache_prev.is_empty() { + bail!( + "cache: cold build saw a pre-existing cache token {cold_cache_prev:?} — \ + the fresh daemon's build cache was not empty" + ); + } + + // Build 2 — unchanged: a full cache hit. Image IDs are NOT compared: + // BuildKit stamps a fresh `created` timestamp into the config on + // every build, so even a fully-cached rebuild gets a new ID. The + // per-execution stamps are the execution truth — both must match + // the cold build's — plus a wall bound far below cold (with the + // usual CI-slack floor). + let (_, warm_wall) = timed_build( + data_dir, + metrics, + "cache_warm_wall", + &ctx, + warm_tag, + &[], + CACHE_BUILD_DEADLINE, + )?; + let warm_stamp_base = image_file(data_dir, warm_tag, "/stamp-base")?; + let warm_stamp_leaf = image_file(data_dir, warm_tag, "/stamp-leaf")?; + if warm_stamp_base != cold_stamp_base || warm_stamp_leaf != cold_stamp_leaf { + bail!("cache: unchanged rebuild re-executed a step (stamps changed)"); + } + let warm_bound = cold_wall.div_f64(10.0).max(CACHE_WARM_FLOOR); + if warm_wall > warm_bound { + bail!( + "cache: unchanged rebuild took {warm_wall:?} (bound {warm_bound:?}, \ + cold {cold_wall:?}) — cache hit not engaging" + ); + } + + // Build 3 — leaf change: the step above the change stays cached, + // the steps at and below it re-execute. + std::fs::write(ctx.join("leaf.txt"), "leaf-v2\n")?; + timed_build( + data_dir, + metrics, + "cache_leaf_wall", + &ctx, + leaf_tag, + &[], + CACHE_BUILD_DEADLINE, + )?; + let leaf_stamp_base = image_file(data_dir, leaf_tag, "/stamp-base")?; + let leaf_stamp_leaf = image_file(data_dir, leaf_tag, "/stamp-leaf")?; + if leaf_stamp_base != cold_stamp_base { + bail!("cache: leaf-only change re-executed the upstream cached step"); + } + if leaf_stamp_leaf == cold_stamp_leaf { + bail!("cache: leaf change did not re-execute the downstream step"); + } + + // Build 4 — base change: both steps re-execute, and the re-run + // cached step reads back the token build 1 wrote — the cache mount + // persisted across builds. + std::fs::write(ctx.join("base.txt"), "base-v2\n")?; + timed_build( + data_dir, + metrics, + "cache_base_wall", + &ctx, + base_tag, + &[], + CACHE_BUILD_DEADLINE, + )?; + let base_stamp_base = image_file(data_dir, base_tag, "/stamp-base")?; + let base_stamp_leaf = image_file(data_dir, base_tag, "/stamp-leaf")?; + let base_cache_prev = image_file(data_dir, base_tag, "/cache-prev")?; + if base_stamp_base == cold_stamp_base { + bail!("cache: base change did not re-execute the cached step"); + } + if base_stamp_leaf == leaf_stamp_leaf { + bail!("cache: base change did not invalidate the downstream step"); + } + if base_cache_prev != "base-v1" { + bail!( + "cache: cache-mount token did not survive across builds \ + (read {base_cache_prev:?}, expected \"base-v1\")" + ); + } + Ok(()) + })(); + + for tag in &tags { + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.clone()]); + } + result +} + +/// D1: 512 MiB incompressible payload plus a 100k-file tree through the +/// context upload, `.dockerignore` honored. All integrity assertions run +/// *inside* the build: the payload sha, the file count, and a whole-tree +/// sha computed in deterministic path order, so a single lost, truncated, +/// or corrupted entry anywhere in the transfer fails the build. +fn large_context(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d1-ctx"); + std::fs::create_dir_all(&ctx).context("creating d1 context dir")?; + + let generated = Instant::now(); + let mut payload = vec![0u8; LARGE_CTX_PAYLOAD]; + fill_incompressible(&mut payload); + std::fs::write(ctx.join("payload.bin"), &payload).context("writing payload")?; + let payload_sha = format!("{:x}", sha2::Sha256::digest(&payload)); + drop(payload); + + // Zero-padded names make guest-side `find . | sort` order equal this + // generation order, so one incremental host hash matches the in-build + // whole-tree hash. + let mut tree_hasher = sha2::Sha256::new(); + for dir in 0..LARGE_CTX_TREE_DIRS { + let dir_name = format!("d{dir:03}"); + let dir_path = ctx.join("tree").join(&dir_name); + std::fs::create_dir_all(&dir_path)?; + for file in 0..LARGE_CTX_FILES_PER_DIR { + let content = format!("{dir_name}/f{file:03}\n"); + std::fs::write(dir_path.join(format!("f{file:03}")), &content)?; + tree_hasher.update(content.as_bytes()); + } + } + let tree_sha = format!("{:x}", tree_hasher.finalize()); + let file_count = LARGE_CTX_TREE_DIRS * LARGE_CTX_FILES_PER_DIR; + + std::fs::write(ctx.join(".dockerignore"), "excluded/\n")?; + std::fs::create_dir_all(ctx.join("excluded"))?; + std::fs::write(ctx.join("excluded/marker.txt"), "must-not-transfer\n")?; + tracing::info!( + elapsed = ?generated.elapsed(), + payload_mib = LARGE_CTX_PAYLOAD / (1024 * 1024), + file_count, + "large_context: context generated" + ); + + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +COPY . /ctx +RUN echo "{payload_sha} /ctx/payload.bin" | sha256sum -c - +RUN test "$(find /ctx/tree -type f | wc -l)" -eq {file_count} +RUN cd /ctx/tree && test "$(find . -type f | sort | xargs cat | sha256sum | cut -d' ' -f1)" = "{tree_sha}" +RUN test ! -e /ctx/excluded +"# + ), + ) + .context("writing d1 Dockerfile")?; + + let result = (|| { + let (output, elapsed) = timed_build( + data_dir, + metrics, + "large_context_build_wall", + &ctx, + LARGE_CTX_TAG, + &[], + LARGE_CTX_DEADLINE, + )?; + match context_transfer_seconds(&output) { + Some(seconds) => metrics.record("large_context_transfer", seconds), + None => tracing::info!("large_context: no transfer timing in build output"), + } + tracing::info!(?elapsed, "large_context: build done"); + Ok(()) + })(); + + docker_ignore(data_dir, &["rmi".into(), "-f".into(), LARGE_CTX_TAG.into()]); + result +} + +/// A throwaway host `ssh-agent` on a private socket, killed on drop. The +/// agent carries one fresh ed25519 key so the forwarded socket is a +/// realistic one, not an empty stub. +struct ThrowawaySshAgent { + pid: String, + sock: std::path::PathBuf, + /// `SHA256:…` fingerprint of the loaded key. The build asserts the + /// forwarded agent reports exactly this, which only a working + /// round-trip can produce. + fingerprint: String, +} + +impl ThrowawaySshAgent { + fn spawn(dir: &Path) -> Result { + let sock = dir.join("d4-agent.sock"); + let output = std::process::Command::new("ssh-agent") + .args(["-a", &sock.display().to_string()]) + .output() + .context("spawning ssh-agent")?; + if !output.status.success() { + bail!( + "ssh-agent failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let pid = stdout + .split("SSH_AGENT_PID=") + .nth(1) + .and_then(|rest| rest.split(';').next()) + .map(str::to_owned) + .ok_or_else(|| anyhow::anyhow!("no SSH_AGENT_PID in ssh-agent output: {stdout}"))?; + let key = dir.join("d4-ssh-key"); + let keygen = std::process::Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&key) + .output() + .context("generating throwaway ssh key")?; + if !keygen.status.success() { + bail!( + "ssh-keygen failed: {}", + String::from_utf8_lossy(&keygen.stderr) + ); + } + let add = std::process::Command::new("ssh-add") + .arg(&key) + .env("SSH_AUTH_SOCK", &sock) + .output() + .context("loading key into ssh-agent")?; + if !add.status.success() { + bail!("ssh-add failed: {}", String::from_utf8_lossy(&add.stderr)); + } + + // `ssh-keygen -lf` prints " SHA256: (ED25519)"; + // the fingerprint token is what `ssh-add -l` echoes back inside the + // build, so the two are directly comparable. + let pub_key = key.with_extension("pub"); + let show = std::process::Command::new("ssh-keygen") + .arg("-lf") + .arg(&pub_key) + .output() + .context("fingerprinting throwaway ssh key")?; + if !show.status.success() { + bail!( + "ssh-keygen -lf failed: {}", + String::from_utf8_lossy(&show.stderr) + ); + } + let listing = String::from_utf8_lossy(&show.stdout); + let fingerprint = listing + .split_whitespace() + .find(|token| token.starts_with("SHA256:")) + .map(str::to_owned) + .ok_or_else(|| anyhow::anyhow!("no SHA256 fingerprint in: {listing}"))?; + + Ok(Self { + pid, + sock, + fingerprint, + }) + } +} + +impl Drop for ThrowawaySshAgent { + fn drop(&mut self) { + let _ = std::process::Command::new("kill").arg(&self.pid).status(); + } +} + +/// Sweeps every byte of `docker save ` for `needle` — including inside +/// gzip'd layer blobs (the containerd image store compresses them). A hit +/// anywhere fails: build secrets must never persist into image content. +fn assert_absent_from_saved_image( + data_dir: &Path, + tag: &str, + needle: &str, + label: &str, +) -> Result<()> { + let tar = data_dir.join(format!("{label}-save.tar")); + let tar_arg = tar.display().to_string(); + docker_output( + data_dir, + &["save", "-o", &tar_arg, tag], + Duration::from_secs(120), + ) + .context("docker save for secret sweep")?; + + // No `X && echo LEAK` shorthand under `set -e`: a non-matching grep + // returns 1, fails the AND-list, and aborts the sweep before + // SWEEP-DONE — every clean image would read as an incomplete sweep. + let script = format!( + r#"set -e +d=$(mktemp -d) +trap 'rm -rf "$d"' EXIT +tar -xf "{tar_arg}" -C "$d" +find "$d" -type f | while read -r f; do + if [ "$(head -c 2 "$f" | od -An -tx1 | tr -d ' \n')" = "1f8b" ]; then + if gunzip -c "$f" 2>/dev/null | grep -aq -- "{needle}"; then echo "LEAK:$f"; fi + elif grep -aq -- "{needle}" "$f"; then + echo "LEAK:$f" + fi +done +echo SWEEP-DONE"# + ); + let sweep = std::process::Command::new("sh") + .args(["-c", &script]) + .output() + .context("running secret sweep")?; + let stdout = String::from_utf8_lossy(&sweep.stdout); + if !stdout.contains("SWEEP-DONE") { + bail!( + "{label}: secret sweep did not complete: {stdout}{}", + String::from_utf8_lossy(&sweep.stderr) + ); + } + if stdout.contains("LEAK:") { + bail!("{label}: secret bytes leaked into the saved image:\n{stdout}"); + } + Ok(()) +} + +/// D4: the BuildKit session channel — `--secret` and `--ssh` both ride the +/// `/session` HTTP upgrade through the proxy. The secret must be readable +/// inside its mounting RUN, gone from the next layer, and absent from every +/// byte of the exported image; the ssh mount must expose a live forwarded +/// agent socket. +fn session_secret_ssh(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d4-secret-ctx"); + std::fs::create_dir_all(&ctx).context("creating d4 secret context dir")?; + std::fs::write(ctx.join("token.txt"), SECRET_VALUE)?; + // The Dockerfile must carry only the HASH of the secret: RUN command + // lines persist verbatim into the image config's history, so inlining + // the value would plant exactly the leak the sweep below hunts for. + let secret_sha = format!("{:x}", sha2::Sha256::digest(SECRET_VALUE.as_bytes())); + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +RUN --mount=type=secret,id=build_token echo "{secret_sha} /run/secrets/build_token" | sha256sum -c - && echo secret-visible > /probe +RUN test ! -e /run/secrets/build_token +"# + ), + ) + .context("writing d4 secret Dockerfile")?; + + // Absolute path: BuildKit resolves a relative `src` against the CLI's + // working directory, not the build context. + let secret_src = format!("id=build_token,src={}", ctx.join("token.txt").display()); + let secret_result = (|| { + timed_build( + data_dir, + metrics, + "session_secret_build_wall", + &ctx, + SECRET_TAG, + &["--secret", &secret_src], + SESSION_DEADLINE, + )?; + if image_file(data_dir, SECRET_TAG, "/probe")? != "secret-visible" { + bail!("secret: probe layer missing"); + } + assert_absent_from_saved_image(data_dir, SECRET_TAG, SECRET_VALUE, "session_secret") + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), SECRET_TAG.into()]); + secret_result?; + + let agent = ThrowawaySshAgent::spawn(data_dir)?; + let ctx = data_dir.join("d4-ssh-ctx"); + std::fs::create_dir_all(&ctx).context("creating d4 ssh context dir")?; + // `test -S` only stats the socket inode BuildKit created — it passes even + // when no agent traffic survives the crossing, which is the half that can + // actually break. Query the agent instead and require the throwaway key's + // own fingerprint back: that answer can only come from a completed + // request/response round-trip over the forwarded socket. + // + // `apk` is the one image assumption in this suite; it holds for the + // default `alpine:latest` and any alpine mirror `ARCBOX_E2E_IMAGE` points + // at. A non-alpine override fails here loudly rather than silently + // weakening the check. + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +RUN --mount=type=ssh set -e; \ + test -S "$SSH_AUTH_SOCK"; \ + apk add --no-cache openssh-client >/dev/null; \ + ssh-add -l | tee /probe-identities; \ + grep -q '{fingerprint}' /probe-identities; \ + echo ssh-forwarded > /probe +"#, + fingerprint = agent.fingerprint, + ), + ) + .context("writing d4 ssh Dockerfile")?; + + let ssh_arg = format!("default={}", agent.sock.display()); + let ssh_result = (|| { + timed_build( + data_dir, + metrics, + "session_ssh_build_wall", + &ctx, + SSH_TAG, + &["--ssh", &ssh_arg], + SESSION_DEADLINE, + )?; + if image_file(data_dir, SSH_TAG, "/probe")? != "ssh-forwarded" { + bail!("ssh: probe layer missing"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), SSH_TAG.into()]); + ssh_result +} + +/// D5: `RUN --mount=type=bind` consumes a context file with no COPY layer — +/// the pnpm-lockfile shape. The mounted file must be readable in its RUN, +/// derived content must land in the image, and the mount itself must leave +/// no trace in the final filesystem. +fn bind_mounts(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d5-ctx"); + std::fs::create_dir_all(&ctx).context("creating d5 context dir")?; + let lockfile = "lockfile-v1\n"; + std::fs::write(ctx.join("lockfile.txt"), lockfile)?; + let lockfile_sha = format!("{:x}", sha2::Sha256::digest(lockfile.as_bytes())); + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} +RUN --mount=type=bind,source=lockfile.txt,target=/lockfile.txt sha256sum /lockfile.txt | cut -d' ' -f1 > /derived +RUN test ! -e /lockfile.txt +" + ), + ) + .context("writing d5 Dockerfile")?; + + let result = (|| { + timed_build( + data_dir, + metrics, + "bind_mounts_build_wall", + &ctx, + BIND_TAG, + &[], + BIND_DEADLINE, + )?; + let derived = image_file(data_dir, BIND_TAG, "/derived")?; + if derived != lockfile_sha { + bail!("bind_mounts: derived hash {derived} != host hash {lockfile_sha}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), BIND_TAG.into()]); + result +} + +/// D6: cross-platform builds. (a) A native build expands +/// `--platform=$BUILDPLATFORM` and `TARGETARCH` correctly. (b) A +/// `--platform linux/amd64` build executes its RUN step through the binfmt +/// translator — asserted in-build via `uname -m`, plus the image's +/// recorded architecture. This is the surface ABX-494 wedged: the arch +/// probe behind it now runs on every ListWorkers. +fn cross_platform(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d6-args-ctx"); + std::fs::create_dir_all(&ctx).context("creating d6 args context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM --platform=$BUILDPLATFORM {image} AS build +ARG BUILDPLATFORM +ARG TARGETARCH +RUN echo "$BUILDPLATFORM:$TARGETARCH" > /args.txt +FROM {image} +COPY --from=build /args.txt /args.txt +"# + ), + ) + .context("writing d6 args Dockerfile")?; + + let args_result = (|| { + timed_build( + data_dir, + metrics, + "xplat_args_build_wall", + &ctx, + XPLAT_ARGS_TAG, + &[], + XPLAT_DEADLINE, + )?; + let args = image_file(data_dir, XPLAT_ARGS_TAG, "/args.txt")?; + if args != "linux/arm64:arm64" { + bail!("cross_platform: platform args expanded to {args:?}, expected linux/arm64:arm64"); + } + Ok(()) + })(); + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), XPLAT_ARGS_TAG.into()], + ); + args_result?; + + // The amd64 base variant is not in the arm64 tar cache; pull it with + // the same retry discipline as `ensure_image`. + let mut last_err = None; + for attempt in 1..=3 { + match docker_output( + data_dir, + &["pull", "--platform", "linux/amd64", image], + Duration::from_secs(90), + ) { + Ok(_) => { + last_err = None; + break; + } + Err(e) => { + tracing::warn!(attempt, "amd64 variant pull failed: {e:#}"); + last_err = Some(e); + } + } + } + if let Some(e) = last_err { + return Err(e).context("pulling amd64 base variant (3 attempts)"); + } + + let ctx = data_dir.join("d6-amd64-ctx"); + std::fs::create_dir_all(&ctx).context("creating d6 amd64 context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r#"FROM {image} +RUN uname -m > /arch && test "$(cat /arch)" = "x86_64" +"# + ), + ) + .context("writing d6 amd64 Dockerfile")?; + + let amd64_result = (|| { + timed_build( + data_dir, + metrics, + "xplat_amd64_build_wall", + &ctx, + XPLAT_AMD64_TAG, + &["--platform", "linux/amd64"], + XPLAT_DEADLINE, + )?; + let arch = docker_output( + data_dir, + &[ + "image", + "inspect", + "-f", + "{{.Architecture}}", + XPLAT_AMD64_TAG, + ], + Duration::from_secs(30), + )?; + if arch.trim() != "amd64" { + bail!( + "cross_platform: built image records architecture {:?}, expected amd64", + arch.trim() + ); + } + Ok(()) + })(); + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), XPLAT_AMD64_TAG.into()], + ); + amd64_result +} + +/// D7: concurrent builds — the CI shape. Four builds with distinct +/// contexts plus two racing on one shared context all run at once; each +/// image must carry exactly its own marker (no cross-talk between build +/// sessions sharing the daemon and builder). +fn concurrent_builds(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let mut jobs: Vec<(String, std::path::PathBuf, String)> = Vec::new(); + for i in 0..CONCURRENT_DISTINCT { + let ctx = data_dir.join(format!("d7-ctx-{i}")); + std::fs::create_dir_all(&ctx)?; + let marker = format!("marker-{i}"); + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nRUN sleep 2 && echo {marker} > /marker\n"), + )?; + jobs.push((format!("arcbox-e2e-build:conc-{i}"), ctx, marker)); + } + let shared = data_dir.join("d7-ctx-shared"); + std::fs::create_dir_all(&shared)?; + std::fs::write( + shared.join("Dockerfile"), + format!("FROM {image}\nRUN sleep 2 && echo marker-shared > /marker\n"), + )?; + for j in 0..CONCURRENT_SHARED { + jobs.push(( + format!("arcbox-e2e-build:conc-shared-{j}"), + shared.clone(), + "marker-shared".to_owned(), + )); + } + + let started = Instant::now(); + let results: Vec<(String, Result)> = std::thread::scope(|scope| { + #[expect( + clippy::needless_collect, + reason = "collect forces every spawn before the first join; a lazy chain would run the builds serially" + )] + let handles: Vec<_> = jobs + .iter() + .map(|(tag, ctx, _)| { + scope.spawn(move || { + let ctx_arg = ctx.display().to_string(); + docker_output( + data_dir, + &["build", "-t", tag, &ctx_arg], + CONCURRENT_DEADLINE, + ) + }) + }) + .collect(); + handles + .into_iter() + .zip(&jobs) + .map(|(handle, (tag, _, _))| { + ( + tag.clone(), + handle + .join() + .unwrap_or_else(|_| Err(anyhow!("build thread panicked"))), + ) + }) + .collect() + }); + let elapsed = started.elapsed(); + metrics.record("concurrent_builds_wall", elapsed.as_secs_f64()); + + let result = (|| { + for (tag, result) in &results { + if let Err(error) = result { + bail!("concurrent: {tag} failed: {error:#}"); + } + } + if elapsed >= CONCURRENT_DEADLINE { + bail!("concurrent: builds took {elapsed:?} (>= {CONCURRENT_DEADLINE:?})"); + } + for (tag, _, marker) in &jobs { + let got = image_file(data_dir, tag, "/marker")?; + if got != *marker { + bail!("concurrent: {tag} carries marker {got:?}, expected {marker:?} — cross-talk"); + } + } + Ok(()) + })(); + for (tag, _, _) in &jobs { + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.clone()]); + } + result +} + +/// Spawns a `docker build` that the caller intends to kill — output goes +/// to /dev/null since nothing will ever collect it. +fn spawn_build(data_dir: &Path, ctx: &Path, tag: &str) -> Result { + std::process::Command::new("docker") + .env("DOCKER_HOST", docker_host(data_dir)) + .args(["build", "-t", tag]) + .arg(ctx) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .context("spawning cancellable docker build") +} + +/// D8: cancellation — the client is SIGKILLed mid-RUN and mid-context- +/// upload. The guest must reap the cancelled RUN's process, and the next +/// build must succeed promptly. The suite-level quiet-log check asserts +/// the daemon rode both kills without proxy ERRORs. +fn cancellation(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + // (1) Kill mid-RUN: the sleep marker is uniquely greppable guest-side. + let ctx = data_dir.join("d8-run-ctx"); + std::fs::create_dir_all(&ctx).context("creating d8 run context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nRUN sleep 6543\n"), + )?; + let mut child = spawn_build(data_dir, &ctx, "arcbox-e2e-build:cancel-run")?; + std::thread::sleep(Duration::from_secs(8)); + let _ = child.kill(); + let _ = child.wait(); + + let mut reaped = false; + for _ in 0..15 { + let count = docker_output( + data_dir, + &[ + "run", + "--rm", + "--pid=host", + image, + "sh", + "-c", + "ps | grep 'sleep [6]543' | wc -l", + ], + Duration::from_secs(60), + ) + .context("guest process sweep")?; + if count.trim() == "0" { + reaped = true; + break; + } + std::thread::sleep(Duration::from_secs(2)); + } + if !reaped { + bail!("cancellation: the cancelled RUN's process is still alive in the guest"); + } + + // (2) Kill mid-context-upload: 256 MiB is several seconds of transfer. + let ctx = data_dir.join("d8-upload-ctx"); + std::fs::create_dir_all(&ctx).context("creating d8 upload context dir")?; + let mut payload = vec![0u8; CANCEL_UPLOAD_PAYLOAD]; + fill_incompressible(&mut payload); + std::fs::write(ctx.join("payload.bin"), &payload).context("writing d8 payload")?; + drop(payload); + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nCOPY payload.bin /payload.bin\n"), + )?; + let mut child = spawn_build(data_dir, &ctx, "arcbox-e2e-build:cancel-upload")?; + std::thread::sleep(Duration::from_millis(1500)); + let _ = child.kill(); + let _ = child.wait(); + + // (3) The daemon must build normally right after both kills. + let ctx = data_dir.join("d8-followup-ctx"); + std::fs::create_dir_all(&ctx).context("creating d8 followup context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!("FROM {image}\nRUN echo recovered > /marker\n"), + )?; + let result = (|| { + timed_build( + data_dir, + metrics, + "cancel_followup_wall", + &ctx, + "arcbox-e2e-build:cancel-followup", + &[], + CANCEL_DEADLINE, + )?; + if image_file(data_dir, "arcbox-e2e-build:cancel-followup", "/marker")? != "recovered" { + bail!("cancellation: follow-up build produced a wrong image"); + } + Ok(()) + })(); + for tag in ["cancel-run", "cancel-upload", "cancel-followup"] { + docker_ignore( + data_dir, + &["rmi".into(), "-f".into(), format!("arcbox-e2e-build:{tag}")], + ); + } + result +} + +/// D9: build-output streaming. A RUN step emitting ordered lines, paced +/// under BuildKit's step-log rate clip, must arrive complete through the +/// proxy (a wedge shows up as the deadline, truncation as the clip marker +/// or a missing tail), and `-q` mode must reduce to one image-ID line. +fn output_streaming(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d9-ctx"); + std::fs::create_dir_all(&ctx).context("creating d9 context dir")?; + let total_lines = STREAM_CHUNKS * STREAM_CHUNK_LINES; + let chunk_indices = (1..=STREAM_CHUNKS) + .map(|i| i.to_string()) + .collect::>() + .join(" "); + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} +RUN for i in {chunk_indices}; do seq $(( (i-1)*{STREAM_CHUNK_LINES} + 1 )) $(( i*{STREAM_CHUNK_LINES} )); sleep 1; done +RUN echo streamed > /marker +" + ), + ) + .context("writing d9 Dockerfile")?; + + let result = (|| { + let (output, _) = timed_build( + data_dir, + metrics, + "streaming_build_wall", + &ctx, + STREAM_TAG, + &["--no-cache"], + STREAM_DEADLINE, + )?; + if output.contains("output clipped") { + bail!("streaming: BuildKit clipped the step log (proxy delivered a truncated stream)"); + } + let tail_marker = total_lines.to_string(); + let near_tail_marker = (total_lines - 1).to_string(); + if !output.contains(&tail_marker) || !output.contains(&near_tail_marker) { + bail!("streaming: tail of the step log missing ({near_tail_marker}/{tail_marker})"); + } + + // Quiet mode: exactly one image-ID line on stdout. + let ctx_arg = ctx.display().to_string(); + let quiet = docker_output( + data_dir, + &["build", "-q", "-t", QUIET_TAG, &ctx_arg], + STREAM_DEADLINE, + ) + .context("quiet build")?; + let quiet = quiet.trim(); + if !quiet.starts_with("sha256:") || quiet.lines().count() != 1 { + bail!("streaming: -q output is not a single image ID: {quiet:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), STREAM_TAG.into()]); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), QUIET_TAG.into()]); + result +} + +/// D10: exporters — `--output type=local` and `type=tar` stream the build +/// artifact back to the client through the session, the reverse of the +/// context-upload direction. Content must arrive byte-exact. +fn exporters(data_dir: &Path, metrics: &mut RunMetrics, image: &str) -> Result<()> { + let ctx = data_dir.join("d10-ctx"); + std::fs::create_dir_all(&ctx).context("creating d10 context dir")?; + std::fs::write( + ctx.join("Dockerfile"), + format!( + r"FROM {image} AS build +RUN mkdir /out && printf '{}' > /out/artifact.txt +FROM scratch AS export +COPY --from=build /out/ / +", + EXPORT_PAYLOAD.trim_end_matches('\n'), + ), + ) + .context("writing d10 Dockerfile")?; + // printf without a trailing newline directive: the payload constant's + // newline is added back by comparing against the trimmed form. + let expected = EXPORT_PAYLOAD.trim_end_matches('\n'); + let ctx_arg = ctx.display().to_string(); + + let local_dest = data_dir.join("d10-local-out"); + let local_arg = format!("type=local,dest={}", local_dest.display()); + let started = Instant::now(); + let local = docker_output( + data_dir, + &["build", "--progress=plain", "-o", &local_arg, &ctx_arg], + EXPORT_DEADLINE, + ); + metrics.record("exporter_local_wall", started.elapsed().as_secs_f64()); + local.context("local exporter build")?; + let exported = std::fs::read_to_string(local_dest.join("artifact.txt")) + .context("reading locally-exported artifact")?; + if exported != expected { + bail!("exporters: local export mismatch: {exported:?} != {expected:?}"); + } + + let tar_dest = data_dir.join("d10-export.tar"); + let tar_arg = format!("type=tar,dest={}", tar_dest.display()); + let started = Instant::now(); + let tar_build = docker_output( + data_dir, + &["build", "--progress=plain", "-o", &tar_arg, &ctx_arg], + EXPORT_DEADLINE, + ); + metrics.record("exporter_tar_wall", started.elapsed().as_secs_f64()); + tar_build.context("tar exporter build")?; + let extract_dir = data_dir.join("d10-tar-out"); + std::fs::create_dir_all(&extract_dir)?; + let extract = std::process::Command::new("tar") + .args(["-xf", &tar_dest.display().to_string(), "-C"]) + .arg(&extract_dir) + .output() + .context("extracting tar export")?; + if !extract.status.success() { + bail!( + "exporters: tar extract failed: {}", + String::from_utf8_lossy(&extract.stderr) + ); + } + let exported = std::fs::read_to_string(extract_dir.join("artifact.txt")) + .context("reading tar-exported artifact")?; + if exported != expected { + bail!("exporters: tar export mismatch: {exported:?} != {expected:?}"); + } + Ok(()) +} diff --git a/tests/e2e/tests/docker_build_external.rs b/tests/e2e/tests/docker_build_external.rs new file mode 100644 index 000000000..a39f8a011 --- /dev/null +++ b/tests/e2e/tests/docker_build_external.rs @@ -0,0 +1,254 @@ +//! docker build e2e — Tier X of +//! internal-docs/plans/docker-build-e2e-matrix.md: real-project builds at +//! pinned upstream commits, with real registry/package-manager/keyserver +//! traffic through the datapath. +//! +//! Env-gated behind `ARCBOX_E2E_EXTERNAL=1` (the network-workload plan's +//! external-phase convention): results depend on upstream availability by +//! nature, so this suite is run manually when touching the proxy or +//! datapath — it gates nothing. +//! +//! - **X1 postgres** (`docker-library/postgres` `17/bookworm`): the classic +//! official-image shape — tiny context, real `apt` + `wget` + **gpg +//! keyserver** traffic. +//! - **X2 next.js** (`vercel/next.js` `examples/with-docker`): the single +//! most common user Dockerfile shape — multi-stage standalone build with +//! a real `pnpm install --frozen-lockfile`. +//! - **X3 caddy builder** (`caddyserver/caddy-docker` `2.11/builder`): +//! apk + checksum-verified release-binary fetch (xcaddy). +//! +//! X4 (mastodon/immich-scale heavyweights) stays manual-only — see the +//! plan. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use arcbox_e2e::docker::{docker_ignore, docker_output, run_with_timeout}; +use arcbox_e2e::metrics::RunMetrics; +use arcbox_e2e::scenario::run_vz_scenario_with_log; + +const READY_TIMEOUT: Duration = Duration::from_secs(180); +/// Real builds pull base images and run package managers; give each one a +/// generous ceiling — the assertion is "completes", not "fast". +const BUILD_DEADLINE: Duration = Duration::from_secs(900); +const FETCH_TIMEOUT: Duration = Duration::from_secs(180); + +/// Pinned upstream commits (2026-07-22). Bump deliberately: a pin move can +/// change what the build exercises. +const POSTGRES_PIN: (&str, &str, &str) = ( + "https://github.com/docker-library/postgres.git", + "62a714f93cc32220de46fd12235c9d509e3b1ad6", + "17/bookworm", +); +const NEXTJS_PIN: (&str, &str, &str) = ( + "https://github.com/vercel/next.js.git", + "1d3bf10cde7b19093222305c4ded5f5948928419", + "examples/with-docker", +); +const CADDY_PIN: (&str, &str, &str) = ( + "https://github.com/caddyserver/caddy-docker.git", + "70350320b11d6cb04586bb869b798273180aa6d1", + "2.11/builder", +); + +#[test] +#[ignore = "real upstream builds over the internet; set ARCBOX_E2E_EXTERNAL=1 and run manually"] +fn docker_build_external_suite() -> Result<()> { + if !arcbox_e2e::env_flag("ARCBOX_E2E_EXTERNAL") { + eprintln!("skipping: ARCBOX_E2E_EXTERNAL is not set"); + return Ok(()); + } + run_vz_scenario_with_log( + "docker_build_external", + "info", + |daemon, data_dir, metrics| { + metrics.time("daemon_ready", || daemon.wait_ready_blocking(READY_TIMEOUT))?; + + let scenarios: [(&str, ScenarioFn); 3] = [ + ("x3_caddy_builder", x3_caddy_builder), + ("x1_postgres", x1_postgres), + ("x2_nextjs", x2_nextjs), + ]; + let only = std::env::var("ARCBOX_E2E_BUILD_ONLY").ok(); + let mut failures = Vec::new(); + for (name, scenario) in scenarios { + if let Some(ref only) = only + && only != name + { + continue; + } + tracing::info!(scenario = name, "starting"); + match scenario(data_dir, metrics) { + Ok(()) => tracing::info!(scenario = name, "passed"), + Err(error) => { + tracing::warn!(scenario = name, "failed: {error:#}"); + failures.push(format!("{name}: {error:#}")); + } + } + } + if failures.is_empty() { + Ok(()) + } else { + bail!( + "{} of {} external builds failed:\n{}", + failures.len(), + scenarios.len(), + failures.join("\n---\n") + ) + } + }, + ) +} + +type ScenarioFn = fn(&Path, &mut RunMetrics) -> Result<()>; + +/// Sparse-fetches `subdir` of `repo` at the pinned `sha` (GitHub allows +/// arbitrary-SHA fetches) and returns the materialized subdir path. +fn fetch_pinned_subdir( + data_dir: &Path, + label: &str, + (repo, sha, subdir): (&str, &str, &str), +) -> Result { + let dest = data_dir.join(format!("src-{label}")); + std::fs::create_dir_all(&dest)?; + let git = |args: &[&str]| -> Result<()> { + let output = run_with_timeout( + Command::new("git").arg("-C").arg(&dest).args(args), + FETCH_TIMEOUT, + ) + .with_context(|| format!("git {args:?}"))?; + if !output.status.success() { + bail!( + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) + }; + git(&["init", "-q"])?; + git(&["remote", "add", "origin", repo])?; + git(&["sparse-checkout", "init", "--cone"])?; + git(&["sparse-checkout", "set", subdir])?; + git(&[ + "fetch", + "-q", + "--depth", + "1", + "--filter=blob:none", + "origin", + sha, + ])?; + git(&[ + "-c", + "advice.detachedHead=false", + "checkout", + "-q", + "FETCH_HEAD", + ])?; + let ctx = dest.join(subdir); + if !ctx.join("Dockerfile").is_file() { + bail!("{label}: no Dockerfile at {}", ctx.display()); + } + Ok(ctx) +} + +fn build_and_time( + data_dir: &Path, + metrics: &mut RunMetrics, + key: &str, + ctx: &Path, + tag: &str, +) -> Result<()> { + let ctx_arg = ctx.display().to_string(); + let started = std::time::Instant::now(); + let result = docker_output( + data_dir, + &["build", "--progress=plain", "-t", tag, &ctx_arg], + BUILD_DEADLINE, + ); + metrics.record(key, started.elapsed().as_secs_f64()); + result.with_context(|| format!("{key}: docker build"))?; + Ok(()) +} + +/// X1: the official postgres image — apt, wget'd gosu, gpg keyserver +/// verification, locale generation. The smoke check runs the built binary. +fn x1_postgres(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { + let ctx = fetch_pinned_subdir(data_dir, "postgres", POSTGRES_PIN)?; + let tag = "arcbox-e2e-external:postgres"; + let result = (|| { + build_and_time(data_dir, metrics, "x1_postgres_build_wall", &ctx, tag)?; + let version = docker_output( + data_dir, + &["run", "--rm", tag, "postgres", "--version"], + Duration::from_secs(60), + )?; + if !version.contains("PostgreSQL") { + bail!("postgres smoke: unexpected version output {version:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.into()]); + result +} + +/// X2: the canonical Next.js standalone build — multi-stage, corepack + +/// `pnpm install --frozen-lockfile`, `next build`. Smoke: the runtime +/// stage carries a working node. +fn x2_nextjs(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { + let ctx = fetch_pinned_subdir(data_dir, "nextjs", NEXTJS_PIN)?; + // Upstream ships no `packageManager` pin, so corepack pulls the latest + // pnpm at build time, whose supply-chain default refuses sharp's build + // scripts (ERR_PNPM_IGNORED_BUILDS) — the verbatim build fails on ANY + // engine. Apply pnpm's documented escape hatch and keep everything + // else verbatim; bail loudly if a pin bump makes the patch stale. + let dockerfile = ctx.join("Dockerfile"); + let content = std::fs::read_to_string(&dockerfile)?; + let patched = content.replace( + "pnpm install --frozen-lockfile", + "pnpm install --frozen-lockfile --dangerously-allow-all-builds", + ); + if patched == content { + bail!("nextjs: pnpm install line not found — re-check the accommodation against the pin"); + } + std::fs::write(&dockerfile, patched)?; + let tag = "arcbox-e2e-external:nextjs"; + let result = (|| { + build_and_time(data_dir, metrics, "x2_nextjs_build_wall", &ctx, tag)?; + let version = docker_output( + data_dir, + &["run", "--rm", "--entrypoint", "node", tag, "--version"], + Duration::from_secs(60), + )?; + if !version.trim().starts_with('v') { + bail!("nextjs smoke: unexpected node version {version:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.into()]); + result +} + +/// X3: the caddy builder image — apk + checksum-verified xcaddy release +/// fetch. Cheapest of the three; runs first to fail fast on a broken +/// external environment. +fn x3_caddy_builder(data_dir: &Path, metrics: &mut RunMetrics) -> Result<()> { + let ctx = fetch_pinned_subdir(data_dir, "caddy", CADDY_PIN)?; + let tag = "arcbox-e2e-external:caddy-builder"; + let result = (|| { + build_and_time(data_dir, metrics, "x3_caddy_build_wall", &ctx, tag)?; + let version = docker_output( + data_dir, + &["run", "--rm", "--entrypoint", "xcaddy", tag, "version"], + Duration::from_secs(60), + )?; + if !version.contains("v0.4") { + bail!("caddy smoke: unexpected xcaddy version {version:?}"); + } + Ok(()) + })(); + docker_ignore(data_dir, &["rmi".into(), "-f".into(), tag.into()]); + result +} diff --git a/xtask/AGENTS.md b/xtask/AGENTS.md index c9da3e5ba..cb3e357c2 100644 --- a/xtask/AGENTS.md +++ b/xtask/AGENTS.md @@ -34,6 +34,9 @@ Current recipe ↔ self-build mapping (keep both columns identical): | `hv_vmm` | `cargo build --release -p arcbox-e2e --bin hv_e2e`| `hv_vmm.rs` (same) | | `stats_watch` | release daemon + musl `arcbox-agent` | `stats_watch.rs` (same) | | `sandbox` | release cli+daemon + musl `arcbox-agent`/`arcbox-vm` bins | `sandbox.rs::build_binaries` (same) | +| `egress_throughput` | `cargo build --release -p arcbox-daemon` | `scenario.rs::run_vz_scenario_with_log` (same) | +| `docker_build` | same as above | `scenario.rs::run_vz_scenario_with_log` (same) | +| `docker_build_external` | same as above | `scenario.rs::run_vz_scenario_with_log` (same) | | `virtio_debug` | none (self-builds) | `cargo build --release -p arcbox-daemon` (RELEASE only) | | `daemon_failure` | none (self-builds) | `cargo build -p arcbox-daemon` (**DEBUG**) | | anything else | none — fallback notice, no `SKIP_BUILD` | its own build | @@ -42,6 +45,31 @@ Note `virtio_debug` and `daemon_failure` deliberately have NO recipe and their profiles differ (release vs debug daemon). If you ever add a recipe for either, it must reproduce that exact profile. +## `xtask e2e` — `--backend` cannot move a pinned target + +Most e2e targets hardcode `ARCBOX_VM_BACKEND` and stamp that backend into +their metrics, so the runner's `--backend` env has no effect on them. Only a +few (`boot_assets`, `backend_matrix`) actually read it. The runner therefore +**errors** when the requested backend conflicts with a pinned one. WHY: the +alternative is a run archived under the wrong backend's label — the same +ghost-debugging class as a mismatched `SKIP_BUILD` recipe, and it silently +corrupts any HV↔VZ oracle comparison read from the artifacts. + +This cuts both ways: `virtio_debug` and `hv_reboot` pin **hv**, so +`--backend vz` on them is exactly as wrong as `--backend hv` on a vz-pinned +target. `--backend both` conflicts with any pin. + +`pinned_backend` (`commands/e2e.rs`) derives this from the sources rather +than from a list, deliberately: the set changes whenever someone adds a +target, and a list goes stale exactly when a new target needs the guard +most. It scans the target's own source for a line carrying both +`ARCBOX_VM_BACKEND` and a `"vz"`/`"hv"` literal, then the `arcbox_e2e` +modules it imports — one level of indirection, which is where +`scenario::run_vz_scenario*` and the `sandbox` harness keep their pin. A +line that merely reads the variable carries no literal and is correctly +ignored. Nothing to keep in lockstep; the unit tests in that file pin the +shapes. + ### Extending — adding a new e2e target (lockstep set) 1. The test must gate its build on `arcbox_e2e::env_flag("SKIP_BUILD")`, or `xtask e2e` cannot prebuild it and it rebuilds every repeat. @@ -49,6 +77,11 @@ for either, it must reproduce that exact profile. or accept the self-build fallback — never a half-match. 3. If you add a recipe, verify the packages AND profile match the test's own `cargo build` line character-for-character. +4. Nothing to do for the backend pin — `pinned_backend` reads it off the + sources, so a new target gets the `--backend` guard for free as long as + it pins the way every existing one does (an `ARCBOX_VM_BACKEND` line with + a `"vz"`/`"hv"` literal, in the target or in an `arcbox_e2e` module it + imports). ## `xtask e2e` — forensics linkage (fragile string/env coupling) diff --git a/xtask/src/commands/e2e.rs b/xtask/src/commands/e2e.rs index 518a4a606..f21ab6de1 100644 --- a/xtask/src/commands/e2e.rs +++ b/xtask/src/commands/e2e.rs @@ -16,6 +16,84 @@ use xtask_kit::repo; use crate::{E2eArgs, E2eBackend}; +/// The backend `test` hardcodes, if any — `--backend` cannot move it. +/// +/// Targets fall into three groups. A few read `ARCBOX_VM_BACKEND` from the +/// environment and genuinely honor `--backend` (`boot_assets`, +/// `backend_matrix`). Most pin one backend outright, either in their own +/// source (`virtio_debug` and `hv_reboot` pin **hv**; `machine`, +/// `idle_balloon`, `stats_watch`, `nfs_restart_probe` pin vz) or through a +/// shared harness in `tests/e2e/src` (`scenario::run_vz_scenario*`, +/// `sandbox`). For a pinned target the runner's env has no effect, so a +/// mismatched request archives a run under the wrong backend's label and +/// corrupts any comparison read from it — the same ghost-debugging class as +/// a mismatched `SKIP_BUILD` recipe. Note this cuts both ways: `--backend +/// vz` on an hv-pinned target is just as wrong as `--backend hv` on a +/// vz-pinned one. +/// +/// Derived from the sources rather than kept as a list here: the set changes +/// whenever someone adds a target, and a list goes stale exactly when a new +/// target needs the guard most. An unreadable or absent target is treated as +/// unpinned — cargo reports an unknown test better than we can. +fn pinned_backend(root: &std::path::Path, test: &str) -> Result> { + let path = root.join("tests/e2e/tests").join(format!("{test}.rs")); + let source = match fs::read_to_string(&path) { + Ok(source) => source, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())), + }; + + // The target's own source first, then the `arcbox_e2e` modules it pulls + // in — that one level of indirection is where `run_vz_scenario` and the + // sandbox harness hide their pin. + if let Some(backend) = literal_backend_pin(&source) { + return Ok(Some(backend)); + } + for module in imported_e2e_modules(&source) { + let path = root.join("tests/e2e/src").join(format!("{module}.rs")); + if let Ok(helper) = fs::read_to_string(&path) + && let Some(backend) = literal_backend_pin(&helper) + { + return Ok(Some(backend)); + } + } + Ok(None) +} + +/// Finds a hardcoded `ARCBOX_VM_BACKEND` value in `source`. +/// +/// A pin is a line that both names the variable and carries a `"vz"` or +/// `"hv"` literal, which is how every pin in the tree is written. Lines that +/// merely *read* the variable (`env::var("ARCBOX_VM_BACKEND")`) or forward +/// an already-computed value carry no literal and are correctly ignored. +fn literal_backend_pin(source: &str) -> Option { + source + .lines() + .filter(|line| line.contains("ARCBOX_VM_BACKEND")) + .find_map(|line| { + ["vz", "hv"] + .into_iter() + .find(|backend| line.contains(&format!("\"{backend}\""))) + .map(str::to_owned) + }) +} + +/// Module names from every `arcbox_e2e::` path in `source`. +fn imported_e2e_modules(source: &str) -> Vec { + const PREFIX: &str = "arcbox_e2e::"; + let mut modules = Vec::new(); + for (index, _) in source.match_indices(PREFIX) { + let module: String = source[index + PREFIX.len()..] + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect(); + if !module.is_empty() && !modules.contains(&module) { + modules.push(module); + } + } + modules +} + /// One test-run outcome for the final summary. struct RunOutcome { label: String, @@ -42,6 +120,17 @@ pub fn run(args: E2eArgs) -> Result<()> { E2eBackend::Hv => &[Some("hv")], E2eBackend::Both => &[Some("vz"), Some("hv")], }; + if let Some(pinned) = pinned_backend(&root, &args.test)? + && backends.iter().flatten().any(|wanted| *wanted != pinned) + { + bail!( + "test `{}` hardcodes ARCBOX_VM_BACKEND={pinned}, so the requested \ + backend cannot take effect — the run would be {pinned} while the \ + label, metrics, and archive said otherwise. Re-run with \ + `--backend {pinned}`, or make the target read the env first.", + args.test, + ); + } // Build what the selected test would build, once, so every repeat // runs with SKIP_BUILD=1 instead of re-invoking cargo. @@ -153,8 +242,9 @@ fn prebuild(root: &std::path::Path, test: &str) -> Result { .run()?; Ok(true) } - // Must match tests/e2e/tests/egress_throughput.rs's own build. - "egress_throughput" => { + // Must match tests/e2e/src/scenario.rs's self-build (the shared + // daemon-boot scaffolding these targets run through). + "egress_throughput" | "docker_build" | "docker_build_external" => { xshell::cmd!(shell, "cargo build --release -p arcbox-daemon").run()?; Ok(true) } @@ -221,3 +311,46 @@ fn summarize(outcomes: &[RunOutcome], artifacts_dir: &std::path::Path) -> Result } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The shapes every pin in `tests/e2e` is actually written in — a + /// tuple entry, `.into()` instead of `.to_owned()`, and an hv pin. + #[test] + fn literal_pins_are_recognized_in_the_shapes_used() { + let vz = r#"("ARCBOX_VM_BACKEND".to_owned(), "vz".to_owned()),"#; + let vz_into = r#"("ARCBOX_VM_BACKEND".into(), "vz".into()),"#; + let hv = r#"("ARCBOX_VM_BACKEND".to_owned(), "hv".to_owned()),"#; + assert_eq!(literal_backend_pin(vz).as_deref(), Some("vz")); + assert_eq!(literal_backend_pin(vz_into).as_deref(), Some("vz")); + assert_eq!(literal_backend_pin(hv).as_deref(), Some("hv")); + } + + /// A target that *reads* the env honors `--backend` and must not be + /// mistaken for a pinned one — `boot_assets` and `backend_matrix` are + /// the whole reason the runner has a `--backend` flag. + #[test] + fn reading_the_env_is_not_a_pin() { + let reads = r#"let backend = match env::var("ARCBOX_VM_BACKEND") {"#; + let forwards = + r#"env.push(("ARCBOX_VM_BACKEND".to_owned(), backend.as_str().to_owned()));"#; + assert!(literal_backend_pin(reads).is_none()); + assert!(literal_backend_pin(forwards).is_none()); + } + + /// The pin a scenario-based target inherits lives one level away, in + /// the `arcbox_e2e` module it imports — that indirection is what a + /// naive scan of the target's own source misses. + #[test] + fn imported_modules_are_collected_for_the_indirect_pin() { + let source = "use arcbox_e2e::scenario::run_vz_scenario_with_log;\n\ + use arcbox_e2e::metrics::RunMetrics;\n\ + let root = arcbox_e2e::repo_root();\n"; + let modules = imported_e2e_modules(source); + assert!(modules.contains(&"scenario".to_owned())); + assert!(modules.contains(&"metrics".to_owned())); + assert!(modules.contains(&"repo_root".to_owned())); + } +}