diff --git a/.github/workflows/ingest.yml b/.github/workflows/ingest.yml index f11a039..4c71ef4 100644 --- a/.github/workflows/ingest.yml +++ b/.github/workflows/ingest.yml @@ -1,20 +1,22 @@ name: Ingest benchmark run -# Manually pull one results directory from GCS, convert it to a run JSON, -# and commit it to main so the deploy-pages workflow redeploys GitHub Pages. +# Manually pull one campaign result bundle from GCS and open a PR adding it as a +# committed run JSON. +# +# This job is a thin wrapper around scripts/ingest.sh — the same script the local +# `make ingest` flow runs. It delegates rather than inlining the converter call +# for three reasons: +# - one code path: a run ingested from CI is byte-identical to one ingested +# from a laptop, down to the commit message; +# - run identity (id, date, name) comes from the bundle's own metadata.json, so +# this form asks for two things instead of five; +# - runs arrive as a reviewable `run/` PR instead of a push straight to +# main, and pr-preview.yml renders each one in the viewer before merge. on: workflow_dispatch: inputs: gcs_path: - description: "gs:// path to a results directory to ingest (e.g. gs://rpc-full-history/benchmarks/2026-07-13-user-dev-063a)" - required: true - type: string - run_id: - description: "Run slug — becomes docs/runs/.json (e.g. pubnet-2026-07-13)" - required: true - type: string - run_name: - description: "Human-readable run name" + description: "gs:// path to a campaign result bundle (e.g. gs://rpc-full-history/results/phase3-c6id8xl-c48a55c6-20260724T214257Z)" required: true type: string dataset_kind: @@ -24,18 +26,16 @@ on: options: - pubnet - synthetic - run_date: - description: "Run date (YYYY-MM-DD)" - required: true - type: string - unit_facts: - description: "Optional repo path to a --unit-facts sidecar JSON (e.g. converter/facts/synthetic-2026-07-15.json)" + extra_args: + description: "Optional convert.py args, passed after `--` (e.g. --unit-facts converter/facts/synthetic-2026-07-15.json). Word-split on whitespace: a value containing spaces cannot be expressed here — run scripts/ingest.sh locally for that." required: false type: string -# Minimal: write the committed run JSON, and mint an OIDC token for GCP WIF. +# contents + pull-requests: the script pushes run/ and opens its PR. +# id-token: the OIDC token GCP Workload Identity Federation exchanges. permissions: contents: write + pull-requests: write id-token: write jobs: @@ -72,44 +72,23 @@ jobs: - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v2 - - name: Download results from GCS + # Full mode: fetch → convert → commit on run/ → push → gh pr create. + # The PR targets the repository's default branch, which is where runs land. + - name: Ingest bundle and open a run PR env: GCS_PATH: ${{ inputs.gcs_path }} - run: | - rm -rf ./results-in - mkdir -p ./results-in - gcloud storage cp -r "$GCS_PATH" ./results-in - # gcloud nests the copied directory under ./results-in/. - echo "RESULTS_DIR=./results-in/$(basename "${GCS_PATH%/}")" >> "$GITHUB_ENV" - - - name: Convert run into docs/runs - env: - RUN_ID: ${{ inputs.run_id }} - RUN_NAME: ${{ inputs.run_name }} - RUN_DATE: ${{ inputs.run_date }} DATASET_KIND: ${{ inputs.dataset_kind }} - GCS_PATH: ${{ inputs.gcs_path }} - UNIT_FACTS: ${{ inputs.unit_facts }} - run: | - python3 converter/convert.py "$RESULTS_DIR" \ - --run-id "$RUN_ID" \ - --run-name "$RUN_NAME" \ - --run-date "$RUN_DATE" \ - --dataset-kind "$DATASET_KIND" \ - --source-gcs "$GCS_PATH" \ - ${UNIT_FACTS:+--unit-facts "$UNIT_FACTS"} \ - --out-dir docs/runs - - - name: Commit and push the new run - env: - RUN_ID: ${{ inputs.run_id }} + EXTRA_ARGS: ${{ inputs.extra_args }} + GH_TOKEN: ${{ github.token }} run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add docs/runs - if git diff --cached --quiet; then - echo "No changes to commit." - exit 0 + if [ -n "$EXTRA_ARGS" ]; then + # Deliberately unquoted: extra_args is one freeform string that has to + # become several arguments, so it is word-split on whitespace. That is + # the documented limit of this input, not an oversight. + # shellcheck disable=SC2086 + scripts/ingest.sh "$GCS_PATH" --dataset-kind "$DATASET_KIND" -- $EXTRA_ARGS + else + scripts/ingest.sh "$GCS_PATH" --dataset-kind "$DATASET_KIND" fi - git commit -m "Ingest benchmark run ${RUN_ID}" - git push diff --git a/.github/workflows/runner-go.yml b/.github/workflows/runner-go.yml new file mode 100644 index 0000000..30f8cf0 --- /dev/null +++ b/.github/workflows/runner-go.yml @@ -0,0 +1,38 @@ +name: Go runner + +# Vet and test the Go campaign runner. Like the shellcheck job this is a +# static gate only — real campaigns run on the benchmark devbox, not in CI. +on: + push: + branches: [main] + paths: + - "runner/**" + - ".github/workflows/runner-go.yml" + pull_request: + paths: + - "runner/**" + - ".github/workflows/runner-go.yml" + +permissions: + contents: read + +jobs: + go: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: runner/go.mod + cache-dependency-path: runner/go.sum + + - name: Vet + working-directory: runner + run: go vet ./... + + - name: Test + working-directory: runner + run: go test ./... diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 0000000..e4dc081 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,38 @@ +name: Shellcheck runner + +# Lint the shell that is left: the devbox bootstrap, the results-side ingest +# script, and the stub binary the campaign runner's e2e suite drives. This is a +# static gate only — the runner itself is exercised for real on the benchmark +# devbox, not in CI (its Go side is gated by runner-go.yml). +on: + push: + branches: [main] + paths: + - "runner/**" + - "scripts/**" + - ".github/workflows/shellcheck.yml" + pull_request: + paths: + - "runner/**" + - "scripts/**" + - ".github/workflows/shellcheck.yml" + +permissions: + contents: read + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Syntax-check (bash -n) + run: | + for f in runner/*.sh scripts/*.sh runner/cmd/campaign/testdata/*.sh; do + echo "bash -n $f" + bash -n "$f" + done + + - name: Shellcheck + run: shellcheck runner/*.sh scripts/*.sh runner/cmd/campaign/testdata/*.sh diff --git a/Makefile b/Makefile index 7901c5d..2bb0c39 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ # Stellar RPC full-history benchmarks — local ops. -# `make` (or `make help`) lists targets. `make convert` is the primary local flow. +# `make` (or `make help`) lists targets. `make ingest` is the primary local flow; +# `make convert` is the layer underneath it. .DEFAULT_GOAL := help -.PHONY: help convert ingest test smoke serve +.PHONY: help convert ingest test smoke serve runner-build runner-test # Variables required by `convert`. `convert` fails early if any is empty. CONVERT_REQUIRED := RESULTS RUN_ID RUN_NAME KIND RUN_DATE @@ -42,6 +43,12 @@ ingest: ## Ingest a campaign bundle into a run/ PR branch (BUNDLE, KIND; --l test: ## Run the converter unit + golden tests python3 -m unittest discover converter/tests +runner-build: ## Build the Go campaign runner + cd runner && go build ./... + +runner-test: ## Vet and test the Go campaign runner + cd runner && go vet ./... && go test ./... + smoke: ## Run the jsdom viewer smoke test (needs node; installs deps on first run) npm --prefix tests/smoke install --silent npm --prefix tests/smoke test diff --git a/README.md b/README.md index ff78fe3..e1ace1a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ step, no server: the numbers live in git and the site is just HTML/JS reading th ## What this is The bench suite (`stellar-rpc bench-ingest cold|hot`, `bench-query cold|hot`) runs in -campaigns on an AWS NVMe devbox (`m6id.2xlarge`). Each campaign is several configurations +campaigns on an AWS NVMe devbox (`m6id.2xlarge`), driven by the config-driven runner in +[`runner/`](runner/) (see "Run a campaign" below). Each campaign is several configurations × 5 fresh-process runs; every run writes CSVs (`stage,n,n_items,total_ns,p50_ns,p90_ns,p99_ns,max_ns`) into its own directory, and the results are mirrored to GCS under `gs://rpc-full-history/benchmarks/`. @@ -51,14 +52,129 @@ To smoke-test the viewer headlessly (loads each run in a jsdom DOM, asserts zero errors and the expected figure/section counts and sanity values), run `make smoke` (needs Node; installs `jsdom` under `tests/smoke/` on first run). -## Add a run locally (the primary flow today) +## Run a campaign -On a laptop that's authenticated to GCS (`gcloud auth login`), pull a results directory -down, convert it, and commit. Worked example: +Campaigns run on the benchmark devbox via the campaign CLI in [`runner/`](runner/) — this +repo's operations side. The runner treats stellar-rpc as a **black box**: it maintains a +build clone of it under `$BENCH_ROOT/src`, builds the configured ref, and drives the +bench subcommands — no standalone stellar-rpc checkout is needed anywhere. See +[runner/README.md](runner/README.md) for the full CLI and config reference, the bundle +layout it produces, and the minimum stellar-rpc ref it requires (the compatibility floor). ```bash -# 1. Pull the results directory from GCS. (This is the exact path recorded as this -# run's provenance in docs/runs/pubnet-2026-07-13.json.) +# 0. One-time on a fresh devbox (and again after every instance stop/start, +# which wipes the NVMe instance store): provision the machine. +./runner/bootstrap.sh + +# Everything below runs from runner/ (the devbox has Go; bootstrap installs it). +cd runner + +# 1. Write a campaign config (copy example-campaign.toml, adjust the keys) and +# sanity-check the full command plan. --dry-run builds, downloads, and runs +# nothing — it works on any machine, e.g. a laptop: +go run ./cmd/campaign run my-campaign.toml --dry-run + +# 2. Run it (in tmux — campaigns run for hours). Results land in +# $BENCH_ROOT/results/--/, tarred to /tmp so the bundle +# survives an instance stop. +go run ./cmd/campaign run my-campaign.toml + +# 3. Publish the bundle to GCS. This happens automatically when the config +# sets publish_uri; run it by hand otherwise (or to retry a failed upload — +# an upload that died partway leaves objects at the destination, so the +# retry needs --force, which is a MERGE: files already there that this +# bundle lacks survive it. Against a destination that really is empty, +# --force only skips the emptiness check, so it is safe on a first publish): +go run ./cmd/campaign publish /mnt/nvme/bench/results/ \ + gs://rpc-full-history/benchmarks --force +``` + +The published bundle is exactly what the next section ingests into a committed run +JSON — closing the loop: campaign config → run → publish → ingest → viewer. + +## Add a run + +`scripts/ingest.sh` is the one path from a published bundle to a committed run: it +fetches the bundle, converts it, and stages the result as a `run/` branch. The +same script backs `make ingest` and the GitHub Action below, so a run ingested from a +laptop and one ingested from CI are byte-identical, commit message included. + +```bash +# On a laptop authenticated to GCS (gcloud auth login). That gs:// path is the +# one recorded as campaign.source_gcs in docs/runs/phase3-c6id8xl-c48a55c6-20260724T214257Z.json. +make ingest \ + BUNDLE=gs://rpc-full-history/results/phase3-c6id8xl-c48a55c6-20260724T214257Z \ + KIND=synthetic +``` + +`BUNDLE` is auto-detected and may be a `gs://` or `s3://` bundle URI, a local bundle +directory, or a `bench-results-.tgz` tarball — the shapes the campaign CLI's +`run` and `publish` leave behind. `KIND` is the one thing you have to state, because it is +the one fact the bundle doesn't record: `datasets[].kind` in the manifest is the dataset's +*transport* (`packs-gs`, `bsb-s3`, …), not pubnet-vs-synthetic. + +Everything else is derived from the bundle's own `metadata.json` +(see [SCHEMA.md § Inputs](SCHEMA.md#inputs--result-bundle-layouts--manifests)): + +| Derived | From | +|-------------------------|--------------------------------------------------| +| Run id, and so the file | `run_id` → `docs/runs/.json` | +| Run date | `started_at` | +| Run name | `campaign.name` | +| `campaign.source_gcs` | the `gs://` URI you passed (recorded as provenance) | +| Commit / PR body | campaign config, close interval, datasets, hardware | + +Three modes, least to most committal: + +| Mode | Effect | +|-------------|-----------------------------------------------------------------------------| +| `--dry-run` | Converts into a **temp** directory — never `docs/runs/` — and prints the converter output, the derived run id, the would-be commit body, the would-be branch, and the exact git/gh commands full mode would run. Executes none of them. A remote bundle isn't fetched either; the fetch command is printed instead, so a dry run works offline. | +| `--local` | Converts into `docs/runs/`, creates `run/` off HEAD, commits the two changed files. No push, no PR. | +| (default) | `--local`, then `git push -u origin run/` and `gh pr create`. | + +`make ingest` passes `--local` on purpose: it stops at the commit, so you can read the +diff and `make serve` the result before anything leaves the machine. Call the script +directly for the other two modes: + +```bash +# Look before you leap — converts to a temp dir, touches nothing: +scripts/ingest.sh gs://rpc-full-history/results/ --dataset-kind synthetic --dry-run + +# Full: convert, branch, commit, push, open the PR (needs gh authenticated): +scripts/ingest.sh gs://rpc-full-history/results/ --dataset-kind synthetic +``` + +The PR targets the default branch and carries the campaign one-liners, any converter +warnings, and a note pointing the reviewer at the preview: `pr-preview.yml` publishes +that PR's `docs/`, so the new run can be read in the viewer before anyone merges it. +Merging is the deploy. + +Two rails keep the flow from surprising you. The script refuses to run against a working +tree with uncommitted tracked changes, and refuses to overwrite an already-ingested run — +an existing `docs/runs/.json` is an error until you pass `--force`. A `--force` +re-ingest reuses the existing `run/` branch instead of failing on it, and exits +quietly when the reconverted JSON turns out byte-identical to the committed one. + +Anything after a literal `--` is passed straight through to `convert.py`, which is how you +override a derived field without leaving the flow: + +```bash +scripts/ingest.sh gs://rpc-full-history/results/ --dataset-kind synthetic -- \ + --run-name "Phase 3 — c6id.8xlarge rerun" \ + --unit-facts converter/facts/synthetic-2026-07-15.json +``` + +### `make convert` — the layer underneath + +`make convert` calls `converter/convert.py` and nothing else: no fetch, no branch, no +commit. Reach for it when the bundle can't identify itself — the **legacy** pubnet and +synthetic layouts predate `metadata.json`, so `--run-id`/`--run-name`/`--run-date` have no +defaults to fall back on — or when you want to name every field by hand. Worked example +against the archived pubnet run (`docs/runs/archive/pubnet-2026-07-13.json`): + +```bash +# 1. Pull the results directory from GCS. (This is the exact path recorded as that +# run's provenance.) gcloud storage cp -r \ gs://rpc-full-history/benchmarks/2026-07-13-user-dev-063a \ ./results-in @@ -72,11 +188,10 @@ make convert \ RUN_DATE=2026-07-13 \ GCS=gs://rpc-full-history/benchmarks/2026-07-13-user-dev-063a -# 3. Review the diff, then commit + push. The deploy-pages workflow syncs -# docs/ to the gh-pages branch and Pages redeploys. +# 3. Review the diff, then commit on a branch and open a PR — the same place +# `scripts/ingest.sh` would have left you. git add docs/runs git commit -m "Add run pubnet-2026-07-13" -git push ``` `make convert` variables: @@ -112,13 +227,26 @@ GCS path, summary paths). ## GitHub Action flow (`.github/workflows/ingest.yml`) -`workflow_dispatch` with inputs `gcs_path`, `run_id`, `run_name`, `dataset_kind` -(pubnet|synthetic), `run_date`, and optional `unit_facts` (a repo path to a `--unit-facts` -sidecar JSON for synthetic dataset meta). It checks out the repo, authenticates to GCP via Workload -Identity Federation, `gcloud storage cp -r` the results directory into `./results-in`, runs -the converter, and commits the new/updated `docs/runs/*.json` + manifest back to `main`. -Permissions are minimal: `contents: write` (to commit) and `id-token: write` (for the OIDC -token WIF exchanges). +`workflow_dispatch` running `scripts/ingest.sh` in full mode — the same script as the +local flow above, which is the point: CI is not a second implementation that can drift. +Three inputs, two of them required: + +| Input | Required | Meaning | +|----------------|----------|---------------------------------------------------------------| +| `gcs_path` | yes | `gs://` path to the campaign result bundle | +| `dataset_kind` | yes | `pubnet` or `synthetic` | +| `extra_args` | no | Passed to `convert.py` after `--` (e.g. `--unit-facts …`) | + +Run id, date, and name are not asked for — they come from the bundle's `metadata.json`. +`extra_args` is one freeform string word-split on whitespace, so a value containing spaces +can't be expressed there; use the local flow for those. + +The job checks out the repo, authenticates to GCP via Workload Identity Federation, then +hands the `gs://` path to the script, which fetches, converts, commits on `run/`, +pushes, and opens the PR against the default branch. Runs therefore arrive as reviewable +PRs with a `pr-preview.yml` render attached, exactly like the local flow — not as a push +straight to `main`. Permissions: `contents: write` and `pull-requests: write` (push the +branch, open the PR) plus `id-token: write` (the OIDC token WIF exchanges). **This workflow does not run yet — the GCP-side setup is pending.** It fails early with a clear message until two repository variables exist @@ -129,8 +257,9 @@ clear message until two repository variables exist Creating them requires, in GCP project **`dev-hubble`**, a workload identity pool + provider (federating this GitHub repo) and a service account with `roles/storage.objectViewer` on -`gs://rpc-full-history`. That GCP setup is out of this repo's hands; until it lands, use the -local flow above. +`gs://rpc-full-history`. That GCP setup is out of this repo's hands; until it lands, +`make ingest` from a laptop is the way runs get in — and it runs the same script, so +nothing about a run changes when the dispatch starts working. ## Data model @@ -153,12 +282,23 @@ statement. ``` stellar-rpc-benchmarks/ -├── Makefile # convert / test / serve / help +├── Makefile # ingest / convert / test / smoke / serve / help ├── README.md ├── SCHEMA.md # run JSON schema v1 (the data contract) ├── .github/ │ └── workflows/ -│ └── ingest.yml # workflow_dispatch: GCS results dir → committed run +│ ├── ingest.yml # workflow_dispatch: GCS bundle → run PR (delegates to scripts/ingest.sh) +│ ├── deploy-pages.yml # sync main:/docs to the gh-pages branch Pages serves +│ ├── pr-preview.yml # publish each PR's docs/ under gh-pages:/pr-preview/pr-/ +│ ├── runner-go.yml # go vet + go test for the campaign runner +│ └── shellcheck.yml # lint the remaining shell scripts on every PR that touches them +├── runner/ # benchmark operations: the devbox side producing result bundles +│ ├── bootstrap.sh # provision the devbox (idempotent, no builds) +│ ├── cmd/campaign/ # campaign CLI: run · plan · preflight · publish (see runner/README.md) +│ ├── internal/ # config · plan · run · preflight · publish · bundle +│ └── example-campaign.toml # annotated config to copy from +├── scripts/ +│ └── ingest.sh # bundle → converted run → run/ branch → PR (make ingest) ├── converter/ │ ├── convert.py # results dir → docs/runs/.json (+ manifest), stdlib only │ ├── facts/ # per-unit sidecar facts (e.g. synthetic model/tps/pack) @@ -168,10 +308,17 @@ stellar-rpc-benchmarks/ └── docs/ # GitHub Pages root (static vanilla-JS viewer) ├── index.html # the viewer shell (dropdown / ?run=) ├── app.js # renderers (per dataset.kind) + charts - ├── styles.css # design system (light + dark) + ├── styles.css # design system (light + dark), shared by every page + ├── summary.html # stakeholder summary page (summary.js + summary.css) + ├── latency-model.html # end-to-end latency model against the phase targets + ├── tx-submission.html # transaction-submission report (txsub.js) + ├── targets.json # Phase 1/2/3 performance targets — single source of truth + ├── dataset-sizes.json # measured sizes of the synthetic dataset profiles + ├── txsub/ # tx-submission harvest summaries, verbatim (+ index.json) └── runs/ ├── index.json # manifest of runs (oldest date first) - └── .json # one file per run (schema v1) + ├── .json # one file per run (schema v1) + └── archive/ # retired pre-campaign runs, with their own manifest ``` ## Future work diff --git a/SCHEMA.md b/SCHEMA.md index 7233c3d..903d455 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -41,8 +41,8 @@ query `events` rows, `n_items` may vary — keep the per-run array as `items_r`, "build": { "commit": "", "branch": "", "go": "…", "rust": "…", "version": "v20.3.1-412-g…", "build_timestamp": "…" }, // commit/branch/go/rust from the machine-metadata `repo:` line; when a - // campaign bundle carries invocation.json, its binary.{commit_hash, - // branch,version,build_timestamp} override commit/branch and add + // campaign bundle carries invocation.json, its binary.{commitHash, + // branch,version,buildTimestamp} override commit/branch and add // version/build_timestamp (the structured binary identity wins). "hardware": { // optional; verbatim from metadata.json (campaign bundles) "instance_type": "m6id.2xlarge", "instance_id": "i-…", // instance_* omitted off EC2 @@ -80,7 +80,7 @@ query `events` rows, `n_items` may vary — keep the per-run array as `items_r`, // table, copied verbatim at convert time — see // "Phase 1/2/3 performance targets" below. "name": "phase1-synthetic-minspec", // optional; metadata.json campaign.name - "config_file": "…​.cfg", // optional; metadata.json campaign.config_file + "config_file": "…​.toml", // optional; metadata.json campaign.config_file "config": { … } // optional; remaining metadata.json campaign knobs // (ingest/query/runs/query_concurrency/cold_iters/ // hot_iters/workers/hot_num_ledgers/ref/built_commit) @@ -239,7 +239,8 @@ The converter auto-detects the input bundle layout from its subdirectory names: - **synthetic** — `synth-{cold,hot}--run`. - **pubnet** — `ingest-{cold,hot}--run`, `query-{cold,hot}--run`, `golden-download-` (a timed sourcing leg surfaced as the `golden` section). -- **campaign** — produced by `campaign.sh`. Timed dirs sit at the bundle root as +- **campaign** — produced by the campaign CLI in `runner/` (the producer-side bundle layout + is documented in `runner/README.md`). Timed dirs sit at the bundle root as `{ingest,query}-{cold,hot}--c-run`; the unit id is the composite `-c` (e.g. `sac-6000-c1`). Untimed prep dirs `golden--c` are dataset preparation, **not results** — the converter skips them and warns. The @@ -247,20 +248,41 @@ The converter auto-detects the input bundle layout from its subdirectory names: orthogonal to `dataset.kind` (a campaign may carry pubnet or synthetic data). Every bundle also carries a free-text `*machine-metadata*.txt` at the root (parsed into -`machine`). A campaign bundle additionally carries **two JSON manifests** — both optional -and additive, so manifest-less bundles convert unchanged: +`machine`). A campaign bundle additionally carries **JSON manifests** — all optional and +additive, so manifest-less bundles convert unchanged: - **`metadata.json`** at the bundle root (schema_version 1) — the campaign runner's record. Source of truth for run identity (`run_id` → default `run_id`; `started_at` → default `run_date`), the `campaign` config (incl. `close_interval` → `campaign.close_interval_ns`), the structured `hardware` object, and `hostname`. `datasets[].kind` is the dataset **transport** (`packs-local|packs-gs|bsb-s3|fixture`), not pubnet-vs-synthetic, and sets - campaign display order. -- **`invocation.json`** in each per-invocation `--out` dir (schema_version 1) — written by - the four bench subcommands. Source of truth for binary identity (`binary.{commit_hash, - branch,version,build_timestamp}`) and the resolved subcommand `flags`. Consistency of the - binary commit is cross-checked across invocations (and against `metadata.campaign.built_commit`); - a mismatch warns. + campaign display order. `finished_at` is absent until the campaign finishes (the runner + writes the manifest up front), and `campaign.resumed` appears only on a bundle built + across more than one `--resume` session. `status` (`running|finished|failed`) is + additive and written the same way — `running` up front, rewritten at the end; bash-era + bundles have none, so absent means unknown, not healthy. +- **`plan.json`** at the bundle root (schema_version 1) — the campaign as data: the + ordered steps the runner intended to execute, with their ids, kinds, argv, + dependencies, and derived paths. Runner-owned and additive; **the converter ignores it + today**. +- **`leg.json`** in each timed `--out` dir (schema_version 1) — the runner's own + completion sentinel, written after the benchmark process ends whether it succeeded or + not: `id`, `argv`, `exit_code`, `started_at`, `finished_at`, `duration_ns`, and `error` + on a failure. It supersedes the invocation.json-presence heuristic the bash runner used + to decide what a `--resume` could skip — `invocation.json` is written by the process + being measured, so a killed process leaves none. Runner-owned and additive; **the + converter ignores it today**. +- **`invocation.json`** in each per-invocation `--out` dir (`schemaVersion` 1, camelCase + keys — written by the four bench subcommands; stellar-rpc's `invocation.go` is the + producer, merged as stellar-rpc#907). Source of truth for binary identity + (`binary.{commitHash,branch,version,buildTimestamp}`) and the resolved subcommand + `flags`; also carries `hostname`, `startedAt`/`finishedAt`, and — **on a failed run + only** — an `error` field (a failed run still writes the manifest, so presence alone + does not mean success; the converter warns loudly on an error-bearing invocation, whose + CSVs are partial). The converter also accepts the snake_case spellings + (`commit_hash`, `build_timestamp`) that pre-#907 drafts of the schema used. Consistency + of the binary commit is cross-checked across invocations (and against + `metadata.campaign.built_commit`); a mismatch warns. Explicitly-passed CLI args (`--run-id`, `--run-date`, …) always win over manifest defaults. Where free-text machine metadata and the structured manifests overlap, the **structured data diff --git a/converter/convert.py b/converter/convert.py index cce9bec..569bc6d 100644 --- a/converter/convert.py +++ b/converter/convert.py @@ -288,14 +288,22 @@ def load_campaign_manifest(results_dir): def load_invocations(results_dir): - """[(dirname, invocation.json dict)] for every per-invocation dir that has one.""" + """[(dirname, invocation.json dict)] for every per-invocation dir that has one. + A failed run also writes invocation.json, with an `error` field (stellar-rpc#907): + its CSVs are partial, so a bundle carrying one is loudly suspect.""" out = [] for p in sorted(glob.glob(os.path.join(results_dir, "*", "invocation.json"))): try: with open(p) as f: - out.append((os.path.basename(os.path.dirname(p)), json.load(f))) + inv = json.load(f) except (json.JSONDecodeError, OSError) as e: warn(f"could not read {os.path.relpath(p, results_dir)} ({e}); ignoring") + continue + d = os.path.basename(os.path.dirname(p)) + if inv.get("error"): + warn(f"{d} is a FAILED run ({inv['error']!r}); its CSVs are partial " + "— re-run or drop that leg before publishing this conversion") + out.append((d, inv)) return out @@ -316,10 +324,18 @@ def hardware_into_machine(machine, hw): machine["kernel"] = hw["uname"] +def _normalize_binary(b): + """invocation.json binary identity keyed snake_case regardless of producer: + stellar-rpc#907 writes camelCase (commitHash, buildTimestamp); earlier + drafts of the schema used snake_case. Accept both spellings.""" + renames = {"commitHash": "commit_hash", "buildTimestamp": "build_timestamp"} + return {renames.get(k, k): v for k, v in b.items()} + + def resolve_binary(build, metadata, invocations): """Merge invocation.json binary identity into build (authoritative over the machine-metadata `repo:` parse) and warn on any commit mismatch.""" - binaries = [inv["binary"] for _, inv in invocations if inv.get("binary")] + binaries = [_normalize_binary(inv["binary"]) for _, inv in invocations if inv.get("binary")] commits = {b["commit_hash"] for b in binaries if b.get("commit_hash")} if metadata and metadata.get("campaign", {}).get("built_commit"): commits.add(metadata["campaign"]["built_commit"]) diff --git a/converter/tests/fixtures.py b/converter/tests/fixtures.py index a343437..3c680a1 100644 --- a/converter/tests/fixtures.py +++ b/converter/tests/fixtures.py @@ -124,18 +124,20 @@ def _hot_phases(u, run): def _write_invocation(d, subcommand, close_interval): + # Mirrors stellar-rpc's writer (invocation.go, #907): camelCase keys, + # written for failed runs too (then with an `error` field). flags = {"out": d, "num-ledgers": "100", "source": "pack"} if subcommand.endswith("hot"): flags["close-interval"] = close_interval inv = { - "schema_version": 1, + "schemaVersion": 1, "command": "stellar-rpc " + subcommand, "flags": flags, - "binary": {"version": VERSION, "commit_hash": COMMIT, - "build_timestamp": "2026-07-22T00:10:00", "branch": BRANCH}, + "binary": {"version": VERSION, "commitHash": COMMIT, + "buildTimestamp": "2026-07-22T00:10:00", "branch": BRANCH}, "hostname": "user-dev-063a", - "started_at": "2026-07-22T01:00:00Z", - "finished_at": "2026-07-22T03:47:12Z", + "startedAt": "2026-07-22T01:00:00Z", + "finishedAt": "2026-07-22T03:47:12Z", } with open(os.path.join(d, "invocation.json"), "w") as f: json.dump(inv, f, indent=2) diff --git a/converter/tests/test_campaign.py b/converter/tests/test_campaign.py index fcf59e7..f0a0d30 100644 --- a/converter/tests/test_campaign.py +++ b/converter/tests/test_campaign.py @@ -184,13 +184,60 @@ def test_warns_on_commit_mismatch(self): inv_path = os.path.join(root, "ingest-hot-sac-6000-c1-run1", "invocation.json") with open(inv_path) as f: inv = json.load(f) - inv["binary"]["commit_hash"] = "f" * 40 + inv["binary"]["commitHash"] = "f" * 40 with open(inv_path, "w") as f: json.dump(inv, f) _, warnings, _ = run_convert(root) self.assertTrue(any("binary commit mismatch" in w for w in warnings)) +class InvocationSchemaTests(unittest.TestCase): + def _rewrite_invocations(self, root, mutate): + for name in os.listdir(root): + p = os.path.join(root, name, "invocation.json") + if not os.path.isfile(p): + continue + with open(p) as f: + inv = json.load(f) + mutate(inv) + with open(p, "w") as f: + json.dump(inv, f) + + def test_legacy_snake_case_binary_still_resolves(self): + # Pre-#907 drafts of invocation.json used snake_case keys; the + # converter accepts both spellings of the binary identity. + tmp = tempfile.mkdtemp() + root = os.path.join(tmp, "snake") + fixtures.build_campaign_bundle(root, paced=True) + + def to_snake(inv): + b = inv["binary"] + b["commit_hash"] = b.pop("commitHash") + b["build_timestamp"] = b.pop("buildTimestamp") + + self._rewrite_invocations(root, to_snake) + data, warnings, _ = run_convert(root) + self.assertEqual(data["build"]["commit"], fixtures.COMMIT) + self.assertEqual(data["build"]["version"], fixtures.VERSION) + self.assertFalse(any("binary commit mismatch" in w for w in warnings)) + + def test_failed_run_invocation_warns(self): + # A failed run writes invocation.json too, with an `error` field + # (stellar-rpc#907) — its CSVs are partial, so conversion must warn. + tmp = tempfile.mkdtemp() + root = os.path.join(tmp, "failed") + fixtures.build_campaign_bundle(root, paced=True) + p = os.path.join(root, "ingest-hot-sac-6000-c1-run1", "invocation.json") + with open(p) as f: + inv = json.load(f) + inv["error"] = "context deadline exceeded" + with open(p, "w") as f: + json.dump(inv, f) + _, warnings, _ = run_convert(root) + self.assertTrue(any("FAILED run" in w and "ingest-hot-sac-6000-c1-run1" in w + for w in warnings)) + + class LegacyBundleTests(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/converter/tests/test_golden.py b/converter/tests/test_golden.py index f8a0415..ad9aa6d 100644 --- a/converter/tests/test_golden.py +++ b/converter/tests/test_golden.py @@ -1,5 +1,6 @@ """Golden tests against the two real local datasets. Skip cleanly (with a clear -message) when the source directories are absent.""" +message) when the source directories are absent or unreadable (e.g. macOS TCC +denying the terminal access to ~/Downloads).""" import argparse import os import sys @@ -11,6 +12,14 @@ PUBNET_DIR = os.path.expanduser("~/Downloads/bench-063a/results") SYNTH_DIR = os.path.expanduser("~/Downloads/bench-synth/results") + + +def _readable_dir(d): + try: + os.listdir(d) + return True + except OSError: + return False FACTS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "facts", "synthetic-2026-07-15.json") @@ -28,8 +37,8 @@ def run_convert(results_dir, **overrides): return data, list(convert._warnings) -@unittest.skipUnless(os.path.isdir(PUBNET_DIR), - f"pubnet dataset absent at {PUBNET_DIR}; skipping golden test") +@unittest.skipUnless(_readable_dir(PUBNET_DIR), + f"pubnet dataset absent or unreadable at {PUBNET_DIR}; skipping golden test") class PubnetGoldenTests(unittest.TestCase): @classmethod def setUpClass(cls): @@ -75,8 +84,8 @@ def test_vocabulary_and_checks(self): self.assertEqual(self.data["checks"]["kind"], "query_p99_threshold") -@unittest.skipUnless(os.path.isdir(SYNTH_DIR), - f"synthetic dataset absent at {SYNTH_DIR}; skipping golden test") +@unittest.skipUnless(_readable_dir(SYNTH_DIR), + f"synthetic dataset absent or unreadable at {SYNTH_DIR}; skipping golden test") class SynthGoldenTests(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/runner/README.md b/runner/README.md new file mode 100644 index 0000000..d4e705b --- /dev/null +++ b/runner/README.md @@ -0,0 +1,365 @@ +# Campaign runner + +The operations side of this repo: a config-driven runner that produces the result +bundles the rest of the pipeline consumes. It treats **stellar-rpc as a black box** — it +clones it, builds the requested ref, and drives its `bench-ingest` / `bench-query` +subcommands; it never lives inside a stellar-rpc checkout and never modifies one. + +``` +runner/ +├── bootstrap.sh # provision the devbox (NVMe, apt, Go, Rust, native libs, env) +├── cmd/campaign/ # the campaign CLI: run · plan · preflight · publish +├── internal/ +│ ├── config/ # TOML in, validated Config out (unknown keys rejected) +│ ├── plan/ # config → ordered steps; the campaign as data (plan.json) +│ ├── run/ # executes a plan: legs, sentinels, resume, the lock +│ ├── preflight/ # does this machine have what this config needs? +│ ├── publish/ # bundle → gs:// or s3:// +│ └── bundle/ # the bundle root: metadata.json, provenance files, +│ # and the checks a --resume must pass +└── example-campaign.toml # annotated config to copy from +``` + +Everything is invoked from `runner/`: + +```bash +cd runner +BENCH_ROOT=/mnt/nvme/bench go run ./cmd/campaign run my-campaign.toml +``` + +`go run` is the normal way in — the devbox has Go (bootstrap.sh installs a pinned +version), and a campaign spends hours in child processes, so the second it takes to +compile the runner is noise. `make runner-build` from the repo root compiles it instead; +`make runner-test` is the vet-and-test gate CI runs. + +`plan` and `run --dry-run` build nothing, download nothing, and execute nothing, so both +are readable on a laptop: + +```bash +cd runner +BENCH_ROOT=/tmp/bench go run ./cmd/campaign plan example-campaign.toml +``` + +The [top-level README](../README.md#run-a-campaign) walks through the operator flow end +to end, from a fresh devbox to a published bundle. + +## CLI reference + +### `campaign run [flags]` + +Build the configured ref, prepare its datasets, and execute every ingest and query leg +into a results bundle. One campaign per `$BENCH_ROOT`: the runner holds an exclusive +lock on `$BENCH_ROOT/.campaign.lock` and refuses a second campaign immediately rather +than queueing it — two campaigns sharing a machine measure each other. + +| Flag | Effect | +|------|--------| +| `--dry-run` | Print the plan and exit. Nothing is built, fetched, locked, or created. | +| `--resume DIR` | Continue an interrupted campaign into an existing bundle (see [Resuming](#resuming-a-crashed-campaign)). | +| `--fail-fast` | Stop at the first failed step. The default is to keep going: a failure only skips the steps that need it. | +| `--no-preflight` | Skip the up-front tool, credential, mount, and disk checks. | + +A failed step never costs the bundle: the epilogue — machine metadata, the final +`metadata.json`, the tarball — runs even after failures, because a campaign that went +wrong is exactly the one whose bundle has to be complete. The run exits nonzero and ends +with a summary naming every failed and skipped step. + +### `campaign plan ` + +Print the ordered steps the campaign would execute, one `$ command` line each. It +resolves no ref against the network and writes nothing; when the ref is not resolvable in +the local build clone it plans with the placeholder sha `deadbeef` in the derived paths +and says so. This is the same data the runner writes to `plan.json` in the bundle. + +### `campaign preflight ` + +Answer, in seconds, whether this machine has what this config needs: `git` and `make`, +the Go and Rust toolchains when a build is going to happen, `gcloud` for `packs-gs` +datasets and `gs://` publishing, `aws` for `s3://` publishing, a listable `publish_uri` +with today's credentials, the NVMe mount when `BENCH_ROOT` is left at its default, and +free disk. Failures name both the missing thing and the config key that wants it. +`campaign run` does this automatically unless `--no-preflight` is passed. + +### `campaign publish [dest-root] [flags]` + +Upload a finished bundle to `//`, where `run_id` is the bundle's +basename. `dest-root` defaults to `$PUBLISH_URI` from the environment — the same value a +config's `publish_uri` feeds the end of a run. `gs://` uploads with `gcloud storage rsync +-r`, `s3://` with `aws s3 sync`; no other scheme is supported. On success the last line +is `published: `, which scripts grep for. + +| Flag | Effect | +|------|--------| +| `--dry-run` | Print every cloud command, execute none of them. | +| `--force` | Write into a destination that already holds objects. | + +Published runs are immutable, which is why a non-empty destination is refused without +`--force`. **`--force` is a merge, not a replace:** the sync writes every object this +bundle has, but objects already at the destination that this bundle does not contain +survive it. Re-publishing a differently-shaped bundle over an old one therefore leaves +the old one's extra files in place and the destination is the union of the two — delete +the prefix first if you need a clean replace. + +## Config reference (TOML) + +A campaign config is a TOML file. Copy [`example-campaign.toml`](example-campaign.toml), +which carries the same reference as inline comments. **Unknown keys are rejected**, not +ignored: a misspelled key fails in the first second of the campaign, naming itself. + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `name` | string | — (required) | Campaign name, `[A-Za-z0-9._-]+`. The run id is `--`. | +| `repo` | string | `https://github.com/stellar/stellar-rpc.git` | Where stellar-rpc comes from: a git URL or an **absolute** local path to a git repository (relative paths are refused — they would depend on the invocation cwd). The persistent build clone at `$BENCH_ROOT/src` is cloned/fetched from it each campaign; `repo` itself is never modified. To benchmark local work-in-progress, point it at a local checkout — only committed state is benchmarkable. | +| `ref` | string | `feature/full-history` | Git ref to benchmark, resolved inside `$BENCH_ROOT/src` after fetching `repo`'s branches and tags. Built into `$BENCH_ROOT/bin/stellar-rpc-`. | +| `ingest` | string | — (required) | `cold` \| `hot` \| `both` \| `none`. | +| `query` | bool | — (required) | Run the query suites after ingest. Query-cold runs against each dataset's frozen pack root; query-hot needs the hot DB a hot ingest leaves behind, so it only runs when `ingest` is `hot` or `both` (otherwise the runner notes that it is running the cold suite only). | +| `close_interval` | string | `"0"` | `bench-ingest hot --close-interval`: a Go duration (`"2s"`, `"1s"`, `"600ms"`) for phase pacing, or `"0"` for unpaced catch-up. | +| `runs` | int ≥ 1 | `5` | Repetitions per (dataset, chunk) cell. | +| `query_concurrency` | int array | `[1, 4, 16]` | Query concurrency sweep; every entry ≥ 1. | +| `cold_iters` | int ≥ 1 | `100` | `bench-query cold --iters`. | +| `hot_iters` | int ≥ 1 | `200` | `bench-query hot --iters`. | +| `workers` | int ≥ 1 | `1` | `bench-ingest cold --workers`. | +| `hot_num_ledgers` | int ≥ 0 | `0` | Cap the hot ingest at this many ledgers; `0` = the whole range. A capped ingest also caps the hot query sampler (`--sample-ledgers`), so it stays inside what was ingested. | +| `publish_uri` | string | `""` | Object-storage root to publish the finished bundle to; must be `gs://` or `s3://`. Empty = no publish. The bundle lands at `//`. | +| `[[dataset]]` | table array | — (at least one) | See below. | + +Each `[[dataset]]` table: + +| Key | Type | Meaning | +|-----|------|---------| +| `name` | string | `[A-Za-z0-9._-]+`, unique across the config. Names the golden directory, the leg ids, and the converter's unit ids. | +| `kind` | string | `packs-local` \| `packs-gs` \| `bsb-s3` \| `fixture`. | +| `location` | string | Meaning depends on the kind (below). Invalid for `fixture`. | +| `chunks` | int array | Chunk IDs to benchmark; at least one, non-negative, no duplicates. | +| `ledgers` | int | **`fixture` only** (and required there): the per-chunk ledger count. `0` = the whole chunk; otherwise ≥ 10000, because the untimed cold freeze streams a whole 10,000-ledger chunk and cannot freeze a partial one. | + +Every kind converges on a local cold pack root — the directory holding `ledgers/`, +`events/`, `txhash/` — which is all the timed legs ever read. They differ only in how +that root is materialized: + +- **`packs-local`** — `location` **is** the cold pack root, used in place and never + written to. +- **`packs-gs`** — `location` is a `gs://` prefix of the same tree; fetched once into + `$BENCH_ROOT/golden//` and reused by later campaigns. +- **`bsb-s3`** — `location` is an S3 bucket path; an untimed cold backfill materializes + `$BENCH_ROOT/golden//`, one chunk at a time. +- **`fixture`** — no location and no network: `ledgers` chunks are generated, then an + untimed cold ingest freezes them into `$BENCH_ROOT/golden//`. + +Anything the runner materializes is built under `.partial` and renamed onto +`` only once whole, so an interrupted preparation is never mistaken for a finished +one. To force a re-fetch or a regeneration: `rm -rf $BENCH_ROOT/golden/`. + +## Migrating a `.cfg` to `.toml` + +Bash-era configs were sourced shell fragments; the CLI reads TOML. The keys map +one-to-one — lowercase the name, and give the value TOML's type instead of a string: + +| `.cfg` | `.toml` | Note | +|--------|---------|------| +| `NAME=phase4` | `name = "phase4"` | | +| `REPO=…` | `repo = "…"` | | +| `REF=feature/full-history` | `ref = "feature/full-history"` | | +| `INGEST=both` | `ingest = "both"` | | +| `QUERY=yes` / `QUERY=no` | `query = true` / `query = false` | A real bool now. | +| `CLOSE_INTERVAL=2s` | `close_interval = "2s"` | Still a string; `"0"` for unpaced. | +| `RUNS=5` | `runs = 5` | | +| `QC=1,4,16` | `query_concurrency = [1, 4, 16]` | Comma string → int array. | +| `COLD_ITERS=100` | `cold_iters = 100` | | +| `HOT_ITERS=200` | `hot_iters = 200` | | +| `WORKERS=1` | `workers = 1` | | +| `HOT_NUM_LEDGERS=0` | `hot_num_ledgers = 0` | | +| `PUBLISH_URI=gs://…` | `publish_uri = "gs://…"` | | +| `DATASETS=("name\|kind\|location\|chunks")` | one `[[dataset]]` table each | See below. | + +A dataset's four pipe-separated fields become four keys, and the chunk list becomes an +array: + +```bash +# .cfg +DATASETS=( + "mydata|packs-gs|gs://my-bucket/cold|1 2" + "synth|fixture|10000|1" +) +``` + +```toml +# .toml +[[dataset]] +name = "mydata" +kind = "packs-gs" +location = "gs://my-bucket/cold" +chunks = [1, 2] + +[[dataset]] +name = "synth" +kind = "fixture" +ledgers = 10000 # not location: the .cfg overloaded that field with the ledger count +chunks = [1] +``` + +The fixture dataset is the one entry whose shape really changes. Its `.cfg` location was +never a location — it was the per-chunk ledger count — so it moves to its own `ledgers` +key, and a `location` on a fixture dataset is now an error rather than a silently +reinterpreted number. + +Two configs bash accepted are now rejected: a repeated chunk ID within a dataset (it +produced two legs with the same id and the same `--out` directory, the second quietly +overwriting the first) and any key outside the table above. + +## Compatibility floor + +The runner requires a stellar-rpc ref whose bench subcommands **write `invocation.json` +into every `--out` directory** — stellar-rpc#907 (`6f35679f`) or any descendant of it. +That commit is merged into `feature/full-history`, so the default +`ref = "feature/full-history"` satisfies the floor. Older refs produce bundles without +per-invocation manifests, which the converter accepts but with weaker provenance (see +`SCHEMA.md` § Inputs). + +## `$BENCH_ROOT` layout + +Everything the runner touches lives under `$BENCH_ROOT` (default `/mnt/nvme/bench`, the +devbox's NVMe instance store — wiped on instance stop/start; everything here is +re-creatable): + +``` +$BENCH_ROOT/ +├── .campaign.lock the one-campaign-per-root flock; created once, never deleted +├── src/ persistent build clone of repo (re-pointed, fetched, and hard-reset +│ every campaign; gitignored build caches survive, so rebuilds are +│ incremental) +├── bin/ versioned binaries: stellar-rpc- +├── golden/ immutable prepared datasets, one dir per dataset name +│ (rm -rf golden/ to force a re-fetch) +├── fixture/ staging area for generated fixture packs +├── scratch/ cold-ingest output, deleted before and after every cold leg +├── hot/ hot DBs; the last run's DB is kept for the hot query suite +└── results/ campaign bundles: --/ +``` + +The finished bundle is also tarred to `/tmp/bench-results---.tgz` (EBS +root, survives an instance stop) and, when `publish_uri` is set, uploaded to +`/--/`. + +## Resuming a crashed campaign + +A campaign is hours of work — the phase-1 reference run took ~17 hours, with single +hot-ingest legs near 5.5 — so a crash or an OOM kill at the last rep should not cost the +whole thing. `--resume` continues into the existing results directory instead of starting +a new one: + +```bash +cd runner +BENCH_ROOT=/mnt/nvme/bench go run ./cmd/campaign run my-campaign.toml \ + --resume /mnt/nvme/bench/results/-- +``` + +**Identity comes from the bundle, not its name.** The runner reads the bundle's +`metadata.json` and refuses unless its `run_id`, campaign `name`, and `built_commit` +match this config and the commit `ref` resolves to right now — resuming onto a different +commit would mix two binaries inside one bundle. A renamed or copied directory therefore +cannot pass itself off as another campaign, and a bundle with no readable +`metadata.json` (pre-crash-safe, or not a bundle at all) is refused outright. + +**The config must be byte-identical.** The bundle stores the config it was started with; +a resume compares the file you passed against that stored copy and, on any difference, +prints a unified diff and stops. The stored copy is never overwritten — it is the record +of what produced the legs already in the bundle, and a resume with edited knobs would +otherwise produce mixed data under a manifest uniformly claiming the new ones. + +**Completion is decided by the runner's own sentinel.** Every timed leg gets a `leg.json` +written into its `--out` directory when the process ends, success or failure; a leg is +skipped only when its sentinel identifies itself — this schema version and *this leg's* +id — and records a zero exit code with no error. Anything else — a recorded failure, a +corrupt sentinel, a sentinel naming another leg, a leg killed before the sentinel was +written — is wiped and re-run, because a half-written leg silently kept would corrupt the +aggregates the converter computes. Bundles produced by the bash runner have no `leg.json`, so those +fall back to the old heuristic: both `invocation.json` and `driver.csv` present, with no +`error` field in the manifest. + +Add `--dry-run` to print the plan against the real directory, annotated with the legs a +resume would skip, before committing hours to it. That combination is the one dry run +that needs the build clone: a resume is only valid against the commit its bundle was +benchmarked with, so it is refused rather than planned with a placeholder sha — run it on +the box, not on a laptop. The run id is reused, so the bundle +keeps its identity: `metadata.json` still carries the original `started_at` (recovered +from the bundle), `finished_at` is the last session's end, and `campaign.resumed` records +that the bundle took more than one session. `campaign.log` accumulates every session's +console output, so the whole history stays in the bundle. + +A resumed session keeps going past failures like any other, so one broken leg does not +end the session — it fails, the steps that need it are skipped, and the rest of the +campaign runs. `--fail-fast` stops at the first failure instead, which is what you want +when the failure is likely to repeat. + +**Same boot only.** `$BENCH_ROOT` is the NVMe instance store: stopping and starting the +instance wipes the results directory, the golden datasets, and the hot DBs together. If the +results directory survived, resume it; if the box restarted, there is nothing to resume onto +and the campaign starts over. The hot query suite reads the DB the last hot ingest left +behind, so in the unlikely case that the DB is gone but the results directory is not, those +query legs fail (and the summary names them) — re-run the hot ingest for that cell by +deleting its `ingest-hot--c-run*` directories and resuming again. + +## Campaign bundle layout — the cross-repo contract + +A campaign bundle is what `campaign publish` uploads and what `converter/convert.py` +consumes (as the **campaign** input layout). Two repos write into it, so its shape is a +contract: + +``` +--/ # run_id = the bundle basename +├── .toml # the campaign config, verbatim +├── binary.txt # benchmarked binary identity (free text) +├── machine-metadata.txt # machine facts (free text) +├── campaign.log # the runner's console output, one session +│ # appended per --resume (free text) +├── metadata.json # ← written by the campaign CLI (THIS repo) +├── plan.json # ← written by the campaign CLI (THIS repo) +├── golden--c/ # untimed dataset prep — not results; +│ # the converter skips these and warns +├── ingest-{cold,hot}--c-run/ +│ ├── driver.csv, hot.csv, *.csv # ← written by stellar-rpc bench subcommands +│ ├── invocation.json # ← written by stellar-rpc bench subcommands +│ └── leg.json # ← written by the campaign CLI (THIS repo) +└── query-{cold,hot}--c-run/ + └── …same shape… +``` + +Who owns what: + +- **`metadata.json`** (bundle root, `schema_version` 1) — written by the campaign CLI + here. Run identity (`run_id`, `started_at`), the campaign config knobs (incl. + `close_interval`), the dataset list, structured `hardware`, and `hostname`. It is written + as soon as the bundle directory exists — without `finished_at`, which only the + end-of-campaign rewrite adds — so a campaign that is killed still leaves a parseable + bundle. It also carries `status`: `running` up front, rewritten to `finished` or + `failed` at the end (additive; bash-era bundles have none, and readers treat an absent + status as unknown). Only the root-level free files (the config, `binary.txt`, + `machine-metadata.txt`, `campaign.log`) sit outside the contract; the converter reads + named files and per-leg subdirectories, so adding one is safe. +- **`plan.json`** (bundle root, `schema_version` 1) — written by the campaign CLI here: + the campaign as data, the same steps `campaign plan` prints, with their ids, kinds, + argv, dependencies, and derived paths. It is rewritten on every session of a resumed + campaign. Additive changes (new fields, new step kinds) keep the version. The converter + ignores it today; it is there so a bundle can say what it intended to run, not only + what it produced. +- **`leg.json`** (each timed `--out` dir, `schema_version` 1) — written by the campaign + CLI here, after the benchmark process ends, whether it succeeded or not: + `schema_version`, `id`, `argv`, `exit_code`, `started_at`, `finished_at`, + `duration_ns`, and `error` on a failure. It is the runner's completion sentinel and the + reason resume no longer has to infer completion from the presence of + `invocation.json` — that file is written *by the process being measured*, so a process + killed before it got there leaves no trace at all. The converter ignores it today. +- **`invocation.json`** (each `--out` dir, `schemaVersion` 1, camelCase keys) — written by + stellar-rpc's `bench-ingest` / `bench-query` (`invocation.go`, merged as + stellar-rpc#907). Binary identity (`binary.{commitHash, branch, version, + buildTimestamp}`), the resolved subcommand flags, `hostname`, + `startedAt`/`finishedAt`, and — on a failed run only — an `error` field. It is written + for failed runs too, so its presence alone is not a success marker. + +The consumer side of this contract — exactly which fields the converter reads, and the +precedence rules between the manifests, the free-text metadata, and CLI arguments — is +documented in [`SCHEMA.md` § Inputs](../SCHEMA.md#inputs--result-bundle-layouts--manifests). +Changing any of these manifests' shape, the bundle directory naming, or the CSV columns +is a cross-repo change: update the producer (here or in stellar-rpc), the converter, and +`SCHEMA.md` together. diff --git a/runner/bootstrap.sh b/runner/bootstrap.sh new file mode 100755 index 0000000..f731491 --- /dev/null +++ b/runner/bootstrap.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Idempotent bootstrap for a full-history benchmark machine: an EC2 instance +# with a local NVMe instance store (e.g. m6id.2xlarge) running Ubuntu 24.04. +# It only provisions — NVMe mount, apt packages, Go, Rust, native libs, env; +# the campaign CLI does all cloning-current and building. Safe to re-run any time — +# in particular after an instance stop/start, which wipes the NVMe instance +# store (golden packs are re-downloaded and the build clone re-created by the +# next bootstrap/campaign run). +# +# Usage (on the machine): +# ./runner/bootstrap.sh +# +# Overridable: NVME_DEV (default /dev/nvme1n1), BENCH_ROOT (default +# /mnt/nvme/bench), REPO (git URL or local path of stellar-rpc, default +# https://github.com/stellar/stellar-rpc.git). +# +set -euo pipefail + +NVME_DEV="${NVME_DEV:-/dev/nvme1n1}" +MOUNT=/mnt/nvme +BENCH_ROOT="${BENCH_ROOT:-$MOUNT/bench}" +REPO="${REPO:-https://github.com/stellar/stellar-rpc.git}" +SRC=$BENCH_ROOT/src + +note() { echo "== $*"; } + +# --- NVMe instance store: format if raw, mount if unmounted ----------------- +[ -b "$NVME_DEV" ] || { echo "error: $NVME_DEV is not a block device" >&2; exit 1; } +model=$(lsblk -no MODEL "$NVME_DEV" | head -1) +case "$model" in + *"Instance Storage"*) ;; + *) echo "error: refusing to touch $NVME_DEV — model '$model' is not the EC2 instance store" >&2; exit 1 ;; +esac +if ! sudo blkid "$NVME_DEV" >/dev/null 2>&1; then + note "no filesystem on $NVME_DEV (fresh instance store) — formatting" + sudo mkfs.ext4 -m0 "$NVME_DEV" +fi +if ! mountpoint -q "$MOUNT"; then + sudo mkdir -p "$MOUNT" + sudo mount -o noatime "$NVME_DEV" "$MOUNT" + sudo chown "$USER" "$MOUNT" +fi +mkdir -p "$BENCH_ROOT"/{golden,scratch,hot,results} + +# --- fsync honesty probe: the whole reason this machine exists -------------- +probe=$(dd if=/dev/zero of="$MOUNT/.fsync-probe" bs=4k count=2000 oflag=dsync 2>&1 | tail -1) +rm -f "$MOUNT/.fsync-probe" +note "fsync probe: $probe" +case "$probe" in + *GB/s*) echo "WARNING: GB/s-scale dsync writes — fsync is being absorbed; hot-commit numbers would be fiction" >&2 ;; +esac + +# --- system packages --------------------------------------------------------- +note "apt packages" +sudo apt-get update -qq +sudo apt-get install -y -qq build-essential git jq pkg-config cmake ninja-build \ + tmux libsnappy-dev liblz4-dev zlib1g-dev + +# --- cloud CLIs: only some campaigns need them, so warn rather than fail ----- +# gcloud: packs-gs datasets and gs:// publishing. aws: bsb-s3 datasets and +# s3:// publishing. Neither ships in apt in a form worth installing here. +command -v gcloud >/dev/null 2>&1 || + echo "WARNING: gcloud not found — packs-gs datasets and gs:// publish_uri will fail; install it: https://cloud.google.com/sdk/docs/install" >&2 +command -v aws >/dev/null 2>&1 || + echo "WARNING: aws not found — bsb-s3 datasets and s3:// publish_uri will fail; install it: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" >&2 + +# --- Go (pinned; Noble's apt Go is too old) ---------------------------------- +# Pinned so every box benchmarks with the same compiler — a toolchain bump moves +# the numbers. When bumping, bump runner/go.mod's `go` directive with it and +# re-baseline before comparing against older runs. Any other version installed +# here gets the pinned one laid over it. +GOVER=go1.26.5 +if ! /usr/local/go/bin/go version 2>/dev/null | grep -qF " $GOVER "; then + note "installing Go $GOVER" + curl -fsSL "https://go.dev/dl/${GOVER}.linux-amd64.tar.gz" -o /tmp/go.tgz + # decompress as the user: sudo'd tar cannot always exec gzip + gunzip -f /tmp/go.tgz + sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xf /tmp/go.tar +fi + +# --- Rust (pinned) ----------------------------------------------------------- +# Same reasoning as Go: rustc builds the native libs, so its version is part of +# the measurement. When bumping, re-baseline before comparing against older runs. +RUSTVER=1.92.0 +if [ ! -x "$HOME/.cargo/bin/rustc" ]; then + note "installing Rust $RUSTVER" + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUSTVER" +elif ! "$HOME/.cargo/bin/rustc" --version 2>/dev/null | grep -qF "rustc $RUSTVER "; then + # rustup is idempotent, so re-pinning an off-version box is a no-op re-run away + note "pinning Rust to $RUSTVER" + "$HOME/.cargo/bin/rustup" toolchain install "$RUSTVER" && + "$HOME/.cargo/bin/rustup" default "$RUSTVER" +fi + +# --- build clone -------------------------------------------------------------- +# The box needs no standalone stellar-rpc checkout: seed the persistent build +# clone the campaign CLI maintains at $BENCH_ROOT/src, and run the native-lib +# install scripts below from it. `campaign run` re-points, fetches, and checks +# out this clone per campaign (and re-clones it itself if this step is ever +# skipped). +if [ ! -d "$SRC/.git" ]; then + note "cloning $REPO into $SRC" + git clone "$REPO" "$SRC" +fi + +# --- native libs, mirroring CI's setup-go action ------------------------------ +[ -e "$HOME/.zstd/lib/libzstd.so" ] || + (cd "$SRC" && PREFIX="$HOME/.zstd" ./scripts/install-zstd.sh) +[ -e "$HOME/.rocksdb/lib/librocksdb.so" ] || + (cd "$SRC" && PREFIX="$HOME/.rocksdb" ZSTD_HOME="$HOME/.zstd" ./scripts/install-rocksdb.sh) + +# --- environment: persist for future shells, set for this run ---------------- +if ! grep -q '# bench-campaigns env' "$HOME/.bashrc"; then + cat >> "$HOME/.bashrc" <<'EOF' +# bench-campaigns env +export PATH=/usr/local/go/bin:$HOME/go/bin:$HOME/.cargo/bin:$PATH +export CGO_CFLAGS="-I$HOME/.zstd/include -I$HOME/.rocksdb/include" +export CGO_LDFLAGS="-L$HOME/.zstd/lib -L$HOME/.rocksdb/lib" +export LD_LIBRARY_PATH="$HOME/.zstd/lib:$HOME/.rocksdb/lib" +EOF +fi +export PATH=/usr/local/go/bin:$HOME/go/bin:$HOME/.cargo/bin:$PATH +export CGO_CFLAGS="-I$HOME/.zstd/include -I$HOME/.rocksdb/include" +export CGO_LDFLAGS="-L$HOME/.zstd/lib -L$HOME/.rocksdb/lib" +export LD_LIBRARY_PATH="$HOME/.zstd/lib:$HOME/.rocksdb/lib" + +note "bootstrap OK — the campaign CLI builds the benchmark binary on first run" diff --git a/runner/cmd/campaign/e2e_test.go b/runner/cmd/campaign/e2e_test.go new file mode 100644 index 0000000..c241033 --- /dev/null +++ b/runner/cmd/campaign/e2e_test.go @@ -0,0 +1,640 @@ +//go:build !windows + +package main + +// End-to-end tests: the real campaign binary, exec'd against a temp BENCH_ROOT +// with a fake toolchain on PATH and a stub standing in for stellar-rpc. Nothing +// else in this repo proves the pieces compose — config, plan, executor, bundle +// writers, resume, and the epilogue — and scenario ConverterOverBundle proves +// the bundle they produce is one converter/convert.py can convert, which is the +// cross-repo contract these benchmarks exist to keep. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// stubCommit is what the fake git resolves every ref to and what the stub +// binary reports as its build identity; the two must agree or the converter +// warns about a commit mismatch. stubSha8 is the short sha every derived path +// carries. +const ( + stubCommit = "0123456789abcdef0123456789abcdef01234567" + stubSha8 = "01234567" +) + +// waitTimeout caps every poll in this file: a regression that stops a campaign +// from reaching a leg should fail the test in seconds, not hang the suite. +// runTimeout does the same for a whole campaign invocation, and killGrace is +// how long a killed group has to release the output pipe before Wait gives up +// on it. +const ( + waitTimeout = 30 * time.Second + runTimeout = 60 * time.Second + killGrace = 5 * time.Second +) + +// legNames are the eight timed legs the e2e config produces, in plan order. +var legNames = []string{ + "ingest-cold-fix-c1-run1", "ingest-cold-fix-c1-run2", + "ingest-hot-fix-c1-run1", "ingest-hot-fix-c1-run2", + "query-cold-fix-c1-run1", "query-cold-fix-c1-run2", + "query-hot-fix-c1-run1", "query-hot-fix-c1-run2", +} + +var reBundleName = regexp.MustCompile(`^e2e-` + stubSha8 + `-[0-9]{8}T[0-9]{6}Z$`) + +func TestE2E(t *testing.T) { + work := t.TempDir() + bin := buildCampaign(t, work) + cfgPath := absPath(t, filepath.Join("testdata", "e2e-campaign.toml")) + + // ConverterOverBundle converts the bundle HappyPath produced, so the two + // share it through the parent test rather than each running a campaign. + var happyBundle string + + t.Run("HappyPath", func(t *testing.T) { + sc := newScenario(t, bin, filepath.Join(work, "happy"), cfgPath) + code, out := sc.run(t, "run", sc.cfg, "--no-preflight") + if code != 0 { + t.Fatalf("exit code = %d, want 0; output:\n%s", code, out) + } + bundle := sc.bundle(t) + happyBundle = bundle + + if name := filepath.Base(bundle); !reBundleName.MatchString(name) { + t.Errorf("bundle name = %q, want e2e--", name) + } + // The untimed prep dir the golden freeze wrote, and the eight timed legs. + mustExist(t, filepath.Join(bundle, "golden-fix-c1")) + for _, leg := range legNames { + dir := filepath.Join(bundle, leg) + for _, f := range []string{"driver.csv", "invocation.json", "leg.json"} { + mustExist(t, filepath.Join(dir, f)) + } + if got := readLeg(t, dir).ExitCode; got != 0 { + t.Errorf("%s: leg.json exit_code = %d, want 0", leg, got) + } + } + for _, f := range []string{"plan.json", "binary.txt", "machine-metadata.txt", "campaign.log"} { + mustExist(t, filepath.Join(bundle, f)) + } + + meta := readMetadata(t, bundle) + if meta.FinishedAt == "" { + t.Error("metadata.json has no finished_at") + } + if meta.Status != "finished" { + t.Errorf("metadata.json status = %q, want finished", meta.Status) + } + // Absent, not false: the manifest omits resumed entirely on a campaign + // that ran in one session. + if raw := readFile(t, filepath.Join(bundle, "metadata.json")); strings.Contains(raw, `"resumed"`) { + t.Errorf("metadata.json records resumed on a fresh campaign:\n%s", raw) + } + + // The cold scratch store is post-cleaned the moment its rep is done; the + // hot DB is deliberately kept, because query-hot reads what it holds. + mustNotExist(t, filepath.Join(sc.root, "scratch", "fix", "1")) + mustExist(t, filepath.Join(sc.root, "hot", "fix", "1")) + }) + + t.Run("KillMidLegResume", func(t *testing.T) { + sc := newScenario(t, bin, filepath.Join(work, "resume"), cfgPath) + sc.setControl("hang ingest-hot-fix-c1-run2") + + proc := sc.start(t, "run", sc.cfg, "--no-preflight") + bundle := waitForBundle(t, sc.root) + killed := filepath.Join(bundle, "ingest-hot-fix-c1-run2") + waitFor(t, "the hung leg's out dir", func() bool { return exists(killed) }) + // The whole process group: the stub's sleep must die with the runner. + proc.terminate() + if s := proc.out.String(); !strings.Contains(s, "ingest-hot-fix-c1-run2") { + t.Fatalf("killed before the campaign reached the hung leg; output:\n%s", s) + } + + sc.setControl("") + code, resumeOut := sc.run(t, "run", sc.cfg, "--no-preflight", "--resume", bundle) + if code != 0 { + t.Fatalf("resume exit code = %d, want 0; output:\n%s", code, resumeOut) + } + + log := readFile(t, filepath.Join(bundle, "campaign.log")) + for _, want := range []string{ + "session start", + "session resume", + "resume: ingest-cold-fix-c1-run1 already complete — skipping", + "resume: ingest-hot-fix-c1-run2 is a partial leg — wiping and re-running", + } { + if !strings.Contains(log, want) { + t.Errorf("campaign.log missing %q", want) + } + } + if got := readLeg(t, killed).ExitCode; got != 0 { + t.Errorf("re-run leg exit_code = %d, want 0", got) + } + meta := readMetadata(t, bundle) + if !meta.Campaign.Resumed { + t.Error("metadata.json does not record resumed") + } + if meta.Status != "finished" { + t.Errorf("metadata.json status = %q, want finished", meta.Status) + } + }) + + t.Run("FailedLegKeepGoing", func(t *testing.T) { + sc := newScenario(t, bin, filepath.Join(work, "failed"), cfgPath) + // The last hot rep: query-hot depends on it, so its failure is what + // takes the hot query suite down with it. + sc.setControl("fail ingest-hot-fix-c1-run2") + + code, out := sc.run(t, "run", sc.cfg, "--no-preflight") + if code == 0 { + t.Fatalf("exit code = 0, want nonzero; output:\n%s", out) + } + bundle := sc.bundle(t) + log := readFile(t, filepath.Join(bundle, "campaign.log")) + for _, want := range []string{ + "skipping query-hot-fix-c1-run1: needs ingest-hot-fix-c1-run2", + "skipping query-hot-fix-c1-run2: needs ingest-hot-fix-c1-run2", + "== campaign summary:", + } { + if !strings.Contains(log, want) { + t.Errorf("campaign.log missing %q", want) + } + } + // A hot failure must not take the cold query suite with it: it needs + // nothing the failed leg produced. + for _, leg := range []string{"query-cold-fix-c1-run1", "query-cold-fix-c1-run2"} { + if got := readLeg(t, filepath.Join(bundle, leg)).ExitCode; got != 0 { + t.Errorf("%s: leg.json exit_code = %d, want 0", leg, got) + } + } + mustNotExist(t, filepath.Join(bundle, "query-hot-fix-c1-run1")) + + failed := filepath.Join(bundle, "ingest-hot-fix-c1-run2") + leg := readLeg(t, failed) + if leg.ExitCode != 1 || leg.Error == "" { + t.Errorf("failed leg.json = exit_code %d, error %q; want exit_code 1 with an error", + leg.ExitCode, leg.Error) + } + if inv := readFile(t, filepath.Join(failed, "invocation.json")); !strings.Contains(inv, `"error": "induced failure"`) { + t.Errorf("failed leg's invocation.json carries no error:\n%s", inv) + } + + meta := readMetadata(t, bundle) + if meta.Status != "failed" { + t.Errorf("metadata.json status = %q, want failed", meta.Status) + } + if meta.FinishedAt == "" { + t.Error("a failed campaign's metadata.json has no finished_at") + } + }) + + t.Run("ResumeRefusesEditedConfig", func(t *testing.T) { + // The resume runs from a copy: editing the file the operator passed must + // leave the bundle's stored copy — what the guard compares against — + // untouched. + original := readFile(t, cfgPath) + working := filepath.Join(work, "edited-cfg") + mkdirAll(t, working) + copyPath := filepath.Join(working, "e2e-campaign.toml") + writeFile(t, copyPath, original) + + sc := newScenario(t, bin, filepath.Join(work, "edited"), copyPath) + if code, out := sc.run(t, "run", sc.cfg, "--no-preflight"); code != 0 { + t.Fatalf("exit code = %d, want 0; output:\n%s", code, out) + } + bundle := sc.bundle(t) + + writeFile(t, copyPath, original+"\n# an operator edit between sessions\n") + code, out := sc.run(t, "run", sc.cfg, "--no-preflight", "--resume", bundle) + if code != 1 { + t.Fatalf("resume exit code = %d, want 1; output:\n%s", code, out) + } + if !strings.Contains(out, "--resume: the config differs") { + t.Errorf("output does not refuse the edited config:\n%s", out) + } + if stored := readFile(t, filepath.Join(bundle, "e2e-campaign.toml")); stored != original { + t.Errorf("the bundle's stored config changed:\n%s", stored) + } + }) + + t.Run("ConverterOverBundle", func(t *testing.T) { + if happyBundle == "" { + t.Skip("HappyPath produced no bundle to convert") + } + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("python3 not available") + } + convert := absPath(t, filepath.Join("..", "..", "..", "converter", "convert.py")) + outDir := t.TempDir() + ctx, cancel := context.WithTimeout(context.Background(), runTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, python, convert, happyBundle, + "--run-id", "e2e-test", "--run-name", "E2E", "--run-date", "2026-08-01", + "--dataset-kind", "synthetic", "--out-dir", outDir) + // Warnings are expected: the golden-* prep dirs are dataset preparation, + // and the converter says so every time it skips them. + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("convert.py: %v\n%s", err, out) + } + mustExist(t, filepath.Join(outDir, "e2e-test.json")) + }) + + t.Run("TarFailureGatesPublish", func(t *testing.T) { + // A publish-configured campaign whose tar fails: the tarball is the + // artifact publish uploads, so publish must not be reached. + cfgBody := strings.Replace(readFile(t, cfgPath), "[[dataset]]", + "publish_uri = \"gs://e2e-bucket/results\"\n\n[[dataset]]", 1) + cfgCopy := filepath.Join(work, "publish-cfg", "e2e-campaign.toml") + mkdirAll(t, filepath.Dir(cfgCopy)) + writeFile(t, cfgCopy, cfgBody) + + sc := newScenario(t, bin, filepath.Join(work, "tarfail"), cfgCopy) + sc.breakTar() + + code, out := sc.run(t, "run", sc.cfg, "--no-preflight") + if code != 1 { + t.Fatalf("exit code = %d, want 1; output:\n%s", code, out) + } + bundle := sc.bundle(t) + log := readFile(t, filepath.Join(bundle, "campaign.log")) + for _, want := range []string{"warning: tar failed", "skipping publish: tar failed"} { + if !strings.Contains(log, want) { + t.Errorf("campaign.log missing %q", want) + } + } + if strings.Contains(log, "published:") { + t.Errorf("campaign.log reports a publish after a failed tar:\n%s", log) + } + // tar is the epilogue's archiver, not a leg: the legs all passed, and + // the status is theirs. + if meta := readMetadata(t, bundle); meta.Status != "finished" { + t.Errorf("metadata.json status = %q, want finished", meta.Status) + } + }) +} + +// ------------------------------------------------------------------ scenario + +// scenario is one campaign's world: its own BENCH_ROOT, its own control file, +// and a fake toolchain on PATH ahead of the real one, so git, make, and (when a +// test asks for it) tar answer the way that scenario needs while the real sh, +// mv, and python3 stay reachable. +type scenario struct { + bin string // the campaign binary under test + cfg string // the config path passed on the CLI + root string // BENCH_ROOT + control string // STUB_RPC_CONTROL + path []string // dirs prepended to PATH, in order +} + +func newScenario(t *testing.T, bin, root, cfg string) *scenario { + t.Helper() + sc := &scenario{bin: bin, cfg: cfg, root: root, control: filepath.Join(root, "control")} + // The build clone is a black box to the runner: a .git is all EnsureSrc and + // ResolveRef look for before handing the work to git, which is fake here. + src := filepath.Join(root, "src") + mkdirAll(t, filepath.Join(src, ".git")) + writeFile(t, sc.control, "") + + stubs := filepath.Join(root, "toolchain") + mkdirAll(t, stubs) + writeExec(t, filepath.Join(stubs, "git"), fmt.Sprintf(`#!/bin/sh +# Fake git: every rev-parse resolves to one fixed commit, and the clone, +# fetch, reset, and checkout the runner drives are no-ops. +for arg in "$@"; do + if [ "$arg" = rev-parse ]; then + echo %s + exit 0 + fi +done +exit 0 +`, stubCommit)) + writeExec(t, filepath.Join(stubs, "make"), fmt.Sprintf(`#!/bin/sh +# Fake make: build-rpc-v2 installs the stub binary where the real target leaves +# it, for the runner's own mv to move into the versioned path. Every other +# target succeeds without doing anything. +src= +prev= +target= +for arg in "$@"; do + if [ "$prev" = -C ]; then src=$arg; fi + if [ "$arg" = build-rpc-v2 ]; then target=$arg; fi + prev=$arg +done +[ -n "$target" ] || exit 0 +cp %s "$src/stellar-rpc-v2" +chmod +x "$src/stellar-rpc-v2" +`, absPath(t, filepath.Join("testdata", "stub-rpc.sh")))) + sc.path = []string{stubs} + + // Every campaign tars its bundle into /tmp, a path the plan owns; take the + // tarball with the scenario rather than leaving it behind. + t.Cleanup(func() { + bundles, _ := filepath.Glob(filepath.Join(root, "results", "*")) + for _, b := range bundles { + os.Remove("/tmp/bench-results-" + filepath.Base(b) + ".tgz") + } + }) + return sc +} + +// setControl rewrites the stub's control file. An empty body clears it. +func (sc *scenario) setControl(body string) { + if body != "" { + body += "\n" + } + if err := os.WriteFile(sc.control, []byte(body), 0o644); err != nil { + panic(err) + } +} + +// breakTar puts a failing tar ahead of everything else on PATH, for the one +// scenario that needs the archiver to fail. +func (sc *scenario) breakTar() { + dir := filepath.Join(sc.root, "broken-tar") + if err := os.MkdirAll(dir, 0o755); err != nil { + panic(err) + } + body := "#!/bin/sh\necho 'tar: stub failure' >&2\nexit 1\n" + if err := os.WriteFile(filepath.Join(dir, "tar"), []byte(body), 0o755); err != nil { + panic(err) + } + sc.path = append([]string{dir}, sc.path...) +} + +func (sc *scenario) env() []string { + var env []string + for _, kv := range os.Environ() { + switch { + case strings.HasPrefix(kv, "PATH="), + strings.HasPrefix(kv, "BENCH_ROOT="), + // PUBLISH_URI would give `campaign publish` a destination the + // scenario never asked for. + strings.HasPrefix(kv, "PUBLISH_URI="), + strings.HasPrefix(kv, "STUB_RPC_"): + default: + env = append(env, kv) + } + } + path := append(append([]string{}, sc.path...), os.Getenv("PATH")) + return append(env, + "PATH="+strings.Join(path, string(os.PathListSeparator)), + "BENCH_ROOT="+sc.root, + "STUB_RPC_CONTROL="+sc.control, + "STUB_RPC_COMMIT="+stubCommit, + ) +} + +// run executes the campaign binary to completion, returning its exit code and +// its combined output. The campaign gets its own process group and a deadline: +// a runner that wedges fails this test rather than the suite. +func (sc *scenario) run(t *testing.T, args ...string) (int, string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), runTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, sc.bin, args...) + cmd.Env = sc.env() + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + // The group, not the leader: the stub's children hold the output pipe, and + // killing only the campaign would leave them writing into it. + cmd.Cancel = func() error { killGroup(cmd); return nil } + cmd.WaitDelay = killGrace + out, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("campaign did not finish within %s; output:\n%s", runTimeout, out) + } + code := 0 + if err != nil { + code = cmd.ProcessState.ExitCode() + } + return code, string(out) +} + +// started is one running campaign: the output it is still writing, and the one +// call that ends it. +type started struct { + out *syncBuffer + // terminate SIGKILLs the whole process group and reaps it, at most once. + // Signalling a group twice is not safe: after the first call reaps the + // leader the kernel may hand that PGID to an unrelated process, and the + // second signal would land on it. + terminate func() +} + +// start launches the campaign in its own process group, so a test can kill the +// runner and everything it spawned the way an operator's ^C would not. +func (sc *scenario) start(t *testing.T, args ...string) *started { + t.Helper() + cmd := exec.Command(sc.bin, args...) + cmd.Env = sc.env() + buf := &syncBuffer{} + cmd.Stdout, cmd.Stderr = buf, buf + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.WaitDelay = killGrace + if err := cmd.Start(); err != nil { + t.Fatalf("start campaign: %v", err) + } + var once sync.Once + proc := &started{out: buf, terminate: func() { + once.Do(func() { + killGroup(cmd) + _ = cmd.Wait() + }) + }} + // Whatever ends this test — a passing kill, or a poll deadline that fails + // it before the kill — the campaign and its sleeping stub die with it. + t.Cleanup(proc.terminate) + return proc +} + +// killGroup SIGKILLs a started command's whole process group. +func killGroup(cmd *exec.Cmd) { + if cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } +} + +// bundle is the single results directory this scenario produced. +func (sc *scenario) bundle(t *testing.T) string { + t.Helper() + hits, err := filepath.Glob(filepath.Join(sc.root, "results", "e2e-*")) + if err != nil || len(hits) != 1 { + t.Fatalf("results dirs = %v (err %v), want exactly one", hits, err) + } + return hits[0] +} + +// waitForBundle polls for the results directory of a campaign that is still +// running. +func waitForBundle(t *testing.T, root string) string { + t.Helper() + var found string + waitFor(t, "the results directory", func() bool { + hits, _ := filepath.Glob(filepath.Join(root, "results", "e2e-*")) + if len(hits) == 1 { + found = hits[0] + return true + } + return false + }) + return found +} + +// waitFor polls cond until it holds or waitTimeout expires. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(waitTimeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("timed out after %s waiting for %s", waitTimeout, what) +} + +// syncBuffer is a bytes.Buffer a test may read while the child process is still +// writing into it. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// ------------------------------------------------------------------- helpers + +func buildCampaign(t *testing.T, dir string) string { + t.Helper() + bin := filepath.Join(dir, "campaign") + out, err := exec.Command("go", "build", "-o", bin, ".").CombinedOutput() + if err != nil { + t.Fatalf("go build: %v\n%s", err, out) + } + return bin +} + +// legJSON is the runner's completion sentinel, read back the way an operator +// diagnosing a bundle would. +type legJSON struct { + ExitCode int `json:"exit_code"` + Error string `json:"error"` +} + +func readLeg(t *testing.T, dir string) legJSON { + t.Helper() + var leg legJSON + readJSON(t, filepath.Join(dir, "leg.json"), &leg) + return leg +} + +// metadataJSON is the sliver of the bundle manifest these tests assert on. +type metadataJSON struct { + Campaign struct { + Resumed bool `json:"resumed"` + } `json:"campaign"` + FinishedAt string `json:"finished_at"` + Status string `json:"status"` +} + +func readMetadata(t *testing.T, bundle string) metadataJSON { + t.Helper() + var meta metadataJSON + readJSON(t, filepath.Join(bundle, "metadata.json"), &meta) + return meta +} + +func readJSON(t *testing.T, path string, into any) { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if err := json.Unmarshal(b, into); err != nil { + t.Fatalf("parse %s: %v", path, err) + } +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func writeFile(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func writeExec(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func mkdirAll(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir -p %s: %v", dir, err) + } +} + +func absPath(t *testing.T, path string) string { + t.Helper() + abs, err := filepath.Abs(path) + if err != nil { + t.Fatalf("abs %s: %v", path, err) + } + return abs +} + +func exists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func mustExist(t *testing.T, path string) { + t.Helper() + if !exists(path) { + t.Errorf("missing: %s", path) + } +} + +func mustNotExist(t *testing.T, path string) { + t.Helper() + if exists(path) { + t.Errorf("present, want absent: %s", path) + } +} diff --git a/runner/cmd/campaign/main.go b/runner/cmd/campaign/main.go new file mode 100644 index 0000000..8d46b40 --- /dev/null +++ b/runner/cmd/campaign/main.go @@ -0,0 +1,257 @@ +// Command campaign runs config-driven benchmark campaigns for stellar-rpc's +// full-history bench subcommands. It is the Go successor to +// runner/campaign.sh and runner/publish.sh. +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/plan" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/preflight" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/publish" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/run" +) + +// defaultBenchRoot is the benchmark machine's NVMe mount; BENCH_ROOT overrides +// it on any other machine. +const defaultBenchRoot = "/mnt/nvme/bench" + +// placeholderSha stands in for the built commit when the ref cannot be +// resolved locally. It must be 8 hex digits, or a --resume would reject the run +// ids derived from it as malformed. +const placeholderSha = "deadbeef" + +// stampLayout is the run id's UTC timestamp, e.g. 20260101T000000Z. +const stampLayout = "20060102T150405Z" + +const topUsage = `campaign — stellar-rpc full-history benchmark campaigns + +usage: campaign [args] + +subcommands: + run run a campaign from a config, producing a results bundle + plan print the steps a campaign would execute, without running them + preflight check tools, credentials, and disk this config needs + publish upload a finished results bundle to object storage + +environment: + BENCH_ROOT storage root for the build clone, datasets, scratch space, and + results (default /mnt/nvme/bench) +` + +// Usage text for each subcommand, describing the shape it will have once +// ported. Keyed by subcommand name. +var subUsage = map[string]string{ + "run": `usage: campaign run [--dry-run] [--resume ] [--fail-fast] [--no-preflight] + +Run a campaign: build the configured ref, prepare its datasets, and execute +every ingest and query leg into a results bundle. + + --dry-run print the plan; build, download, and run nothing + --resume DIR continue an interrupted campaign into an existing bundle + --fail-fast stop at the first failed step (default: keep going) + --no-preflight skip the up-front tool and credential checks +`, + "plan": `usage: campaign plan + +Print the ordered steps the campaign would execute, one command per line. +`, + "preflight": `usage: campaign preflight + +Check that the tools, credentials, mounts, and free disk this config needs are +available, before a campaign spends hours discovering otherwise. +`, + "publish": `usage: campaign publish [dest-root] [--dry-run] [--force] + +Upload a results bundle to //, where run_id is the bundle's +basename. dest-root defaults to $PUBLISH_URI from the environment; gs:// and +s3:// are the only supported schemes. Published runs are immutable. + + --dry-run print the cloud commands, execute none of them + --force write into a non-empty destination (a merge, not a replace) +`, +} + +// subFlags builds the flag set for one subcommand: the flags the ported +// command will accept, a usage function printing that subcommand's text, and +// ContinueOnError so the dispatcher — not flag — decides the exit code. +func subFlags(name string, stderr io.Writer) *flag.FlagSet { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprint(fs.Output(), subUsage[name]) } + switch name { + case "run": + fs.Bool("dry-run", false, "") + fs.String("resume", "", "") + fs.Bool("fail-fast", false, "") + fs.Bool("no-preflight", false, "") + case "publish": + fs.Bool("dry-run", false, "") + fs.Bool("force", false, "") + } + return fs +} + +// parseArgs parses flags and positionals interleaved in any order, +// returning the positionals. flag.FlagSet.Parse stops at the first +// positional; benchmark operators put the config path first. +func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) { + var pos []string + for { + if err := fs.Parse(args); err != nil { + return nil, err + } + rest := fs.Args() + if len(rest) == 0 { + return pos, nil + } + // Parse stops either at the first positional (consuming nothing) + // or at a "--" terminator (consuming it). After "--" everything + // is positional, verbatim — do not reparse it. + if consumed := len(args) - len(rest); consumed > 0 && args[consumed-1] == "--" { + return append(pos, rest...), nil + } + pos = append(pos, rest[0]) + args = rest[1:] + } +} + +// planCmd prints the steps a campaign would execute. It builds no clone, +// fetches nothing, and writes nothing — it is readable on any machine. +func planCmd(pos []string, stdout, stderr io.Writer) int { + if len(pos) != 1 { + fmt.Fprint(stderr, subUsage["plan"]) + fmt.Fprint(stderr, "error: plan needs exactly one config path\n") + return 2 + } + cfg, err := config.Load(pos[0]) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 2 + } + benchRoot := benchRootFromEnv() + in := plan.Inputs{ + BenchRoot: benchRoot, + Stamp: time.Now().UTC().Format(stampLayout), + } + src := filepath.Join(benchRoot, "src") + if sha, err := run.ResolveRef(src, cfg.Ref); err == nil { + in.BuiltCommit, in.Sha8 = sha, sha[:8] + } else { + // Planning fetches nothing, so the ref may not resolve locally yet: + // plan with the ref itself and a placeholder sha in derived paths. + in.BuiltCommit, in.Sha8 = cfg.Ref, placeholderSha + fmt.Fprintf(stdout, "== note: ref '%s' is not resolvable in %s — using placeholder sha '%s' in paths\n", + cfg.Ref, src, placeholderSha) + } + plan.Build(cfg, in).Print(stdout) + return 0 +} + +// preflightCmd checks the tools, credentials, mount, and free disk this config +// needs, before a campaign spends hours discovering otherwise. It resolves no +// ref and touches no clone: binPath is empty, so the toolchain checks assume a +// build will happen. `campaign run` passes the real versioned binary path, +// which lets an already-built ref skip them. +func preflightCmd(pos []string, stdout, stderr io.Writer) int { + if len(pos) != 1 { + fmt.Fprint(stderr, subUsage["preflight"]) + fmt.Fprint(stderr, "error: preflight needs exactly one config path\n") + return 2 + } + cfg, err := config.Load(pos[0]) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 2 + } + if !printPreflight(preflight.Run(cfg, benchRootFromEnv(), "", preflight.Deps{}), stdout) { + return 1 + } + fmt.Fprintln(stdout, "preflight: ok") + return 0 +} + +// publishCmd uploads a finished bundle to //. The +// destination is the argument or $PUBLISH_URI — the same default the campaign +// config's publish_uri feeds the end of a run. +func publishCmd(pos []string, fs *flag.FlagSet, stdout, stderr io.Writer) int { + if len(pos) == 0 { + fmt.Fprint(stderr, subUsage["publish"]) + fmt.Fprint(stderr, "error: publish needs a results directory\n") + return 2 + } + if len(pos) > 2 { + fmt.Fprintf(stderr, "error: unexpected extra argument: %s\n", pos[2]) + return 2 + } + destRoot := os.Getenv("PUBLISH_URI") + if len(pos) == 2 { + destRoot = pos[1] + } + if destRoot == "" { + fmt.Fprint(stderr, "error: no destination: pass or set PUBLISH_URI\n") + return 2 + } + if err := publish.Run(pos[0], destRoot, boolFlag(fs, "dry-run"), boolFlag(fs, "force"), stdout); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + return 0 +} + +// benchRootFromEnv is the storage root every subcommand works under. +func benchRootFromEnv() string { + if root := os.Getenv("BENCH_ROOT"); root != "" { + return root + } + return defaultBenchRoot +} + +func main() { + os.Exit(dispatch(os.Args[1:], os.Stdout, os.Stderr)) +} + +func dispatch(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, topUsage) + return 2 + } + switch args[0] { + case "-h", "--help", "help": + fmt.Fprint(stdout, topUsage) + return 0 + case "run", "plan", "preflight", "publish": + fs := subFlags(args[0], stderr) + pos, err := parseArgs(fs, args[1:]) + if err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + switch args[0] { + case "run": + return runCmd(pos, fs, stdout, stderr) + case "plan": + return planCmd(pos, stdout, stderr) + case "preflight": + return preflightCmd(pos, stdout, stderr) + case "publish": + return publishCmd(pos, fs, stdout, stderr) + } + fmt.Fprint(stderr, subUsage[args[0]]) + fmt.Fprintf(stderr, "error: %s is not implemented yet\n", args[0]) + return 2 + default: + fmt.Fprintf(stderr, "error: unknown subcommand: %s\n", args[0]) + fmt.Fprint(stderr, topUsage) + return 2 + } +} diff --git a/runner/cmd/campaign/main_test.go b/runner/cmd/campaign/main_test.go new file mode 100644 index 0000000..4d36f1f --- /dev/null +++ b/runner/cmd/campaign/main_test.go @@ -0,0 +1,392 @@ +package main + +import ( + "bytes" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestParseArgs(t *testing.T) { + cases := []struct { + name string + sub string + args []string + pos []string + set map[string]string // flag name → value after parsing + }{ + { + name: "positional then flag", + sub: "run", + args: []string{"cfg.toml", "--dry-run"}, + pos: []string{"cfg.toml"}, + set: map[string]string{"dry-run": "true"}, + }, + { + name: "flag then positional", + sub: "run", + args: []string{"--dry-run", "cfg.toml"}, + pos: []string{"cfg.toml"}, + set: map[string]string{"dry-run": "true"}, + }, + { + name: "flag between two positionals", + sub: "publish", + args: []string{"results", "--force", "dest"}, + pos: []string{"results", "dest"}, + set: map[string]string{"force": "true"}, + }, + { + name: "no arguments at all", + sub: "run", + args: nil, + pos: nil, + set: map[string]string{"dry-run": "false"}, + }, + { + name: "terminator makes flag-shaped positionals verbatim", + sub: "publish", + args: []string{"--", "-results", "-dest"}, + pos: []string{"-results", "-dest"}, + set: map[string]string{"force": "false"}, + }, + { + name: "terminator after a peeled positional", + sub: "publish", + args: []string{"results", "--", "--force"}, + pos: []string{"results", "--force"}, + set: map[string]string{"force": "false"}, + }, + { + name: "flag before the terminator still parses", + sub: "publish", + args: []string{"--force", "--", "x"}, + pos: []string{"x"}, + set: map[string]string{"force": "true"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fs := subFlags(tc.sub, io.Discard) + pos, err := parseArgs(fs, tc.args) + if err != nil { + t.Fatalf("parseArgs(%q) = error %v, want nil", tc.args, err) + } + if !slices.Equal(pos, tc.pos) { + t.Errorf("positionals = %q, want %q", pos, tc.pos) + } + for name, want := range tc.set { + if got := fs.Lookup(name).Value.String(); got != want { + t.Errorf("flag %s = %s, want %s", name, got, want) + } + } + }) + } +} + +func TestParseArgsRejectsUnknownFlagAfterPositional(t *testing.T) { + fs := subFlags("run", io.Discard) + if _, err := parseArgs(fs, []string{"cfg.toml", "--bogus"}); err == nil { + t.Fatal("parseArgs = nil error, want an error for -bogus") + } +} + +const planConfig = ` +name = "cli" +ingest = "cold" +query = false +runs = 1 + +[[dataset]] +name = "ds" +kind = "packs-local" +location = "/packs/ds" +chunks = [1] +` + +func TestPlanCmd(t *testing.T) { + t.Run("unreadable config exits 2 with the config error", func(t *testing.T) { + t.Setenv("BENCH_ROOT", t.TempDir()) + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"plan", filepath.Join(t.TempDir(), "nope.toml")}, &stdout, &stderr); got != 2 { + t.Errorf("exit code = %d, want 2", got) + } + if !strings.Contains(stderr.String(), "config:") { + t.Errorf("stderr = %q, want the config error", stderr.String()) + } + }) + + t.Run("a valid config plans against a bench root with no clone", func(t *testing.T) { + benchRoot := t.TempDir() + t.Setenv("BENCH_ROOT", benchRoot) + cfg := filepath.Join(t.TempDir(), "campaign.toml") + if err := os.WriteFile(cfg, []byte(planConfig), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"plan", cfg}, &stdout, &stderr); got != 0 { + t.Errorf("exit code = %d, want 0 (stderr: %s)", got, stderr.String()) + } + for _, want := range []string{ + "using placeholder sha 'deadbeef' in paths", + "== build\n", + filepath.Join(benchRoot, "bin", "stellar-rpc-deadbeef"), + "== ingest-cold-ds-c1-run1\n", + } { + if !strings.Contains(stdout.String(), want) { + t.Errorf("stdout missing %q, got:\n%s", want, stdout.String()) + } + } + if stderr.Len() != 0 { + t.Errorf("stderr = %q, want empty", stderr.String()) + } + }) +} + +// stubPATH points PATH at a directory of no-op executables, so preflight's +// tool checks see exactly the named tools and nothing else. +func stubPATH(t *testing.T, tools ...string) { + t.Helper() + dir := t.TempDir() + for _, tool := range tools { + if err := os.WriteFile(filepath.Join(dir, tool), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("write %s: %v", tool, err) + } + } + t.Setenv("PATH", dir) +} + +func TestPreflightCmd(t *testing.T) { + t.Run("unreadable config exits 2 with the config error", func(t *testing.T) { + t.Setenv("BENCH_ROOT", t.TempDir()) + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"preflight", filepath.Join(t.TempDir(), "nope.toml")}, &stdout, &stderr); got != 2 { + t.Errorf("exit code = %d, want 2", got) + } + if !strings.Contains(stderr.String(), "config:") { + t.Errorf("stderr = %q, want the config error", stderr.String()) + } + }) + + // planConfig publishes nowhere and uses a local pack root, so the only + // tools it needs are the clone-and-build set. binPath is empty here, so a + // build is always assumed — hence go and cargo. + t.Run("a config needing only the build tools passes", func(t *testing.T) { + t.Setenv("BENCH_ROOT", t.TempDir()) + stubPATH(t, "git", "make", "go", "cargo") + cfg := filepath.Join(t.TempDir(), "campaign.toml") + if err := os.WriteFile(cfg, []byte(planConfig), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"preflight", cfg}, &stdout, &stderr); got != 0 { + t.Errorf("exit code = %d, want 0 (stdout: %s)", got, stdout.String()) + } + if !strings.Contains(stdout.String(), "preflight: ok") { + t.Errorf("stdout = %q, want 'preflight: ok'", stdout.String()) + } + if strings.Contains(stdout.String(), "FAIL") { + t.Errorf("stdout = %q, want no failures", stdout.String()) + } + }) + + t.Run("a missing tool fails, exit 1", func(t *testing.T) { + t.Setenv("BENCH_ROOT", t.TempDir()) + stubPATH(t, "make", "go", "cargo") + cfg := filepath.Join(t.TempDir(), "campaign.toml") + if err := os.WriteFile(cfg, []byte(planConfig), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"preflight", cfg}, &stdout, &stderr); got != 1 { + t.Errorf("exit code = %d, want 1 (stdout: %s)", got, stdout.String()) + } + if !strings.Contains(stdout.String(), "preflight: FAIL — git not found in PATH") { + t.Errorf("stdout = %q, want the git failure line", stdout.String()) + } + }) +} + +func TestPublishCmd(t *testing.T) { + // A gcloud that reports an empty prefix and accepts the upload, so the + // whole subcommand runs end to end without touching a bucket. + stubGcloud := func(t *testing.T) { + t.Helper() + dir := t.TempDir() + script := "#!/bin/sh\ncase \"$2\" in\nls) echo 'ERROR: One or more URLs matched no objects.' >&2; exit 1 ;;\n*) exit 0 ;;\nesac\n" + if err := os.WriteFile(filepath.Join(dir, "gcloud"), []byte(script), 0o755); err != nil { + t.Fatalf("write fake gcloud: %v", err) + } + t.Setenv("PATH", dir) + } + bundleDir := func(t *testing.T) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "phase4-6f35679f-20260715T101500Z") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir bundle: %v", err) + } + return dir + } + + t.Run("a bundle and $PUBLISH_URI are enough", func(t *testing.T) { + stubGcloud(t) + dir := bundleDir(t) + t.Setenv("PUBLISH_URI", "gs://bucket/results") + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"publish", dir}, &stdout, &stderr); got != 0 { + t.Errorf("exit code = %d, want 0 (stderr: %s)", got, stderr.String()) + } + want := "published: gs://bucket/results/" + filepath.Base(dir) + "/" + if !strings.Contains(stdout.String(), want) { + t.Errorf("stdout missing %q, got:\n%s", want, stdout.String()) + } + }) + + t.Run("no destination anywhere exits 2", func(t *testing.T) { + t.Setenv("PUBLISH_URI", "") + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"publish", bundleDir(t)}, &stdout, &stderr); got != 2 { + t.Errorf("exit code = %d, want 2", got) + } + if !strings.Contains(stderr.String(), "no destination: pass or set PUBLISH_URI") { + t.Errorf("stderr = %q, want the no-destination error", stderr.String()) + } + }) + + t.Run("a missing bundle is an operational failure, exit 1", func(t *testing.T) { + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"publish", filepath.Join(t.TempDir(), "nope"), "gs://bucket/results"}, &stdout, &stderr); got != 1 { + t.Errorf("exit code = %d, want 1", got) + } + if !strings.Contains(stderr.String(), "error: results dir not found: ") { + t.Errorf("stderr = %q, want the missing-bundle error", stderr.String()) + } + }) + + t.Run("--dry-run prints the commands and runs none", func(t *testing.T) { + stubPATH(t) // an empty PATH: a dry run must not need the CLI at all + dir := bundleDir(t) + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"publish", dir, "gs://bucket/results", "--dry-run"}, &stdout, &stderr); got != 0 { + t.Errorf("exit code = %d, want 0 (stderr: %s)", got, stderr.String()) + } + for _, want := range []string{" $ gcloud storage ls ", " $ gcloud storage rsync -r ", "dry run complete"} { + if !strings.Contains(stdout.String(), want) { + t.Errorf("stdout missing %q, got:\n%s", want, stdout.String()) + } + } + }) +} + +func TestRunDispatch(t *testing.T) { + cases := []struct { + name string + args []string + exit int + stderr []string // substrings expected on stderr + notStderr []string // substrings that must not appear on stderr + stdout []string // substrings expected on stdout + noStderr bool + }{ + { + name: "no args prints top usage", + args: nil, + exit: 2, + stderr: []string{"usage: campaign ", "run", "plan", "preflight", "publish", "BENCH_ROOT"}, + }, + { + name: "--help prints top usage on stdout", + args: []string{"--help"}, + exit: 0, + stdout: []string{"usage: campaign ", "preflight"}, + noStderr: true, + }, + { + name: "unknown subcommand names the bad argument", + args: []string{"benchmark"}, + exit: 2, + stderr: []string{"error: unknown subcommand: benchmark", "usage: campaign "}, + }, + { + name: "run without a config names what is missing", + args: []string{"run"}, + exit: 2, + stderr: []string{"usage: campaign run ", "--resume", "error: run needs exactly one config path"}, + }, + { + name: "run -h prints its usage and succeeds", + args: []string{"run", "-h"}, + exit: 0, + stderr: []string{"usage: campaign run ", "--no-preflight"}, + notStderr: []string{"error:"}, + }, + { + name: "unknown flag names the flag", + args: []string{"run", "--bogus"}, + exit: 2, + stderr: []string{"not defined: -bogus", "usage: campaign run "}, + }, + { + name: "unknown flag after a positional names the flag", + args: []string{"run", "cfg.toml", "--bogus"}, + exit: 2, + stderr: []string{"not defined: -bogus", "usage: campaign run "}, + }, + { + name: "plan without a config names what is missing", + args: []string{"plan"}, + exit: 2, + stderr: []string{"usage: campaign plan ", "error: plan needs exactly one config path"}, + }, + { + name: "preflight without a config names what is missing", + args: []string{"preflight"}, + exit: 2, + stderr: []string{"usage: campaign preflight ", "error: preflight needs exactly one config path"}, + }, + { + name: "publish without a bundle names what is missing", + args: []string{"publish"}, + exit: 2, + stderr: []string{"usage: campaign publish ", "--force", "error: publish needs a results directory"}, + }, + { + name: "publish with a third positional names it", + args: []string{"publish", "results", "gs://bucket", "extra"}, + exit: 2, + stderr: []string{"error: unexpected extra argument: extra"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + got := dispatch(tc.args, &stdout, &stderr) + if got != tc.exit { + t.Errorf("exit code = %d, want %d", got, tc.exit) + } + for _, want := range tc.stderr { + if !strings.Contains(stderr.String(), want) { + t.Errorf("stderr missing %q, got:\n%s", want, stderr.String()) + } + } + for _, unwanted := range tc.notStderr { + if strings.Contains(stderr.String(), unwanted) { + t.Errorf("stderr contains %q, want it absent, got:\n%s", unwanted, stderr.String()) + } + } + for _, want := range tc.stdout { + if !strings.Contains(stdout.String(), want) { + t.Errorf("stdout missing %q, got:\n%s", want, stdout.String()) + } + } + if tc.noStderr && stderr.Len() != 0 { + t.Errorf("stderr = %q, want empty", stderr.String()) + } + }) + } +} diff --git a/runner/cmd/campaign/run.go b/runner/cmd/campaign/run.go new file mode 100644 index 0000000..f363531 --- /dev/null +++ b/runner/cmd/campaign/run.go @@ -0,0 +1,360 @@ +package main + +import ( + "flag" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/bundle" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/plan" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/preflight" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/publish" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/run" +) + +// startedAtLayout is metadata.json's timestamp format: UTC, second precision. +const startedAtLayout = "2006-01-02T15:04:05Z" + +// runCmd runs a campaign: converge the build clone, resolve the ref, preflight +// the machine, then walk the plan into a results bundle. It is the successor to +// campaign.sh's main sequence, in the same order, for the same reasons. +func runCmd(pos []string, fs *flag.FlagSet, stdout, stderr io.Writer) int { + if len(pos) != 1 { + fmt.Fprint(stderr, subUsage["run"]) + fmt.Fprint(stderr, "error: run needs exactly one config path\n") + return 2 + } + cfgPath, err := filepath.Abs(pos[0]) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 2 + } + cfg, err := config.Load(cfgPath) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 2 + } + + benchRoot := benchRootFromEnv() + src := filepath.Join(benchRoot, "src") + resumeDir := stringFlag(fs, "resume") + if boolFlag(fs, "dry-run") { + return dryRun(cfg, cfgPath, benchRoot, src, resumeDir, stdout, stderr) + } + return realRun(cfg, cfgPath, benchRoot, src, resumeDir, + boolFlag(fs, "fail-fast"), boolFlag(fs, "no-preflight"), stdout, stderr) +} + +// dryRun prints the plan and executes nothing — no clone, no lock, no +// directory, exactly as `campaign.sh --dry-run` did. Whatever the clone already +// knows is used; what it does not know is planned with a placeholder sha. +func dryRun(cfg *config.Config, cfgPath, benchRoot, src, resumeDir string, stdout, stderr io.Writer) int { + run.Notef(stdout, "dry run: printing commands only — nothing is built, downloaded, or executed") + run.Notef(stdout, "source: %s @ %s → %s (the build clone is not touched)", cfg.Repo, cfg.Ref, src) + + in := plan.Inputs{BenchRoot: benchRoot, Stamp: time.Now().UTC().Format(stampLayout)} + builtCommit, resolveErr := run.ResolveRef(src, cfg.Ref) + if resolveErr == nil { + in.BuiltCommit, in.Sha8 = builtCommit, builtCommit[:8] + } else { + // A dry run fetches nothing, so the ref may not resolve locally yet: + // plan with the ref itself and a placeholder sha in derived paths. The + // placeholder is 8 hex digits, or a --resume would reject the run ids + // derived from it as malformed. + in.BuiltCommit, in.Sha8 = cfg.Ref, placeholderSha + run.Notef(stdout, "dry run: ref '%s' not resolvable without the clone — using placeholder sha '%s' in paths", + cfg.Ref, placeholderSha) + } + + var resume *bundle.Resume + if resumeDir != "" { + if resolveErr != nil { + // A resume is only valid against the commit its bundle was + // benchmarked with, and nothing here can know that commit without + // the clone. Refusing beats printing a plan for a run id that a + // real resume would reject. + fmt.Fprintf(stderr, "error: --dry-run --resume needs the build clone at %s to resolve ref '%s' — "+ + "a resume must be validated against the commit its bundle was benchmarked with\n", src, cfg.Ref) + return 1 + } + dir, err := filepath.Abs(resumeDir) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + resume, err = bundle.ValidateResume(dir, cfgPath, cfg, in.BuiltCommit, benchRoot, stdout) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + in.Stamp = resume.Stamp + } + + p := plan.Build(cfg, in) + run.Notef(stdout, "campaign %s → %s", cfg.Name, p.ResultsDir) + if resume != nil { + run.Notef(stdout, "resume: continuing %s — finished legs are skipped", resume.RunID) + for _, s := range p.Steps { + if s.Kind == plan.KindLeg && run.LegComplete(s.OutDir, s.ID) { + run.Notef(stdout, "resume: %s already complete — would skip", s.ID) + } + } + } + p.Print(stdout) + run.Notef(stdout, "dry run complete") + return 0 +} + +// realRun is the campaign itself. +func realRun(cfg *config.Config, cfgPath, benchRoot, src, resumeDir string, + failFast, noPreflight bool, stdout, stderr io.Writer) int { + + // One campaign per BENCH_ROOT: two would fight over the build clone, the + // scratch dirs, and the hot DBs, and each would measure the other. + release, err := run.AcquireLock(benchRoot) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + defer release() + + // Until the bundle exists there is nowhere to tee the log to; these first + // notes go to the terminal only, as they did in bash. + out := io.Writer(stdout) + + run.Notef(out, "source: %s @ %s → %s", cfg.Repo, cfg.Ref, src) + if err := run.EnsureSrc(src, cfg.Repo, out); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + builtCommit, err := run.ResolveRef(src, cfg.Ref) + if err != nil { + fmt.Fprintf(stderr, "error: ref '%s' does not resolve to a commit in %s\n", cfg.Ref, cfg.Repo) + return 1 + } + sha8 := builtCommit[:8] + binPath := filepath.Join(benchRoot, "bin", "stellar-rpc-"+sha8) + + if !noPreflight { + // Everything this campaign needs and does not have, named now rather + // than seventeen hours from now. + if !printPreflight(preflight.Run(cfg, benchRoot, binPath, preflight.Deps{}), out) { + return 1 + } + } + + session := "start" + stamp := time.Now().UTC().Format(stampLayout) + startedAt := time.Now().UTC().Format(startedAtLayout) + configFile := filepath.Base(cfgPath) + var resume *bundle.Resume + if resumeDir != "" { + dir, err := filepath.Abs(resumeDir) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + resume, err = bundle.ValidateResume(dir, cfgPath, cfg, builtCommit, benchRoot, stderr) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + session, stamp, configFile = "resume", resume.Stamp, resume.ConfigFile + // started_at comes from the bundle so metadata.json still spans the + // whole campaign; bundles written before it was recorded have none. + if resume.StartedAt != "" { + startedAt = resume.StartedAt + } + } + + p := plan.Build(cfg, plan.Inputs{BenchRoot: benchRoot, BuiltCommit: builtCommit, Sha8: sha8, Stamp: stamp}) + res := p.ResultsDir + if resume != nil && filepath.Clean(res) != filepath.Clean(resume.Dir) { + fmt.Fprintf(stderr, "error: --resume: '%s' is not the bundle this config and commit produce (%s)\n", + resume.Dir, res) + return 1 + } + + run.Notef(out, "campaign %s → %s", cfg.Name, res) + for _, dir := range []string{"bin", "golden", "scratch", "hot", "fixture"} { + if err := os.MkdirAll(filepath.Join(benchRoot, dir), 0o755); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + } + if err := os.MkdirAll(res, 0o755); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + if resume == nil { + // Only a fresh bundle stores the config. On a resume the stored copy + // has already been proven byte-identical to this one, and overwriting + // it is precisely how the bash runner lost the record of what a + // campaign was started with. + if err := copyFile(cfgPath, filepath.Join(res, configFile)); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + } + + // From here on the runner's console is part of the bundle: on a campaign + // that dies it is the only record of how far it got. Appended, so the + // sessions of a resumed campaign accumulate in one file. + logFile, err := run.OpenCampaignLog(res) + if err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + defer logFile.Close() + out = io.MultiWriter(stdout, logFile) + run.Notef(out, "session %s %s — logging to %s", session, time.Now().UTC().Format(startedAtLayout), logFile.Name()) + + hostname, _ := os.Hostname() + meta := bundle.MetadataInput{ + Cfg: cfg, + ConfigFile: configFile, + RunID: p.RunID, + BuiltCommit: builtCommit, + Resumed: resume != nil, + Hardware: bundle.CollectHardware(bundle.DefaultIMDSBase), + Hostname: hostname, + StartedAt: startedAt, + Status: bundle.StatusRunning, + } + if resume != nil && resume.StartedAt == "" { + run.Notef(out, "resume: no started_at in %s/%s — recording this session's start", resume.RunID, bundle.MetadataName) + } + // A manifest up front makes a killed campaign's partial bundle parseable; + // the end-of-campaign rewrite adds finished_at and the final status. + if err := bundle.WriteMetadata(res, meta); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + // The plan is written on every session: a resume builds the same plan by + // construction, and rewriting it keeps the file current with the runner + // that produced the bundle's newest legs. + if err := p.WriteFile(filepath.Join(res, "plan.json")); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + return 1 + } + + _, execErr := run.Execute(p, run.Options{ + Output: out, + Resume: resume != nil, + FailFast: failFast, + OnStepDone: func(s plan.Step, r run.StepResult) { + // Right after the build, not at the end: a campaign that dies in + // its legs still says which binary produced them. + if s.Kind != plan.KindBuild || r.Status != run.StatusOK { + return + } + if err := bundle.WriteBinaryInfo(res, p.Bin, cfg.Repo, cfg.Ref, builtCommit); err != nil { + run.Notef(out, "warning: %s", err) + } + }, + }) + + // The epilogue runs even after a failed campaign: a campaign that went + // wrong is exactly the one whose bundle has to be complete and preserved. + run.Notef(out, "machine metadata") + if err := bundle.WriteMachineMetadata(res, bundle.MachineInput{ + Cfg: cfg, Repo: cfg.Repo, Ref: cfg.Ref, BuiltCommit: builtCommit, + BinPath: p.Bin, BenchRoot: benchRoot, Hardware: meta.Hardware, + }); err != nil { + run.Notef(out, "warning: %s", err) + } + meta.FinishedAt = time.Now().UTC().Format(startedAtLayout) + meta.Status = bundle.StatusFinished + if execErr != nil { + meta.Status = bundle.StatusFailed + } + if err := bundle.WriteMetadata(res, meta); err != nil { + run.Notef(out, "warning: %s", err) + } + + // Tar last, so the bundle it preserves contains every file above. + tarErr := tarBundle(p, out) + if tarErr != nil { + // Reported, never masking a leg failure: the data is still in res. + run.Notef(out, "warning: tar failed: %s — the bundle is intact at %s", tarErr, res) + } else { + run.Notef(out, "campaign done: %s", p.Tarball) + } + // Publishing is a separate final step, after the "campaign done" note: the + // data is already safe in res and the tarball, so a publish failure is not + // a benchmark failure — it exits 1 with the exact retry command rather than + // corrupting that signal. + var publishErr error + if cfg.PublishURI != "" { + if tarErr != nil { + // The plan's publish step needs the tarball, and bash's set -e + // aborted before publish whenever tar failed. Publishing here would + // upload a bundle the retry note cannot honestly call safe. + // covered by the task-10 e2e (tar-failure scenario) + run.Notef(out, "skipping publish: tar failed — publish manually once the bundle is intact: campaign publish %s %s", + res, cfg.PublishURI) + } else if publishErr = publish.Run(res, cfg.PublishURI, false, false, out); publishErr != nil { + fmt.Fprintf(out, "error: %s\n", publishErr) + // A failed upload can still have written objects, which the retry's + // immutability check then refuses: name --force here rather than + // leaving the operator to rediscover it. + run.Notef(out, "publish failed — data is safe in %s and %s; retry with: campaign publish %s %s "+ + "(add --force if the failed upload left objects behind)", + res, p.Tarball, res, cfg.PublishURI) + } + } + + if execErr != nil || tarErr != nil || publishErr != nil { + return 1 + } + return 0 +} + +// tarBundle runs the plan's tarball step. Execute leaves it to the wiring; the +// step still comes from the plan, so what runs is what the plan printed. +func tarBundle(p *plan.Plan, out io.Writer) error { + for _, s := range p.Steps { + if s.Kind == plan.KindTarball { + return run.RunStep(s, out) + } + } + return fmt.Errorf("plan has no tarball step") +} + +// printPreflight reports what preflight found and says whether the campaign may +// proceed. Failures print before anything is built or fetched. +func printPreflight(res preflight.Result, w io.Writer) bool { + for _, failure := range res.Failures { + fmt.Fprintf(w, "preflight: FAIL — %s\n", failure) + } + for _, warning := range res.Warnings { + fmt.Fprintf(w, "preflight: warn — %s\n", warning) + } + return len(res.Failures) == 0 +} + +// copyFile copies src to dst verbatim: the bundle's stored config must be the +// bytes the operator ran, byte for byte, since that is what a later resume +// compares against. +func copyFile(src, dst string) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + return os.WriteFile(dst, b, 0o644) +} + +// boolFlag and stringFlag read a parsed flag by name. The flag sets are built +// in one place (subFlags) and consumed here, so a lookup by name keeps the two +// from drifting apart through a forgotten pointer. +func boolFlag(fs *flag.FlagSet, name string) bool { + return fs.Lookup(name).Value.String() == "true" +} + +func stringFlag(fs *flag.FlagSet, name string) string { + return fs.Lookup(name).Value.String() +} diff --git a/runner/cmd/campaign/run_test.go b/runner/cmd/campaign/run_test.go new file mode 100644 index 0000000..1067bb6 --- /dev/null +++ b/runner/cmd/campaign/run_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// planTestdata is the golden plan config from internal/plan — a two-dataset, +// four-suite campaign, which makes it the widest dry run available here. +const planTestdata = "../../internal/plan/testdata/campaign.toml" + +func TestRunCmdRejectsABadConfig(t *testing.T) { + t.Setenv("BENCH_ROOT", t.TempDir()) + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"run", filepath.Join(t.TempDir(), "nope.toml")}, &stdout, &stderr); got != 2 { + t.Errorf("exit code = %d, want 2", got) + } + if !strings.Contains(stderr.String(), "config:") { + t.Errorf("stderr = %q, want the config error", stderr.String()) + } +} + +func TestRunCmdDryRunTouchesNothing(t *testing.T) { + // A path that does not exist yet: the point of the test is that a dry run + // does not bring it into being — no lock file, no results dir, no clone. + benchRoot := filepath.Join(t.TempDir(), "bench") + t.Setenv("BENCH_ROOT", benchRoot) + + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"run", planTestdata, "--dry-run"}, &stdout, &stderr); got != 0 { + t.Fatalf("exit code = %d, want 0 (stderr: %s)", got, stderr.String()) + } + for _, want := range []string{ + "dry run: printing commands only", + "using placeholder sha 'deadbeef' in paths", + "== build\n", + filepath.Join(benchRoot, "bin", "stellar-rpc-deadbeef"), + "== dataset-packs\n", + "== ingest-cold-packs-c1-run1\n", + "== query-hot-fix-c2-run2\n", + "$ campaign publish " + filepath.Join(benchRoot, "results"), + "dry run complete", + } { + if !strings.Contains(stdout.String(), want) { + t.Errorf("stdout missing %q, got:\n%s", want, stdout.String()) + } + } + if stderr.Len() != 0 { + t.Errorf("stderr = %q, want empty", stderr.String()) + } + if _, err := os.Stat(benchRoot); !os.IsNotExist(err) { + entries, _ := os.ReadDir(benchRoot) + t.Errorf("%s exists after a dry run (stat err = %v), holding %v", benchRoot, err, entries) + } +} + +func TestRunCmdDryRunResumeNeedsTheClone(t *testing.T) { + benchRoot := t.TempDir() + t.Setenv("BENCH_ROOT", benchRoot) + resumeDir := filepath.Join(benchRoot, "results", "golden-deadbeef-20260101T000000Z") + if err := os.MkdirAll(resumeDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + var stdout, stderr bytes.Buffer + if got := dispatch([]string{"run", planTestdata, "--dry-run", "--resume", resumeDir}, &stdout, &stderr); got != 1 { + t.Errorf("exit code = %d, want 1 (stdout: %s)", got, stdout.String()) + } + // Without the clone there is no commit to validate the bundle against, and + // a plan printed against a placeholder sha would describe a resume that a + // real run would refuse. + if !strings.Contains(stderr.String(), "needs the build clone") { + t.Errorf("stderr = %q, want it to name the missing clone", stderr.String()) + } +} diff --git a/runner/cmd/campaign/testdata/e2e-campaign.toml b/runner/cmd/campaign/testdata/e2e-campaign.toml new file mode 100644 index 0000000..de0e16b --- /dev/null +++ b/runner/cmd/campaign/testdata/e2e-campaign.toml @@ -0,0 +1,16 @@ +# End-to-end campaign config (cmd/campaign/e2e_test.go): the smallest shape +# that still exercises all four suites — both ingest tiers, both query tiers — +# over one chunk of one generated fixture, twice. A fixture dataset is the only +# kind that needs no network and no operator-supplied pack tree, so the whole +# campaign runs against the stub binary alone. +name = "e2e" +ingest = "both" +query = true +runs = 2 +close_interval = "0" + +[[dataset]] +name = "fix" +kind = "fixture" +ledgers = 10000 +chunks = [1] diff --git a/runner/cmd/campaign/testdata/stub-rpc.sh b/runner/cmd/campaign/testdata/stub-rpc.sh new file mode 100755 index 0000000..3f3a6fb --- /dev/null +++ b/runner/cmd/campaign/testdata/stub-rpc.sh @@ -0,0 +1,196 @@ +#!/bin/sh +# Stub stellar-rpc binary for the campaign runner's end-to-end test +# (cmd/campaign/e2e_test.go). It answers the bench subcommands the campaign +# drives, materializes the directories they would materialize, and writes the +# CSV and invocation.json shapes converter/tests/fixtures.py documents — enough +# for the real converter to convert a bundle this stub produced. +# +# Environment: +# STUB_RPC_COMMIT 40-hex commit to report as the build identity. It must be +# the sha the test's fake git resolves to, or the converter +# warns about a binary/manifest commit mismatch. +# STUB_RPC_CONTROL optional control file, one `fail ` or +# `hang ` line per leg to derail. A failed leg +# writes a partial driver.csv and an invocation.json +# carrying an error (stellar-rpc#907) and exits 1; a hung +# leg writes nothing and sleeps until it is killed. +# +# Flags are read in the --key=value form the runner emits, and only the ones +# that decide what this stub writes. +set -eu + +version=v20.3.1-999-gstubbed +branch=stub/e2e +commit=${STUB_RPC_COMMIT:-0123456789abcdef0123456789abcdef01234567} +header='stage,n,n_items,total_ns,p50_ns,p90_ns,p99_ns,max_ns' + +sub=${1:-} +tier=${2:-} + +if [ "$sub" = version ]; then + echo "stellar-rpc $version" + echo "commit: $commit" + echo "branch: $branch" + exit 0 +fi + +out= +pack_dir= +cold_out_dir= +hot_dir= +close_interval= +types=ledgers,txpage,txhash,events +concurrency=1 +for arg in "$@"; do + case $arg in + --out=*) out=${arg#--out=} ;; + --pack-dir=*) pack_dir=${arg#--pack-dir=} ;; + --cold-out-dir=*) cold_out_dir=${arg#--cold-out-dir=} ;; + --hot-dir=*) hot_dir=${arg#--hot-dir=} ;; + --close-interval=*) close_interval=${arg#--close-interval=} ;; + --types=*) types=${arg#--types=} ;; + --query-concurrency=*) concurrency=${arg#--query-concurrency=} ;; + esac +done + +# csv_start writes the header; csv_row appends one row. Splitting them keeps the +# per-leg row lists readable as the tables they are. +csv_start() { echo "$header" >"$1"; } +csv_row() { echo "$2" >>"$1"; } + +# write_invocation mirrors stellar-rpc's own writer: camelCase keys, and an +# `error` field on the runs that failed. +write_invocation() { + inv_error=${1:-} + { + echo '{' + echo ' "schemaVersion": 1,' + printf ' "command": "stellar-rpc %s %s",\n' "$sub" "$tier" + echo ' "flags": {' + if [ -n "$close_interval" ]; then + printf ' "close-interval": "%s",\n' "$close_interval" + fi + printf ' "out": "%s"\n' "$out" + echo ' },' + echo ' "binary": {' + printf ' "version": "%s",\n' "$version" + printf ' "commitHash": "%s",\n' "$commit" + echo ' "buildTimestamp": "2026-07-30T00:00:00",' + printf ' "branch": "%s"\n' "$branch" + echo ' },' + echo ' "hostname": "e2e-stub",' + echo ' "startedAt": "2026-07-30T01:00:00Z",' + if [ -n "$inv_error" ]; then + printf ' "error": "%s",\n' "$inv_error" + fi + echo ' "finishedAt": "2026-07-30T01:00:01Z"' + echo '}' + } >"$out/invocation.json" +} + +# control_mode is what the control file says to do with this leg: run, fail, or +# hang. The last matching line wins. +control_mode() { + mode=run + if [ -n "${STUB_RPC_CONTROL:-}" ] && [ -f "$STUB_RPC_CONTROL" ]; then + while read -r verb target; do + if [ "$target" = "$1" ] && { [ "$verb" = fail ] || [ "$verb" = hang ]; }; then + mode=$verb + fi + done <"$STUB_RPC_CONTROL" + fi + echo "$mode" +} + +if [ -n "$out" ]; then + case "$(control_mode "${out##*/}")" in + hang) + # Nothing on disk: the leg the test kills must look partial, not failed. + sleep 600 + exit 0 + ;; + fail) + mkdir -p "$out" + csv_start "$out/driver.csv" + csv_row "$out/driver.csv" 'backfill_wall,1,0,50000,50000,50000,50000,50000' + write_invocation 'induced failure' + exit 1 + ;; + esac +fi + +case "$sub $tier" in +"bench-ingest fixture") + # Generation only stages a pack tree; it is untimed and writes no --out. + mkdir -p "$pack_dir" + echo 'stub fixture pack' >"$pack_dir/pack-0001.stub" + ;; + +"bench-ingest cold") + # One code path for the untimed golden freeze and the timed cold legs: both + # read a pack tree and write a cold store. + mkdir -p "$cold_out_dir/ledgers" + echo 'stub cold store' >"$cold_out_dir/ledgers/chunk-0001.stub" + mkdir -p "$out" + csv_start "$out/driver.csv" + csv_row "$out/driver.csv" 'backfill_wall,1,0,50000,50000,50000,50000,50000' + csv_row "$out/driver.csv" 'index_rebuild,1,0,3000,3000,3000,3000,3000' + csv_row "$out/driver.csv" 'chunk_total,1,0,40000,40000,40000,40000,40000' + csv_row "$out/driver.csv" 'ledgers_total,100,100,8000,80,90,99,120' + csv_row "$out/driver.csv" 'txhash_total,600,600,6000,60,70,90,100' + csv_row "$out/driver.csv" 'events_total,600,600,9000,90,95,99,110' + csv_row "$out/driver.csv" 'cold_extract,1,0,7000,70,80,90,100' + csv_row "$out/driver.csv" \ + 'peak_rss_bytes,1,0,20000000000,20000000000,20000000000,20000000000,20000000000' + csv_start "$out/events.csv" + csv_row "$out/events.csv" 'term_index,600,600,4000,40,45,49,60' + csv_row "$out/events.csv" 'write,600,600,3000,30,35,39,50' + csv_start "$out/ledgers.csv" + csv_row "$out/ledgers.csv" 'write,100,100,3000,30,35,39,50' + csv_start "$out/txhash.csv" + csv_row "$out/txhash.csv" 'finalize,1,0,1500,1500,1500,1500,1500' + write_invocation + ;; + +"bench-ingest hot") + mkdir -p "$hot_dir" + echo 'stub hot db' >"$hot_dir/hot.stub" + mkdir -p "$out" + csv_start "$out/driver.csv" + csv_row "$out/driver.csv" 'ingest_total,100,100,2000,100,200,500,900' + csv_row "$out/driver.csv" 'run_wall,1,100,60000,60000,60000,60000,60000' + csv_row "$out/driver.csv" \ + 'peak_rss_bytes,1,0,14000000000,14000000000,14000000000,14000000000,14000000000' + csv_start "$out/hot.csv" + csv_row "$out/hot.csv" 'extract,100,0,300,30,40,50,80' + csv_row "$out/hot.csv" 'ledgers,100,100,400,40,50,60,90' + csv_row "$out/hot.csv" 'txhash,100,600,200,20,25,30,50' + csv_row "$out/hot.csv" 'events,100,600,500,50,55,60,90' + csv_row "$out/hot.csv" 'commit,100,0,800,80,90,120,200' + csv_row "$out/hot.csv" 'apply,100,0,250,25,30,40,70' + write_invocation + ;; + +"bench-query cold" | "bench-query hot") + mkdir -p "$out" + csv_start "$out/driver.csv" + csv_row "$out/driver.csv" 'open,1,0,5000,5000,5000,5000,5000' + # The comma-separated --types and --query-concurrency lists are split on + # purpose: one per-type CSV, one column set per concurrency level. + IFS=, + for qt in $types; do + csv_start "$out/$qt.csv" + for w in $concurrency; do + csv_row "$out/driver.csv" "${qt}_c${w},1,0,1000000,1000000,1000000,1000000,1000000" + csv_row "$out/$qt.csv" "total_c${w},100,100,900000,9000,11000,13000,20000" + done + done + unset IFS + write_invocation + ;; + +*) + echo "stub-rpc: unsupported command: $*" >&2 + exit 64 + ;; +esac diff --git a/runner/example-campaign.toml b/runner/example-campaign.toml new file mode 100644 index 0000000..2e4bfca --- /dev/null +++ b/runner/example-campaign.toml @@ -0,0 +1,110 @@ +# Example campaign config. Copy this file, adjust the keys, and run it from +# runner/: +# +# cd runner +# BENCH_ROOT=/mnt/nvme/bench go run ./cmd/campaign run my-campaign.toml +# +# Swap `run` for `plan` to print the command plan without touching anything, or +# add --dry-run to `run` for the same plan with the resume and bundle paths +# resolved. Both work on any machine — nothing is built, downloaded, or +# benchmarked. The one exception is `run --dry-run --resume`, which needs the +# build clone on the box: a resume is only valid against the commit its bundle +# was benchmarked with, so the ref has to resolve for real. +# +# Every key is listed below with its default. Unknown keys are rejected rather +# than ignored, so a typo fails in the first second instead of an hour in. +# runner/README.md § Config reference is the authoritative description of each. + +# Campaign name (required; [A-Za-z0-9._-]+). Results land in +# $BENCH_ROOT/results/--/ and the bundle tarball carries the +# same run id. +name = "example" + +# Where stellar-rpc comes from: a git URL or an absolute local path. The runner +# maintains a persistent build clone of it at $BENCH_ROOT/src; the source itself +# is never modified. To benchmark local work-in-progress, point this at a local +# stellar-rpc checkout — only committed state reachable from repo is +# benchmarkable. +#repo = "https://github.com/stellar/stellar-rpc.git" + +# Git ref to benchmark, resolved in the build clone after fetching repo's +# branches and tags. Default: feature/full-history, the branch this whole suite +# benchmarks. The ref is built into a versioned binary +# ($BENCH_ROOT/bin/stellar-rpc-). +#ref = "feature/full-history" + +# Which ingest benchmarks to run per (dataset, chunk): cold | hot | both | none +# (required). +ingest = "both" + +# Whether to run the query benchmark suites after ingest: true | false +# (required — there is no default). Query-hot additionally needs the hot DB a +# hot ingest leaves behind, so it only runs when ingest is hot or both. +query = false + +# Pace the hot ingest at a fixed ledger close interval — a Go duration, e.g. +# "2s", "1s", "600ms". Default "0" = unpaced catch-up ingestion. +#close_interval = "2s" + +# Repetitions per (dataset, chunk) cell. Default 5, which is what the +# converter's median/min-max aggregation assumes. +#runs = 5 + +# Query tuning, used only when query = true: the concurrency sweep and the +# per-benchmark iteration counts. Defaults shown. +#query_concurrency = [1, 4, 16] +#cold_iters = 100 +#hot_iters = 200 + +# bench-ingest cold --workers. Default 1. +#workers = 1 + +# Cap the hot ingest at this many ledgers. Default 0 = the whole range. +#hot_num_ledgers = 0 + +# Publish the finished bundle to object storage (gs:// or s3://) at the end of +# the campaign. Default "" = keep results local; publish later by hand with +# `campaign publish `. +#publish_uri = "gs://rpc-full-history/benchmarks" + +# Datasets to benchmark: one [[dataset]] table each, with a unique name and the +# chunk IDs this campaign runs. Every kind converges on a local cold pack root +# (the directory holding ledgers/, events/, txhash/) that the legs then read; +# they differ only in how that root is materialized. Uncomment the kind you +# want — a copy-paste of this file starts with the fetched-packs one. + +# packs-gs: location is a gs:// prefix of a cold pack tree, fetched once into +# $BENCH_ROOT/golden// and reused by every later campaign (rm -rf that +# directory to force a re-fetch). +[[dataset]] +name = "mydata" +kind = "packs-gs" +location = "gs://my-bucket/ledgers/mydata/packs/cold" +chunks = [1, 2] + +# packs-local: location is a cold pack root you already have on this machine. +# It is used in place and never written to. +#[[dataset]] +#name = "mylocal" +#kind = "packs-local" +#location = "/mnt/nvme/packs/mydata" +#chunks = [1] + +# bsb-s3: location is an S3 bucket path; an untimed cold backfill materializes +# $BENCH_ROOT/golden// one chunk at a time. +#[[dataset]] +#name = "frompubnet" +#kind = "bsb-s3" +#location = "my-bucket/ledgers/pubnet" +#chunks = [1] + +# fixture: generated data, no network and no pack tree of your own. It takes +# `ledgers` instead of a location — the per-chunk ledger count, 0 for the whole +# 10,000-ledger chunk or at least 10000, because the untimed cold freeze that +# turns the generated pack into golden packs streams a whole chunk and cannot +# freeze a partial one. +#[[dataset]] +#name = "myfixture" +#kind = "fixture" +#ledgers = 10000 +#chunks = [1] diff --git a/runner/go.mod b/runner/go.mod new file mode 100644 index 0000000..2e00a62 --- /dev/null +++ b/runner/go.mod @@ -0,0 +1,7 @@ +module github.com/stellar/stellar-rpc-benchmarks/runner + +go 1.26 + +// The campaign config parser (internal/config) — the only third-party +// dependency. +require github.com/BurntSushi/toml v1.6.0 diff --git a/runner/go.sum b/runner/go.sum new file mode 100644 index 0000000..f74b269 --- /dev/null +++ b/runner/go.sum @@ -0,0 +1,2 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= diff --git a/runner/internal/bundle/bundle.go b/runner/internal/bundle/bundle.go new file mode 100644 index 0000000..5c68f74 --- /dev/null +++ b/runner/internal/bundle/bundle.go @@ -0,0 +1,205 @@ +// Package bundle owns the bundle root's manifest: reading metadata.json, the +// status vocabulary it carries, and the checks a --resume must pass before any +// step of a resumed campaign runs. +// +// metadata.json is a cross-repo contract with converter/convert.py — see +// SCHEMA.md § Inputs and runner/README.md § Campaign bundle layout. Only the +// fields the runner itself consumes are modelled here; the rest are carried +// through untouched by readers. +package bundle + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +// Status values metadata.json carries in its (additive) status field. +// "running" is written up front; the end-of-campaign rewrite makes it +// "finished", or "failed" when any leg failed or the campaign aborted. +// Bash-era bundles have no status; readers treat absent as unknown. +const ( + StatusRunning = "running" + StatusFinished = "finished" + StatusFailed = "failed" +) + +// MetadataName is the bundle-root manifest's filename. +const MetadataName = "metadata.json" + +// Metadata is the read-side of the bundle manifest — only the fields the +// runner consumes; the converter reads more. +type Metadata struct { + SchemaVersion int `json:"schema_version"` + RunID string `json:"run_id"` + Campaign struct { + Name string `json:"name"` + ConfigFile string `json:"config_file"` + Ref string `json:"ref"` + BuiltCommit string `json:"built_commit"` + } `json:"campaign"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + Status string `json:"status"` +} + +// ReadMetadata reads and parses bundleDir/metadata.json. Fields the runner does +// not model (datasets, hardware, and the rest of the campaign block) are +// ignored, not an error: this reader must keep working on bundles written by a +// newer writer, and on the bash runner's. +func ReadMetadata(bundleDir string) (*Metadata, error) { + path := filepath.Join(bundleDir, MetadataName) + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("bundle: read %s: %w", path, err) + } + var m Metadata + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("bundle: parse %s: %w", path, err) + } + return &m, nil +} + +// Resume is what a validated --resume hands the run wiring. +type Resume struct { + Dir string + RunID string + Sha8 string // parsed from RunID's fixed tail + Stamp string // parsed from RunID's fixed tail + // ConfigFile is the bundle-root name of the stored config copy, carried + // forward so the resumed session's metadata keeps pointing at the file the + // campaign was started with even when the operator invoked a differently + // named path holding the same bytes. + ConfigFile string + StartedAt string // recovered; "" for pre-crash-safe bundles (caller + // records this session's start, like bash) +} + +// reRunID matches a run id. NAME may contain '-', so the sha and stamp are +// matched as the fixed tail. +var reRunID = regexp.MustCompile(`^(.+)-([0-9a-f]{8})-([0-9]{8}T[0-9]{6}Z)$`) + +// ValidateResume refuses to resume dir unless it is provably the same +// campaign: same name, same binary, byte-identical config. Identity comes from +// the bundle's metadata.json, not from parsing the directory basename, so a +// renamed or copied bundle cannot pass itself off as another campaign. +// diffOut receives the unified diff when the config guard fires. +// +// Because every one of these checks runs before the first step, a resumed +// campaign never needs to re-copy the config into the bundle: the stored copy +// is already known to be the config being run. The run wiring copies it only +// when creating a fresh bundle. +func ValidateResume(dir, cfgPath string, cfg *config.Config, + builtCommit, benchRoot string, diffOut io.Writer) (*Resume, error) { + + fi, err := os.Stat(dir) + if err != nil { + return nil, fmt.Errorf("--resume: cannot read '%s': %w", dir, err) + } + if !fi.IsDir() { + return nil, fmt.Errorf("--resume: '%s' is not a directory", dir) + } + + meta, err := ReadMetadata(dir) + if err != nil { + return nil, fmt.Errorf("--resume: no readable %s in %s — this bundle predates "+ + "crash-safe metadata (or is not a campaign bundle); resume it with the bash "+ + "runner or start a fresh campaign", MetadataName, dir) + } + + m := reRunID.FindStringSubmatch(meta.RunID) + if m == nil { + return nil, fmt.Errorf("--resume: '%s' is not a -- run id", meta.RunID) + } + sha8, stamp := m[2], m[3] + + if meta.Campaign.Name != cfg.Name { + return nil, fmt.Errorf("--resume: '%s' belongs to campaign '%s', but this config's name is '%s'", + meta.RunID, meta.Campaign.Name, cfg.Name) + } + if meta.Campaign.BuiltCommit != builtCommit { + return nil, fmt.Errorf("--resume: '%s' was benchmarked with commit %s, but ref '%s' now resolves "+ + "to %s — resuming would mix two binaries in one bundle; check out the same ref or "+ + "start a fresh campaign", meta.RunID, meta.Campaign.BuiltCommit, cfg.Ref, builtCommit) + } + if expected := filepath.Join(benchRoot, "results", meta.RunID); filepath.Clean(dir) != filepath.Clean(expected) { + return nil, fmt.Errorf("--resume: '%s' is not this BENCH_ROOT's results directory (expected %s) — "+ + "set BENCH_ROOT to the original campaign's root", dir, expected) + } + + if err := checkStoredConfig(dir, cfgPath, meta, diffOut); err != nil { + return nil, err + } + + // An empty started_at is not an error: bundles written before metadata.json + // was written up front have none, and the caller records this session's + // start instead, as bash did. + return &Resume{ + Dir: dir, + RunID: meta.RunID, + Sha8: sha8, + Stamp: stamp, + ConfigFile: meta.Campaign.ConfigFile, + StartedAt: meta.StartedAt, + }, nil +} + +// checkStoredConfig is the config-diff guard: it proves the config about to be +// run is byte-identical to the one this campaign started with. Bash checked +// nothing here and overwrote the bundle's stored copy with the current config, +// so a resume with edited knobs produced mixed data under a manifest that +// uniformly claimed the new knobs. +func checkStoredConfig(dir, cfgPath string, meta *Metadata, diffOut io.Writer) error { + if meta.Campaign.ConfigFile == "" { + return fmt.Errorf("--resume: '%s' records no config_file — cannot verify what this campaign "+ + "was started with; start a fresh campaign", meta.RunID) + } + // metadata.json travels with the bundle and may be hand-edited; the guard + // must only ever compare against the copy stored in the bundle root. + if meta.Campaign.ConfigFile != filepath.Base(meta.Campaign.ConfigFile) { + return fmt.Errorf("--resume: '%s' records a config_file that is not a bundle-root filename "+ + "('%s') — refusing to compare against a path outside the bundle", + meta.RunID, meta.Campaign.ConfigFile) + } + stored := filepath.Join(dir, meta.Campaign.ConfigFile) + storedBytes, err := os.ReadFile(stored) + if err != nil { + return fmt.Errorf("--resume: cannot read the config this campaign started with (%s): %w — "+ + "start a fresh campaign", stored, err) + } + currentBytes, err := os.ReadFile(cfgPath) + if err != nil { + return fmt.Errorf("--resume: cannot read %s: %w", cfgPath, err) + } + if string(storedBytes) == string(currentBytes) { + return nil + } + writeDiff(diffOut, stored, cfgPath) + return fmt.Errorf("--resume: the config differs from the one this campaign started with (%s) — "+ + "a resumed campaign must run the exact config it began with; restore it or start a "+ + "fresh campaign", stored) +} + +// writeDiff shows the operator what changed. diff(1) is universally present and +// exits 1 for "files differ", which is the expected case here; anything else it +// says is reported in place of the diff, never in place of the refusal. +func writeDiff(diffOut io.Writer, stored, current string) { + if diffOut == nil { + return + } + out, err := exec.Command("diff", "-u", stored, current).Output() + if len(out) > 0 { + _, _ = diffOut.Write(out) + } + var exitErr *exec.ExitError + if err != nil && !(errors.As(err, &exitErr) && exitErr.ExitCode() == 1) { + _, _ = fmt.Fprintf(diffOut, "(cannot show the diff: diff -u %s %s: %v)\n", stored, current, err) + } +} diff --git a/runner/internal/bundle/bundle_test.go b/runner/internal/bundle/bundle_test.go new file mode 100644 index 0000000..1faf8a1 --- /dev/null +++ b/runner/internal/bundle/bundle_test.go @@ -0,0 +1,429 @@ +package bundle + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +// --- helpers -------------------------------------------------------------- + +const ( + testName = "nvme-full" + testCommit = "1e0e7f9c0d2b4a6f8e0a1c3d5f7b9d1e2a4c6e80" + testSha8 = "1e0e7f9c" + testStamp = "20260715T101500Z" + testRunID = testName + "-" + testSha8 + "-" + testStamp + testStart = "2026-07-15T10:15:00Z" +) + +const testCfgBody = `name = "nvme-full" +ref = "feature/full-history" +ingest = "both" +query = true +runs = 5 + +[[dataset]] +name = "pubnet" +kind = "packs-local" +location = "/mnt/nvme/packs" +chunks = [63] +` + +// bashMetadata is the manifest shape write_campaign_metadata emits, extra +// fields and all: readers must ignore what they do not model. +func bashMetadata(runID, name, configFile, builtCommit, startedAt string) string { + return `{ + "schema_version": 1, + "run_id": "` + runID + `", + "campaign": { + "name": "` + name + `", + "config_file": "` + configFile + `", + "ref": "feature/full-history", + "built_commit": "` + builtCommit + `", + "ingest": "both", + "query": "yes", + "close_interval": "0", + "runs": 5, + "query_concurrency": "1,4,16", + "cold_iters": 100, + "hot_iters": 200, + "workers": 1, + "hot_num_ledgers": 0, + "resumed": true + }, + "datasets": [ + {"name": "pubnet", "kind": "packs-local", "location": "/mnt/nvme/packs", "chunks": [63]} + ], + "hardware": {"instance_type": "i4i.4xlarge", "uname": "Linux 6.8.0 x86_64", "cpus": 16}, + "hostname": "bench-devbox", + "started_at": "` + startedAt + `" +} +` +} + +type bundleOpts struct { + runID string + name string + configFile string // name of the stored copy; "" means write no copy + metaConfig string // config_file as recorded in metadata.json; "" uses configFile + builtCommit string + startedAt string + storedCfg string // stored copy's body; "" uses testCfgBody + metadata string // whole metadata.json body; "" builds the bash shape + noMetadata bool +} + +// makeBundle writes /results/ and returns benchRoot and the +// bundle dir. +func makeBundle(t *testing.T, o bundleOpts) (benchRoot, dir string) { + t.Helper() + if o.runID == "" { + o.runID = testRunID + } + if o.name == "" { + o.name = testName + } + if o.builtCommit == "" { + o.builtCommit = testCommit + } + if o.metaConfig == "" { + o.metaConfig = o.configFile + } + benchRoot = t.TempDir() + dir = filepath.Join(benchRoot, "results", o.runID) + mustMkdir(t, dir) + if !o.noMetadata { + body := o.metadata + if body == "" { + body = bashMetadata(o.runID, o.name, o.metaConfig, o.builtCommit, o.startedAt) + } + mustWrite(t, filepath.Join(dir, MetadataName), body) + } + if o.configFile != "" { + stored := o.storedCfg + if stored == "" { + stored = testCfgBody + } + mustWrite(t, filepath.Join(dir, o.configFile), stored) + } + return benchRoot, dir +} + +func mustMkdir(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } +} + +func mustWrite(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// currentConfig writes the config the resuming session would pass on the +// command line. +func currentConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "campaign.cfg") + mustWrite(t, path, body) + return path +} + +func testConfig() *config.Config { + return &config.Config{Name: testName, Ref: "feature/full-history"} +} + +// validate runs ValidateResume with the standard inputs, capturing diff output. +func validate(t *testing.T, dir, cfgPath string, cfg *config.Config, builtCommit, benchRoot string) (*Resume, string, error) { + t.Helper() + var diff bytes.Buffer + r, err := ValidateResume(dir, cfgPath, cfg, builtCommit, benchRoot, &diff) + return r, diff.String(), err +} + +// --- ReadMetadata --------------------------------------------------------- + +func TestReadMetadataBashShape(t *testing.T) { + _, dir := makeBundle(t, bundleOpts{configFile: "campaign.cfg", startedAt: testStart}) + meta, err := ReadMetadata(dir) + if err != nil { + t.Fatalf("ReadMetadata: %v", err) + } + if meta.SchemaVersion != 1 || meta.RunID != testRunID { + t.Errorf("schema_version/run_id = %d/%q, want 1/%q", meta.SchemaVersion, meta.RunID, testRunID) + } + if meta.Campaign.Name != testName || meta.Campaign.ConfigFile != "campaign.cfg" { + t.Errorf("campaign name/config_file = %q/%q", meta.Campaign.Name, meta.Campaign.ConfigFile) + } + if meta.Campaign.BuiltCommit != testCommit || meta.Campaign.Ref != "feature/full-history" { + t.Errorf("campaign built_commit/ref = %q/%q", meta.Campaign.BuiltCommit, meta.Campaign.Ref) + } + if meta.StartedAt != testStart { + t.Errorf("started_at = %q, want %q", meta.StartedAt, testStart) + } + // A bash-era bundle has neither field; absent status is unknown, not an error. + if meta.Status != "" { + t.Errorf("status = %q, want \"\" for a bash-era bundle", meta.Status) + } + if meta.FinishedAt != "" { + t.Errorf("finished_at = %q, want \"\"", meta.FinishedAt) + } +} + +func TestReadMetadataStatus(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, MetadataName), + `{"run_id":"x","status":"`+StatusFinished+`","finished_at":"2026-07-15T20:00:00Z"}`) + meta, err := ReadMetadata(dir) + if err != nil { + t.Fatalf("ReadMetadata: %v", err) + } + if meta.Status != StatusFinished || meta.FinishedAt != "2026-07-15T20:00:00Z" { + t.Errorf("status/finished_at = %q/%q", meta.Status, meta.FinishedAt) + } +} + +func TestReadMetadataErrors(t *testing.T) { + if _, err := ReadMetadata(t.TempDir()); err == nil { + t.Fatal("ReadMetadata of a bundle with no metadata.json: want error") + } + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, MetadataName), "{not json") + if _, err := ReadMetadata(dir); err == nil { + t.Fatal("ReadMetadata of corrupt metadata.json: want error") + } +} + +// --- ValidateResume: happy paths ------------------------------------------ + +func TestValidateResumeHappyPath(t *testing.T) { + benchRoot, dir := makeBundle(t, bundleOpts{configFile: "campaign.cfg", startedAt: testStart}) + cfgPath := currentConfig(t, testCfgBody) + + r, diff, err := validate(t, dir, cfgPath, testConfig(), testCommit, benchRoot) + if err != nil { + t.Fatalf("ValidateResume: %v", err) + } + if diff != "" { + t.Errorf("diff output on an identical config: %q", diff) + } + if r.Dir != dir || r.RunID != testRunID { + t.Errorf("Dir/RunID = %q/%q", r.Dir, r.RunID) + } + if r.Sha8 != testSha8 || r.Stamp != testStamp { + t.Errorf("Sha8/Stamp = %q/%q, want %q/%q", r.Sha8, r.Stamp, testSha8, testStamp) + } + if r.StartedAt != testStart { + t.Errorf("StartedAt = %q, want %q", r.StartedAt, testStart) + } +} + +// A bundle from before metadata.json was written up front has no started_at: +// resumable, with the caller left to record this session's start. +func TestValidateResumeNoStartedAt(t *testing.T) { + benchRoot, dir := makeBundle(t, bundleOpts{configFile: "campaign.cfg"}) + r, _, err := validate(t, dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot) + if err != nil { + t.Fatalf("ValidateResume: %v", err) + } + if r.StartedAt != "" { + t.Errorf("StartedAt = %q, want \"\"", r.StartedAt) + } +} + +func TestValidateResumeNameWithDashes(t *testing.T) { + const name = "my-camp-aign" + runID := name + "-deadbeef-" + testStamp + benchRoot, dir := makeBundle(t, bundleOpts{runID: runID, name: name, configFile: "campaign.cfg"}) + cfg := testConfig() + cfg.Name = name + + r, _, err := validate(t, dir, currentConfig(t, testCfgBody), cfg, testCommit, benchRoot) + if err != nil { + t.Fatalf("ValidateResume: %v", err) + } + if r.Sha8 != "deadbeef" || r.Stamp != testStamp { + t.Errorf("Sha8/Stamp = %q/%q, want deadbeef/%q", r.Sha8, r.Stamp, testStamp) + } +} + +// --- ValidateResume: refusals --------------------------------------------- + +func TestValidateResumeRefusals(t *testing.T) { + tests := []struct { + name string + // setup returns the dir to resume, the current config's path, the + // config, the commit the ref now resolves to, and the bench root. + setup func(t *testing.T) (dir, cfgPath string, cfg *config.Config, builtCommit, benchRoot string) + want string + }{ + { + name: "missing dir", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot := t.TempDir() + return filepath.Join(benchRoot, "results", testRunID), currentConfig(t, testCfgBody), + testConfig(), testCommit, benchRoot + }, + want: "--resume: cannot read", + }, + { + name: "not a directory", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot := t.TempDir() + dir := filepath.Join(benchRoot, "results", testRunID) + mustMkdir(t, filepath.Dir(dir)) + mustWrite(t, dir, "not a bundle") + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "is not a directory", + }, + { + name: "missing metadata.json", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{noMetadata: true, configFile: "campaign.cfg"}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "no readable metadata.json in", + }, + { + name: "corrupt metadata.json", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{ + metadata: `{"run_id": "` + testRunID + `", "campaign": {`, configFile: "campaign.cfg"}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "no readable metadata.json in", + }, + { + name: "malformed run_id", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{runID: "nvme-full-20260715", configFile: "campaign.cfg"}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "is not a -- run id", + }, + { + name: "campaign name mismatch", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{name: "other-campaign", configFile: "campaign.cfg"}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "belongs to campaign 'other-campaign', but this config's name is 'nvme-full'", + }, + { + // The run id's short sha still matches: only the full built_commit + // in metadata.json can tell these two apart. + name: "built_commit mismatch", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{ + builtCommit: testSha8 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", configFile: "campaign.cfg"}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "resuming would mix two binaries in one bundle", + }, + { + name: "wrong bench root", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + _, dir := makeBundle(t, bundleOpts{configFile: "campaign.cfg"}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, t.TempDir() + }, + want: "is not this BENCH_ROOT's results directory (expected", + }, + { + name: "no config_file recorded", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "records no config_file", + }, + { + // The traversal target is a byte-identical config: without the + // guard this resume would be accepted, so the refusal proves the + // file outside the bundle is never read. + name: "config_file escapes the bundle root", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{metaConfig: "../evil.toml"}) + mustWrite(t, filepath.Join(filepath.Dir(dir), "evil.toml"), testCfgBody) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "records a config_file that is not a bundle-root filename ('../evil.toml') — " + + "refusing to compare against a path outside the bundle", + }, + { + name: "stored config copy missing", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{metaConfig: "campaign.cfg"}) + return dir, currentConfig(t, testCfgBody), testConfig(), testCommit, benchRoot + }, + want: "cannot read the config this campaign started with", + }, + { + name: "current config unreadable", + setup: func(t *testing.T) (string, string, *config.Config, string, string) { + benchRoot, dir := makeBundle(t, bundleOpts{configFile: "campaign.cfg"}) + return dir, filepath.Join(t.TempDir(), "gone.cfg"), testConfig(), testCommit, benchRoot + }, + want: "--resume: cannot read", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir, cfgPath, cfg, builtCommit, benchRoot := tc.setup(t) + r, _, err := validate(t, dir, cfgPath, cfg, builtCommit, benchRoot) + if err == nil { + t.Fatalf("ValidateResume: want refusal, got %+v", r) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to contain %q", err, tc.want) + } + if !strings.HasPrefix(err.Error(), "--resume: ") { + t.Errorf("error = %q, want the --resume: prefix", err) + } + }) + } +} + +// The guard finding 1 is about: an edited config is refused, the operator is +// shown what changed, and the bundle's own copy is left alone. +func TestValidateResumeEditedConfig(t *testing.T) { + benchRoot, dir := makeBundle(t, bundleOpts{configFile: "campaign.cfg", startedAt: testStart}) + edited := strings.Replace(testCfgBody, "runs = 5", "runs = 3", 1) + cfgPath := currentConfig(t, edited) + + _, diff, err := validate(t, dir, cfgPath, testConfig(), testCommit, benchRoot) + if err == nil { + t.Fatal("ValidateResume with an edited config: want refusal") + } + if !strings.Contains(err.Error(), "a resumed campaign must run the exact config it began with") { + t.Errorf("error = %q", err) + } + if !strings.Contains(diff, "-runs = 5") || !strings.Contains(diff, "+runs = 3") { + t.Errorf("diff output = %q, want a -runs = 5 / +runs = 3 pair", diff) + } + stored, readErr := os.ReadFile(filepath.Join(dir, "campaign.cfg")) + if readErr != nil { + t.Fatalf("read stored config: %v", readErr) + } + if string(stored) != testCfgBody { + t.Errorf("the stored config was modified:\n%s", stored) + } +} + +// A nil diff writer is a caller that does not want the diff, not a panic. +func TestValidateResumeNilDiffWriter(t *testing.T) { + benchRoot, dir := makeBundle(t, bundleOpts{configFile: "campaign.cfg"}) + cfgPath := currentConfig(t, strings.Replace(testCfgBody, "runs = 5", "runs = 3", 1)) + if _, err := ValidateResume(dir, cfgPath, testConfig(), testCommit, benchRoot, nil); err == nil { + t.Fatal("ValidateResume with an edited config: want refusal") + } +} diff --git a/runner/internal/bundle/metadata.go b/runner/internal/bundle/metadata.go new file mode 100644 index 0000000..db991b7 --- /dev/null +++ b/runner/internal/bundle/metadata.go @@ -0,0 +1,249 @@ +package bundle + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +// SchemaVersion is the version of the metadata.json contract. Additive changes +// (new fields) keep the version; a change that breaks the converter bumps it. +const SchemaVersion = 1 + +// Hardware is metadata.json's hardware object. Empty facts are omitted +// entirely, never "" — the bash writer dropped them with jq's with_entries. +type Hardware struct { + InstanceType string `json:"instance_type,omitempty"` + InstanceID string `json:"instance_id,omitempty"` + Uname string `json:"uname"` + CPUs int `json:"cpus,omitempty"` + MemTotalKB int64 `json:"mem_total_kb,omitempty"` +} + +// MetadataInput is everything the writer records that no single invocation +// knows. StartedAt/FinishedAt/Status/Resumed come from the run wiring. +type MetadataInput struct { + Cfg *config.Config + ConfigFile string // basename of the config, e.g. "phase4.toml" + RunID string + BuiltCommit string + Resumed bool + Hardware Hardware + Hostname string + StartedAt string // RFC3339-like UTC, 2026-07-31T22:00:00Z + FinishedAt string // "" on the up-front write + Status string // StatusRunning / StatusFinished / StatusFailed +} + +// metadataFile is the write-side of the manifest. It is a separate type from +// Metadata (the read side, which models only what the runner consumes) because +// this one is the contract: field order is declaration order, and the quirks +// the bash writer had — "yes"/"no", the comma-string concurrency, the fields +// that vanish when empty — live in these tags and in config's *String helpers. +type metadataFile struct { + SchemaVersion int `json:"schema_version"` + RunID string `json:"run_id"` + Campaign campaignMetadata `json:"campaign"` + Datasets []datasetMetadata `json:"datasets"` + Hardware Hardware `json:"hardware"` + Hostname string `json:"hostname"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at,omitempty"` + Status string `json:"status"` +} + +type campaignMetadata struct { + Name string `json:"name"` + ConfigFile string `json:"config_file"` + Ref string `json:"ref"` + BuiltCommit string `json:"built_commit"` + Ingest string `json:"ingest"` + Query string `json:"query"` + CloseInterval string `json:"close_interval"` + Runs int `json:"runs"` + QueryConcurrency string `json:"query_concurrency"` + ColdIters int `json:"cold_iters"` + HotIters int `json:"hot_iters"` + Workers int `json:"workers"` + HotNumLedgers int `json:"hot_num_ledgers"` + // Resumed is present only on resumed bundles, matching bash's del(). + Resumed bool `json:"resumed,omitempty"` +} + +type datasetMetadata struct { + Name string `json:"name"` + Kind string `json:"kind"` + Location string `json:"location"` + Chunks []int `json:"chunks"` +} + +// WriteMetadata writes /metadata.json. Written twice per campaign: up +// front (no finished_at, status running) so a killed campaign leaves a +// parseable bundle, and at the end with finished_at and a final status. +func WriteMetadata(dir string, in MetadataInput) error { + b, err := marshalMetadata(in) + if err != nil { + return err + } + path := filepath.Join(dir, MetadataName) + if err := os.WriteFile(path, b, 0o644); err != nil { + return fmt.Errorf("bundle: write %s: %w", path, err) + } + return nil +} + +// marshalMetadata renders the manifest bytes, trailing newline and all, so the +// contract test can compare them without touching the filesystem. +func marshalMetadata(in MetadataInput) ([]byte, error) { + cfg := in.Cfg + m := metadataFile{ + SchemaVersion: SchemaVersion, + RunID: in.RunID, + Campaign: campaignMetadata{ + Name: cfg.Name, + ConfigFile: in.ConfigFile, + Ref: cfg.Ref, + BuiltCommit: in.BuiltCommit, + Ingest: cfg.Ingest, + Query: cfg.QueryString(), + CloseInterval: cfg.CloseInterval, + Runs: cfg.Runs, + QueryConcurrency: cfg.QueryConcurrencyString(), + ColdIters: cfg.ColdIters, + HotIters: cfg.HotIters, + Workers: cfg.Workers, + HotNumLedgers: cfg.HotNumLedgers, + Resumed: in.Resumed, + }, + Datasets: make([]datasetMetadata, 0, len(cfg.Datasets)), + Hardware: in.Hardware, + Hostname: in.Hostname, + StartedAt: in.StartedAt, + FinishedAt: in.FinishedAt, + Status: in.Status, + } + for i := range cfg.Datasets { + d := &cfg.Datasets[i] + m.Datasets = append(m.Datasets, datasetMetadata{ + Name: d.Name, + Kind: d.Kind, + Location: d.LocationString(), + Chunks: d.Chunks, + }) + } + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return nil, fmt.Errorf("bundle: marshal %s: %w", MetadataName, err) + } + return append(b, '\n'), nil +} + +// DefaultIMDSBase is the EC2 instance metadata service root. +const DefaultIMDSBase = "http://169.254.169.254" + +// imdsTimeout caps every metadata-service request: off EC2 the address is +// unroutable, and the campaign must not stall on it. +const imdsTimeout = 2 * time.Second + +// CollectHardware gathers the structured hardware facts: EC2 instance identity +// via IMDSv2 (absent off EC2), CPU count, and MemTotal on Linux. imdsBase is +// the metadata service root, overridable for tests. Every fact is best-effort: +// what cannot be gathered stays zero, and the writer omits it. +func CollectHardware(imdsBase string) Hardware { + hw := Hardware{ + Uname: commandOutput("uname", "-srm"), + CPUs: runtime.NumCPU(), + } + hw.InstanceType, hw.InstanceID = ec2Identity(imdsBase) + hw.MemTotalKB = memTotalKB() + return hw +} + +// ec2Identity asks IMDSv2 for the instance type and id. Any failure means "not +// on EC2" — normal, not an error. +func ec2Identity(imdsBase string) (instanceType, instanceID string) { + client := &http.Client{Timeout: imdsTimeout} + req, err := http.NewRequest(http.MethodPut, imdsBase+"/latest/api/token", nil) + if err != nil { + return "", "" + } + req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "60") + token, err := readBody(client, req) + if err != nil { + return "", "" + } + get := func(path string) string { + req, err := http.NewRequest(http.MethodGet, imdsBase+path, nil) + if err != nil { + return "" + } + req.Header.Set("X-aws-ec2-metadata-token", token) + v, err := readBody(client, req) + if err != nil { + return "" + } + return v + } + return get("/latest/meta-data/instance-type"), get("/latest/meta-data/instance-id") +} + +// readBody performs req and returns its trimmed body, treating any non-2xx as +// an error the way curl -f does. +func readBody(client *http.Client, req *http.Request) (string, error) { + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return "", fmt.Errorf("%s: %s", req.URL, resp.Status) + } + b, err := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +// memTotalKB reads MemTotal from /proc/meminfo, in kB. Absent (macOS) → 0, and +// the field is omitted, exactly as bash only set it when /proc/meminfo existed. +func memTotalKB() int64 { + f, err := os.Open("/proc/meminfo") + if err != nil { + return 0 + } + defer f.Close() + scan := bufio.NewScanner(f) + for scan.Scan() { + fields := strings.Fields(scan.Text()) + if len(fields) >= 2 && fields[0] == "MemTotal:" { + kb, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0 + } + return kb + } + } + return 0 +} + +// commandOutput runs a fact-gathering command and returns its trimmed stdout, +// or "" if it cannot be run. Every caller here is best-effort by contract. +func commandOutput(name string, args ...string) string { + out, err := exec.Command(name, args...).Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} diff --git a/runner/internal/bundle/metadata_test.go b/runner/internal/bundle/metadata_test.go new file mode 100644 index 0000000..825b2fe --- /dev/null +++ b/runner/internal/bundle/metadata_test.go @@ -0,0 +1,263 @@ +package bundle + +import ( + "flag" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +var update = flag.Bool("update", false, "rewrite testdata/metadata.golden.json from the current writer output") + +const metadataGoldenPath = "testdata/metadata.golden.json" + +// goldenInput pins everything the writer cannot derive from the config, so the +// manifest below is a function of the committed testdata config alone. +func goldenInput(t *testing.T) MetadataInput { + t.Helper() + cfg, err := config.Load("testdata/campaign.toml") + if err != nil { + t.Fatalf("config.Load(testdata/campaign.toml): %v", err) + } + return MetadataInput{ + Cfg: cfg, + ConfigFile: "campaign.toml", + RunID: "golden-deadbeef-20260101T000000Z", + BuiltCommit: "deadbeefcafebabefeedface1234567890abcdef", + Hardware: Hardware{ + InstanceType: "i4i.4xlarge", + InstanceID: "i-0123456789abcdef0", + Uname: "Linux 6.8.0-1029-aws x86_64", + CPUs: 16, + MemTotalKB: 131033600, + }, + Hostname: "bench-devbox", + StartedAt: "2026-01-01T00:00:00Z", + FinishedAt: "2026-01-01T04:30:00Z", + Status: StatusFinished, + } +} + +// marshal renders the manifest bytes for in, failing the test on error. +func marshal(t *testing.T, in MetadataInput) string { + t.Helper() + b, err := marshalMetadata(in) + if err != nil { + t.Fatalf("marshalMetadata: %v", err) + } + return string(b) +} + +// TestMetadataGolden is the contract test: metadata.json is read by +// converter/convert.py in another repo, so its bytes are pinned. +func TestMetadataGolden(t *testing.T) { + got := marshal(t, goldenInput(t)) + if *update { + if err := os.WriteFile(metadataGoldenPath, []byte(got), 0o644); err != nil { + t.Fatalf("write golden: %v", err) + } + t.Logf("wrote %s", metadataGoldenPath) + return + } + want, err := os.ReadFile(metadataGoldenPath) + if err != nil { + t.Fatalf("read golden: %v (run: go test ./internal/bundle -run Golden -update)", err) + } + if got != string(want) { + t.Errorf("metadata.json differs from %s.\n--- got ---\n%s", metadataGoldenPath, got) + } +} + +// TestMetadataQuirks asserts the shapes the bash writer had, independently of +// the golden bytes: the converter reads these three and nothing else would +// catch a "fix" that made them natural Go types. +func TestMetadataQuirks(t *testing.T) { + got := marshal(t, goldenInput(t)) + for _, want := range []string{ + `"query": "yes"`, + `"query_concurrency": "1,4,16"`, + `"close_interval": "2s"`, + `"schema_version": 1`, + } { + if !strings.Contains(got, want) { + t.Errorf("metadata.json missing %s:\n%s", want, got) + } + } + // The fixture dataset has no location; bash recorded its per-chunk ledger + // count in that field, and the converter reads it there. + if !strings.Contains(got, `"kind": "fixture",`) || !strings.Contains(got, `"location": "10000"`) { + t.Errorf("fixture dataset should record location \"10000\":\n%s", got) + } +} + +func TestMetadataQueryNo(t *testing.T) { + in := goldenInput(t) + in.Cfg.Query = false + in.Cfg.QueryConcurrency = []int{8} + got := marshal(t, in) + if !strings.Contains(got, `"query": "no"`) || !strings.Contains(got, `"query_concurrency": "8"`) { + t.Errorf("query = false should render \"no\" and a one-element sweep:\n%s", got) + } +} + +// TestMetadataUpFrontWrite covers the write that happens before any leg runs: +// no finished_at, status running, and no resumed key on a fresh campaign. +func TestMetadataUpFrontWrite(t *testing.T) { + in := goldenInput(t) + in.FinishedAt = "" + in.Status = StatusRunning + got := marshal(t, in) + if strings.Contains(got, "finished_at") { + t.Errorf("up-front write must omit finished_at entirely:\n%s", got) + } + if !strings.Contains(got, `"status": "running"`) { + t.Errorf("up-front write should be status running:\n%s", got) + } + if strings.Contains(got, "resumed") { + t.Errorf("a fresh campaign must omit campaign.resumed:\n%s", got) + } +} + +func TestMetadataResumed(t *testing.T) { + in := goldenInput(t) + in.Resumed = true + if got := marshal(t, in); !strings.Contains(got, `"resumed": true`) { + t.Errorf("a resumed campaign should record resumed: true:\n%s", got) + } +} + +// TestMetadataHardwareOmitted checks that an unavailable fact is absent rather +// than "" or 0 — the with_entries filter bash's jq applied. +func TestMetadataHardwareOmitted(t *testing.T) { + in := goldenInput(t) + in.Hardware = Hardware{Uname: "Darwin 25.5.0 arm64"} + got := marshal(t, in) + for _, absent := range []string{"instance_type", "instance_id", "cpus", "mem_total_kb"} { + if strings.Contains(got, absent) { + t.Errorf("unavailable hardware fact %s should be omitted:\n%s", absent, got) + } + } + if !strings.Contains(got, `"uname": "Darwin 25.5.0 arm64"`) { + t.Errorf("uname is always recorded:\n%s", got) + } +} + +// TestWriteMetadataRoundTrip proves the writer and the resume reader agree on +// the fields --resume recovers. +func TestWriteMetadataRoundTrip(t *testing.T) { + dir := t.TempDir() + in := goldenInput(t) + if err := WriteMetadata(dir, in); err != nil { + t.Fatalf("WriteMetadata: %v", err) + } + meta, err := ReadMetadata(dir) + if err != nil { + t.Fatalf("ReadMetadata: %v", err) + } + for _, f := range []struct{ name, got, want string }{ + {"run_id", meta.RunID, in.RunID}, + {"campaign.name", meta.Campaign.Name, in.Cfg.Name}, + {"campaign.config_file", meta.Campaign.ConfigFile, in.ConfigFile}, + {"campaign.ref", meta.Campaign.Ref, in.Cfg.Ref}, + {"campaign.built_commit", meta.Campaign.BuiltCommit, in.BuiltCommit}, + {"started_at", meta.StartedAt, in.StartedAt}, + {"finished_at", meta.FinishedAt, in.FinishedAt}, + {"status", meta.Status, in.Status}, + } { + if f.got != f.want { + t.Errorf("%s = %q, want %q", f.name, f.got, f.want) + } + } + if meta.SchemaVersion != SchemaVersion { + t.Errorf("schema_version = %d, want %d", meta.SchemaVersion, SchemaVersion) + } +} + +// --- hardware collection -------------------------------------------------- + +// imdsServer serves IMDSv2 the way EC2 does: a token PUT, then facts that +// require the token header. +func imdsServer(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/latest/api/token" { + if r.Method != http.MethodPut || r.Header.Get("X-aws-ec2-metadata-token-ttl-seconds") == "" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = w.Write([]byte("TOKEN123")) + return + } + if r.Header.Get("X-aws-ec2-metadata-token") != "TOKEN123" { + w.WriteHeader(http.StatusUnauthorized) + return + } + switch r.URL.Path { + case "/latest/meta-data/instance-type": + _, _ = w.Write([]byte("i4i.4xlarge")) + case "/latest/meta-data/instance-id": + _, _ = w.Write([]byte("i-0123456789abcdef0")) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestCollectHardwareOnEC2(t *testing.T) { + hw := CollectHardware(imdsServer(t).URL) + if hw.InstanceType != "i4i.4xlarge" || hw.InstanceID != "i-0123456789abcdef0" { + t.Errorf("instance identity = %q/%q, want i4i.4xlarge/i-0123456789abcdef0", hw.InstanceType, hw.InstanceID) + } + if hw.Uname == "" { + t.Error("uname should be recorded on every platform") + } + if hw.CPUs < 1 { + t.Errorf("cpus = %d, want >= 1", hw.CPUs) + } +} + +// TestCollectHardwareNoToken covers a metadata service that refuses IMDSv2 +// tokens: the facts are absent, not an error. +func TestCollectHardwareNoToken(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + hw := CollectHardware(srv.URL) + if hw.InstanceType != "" || hw.InstanceID != "" { + t.Errorf("instance identity = %q/%q, want both empty", hw.InstanceType, hw.InstanceID) + } + if hw.Uname == "" { + t.Error("uname should still be recorded") + } +} + +// TestCollectHardwareOffEC2 is the normal laptop case: nothing answers at the +// metadata address, and the campaign must not stall on it. +func TestCollectHardwareOffEC2(t *testing.T) { + // A closed port on loopback: refused immediately, and even a black-holed + // address would be capped by the 2s client timeout. + start := time.Now() + hw := CollectHardware("http://127.0.0.1:1") + if elapsed := time.Since(start); elapsed > 3*time.Second { + t.Errorf("CollectHardware took %s off EC2, want well under the 2s-per-request timeout", elapsed) + } + if hw.InstanceType != "" || hw.InstanceID != "" { + t.Errorf("instance identity = %q/%q, want both empty", hw.InstanceType, hw.InstanceID) + } +} + +func TestWriteMetadataUnwritableDir(t *testing.T) { + err := WriteMetadata(filepath.Join(t.TempDir(), "nope"), goldenInput(t)) + if err == nil { + t.Fatal("WriteMetadata into a missing directory should fail") + } +} diff --git a/runner/internal/bundle/provenance.go b/runner/internal/bundle/provenance.go new file mode 100644 index 0000000..7ccf24a --- /dev/null +++ b/runner/internal/bundle/provenance.go @@ -0,0 +1,197 @@ +package bundle + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +// BinaryInfoName and MachineMetadataName are the bundle root's two free-text +// provenance files. Nothing parses them; they are what an operator reads six +// months later to know what ran on what. +const ( + BinaryInfoName = "binary.txt" + MachineMetadataName = "machine-metadata.txt" +) + +// WriteBinaryInfo writes /binary.txt, the benchmarked binary's identity in +// free text: binary path, commit, ref, repo, and the binary's own `version` +// output (first 3 lines, stdout+stderr). +func WriteBinaryInfo(dir, binPath, repo, ref, builtCommit string) error { + lines := []string{ + "binary: " + binPath, + "commit: " + builtCommit, + "ref: " + ref, + "repo: " + repo, + } + lines = append(lines, binaryVersion(binPath)...) + return writeLines(filepath.Join(dir, BinaryInfoName), lines) +} + +// MachineInput is what the machine-metadata writer needs beyond the shared +// hardware facts. +type MachineInput struct { + Cfg *config.Config + Repo, Ref, BuiltCommit, BinPath string + BenchRoot string // for the fsync probe file + Hardware Hardware // the shared IMDS facts (no second query) +} + +// WriteMachineMetadata writes /machine-metadata.txt: what the campaign ran +// on, in free text. Every fact is best-effort — one that cannot be gathered is +// absent from the file rather than an error, because a missing lsblk must never +// cost a campaign its bundle. +func WriteMachineMetadata(dir string, in MachineInput) error { + var lines []string + add := func(s string) { + if s = strings.TrimRight(s, "\n"); s != "" { + lines = append(lines, strings.Split(s, "\n")...) + } + } + + add(time.Now().UTC().Format("Mon Jan 2 15:04:05 MST 2006")) + if in.Hardware.InstanceType != "" { + add("instance-type: " + in.Hardware.InstanceType) + } + if in.Hardware.InstanceID != "" { + add("instance-id: " + in.Hardware.InstanceID) + } + add(commandOutput("uname", "-a")) + add(commandOutput("lsb_release", "-ds")) + add(cpuFacts()) + add(head(commandOutput("free", "-h"), 2)) + add(commandOutput("lsblk", "-o", "NAME,SIZE,MODEL")) + add("repo: " + in.Repo) + add(fmt.Sprintf("ref: %s (%s)", in.Ref, in.BuiltCommit)) + add(fmt.Sprintf("binary: %s (commit %s)", in.BinPath, in.BuiltCommit)) + lines = append(lines, binaryVersion(in.BinPath)...) + add(commandOutput("go", "version")) + add(rustcVersion()) + + cfg := in.Cfg + add(fmt.Sprintf("campaign: %s · ingest: %s · query: %s · runs: %d · concurrency: %s", + cfg.Name, cfg.Ingest, cfg.QueryString(), cfg.Runs, cfg.QueryConcurrencyString())) + add(fmt.Sprintf("cold-iters: %d · hot-iters: %d · close-interval: %s · workers: %d · hot-num-ledgers: %d", + cfg.ColdIters, cfg.HotIters, cfg.CloseInterval, cfg.Workers, cfg.HotNumLedgers)) + add(fsyncProbe(in.BenchRoot)) + + return writeLines(filepath.Join(dir, MachineMetadataName), lines) +} + +// cpuFacts is lscpu's model and CPU count on Linux, sysctl's equivalents on +// macOS, and nothing at all where neither exists. +func cpuFacts() string { + if out := commandOutput("lscpu"); out != "" { + var keep []string + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "Model name") || strings.HasPrefix(line, "CPU(s)") { + keep = append(keep, line) + } + } + if len(keep) > 0 { + return strings.Join(keep, "\n") + } + } + return commandOutput("sysctl", "-n", "machdep.cpu.brand_string", "hw.memsize", "hw.ncpu") +} + +// rustcVersion tries PATH first, then the rustup default install location — +// rustc is often installed for a user whose PATH the campaign does not inherit. +func rustcVersion() string { + if out := commandOutput("rustc", "--version"); out != "" { + return out + } + if home, err := os.UserHomeDir(); err == nil { + return commandOutput(filepath.Join(home, ".cargo", "bin", "rustc"), "--version") + } + return "" +} + +// fsyncProbeWrites and fsyncProbeBlock size the probe: 2000 synchronous 4 KiB +// writes, the same shape as the dd probe bash ran. +const ( + fsyncProbeWrites = 2000 + fsyncProbeBlock = 4096 +) + +// fsyncProbe measures synchronous write throughput on the bench root's disk — +// the single number that explains an ingest campaign that came out slow. It is +// native Go rather than dd because dd's oflag=dsync does not exist on macOS, +// and the reported line says how it was measured so no one has to guess. +func fsyncProbe(benchRoot string) string { + if benchRoot == "" { + return "fsync probe: unavailable (no BENCH_ROOT)" + } + path := filepath.Join(benchRoot, ".fsync-probe") + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0o644) + if err != nil { + return fmt.Sprintf("fsync probe: unavailable (%v)", err) + } + buf := make([]byte, fsyncProbeBlock) + start := time.Now() + for i := 0; i < fsyncProbeWrites; i++ { + n, err := f.Write(buf) + if err == nil && n != fsyncProbeBlock { + err = fmt.Errorf("short write: %d of %d bytes", n, fsyncProbeBlock) + } + if err != nil { + f.Close() + os.Remove(path) + return fmt.Sprintf("fsync probe: unavailable (%v)", err) + } + } + elapsed := time.Since(start) + closeErr := f.Close() + os.Remove(path) // best-effort cleanup, like bash's rm -f: a leftover probe file is truncated by the next probe + if closeErr != nil { + return fmt.Sprintf("fsync probe: unavailable (%v)", closeErr) + } + mbps := float64(fsyncProbeWrites*fsyncProbeBlock) / 1e6 / elapsed.Seconds() + return fmt.Sprintf("fsync probe: %.1f MB/s (native Go O_SYNC probe, %dKiB x %d)", + mbps, fsyncProbeBlock/1024, fsyncProbeWrites) +} + +// binaryVersion is the benchmarked binary's own version output, first 3 lines, +// stdout and stderr together — the binary reports its build stamp there. A +// binary that cannot run says so in place of its version. +func binaryVersion(binPath string) []string { + out, err := exec.Command(binPath, "version").CombinedOutput() + lines := splitLines(head(strings.TrimRight(string(out), "\n"), 3)) + if err != nil && len(lines) == 0 { + return []string{fmt.Sprintf("version: %v", err)} + } + return lines +} + +// head is the first n lines of s. +func head(s string, n int) string { + lines := splitLines(s) + if len(lines) > n { + lines = lines[:n] + } + return strings.Join(lines, "\n") +} + +func splitLines(s string) []string { + if s == "" { + return nil + } + return strings.Split(s, "\n") +} + +// writeLines writes one line per element, with a trailing newline. +func writeLines(path string, lines []string) error { + body := "" + if len(lines) > 0 { + body = strings.Join(lines, "\n") + "\n" + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + return fmt.Errorf("bundle: write %s: %w", path, err) + } + return nil +} diff --git a/runner/internal/bundle/provenance_test.go b/runner/internal/bundle/provenance_test.go new file mode 100644 index 0000000..d5379c2 --- /dev/null +++ b/runner/internal/bundle/provenance_test.go @@ -0,0 +1,168 @@ +package bundle + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +// stubBinary writes an executable that answers `version` with four lines, so +// the writers' head -3 equivalent has something to cut. +func stubBinary(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "stellar-rpc-v2") + script := "#!/bin/sh\n" + + "echo 'stellar-rpc-v2 v23.0.0'\n" + + "echo 'commit: deadbeef'\n" + + "echo 'build: 2026-01-01'\n" + + "echo 'fourth line should be cut'\n" + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write stub binary: %v", err) + } + return path +} + +// readFile is the written provenance file's contents. +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func TestWriteBinaryInfo(t *testing.T) { + dir := t.TempDir() + bin := stubBinary(t) + err := WriteBinaryInfo(dir, bin, "https://github.com/stellar/stellar-rpc.git", + "feature/full-history", "deadbeefcafebabe") + if err != nil { + t.Fatalf("WriteBinaryInfo: %v", err) + } + got := readFile(t, filepath.Join(dir, BinaryInfoName)) + for _, want := range []string{ + "binary: " + bin, + "commit: deadbeefcafebabe", + "ref: feature/full-history", + "repo: https://github.com/stellar/stellar-rpc.git", + "stellar-rpc-v2 v23.0.0", + "build: 2026-01-01", + } { + if !strings.Contains(got, want) { + t.Errorf("binary.txt missing %q:\n%s", want, got) + } + } + if strings.Contains(got, "fourth line") { + t.Errorf("binary.txt should keep only the first 3 version lines:\n%s", got) + } +} + +// TestWriteBinaryInfoMissingBinary covers the best-effort contract: a binary +// that cannot run costs the file its version lines, not the campaign its +// bundle. +func TestWriteBinaryInfoMissingBinary(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "not-a-binary") + if err := WriteBinaryInfo(dir, missing, "repo", "ref", "commit"); err != nil { + t.Fatalf("WriteBinaryInfo: %v", err) + } + got := readFile(t, filepath.Join(dir, BinaryInfoName)) + if !strings.Contains(got, "binary: "+missing) { + t.Errorf("binary.txt should still record the identity lines:\n%s", got) + } + if !strings.Contains(got, "version:") { + t.Errorf("binary.txt should report why the version is missing:\n%s", got) + } +} + +func machineInput(t *testing.T, benchRoot, bin string) MachineInput { + t.Helper() + cfg, err := config.Load("testdata/campaign.toml") + if err != nil { + t.Fatalf("config.Load(testdata/campaign.toml): %v", err) + } + return MachineInput{ + Cfg: cfg, + Repo: "https://github.com/stellar/stellar-rpc.git", + Ref: "feature/full-history", + BuiltCommit: "deadbeefcafebabe", + BinPath: bin, + BenchRoot: benchRoot, + Hardware: Hardware{ + InstanceType: "i4i.4xlarge", + InstanceID: "i-0123456789abcdef0", + Uname: "Linux 6.8.0-1029-aws x86_64", + }, + } +} + +func TestWriteMachineMetadata(t *testing.T) { + dir := t.TempDir() + benchRoot := t.TempDir() + in := machineInput(t, benchRoot, stubBinary(t)) + if err := WriteMachineMetadata(dir, in); err != nil { + t.Fatalf("WriteMachineMetadata: %v", err) + } + got := readFile(t, filepath.Join(dir, MachineMetadataName)) + for _, want := range []string{ + "instance-type: i4i.4xlarge", + "instance-id: i-0123456789abcdef0", + "repo: https://github.com/stellar/stellar-rpc.git", + "ref: feature/full-history (deadbeefcafebabe)", + "binary: " + in.BinPath + " (commit deadbeefcafebabe)", + "stellar-rpc-v2 v23.0.0", + "campaign: golden · ingest: both · query: yes · runs: 5 · concurrency: 1,4,16", + "cold-iters: 100 · hot-iters: 200 · close-interval: 2s · workers: 1 · hot-num-ledgers: 50000", + "fsync probe: ", + } { + if !strings.Contains(got, want) { + t.Errorf("machine-metadata.txt missing %q:\n%s", want, got) + } + } + if strings.Contains(got, "fsync probe: unavailable") { + t.Errorf("the probe should have run against a writable BENCH_ROOT:\n%s", got) + } + if _, err := os.Stat(filepath.Join(benchRoot, ".fsync-probe")); !os.IsNotExist(err) { + t.Errorf("the probe file should be removed, stat err = %v", err) + } + // Every fact is best-effort, so the file must never carry a blank line + // where an absent one would have been. + for i, line := range strings.Split(strings.TrimRight(got, "\n"), "\n") { + if strings.TrimSpace(line) == "" { + t.Errorf("machine-metadata.txt line %d is blank — absent facts should leave no gap", i+1) + } + } +} + +// TestWriteMachineMetadataNoHardware covers a machine off EC2: the instance +// lines are absent rather than empty. +func TestWriteMachineMetadataNoHardware(t *testing.T) { + dir := t.TempDir() + in := machineInput(t, t.TempDir(), stubBinary(t)) + in.Hardware = Hardware{} + if err := WriteMachineMetadata(dir, in); err != nil { + t.Fatalf("WriteMachineMetadata: %v", err) + } + got := readFile(t, filepath.Join(dir, MachineMetadataName)) + if strings.Contains(got, "instance-type") || strings.Contains(got, "instance-id") { + t.Errorf("off EC2 the instance lines should be absent:\n%s", got) + } +} + +// TestWriteMachineMetadataUnwritableProbe covers a BENCH_ROOT the probe cannot +// write: the file still gets written, with the probe reporting why. +func TestWriteMachineMetadataUnwritableProbe(t *testing.T) { + dir := t.TempDir() + in := machineInput(t, filepath.Join(t.TempDir(), "missing"), stubBinary(t)) + if err := WriteMachineMetadata(dir, in); err != nil { + t.Fatalf("WriteMachineMetadata: %v", err) + } + got := readFile(t, filepath.Join(dir, MachineMetadataName)) + if !strings.Contains(got, "fsync probe: unavailable") { + t.Errorf("an unwritable BENCH_ROOT should report an unavailable probe:\n%s", got) + } +} diff --git a/runner/internal/bundle/testdata/campaign.toml b/runner/internal/bundle/testdata/campaign.toml new file mode 100644 index 0000000..42eed26 --- /dev/null +++ b/runner/internal/bundle/testdata/campaign.toml @@ -0,0 +1,22 @@ +# Golden-test campaign for the metadata.json contract: one fetched pack tree +# and one generated fixture, so the manifest exercises both a real location and +# the fixture quirk (ledger count recorded in the location field). +name = "golden" +ref = "feature/full-history" +ingest = "both" +query = true +runs = 5 +close_interval = "2s" +hot_num_ledgers = 50000 + +[[dataset]] +name = "packs" +kind = "packs-gs" +location = "gs://bucket/cold" +chunks = [1, 2] + +[[dataset]] +name = "fix" +kind = "fixture" +ledgers = 10000 +chunks = [1] diff --git a/runner/internal/bundle/testdata/metadata.golden.json b/runner/internal/bundle/testdata/metadata.golden.json new file mode 100644 index 0000000..931bb6b --- /dev/null +++ b/runner/internal/bundle/testdata/metadata.golden.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "run_id": "golden-deadbeef-20260101T000000Z", + "campaign": { + "name": "golden", + "config_file": "campaign.toml", + "ref": "feature/full-history", + "built_commit": "deadbeefcafebabefeedface1234567890abcdef", + "ingest": "both", + "query": "yes", + "close_interval": "2s", + "runs": 5, + "query_concurrency": "1,4,16", + "cold_iters": 100, + "hot_iters": 200, + "workers": 1, + "hot_num_ledgers": 50000 + }, + "datasets": [ + { + "name": "packs", + "kind": "packs-gs", + "location": "gs://bucket/cold", + "chunks": [ + 1, + 2 + ] + }, + { + "name": "fix", + "kind": "fixture", + "location": "10000", + "chunks": [ + 1 + ] + } + ], + "hardware": { + "instance_type": "i4i.4xlarge", + "instance_id": "i-0123456789abcdef0", + "uname": "Linux 6.8.0-1029-aws x86_64", + "cpus": 16, + "mem_total_kb": 131033600 + }, + "hostname": "bench-devbox", + "started_at": "2026-01-01T00:00:00Z", + "finished_at": "2026-01-01T04:30:00Z", + "status": "finished" +} diff --git a/runner/internal/config/config.go b/runner/internal/config/config.go new file mode 100644 index 0000000..dd182c9 --- /dev/null +++ b/runner/internal/config/config.go @@ -0,0 +1,319 @@ +// Package config loads and validates campaign configs: TOML in, a validated +// Config out, with unknown keys and out-of-range values rejected up front. +// +// The rules here are ported from runner/campaign.sh's validation block; its +// error messages are operator UX, so they are reproduced with the same +// specificity, naming the TOML key instead of the bash variable. +package config + +import ( + "errors" + "fmt" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/BurntSushi/toml" +) + +// Defaults for the keys that have one, matching campaign.sh. +const ( + DefaultRepo = "https://github.com/stellar/stellar-rpc.git" + DefaultRef = "feature/full-history" + DefaultCloseInterval = "0" + DefaultRuns = 5 + DefaultColdIters = 100 + DefaultHotIters = 200 + DefaultWorkers = 1 +) + +// Dataset kinds. Each names a different way of materializing a cold pack root; +// see runner/README.md for what location means for each. +const ( + KindPacksLocal = "packs-local" + KindPacksGS = "packs-gs" + KindBSBS3 = "bsb-s3" + KindFixture = "fixture" +) + +// minFixtureLedgers is the smallest non-zero fixture ledger count: the cold +// freeze streams a whole 10,000-ledger chunk, so a partial chunk cannot be +// frozen. +const minFixtureLedgers = 10000 + +// Config is a whole campaign config file. +type Config struct { + Name string `toml:"name"` + Repo string `toml:"repo"` + Ref string `toml:"ref"` + Ingest string `toml:"ingest"` + Query bool `toml:"query"` + CloseInterval string `toml:"close_interval"` + Runs int `toml:"runs"` + QueryConcurrency []int `toml:"query_concurrency"` + ColdIters int `toml:"cold_iters"` + HotIters int `toml:"hot_iters"` + Workers int `toml:"workers"` + HotNumLedgers int `toml:"hot_num_ledgers"` + PublishURI string `toml:"publish_uri"` + Datasets []Dataset `toml:"dataset"` +} + +// Dataset is one [[dataset]] table: a named pack tree and the chunks of it +// this campaign benchmarks. +type Dataset struct { + Name string `toml:"name"` + Kind string `toml:"kind"` + Location string `toml:"location"` + Chunks []int `toml:"chunks"` + // Ledgers is the per-chunk ledger count of a fixture dataset, and is + // invalid for every other kind. It is a pointer because 0 is a + // meaningful value (the whole chunk) that must be told from unset. + Ledgers *int `toml:"ledgers"` +} + +var reName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +// defaults returns a Config pre-filled with the documented defaults. Decoding +// on top of it leaves absent keys at their default and lets present keys win. +func defaults() Config { + return Config{ + Repo: DefaultRepo, + Ref: DefaultRef, + CloseInterval: DefaultCloseInterval, + Runs: DefaultRuns, + QueryConcurrency: []int{1, 4, 16}, + ColdIters: DefaultColdIters, + HotIters: DefaultHotIters, + Workers: DefaultWorkers, + } +} + +// Load parses the TOML config at path, applies defaults, and validates it. +func Load(path string) (*Config, error) { + cfg := defaults() + md, err := toml.DecodeFile(path, &cfg) + if err != nil { + return nil, fmt.Errorf("config: %w", err) + } + if undecoded := md.Undecoded(); len(undecoded) > 0 { + keys := make([]string, len(undecoded)) + for i, k := range undecoded { + keys[i] = k.String() + } + plural := "" + if len(keys) > 1 { + plural = "s" + } + return nil, fmt.Errorf("config: unknown key%s: %s", plural, strings.Join(keys, ", ")) + } + // A bool cannot express "unset", so ask the metadata whether the + // operator actually chose. + if !md.IsDefined("query") { + return nil, errors.New("config: query is required (true|false)") + } + if err := cfg.validate(); err != nil { + return nil, err + } + return &cfg, nil +} + +func (c *Config) validate() error { + if c.Name == "" { + return errors.New("config: name is required") + } + if !reName.MatchString(c.Name) { + return fmt.Errorf("config: name must match [A-Za-z0-9._-]+ (got '%s')", c.Name) + } + if err := validateRepo(c.Repo); err != nil { + return err + } + if c.Ref == "" { + return errors.New("config: ref must not be empty") + } + switch c.Ingest { + case "cold", "hot", "both", "none": + default: + got := c.Ingest + if got == "" { + got = "" + } + return fmt.Errorf("config: ingest must be cold|hot|both|none (got '%s')", got) + } + if err := validateCloseInterval(c.CloseInterval); err != nil { + return err + } + for _, f := range []struct { + key string + value int + }{ + {"runs", c.Runs}, + {"cold_iters", c.ColdIters}, + {"hot_iters", c.HotIters}, + {"workers", c.Workers}, + } { + if f.value < 1 { + return fmt.Errorf("config: %s must be an integer >= 1 (got '%d')", f.key, f.value) + } + } + if c.HotNumLedgers < 0 { + return fmt.Errorf("config: hot_num_ledgers must be an integer >= 0 (got '%d')", c.HotNumLedgers) + } + if len(c.QueryConcurrency) == 0 { + return errors.New("config: query_concurrency must list at least one concurrency level") + } + for _, qc := range c.QueryConcurrency { + if qc < 1 { + return fmt.Errorf("config: query_concurrency entries must be integers >= 1 (got '%d')", qc) + } + } + if c.PublishURI != "" && !strings.HasPrefix(c.PublishURI, "gs://") && !strings.HasPrefix(c.PublishURI, "s3://") { + return fmt.Errorf("config: publish_uri must be a gs:// or s3:// URI (got '%s')", c.PublishURI) + } + if len(c.Datasets) == 0 { + return errors.New("config: at least one [[dataset]] is required") + } + seen := make(map[string]bool, len(c.Datasets)) + for i := range c.Datasets { + if err := c.Datasets[i].validate(seen); err != nil { + return err + } + } + return nil +} + +func validateRepo(repo string) error { + if repo == "" { + return errors.New("config: repo must not be empty") + } + if strings.Contains(repo, "://") || isSCPLike(repo) { + return nil + } + // Not a URL: must be an absolute path to a local git repository. + // Relative paths are refused — they would silently depend on the + // invocation cwd. + if !filepath.IsAbs(repo) { + return fmt.Errorf("config: repo must be a git URL or an absolute local path (got '%s')", repo) + } + if err := exec.Command("git", "-C", repo, "rev-parse", "--git-dir").Run(); err != nil { + return fmt.Errorf("config: repo path '%s' is not a git repository", repo) + } + return nil +} + +// isSCPLike reports whether repo has git's scp-like remote shape, user@host:path. +func isSCPLike(repo string) bool { + at := strings.Index(repo, "@") + return at >= 0 && strings.Contains(repo[at+1:], ":") +} + +func validateCloseInterval(s string) error { + if s == "0" { + return nil + } + d, err := time.ParseDuration(s) + if err != nil || d < 0 { + return fmt.Errorf("config: close_interval must be a Go duration or 0 (got '%s')", s) + } + return nil +} + +// validate checks one dataset, recording its name in seen to catch duplicates. +func (d *Dataset) validate(seen map[string]bool) error { + if !reName.MatchString(d.Name) { + return fmt.Errorf("config: dataset name must match [A-Za-z0-9._-]+ (got '%s')", d.Name) + } + if seen[d.Name] { + return fmt.Errorf("config: duplicate dataset name '%s'", d.Name) + } + seen[d.Name] = true + if len(d.Chunks) == 0 { + return fmt.Errorf("config: dataset '%s': chunks must list at least one chunk ID", d.Name) + } + // Duplicate chunk IDs are rejected outright, which bash did not do: step + // IDs embed -c-run, so a repeated chunk would produce + // two steps with the same ID and the same output directory — ambiguous to + // depend on and, on resume, indistinguishable from finished work. Bash + // silently let the second copy overwrite the first's out dirs. + seenChunks := make(map[int]bool, len(d.Chunks)) + for _, chunk := range d.Chunks { + if chunk < 0 { + return fmt.Errorf("config: dataset '%s': chunk IDs must be non-negative integers (got '%d')", d.Name, chunk) + } + if seenChunks[chunk] { + return fmt.Errorf("config: dataset '%s': duplicate chunk ID '%d'", d.Name, chunk) + } + seenChunks[chunk] = true + } + switch d.Kind { + case KindPacksLocal: + // The root's contents are checked at prep time, as in bash. + if d.Location == "" { + return fmt.Errorf("config: dataset '%s': packs-local location must be a local cold pack root", d.Name) + } + case KindPacksGS: + if !strings.HasPrefix(d.Location, "gs://") { + return fmt.Errorf("config: dataset '%s': packs-gs location must start with gs:// (got '%s')", d.Name, d.Location) + } + case KindBSBS3: + if d.Location == "" { + return fmt.Errorf("config: dataset '%s': bsb-s3 location must be an S3 bucket path", d.Name) + } + case KindFixture: + if d.Location != "" { + return fmt.Errorf("config: dataset '%s': fixture datasets use ledgers, not location (got location '%s')", d.Name, d.Location) + } + if d.Ledgers == nil { + return fmt.Errorf("config: dataset '%s': fixture datasets need ledgers = (0 or >= %d)", d.Name, minFixtureLedgers) + } + if n := *d.Ledgers; n != 0 && n < minFixtureLedgers { + return fmt.Errorf("config: dataset '%s': fixture ledger count must be 0 or >= %d — the cold freeze streams the whole 10,000-ledger chunk (got '%d')", d.Name, minFixtureLedgers, n) + } + default: + return fmt.Errorf("config: dataset '%s': kind must be packs-local|packs-gs|bsb-s3|fixture (got '%s')", d.Name, d.Kind) + } + if d.Kind != KindFixture && d.Ledgers != nil { + return fmt.Errorf("config: dataset '%s': ledgers is only valid for fixture datasets (kind is %s)", d.Name, d.Kind) + } + return nil +} + +// The three helpers below are the metadata.json compatibility layer: the +// manifest is a cross-repo contract with converter/convert.py, and it records +// these fields in the shapes the bash runner wrote. Keep them here so the +// quirks live in one place. + +// QueryString renders query as metadata.json records it: the bash config key +// took the strings "yes" and "no". +func (c *Config) QueryString() string { + if c.Query { + return "yes" + } + return "no" +} + +// QueryConcurrencyString renders the concurrency sweep as metadata.json +// records it: a comma-separated list, e.g. "1,4,16". +func (c *Config) QueryConcurrencyString() string { + parts := make([]string, len(c.QueryConcurrency)) + for i, qc := range c.QueryConcurrency { + parts[i] = strconv.Itoa(qc) + } + return strings.Join(parts, ",") +} + +// LocationString renders the dataset's location as metadata.json records it. +// Fixture datasets have no location: the bash runner overloaded that field +// with the per-chunk ledger count, so the manifest keeps the decimal count. +func (d *Dataset) LocationString() string { + if d.Kind == KindFixture { + if d.Ledgers == nil { + return "" + } + return strconv.Itoa(*d.Ledgers) + } + return d.Location +} diff --git a/runner/internal/config/config_test.go b/runner/internal/config/config_test.go new file mode 100644 index 0000000..2da6656 --- /dev/null +++ b/runner/internal/config/config_test.go @@ -0,0 +1,531 @@ +package config + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" +) + +// write puts src in a temp file and returns its path. +func write(t *testing.T, src string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "campaign.toml") + if err := os.WriteFile(path, []byte(src), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +func load(t *testing.T, src string) *Config { + t.Helper() + cfg, err := Load(write(t, src)) + if err != nil { + t.Fatalf("Load = error %v, want nil", err) + } + return cfg +} + +// minimal is the smallest accepted config: the four required things. It is +// split so that top-level keys can be added before the [[dataset]] table — +// text appended to the whole config lands inside that table instead. +const ( + minimalTop = ` +name = "min" +ingest = "both" +query = true +` + minimalDataset = ` +[[dataset]] +name = "local" +kind = "packs-local" +location = "/data/packs" +chunks = [7] +` + minimal = minimalTop + minimalDataset +) + +// withTop returns the minimal config plus extra top-level keys. +func withTop(extra string) string { + return minimalTop + extra + "\n" + minimalDataset +} + +func TestLoadFullConfig(t *testing.T) { + cfg := load(t, ` +name = "phase4" +repo = "git@github.com:stellar/stellar-rpc.git" +ref = "v1.2.3" +ingest = "hot" +query = false +close_interval = "2s" +runs = 3 +query_concurrency = [2, 8] +cold_iters = 50 +hot_iters = 60 +workers = 4 +hot_num_ledgers = 1000 +publish_uri = "s3://bucket/bench" + +[[dataset]] +name = "pubnet" +kind = "packs-gs" +location = "gs://bucket/cold" +chunks = [1, 2] + +[[dataset]] +name = "synth" +kind = "fixture" +ledgers = 10000 +chunks = [0] +`) + + if cfg.Name != "phase4" { + t.Errorf("name = %q, want phase4", cfg.Name) + } + if cfg.Repo != "git@github.com:stellar/stellar-rpc.git" { + t.Errorf("repo = %q", cfg.Repo) + } + if cfg.Ref != "v1.2.3" { + t.Errorf("ref = %q, want v1.2.3", cfg.Ref) + } + if cfg.Ingest != "hot" { + t.Errorf("ingest = %q, want hot", cfg.Ingest) + } + if cfg.Query { + t.Error("query = true, want false") + } + if cfg.CloseInterval != "2s" { + t.Errorf("close_interval = %q, want 2s", cfg.CloseInterval) + } + if cfg.Runs != 3 { + t.Errorf("runs = %d, want 3", cfg.Runs) + } + if !slices.Equal(cfg.QueryConcurrency, []int{2, 8}) { + t.Errorf("query_concurrency = %v, want [2 8]", cfg.QueryConcurrency) + } + if cfg.ColdIters != 50 || cfg.HotIters != 60 { + t.Errorf("iters = %d/%d, want 50/60", cfg.ColdIters, cfg.HotIters) + } + if cfg.Workers != 4 { + t.Errorf("workers = %d, want 4", cfg.Workers) + } + if cfg.HotNumLedgers != 1000 { + t.Errorf("hot_num_ledgers = %d, want 1000", cfg.HotNumLedgers) + } + if cfg.PublishURI != "s3://bucket/bench" { + t.Errorf("publish_uri = %q", cfg.PublishURI) + } + if len(cfg.Datasets) != 2 { + t.Fatalf("datasets = %d, want 2", len(cfg.Datasets)) + } + gs := cfg.Datasets[0] + if gs.Name != "pubnet" || gs.Kind != KindPacksGS || gs.Location != "gs://bucket/cold" { + t.Errorf("dataset[0] = %+v", gs) + } + if !slices.Equal(gs.Chunks, []int{1, 2}) { + t.Errorf("dataset[0].chunks = %v, want [1 2]", gs.Chunks) + } + if gs.Ledgers != nil { + t.Errorf("dataset[0].ledgers = %d, want unset", *gs.Ledgers) + } + fixture := cfg.Datasets[1] + if fixture.Name != "synth" || fixture.Kind != KindFixture || fixture.Location != "" { + t.Errorf("dataset[1] = %+v", fixture) + } + if fixture.Ledgers == nil || *fixture.Ledgers != 10000 { + t.Errorf("dataset[1].ledgers = %v, want 10000", fixture.Ledgers) + } + if !slices.Equal(fixture.Chunks, []int{0}) { + t.Errorf("dataset[1].chunks = %v, want [0]", fixture.Chunks) + } +} + +func TestLoadMinimalConfigAppliesDefaults(t *testing.T) { + cfg := load(t, minimal) + + if cfg.Repo != DefaultRepo { + t.Errorf("repo = %q, want %q", cfg.Repo, DefaultRepo) + } + if cfg.Ref != DefaultRef { + t.Errorf("ref = %q, want %q", cfg.Ref, DefaultRef) + } + if cfg.CloseInterval != "0" { + t.Errorf("close_interval = %q, want 0", cfg.CloseInterval) + } + if cfg.Runs != 5 { + t.Errorf("runs = %d, want 5", cfg.Runs) + } + if !slices.Equal(cfg.QueryConcurrency, []int{1, 4, 16}) { + t.Errorf("query_concurrency = %v, want [1 4 16]", cfg.QueryConcurrency) + } + if cfg.ColdIters != 100 { + t.Errorf("cold_iters = %d, want 100", cfg.ColdIters) + } + if cfg.HotIters != 200 { + t.Errorf("hot_iters = %d, want 200", cfg.HotIters) + } + if cfg.Workers != 1 { + t.Errorf("workers = %d, want 1", cfg.Workers) + } + if cfg.HotNumLedgers != 0 { + t.Errorf("hot_num_ledgers = %d, want 0", cfg.HotNumLedgers) + } + if cfg.PublishURI != "" { + t.Errorf("publish_uri = %q, want empty", cfg.PublishURI) + } +} + +func TestLoadAcceptedConfigs(t *testing.T) { + cases := []struct { + name string + src string + }{ + {"query false", strings.Replace(minimal, "query = true", "query = false", 1)}, + {"ingest none", strings.Replace(minimal, `ingest = "both"`, `ingest = "none"`, 1)}, + {"bare zero close_interval", withTop(`close_interval = "0"`)}, + {"sub-second close_interval", withTop(`close_interval = "600ms"`)}, + {"compound close_interval", withTop(`close_interval = "1m30s"`)}, + {"gs publish uri", withTop(`publish_uri = "gs://bucket/prefix"`)}, + {"chunk id zero", strings.Replace(minimal, "chunks = [7]", "chunks = [0, 1]", 1)}, + {"fixture whole chunk", ` +name = "f" +ingest = "cold" +query = true + +[[dataset]] +name = "synth" +kind = "fixture" +ledgers = 0 +chunks = [0] +`}, + {"bsb-s3 dataset", ` +name = "b" +ingest = "cold" +query = true + +[[dataset]] +name = "bsb" +kind = "bsb-s3" +location = "s3://bucket/prefix" +chunks = [1] +`}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := Load(write(t, tc.src)); err != nil { + t.Fatalf("Load = error %v, want nil", err) + } + }) + } +} + +func TestLoadRejects(t *testing.T) { + cases := []struct { + name string + src string + want []string // substrings the error must contain + }{ + { + name: "unknown top-level key", + src: withTop("runz = 3"), + want: []string{"unknown key", "runz"}, + }, + { + name: "unknown dataset key", + src: minimal + "\nlocations = \"/x\"\n", + want: []string{"unknown key", "dataset.locations"}, + }, + { + name: "several unknown keys", + src: withTop("runz = 3\nqc = \"1,4\""), + want: []string{"unknown keys", "runz", "qc"}, + }, + { + name: "missing query", + src: strings.Replace(minimal, "query = true", "", 1), + want: []string{"query is required", "true|false"}, + }, + { + name: "missing name", + src: strings.Replace(minimal, `name = "min"`, "", 1), + want: []string{"name is required"}, + }, + { + name: "bad name charset", + src: strings.Replace(minimal, `name = "min"`, `name = "phase 4!"`, 1), + want: []string{"name must match [A-Za-z0-9._-]+", "phase 4!"}, + }, + { + name: "empty repo", + src: withTop(`repo = ""`), + want: []string{"repo must not be empty"}, + }, + { + name: "relative repo path", + src: withTop(`repo = "../stellar-rpc"`), + want: []string{"repo must be a git URL or an absolute local path", "../stellar-rpc"}, + }, + { + name: "empty ref", + src: withTop(`ref = ""`), + want: []string{"ref must not be empty"}, + }, + { + name: "missing ingest", + src: strings.Replace(minimal, `ingest = "both"`, "", 1), + want: []string{"ingest must be cold|hot|both|none", ""}, + }, + { + name: "bad ingest", + src: strings.Replace(minimal, `ingest = "both"`, `ingest = "warm"`, 1), + want: []string{"ingest must be cold|hot|both|none", "warm"}, + }, + { + name: "unparsable close_interval", + src: withTop(`close_interval = "2 seconds"`), + want: []string{"close_interval must be a Go duration or 0", "2 seconds"}, + }, + { + name: "negative close_interval", + src: withTop(`close_interval = "-2s"`), + want: []string{"close_interval must be a Go duration or 0", "-2s"}, + }, + { + name: "zero runs", + src: withTop("runs = 0"), + want: []string{"runs must be an integer >= 1", "0"}, + }, + { + name: "negative cold_iters", + src: withTop("cold_iters = -1"), + want: []string{"cold_iters must be an integer >= 1", "-1"}, + }, + { + name: "zero hot_iters", + src: withTop("hot_iters = 0"), + want: []string{"hot_iters must be an integer >= 1", "0"}, + }, + { + name: "zero workers", + src: withTop("workers = 0"), + want: []string{"workers must be an integer >= 1", "0"}, + }, + { + name: "negative hot_num_ledgers", + src: withTop("hot_num_ledgers = -5"), + want: []string{"hot_num_ledgers must be an integer >= 0", "-5"}, + }, + { + name: "empty query_concurrency", + src: withTop("query_concurrency = []"), + want: []string{"query_concurrency must list at least one concurrency level"}, + }, + { + name: "zero query_concurrency entry", + src: withTop("query_concurrency = [1, 0]"), + want: []string{"query_concurrency entries must be integers >= 1", "0"}, + }, + { + name: "bad publish_uri scheme", + src: withTop(`publish_uri = "https://bucket/bench"`), + want: []string{"publish_uri must be a gs:// or s3:// URI", "https://bucket/bench"}, + }, + { + name: "no datasets", + src: ` +name = "nods" +ingest = "both" +query = true +`, + want: []string{"at least one [[dataset]] is required"}, + }, + { + name: "bad dataset name charset", + src: strings.Replace(minimal, `name = "local"`, `name = "my data"`, 1), + want: []string{"dataset name must match [A-Za-z0-9._-]+", "my data"}, + }, + { + name: "duplicate dataset names", + src: minimal + ` +[[dataset]] +name = "local" +kind = "packs-local" +location = "/data/other" +chunks = [8] +`, + want: []string{"duplicate dataset name", "local"}, + }, + { + name: "empty chunks", + src: strings.Replace(minimal, "chunks = [7]", "chunks = []", 1), + want: []string{"dataset 'local'", "chunks must list at least one chunk ID"}, + }, + { + name: "negative chunk id", + src: strings.Replace(minimal, "chunks = [7]", "chunks = [1, -2]", 1), + want: []string{"dataset 'local'", "chunk IDs must be non-negative integers", "-2"}, + }, + { + name: "duplicate chunk id", + src: strings.Replace(minimal, "chunks = [7]", "chunks = [1, 1]", 1), + want: []string{"dataset 'local'", "duplicate chunk ID", "1"}, + }, + { + name: "unknown dataset kind", + src: strings.Replace(minimal, `kind = "packs-local"`, `kind = "packs-http"`, 1), + want: []string{"dataset 'local'", "kind must be packs-local|packs-gs|bsb-s3|fixture", "packs-http"}, + }, + { + name: "packs-local without location", + src: strings.Replace(minimal, `location = "/data/packs"`, "", 1), + want: []string{"dataset 'local'", "packs-local location must be a local cold pack root"}, + }, + { + name: "packs-gs location is not gs://", + src: strings.NewReplacer( + `kind = "packs-local"`, `kind = "packs-gs"`, + `location = "/data/packs"`, `location = "s3://bucket/cold"`, + ).Replace(minimal), + want: []string{"dataset 'local'", "packs-gs location must start with gs://", "s3://bucket/cold"}, + }, + { + name: "bsb-s3 without location", + src: strings.NewReplacer( + `kind = "packs-local"`, `kind = "bsb-s3"`, + `location = "/data/packs"`, "", + ).Replace(minimal), + want: []string{"dataset 'local'", "bsb-s3 location must be an S3 bucket path"}, + }, + { + name: "fixture with a location", + src: strings.NewReplacer( + `kind = "packs-local"`, `kind = "fixture"`, + `location = "/data/packs"`, "location = \"/data/packs\"\nledgers = 10000", + ).Replace(minimal), + want: []string{"dataset 'local'", "fixture datasets use ledgers, not location", "/data/packs"}, + }, + { + name: "fixture without ledgers", + src: strings.NewReplacer( + `kind = "packs-local"`, `kind = "fixture"`, + `location = "/data/packs"`, "", + ).Replace(minimal), + want: []string{"dataset 'local'", "fixture datasets need ledgers", ">= 10000"}, + }, + { + name: "fixture with a partial chunk", + src: strings.NewReplacer( + `kind = "packs-local"`, `kind = "fixture"`, + `location = "/data/packs"`, "ledgers = 5000", + ).Replace(minimal), + want: []string{ + "dataset 'local'", + "fixture ledger count must be 0 or >= 10000", + "the cold freeze streams the whole 10,000-ledger chunk", + "5000", + }, + }, + { + name: "non-fixture with ledgers", + src: strings.Replace(minimal, "chunks = [7]", "chunks = [7]\nledgers = 10000", 1), + want: []string{"dataset 'local'", "ledgers is only valid for fixture datasets", "packs-local"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := Load(write(t, tc.src)) + if err == nil { + t.Fatal("Load = nil error, want a rejection") + } + if !strings.HasPrefix(err.Error(), "config: ") { + t.Errorf("error %q does not start with %q", err, "config: ") + } + for _, want := range tc.want { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not contain %q", err, want) + } + } + }) + } +} + +func TestLoadRejectsMissingFile(t *testing.T) { + _, err := Load(filepath.Join(t.TempDir(), "absent.toml")) + if err == nil { + t.Fatal("Load = nil error, want an error for a missing file") + } + if !strings.HasPrefix(err.Error(), "config: ") { + t.Errorf("error %q does not start with %q", err, "config: ") + } +} + +func TestLoadLocalRepoPath(t *testing.T) { + dir := t.TempDir() + if out, err := exec.Command("git", "init", "-q", dir).CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, out) + } + + if _, err := Load(write(t, withTop(`repo = "`+dir+`"`))); err != nil { + t.Fatalf("Load with a local git repo = error %v, want nil", err) + } + + notARepo := t.TempDir() + _, err := Load(write(t, withTop(`repo = "`+notARepo+`"`))) + if err == nil { + t.Fatal("Load with a non-git absolute path = nil error, want a rejection") + } + for _, want := range []string{"is not a git repository", notARepo} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not contain %q", err, want) + } + } +} + +func TestMetadataMappings(t *testing.T) { + cfg := load(t, ` +name = "map" +ingest = "both" +query = true +query_concurrency = [1, 4, 16] + +[[dataset]] +name = "local" +kind = "packs-local" +location = "/data/packs" +chunks = [1] + +[[dataset]] +name = "synth" +kind = "fixture" +ledgers = 20000 +chunks = [0] +`) + + if got := cfg.QueryString(); got != "yes" { + t.Errorf("QueryString() = %q, want yes", got) + } + cfg.Query = false + if got := cfg.QueryString(); got != "no" { + t.Errorf("QueryString() = %q, want no", got) + } + if got := cfg.QueryConcurrencyString(); got != "1,4,16" { + t.Errorf("QueryConcurrencyString() = %q, want 1,4,16", got) + } + if got := cfg.Datasets[0].LocationString(); got != "/data/packs" { + t.Errorf("LocationString() = %q, want /data/packs", got) + } + if got := cfg.Datasets[1].LocationString(); got != "20000" { + t.Errorf("fixture LocationString() = %q, want 20000", got) + } +} + +func TestQueryConcurrencyStringSingleEntry(t *testing.T) { + cfg := load(t, withTop("query_concurrency = [8]")) + if got := cfg.QueryConcurrencyString(); got != "8" { + t.Errorf("QueryConcurrencyString() = %q, want 8", got) + } +} diff --git a/runner/internal/plan/doc.go b/runner/internal/plan/doc.go new file mode 100644 index 0000000..ee3b483 --- /dev/null +++ b/runner/internal/plan/doc.go @@ -0,0 +1,16 @@ +// Package plan turns a validated config into an ordered list of steps with +// explicit dependencies — a pure function, and the campaign as data. +// +// Build is the whole package: config in, [Plan] out, with everything +// environment-dependent (bench root, resolved commit, timestamp) entering +// through [Inputs]. It reads no clock, touches no filesystem, and executes +// nothing, so the same config and inputs always yield the same plan — which is +// what makes the plan committable as a golden fixture and lets dry-run, resume, +// and progress reporting be plain consumers of it. +// +// Step ordering and argv are ported from runner/campaign.sh's loop functions +// (build_binary, prepare_dataset, run_ingest_cold, run_ingest_hot, +// run_query_cold, run_query_hot, and the end-of-campaign tar and publish). The +// flags and their order are copied exactly: the bench subcommands are a +// cross-repo contract, and a stable argv keeps the golden plan meaningful. +package plan diff --git a/runner/internal/plan/plan.go b/runner/internal/plan/plan.go new file mode 100644 index 0000000..8f521e8 --- /dev/null +++ b/runner/internal/plan/plan.go @@ -0,0 +1,501 @@ +package plan + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +// SchemaVersion is the version of the plan.json contract. Additive changes +// (new fields, new step kinds) keep the version; a change that breaks a +// consumer bumps it. +const SchemaVersion = 1 + +// Step kinds. +const ( + KindBuild = "build" + KindDataset = "dataset" + KindLeg = "leg" + KindTarball = "tarball" + KindPublish = "publish" +) + +// queryTypes is the query type list every bench-query invocation sweeps. +const queryTypes = "--types=ledgers,txpage,txhash,events" + +// Inputs carries everything about the environment that Build would otherwise +// have to discover for itself. Keeping them in a struct is what makes Build +// pure: the caller resolves the ref, reads the clock, and picks the storage +// root, so a test can pin all three. +type Inputs struct { + BenchRoot string // e.g. /mnt/nvme/bench + BuiltCommit string // full sha the ref resolved to, or the ref itself in placeholder mode + Sha8 string // 8-hex short sha ("deadbeef" placeholder when unresolvable) + Stamp string // UTC, e.g. 20260101T000000Z +} + +// Step is one unit of scheduling, skipping, and failure. Argv is a *list* of +// commands because a build or a dataset preparation runs several external +// commands that only make sense together — bash ran them inside one function, +// and a campaign that stopped between two of them would leave a half-built +// binary or a half-fetched pack tree. A timed leg always has exactly one +// command: the measurement is the process. +type Step struct { + ID string `json:"id"` + Kind string `json:"kind"` // build|dataset|leg|tarball|publish + Timed bool `json:"timed"` // true only for benchmark legs + Argv [][]string `json:"argv"` // external commands, in order + Env map[string]string `json:"env,omitempty"` + // OutDir is a leg's --out directory: where the bench subcommand writes + // driver.csv and invocation.json, and what the executor inspects to decide + // whether a resumed leg is already finished. + OutDir string `json:"out_dir,omitempty"` + // PreClean lists directories to rm -rf before the step runs, so every leg + // starts from a known-empty scratch or hot DB and every dataset + // materializes onto bare ground. It is the single source of truth for those + // wipes: the executor removes exactly this list, and the dry run prints it. + PreClean []string `json:"pre_clean,omitempty"` + // PostClean lists directories to rm -rf after the step succeeds. This is a + // deliberate change from bash, which only cleaned before the next rep and + // so left the final rep's cold scratch on disk for the rest of the + // campaign; freeing it immediately keeps peak disk to one cell's worth. + PostClean []string `json:"post_clean,omitempty"` + Needs []string `json:"needs,omitempty"` // step ids + // PublishURI is the destination root a publish step uploads to. It is not + // an argv: the publish subcommand owns the destination checks and the + // cloud tooling, so the plan records the intent and nothing more. + PublishURI string `json:"publish_uri,omitempty"` + Dataset *DatasetSpec `json:"dataset,omitempty"` // dataset steps only +} + +// DatasetSpec is what the executor needs to run the .partial dance around a +// dataset step's commands: materialize into .partial, then rename onto +// Root once whole, so an interrupted preparation is never mistaken for a +// finished one. The choreography is the executor's, not argv's. +type DatasetSpec struct { + Name string `json:"name"` + Kind string `json:"kind"` + Location string `json:"location,omitempty"` // absent for fixture + Ledgers *int `json:"ledgers,omitempty"` // fixture only + Root string `json:"root"` // the cold pack root this converges on + Stage string `json:"stage,omitempty"` // fixture staging pack dir +} + +// Plan is a whole campaign as data: the paths it derives and the steps it +// would execute, in order. +type Plan struct { + SchemaVersion int `json:"schema_version"` + RunID string `json:"run_id"` // -- + BenchRoot string `json:"bench_root"` + Src string `json:"src"` // /src + Bin string `json:"bin"` // /bin/stellar-rpc- + ResultsDir string `json:"results_dir"` // /results/ + Tarball string `json:"tarball"` // /tmp/bench-results-.tgz + Notes []string `json:"notes,omitempty"` + Steps []Step `json:"steps"` +} + +// Build turns a validated config into the ordered steps a campaign executes. +// It is pure: same cfg, same Inputs, same plan. +func Build(cfg *config.Config, in Inputs) *Plan { + runID := cfg.Name + "-" + in.Sha8 + "-" + in.Stamp + p := &Plan{ + SchemaVersion: SchemaVersion, + RunID: runID, + BenchRoot: in.BenchRoot, + Src: filepath.Join(in.BenchRoot, "src"), + Bin: filepath.Join(in.BenchRoot, "bin", "stellar-rpc-"+in.Sha8), + ResultsDir: filepath.Join(in.BenchRoot, "results", runID), + Tarball: "/tmp/bench-results-" + runID + ".tgz", + } + + // Query-hot needs the hot DB a hot ingest leaves behind, so it only runs + // when this campaign also ingests hot. + ingestCold := cfg.Ingest == "cold" || cfg.Ingest == "both" + ingestHot := cfg.Ingest == "hot" || cfg.Ingest == "both" + queryCold := cfg.Query + queryHot := cfg.Query && ingestHot + if cfg.Query && !ingestHot { + p.Notes = append(p.Notes, fmt.Sprintf( + "query = true with ingest = %s leaves no hot DB — running the cold query suite only", cfg.Ingest)) + } + if cfg.Ingest == "none" && !cfg.Query { + p.Notes = append(p.Notes, "ingest = none and query = false — this campaign only prepares datasets") + } + + p.Steps = append(p.Steps, buildStep(p, in)) + for i := range cfg.Datasets { + p.Steps = append(p.Steps, datasetStep(p, in, &cfg.Datasets[i])) + } + + // The four suites run in bash's order: all of one before any of the next, + // so a campaign killed partway through still has whole suites. + if ingestCold { + forEachCell(cfg, func(d *config.Dataset, chunk, rep int) { + p.Steps = append(p.Steps, ingestColdLeg(p, in, cfg, d, chunk, rep)) + }) + } + if ingestHot { + forEachCell(cfg, func(d *config.Dataset, chunk, rep int) { + p.Steps = append(p.Steps, ingestHotLeg(p, in, cfg, d, chunk, rep)) + }) + } + if queryCold { + forEachCell(cfg, func(d *config.Dataset, chunk, rep int) { + p.Steps = append(p.Steps, queryColdLeg(p, in, cfg, d, chunk, rep)) + }) + } + if queryHot { + forEachCell(cfg, func(d *config.Dataset, chunk, rep int) { + p.Steps = append(p.Steps, queryHotLeg(p, in, cfg, d, chunk, rep)) + }) + } + + // The tarball needs nothing: it runs even after failed legs, because a + // campaign that went wrong is precisely the one whose bundle must be + // preserved for diagnosis. + p.Steps = append(p.Steps, Step{ + ID: "tarball", + Kind: KindTarball, + Argv: [][]string{{"tar", "-C", filepath.Join(in.BenchRoot, "results"), "-czf", p.Tarball, runID}}, + }) + if cfg.PublishURI != "" { + p.Steps = append(p.Steps, Step{ + ID: "publish", + Kind: KindPublish, + // A publish step runs no external command, but argv is a required + // plan.json field: an empty list, never null. + Argv: [][]string{}, + Needs: []string{"tarball"}, + PublishURI: cfg.PublishURI, + }) + } + return p +} + +// forEachCell visits every (dataset, chunk, rep) cell in config order — the +// loop nesting every bash suite function shares. +func forEachCell(cfg *config.Config, visit func(d *config.Dataset, chunk, rep int)) { + for i := range cfg.Datasets { + d := &cfg.Datasets[i] + for _, chunk := range d.Chunks { + for rep := 1; rep <= cfg.Runs; rep++ { + visit(d, chunk, rep) + } + } + } +} + +func buildStep(p *Plan, in Inputs) Step { + return Step{ + ID: "build", + Kind: KindBuild, + Argv: [][]string{ + {"git", "-C", p.Src, "-c", "advice.detachedHead=false", "checkout", "-q", "--detach", in.BuiltCommit}, + {"make", "-C", p.Src, "build-libs"}, + // build-rpc-v2 goes through the Makefile so the binary carries the + // repo's GOLDFLAGS (version, commit, branch, build timestamp) that + // `stellar-rpc-v2 version` and invocation.json report. The target + // writes ./stellar-rpc-v2 in the clone root; move it into the + // versioned path the campaign runs. + {"make", "-C", p.Src, "build-rpc-v2"}, + {"mv", filepath.Join(p.Src, "stellar-rpc-v2"), p.Bin}, + }, + } +} + +// datasetStep converges one dataset on a local cold pack root, whatever kind it +// is. Only the kinds that invoke the binary under test depend on the build. +func datasetStep(p *Plan, in Inputs, d *config.Dataset) Step { + root := datasetRoot(in, d) + step := Step{ + ID: "dataset-" + d.Name, + Kind: KindDataset, + // Starts empty rather than nil so a packs-local dataset, which appends + // nothing below, still marshals argv as [] — the field is required. + Argv: [][]string{}, + Dataset: &DatasetSpec{ + Name: d.Name, + Kind: d.Kind, + Root: root, + }, + } + switch d.Kind { + case config.KindPacksLocal: + // Nothing to materialize: the operator already has the pack root, and + // the executor only validates it. + step.Dataset.Location = d.Location + case config.KindPacksGS: + step.Dataset.Location = d.Location + // Clear the empty leftover of an earlier cleared-out fetch, or the + // rename below would nest the partial inside it. The .partial itself is + // deliberately not listed: rsync resumes into a half-fetched tree. + step.PreClean = []string{root} + step.Argv = [][]string{{"gcloud", "storage", "rsync", "-r", d.Location, root + ".partial"}} + case config.KindBSBS3: + step.Dataset.Location = d.Location + step.Needs = []string{"build"} + // Unlike a fetch, a cold backfill cannot resume: restarting on top of a + // half-written pack tree would double-write it. + step.PreClean = []string{root, root + ".partial"} + // AWS_EC2_METADATA_DISABLED is set on these commands only: without it + // the SDK signs requests with the machine's IAM role and the public + // bucket 403s, but setting it for the whole campaign would also hide + // those same instance-role credentials from the publish step's `aws + // s3` calls. + step.Env = map[string]string{"AWS_EC2_METADATA_DISABLED": "true"} + for _, chunk := range d.Chunks { + step.Argv = append(step.Argv, []string{ + p.Bin, "bench-ingest", "cold", + "--source=bsb", "--datastore-type=S3", "--region=us-east-2", + "--bucket-path=" + d.Location, + "--start-chunk=" + strconv.Itoa(chunk), "--num-chunks=1", + "--cold-out-dir=" + root + ".partial", + "--out=" + filepath.Join(p.ResultsDir, fmt.Sprintf("golden-%s-c%d", d.Name, chunk)), + }) + } + case config.KindFixture: + stage := filepath.Join(in.BenchRoot, "fixture", d.Name, "ledgers") + step.Needs = []string{"build"} + step.Dataset.Stage = stage + // The staging tree goes too, and by its parent: generation writes the + // ledgers/ dir itself, so a stale one from a killed run must not survive. + step.PreClean = []string{filepath.Dir(stage), root, root + ".partial"} + if d.Ledgers != nil { + ledgers := *d.Ledgers + step.Dataset.Ledgers = &ledgers + } + for _, chunk := range d.Chunks { + step.Argv = append(step.Argv, []string{ + p.Bin, "bench-ingest", "fixture", + "--pack-dir=" + stage, "--chunk=" + strconv.Itoa(chunk), + "--num-ledgers=" + strconv.Itoa(derefLedgers(d)), "--seed=1", + }) + } + // Freezing every chunk after generating every chunk, rather than + // interleaving, is bash's order: the freeze reads the whole staged + // pack tree. + for _, chunk := range d.Chunks { + step.Argv = append(step.Argv, []string{ + p.Bin, "bench-ingest", "cold", + "--source=pack", "--pack-dir=" + stage, + "--start-chunk=" + strconv.Itoa(chunk), "--num-chunks=1", + "--cold-out-dir=" + root + ".partial", + "--out=" + filepath.Join(p.ResultsDir, fmt.Sprintf("golden-%s-c%d", d.Name, chunk)), + }) + } + } + return step +} + +func ingestColdLeg(p *Plan, in Inputs, cfg *config.Config, d *config.Dataset, chunk, rep int) Step { + id := fmt.Sprintf("ingest-cold-%s-c%d-run%d", d.Name, chunk, rep) + out := filepath.Join(p.ResultsDir, id) + scratch := filepath.Join(in.BenchRoot, "scratch", d.Name, strconv.Itoa(chunk)) + return Step{ + ID: id, + Kind: KindLeg, + Timed: true, + Argv: [][]string{{ + p.Bin, "bench-ingest", "cold", + "--source=pack", "--pack-dir=" + filepath.Join(datasetRoot(in, d), "ledgers"), + "--start-chunk=" + strconv.Itoa(chunk), "--num-chunks=1", + "--workers=" + strconv.Itoa(cfg.Workers), + "--cold-out-dir=" + scratch, + "--out=" + out, + }}, + OutDir: out, + // Nothing reads the cold scratch DB afterwards, so it goes away as + // soon as the rep is done instead of squatting until the next rep. + PreClean: []string{scratch}, + PostClean: []string{scratch}, + Needs: []string{"build", "dataset-" + d.Name}, + } +} + +func ingestHotLeg(p *Plan, in Inputs, cfg *config.Config, d *config.Dataset, chunk, rep int) Step { + id := fmt.Sprintf("ingest-hot-%s-c%d-run%d", d.Name, chunk, rep) + out := filepath.Join(p.ResultsDir, id) + hot := hotDir(in, d, chunk) + argv := []string{ + p.Bin, "bench-ingest", "hot", + "--source=pack", "--pack-dir=" + filepath.Join(datasetRoot(in, d), "ledgers"), + "--start-chunk=" + strconv.Itoa(chunk), "--hot-dir=" + hot, + "--close-interval=" + cfg.CloseInterval, + } + if cfg.HotNumLedgers > 0 { + argv = append(argv, "--num-ledgers="+strconv.Itoa(cfg.HotNumLedgers)) + } + argv = append(argv, "--out="+out) + return Step{ + ID: id, + Kind: KindLeg, + Timed: true, + Argv: [][]string{argv}, + OutDir: out, + // The hot DB is wiped before each rep but deliberately kept after the + // last one: the hot query suite reads what that rep left behind. Every + // rep of a cell ingests the same chunk, so the DB the last one leaves + // is the whole DB. + PreClean: []string{hot}, + Needs: []string{"build", "dataset-" + d.Name}, + } +} + +func queryColdLeg(p *Plan, in Inputs, cfg *config.Config, d *config.Dataset, chunk, rep int) Step { + id := fmt.Sprintf("query-cold-%s-c%d-run%d", d.Name, chunk, rep) + out := filepath.Join(p.ResultsDir, id) + return Step{ + ID: id, + Kind: KindLeg, + Timed: true, + Argv: [][]string{{ + p.Bin, "bench-query", "cold", + "--cold-dir=" + datasetRoot(in, d), + "--start-chunk=" + strconv.Itoa(chunk), "--num-chunks=1", + queryTypes, + "--query-concurrency=" + cfg.QueryConcurrencyString(), + "--iters=" + strconv.Itoa(cfg.ColdIters), + "--out=" + out, + }}, + OutDir: out, + Needs: []string{"build", "dataset-" + d.Name}, + } +} + +func queryHotLeg(p *Plan, in Inputs, cfg *config.Config, d *config.Dataset, chunk, rep int) Step { + id := fmt.Sprintf("query-hot-%s-c%d-run%d", d.Name, chunk, rep) + out := filepath.Join(p.ResultsDir, id) + hot := hotDir(in, d, chunk) + argv := []string{ + p.Bin, "bench-query", "hot", + "--hot-dir=" + hot, "--chunk=" + strconv.Itoa(chunk), + queryTypes, + "--query-concurrency=" + cfg.QueryConcurrencyString(), + "--iters=" + strconv.Itoa(cfg.HotIters), "--warmup=20", + } + // A capped hot ingest leaves a truncated DB; keep the query sampler inside + // what was ingested. + if cfg.HotNumLedgers > 0 { + argv = append(argv, "--sample-ledgers="+strconv.Itoa(cfg.HotNumLedgers)) + } + argv = append(argv, "--out="+out) + return Step{ + ID: id, + Kind: KindLeg, + Timed: true, + Argv: [][]string{argv}, + OutDir: out, + // The dependency is on the *last* hot-ingest rep of this cell, not on + // this leg's own rep number: each rep wipes and rewrites the hot DB, + // so only the final rep's success guarantees a whole DB to query. + Needs: []string{ + "build", + "dataset-" + d.Name, + fmt.Sprintf("ingest-hot-%s-c%d-run%d", d.Name, chunk, cfg.Runs), + }, + } +} + +// datasetRoot is the local cold pack root a dataset converges on: its own +// location when the operator already has the packs, a campaign-owned golden +// directory when the runner has to materialize them. +func datasetRoot(in Inputs, d *config.Dataset) string { + if d.Kind == config.KindPacksLocal { + return d.Location + } + return filepath.Join(in.BenchRoot, "golden", d.Name) +} + +func hotDir(in Inputs, d *config.Dataset, chunk int) string { + return filepath.Join(in.BenchRoot, "hot", d.Name, strconv.Itoa(chunk)) +} + +// derefLedgers is the fixture ledger count; config validation guarantees +// fixture datasets have one, and 0 means the whole chunk. +func derefLedgers(d *config.Dataset) int { + if d.Ledgers == nil { + return 0 + } + return *d.Ledgers +} + +// WriteFile writes the plan as indented JSON with a trailing newline. +func (p *Plan) WriteFile(path string) error { + b, err := json.MarshalIndent(p, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(b, '\n'), 0o644) +} + +// Print writes the plan the way bash's --dry-run did: a header per step and one +// `$ command` line per command, with no quoting or escaping — these lines are +// for reading, not for pasting into a shell. +func (p *Plan) Print(w io.Writer) { + for _, note := range p.Notes { + fmt.Fprintf(w, "== note: %s\n", note) + } + for _, s := range p.Steps { + fmt.Fprintf(w, "== %s\n", s.ID) + if len(s.PreClean) > 0 { + // One line for the whole list, as the executor's single rm -rf. + fmt.Fprintf(w, " $ rm -rf %s\n", strings.Join(s.PreClean, " ")) + } + // A dataset step's .partial dance belongs to the executor, not to argv, + // so the argv lines alone would hide the destructive half of what a + // dataset preparation does. These lines are derived from the same + // DatasetSpec the executor keys off, so a dry run and campaign.log read + // identically. + if s.Dataset != nil && s.Dataset.Kind == config.KindPacksGS { + fmt.Fprintf(w, " $ mkdir -p %s.partial\n", s.Dataset.Root) + } + prefix := envPrefix(s.Env) + for _, argv := range s.Argv { + fmt.Fprintf(w, " $ %s%s\n", prefix, strings.Join(argv, " ")) + } + if s.Dataset != nil && materializes(s.Dataset.Kind) { + fmt.Fprintf(w, " $ mv %s.partial %s\n", s.Dataset.Root, s.Dataset.Root) + } + if s.Kind == KindPublish { + fmt.Fprintf(w, " $ campaign publish %s %s\n", p.ResultsDir, s.PublishURI) + } + } +} + +// materializes reports whether a dataset kind builds its pack root under +// .partial and renames it into place once whole. packs-local is the one +// kind that does not: the operator already has the packs. +func materializes(kind string) bool { + switch kind { + case config.KindPacksGS, config.KindBSBS3, config.KindFixture: + return true + } + return false +} + +// envPrefix renders a step's extra environment as bash printed it: an `env +// K=V ` prefix on the command line. Keys are sorted so the output is stable. +func envPrefix(env map[string]string) string { + if len(env) == 0 { + return "" + } + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + b.WriteString("env ") + for _, k := range keys { + fmt.Fprintf(&b, "%s=%s ", k, env[k]) + } + return b.String() +} diff --git a/runner/internal/plan/plan_test.go b/runner/internal/plan/plan_test.go new file mode 100644 index 0000000..dd4fedc --- /dev/null +++ b/runner/internal/plan/plan_test.go @@ -0,0 +1,548 @@ +package plan + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +var update = flag.Bool("update", false, "rewrite testdata/plan.golden.json from the current Build output") + +const goldenPath = "testdata/plan.golden.json" + +// goldenInputs pins everything environment-dependent, so the plan below is a +// function of the config alone. +func goldenInputs() Inputs { + return Inputs{ + BenchRoot: "/bench", + BuiltCommit: "deadbeefcafebabefeedface1234567890abcdef", + Sha8: "deadbeef", + Stamp: "20260101T000000Z", + } +} + +// load writes src to a temp file and loads it as a campaign config. +func load(t *testing.T, src string) *config.Config { + t.Helper() + path := filepath.Join(t.TempDir(), "campaign.toml") + if err := os.WriteFile(path, []byte(src), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + cfg, err := config.Load(path) + if err != nil { + t.Fatalf("config.Load: %v", err) + } + return cfg +} + +// buildGolden builds the plan the committed testdata config describes. +func buildGolden(t *testing.T) *Plan { + t.Helper() + cfg, err := config.Load("testdata/campaign.toml") + if err != nil { + t.Fatalf("config.Load(testdata/campaign.toml): %v", err) + } + return Build(cfg, goldenInputs()) +} + +// stepByID finds one step, failing the test when the id is absent. +func stepByID(t *testing.T, p *Plan, id string) Step { + t.Helper() + for _, s := range p.Steps { + if s.ID == id { + return s + } + } + t.Fatalf("plan has no step %q", id) + return Step{} +} + +// assertEmptyArgvJSON checks that a commandless step still serializes argv as +// an empty list: argv is a required plan.json field, and null is not a list. +func assertEmptyArgvJSON(t *testing.T, s Step) { + t.Helper() + b, err := json.Marshal(s) + if err != nil { + t.Fatalf("marshal %s: %v", s.ID, err) + } + if !strings.Contains(string(b), `"argv":[]`) { + t.Errorf("%s marshals as %s, want it to contain `\"argv\":[]`", s.ID, b) + } +} + +func stepIDs(p *Plan) []string { + ids := make([]string, len(p.Steps)) + for i, s := range p.Steps { + ids[i] = s.ID + } + return ids +} + +func TestGolden(t *testing.T) { + p := buildGolden(t) + got := filepath.Join(t.TempDir(), "plan.json") + if err := p.WriteFile(got); err != nil { + t.Fatalf("WriteFile: %v", err) + } + gotBytes, err := os.ReadFile(got) + if err != nil { + t.Fatalf("read written plan: %v", err) + } + if *update { + if err := os.WriteFile(goldenPath, gotBytes, 0o644); err != nil { + t.Fatalf("update golden: %v", err) + } + t.Logf("wrote %s", goldenPath) + return + } + wantBytes, err := os.ReadFile(goldenPath) + if err != nil { + t.Fatalf("read golden: %v", err) + } + if !bytes.Equal(gotBytes, wantBytes) { + t.Errorf("plan.json differs from %s — rerun with -update to accept:\n%s", goldenPath, diffLines(wantBytes, gotBytes)) + } +} + +// diffLines reports the first differing line, which is all a golden mismatch +// usually needs. +func diffLines(want, got []byte) string { + w := strings.Split(string(want), "\n") + g := strings.Split(string(got), "\n") + for i := 0; i < len(w) && i < len(g); i++ { + if w[i] != g[i] { + return fmt.Sprintf("line %d:\n want: %s\n got: %s", i+1, w[i], g[i]) + } + } + return fmt.Sprintf("want %d lines, got %d", len(w), len(g)) +} + +func TestQueryHotNeedsLastIngestHotRep(t *testing.T) { + p := buildGolden(t) + seen := 0 + for _, s := range p.Steps { + if !strings.HasPrefix(s.ID, "query-hot-") { + continue + } + seen++ + // runs = 2, so every rep — not just rep 2 — waits on rep 2. + cell, _, _ := strings.Cut(strings.TrimPrefix(s.ID, "query-hot-"), "-run") + want := "ingest-hot-" + cell + "-run2" + if !slices.Contains(s.Needs, want) { + t.Errorf("%s needs %v, want it to include %q", s.ID, s.Needs, want) + } + if !slices.Contains(s.Needs, "build") { + t.Errorf("%s needs %v, want it to include \"build\"", s.ID, s.Needs) + } + } + if seen != 8 { // 2 datasets x 2 chunks x 2 reps + t.Errorf("found %d query-hot steps, want 8", seen) + } +} + +func TestSuiteOrdering(t *testing.T) { + p := buildGolden(t) + // Collapse each step id to its suite, then to runs of consecutive suites: + // an interleaved plan would repeat a label. + suites := []string{"dataset", "ingest-cold", "ingest-hot", "query-cold", "query-hot"} + var blocks []string + for _, id := range stepIDs(p) { + label := id + for _, s := range suites { + if strings.HasPrefix(id, s+"-") { + label = s + break + } + } + if len(blocks) == 0 || blocks[len(blocks)-1] != label { + blocks = append(blocks, label) + } + } + want := []string{"build", "dataset", "ingest-cold", "ingest-hot", "query-cold", "query-hot", "tarball", "publish"} + if !slices.Equal(blocks, want) { + t.Errorf("step blocks = %v, want %v", blocks, want) + } +} + +const hotConfig = ` +name = "hot" +ingest = "hot" +query = true +runs = 1 +hot_num_ledgers = %d + +[[dataset]] +name = "ds" +kind = "packs-local" +location = "/packs/ds" +chunks = [7] +` + +func TestHotLedgerCaps(t *testing.T) { + capped := Build(load(t, fmt.Sprintf(hotConfig, 50000)), goldenInputs()) + uncapped := Build(load(t, fmt.Sprintf(hotConfig, 0)), goldenInputs()) + + cases := []struct { + plan *Plan + id string + flag string + want bool + }{ + {capped, "ingest-hot-ds-c7-run1", "--num-ledgers=50000", true}, + {uncapped, "ingest-hot-ds-c7-run1", "--num-ledgers=", false}, + {capped, "query-hot-ds-c7-run1", "--sample-ledgers=50000", true}, + {uncapped, "query-hot-ds-c7-run1", "--sample-ledgers=", false}, + } + for _, tc := range cases { + argv := stepByID(t, tc.plan, tc.id).Argv[0] + got := slices.ContainsFunc(argv, func(a string) bool { return strings.HasPrefix(a, tc.flag) }) + if got != tc.want { + t.Errorf("%s argv %v: has %q = %v, want %v", tc.id, argv, tc.flag, got, tc.want) + } + } + + // The --out flag stays last, after the optional cap. + for _, p := range []*Plan{capped, uncapped} { + argv := stepByID(t, p, "ingest-hot-ds-c7-run1").Argv[0] + if last := argv[len(argv)-1]; !strings.HasPrefix(last, "--out=") { + t.Errorf("last ingest-hot arg = %q, want an --out flag", last) + } + } +} + +func TestPacksLocalRootIsTheConfiguredLocation(t *testing.T) { + p := Build(load(t, fmt.Sprintf(hotConfig, 0)), goldenInputs()) + ds := stepByID(t, p, "dataset-ds") + if len(ds.Argv) != 0 { + t.Errorf("packs-local dataset argv = %v, want no commands", ds.Argv) + } + if len(ds.Needs) != 0 { + t.Errorf("packs-local dataset needs = %v, want none — it does not invoke the binary", ds.Needs) + } + if ds.Dataset.Root != "/packs/ds" { + t.Errorf("packs-local root = %q, want the configured location", ds.Dataset.Root) + } + assertEmptyArgvJSON(t, ds) +} + +func TestColdIngestCleansScratchBothSides(t *testing.T) { + p := Build(load(t, ` +name = "c" +ingest = "cold" +query = false +runs = 1 + +[[dataset]] +name = "ds" +kind = "packs-local" +location = "/packs/ds" +chunks = [3] +`), goldenInputs()) + leg := stepByID(t, p, "ingest-cold-ds-c3-run1") + want := []string{"/bench/scratch/ds/3"} + if !slices.Equal(leg.PreClean, want) { + t.Errorf("pre_clean = %v, want %v", leg.PreClean, want) + } + if !slices.Equal(leg.PostClean, want) { + t.Errorf("post_clean = %v, want %v", leg.PostClean, want) + } + hot := Build(load(t, fmt.Sprintf(hotConfig, 0)), goldenInputs()) + if pc := stepByID(t, hot, "ingest-hot-ds-c7-run1").PostClean; pc != nil { + t.Errorf("ingest-hot post_clean = %v, want none — the query suite reads that DB", pc) + } +} + +func TestNotes(t *testing.T) { + cases := []struct { + name string + ingest string + query bool + want []string + }{ + { + name: "query with cold ingest gets the cold-suite-only note", + ingest: "cold", + query: true, + want: []string{"query = true with ingest = cold leaves no hot DB — running the cold query suite only"}, + }, + { + name: "query with no ingest gets the same note", + ingest: "none", + query: true, + want: []string{"query = true with ingest = none leaves no hot DB — running the cold query suite only"}, + }, + { + name: "datasets-only campaign says so", + ingest: "none", + query: false, + want: []string{"ingest = none and query = false — this campaign only prepares datasets"}, + }, + { + name: "a full campaign has nothing to warn about", + ingest: "both", + query: true, + want: nil, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + src := "name = \"n\"\ningest = \"" + tc.ingest + "\"\nquery = " + boolString(tc.query) + ` +runs = 1 + +[[dataset]] +name = "ds" +kind = "packs-local" +location = "/packs/ds" +chunks = [1] +` + p := Build(load(t, src), goldenInputs()) + if !slices.Equal(p.Notes, tc.want) { + t.Errorf("notes = %q, want %q", p.Notes, tc.want) + } + }) + } +} + +func boolString(b bool) string { + if b { + return "true" + } + return "false" +} + +func TestQuerySuitesFollowIngest(t *testing.T) { + p := Build(load(t, ` +name = "n" +ingest = "cold" +query = true +runs = 1 + +[[dataset]] +name = "ds" +kind = "packs-local" +location = "/packs/ds" +chunks = [1] +`), goldenInputs()) + for _, id := range stepIDs(p) { + if strings.HasPrefix(id, "query-hot-") { + t.Errorf("plan has %s, but ingest = cold leaves no hot DB", id) + } + } + stepByID(t, p, "query-cold-ds-c1-run1") +} + +func TestBSBS3Step(t *testing.T) { + p := Build(load(t, ` +name = "n" +ingest = "none" +query = false +runs = 1 + +[[dataset]] +name = "mainnet" +kind = "bsb-s3" +location = "s3://bucket/prefix" +chunks = [4] +`), goldenInputs()) + ds := stepByID(t, p, "dataset-mainnet") + if got := ds.Env["AWS_EC2_METADATA_DISABLED"]; got != "true" { + t.Errorf("env = %v, want AWS_EC2_METADATA_DISABLED=true", ds.Env) + } + if !slices.Equal(ds.Needs, []string{"build"}) { + t.Errorf("needs = %v, want [build] — the backfill runs the binary under test", ds.Needs) + } + if len(ds.Argv) != 1 { + t.Fatalf("argv = %v, want one command per chunk", ds.Argv) + } + if !slices.Contains(ds.Argv[0], "--cold-out-dir=/bench/golden/mainnet.partial") { + t.Errorf("argv = %v, want it to materialize into the .partial root", ds.Argv[0]) + } +} + +func TestFixtureStepGeneratesThenFreezes(t *testing.T) { + p := buildGolden(t) + ds := stepByID(t, p, "dataset-fix") + if len(ds.Argv) != 4 { // 2 chunks generated, then 2 frozen + t.Fatalf("argv has %d commands, want 4", len(ds.Argv)) + } + for i, want := range []string{"fixture", "fixture", "cold", "cold"} { + if got := ds.Argv[i][2]; got != want { + t.Errorf("argv[%d] subcommand = %q, want %q", i, got, want) + } + } + if ds.Dataset.Stage != "/bench/fixture/fix/ledgers" { + t.Errorf("stage = %q, want the fixture staging pack dir", ds.Dataset.Stage) + } + if ds.Dataset.Location != "" { + t.Errorf("location = %q, want it absent for a fixture", ds.Dataset.Location) + } + if ds.Dataset.Ledgers == nil || *ds.Dataset.Ledgers != 10000 { + t.Errorf("ledgers = %v, want 10000", ds.Dataset.Ledgers) + } +} + +func TestDatasetPreClean(t *testing.T) { + p := buildGolden(t) + // packs-gs clears the leftover root so the rename cannot nest the partial + // inside it, but keeps the partial itself: rsync resumes into it. + if got, want := stepByID(t, p, "dataset-packs").PreClean, []string{"/bench/golden/packs"}; !slices.Equal(got, want) { + t.Errorf("packs-gs pre_clean = %v, want %v — the .partial is resumable", got, want) + } + // A fixture wipes the staging tree by its parent, plus both roots. + want := []string{"/bench/fixture/fix", "/bench/golden/fix", "/bench/golden/fix.partial"} + if got := stepByID(t, p, "dataset-fix").PreClean; !slices.Equal(got, want) { + t.Errorf("fixture pre_clean = %v, want %v", got, want) + } + + bsb := Build(load(t, ` +name = "n" +ingest = "none" +query = false +runs = 1 + +[[dataset]] +name = "mainnet" +kind = "bsb-s3" +location = "s3://bucket/prefix" +chunks = [4] +`), goldenInputs()) + want = []string{"/bench/golden/mainnet", "/bench/golden/mainnet.partial"} + if got := stepByID(t, bsb, "dataset-mainnet").PreClean; !slices.Equal(got, want) { + t.Errorf("bsb-s3 pre_clean = %v, want %v — a cold backfill cannot resume", got, want) + } + + local := Build(load(t, fmt.Sprintf(hotConfig, 0)), goldenInputs()) + if got := stepByID(t, local, "dataset-ds").PreClean; got != nil { + t.Errorf("packs-local pre_clean = %v, want none — those packs are the operator's", got) + } +} + +func TestTarballRunsUnconditionally(t *testing.T) { + p := buildGolden(t) + tarball := stepByID(t, p, "tarball") + if len(tarball.Needs) != 0 { + t.Errorf("tarball needs = %v, want none — the bundle matters most when legs failed", tarball.Needs) + } + want := []string{"tar", "-C", "/bench/results", "-czf", p.Tarball, p.RunID} + if !slices.Equal(tarball.Argv[0], want) { + t.Errorf("tarball argv = %v, want %v", tarball.Argv[0], want) + } +} + +func TestPublishStepRecordsIntentOnly(t *testing.T) { + p := buildGolden(t) + pub := stepByID(t, p, "publish") + if len(pub.Argv) != 0 { + t.Errorf("publish argv = %v, want none — the publish subcommand owns the tooling", pub.Argv) + } + if !slices.Equal(pub.Needs, []string{"tarball"}) { + t.Errorf("publish needs = %v, want [tarball]", pub.Needs) + } + if pub.PublishURI != "gs://bucket/results" { + t.Errorf("publish_uri = %q, want the configured destination", pub.PublishURI) + } + assertEmptyArgvJSON(t, pub) + + noPublish := Build(load(t, ` +name = "n" +ingest = "none" +query = false +runs = 1 + +[[dataset]] +name = "ds" +kind = "packs-local" +location = "/packs/ds" +chunks = [1] +`), goldenInputs()) + for _, id := range stepIDs(noPublish) { + if id == "publish" { + t.Error("plan has a publish step, but publish_uri is unset") + } + } +} + +func TestBuildIsPure(t *testing.T) { + cfg, err := config.Load("testdata/campaign.toml") + if err != nil { + t.Fatalf("config.Load: %v", err) + } + first := Build(cfg, goldenInputs()) + second := Build(cfg, goldenInputs()) + if !reflect.DeepEqual(first, second) { + t.Error("two Build calls with identical inputs produced different plans") + } +} + +func TestPrint(t *testing.T) { + p := Build(load(t, ` +name = "n" +ingest = "cold" +query = true +runs = 1 +publish_uri = "gs://bucket/results" + +[[dataset]] +name = "mainnet" +kind = "bsb-s3" +location = "s3://bucket/prefix" +chunks = [4] +`), goldenInputs()) + var out bytes.Buffer + p.Print(&out) + got := out.String() + for _, want := range []string{ + "== note: query = true with ingest = cold leaves no hot DB", + "== build\n $ git -C /bench/src -c advice.detachedHead=false checkout -q --detach deadbeefcafebabefeedface1234567890abcdef\n", + " $ env AWS_EC2_METADATA_DISABLED=true /bench/bin/stellar-rpc-deadbeef bench-ingest cold", + "== ingest-cold-mainnet-c4-run1\n $ rm -rf /bench/scratch/mainnet/4\n", + // The whole pre_clean list on one line, as the executor wipes it. + "== dataset-mainnet\n $ rm -rf /bench/golden/mainnet /bench/golden/mainnet.partial\n", + " $ mv /bench/golden/mainnet.partial /bench/golden/mainnet\n", + " $ campaign publish /bench/results/n-deadbeef-20260101T000000Z gs://bucket/results\n", + } { + if !strings.Contains(got, want) { + t.Errorf("Print output missing %q, got:\n%s", want, got) + } + } + // The backfill writes its own --cold-out-dir; only a fetch needs the + // partial to exist first. + if strings.Contains(got, "mkdir -p") { + t.Errorf("Print output has a mkdir for a bsb-s3 dataset, got:\n%s", got) + } +} + +// TestPrintDatasetChoreography pins the destructive half of a dataset +// preparation in the dry run: the wipes, the partial, and the rename are the +// executor's, so nothing but this printer would show them. +func TestPrintDatasetChoreography(t *testing.T) { + var out bytes.Buffer + buildGolden(t).Print(&out) + got := out.String() + for _, want := range []string{ + "== dataset-packs\n" + + " $ rm -rf /bench/golden/packs\n" + + " $ mkdir -p /bench/golden/packs.partial\n" + + " $ gcloud storage rsync -r gs://bucket/cold /bench/golden/packs.partial\n" + + " $ mv /bench/golden/packs.partial /bench/golden/packs\n", + "== dataset-fix\n $ rm -rf /bench/fixture/fix /bench/golden/fix /bench/golden/fix.partial\n", + " $ mv /bench/golden/fix.partial /bench/golden/fix\n", + } { + if !strings.Contains(got, want) { + t.Errorf("Print output missing %q, got:\n%s", want, got) + } + } + // A fixture freezes straight into --cold-out-dir, so it gets no mkdir. + if strings.Contains(got, "mkdir -p /bench/golden/fix.partial") { + t.Errorf("Print output has a mkdir for the fixture partial, got:\n%s", got) + } +} diff --git a/runner/internal/plan/testdata/campaign.toml b/runner/internal/plan/testdata/campaign.toml new file mode 100644 index 0000000..cf23302 --- /dev/null +++ b/runner/internal/plan/testdata/campaign.toml @@ -0,0 +1,22 @@ +# Golden-test campaign: both ingest suites and both query suites, two reps per +# cell, over two chunks of a fetched pack tree and a generated fixture — enough +# shape to pin step ordering, dependencies, and argv without being unreadable. +name = "golden" +ingest = "both" +query = true +runs = 2 +close_interval = "2s" +hot_num_ledgers = 50000 +publish_uri = "gs://bucket/results" + +[[dataset]] +name = "packs" +kind = "packs-gs" +location = "gs://bucket/cold" +chunks = [1, 2] + +[[dataset]] +name = "fix" +kind = "fixture" +ledgers = 10000 +chunks = [1, 2] diff --git a/runner/internal/plan/testdata/plan.golden.json b/runner/internal/plan/testdata/plan.golden.json new file mode 100644 index 0000000..41e7090 --- /dev/null +++ b/runner/internal/plan/testdata/plan.golden.json @@ -0,0 +1,1013 @@ +{ + "schema_version": 1, + "run_id": "golden-deadbeef-20260101T000000Z", + "bench_root": "/bench", + "src": "/bench/src", + "bin": "/bench/bin/stellar-rpc-deadbeef", + "results_dir": "/bench/results/golden-deadbeef-20260101T000000Z", + "tarball": "/tmp/bench-results-golden-deadbeef-20260101T000000Z.tgz", + "steps": [ + { + "id": "build", + "kind": "build", + "timed": false, + "argv": [ + [ + "git", + "-C", + "/bench/src", + "-c", + "advice.detachedHead=false", + "checkout", + "-q", + "--detach", + "deadbeefcafebabefeedface1234567890abcdef" + ], + [ + "make", + "-C", + "/bench/src", + "build-libs" + ], + [ + "make", + "-C", + "/bench/src", + "build-rpc-v2" + ], + [ + "mv", + "/bench/src/stellar-rpc-v2", + "/bench/bin/stellar-rpc-deadbeef" + ] + ] + }, + { + "id": "dataset-packs", + "kind": "dataset", + "timed": false, + "argv": [ + [ + "gcloud", + "storage", + "rsync", + "-r", + "gs://bucket/cold", + "/bench/golden/packs.partial" + ] + ], + "pre_clean": [ + "/bench/golden/packs" + ], + "dataset": { + "name": "packs", + "kind": "packs-gs", + "location": "gs://bucket/cold", + "root": "/bench/golden/packs" + } + }, + { + "id": "dataset-fix", + "kind": "dataset", + "timed": false, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "fixture", + "--pack-dir=/bench/fixture/fix/ledgers", + "--chunk=1", + "--num-ledgers=10000", + "--seed=1" + ], + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "fixture", + "--pack-dir=/bench/fixture/fix/ledgers", + "--chunk=2", + "--num-ledgers=10000", + "--seed=1" + ], + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/fixture/fix/ledgers", + "--start-chunk=1", + "--num-chunks=1", + "--cold-out-dir=/bench/golden/fix.partial", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/golden-fix-c1" + ], + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/fixture/fix/ledgers", + "--start-chunk=2", + "--num-chunks=1", + "--cold-out-dir=/bench/golden/fix.partial", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/golden-fix-c2" + ] + ], + "pre_clean": [ + "/bench/fixture/fix", + "/bench/golden/fix", + "/bench/golden/fix.partial" + ], + "needs": [ + "build" + ], + "dataset": { + "name": "fix", + "kind": "fixture", + "ledgers": 10000, + "root": "/bench/golden/fix", + "stage": "/bench/fixture/fix/ledgers" + } + }, + { + "id": "ingest-cold-packs-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=1", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/packs/1", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c1-run1", + "pre_clean": [ + "/bench/scratch/packs/1" + ], + "post_clean": [ + "/bench/scratch/packs/1" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-cold-packs-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=1", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/packs/1", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c1-run2", + "pre_clean": [ + "/bench/scratch/packs/1" + ], + "post_clean": [ + "/bench/scratch/packs/1" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-cold-packs-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=2", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/packs/2", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c2-run1", + "pre_clean": [ + "/bench/scratch/packs/2" + ], + "post_clean": [ + "/bench/scratch/packs/2" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-cold-packs-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=2", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/packs/2", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-packs-c2-run2", + "pre_clean": [ + "/bench/scratch/packs/2" + ], + "post_clean": [ + "/bench/scratch/packs/2" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-cold-fix-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=1", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/fix/1", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c1-run1", + "pre_clean": [ + "/bench/scratch/fix/1" + ], + "post_clean": [ + "/bench/scratch/fix/1" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "ingest-cold-fix-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=1", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/fix/1", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c1-run2", + "pre_clean": [ + "/bench/scratch/fix/1" + ], + "post_clean": [ + "/bench/scratch/fix/1" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "ingest-cold-fix-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=2", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/fix/2", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c2-run1", + "pre_clean": [ + "/bench/scratch/fix/2" + ], + "post_clean": [ + "/bench/scratch/fix/2" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "ingest-cold-fix-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "cold", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=2", + "--num-chunks=1", + "--workers=1", + "--cold-out-dir=/bench/scratch/fix/2", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-cold-fix-c2-run2", + "pre_clean": [ + "/bench/scratch/fix/2" + ], + "post_clean": [ + "/bench/scratch/fix/2" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "ingest-hot-packs-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=1", + "--hot-dir=/bench/hot/packs/1", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c1-run1", + "pre_clean": [ + "/bench/hot/packs/1" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-hot-packs-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=1", + "--hot-dir=/bench/hot/packs/1", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c1-run2", + "pre_clean": [ + "/bench/hot/packs/1" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-hot-packs-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=2", + "--hot-dir=/bench/hot/packs/2", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c2-run1", + "pre_clean": [ + "/bench/hot/packs/2" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-hot-packs-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/packs/ledgers", + "--start-chunk=2", + "--hot-dir=/bench/hot/packs/2", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-packs-c2-run2", + "pre_clean": [ + "/bench/hot/packs/2" + ], + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "ingest-hot-fix-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=1", + "--hot-dir=/bench/hot/fix/1", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c1-run1", + "pre_clean": [ + "/bench/hot/fix/1" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "ingest-hot-fix-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=1", + "--hot-dir=/bench/hot/fix/1", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c1-run2", + "pre_clean": [ + "/bench/hot/fix/1" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "ingest-hot-fix-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=2", + "--hot-dir=/bench/hot/fix/2", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c2-run1", + "pre_clean": [ + "/bench/hot/fix/2" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "ingest-hot-fix-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-ingest", + "hot", + "--source=pack", + "--pack-dir=/bench/golden/fix/ledgers", + "--start-chunk=2", + "--hot-dir=/bench/hot/fix/2", + "--close-interval=2s", + "--num-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/ingest-hot-fix-c2-run2", + "pre_clean": [ + "/bench/hot/fix/2" + ], + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "query-cold-packs-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/packs", + "--start-chunk=1", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c1-run1", + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "query-cold-packs-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/packs", + "--start-chunk=1", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c1-run2", + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "query-cold-packs-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/packs", + "--start-chunk=2", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c2-run1", + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "query-cold-packs-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/packs", + "--start-chunk=2", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-packs-c2-run2", + "needs": [ + "build", + "dataset-packs" + ] + }, + { + "id": "query-cold-fix-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/fix", + "--start-chunk=1", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c1-run1", + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "query-cold-fix-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/fix", + "--start-chunk=1", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c1-run2", + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "query-cold-fix-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/fix", + "--start-chunk=2", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c2-run1", + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "query-cold-fix-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "cold", + "--cold-dir=/bench/golden/fix", + "--start-chunk=2", + "--num-chunks=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=100", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-cold-fix-c2-run2", + "needs": [ + "build", + "dataset-fix" + ] + }, + { + "id": "query-hot-packs-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/packs/1", + "--chunk=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c1-run1", + "needs": [ + "build", + "dataset-packs", + "ingest-hot-packs-c1-run2" + ] + }, + { + "id": "query-hot-packs-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/packs/1", + "--chunk=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c1-run2", + "needs": [ + "build", + "dataset-packs", + "ingest-hot-packs-c1-run2" + ] + }, + { + "id": "query-hot-packs-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/packs/2", + "--chunk=2", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c2-run1", + "needs": [ + "build", + "dataset-packs", + "ingest-hot-packs-c2-run2" + ] + }, + { + "id": "query-hot-packs-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/packs/2", + "--chunk=2", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-packs-c2-run2", + "needs": [ + "build", + "dataset-packs", + "ingest-hot-packs-c2-run2" + ] + }, + { + "id": "query-hot-fix-c1-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/fix/1", + "--chunk=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c1-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c1-run1", + "needs": [ + "build", + "dataset-fix", + "ingest-hot-fix-c1-run2" + ] + }, + { + "id": "query-hot-fix-c1-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/fix/1", + "--chunk=1", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c1-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c1-run2", + "needs": [ + "build", + "dataset-fix", + "ingest-hot-fix-c1-run2" + ] + }, + { + "id": "query-hot-fix-c2-run1", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/fix/2", + "--chunk=2", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c2-run1" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c2-run1", + "needs": [ + "build", + "dataset-fix", + "ingest-hot-fix-c2-run2" + ] + }, + { + "id": "query-hot-fix-c2-run2", + "kind": "leg", + "timed": true, + "argv": [ + [ + "/bench/bin/stellar-rpc-deadbeef", + "bench-query", + "hot", + "--hot-dir=/bench/hot/fix/2", + "--chunk=2", + "--types=ledgers,txpage,txhash,events", + "--query-concurrency=1,4,16", + "--iters=200", + "--warmup=20", + "--sample-ledgers=50000", + "--out=/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c2-run2" + ] + ], + "out_dir": "/bench/results/golden-deadbeef-20260101T000000Z/query-hot-fix-c2-run2", + "needs": [ + "build", + "dataset-fix", + "ingest-hot-fix-c2-run2" + ] + }, + { + "id": "tarball", + "kind": "tarball", + "timed": false, + "argv": [ + [ + "tar", + "-C", + "/bench/results", + "-czf", + "/tmp/bench-results-golden-deadbeef-20260101T000000Z.tgz", + "golden-deadbeef-20260101T000000Z" + ] + ] + }, + { + "id": "publish", + "kind": "publish", + "timed": false, + "argv": [], + "needs": [ + "tarball" + ], + "publish_uri": "gs://bucket/results" + } + ] +} diff --git a/runner/internal/preflight/preflight.go b/runner/internal/preflight/preflight.go new file mode 100644 index 0000000..d5a33f0 --- /dev/null +++ b/runner/internal/preflight/preflight.go @@ -0,0 +1,217 @@ +// Package preflight answers one question before a campaign starts: does this +// machine have what this config needs? A missing gcloud credential is cheap to +// fix and catastrophic to discover seventeen hours in, when the campaign +// finally reaches its publish step. +// +// Every check is derived from the config — a campaign that publishes nowhere +// is never asked about gcloud — and every environment probe goes through Deps, +// so a test can trigger exactly one failure at a time. +package preflight + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/publish" +) + +// The bench devbox's NVMe layout. campaign.sh only verified the mount when +// BENCH_ROOT was left at its default, on the reasoning that an operator who +// pointed BENCH_ROOT elsewhere knows what they mounted; this keeps that. +const ( + defaultBenchRoot = "/mnt/nvme/bench" + nvmeMount = "/mnt/nvme" +) + +const gib = 1 << 30 + +// minFree is the free-space line below which preflight warns. It is a warning, +// not a failure: how much a campaign really needs depends on its datasets and +// rep count, and a small fixture campaign runs happily under it. +const minFree = 100 * gib + +// Deps are the environment probes, injectable so tests can stub each check. +type Deps struct { + LookPath func(file string) (string, error) // default exec.LookPath + // ListRoot lists an object-storage root to prove the current credentials + // can reach it. An empty root is success; auth/network errors are not. + ListRoot func(uri string) error // default ListRoot + Mountpoint func(dir string) error // default: exec `mountpoint -q ` + DiskFree func(dir string) (uint64, error) // free bytes; default syscall.Statfs +} + +// Result is what preflight found. Failures are things the campaign will need +// and does not have; warnings are things an operator should see before +// committing the machine for a day. +type Result struct { + Failures []string // each names the missing thing AND the config key that needs it + Warnings []string +} + +func (r *Result) failf(format string, a ...any) { + r.Failures = append(r.Failures, fmt.Sprintf(format, a...)) +} +func (r *Result) warnf(format string, a ...any) { + r.Warnings = append(r.Warnings, fmt.Sprintf(format, a...)) +} + +// requireTool records a failure when tool is not on PATH, saying which part of +// this config wants it. +func (r *Result) requireTool(d Deps, tool, why string) { + if _, err := d.LookPath(tool); err != nil { + r.failf("%s not found in PATH — %s", tool, why) + } +} + +// Run performs every check cfg needs. benchRoot is the storage root the +// campaign would use; binPath is the versioned binary it would run, or "" when +// the ref is not resolvable yet (a standalone preflight on a fresh machine). +// Toolchain checks assume a build will happen unless binPath already exists +// and is executable. +func Run(cfg *config.Config, benchRoot, binPath string, d Deps) Result { + d = withDefaults(d) + var res Result + + // git and make drive every campaign: the build clone is a git checkout, + // and the binary comes out of the repo's Makefile. + res.requireTool(d, "git", fmt.Sprintf("the campaign clones and builds repo '%s'", cfg.Repo)) + res.requireTool(d, "make", fmt.Sprintf("the campaign builds repo '%s'", cfg.Repo)) + + if willBuild(binPath) { + res.requireTool(d, "go", fmt.Sprintf("building ref '%s' needs the Go toolchain", cfg.Ref)) + // rustup installs outside PATH on the devbox, so a cargo that + // LookPath cannot see may still be the one the build uses — the same + // fallback campaign.sh does when recording the rustc version. + if _, err := d.LookPath("cargo"); err != nil && !isExecutable(cargoBin()) { + res.failf("cargo not found in PATH or %s — building ref '%s' needs the Rust toolchain", cargoBin(), cfg.Ref) + } + } + + var wantsGcloud []string + for _, ds := range cfg.Datasets { + if ds.Kind == config.KindPacksGS { + wantsGcloud = append(wantsGcloud, fmt.Sprintf("dataset '%s' fetches its packs from '%s'", ds.Name, ds.Location)) + } + } + if strings.HasPrefix(cfg.PublishURI, "gs://") { + wantsGcloud = append(wantsGcloud, fmt.Sprintf("publish_uri '%s'", cfg.PublishURI)) + } + if len(wantsGcloud) > 0 { + res.requireTool(d, "gcloud", "needed by "+strings.Join(wantsGcloud, ", ")) + } + + // Only publishing needs the aws CLI. bsb-s3 datasets deliberately do not + // appear here: the bench binary's own SDK reads that public bucket, with + // no CLI and no credentials. + if strings.HasPrefix(cfg.PublishURI, "s3://") { + res.requireTool(d, "aws", fmt.Sprintf("needed by publish_uri '%s'", cfg.PublishURI)) + } + + if cfg.PublishURI != "" { + if err := d.ListRoot(cfg.PublishURI); err != nil { + res.failf("cannot list publish_uri '%s': %s — the campaign would only discover this at its publish step, hours from now", cfg.PublishURI, err) + } + } + + // Port of campaign.sh: the mount is only checked for the default root, and + // only where a mountpoint command exists (macOS has none). + if benchRoot == defaultBenchRoot { + if _, err := d.LookPath("mountpoint"); err == nil { + if err := d.Mountpoint(nvmeMount); err != nil { + res.failf("%s not mounted — run bootstrap.sh first, or set BENCH_ROOT", nvmeMount) + } + } + } + + free, err := d.DiskFree(benchRoot) + switch { + case err != nil: + res.warnf("could not measure free disk under %s: %s", benchRoot, err) + case free < minFree: + res.warnf("only %d GiB free under %s — a full campaign may need more", free/gib, benchRoot) + } + + return res +} + +func withDefaults(d Deps) Deps { + if d.LookPath == nil { + d.LookPath = exec.LookPath + } + if d.ListRoot == nil { + d.ListRoot = ListRoot + } + if d.Mountpoint == nil { + d.Mountpoint = mountpoint + } + if d.DiskFree == nil { + d.DiskFree = diskFree + } + return d +} + +// willBuild reports whether the campaign will compile the ref, which is what +// makes the Go and Rust toolchains a hard requirement. An unresolved ref ("") +// counts as a build: assuming otherwise would skip the check on exactly the +// fresh machine that most likely lacks a toolchain. +func willBuild(binPath string) bool { + return binPath == "" || !isExecutable(binPath) +} + +func isExecutable(path string) bool { + fi, err := os.Stat(path) + return err == nil && fi.Mode().IsRegular() && fi.Mode().Perm()&0o111 != 0 +} + +func cargoBin() string { return filepath.Join(os.Getenv("HOME"), ".cargo", "bin", "cargo") } + +// ListRoot reports whether the current credentials can list the +// object-storage root at uri. An empty root is a listable root; an auth, +// network, or missing-bucket error is not. +// +// The listing itself is publish.List — one implementation shared with the +// publish step, because both are asking the same question of the same two CLIs +// and a listing wrongly read as "empty" fails the same way in both places. +func ListRoot(uri string) error { + _, err := publish.List(uri) + return err +} + +func mountpoint(dir string) error { return exec.Command("mountpoint", "-q", dir).Run() } + +// diskFree returns the free bytes on the filesystem holding dir. BENCH_ROOT +// often does not exist yet, so it walks up to the nearest existing parent — +// that is the filesystem the root will land on. +func diskFree(dir string) (uint64, error) { + target, err := nearestExisting(dir) + if err != nil { + return 0, err + } + var st syscall.Statfs_t + if err := syscall.Statfs(target, &st); err != nil { + return 0, fmt.Errorf("statfs %s: %w", target, err) + } + return uint64(st.Bavail) * uint64(st.Bsize), nil +} + +func nearestExisting(dir string) (string, error) { + path, err := filepath.Abs(dir) + if err != nil { + return "", err + } + for { + if _, err := os.Stat(path); err == nil { + return path, nil + } + parent := filepath.Dir(path) + if parent == path { + return "", fmt.Errorf("no existing directory above %s", dir) + } + path = parent + } +} diff --git a/runner/internal/preflight/preflight_test.go b/runner/internal/preflight/preflight_test.go new file mode 100644 index 0000000..ab822c9 --- /dev/null +++ b/runner/internal/preflight/preflight_test.go @@ -0,0 +1,450 @@ +package preflight + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" +) + +// testConfig is a campaign that needs nothing beyond a toolchain: a local pack +// root, no publishing. Each test mutates the one field its check reads. +func testConfig(mods ...func(*config.Config)) *config.Config { + cfg := &config.Config{ + Name: "pf", + Repo: config.DefaultRepo, + Ref: config.DefaultRef, + Datasets: []config.Dataset{ + {Name: "ds", Kind: config.KindPacksLocal, Location: "/packs/ds", Chunks: []int{1}}, + }, + } + for _, mod := range mods { + mod(cfg) + } + return cfg +} + +// okDeps is an environment where every check passes, so a test that changes +// one dep sees exactly one failure. +func okDeps() Deps { + return Deps{ + LookPath: func(file string) (string, error) { return "/usr/bin/" + file, nil }, + ListRoot: func(string) error { return nil }, + Mountpoint: func(string) error { return nil }, + DiskFree: func(string) (uint64, error) { return 500 * gib, nil }, + } +} + +// missing returns a LookPath that finds everything except the named tools. +func missing(tools ...string) func(string) (string, error) { + return func(file string) (string, error) { + for _, t := range tools { + if t == file { + return "", os.ErrNotExist + } + } + return "/usr/bin/" + file, nil + } +} + +func has(t *testing.T, msgs []string, want string) { + t.Helper() + for _, m := range msgs { + if strings.Contains(m, want) { + return + } + } + t.Errorf("no message contains %q, got %q", want, msgs) +} + +// stubPATH points PATH at a directory of no-op executables, so the real +// exec.LookPath finds exactly the named tools and nothing else. +func stubPATH(t *testing.T, tools ...string) string { + t.Helper() + dir := t.TempDir() + for _, tool := range tools { + writeScript(t, filepath.Join(dir, tool), "#!/bin/sh\nexit 0\n") + } + t.Setenv("PATH", dir) + return dir +} + +func writeScript(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func TestToolChecks(t *testing.T) { + cases := []struct { + name string + absent []string + want string // substring of the expected failure + }{ + {name: "git", absent: []string{"git"}, want: "git not found in PATH — the campaign clones and builds repo '" + config.DefaultRepo + "'"}, + {name: "make", absent: []string{"make"}, want: "make not found in PATH — the campaign builds repo '" + config.DefaultRepo + "'"}, + {name: "go", absent: []string{"go"}, want: "go not found in PATH — building ref '" + config.DefaultRef + "'"}, + {name: "cargo", absent: []string{"cargo"}, want: "cargo not found in PATH or "}, + } + for _, tc := range cases { + t.Run("missing "+tc.name, func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) // no rustup install to fall back to + d := okDeps() + d.LookPath = missing(tc.absent...) + res := Run(testConfig(), t.TempDir(), "", d) + if len(res.Failures) != 1 { + t.Fatalf("failures = %q, want exactly one", res.Failures) + } + has(t, res.Failures, tc.want) + }) + } +} + +func TestCargoFoundUnderHome(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + writeScript(t, filepath.Join(home, ".cargo", "bin", "cargo"), "#!/bin/sh\nexit 0\n") + + d := okDeps() + d.LookPath = missing("cargo") + res := Run(testConfig(), t.TempDir(), "", d) + if len(res.Failures) != 0 { + t.Errorf("failures = %q, want none: rustup's cargo is outside PATH but usable", res.Failures) + } +} + +func TestExistingBinarySkipsToolchainChecks(t *testing.T) { + bin := filepath.Join(t.TempDir(), "stellar-rpc-deadbeef") + writeScript(t, bin, "#!/bin/sh\nexit 0\n") + t.Setenv("HOME", t.TempDir()) + + d := okDeps() + d.LookPath = missing("go", "cargo", "git", "make") + res := Run(testConfig(), t.TempDir(), bin, d) + + // Nothing is built, so no toolchain is needed — but the clone and the + // Makefile-driven steps still are. + if len(res.Failures) != 2 { + t.Fatalf("failures = %q, want git and make only", res.Failures) + } + has(t, res.Failures, "git not found") + has(t, res.Failures, "make not found") + + t.Run("a non-executable binary still means a build", func(t *testing.T) { + plain := filepath.Join(t.TempDir(), "stellar-rpc-deadbeef") + if err := os.WriteFile(plain, []byte("stale"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + res := Run(testConfig(), t.TempDir(), plain, d) + has(t, res.Failures, "go not found") + has(t, res.Failures, "cargo not found") + }) +} + +func TestCloudToolChecks(t *testing.T) { + packsGS := func(c *config.Config) { + c.Datasets = []config.Dataset{{Name: "packs", Kind: config.KindPacksGS, Location: "gs://bucket/cold", Chunks: []int{1}}} + } + bsbS3 := func(c *config.Config) { + c.Datasets = []config.Dataset{{Name: "bsb", Kind: config.KindBSBS3, Location: "s3://bucket/ledgers", Chunks: []int{1}}} + } + publish := func(uri string) func(*config.Config) { + return func(c *config.Config) { c.PublishURI = uri } + } + + cases := []struct { + name string + mods []func(*config.Config) + absent []string + want []string + noFails bool + }{ + { + name: "packs-gs dataset needs gcloud", + mods: []func(*config.Config){packsGS}, + absent: []string{"gcloud"}, + want: []string{"gcloud not found in PATH", "dataset 'packs' fetches its packs from 'gs://bucket/cold'"}, + }, + { + name: "gs:// publish_uri needs gcloud", + mods: []func(*config.Config){publish("gs://bucket/results")}, + absent: []string{"gcloud"}, + want: []string{"gcloud not found in PATH", "publish_uri 'gs://bucket/results'"}, + }, + { + name: "no gs:// anywhere needs no gcloud", + absent: []string{"gcloud"}, + noFails: true, + }, + { + name: "s3:// publish_uri needs aws", + mods: []func(*config.Config){publish("s3://bucket/results")}, + absent: []string{"aws"}, + want: []string{"aws not found in PATH", "publish_uri 's3://bucket/results'"}, + }, + { + // The bench binary's SDK reads the public bucket itself. + name: "bsb-s3 dataset needs no aws CLI", + mods: []func(*config.Config){bsbS3}, + absent: []string{"aws", "gcloud"}, + noFails: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := okDeps() + d.LookPath = missing(tc.absent...) + res := Run(testConfig(tc.mods...), t.TempDir(), "", d) + if tc.noFails { + if len(res.Failures) != 0 { + t.Fatalf("failures = %q, want none", res.Failures) + } + return + } + if len(res.Failures) != 1 { + t.Fatalf("failures = %q, want exactly one", res.Failures) + } + for _, want := range tc.want { + has(t, res.Failures, want) + } + }) + } +} + +func TestPublishRootListable(t *testing.T) { + t.Run("a listing error fails, naming publish_uri", func(t *testing.T) { + d := okDeps() + d.ListRoot = func(string) error { return os.ErrPermission } + cfg := testConfig(func(c *config.Config) { c.PublishURI = "gs://bucket/results" }) + res := Run(cfg, t.TempDir(), "", d) + if len(res.Failures) != 1 { + t.Fatalf("failures = %q, want exactly one", res.Failures) + } + has(t, res.Failures, "cannot list publish_uri 'gs://bucket/results': permission denied") + has(t, res.Failures, "hours from now") + }) + + t.Run("no publish_uri lists nothing", func(t *testing.T) { + called := false + d := okDeps() + d.ListRoot = func(string) error { called = true; return nil } + res := Run(testConfig(), t.TempDir(), "", d) + if called { + t.Error("ListRoot was called for a config that publishes nowhere") + } + if len(res.Failures) != 0 { + t.Errorf("failures = %q, want none", res.Failures) + } + }) +} + +func TestMountCheck(t *testing.T) { + t.Run("the default bench root must be mounted", func(t *testing.T) { + d := okDeps() + d.Mountpoint = func(string) error { return os.ErrNotExist } + res := Run(testConfig(), defaultBenchRoot, "", d) + if len(res.Failures) != 1 { + t.Fatalf("failures = %q, want exactly one", res.Failures) + } + has(t, res.Failures, "/mnt/nvme not mounted — run bootstrap.sh first, or set BENCH_ROOT") + }) + + t.Run("a machine without mountpoint is not checked", func(t *testing.T) { + called := false + d := okDeps() + d.LookPath = missing("mountpoint") + d.Mountpoint = func(string) error { called = true; return os.ErrNotExist } + res := Run(testConfig(), defaultBenchRoot, "", d) + if called { + t.Error("Mountpoint was called although no mountpoint command exists") + } + if len(res.Failures) != 0 { + t.Errorf("failures = %q, want none", res.Failures) + } + }) + + t.Run("a custom bench root is the operator's business", func(t *testing.T) { + called := false + d := okDeps() + d.Mountpoint = func(string) error { called = true; return os.ErrNotExist } + res := Run(testConfig(), t.TempDir(), "", d) + if called { + t.Error("Mountpoint was called for a non-default BENCH_ROOT") + } + if len(res.Failures) != 0 { + t.Errorf("failures = %q, want none", res.Failures) + } + }) +} + +func TestDiskCheck(t *testing.T) { + cases := []struct { + name string + free uint64 + err error + want string // substring of the expected warning, "" for no warning + }{ + {name: "tight disk warns", free: 50 * gib, want: "only 50 GiB free under "}, + {name: "roomy disk is quiet", free: 200 * gib}, + {name: "an unmeasurable filesystem warns", err: os.ErrNotExist, want: "could not measure free disk under "}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + d := okDeps() + d.DiskFree = func(string) (uint64, error) { return tc.free, tc.err } + res := Run(testConfig(), root, "", d) + if len(res.Failures) != 0 { + t.Errorf("failures = %q, want none: disk space is only ever a warning", res.Failures) + } + if tc.want == "" { + if len(res.Warnings) != 0 { + t.Fatalf("warnings = %q, want none", res.Warnings) + } + return + } + if len(res.Warnings) != 1 { + t.Fatalf("warnings = %q, want exactly one", res.Warnings) + } + has(t, res.Warnings, tc.want+root) + }) + } +} + +// The whole check set through the real exec.LookPath, with PATH stubbed to a +// directory of fake tools. +func TestRunWithRealLookPath(t *testing.T) { + d := Deps{ + ListRoot: func(string) error { return nil }, + Mountpoint: func(string) error { return nil }, + DiskFree: func(string) (uint64, error) { return 500 * gib, nil }, + } + + t.Run("a bare PATH is missing the toolchain", func(t *testing.T) { + stubPATH(t, "git", "make") + t.Setenv("HOME", t.TempDir()) + res := Run(testConfig(), t.TempDir(), "", d) + if len(res.Failures) != 2 { + t.Fatalf("failures = %q, want go and cargo", res.Failures) + } + has(t, res.Failures, "go not found") + has(t, res.Failures, "cargo not found") + }) + + t.Run("a fully equipped PATH passes", func(t *testing.T) { + stubPATH(t, "git", "make", "go", "cargo", "gcloud", "aws") + t.Setenv("HOME", t.TempDir()) + cfg := testConfig(func(c *config.Config) { c.PublishURI = "gs://bucket/results" }) + res := Run(cfg, t.TempDir(), "", d) + if len(res.Failures) != 0 { + t.Fatalf("failures = %q, want none", res.Failures) + } + if len(res.Warnings) != 0 { + t.Fatalf("warnings = %q, want none", res.Warnings) + } + }) +} + +// The default ListRoot, exercised against fake gcloud and aws binaries on +// PATH: it is the empty-vs-error distinction that matters, not the real CLIs. +func TestListRoot(t *testing.T) { + cases := []struct { + name string + tool string + script string + uri string + wantErr string // "" means the root must count as listable + }{ + { + name: "gcloud lists a populated root", + tool: "gcloud", + script: "#!/bin/sh\necho gs://bucket/results/run-1/\n", + uri: "gs://bucket/results", + }, + { + name: "gcloud on an empty root", + tool: "gcloud", + script: "#!/bin/sh\necho 'ERROR: One or more URLs matched no objects.' >&2\nexit 1\n", + uri: "gs://bucket/results", + }, + { + name: "gcloud without credentials", + tool: "gcloud", + script: "#!/bin/sh\necho 'ERROR: HTTPError 403: AccessDenied' >&2\nexit 1\n", + uri: "gs://bucket/results", + wantErr: "AccessDenied", + }, + { + // Silence is aws's empty signature, not gcloud's: gcloud always + // says so, and treating its silence as empty would pass a + // credential check that never ran. + name: "gcloud failing silently is a real failure", + tool: "gcloud", + script: "#!/bin/sh\nexit 1\n", + uri: "gs://bucket/results", + wantErr: "no output", + }, + { + name: "aws on an empty prefix says nothing at all", + tool: "aws", + script: "#!/bin/sh\nexit 1\n", + uri: "s3://bucket/results", + }, + { + // Anything printed anywhere means the exit was not emptiness. + name: "aws failing on stdout only is a real failure", + tool: "aws", + script: "#!/bin/sh\necho 'An error occurred (AccessDenied)'\nexit 1\n", + uri: "s3://bucket/results", + wantErr: "An error occurred (AccessDenied)", + }, + { + name: "aws without credentials", + tool: "aws", + script: "#!/bin/sh\necho 'fatal error: Unable to locate credentials' >&2\nexit 1\n", + uri: "s3://bucket/results", + wantErr: "Unable to locate credentials", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + writeScript(t, filepath.Join(dir, tc.tool), tc.script) + t.Setenv("PATH", dir) + err := ListRoot(tc.uri) + switch { + case tc.wantErr == "" && err != nil: + t.Fatalf("ListRoot(%s) = %v, want nil", tc.uri, err) + case tc.wantErr != "" && err == nil: + t.Fatalf("ListRoot(%s) = nil, want an error mentioning %q", tc.uri, tc.wantErr) + case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): + t.Fatalf("ListRoot(%s) = %v, want it to mention %q", tc.uri, err, tc.wantErr) + } + }) + } + + t.Run("an unsupported scheme is an error, not a listing", func(t *testing.T) { + if err := ListRoot("https://example.com/results"); err == nil { + t.Fatal("ListRoot(https://…) = nil, want an unsupported-scheme error") + } + }) +} + +func TestDiskFreeMeasuresTheNearestExistingParent(t *testing.T) { + root := t.TempDir() + free, err := diskFree(filepath.Join(root, "not", "created", "yet")) + if err != nil { + t.Fatalf("diskFree = error %v, want the parent's filesystem", err) + } + if free == 0 { + t.Error("diskFree = 0 bytes, want the temp filesystem's free space") + } +} diff --git a/runner/internal/publish/publish.go b/runner/internal/publish/publish.go new file mode 100644 index 0000000..16b1f22 --- /dev/null +++ b/runner/internal/publish/publish.go @@ -0,0 +1,234 @@ +// Package publish safeguards a finished benchmark campaign bundle to object +// storage. It uploads a campaign results directory to //, +// where run_id is the bundle's basename (the same run_id recorded in the +// bundle's metadata.json). The uploader is idempotent, but published runs are +// immutable: it refuses to write into a destination that already holds objects +// unless Force is given. +// +// It is the port of runner/publish.sh, and a leaf package on purpose: preflight +// borrows List to prove, before a campaign starts, that the credentials it will +// need seventeen hours from now exist today. +package publish + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// listTimeout bounds a destination listing. An unreachable endpoint should cost +// seconds, not the TCP stack's idea of patience. +const listTimeout = 30 * time.Second + +// metadataName is the bundle manifest whose absence marks a pre-manifest +// bundle. The file itself is internal/bundle's contract; publish only looks. +const metadataName = "metadata.json" + +// State is what a destination listing found. +type State int + +const ( + Empty State = iota // the prefix holds no objects + HasObjects // the prefix holds at least one object +) + +// ListError is a listing that failed for a reason other than emptiness: bad +// credentials, no network, no such bucket. It carries the tool's exit status +// and the first line of its complaint, which is what an operator needs. +type ListError struct { + ExitCode int + Detail string +} + +func (e *ListError) Error() string { return fmt.Sprintf("exit %d: %s", e.ExitCode, e.Detail) } + +// tool is one object-storage CLI: how it lists a prefix, how it syncs a +// directory into one, and how it says "no objects here". +type tool struct { + name string + ls func(uri string) []string + sync func(dir, dest string) []string + // empty reports whether a nonzero exit was this tool's way of saying the + // prefix holds no objects. + empty func(stdout, stderr string) bool +} + +// toolFor dispatches on the URI's scheme. gs:// uploads with `gcloud storage +// rsync -r`, s3:// with `aws s3 sync`. No other scheme is supported. +func toolFor(uri string) (tool, error) { + switch { + case strings.HasPrefix(uri, "gs://"): + return tool{ + name: "gcloud", + ls: func(uri string) []string { return []string{"gcloud", "storage", "ls", uri} }, + sync: func(dir, dest string) []string { + return []string{"gcloud", "storage", "rsync", "-r", dir, dest} + }, + empty: func(_, stderr string) bool { return strings.Contains(stderr, "matched no objects") }, + }, nil + case strings.HasPrefix(uri, "s3://"): + return tool{ + name: "aws", + ls: func(uri string) []string { return []string{"aws", "s3", "ls", uri} }, + sync: func(dir, dest string) []string { return []string{"aws", "s3", "sync", dir, dest} }, + empty: func(stdout, stderr string) bool { return stdout == "" && stderr == "" }, + }, nil + } + return tool{}, fmt.Errorf("unsupported scheme (supported: gs://, s3://)") +} + +// List reports whether uri already holds objects. +// +// Both CLIs report an empty prefix through a nonzero exit, but each has its own +// signature: aws s3 ls says nothing at all, gcloud storage ls says the URL +// matched no objects. runner/publish.sh accepts either signature from either +// CLI, out of shell expedience; here each tool is held to its own signature +// only, because the false-pass direction is the dangerous one — reading a +// credential failure as "empty prefix" would skip the immutability check that +// exists to keep a published run immutable, and would pass a preflight +// credential check that never actually ran. +func List(uri string) (State, error) { + t, err := toolFor(uri) + if err != nil { + return Empty, err + } + + ctx, cancel := context.WithTimeout(context.Background(), listTimeout) + defer cancel() + argv := t.ls(uri) + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + runErr := cmd.Run() + out, errOut := strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()) + + if runErr == nil { + if out == "" { + return Empty, nil + } + return HasObjects, nil + } + if ctx.Err() != nil { + return Empty, fmt.Errorf("%s took longer than %s", t.name, listTimeout) + } + if t.empty(out, errOut) { + return Empty, nil + } + detail := diagnosis(errOut) + if detail == "" { + detail = diagnosis(out) + } + if detail == "" { + detail = "no output" + } + return Empty, &ListError{ExitCode: exitCode(runErr), Detail: detail} +} + +// Run publishes resultsDir to //, where run_id is the +// bundle's basename. +// +// force skips the immutability check, and a forced publish is a MERGE, not a +// replace: the sync writes every object the bundle has, but objects already at +// the destination that this bundle does not contain survive it. Re-publishing a +// differently-shaped bundle over an old one therefore leaves the old one's +// extra files in place, and the destination is the union of the two. +// +// dryRun prints every cloud command and executes none of them. On success the +// final line written to out is `published: ` (machine-greppable; scripts +// depend on it). +func Run(resultsDir, destRoot string, dryRun, force bool, out io.Writer) error { + resultsDir = strings.TrimSuffix(resultsDir, "/") + if fi, err := os.Stat(resultsDir); err != nil || !fi.IsDir() { + return fmt.Errorf("results dir not found: %s", resultsDir) + } + if destRoot == "" { + return errors.New("no destination: pass or set PUBLISH_URI") + } + + runID := filepath.Base(resultsDir) + if _, err := os.Stat(filepath.Join(resultsDir, metadataName)); err != nil { + notef(out, "warning: %s/%s missing — pre-manifest bundle", resultsDir, metadataName) + } + dest := strings.TrimSuffix(destRoot, "/") + "/" + runID + "/" + + t, err := toolFor(dest) + if err != nil { + return fmt.Errorf("unsupported destination scheme: %s (supported: gs://, s3://)", dest) + } + + // Published runs are immutable: a destination that already holds objects is + // only written to with force. The listing command is printed even under a + // dry run — it is part of what a real publish would do. + if !force { + printCmd(out, t.ls(dest)) + if !dryRun { + state, err := List(dest) + if err != nil { + var le *ListError + if errors.As(err, &le) { + return fmt.Errorf("cannot list destination %s (exit %d): %s", dest, le.ExitCode, le.Detail) + } + return fmt.Errorf("cannot list destination %s: %s", dest, err) + } + if state == HasObjects { + return fmt.Errorf("destination already has objects: %s — published runs are immutable; pass --force to overwrite", dest) + } + } + } + + notef(out, "publish %s → %s", runID, dest) + argv := t.sync(resultsDir, dest) + printCmd(out, argv) + if dryRun { + notef(out, "dry run complete") + return nil + } + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Stdout, cmd.Stderr = out, out + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s: %w", strings.Join(argv, " "), err) + } + fmt.Fprintf(out, "published: %s\n", dest) + return nil +} + +// diagnosis reduces a CLI's output stream to its first non-empty line. Both +// tools put the actual error on stderr ("ERROR: (gcloud.storage.ls) …", "fatal +// error: Unable to locate credentials") and follow it with several lines of +// remediation prose, which would bury the failure in the report. +func diagnosis(stream string) string { + for _, line := range strings.Split(stream, "\n") { + if line = strings.TrimSpace(line); line != "" { + return line + } + } + return "" +} + +// exitCode is the tool's status, or -1 when it never got far enough to have one +// (binary missing, permission denied, signal). +func exitCode(err error) int { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() + } + return -1 +} + +// printCmd echoes a command the way bash's run() and the plan printer do. +func printCmd(w io.Writer, argv []string) { + fmt.Fprintf(w, " $ %s\n", strings.Join(argv, " ")) +} + +// notef is run.Notef's line format, duplicated rather than imported: preflight +// imports this package, and a leaf package keeps that dependency honest. +func notef(w io.Writer, format string, args ...any) { + fmt.Fprintf(w, "== [%s] %s\n", time.Now().UTC().Format("15:04:05"), fmt.Sprintf(format, args...)) +} diff --git a/runner/internal/publish/publish_test.go b/runner/internal/publish/publish_test.go new file mode 100644 index 0000000..3732e96 --- /dev/null +++ b/runner/internal/publish/publish_test.go @@ -0,0 +1,398 @@ +package publish + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// runID is the bundle basename every test publishes, and therefore the last +// path component of every destination below. +const runID = "phase4-6f35679f-20260715T101500Z" + +// cli is a fake gcloud or aws on PATH. It appends every invocation's argv to a +// file, so a test can assert not only what the publish did but what it never +// ran — a skipped listing is the whole point of --force, and a skipped sync is +// the whole point of the immutability check. +type cli struct{ record string } + +// stubCLI writes an executable named tool (gcloud or aws) into a temp dir and +// points PATH at it. lsBody is the shell run for the listing subcommand — the +// verb sits at $2 for both tools (`gcloud storage ls`, `aws s3 ls`); every +// other subcommand succeeds silently, standing in for the upload. +func stubCLI(t *testing.T, tool, lsBody string) *cli { + t.Helper() + dir := t.TempDir() + rec := filepath.Join(dir, "argv.log") + script := fmt.Sprintf(`#!/bin/sh +printf '%%s\n' "$*" >> %q +case "$2" in +ls) +%s +;; +*) exit 0 ;; +esac +`, rec, lsBody) + if err := os.WriteFile(filepath.Join(dir, tool), []byte(script), 0o755); err != nil { + t.Fatalf("write fake %s: %v", tool, err) + } + t.Setenv("PATH", dir) + return &cli{record: rec} +} + +// argv is every invocation the fake CLI saw, in order, as space-joined argv. +func (c *cli) argv(t *testing.T) []string { + t.Helper() + b, err := os.ReadFile(c.record) + if os.IsNotExist(err) { + return nil + } + if err != nil { + t.Fatalf("read recorded argv: %v", err) + } + return strings.Split(strings.TrimSpace(string(b)), "\n") +} + +// bundle makes a results directory named runID, with a metadata.json unless +// told otherwise. +func bundle(t *testing.T, withMetadata bool) string { + t.Helper() + dir := filepath.Join(t.TempDir(), runID) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir bundle: %v", err) + } + if withMetadata { + if err := os.WriteFile(filepath.Join(dir, metadataName), []byte(`{"run_id":"x"}`), 0o644); err != nil { + t.Fatalf("write metadata: %v", err) + } + } + return dir +} + +// Empty-prefix signatures, one per tool, as each CLI really writes them. +const ( + gcloudEmpty = "echo 'ERROR: One or more URLs matched no objects.' >&2\nexit 1" + gcloudFull = "echo gs://bucket/results/phase4-6f35679f-20260715T101500Z/metadata.json\nexit 0" + gcloudDenied = "echo 'ERROR: (gcloud.storage.ls) HTTPError 403: AccessDenied' >&2\necho 'Try gcloud auth login' >&2\nexit 1" + awsEmpty = "exit 1" +) + +func TestPublishesIntoAnEmptyPrefix(t *testing.T) { + c := stubCLI(t, "gcloud", gcloudEmpty) + dir := bundle(t, true) + var out bytes.Buffer + + if err := Run(dir, "gs://bucket/results", false, false, &out); err != nil { + t.Fatalf("Run = %v, want nil\n%s", err, out.String()) + } + + dest := "gs://bucket/results/" + runID + "/" + want := []string{ + "storage ls " + dest, + "storage rsync -r " + dir + " " + dest, + } + if got := c.argv(t); !equal(got, want) { + t.Errorf("cloud commands = %q, want %q", got, want) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if last := lines[len(lines)-1]; last != "published: "+dest { + t.Errorf("last line = %q, want %q — scripts grep for it", last, "published: "+dest) + } + if strings.Contains(out.String(), "pre-manifest bundle") { + t.Errorf("output warns about a missing metadata.json although the bundle has one:\n%s", out.String()) + } +} + +func TestRefusesANonEmptyDestination(t *testing.T) { + c := stubCLI(t, "gcloud", gcloudFull) + dir := bundle(t, true) + var out bytes.Buffer + + err := Run(dir, "gs://bucket/results", false, false, &out) + if err == nil { + t.Fatal("Run = nil, want the immutability refusal") + } + dest := "gs://bucket/results/" + runID + "/" + want := "destination already has objects: " + dest + " — published runs are immutable; pass --force to overwrite" + if err.Error() != want { + t.Errorf("error = %q, want %q", err, want) + } + if got := c.argv(t); !equal(got, []string{"storage ls " + dest}) { + t.Errorf("cloud commands = %q, want the listing only — nothing may be uploaded over a published run", got) + } +} + +func TestAbortsOnAListingItCannotRead(t *testing.T) { + c := stubCLI(t, "gcloud", gcloudDenied) + dir := bundle(t, true) + var out bytes.Buffer + + err := Run(dir, "gs://bucket/results", false, false, &out) + if err == nil { + t.Fatal("Run = nil, want the listing error — a credential failure must never read as an empty prefix") + } + for _, want := range []string{ + "cannot list destination gs://bucket/results/" + runID + "/", + "(exit 1)", + "HTTPError 403: AccessDenied", + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to mention %q", err, want) + } + } + if strings.Contains(err.Error(), "Try gcloud auth login") { + t.Errorf("error = %q, want the failure only, not the remediation prose", err) + } + if got := c.argv(t); len(got) != 1 { + t.Errorf("cloud commands = %q, want the listing only", got) + } +} + +func TestForceSkipsTheListingAndUploads(t *testing.T) { + // The listing would refuse this destination; --force must not run it at all. + c := stubCLI(t, "gcloud", gcloudFull) + dir := bundle(t, true) + var out bytes.Buffer + + if err := Run(dir, "gs://bucket/results", false, true, &out); err != nil { + t.Fatalf("Run = %v, want nil\n%s", err, out.String()) + } + dest := "gs://bucket/results/" + runID + "/" + if got := c.argv(t); !equal(got, []string{"storage rsync -r " + dir + " " + dest}) { + t.Errorf("cloud commands = %q, want the sync only", got) + } +} + +func TestDryRunPrintsEverythingAndRunsNothing(t *testing.T) { + c := stubCLI(t, "gcloud", gcloudEmpty) + dir := bundle(t, true) + var out bytes.Buffer + + if err := Run(dir, "gs://bucket/results", true, false, &out); err != nil { + t.Fatalf("Run = %v, want nil\n%s", err, out.String()) + } + if got := c.argv(t); got != nil { + t.Errorf("cloud commands = %q, want none under --dry-run", got) + } + dest := "gs://bucket/results/" + runID + "/" + for _, want := range []string{ + " $ gcloud storage ls " + dest, + " $ gcloud storage rsync -r " + dir + " " + dest, + "dry run complete", + } { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q, got:\n%s", want, out.String()) + } + } + if strings.Contains(out.String(), "published:") { + t.Errorf("output claims a publish that never happened:\n%s", out.String()) + } +} + +func TestS3Dispatch(t *testing.T) { + // Silence is aws's empty-prefix signature: no stdout, no stderr, exit 1. + c := stubCLI(t, "aws", awsEmpty) + dir := bundle(t, true) + var out bytes.Buffer + + if err := Run(dir, "s3://bucket/results/", false, false, &out); err != nil { + t.Fatalf("Run = %v, want nil\n%s", err, out.String()) + } + dest := "s3://bucket/results/" + runID + "/" + want := []string{ + "s3 ls " + dest, + "s3 sync " + dir + " " + dest, + } + if got := c.argv(t); !equal(got, want) { + t.Errorf("cloud commands = %q, want %q", got, want) + } + if !strings.Contains(out.String(), "published: "+dest) { + t.Errorf("output missing the published line, got:\n%s", out.String()) + } +} + +func TestTrailingSlashesCollapse(t *testing.T) { + c := stubCLI(t, "gcloud", gcloudEmpty) + dir := bundle(t, true) + var out bytes.Buffer + + // A results dir with a trailing slash still has the run id as its basename. + if err := Run(dir+"/", "gs://bucket/results/", false, false, &out); err != nil { + t.Fatalf("Run = %v, want nil\n%s", err, out.String()) + } + dest := "gs://bucket/results/" + runID + "/" + if got := c.argv(t); !equal(got, []string{"storage ls " + dest, "storage rsync -r " + dir + " " + dest}) { + t.Errorf("cloud commands = %q, want them under %s", got, dest) + } +} + +func TestWarnsAboutAPreManifestBundle(t *testing.T) { + stubCLI(t, "gcloud", gcloudEmpty) + dir := bundle(t, false) + var out bytes.Buffer + + if err := Run(dir, "gs://bucket/results", false, false, &out); err != nil { + t.Fatalf("Run = %v, want nil\n%s", err, out.String()) + } + want := "warning: " + filepath.Join(dir, metadataName) + " missing — pre-manifest bundle" + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q, got:\n%s", want, out.String()) + } +} + +func TestRefusalsBeforeAnyCloudCall(t *testing.T) { + cases := []struct { + name string + dir func(t *testing.T) string + destRoot string + want string + }{ + { + name: "a results dir that does not exist", + dir: func(t *testing.T) string { return filepath.Join(t.TempDir(), "nope") }, + destRoot: "gs://bucket/results", + want: "results dir not found: ", + }, + { + name: "a results path that is a file", + dir: func(t *testing.T) string { + path := filepath.Join(t.TempDir(), "bundle.tgz") + if err := os.WriteFile(path, nil, 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + return path + }, + destRoot: "gs://bucket/results", + want: "results dir not found: ", + }, + { + name: "no destination at all", + dir: func(t *testing.T) string { return bundle(t, true) }, + destRoot: "", + want: "no destination: pass or set PUBLISH_URI", + }, + { + name: "a scheme neither CLI speaks", + dir: func(t *testing.T) string { return bundle(t, true) }, + destRoot: "https://example.com/results", + want: "unsupported destination scheme: https://example.com/results/" + runID + "/ (supported: gs://, s3://)", + }, + { + name: "a bare path is not object storage", + dir: func(t *testing.T) string { return bundle(t, true) }, + destRoot: "/mnt/backup", + want: "unsupported destination scheme: /mnt/backup/" + runID + "/ (supported: gs://, s3://)", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := stubCLI(t, "gcloud", gcloudEmpty) + var out bytes.Buffer + err := Run(tc.dir(t), tc.destRoot, false, false, &out) + if err == nil { + t.Fatalf("Run = nil, want an error mentioning %q", tc.want) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to mention %q", err, tc.want) + } + if got := c.argv(t); got != nil { + t.Errorf("cloud commands = %q, want none", got) + } + }) + } +} + +func TestListStates(t *testing.T) { + cases := []struct { + name string + tool string + body string + uri string + want State + wantErr string + }{ + {name: "gcloud empty prefix", tool: "gcloud", body: gcloudEmpty, uri: "gs://bucket/results", want: Empty}, + {name: "gcloud populated prefix", tool: "gcloud", body: gcloudFull, uri: "gs://bucket/results", want: HasObjects}, + {name: "gcloud denied", tool: "gcloud", body: gcloudDenied, uri: "gs://bucket/results", wantErr: "AccessDenied"}, + { + // Silence is aws's empty signature, not gcloud's: gcloud always + // says so, and reading its silence as empty would skip the + // immutability check on a listing that never worked. + name: "gcloud silent failure", tool: "gcloud", body: "exit 1", + uri: "gs://bucket/results", wantErr: "no output", + }, + {name: "aws empty prefix says nothing at all", tool: "aws", body: awsEmpty, uri: "s3://bucket/results", want: Empty}, + { + name: "aws complaining on stdout only", tool: "aws", body: "echo 'An error occurred (AccessDenied)'\nexit 1", + uri: "s3://bucket/results", wantErr: "An error occurred (AccessDenied)", + }, + {name: "an unsupported scheme", tool: "gcloud", body: gcloudEmpty, uri: "https://example.com", wantErr: "unsupported scheme"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stubCLI(t, tc.tool, tc.body) + got, err := List(tc.uri) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("List(%s) = %v, nil; want an error mentioning %q", tc.uri, got, tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("List(%s) = error %v, want it to mention %q", tc.uri, err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("List(%s) = error %v, want %v", tc.uri, err, tc.want) + } + if got != tc.want { + t.Errorf("List(%s) = %v, want %v", tc.uri, got, tc.want) + } + }) + } +} + +func TestDiagnosisKeepsTheErrorNotTheRemediation(t *testing.T) { + stderr := "ERROR: (gcloud.storage.ls) There was a problem refreshing your current auth tokens\nPlease run:\n\n $ gcloud auth login\n" + if got, want := diagnosis(stderr), "ERROR: (gcloud.storage.ls) There was a problem refreshing your current auth tokens"; got != want { + t.Errorf("diagnosis = %q, want %q", got, want) + } +} + +func TestUploadFailurePropagates(t *testing.T) { + // An empty prefix, then a sync that fails: the publish must not claim it. + dir := t.TempDir() + script := "#!/bin/sh\ncase \"$2\" in\nls) echo 'ERROR: One or more URLs matched no objects.' >&2; exit 1 ;;\n*) echo 'ERROR: upload interrupted' >&2; exit 3 ;;\nesac\n" + if err := os.WriteFile(filepath.Join(dir, "gcloud"), []byte(script), 0o755); err != nil { + t.Fatalf("write fake gcloud: %v", err) + } + t.Setenv("PATH", dir) + + var out bytes.Buffer + err := Run(bundle(t, true), "gs://bucket/results", false, false, &out) + if err == nil { + t.Fatal("Run = nil, want the sync's failure") + } + if !strings.Contains(err.Error(), "gcloud storage rsync") { + t.Errorf("error = %q, want it to name the command that failed", err) + } + if strings.Contains(out.String(), "published:") { + t.Errorf("output claims a publish that failed:\n%s", out.String()) + } +} + +func equal(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} diff --git a/runner/internal/run/dataset.go b/runner/internal/run/dataset.go new file mode 100644 index 0000000..396b7dd --- /dev/null +++ b/runner/internal/run/dataset.go @@ -0,0 +1,177 @@ +package run + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/plan" +) + +// runDataset converges one dataset on a local cold pack root, whatever kind it +// is: the legs downstream all read /ledgers and neither know nor care +// whether it was fetched, backfilled, generated, or already there. +// +// Everything a kind materializes lands in .partial and is renamed onto +// only once whole, so an interrupted preparation can never be mistaken +// for a finished one — the golden-present check below is exactly that +// distinction, and `rm -rf ` is the documented lever to force a re-fetch. +func runDataset(s plan.Step, opts Options) StepResult { + if s.Dataset == nil { + return failure(s, errors.New("dataset step has no dataset spec")) + } + if err := prepareDataset(s, opts.Output); err != nil { + return failure(s, err) + } + return StepResult{ID: s.ID, Status: StatusOK} +} + +func prepareDataset(s plan.Step, out io.Writer) error { + d := s.Dataset + partial := d.Root + ".partial" + + switch d.Kind { + case config.KindPacksLocal: + Notef(out, "dataset %s: local cold pack root %s", d.Name, d.Root) + if !dirExists(filepath.Join(d.Root, "ledgers")) { + return fmt.Errorf("dataset '%s': %s/ledgers not found — location must be a cold pack root", d.Name, d.Root) + } + + case config.KindPacksGS: + if goldenPresent(d.Root) { + Notef(out, "dataset %s: golden packs already at %s — skipping fetch", d.Name, d.Root) + break + } + Notef(out, "dataset %s: fetch %s", d.Name, d.Location) + // golden_present was false, so root is absent or an empty leftover, and + // the plan's pre_clean says to clear it — but not the partial, which + // rsync resumes into. + if err := preClean(s, out); err != nil { + return err + } + if err := mkdirAll(out, partial); err != nil { + return err + } + if err := runDatasetCommands(s, out); err != nil { + return err + } + if err := rename(out, partial, d.Root); err != nil { + return err + } + + case config.KindBSBS3: + if goldenPresent(d.Root) { + Notef(out, "dataset %s: golden packs already at %s — skipping backfill", d.Name, d.Root) + break + } + Notef(out, "dataset %s: golden backfill of %s from S3 (untimed)", d.Name, d.Location) + if err := preClean(s, out); err != nil { + return err + } + // One untimed cold backfill per chunk. The step's env carries + // AWS_EC2_METADATA_DISABLED=true: without it the SDK signs requests + // with the machine's IAM role and the public bucket 403s, but setting + // it for the whole campaign would also hide those same instance-role + // credentials from the publish step's `aws s3` calls. + if err := runDatasetCommands(s, out); err != nil { + return err + } + if err := rename(out, partial, d.Root); err != nil { + return err + } + + case config.KindFixture: + if goldenPresent(d.Root) { + Notef(out, "dataset %s: golden packs already at %s — skipping generation", d.Name, d.Root) + break + } + if d.Stage == "" { + // The generation commands write into the staging tree, and the + // plan's pre_clean is derived from it; a spec without one describes + // a preparation nobody can reason about. + return fmt.Errorf("dataset '%s': fixture step has no staging pack dir", d.Name) + } + Notef(out, "dataset %s: generate a fixture pack tree", d.Name) + if err := preClean(s, out); err != nil { + return err + } + // Generate every chunk into the staging pack tree, then freeze every + // chunk into the golden packs — both untimed. + if err := runDatasetCommands(s, out); err != nil { + return err + } + if err := rename(out, partial, d.Root); err != nil { + return err + } + + default: + return fmt.Errorf("dataset '%s': unknown kind %q", d.Name, d.Kind) + } + + if !dirExists(filepath.Join(d.Root, "ledgers")) { + return fmt.Errorf("dataset '%s': %s/ledgers missing after preparation", d.Name, d.Root) + } + return nil +} + +// preClean wipes what the plan says to wipe before a dataset materializes. +// Which directories a kind clears — and, for packs-gs, which it deliberately +// keeps — is a property of the kind, so the plan owns the list and a dry run +// prints exactly the wipes the run performs. +func preClean(s plan.Step, out io.Writer) error { + if len(s.PreClean) == 0 { + return nil + } + return removeAll(out, s.PreClean...) +} + +// runDatasetCommands runs the step's commands in order, stopping at the first +// failure: what they build together is one pack tree, and half of one is worth +// nothing. +func runDatasetCommands(s plan.Step, out io.Writer) error { + for _, argv := range s.Argv { + if err := runCommand(argv, s.Env, out); err != nil { + return err + } + } + return nil +} + +// goldenPresent reports whether a pack root is there and non-empty, the port of +// bash's golden_present. +func goldenPresent(dir string) bool { + f, err := os.Open(dir) + if err != nil { + return false + } + defer f.Close() + names, err := f.Readdirnames(1) + return err == nil && len(names) > 0 +} + +func dirExists(path string) bool { + fi, err := os.Stat(path) + return err == nil && fi.IsDir() +} + +// mkdirAll and rename log themselves as the commands bash ran, so a campaign +// log shows every filesystem move the runner made, not just the ones that +// happened to be external processes. +func mkdirAll(out io.Writer, dir string) error { + fmt.Fprintf(out, " $ mkdir -p %s\n", dir) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("mkdir -p %s: %w", dir, err) + } + return nil +} + +func rename(out io.Writer, from, to string) error { + fmt.Fprintf(out, " $ mv %s %s\n", from, to) + if err := os.Rename(from, to); err != nil { + return fmt.Errorf("mv %s %s: %w", from, to, err) + } + return nil +} diff --git a/runner/internal/run/dataset_test.go b/runner/internal/run/dataset_test.go new file mode 100644 index 0000000..672991f --- /dev/null +++ b/runner/internal/run/dataset_test.go @@ -0,0 +1,248 @@ +package run + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/config" + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/plan" +) + +// dsStep is a dataset step whose commands are shell scripts standing in for +// gcloud rsync and the bench binary: they write into .partial exactly +// where the real tools do, which is all the choreography under test cares +// about. PreClean mirrors what plan.Build populates for the kind, since that +// list — not the executor — is what decides which directories get wiped. +func dsStep(name, kind, root string, argv ...[]string) plan.Step { + if argv == nil { + argv = [][]string{} + } + s := plan.Step{ + ID: "dataset-" + name, + Kind: plan.KindDataset, + Argv: argv, + Dataset: &plan.DatasetSpec{Name: name, Kind: kind, Location: "gs://bucket/cold", Root: root}, + } + switch kind { + case config.KindPacksGS: + s.PreClean = []string{root} + case config.KindBSBS3: + s.PreClean = []string{root, root + ".partial"} + } + return s +} + +// fixtureStep is dsStep for the one kind that also owns a staging pack tree, +// which the plan wipes by its parent along with both roots. +func fixtureStep(name, root, stage string, argv ...[]string) plan.Step { + s := dsStep(name, config.KindFixture, root, argv...) + s.Dataset.Stage = stage + s.PreClean = []string{filepath.Dir(stage), root, root + ".partial"} + return s +} + +// materialize is a command that fills dir with a pack tree, the way a fetch, a +// backfill, or a freeze would. +func materialize(dir string) []string { + return []string{"/bin/sh", "-c", `mkdir -p "$1/ledgers" && : > "$1/ledgers/chunk-1.pack"`, "sh", dir} +} + +// marker is a command that records that it ran. Preparations that should have +// been short-circuited are proven by its absence. +func marker(path string) []string { + return []string{"/bin/sh", "-c", `: > "$1"`, "sh", path} +} + +func prepare(t *testing.T, step plan.Step) outcome { + t.Helper() + return walk(t, &plan.Plan{Steps: []plan.Step{step}}, Options{}) +} + +func TestPrepareDatasetPacksLocal(t *testing.T) { + t.Run("a root holding packs is accepted as it is", func(t *testing.T) { + root := t.TempDir() + mustWrite(t, filepath.Join(root, "ledgers", "chunk-1.pack"), "packs") + got := prepare(t, dsStep("local", config.KindPacksLocal, root)) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "dataset local: local cold pack root "+root) + }) + + t.Run("a root without ledgers/ is refused in the operator's own words", func(t *testing.T) { + root := t.TempDir() + got := prepare(t, dsStep("local", config.KindPacksLocal, root)) + got.assertStatuses(t, StatusFailed) + want := "dataset 'local': " + root + "/ledgers not found — location must be a cold pack root" + if err := got.results[0].Err; err == nil || err.Error() != want { + t.Errorf("error = %v, want %q", err, want) + } + }) +} + +func TestPrepareDatasetPacksGS(t *testing.T) { + t.Run("an empty leftover root is cleared and the partial renamed onto it", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "pubnet") + mustMkdir(t, root) // the leftover of an earlier, cleared-out fetch + // A half-fetched tree from a killed session: the partial is deliberately + // kept, because rsync resumes into it. + mustWrite(t, root+".partial/half.pack", "resumable") + + got := prepare(t, dsStep("pubnet", config.KindPacksGS, root, materialize(root+".partial"))) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "dataset pubnet: fetch gs://bucket/cold") + got.assertLogHas(t, " $ rm -rf "+root+"\n") + got.assertLogHas(t, " $ mkdir -p "+root+".partial") + got.assertLogHas(t, " $ mv "+root+".partial "+root) + + assertExists(t, filepath.Join(root, "ledgers", "chunk-1.pack")) + assertExists(t, filepath.Join(root, "half.pack")) // the resumed bytes + assertGone(t, root+".partial") + assertGone(t, filepath.Join(root, "pubnet.partial")) // never nested + }) + + t.Run("golden packs already there short-circuit the fetch", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "pubnet") + mustWrite(t, filepath.Join(root, "ledgers", "chunk-1.pack"), "packs") + ran := filepath.Join(tmp, "ran.txt") + + got := prepare(t, dsStep("pubnet", config.KindPacksGS, root, marker(ran))) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "dataset pubnet: golden packs already at "+root+" — skipping fetch") + assertGone(t, ran) + // The short-circuit comes before the wipes: present golden packs are + // never touched, whatever pre_clean says. + got.assertLogLacks(t, "rm -rf") + assertExists(t, filepath.Join(root, "ledgers", "chunk-1.pack")) + }) + + t.Run("a preparation that leaves no ledgers/ fails", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "pubnet") + got := prepare(t, dsStep("pubnet", config.KindPacksGS, root, marker(root+".partial/events.db"))) + got.assertStatuses(t, StatusFailed) + want := "dataset 'pubnet': " + root + "/ledgers missing after preparation" + if err := got.results[0].Err; err == nil || err.Error() != want { + t.Errorf("error = %v, want %q", err, want) + } + }) + + t.Run("a failed fetch leaves the partial behind and never renames it", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "pubnet") + got := prepare(t, dsStep("pubnet", config.KindPacksGS, root, []string{"/bin/sh", "-c", "exit 1"})) + got.assertStatuses(t, StatusFailed) + assertGone(t, root) + assertExists(t, root+".partial") + }) +} + +func TestPrepareDatasetBSBS3(t *testing.T) { + t.Run("a stale partial is wiped before the backfill", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "bsb") + stale := root + ".partial/stale.pack" + mustWrite(t, stale, "from a backfill that died") + + // The backfill only materializes when the stale bytes are gone: a + // resumed cold backfill would double-write the pack tree. + argv := []string{"/bin/sh", "-c", `test ! -e "$2" && mkdir -p "$1/ledgers"`, "sh", root + ".partial", stale} + got := prepare(t, dsStep("bsb", config.KindBSBS3, root, argv)) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, " $ rm -rf "+root+" "+root+".partial") + assertExists(t, filepath.Join(root, "ledgers")) + assertGone(t, filepath.Join(root, "stale.pack")) + }) + + t.Run("golden packs already there short-circuit the backfill", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "bsb") + mustWrite(t, filepath.Join(root, "ledgers", "chunk-1.pack"), "packs") + ran := filepath.Join(tmp, "ran.txt") + + got := prepare(t, dsStep("bsb", config.KindBSBS3, root, marker(ran))) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "dataset bsb: golden packs already at "+root+" — skipping backfill") + assertGone(t, ran) + }) + + t.Run("the S3 env is set on the backfill commands", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "bsb") + argv := []string{"/bin/sh", "-c", `test "$AWS_EC2_METADATA_DISABLED" = true && mkdir -p "$1/ledgers"`, "sh", root + ".partial"} + step := dsStep("bsb", config.KindBSBS3, root, argv) + step.Env = map[string]string{"AWS_EC2_METADATA_DISABLED": "true"} + + got := prepare(t, step) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, " $ env AWS_EC2_METADATA_DISABLED=true /bin/sh -c") + }) +} + +func TestPrepareDatasetFixture(t *testing.T) { + t.Run("a stale staging tree is wiped before generation", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "fix") + stage := filepath.Join(tmp, "fixture", "fix", "ledgers") + stale := filepath.Join(tmp, "fixture", "fix", "ledgers", "chunk-1.pack") + mustWrite(t, stale, "half a chunk from a killed generation") + + // Generation refuses to run on top of the stale chunk; the freeze then + // fills the partial. + generate := []string{"/bin/sh", "-c", `test ! -e "$2" && mkdir -p "$1"`, "sh", stage, stale} + freeze := materialize(root + ".partial") + got := prepare(t, fixtureStep("fix", root, stage, generate, freeze)) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "dataset fix: generate a fixture pack tree") + got.assertLogHas(t, " $ rm -rf "+filepath.Join(tmp, "fixture", "fix")+" "+root+" "+root+".partial") + assertExists(t, filepath.Join(root, "ledgers", "chunk-1.pack")) + }) + + t.Run("golden packs already there short-circuit the generation", func(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "fix") + mustWrite(t, filepath.Join(root, "ledgers", "chunk-1.pack"), "packs") + ran := filepath.Join(tmp, "ran.txt") + stage := filepath.Join(tmp, "fixture", "fix", "ledgers") + + staged := filepath.Join(stage, "chunk-1.pack") + mustWrite(t, staged, "left by the generation that made these packs") + + got := prepare(t, fixtureStep("fix", root, stage, marker(ran))) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "dataset fix: golden packs already at "+root+" — skipping generation") + assertGone(t, ran) + assertExists(t, staged) // the short-circuit precedes the staging wipe + }) +} + +// TestPrepareDatasetWipesWhatThePlanSays pins where the wipe list lives: the +// executor removes the step's pre_clean and nothing else, so a plan that asks +// for the partial gets it removed even for the kind that normally resumes. +func TestPrepareDatasetWipesWhatThePlanSays(t *testing.T) { + tmp := t.TempDir() + root := filepath.Join(tmp, "golden", "pubnet") + mustWrite(t, root+".partial/half.pack", "resumable, but this plan says otherwise") + + step := dsStep("pubnet", config.KindPacksGS, root, materialize(root+".partial")) + step.PreClean = []string{root, root + ".partial"} + + got := prepare(t, step) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, " $ rm -rf "+root+" "+root+".partial") + assertGone(t, filepath.Join(root, "half.pack")) +} + +func TestPrepareDatasetRejectsAnUnknownKind(t *testing.T) { + got := prepare(t, dsStep("odd", "packs-ftp", filepath.Join(t.TempDir(), "golden", "odd"))) + got.assertStatuses(t, StatusFailed) + if err := got.results[0].Err; err == nil || !strings.Contains(err.Error(), `unknown kind "packs-ftp"`) { + t.Errorf("error = %v, want it to name the unknown kind", err) + } +} + +func TestRunDatasetWithoutASpec(t *testing.T) { + got := prepare(t, plan.Step{ID: "dataset-x", Kind: plan.KindDataset, Argv: [][]string{}}) + got.assertStatuses(t, StatusFailed) +} diff --git a/runner/internal/run/doc.go b/runner/internal/run/doc.go new file mode 100644 index 0000000..b629bfd --- /dev/null +++ b/runner/internal/run/doc.go @@ -0,0 +1,3 @@ +// Package run executes a plan: it walks the steps, writes the per-leg +// completion sentinels, and decides what a resumed campaign may skip. +package run diff --git a/runner/internal/run/lock.go b/runner/internal/run/lock.go new file mode 100644 index 0000000..db788ae --- /dev/null +++ b/runner/internal/run/lock.go @@ -0,0 +1,39 @@ +package run + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +// lockName is the lock file at the root of a BENCH_ROOT. It is created once +// and never deleted: unlinking a lock file races with the next campaign, which +// may already hold the old inode open. +const lockName = ".campaign.lock" + +// AcquireLock takes an exclusive, non-blocking flock on /.campaign.lock +// and returns the function that releases it. Two campaigns sharing a BENCH_ROOT +// would fight over the same build clone, scratch dirs, and hot DBs, so the +// second one is refused immediately rather than queued: the operator wants to +// know now, not in six hours. +func AcquireLock(benchRoot string) (release func(), err error) { + if err := os.MkdirAll(benchRoot, 0o755); err != nil { + return nil, fmt.Errorf("create BENCH_ROOT %s: %w", benchRoot, err) + } + path := filepath.Join(benchRoot, lockName) + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return nil, fmt.Errorf("open lock file %s: %w", path, err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + return nil, fmt.Errorf("another campaign is already running on this BENCH_ROOT (held lock: %s)", path) + } + // The lock lives on the open file description, so the fd stays open until + // release; closing it anywhere earlier would drop the lock silently. + return func() { + syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + f.Close() + }, nil +} diff --git a/runner/internal/run/log.go b/runner/internal/run/log.go new file mode 100644 index 0000000..618a6e8 --- /dev/null +++ b/runner/internal/run/log.go @@ -0,0 +1,26 @@ +package run + +import ( + "fmt" + "io" + "os" + "path/filepath" + "time" +) + +// campaignLogName is the per-bundle log every session appends to. +const campaignLogName = "campaign.log" + +// Notef prints a bash-style note — `== [HH:MM:SS] msg`, the clock in UTC — +// the line format every operator reading a campaign log already knows from +// campaign.sh's note(). +func Notef(w io.Writer, format string, args ...any) { + fmt.Fprintf(w, "== [%s] %s\n", time.Now().UTC().Format("15:04:05"), fmt.Sprintf(format, args...)) +} + +// OpenCampaignLog opens /campaign.log for appending. Append, not +// truncate: a campaign that is resumed twice leaves all three sessions in one +// file, in the order they happened. +func OpenCampaignLog(resultsDir string) (*os.File, error) { + return os.OpenFile(filepath.Join(resultsDir, campaignLogName), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) +} diff --git a/runner/internal/run/resume.go b/runner/internal/run/resume.go new file mode 100644 index 0000000..94f6f86 --- /dev/null +++ b/runner/internal/run/resume.go @@ -0,0 +1,140 @@ +package run + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// legStateKind is what an existing --out directory means to a resumed campaign. +type legStateKind int + +const ( + legAbsent legStateKind = iota // no directory at all + legPartial // something is there, but nothing says it finished + legFailedEarlier // a previous session ran this leg and it failed + legComplete // a previous session ran this leg and it succeeded +) + +// legState is a classification plus, for a failure, the reason to show the +// operator — the sentinel's error, the exit status, or the invocation manifest's +// own error field. +type legState struct { + kind legStateKind + reason string +} + +// legSentinelName is the runner-owned completion marker. The bench subcommands +// own invocation.json; nothing but this runner writes leg.json, which is why it +// is the sentinel resume trusts first. +const legSentinelName = "leg.json" + +// LegComplete reports whether a leg's --out directory already holds a leg an +// earlier session finished successfully. wantID is the plan's id for the leg, +// which the sentinel must name to be trusted. It is the read-only half of the +// resume decision: `run --dry-run --resume` annotates the plan with it, +// touching nothing. +func LegComplete(dir, wantID string) bool { + return classifyLegDir(dir, wantID).kind == legComplete +} + +// classifyLegDir decides what a resumed campaign should do with a leg's +// existing --out directory. Only a marker that positively records success +// counts as complete; every ambiguous state resolves to "wipe and re-run", +// because a half-written leg silently kept would corrupt the aggregates the +// converter computes over the bundle. +func classifyLegDir(dir, wantID string) legState { + // Only positive absence is absence; everything unreadable resolves to + // wipe-and-re-run. Lstat rather than Stat so a dangling symlink is seen as + // something-is-there, and permission or I/O errors fall to partial too — + // otherwise resume would call MkdirAll on a path that can never be created. + if _, err := os.Lstat(dir); err != nil { + if os.IsNotExist(err) { + return legState{kind: legAbsent} + } + return legState{kind: legPartial} + } + + // A sentinel is trusted only once it has identified itself: this runner's + // schema version and this leg's id. A `{}` that happens to parse, a record + // copied in from another leg, or a future schema whose fields mean something + // else all prove nothing about this directory, so they are partial. + switch sentinel, err := readLegSentinel(filepath.Join(dir, legSentinelName)); { + case err == nil && (sentinel.SchemaVersion != LegSchemaVersion || sentinel.ID != wantID): + return legState{kind: legPartial, reason: "sentinel does not match this leg"} + case err == nil && sentinel.ExitCode == nil: + return legState{kind: legPartial, reason: "sentinel records no exit"} + case err == nil && *sentinel.ExitCode == 0 && sentinel.Error == "": + return legState{kind: legComplete} + case err == nil && sentinel.Error != "": + return legState{kind: legFailedEarlier, reason: sentinel.Error} + case err == nil: + return legState{kind: legFailedEarlier, reason: fmt.Sprintf("exit status %d", *sentinel.ExitCode)} + case !os.IsNotExist(err): + // The sentinel is there but unreadable or corrupt: it proves nothing, + // so the leg is treated as partial rather than trusted either way. + return legState{kind: legPartial} + } + + // No leg.json: this may be a bundle the bash runner produced, which had no + // sentinel of its own and inferred completion from the manifests the bench + // subcommand writes. A failed run also writes invocation.json — with an + // `error` field (stellar-rpc#907) — so completion means both files present + // AND no error recorded. + inv, err := readInvocation(filepath.Join(dir, "invocation.json")) + if err != nil { + return legState{kind: legPartial} + } + if _, err := os.Stat(filepath.Join(dir, "driver.csv")); err != nil { + return legState{kind: legPartial} + } + if inv.Error != "" { + return legState{kind: legFailedEarlier, reason: inv.Error} + } + return legState{kind: legComplete} +} + +// legSentinelView is the read side of leg.json: the fields resume decides on. +// ExitCode is a pointer because "the runner recorded exit 0" and "there is no +// exit_code here at all" must not read alike — decoding an absent field into an +// int would turn a sentinel that records no exit into a claim of success. +type legSentinelView struct { + SchemaVersion int `json:"schema_version"` + ID string `json:"id"` + ExitCode *int `json:"exit_code"` + Error string `json:"error"` +} + +// readLegSentinel reads and parses a leg.json. A missing file is reported as +// os.IsNotExist so the caller can fall back to the bash-era manifests. +func readLegSentinel(path string) (legSentinelView, error) { + b, err := os.ReadFile(path) + if err != nil { + return legSentinelView{}, err + } + var s legSentinelView + if err := json.Unmarshal(b, &s); err != nil { + return legSentinelView{}, err + } + return s, nil +} + +// invocationManifest is the sliver of stellar-rpc's invocation.json this runner +// reads: whether the run recorded a failure. The file is camelCase and owned by +// the other repo; the runner never writes it. +type invocationManifest struct { + Error string `json:"error"` +} + +func readInvocation(path string) (invocationManifest, error) { + b, err := os.ReadFile(path) + if err != nil { + return invocationManifest{}, err + } + var m invocationManifest + if err := json.Unmarshal(b, &m); err != nil { + return invocationManifest{}, err + } + return m, nil +} diff --git a/runner/internal/run/run.go b/runner/internal/run/run.go new file mode 100644 index 0000000..0499571 --- /dev/null +++ b/runner/internal/run/run.go @@ -0,0 +1,376 @@ +package run + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/plan" +) + +// LegSchemaVersion is the version of the leg.json contract. Like plan.json, +// additive changes keep the version. +const LegSchemaVersion = 1 + +// Status is what became of one step. +type Status string + +const ( + StatusOK Status = "ok" + StatusFailed Status = "failed" + StatusSkipped Status = "skipped" // a need (transitively) failed + StatusResumed Status = "resumed" // already complete in an earlier session +) + +// StepResult is one line of the campaign's outcome. +type StepResult struct { + ID string + Status Status + Err error // nil unless failed +} + +// Options configures a walk. +type Options struct { + // Output receives the runner's notes, the command lines, and the child + // processes' stdout and stderr. The caller composes the tee (terminal plus + // campaign.log); the executor just writes. Defaults to os.Stdout, which is + // where bash sent everything. + Output io.Writer + // Resume inspects each leg's existing --out directory and skips the ones an + // earlier session finished. Off, nothing existing is read or wiped by the + // resume path. + Resume bool + // FailFast stops the walk at the first failed step. Default (keep going): a + // failure only skips the steps that need it. + FailFast bool + // OnStepDone is called after every step the walk executed — including the + // ones a resume or an existing binary made a no-op, excluding only the ones + // skipped because a need failed. The run wiring writes binary.txt from it + // the moment the build succeeds, so a campaign that dies during its legs + // still leaves the binary's identity in the bundle. + OnStepDone func(s plan.Step, res StepResult) +} + +// legSentinel is leg.json: the runner's own record that a leg ran to +// completion, written whether it succeeded or not. The bench subcommands' +// invocation.json cannot play this role — it is written by the process being +// measured, so a process killed before it got there leaves no trace at all. +// +// ExitCode carries no omitempty on purpose: exit 0 is the success record resume +// reads, and a sentinel without the field is not trusted (see legSentinelView). +type legSentinel struct { + SchemaVersion int `json:"schema_version"` + ID string `json:"id"` + Argv []string `json:"argv"` + ExitCode int `json:"exit_code"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + DurationNS int64 `json:"duration_ns"` + Error string `json:"error,omitempty"` +} + +// Execute walks the plan in order — sequentially, always: these are benchmarks, +// and two of them sharing the machine measure the sharing. It returns one +// result per executed step and a non-nil error when any step failed; the +// failure summary has already been printed to Output, so a caller that exits +// nonzero needs to print nothing more. +// +// Under FailFast the walk stops at the first failure, so the returned slice is +// short: steps after the failed one have no result at all, rather than a +// skipped one. +func Execute(p *plan.Plan, opts Options) ([]StepResult, error) { + if opts.Output == nil { + opts.Output = os.Stdout + } + results := make([]StepResult, 0, len(p.Steps)) + // bad holds every step that failed or was skipped. Skipped steps being bad + // is what makes the propagation transitive: the dependent of a skipped step + // is skipped too, without walking the graph. + bad := map[string]bool{} + skipNeed := map[string]string{} + + for _, step := range p.Steps { + // The tarball and the publish are the wiring's epilogue, not the walk's: + // tar must run after the final provenance writes so the bundle it + // preserves contains them, and a publish failure is not a benchmark + // failure. They are in the plan because they are part of the campaign; + // they are skipped here because they belong after every step of it. + if step.Kind == plan.KindTarball || step.Kind == plan.KindPublish { + continue + } + if need := firstBadNeed(step, bad); need != "" { + Notef(opts.Output, "skipping %s: needs %s, which failed or was skipped", step.ID, need) + bad[step.ID] = true + skipNeed[step.ID] = need + results = append(results, StepResult{ID: step.ID, Status: StatusSkipped}) + continue + } + Notef(opts.Output, "%s", step.ID) + res := executeStep(p, step, opts) + results = append(results, res) + if opts.OnStepDone != nil { + opts.OnStepDone(step, res) + } + if res.Status == StatusFailed { + Notef(opts.Output, "%s failed: %v", step.ID, res.Err) + bad[step.ID] = true + if opts.FailFast { + break + } + } + } + return results, summarize(results, skipNeed, opts.Output) +} + +// firstBadNeed returns the first of a step's needs that failed or was skipped, +// or "" when the step is clear to run. +func firstBadNeed(s plan.Step, bad map[string]bool) string { + for _, need := range s.Needs { + if bad[need] { + return need + } + } + return "" +} + +// summarize prints the end-of-campaign failure block and returns the error +// Execute hands back. An all-ok campaign prints nothing and returns nil. +func summarize(results []StepResult, skipNeed map[string]string, w io.Writer) error { + var failed, skipped []StepResult + for _, r := range results { + switch r.Status { + case StatusFailed: + failed = append(failed, r) + case StatusSkipped: + skipped = append(skipped, r) + } + } + if len(failed) == 0 && len(skipped) == 0 { + return nil + } + fmt.Fprintf(w, "== campaign summary: %d failed, %d skipped\n", len(failed), len(skipped)) + for _, r := range failed { + fmt.Fprintf(w, "== failed: %s (%v)\n", r.ID, r.Err) + } + for _, r := range skipped { + fmt.Fprintf(w, "== skipped: %s (needs %s)\n", r.ID, skipNeed[r.ID]) + } + if len(failed) == 0 { + return nil + } + return fmt.Errorf("%d step(s) failed", len(failed)) +} + +func executeStep(p *plan.Plan, s plan.Step, opts Options) StepResult { + switch s.Kind { + case plan.KindLeg: + return runLeg(s, opts) + case plan.KindBuild: + return runBuild(p, s, opts) + case plan.KindDataset: + return runDataset(s, opts) + default: + return failure(s, fmt.Errorf("unknown step kind %q", s.Kind)) + } +} + +// runLeg runs one timed benchmark leg: the whole point of the campaign, and the +// only step kind with a completion sentinel. +func runLeg(s plan.Step, opts Options) StepResult { + if len(s.Argv) != 1 { + return failure(s, fmt.Errorf("leg has %d commands, want exactly 1 (the measurement is the process)", len(s.Argv))) + } + if opts.Resume { + base := filepath.Base(s.OutDir) + state := classifyLegDir(s.OutDir, s.ID) + switch { + case state.kind == legComplete: + Notef(opts.Output, "resume: %s already complete — skipping", base) + return StepResult{ID: s.ID, Status: StatusResumed} + case state.kind == legFailedEarlier: + Notef(opts.Output, "resume: %s failed in an earlier session (%s) — wiping and re-running", base, state.reason) + case state.kind == legPartial && state.reason != "": + Notef(opts.Output, "resume: %s is a partial leg (%s) — wiping and re-running", base, state.reason) + case state.kind == legPartial: + Notef(opts.Output, "resume: %s is a partial leg — wiping and re-running", base) + } + if state.kind != legAbsent { + if err := removeAll(opts.Output, s.OutDir); err != nil { + return failure(s, err) + } + } + } + for _, dir := range s.PreClean { + if err := removeAll(opts.Output, dir); err != nil { + return failure(s, err) + } + } + + // The bench subcommand creates its own --out dir, but creating it here too + // means the sentinel has somewhere to land even when the binary dies + // instantly — which is exactly the case resume most needs to classify. + started := time.Now() + runErr := os.MkdirAll(s.OutDir, 0o755) + if runErr == nil { + runErr = runCommand(s.Argv[0], s.Env, opts.Output) + } + finished := time.Now() + + if err := writeLegSentinel(s, started, finished, runErr); err != nil { + if runErr == nil { + // A leg whose completion cannot be recorded is not complete: a + // resume would re-run it anyway, so call it failed now. + runErr = err + } else { + Notef(opts.Output, "warning: %s: %v", s.ID, err) + } + } + if runErr != nil { + return failure(s, runErr) + } + // Post-cleaning only on success keeps a failed leg's scratch around for + // diagnosis. A failure to clean is not a failure of the measurement. + for _, dir := range s.PostClean { + if err := removeAll(opts.Output, dir); err != nil { + Notef(opts.Output, "warning: %s: %v", s.ID, err) + } + } + return StepResult{ID: s.ID, Status: StatusOK} +} + +// writeLegSentinel records how the leg went, success or failure, into its --out +// directory. +func writeLegSentinel(s plan.Step, started, finished time.Time, runErr error) error { + sentinel := legSentinel{ + SchemaVersion: LegSchemaVersion, + ID: s.ID, + Argv: s.Argv[0], + StartedAt: started.UTC().Format(time.RFC3339), + FinishedAt: finished.UTC().Format(time.RFC3339), + DurationNS: finished.Sub(started).Nanoseconds(), + } + if runErr != nil { + sentinel.ExitCode = exitCode(runErr) + sentinel.Error = runErr.Error() + } + b, err := json.MarshalIndent(sentinel, "", " ") + if err != nil { + return fmt.Errorf("marshal %s: %w", legSentinelName, err) + } + if err := os.WriteFile(filepath.Join(s.OutDir, legSentinelName), append(b, '\n'), 0o644); err != nil { + return fmt.Errorf("write %s: %w", legSentinelName, err) + } + return nil +} + +// exitCode is the child's status, or -1 when it never got far enough to have +// one (binary missing, permission denied, signal). +func exitCode(err error) int { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() + } + return -1 +} + +// runBuild builds the binary under test unless it is already there. The binary +// at its versioned path is its own completion marker — the path contains the +// commit's short sha, so a stale binary cannot be mistaken for this one. +func runBuild(p *plan.Plan, s plan.Step, opts Options) StepResult { + if executableExists(p.Bin) { + Notef(opts.Output, "binary %s already built — skipping build", p.Bin) + return StepResult{ID: s.ID, Status: StatusOK} + } + return runCommands(s, opts) +} + +// executableExists reports whether path is a regular file anyone may execute. +func executableExists(path string) bool { + fi, err := os.Stat(path) + return err == nil && fi.Mode().IsRegular() && fi.Mode().Perm()&0o111 != 0 +} + +// runCommands runs a step's commands in order, stopping at the first failure. +// They belong to one step precisely because stopping between them would leave +// something half-made. +func runCommands(s plan.Step, opts Options) StepResult { + for _, argv := range s.Argv { + if err := runCommand(argv, s.Env, opts.Output); err != nil { + return failure(s, err) + } + } + return StepResult{ID: s.ID, Status: StatusOK} +} + +// RunStep runs one step's commands outside the walk, printed and plumbed +// exactly as Execute would. It exists for the steps Execute deliberately leaves +// to the wiring's epilogue — the tarball, which must be made after the final +// provenance writes. +func RunStep(s plan.Step, out io.Writer) error { + if res := runCommands(s, Options{Output: out}); res.Status != StatusOK { + return res.Err + } + return nil +} + +// runCommand prints the command the way the plan printer and bash's run() do, +// then executes it with its output going wherever the notes go. +func runCommand(argv []string, env map[string]string, out io.Writer) error { + if len(argv) == 0 { + return errors.New("empty command") + } + fmt.Fprintf(out, " $ %s%s\n", envPrefix(env), strings.Join(argv, " ")) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Stdout, cmd.Stderr = out, out + cmd.Env = os.Environ() + for _, k := range sortedKeys(env) { + cmd.Env = append(cmd.Env, k+"="+env[k]) + } + return cmd.Run() +} + +// removeAll wipes directories, logging them as the single rm -rf bash ran. +func removeAll(out io.Writer, dirs ...string) error { + fmt.Fprintf(out, " $ rm -rf %s\n", strings.Join(dirs, " ")) + for _, dir := range dirs { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("rm -rf %s: %w", dir, err) + } + } + return nil +} + +// envPrefix renders a step's extra environment as an `env K=V ` command prefix, +// matching plan.Plan.Print so a log line and a plan line for the same command +// read identically. +func envPrefix(env map[string]string) string { + if len(env) == 0 { + return "" + } + var b strings.Builder + b.WriteString("env ") + for _, k := range sortedKeys(env) { + fmt.Fprintf(&b, "%s=%s ", k, env[k]) + } + return b.String() +} + +func sortedKeys(env map[string]string) []string { + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func failure(s plan.Step, err error) StepResult { + return StepResult{ID: s.ID, Status: StatusFailed, Err: err} +} diff --git a/runner/internal/run/run_test.go b/runner/internal/run/run_test.go new file mode 100644 index 0000000..67c2816 --- /dev/null +++ b/runner/internal/run/run_test.go @@ -0,0 +1,707 @@ +package run + +import ( + "bytes" + "encoding/json" + "io" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "github.com/stellar/stellar-rpc-benchmarks/runner/internal/plan" +) + +// --- helpers -------------------------------------------------------------- + +func mustMkdir(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } +} + +func mustWrite(t *testing.T, path, body string) { + t.Helper() + mustMkdir(t, filepath.Dir(path)) + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// shLeg is a timed leg whose measurement is a shell script. The script's +// positional parameters carry the paths it needs — passing them as arguments +// rather than environment keeps the leg's Env free for the env-propagation +// test. +func shLeg(id, outDir, script string, args ...string) plan.Step { + argv := append([]string{"/bin/sh", "-c", script, "sh", outDir}, args...) + return plan.Step{ID: id, Kind: plan.KindLeg, Timed: true, OutDir: outDir, Argv: [][]string{argv}} +} + +// outcome is everything one Execute call produced. +type outcome struct { + results []StepResult + err error + log string +} + +func walk(t *testing.T, p *plan.Plan, opts Options) outcome { + t.Helper() + var buf bytes.Buffer + opts.Output = &buf + results, err := Execute(p, opts) + return outcome{results: results, err: err, log: buf.String()} +} + +func (o outcome) statuses() []Status { + got := make([]Status, len(o.results)) + for i, r := range o.results { + got[i] = r.Status + } + return got +} + +func (o outcome) assertStatuses(t *testing.T, want ...Status) { + t.Helper() + if got := o.statuses(); !slices.Equal(got, want) { + t.Errorf("statuses = %v, want %v\nlog:\n%s", got, want, o.log) + } +} + +func (o outcome) assertLogHas(t *testing.T, want string) { + t.Helper() + if !strings.Contains(o.log, want) { + t.Errorf("log does not contain %q\nlog:\n%s", want, o.log) + } +} + +func (o outcome) assertLogLacks(t *testing.T, unwanted string) { + t.Helper() + if strings.Contains(o.log, unwanted) { + t.Errorf("log contains %q, want it not to\nlog:\n%s", unwanted, o.log) + } +} + +func readSentinel(t *testing.T, outDir string) legSentinel { + t.Helper() + path := filepath.Join(outDir, legSentinelName) + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var s legSentinel + if err := json.Unmarshal(b, &s); err != nil { + t.Fatalf("unmarshal %s: %v", path, err) + } + return s +} + +func assertGone(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("%s still exists (stat err = %v)", path, err) + } +} + +func assertExists(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Errorf("%s missing: %v", path, err) + } +} + +// --- resume decision table ------------------------------------------------- + +func TestClassifyLegDir(t *testing.T) { + // The leg being classified: its plan id is its --out directory's basename, + // and the sentinel has to name it to be believed. + const legID = "ingest-cold-ds-c0-run1" + cases := []struct { + name string + setup func(t *testing.T, dir string) + want legStateKind + wantReason string + }{ + { + name: "no directory at all", + setup: func(t *testing.T, dir string) { os.RemoveAll(dir) }, + want: legAbsent, + }, + { + name: "empty directory", + setup: func(t *testing.T, dir string) {}, + want: legPartial, + }, + { + name: "dangling symlink where the out-dir should be", + setup: func(t *testing.T, dir string) { + if err := os.RemoveAll(dir); err != nil { + t.Fatalf("RemoveAll: %v", err) + } + if err := os.Symlink(dir+"-gone", dir); err != nil { + t.Fatalf("Symlink: %v", err) + } + }, + want: legPartial, + }, + { + name: "sentinel says success", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), `{"schema_version":1,"id":"`+legID+`","exit_code":0}`) + }, + want: legComplete, + }, + { + name: "sentinel says failure", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), + `{"schema_version":1,"id":"`+legID+`","exit_code":1,"error":"exit status 1"}`) + }, + want: legFailedEarlier, + wantReason: "exit status 1", + }, + { + name: "sentinel with a nonzero exit and no error field", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), `{"schema_version":1,"id":"`+legID+`","exit_code":2}`) + }, + want: legFailedEarlier, + wantReason: "exit status 2", + }, + { + // The degenerate sentinel: valid JSON, zero exit code by omission, + // and no claim to be anything. Believing it would skip a leg that + // never ran. + name: "empty JSON object as a sentinel", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), `{}`) + }, + want: legPartial, + wantReason: "sentinel does not match this leg", + }, + { + // The right leg, the right schema, and no record of how it ended: + // an absent exit_code must not decode into the 0 that means success. + name: "sentinel without an exit_code", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), `{"schema_version":1,"id":"`+legID+`"}`) + }, + want: legPartial, + wantReason: "sentinel records no exit", + }, + { + name: "sentinel records a different leg", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), + `{"schema_version":1,"id":"ingest-cold-ds-c0-run2","exit_code":0}`) + }, + want: legPartial, + wantReason: "sentinel does not match this leg", + }, + { + name: "sentinel from an unknown schema version", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), `{"schema_version":0,"id":"`+legID+`","exit_code":0}`) + }, + want: legPartial, + wantReason: "sentinel does not match this leg", + }, + { + name: "corrupt sentinel", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, legSentinelName), `{"schema_version":1,`) + }, + want: legPartial, + }, + { + name: "bash-era bundle: invocation.json and driver.csv, no error", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, "invocation.json"), + `{"schemaVersion":1,"command":"bench-ingest cold","startedAt":"2026-07-01T00:00:00Z","finishedAt":"2026-07-01T00:10:00Z"}`) + mustWrite(t, filepath.Join(dir, "driver.csv"), "stage,wall\n") + }, + want: legComplete, + }, + { + name: "bash-era bundle: invocation.json records an error", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, "invocation.json"), `{"schemaVersion":1,"error":"datastore unreachable"}`) + mustWrite(t, filepath.Join(dir, "driver.csv"), "stage,wall\n") + }, + want: legFailedEarlier, + wantReason: "datastore unreachable", + }, + { + name: "invocation.json without driver.csv", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, "invocation.json"), `{"schemaVersion":1}`) + }, + want: legPartial, + }, + { + name: "driver.csv without invocation.json", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, "driver.csv"), "stage,wall\n") + }, + want: legPartial, + }, + { + name: "unreadable invocation.json", + setup: func(t *testing.T, dir string) { + mustWrite(t, filepath.Join(dir, "invocation.json"), `{"schemaVersion":`) + mustWrite(t, filepath.Join(dir, "driver.csv"), "stage,wall\n") + }, + want: legPartial, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := filepath.Join(t.TempDir(), legID) + mustMkdir(t, dir) + tc.setup(t, dir) + got := classifyLegDir(dir, legID) + if got.kind != tc.want { + t.Errorf("kind = %v, want %v", got.kind, tc.want) + } + if got.reason != tc.wantReason { + t.Errorf("reason = %q, want %q", got.reason, tc.wantReason) + } + }) + } +} + +// --- executor mechanics ----------------------------------------------------- + +func TestExecuteHappyPath(t *testing.T) { + tmp := t.TempDir() + scratch := filepath.Join(tmp, "scratch") + post := filepath.Join(tmp, "post") + mustWrite(t, filepath.Join(scratch, "stale"), "left over from the previous rep") + mustWrite(t, filepath.Join(post, "hot.db"), "x") + outA := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run1") + outB := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run2") + + a := shLeg("a", outA, `test ! -e "$2/stale" && : > "$1/driver.csv"`, scratch) + a.PreClean = []string{scratch} + a.PostClean = []string{post} + b := shLeg("b", outB, `: > "$1/driver.csv"`) + b.Needs = []string{"a"} + p := &plan.Plan{Steps: []plan.Step{a, b}} + + got := walk(t, p, Options{}) + if got.err != nil { + t.Fatalf("Execute: %v\nlog:\n%s", got.err, got.log) + } + got.assertStatuses(t, StatusOK, StatusOK) + + for _, out := range []string{outA, outB} { + assertExists(t, filepath.Join(out, "driver.csv")) + s := readSentinel(t, out) + if s.SchemaVersion != LegSchemaVersion || s.ExitCode != 0 || s.Error != "" { + t.Errorf("%s sentinel = %+v, want schema %d, exit 0, no error", out, s, LegSchemaVersion) + } + if s.DurationNS <= 0 { + t.Errorf("%s duration_ns = %d, want > 0", out, s.DurationNS) + } + if s.StartedAt == "" || s.FinishedAt == "" { + t.Errorf("%s sentinel is missing timestamps: %+v", out, s) + } + } + if s := readSentinel(t, outA); !slices.Equal(s.Argv, p.Steps[0].Argv[0]) { + t.Errorf("sentinel argv = %v, want %v", s.Argv, p.Steps[0].Argv[0]) + } + // PostClean runs only on success, and it ran: the hot DB is gone. + assertGone(t, post) + got.assertLogHas(t, " $ rm -rf "+scratch) +} + +func TestExecuteKeepGoing(t *testing.T) { + tmp := t.TempDir() + out := func(id string) string { return filepath.Join(tmp, "res", id) } + + fail := shLeg("fail", out("fail"), `exit 1`) + dep := shLeg("dep", out("dep"), `: > "$1/driver.csv"`) + dep.Needs = []string{"fail"} + dep2 := shLeg("dep2", out("dep2"), `: > "$1/driver.csv"`) + dep2.Needs = []string{"dep"} + indep := shLeg("indep", out("indep"), `: > "$1/driver.csv"`) + p := &plan.Plan{Steps: []plan.Step{fail, dep, dep2, indep}} + + got := walk(t, p, Options{}) + if got.err == nil { + t.Fatalf("Execute returned nil error after a failed leg\nlog:\n%s", got.log) + } + if want := "1 step(s) failed"; got.err.Error() != want { + t.Errorf("Execute error = %q, want %q", got.err, want) + } + got.assertStatuses(t, StatusFailed, StatusSkipped, StatusSkipped, StatusOK) + + got.assertLogHas(t, "== campaign summary: 1 failed, 2 skipped") + got.assertLogHas(t, "== failed: fail (exit status 1)") + got.assertLogHas(t, "== skipped: dep (needs fail)") + // dep2 needs dep, which was skipped rather than failed: the propagation is + // transitive without the executor walking the graph. + got.assertLogHas(t, "== skipped: dep2 (needs dep)") + + s := readSentinel(t, out("fail")) + if s.ExitCode != 1 || s.Error != "exit status 1" { + t.Errorf("failed leg sentinel = %+v, want exit_code 1 and error \"exit status 1\"", s) + } + assertExists(t, filepath.Join(out("indep"), legSentinelName)) + assertGone(t, out("dep")) +} + +func TestExecuteFailFast(t *testing.T) { + tmp := t.TempDir() + out := func(id string) string { return filepath.Join(tmp, "res", id) } + fail := shLeg("fail", out("fail"), `exit 3`) + indep := shLeg("indep", out("indep"), `: > "$1/driver.csv"`) + p := &plan.Plan{Steps: []plan.Step{fail, indep}} + + got := walk(t, p, Options{FailFast: true}) + if got.err == nil { + t.Fatalf("Execute returned nil error under fail-fast\nlog:\n%s", got.log) + } + got.assertStatuses(t, StatusFailed) + // The walk stopped: the later step has no result and never ran. + assertGone(t, out("indep")) + if s := readSentinel(t, out("fail")); s.ExitCode != 3 { + t.Errorf("sentinel exit_code = %d, want 3", s.ExitCode) + } +} + +func TestExecuteResume(t *testing.T) { + tmp := t.TempDir() + ran := filepath.Join(tmp, "ran.txt") + outDone := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run1") + outPartial := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run2") + + // An earlier session finished run1 and was killed during run2, which left a + // half-written directory with no manifests in it. + mustWrite(t, filepath.Join(outDone, legSentinelName), `{"schema_version":1,"id":"done","exit_code":0}`) + mustWrite(t, filepath.Join(outDone, "driver.csv"), "stage,wall\n") + mustWrite(t, filepath.Join(outPartial, "driver.csv.tmp"), "half a row") + + script := `echo "$2" >> "$3"; : > "$1/driver.csv"` + p := &plan.Plan{Steps: []plan.Step{ + shLeg("done", outDone, script, "done", ran), + shLeg("partial", outPartial, script, "partial", ran), + }} + + got := walk(t, p, Options{Resume: true}) + if got.err != nil { + t.Fatalf("Execute: %v\nlog:\n%s", got.err, got.log) + } + got.assertStatuses(t, StatusResumed, StatusOK) + got.assertLogHas(t, "resume: ingest-cold-ds-c0-run1 already complete — skipping") + got.assertLogHas(t, "resume: ingest-cold-ds-c0-run2 is a partial leg — wiping and re-running") + + b, err := os.ReadFile(ran) + if err != nil { + t.Fatalf("read %s: %v", ran, err) + } + if got := strings.Fields(string(b)); !slices.Equal(got, []string{"partial"}) { + t.Errorf("legs that ran = %v, want only [partial]", got) + } + // The partial directory was wiped, not merged into. + assertGone(t, filepath.Join(outPartial, "driver.csv.tmp")) + assertExists(t, filepath.Join(outPartial, legSentinelName)) +} + +func TestExecuteResumeAfterRecordedFailure(t *testing.T) { + tmp := t.TempDir() + out := filepath.Join(tmp, "res", "query-cold-ds-c0-run1") + mustWrite(t, filepath.Join(out, legSentinelName), `{"schema_version":1,"id":"leg","exit_code":1,"error":"exit status 1"}`) + + p := &plan.Plan{Steps: []plan.Step{shLeg("leg", out, `: > "$1/driver.csv"`)}} + got := walk(t, p, Options{Resume: true}) + if got.err != nil { + t.Fatalf("Execute: %v\nlog:\n%s", got.err, got.log) + } + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "resume: query-cold-ds-c0-run1 failed in an earlier session (exit status 1) — wiping and re-running") + if s := readSentinel(t, out); s.ExitCode != 0 || s.Error != "" { + t.Errorf("sentinel after re-run = %+v, want a clean success", s) + } +} + +// A sentinel that does not name this leg proves nothing about the directory it +// sits in — a copied or hand-made leg.json must not skip a leg that never ran. +func TestExecuteResumeRejectsForeignSentinel(t *testing.T) { + tmp := t.TempDir() + out := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run1") + mustWrite(t, filepath.Join(out, legSentinelName), `{"schema_version":1,"id":"ingest-cold-ds-c0-run2","exit_code":0}`) + ran := filepath.Join(tmp, "ran.txt") + + p := &plan.Plan{Steps: []plan.Step{shLeg("ingest-cold-ds-c0-run1", out, `echo ran >> "$2"; : > "$1/driver.csv"`, ran)}} + got := walk(t, p, Options{Resume: true}) + if got.err != nil { + t.Fatalf("Execute: %v\nlog:\n%s", got.err, got.log) + } + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "resume: ingest-cold-ds-c0-run1 is a partial leg (sentinel does not match this leg) — wiping and re-running") + assertExists(t, ran) + if s := readSentinel(t, out); s.ID != "ingest-cold-ds-c0-run1" { + t.Errorf("sentinel after re-run = %+v, want this leg's id", s) + } +} + +func TestExecuteWithoutResumeIgnoresExistingOutput(t *testing.T) { + tmp := t.TempDir() + out := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run1") + mustWrite(t, filepath.Join(out, legSentinelName), `{"schema_version":1,"exit_code":0}`) + ran := filepath.Join(tmp, "ran.txt") + + p := &plan.Plan{Steps: []plan.Step{shLeg("leg", out, `echo ran >> "$2"`, ran)}} + got := walk(t, p, Options{}) + got.assertStatuses(t, StatusOK) + assertExists(t, ran) + if strings.Contains(got.log, "resume:") { + t.Errorf("a non-resume walk inspected existing output\nlog:\n%s", got.log) + } +} + +func TestExecuteLegEnv(t *testing.T) { + tmp := t.TempDir() + out := filepath.Join(tmp, "res", "golden-ds-c0") + step := shLeg("env", out, `test "$FOO" = bar`) + step.Env = map[string]string{"FOO": "bar"} + + got := walk(t, &plan.Plan{Steps: []plan.Step{step}}, Options{}) + if got.err != nil { + t.Fatalf("Execute: %v\nlog:\n%s", got.err, got.log) + } + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, " $ env FOO=bar /bin/sh -c") +} + +func TestExecuteBuild(t *testing.T) { + t.Run("existing binary skips the build", func(t *testing.T) { + tmp := t.TempDir() + bin := filepath.Join(tmp, "bin", "stellar-rpc-deadbeef") + mustWrite(t, bin, "#!/bin/sh\n") + if err := os.Chmod(bin, 0o755); err != nil { + t.Fatalf("chmod: %v", err) + } + p := &plan.Plan{Bin: bin, Steps: []plan.Step{{ + ID: "build", Kind: plan.KindBuild, Argv: [][]string{{"/bin/sh", "-c", "exit 1"}}, + }}} + got := walk(t, p, Options{}) + got.assertStatuses(t, StatusOK) + got.assertLogHas(t, "binary "+bin+" already built — skipping build") + if strings.Contains(got.log, " $ ") { + t.Errorf("build ran a command despite the binary being there\nlog:\n%s", got.log) + } + }) + + t.Run("missing binary runs every command in order", func(t *testing.T) { + tmp := t.TempDir() + order := filepath.Join(tmp, "order.txt") + p := &plan.Plan{Bin: filepath.Join(tmp, "bin", "stellar-rpc-deadbeef"), Steps: []plan.Step{{ + ID: "build", Kind: plan.KindBuild, Argv: [][]string{ + {"/bin/sh", "-c", `echo checkout >> "$1"`, "sh", order}, + {"/bin/sh", "-c", `echo make >> "$1"`, "sh", order}, + }, + }}} + got := walk(t, p, Options{}) + got.assertStatuses(t, StatusOK) + b, err := os.ReadFile(order) + if err != nil { + t.Fatalf("read %s: %v", order, err) + } + if want := []string{"checkout", "make"}; !slices.Equal(strings.Fields(string(b)), want) { + t.Errorf("commands ran as %q, want %v", b, want) + } + }) + + t.Run("a failed command stops the rest of the step", func(t *testing.T) { + tmp := t.TempDir() + marker := filepath.Join(tmp, "second.txt") + p := &plan.Plan{Bin: filepath.Join(tmp, "bin", "stellar-rpc-deadbeef"), Steps: []plan.Step{{ + ID: "build", Kind: plan.KindBuild, Argv: [][]string{ + {"/bin/sh", "-c", "exit 1"}, + {"/bin/sh", "-c", `: > "$1"`, "sh", marker}, + }, + }}} + got := walk(t, p, Options{}) + got.assertStatuses(t, StatusFailed) + assertGone(t, marker) + }) +} + +// TestExecuteSkipsTheEpilogue pins the division of labour: the tarball and the +// publish are in the plan, but Execute leaves them to the run wiring, which +// makes the tarball only after the final provenance writes. +func TestExecuteSkipsTheEpilogue(t *testing.T) { + tmp := t.TempDir() + marker := filepath.Join(tmp, "tarball.txt") + tarball := plan.Step{ + ID: "tarball", Kind: plan.KindTarball, + Argv: [][]string{{"/bin/sh", "-c", `: > "$1"`, "sh", marker}}, + } + p := &plan.Plan{Steps: []plan.Step{ + shLeg("leg", filepath.Join(tmp, "res", "leg"), `: > "$1/driver.csv"`), + tarball, + {ID: "publish", Kind: plan.KindPublish, Argv: [][]string{}, PublishURI: "gs://bucket/runs", Needs: []string{"tarball"}}, + }} + + got := walk(t, p, Options{}) + if got.err != nil { + t.Fatalf("Execute: %v\nlog:\n%s", got.err, got.log) + } + // Only the leg has a result, and no tar ran. + got.assertStatuses(t, StatusOK) + assertGone(t, marker) + + // The wiring runs the same step itself, out of the same plan. + if err := RunStep(tarball, io.Discard); err != nil { + t.Fatalf("RunStep(tarball): %v", err) + } + assertExists(t, marker) +} + +func TestOnStepDoneFiresForExecutedSteps(t *testing.T) { + tmp := t.TempDir() + bin := filepath.Join(tmp, "bin", "stellar-rpc-deadbeef") + p := &plan.Plan{Bin: bin, Steps: []plan.Step{ + {ID: "build", Kind: plan.KindBuild, Argv: [][]string{{"/bin/sh", "-c", `mkdir -p "$(dirname "$1")" && : > "$1"`, "sh", bin}}}, + shLeg("fail", filepath.Join(tmp, "res", "fail"), `exit 1`), + shLeg("dep", filepath.Join(tmp, "res", "dep"), `: > "$1/driver.csv"`), + }} + p.Steps[2].Needs = []string{"fail"} + + var seen []string + var buf bytes.Buffer + if _, err := Execute(p, Options{ + Output: &buf, + OnStepDone: func(s plan.Step, r StepResult) { + seen = append(seen, s.ID+"="+string(r.Status)) + }, + }); err == nil { + t.Fatalf("Execute returned nil error after a failed leg\nlog:\n%s", buf.String()) + } + // The skipped step is the one exception: it never ran, so nothing is done. + if want := []string{"build=ok", "fail=failed"}; !slices.Equal(seen, want) { + t.Errorf("OnStepDone saw %v, want %v", seen, want) + } +} + +func TestExecuteLegWithMissingBinaryIsFailedWithASentinel(t *testing.T) { + tmp := t.TempDir() + out := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run1") + p := &plan.Plan{Steps: []plan.Step{{ + ID: "leg", Kind: plan.KindLeg, Timed: true, OutDir: out, + Argv: [][]string{{filepath.Join(tmp, "bin", "stellar-rpc-deadbeef"), "bench-ingest", "cold"}}, + }}} + got := walk(t, p, Options{}) + got.assertStatuses(t, StatusFailed) + // The binary never started, so it wrote no invocation.json — the sentinel + // is the only record that this leg was attempted, which is exactly why the + // executor creates the out dir itself. + s := readSentinel(t, out) + if s.ExitCode != -1 || s.Error == "" { + t.Errorf("sentinel = %+v, want exit_code -1 and an error", s) + } +} + +func TestExecuteAllOKPrintsNoSummary(t *testing.T) { + tmp := t.TempDir() + p := &plan.Plan{Steps: []plan.Step{shLeg("a", filepath.Join(tmp, "a"), `: > "$1/driver.csv"`)}} + got := walk(t, p, Options{}) + if got.err != nil { + t.Fatalf("Execute: %v", got.err) + } + if strings.Contains(got.log, "campaign summary") { + t.Errorf("an all-ok campaign printed a summary\nlog:\n%s", got.log) + } +} + +// --- lock ------------------------------------------------------------------- + +func TestAcquireLock(t *testing.T) { + benchRoot := filepath.Join(t.TempDir(), "bench") + release, err := AcquireLock(benchRoot) + if err != nil { + t.Fatalf("first AcquireLock: %v", err) + } + if _, err := AcquireLock(benchRoot); err == nil { + t.Fatal("second AcquireLock succeeded while the lock was held") + } else if want := "another campaign is already running on this BENCH_ROOT"; !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err, want) + } + + release() + release2, err := AcquireLock(benchRoot) + if err != nil { + t.Fatalf("AcquireLock after release: %v", err) + } + release2() + // The lock file outlives the lock: deleting it would race the next campaign. + assertExists(t, filepath.Join(benchRoot, lockName)) +} + +// --- logging ------------------------------------------------------------------ + +func TestNotef(t *testing.T) { + var buf bytes.Buffer + Notef(&buf, "build %s → %s", "abc1234", "/bench/bin/stellar-rpc-abc1234") + want := regexp.MustCompile(`^== \[\d{2}:\d{2}:\d{2}\] build abc1234 → /bench/bin/stellar-rpc-abc1234\n$`) + if !want.MatchString(buf.String()) { + t.Errorf("Notef wrote %q, want it to match %s", buf.String(), want) + } +} + +func TestOpenCampaignLogAppends(t *testing.T) { + dir := t.TempDir() + for _, session := range []string{"first\n", "second\n"} { + f, err := OpenCampaignLog(dir) + if err != nil { + t.Fatalf("OpenCampaignLog: %v", err) + } + if _, err := f.WriteString(session); err != nil { + t.Fatalf("write: %v", err) + } + f.Close() + } + b, err := os.ReadFile(filepath.Join(dir, campaignLogName)) + if err != nil { + t.Fatalf("read log: %v", err) + } + if string(b) != "first\nsecond\n" { + t.Errorf("campaign.log = %q, want both sessions in order", b) + } +} + +// TestLegSentinelJSONShape pins the wire format: the sentinel is read by resume +// and by anything inspecting a bundle, so its keys are a contract. +func TestLegSentinelJSONShape(t *testing.T) { + tmp := t.TempDir() + out := filepath.Join(tmp, "res", "ingest-cold-ds-c0-run1") + p := &plan.Plan{Steps: []plan.Step{shLeg("ingest-cold-ds-c0-run1", out, `exit 0`)}} + if got := walk(t, p, Options{}); got.err != nil { + t.Fatalf("Execute: %v\nlog:\n%s", got.err, got.log) + } + b, err := os.ReadFile(filepath.Join(out, legSentinelName)) + if err != nil { + t.Fatalf("read sentinel: %v", err) + } + var raw map[string]any + if err := json.Unmarshal(b, &raw); err != nil { + t.Fatalf("unmarshal sentinel: %v", err) + } + want := []string{"argv", "duration_ns", "exit_code", "finished_at", "id", "schema_version", "started_at"} + got := make([]string, 0, len(raw)) + for k := range raw { + got = append(got, k) + } + slices.Sort(got) + if !slices.Equal(got, want) { + t.Errorf("sentinel keys = %v, want %v", got, want) + } + // error is omitted on success, and only then. + if _, ok := raw["error"]; ok { + t.Errorf("a successful leg recorded an error: %s", b) + } +} diff --git a/runner/internal/run/source.go b/runner/internal/run/source.go new file mode 100644 index 0000000..88036b3 --- /dev/null +++ b/runner/internal/run/source.go @@ -0,0 +1,57 @@ +package run + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// EnsureSrc converges the persistent build clone at src onto repo: clone once, +// then per campaign point origin at repo (it may have changed since the clone +// was made), fetch its branches and tags, and hard-reset. Gitignored build +// caches (cargo target/, Go cache) survive the reset — clean -fd, deliberately +// no -x — so rebuilding a nearby commit is incremental. repo itself is never +// modified. +func EnsureSrc(src, repo string, out io.Writer) error { + if _, err := os.Stat(filepath.Join(src, ".git")); err != nil { + if err := runCommand([]string{"git", "clone", repo, src}, nil, out); err != nil { + return fmt.Errorf("clone %s into %s: %w", repo, src, err) + } + } + for _, argv := range [][]string{ + {"git", "-C", src, "remote", "set-url", "origin", repo}, + {"git", "-C", src, "fetch", "-q", "--prune", "origin", + "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*"}, + {"git", "-C", src, "reset", "-q", "--hard"}, + {"git", "-C", src, "clean", "-qfd"}, + } { + if err := runCommand(argv, nil, out); err != nil { + return fmt.Errorf("%s: %w", strings.Join(argv, " "), err) + } + } + return nil +} + +// ResolveRef returns the full commit ref resolves to inside src. +// Remote-tracking branches are tried first so a stale local ref never shadows +// the fetched branch tip; the fallback covers tags and raw hashes. +func ResolveRef(src, ref string) (string, error) { + // Without this guard git would search upwards from src and answer out of + // whatever repository happens to contain it. + if _, err := os.Stat(filepath.Join(src, ".git")); err != nil { + return "", fmt.Errorf("no build clone at %s", src) + } + for _, rev := range []string{"refs/remotes/origin/" + ref + "^{commit}", ref + "^{commit}"} { + out, err := exec.Command("git", "-C", src, "rev-parse", "--verify", "--quiet", rev).Output() + if err != nil { + continue + } + if sha := strings.TrimSpace(string(out)); len(sha) >= 8 { + return sha, nil + } + } + return "", fmt.Errorf("ref '%s' does not resolve to a commit in %s", ref, src) +} diff --git a/runner/internal/run/source_test.go b/runner/internal/run/source_test.go new file mode 100644 index 0000000..6b5e174 --- /dev/null +++ b/runner/internal/run/source_test.go @@ -0,0 +1,211 @@ +package run + +import ( + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// git runs a git command in dir and fails the test if it does not succeed. +// Identity and hooks come from the flags, not the machine: the test must behave +// the same on a devbox with a global gitconfig and on a bare CI runner. +func git(t *testing.T, dir string, args ...string) string { + t.Helper() + argv := append([]string{ + "-c", "user.name=bench", "-c", "user.email=bench@example.com", + "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main", + }, args...) + cmd := exec.Command("git", argv...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s in %s: %v\n%s", strings.Join(args, " "), dir, err, out) + } + return strings.TrimSpace(string(out)) +} + +// commit writes a file and commits it, returning the new commit's sha. +func commit(t *testing.T, repo, name, body string) string { + t.Helper() + mustWrite(t, filepath.Join(repo, name), body) + git(t, repo, "add", "-A") + git(t, repo, "commit", "-q", "-m", "add "+name) + return git(t, repo, "rev-parse", "HEAD") +} + +// originRepo is a local stellar-rpc stand-in: one commit, one gitignore, on +// branch main. +func originRepo(t *testing.T) (dir, head string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + dir = filepath.Join(t.TempDir(), "origin") + mustMkdir(t, dir) + git(t, dir, "init", "-q") + mustWrite(t, filepath.Join(dir, ".gitignore"), "target/\n") + return dir, commit(t, dir, "README", "first\n") +} + +func TestEnsureSrcClonesThenFetches(t *testing.T) { + origin, first := originRepo(t) + src := filepath.Join(t.TempDir(), "src") + + if err := EnsureSrc(src, origin, io.Discard); err != nil { + t.Fatalf("first EnsureSrc: %v", err) + } + assertExists(t, filepath.Join(src, ".git")) + got, err := ResolveRef(src, "main") + if err != nil { + t.Fatalf("ResolveRef after clone: %v", err) + } + if got != first { + t.Errorf("main = %s, want the origin head %s", got, first) + } + + // The origin moves on, as feature/full-history does between campaigns. + second := commit(t, origin, "NOTES", "second\n") + if err := EnsureSrc(src, origin, io.Discard); err != nil { + t.Fatalf("second EnsureSrc: %v", err) + } + got, err = ResolveRef(src, "main") + if err != nil { + t.Fatalf("ResolveRef after fetch: %v", err) + } + // This is the stale-local-ref case: the clone's own main still points at + // the first commit (a fetch updates only the remote-tracking refs), and + // resolving must follow origin/main, not it. + if local := git(t, src, "rev-parse", "refs/heads/main"); local != first { + t.Fatalf("test premise broken: local main = %s, want the stale %s", local, first) + } + if got != second { + t.Errorf("main = %s, want the fetched tip %s (a stale local ref shadowed it)", got, second) + } +} + +func TestEnsureSrcKeepsBuildCaches(t *testing.T) { + origin, _ := originRepo(t) + src := filepath.Join(t.TempDir(), "src") + if err := EnsureSrc(src, origin, io.Discard); err != nil { + t.Fatalf("EnsureSrc: %v", err) + } + + // target/ is gitignored (a cargo build cache); scratch.go is merely + // untracked. The reset must keep the first and drop the second, which is + // what clean -fd without -x buys: rebuilding a nearby commit stays + // incremental. + mustWrite(t, filepath.Join(src, "target", "libpreflight.a"), "cached") + mustWrite(t, filepath.Join(src, "scratch.go"), "package main") + mustWrite(t, filepath.Join(src, "README"), "locally edited\n") + + if err := EnsureSrc(src, origin, io.Discard); err != nil { + t.Fatalf("second EnsureSrc: %v", err) + } + assertExists(t, filepath.Join(src, "target", "libpreflight.a")) + assertGone(t, filepath.Join(src, "scratch.go")) + if b, err := os.ReadFile(filepath.Join(src, "README")); err != nil || string(b) != "first\n" { + t.Errorf("README = %q (err %v), want the reset content", b, err) + } +} + +func TestEnsureSrcRepointsOrigin(t *testing.T) { + first, _ := originRepo(t) + src := filepath.Join(t.TempDir(), "src") + if err := EnsureSrc(src, first, io.Discard); err != nil { + t.Fatalf("EnsureSrc: %v", err) + } + + // A campaign later points repo at a different checkout — a fork, or the + // operator's own work in progress. The clone follows it. + second, secondHead := originRepo(t) + git(t, second, "checkout", "-q", "-b", "feature/full-history") + secondHead = commit(t, second, "FEATURE", "wip\n") + if err := EnsureSrc(src, second, io.Discard); err != nil { + t.Fatalf("EnsureSrc onto the second origin: %v", err) + } + got, err := ResolveRef(src, "feature/full-history") + if err != nil { + t.Fatalf("ResolveRef: %v", err) + } + if got != secondHead { + t.Errorf("feature/full-history = %s, want %s from the new origin", got, secondHead) + } +} + +func TestEnsureSrcPrintsItsCommands(t *testing.T) { + origin, _ := originRepo(t) + src := filepath.Join(t.TempDir(), "src") + var buf strings.Builder + if err := EnsureSrc(src, origin, &buf); err != nil { + t.Fatalf("EnsureSrc: %v", err) + } + for _, want := range []string{ + " $ git clone " + origin + " " + src, + " $ git -C " + src + " remote set-url origin " + origin, + " $ git -C " + src + " fetch -q --prune origin +refs/heads/*:refs/remotes/origin/* +refs/tags/*:refs/tags/*", + " $ git -C " + src + " reset -q --hard", + " $ git -C " + src + " clean -qfd", + } { + if !strings.Contains(buf.String(), want) { + t.Errorf("log missing %q, got:\n%s", want, buf.String()) + } + } +} + +func TestEnsureSrcFailsOnAnUnreachableRepo(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + tmp := t.TempDir() + err := EnsureSrc(filepath.Join(tmp, "src"), filepath.Join(tmp, "no-such-repo"), io.Discard) + if err == nil { + t.Fatal("EnsureSrc succeeded on a repo that does not exist") + } + if !strings.Contains(err.Error(), "clone") { + t.Errorf("error = %v, want it to name the clone", err) + } +} + +func TestResolveRef(t *testing.T) { + origin, head := originRepo(t) + git(t, origin, "tag", "v1.2.3") + src := filepath.Join(t.TempDir(), "src") + if err := EnsureSrc(src, origin, io.Discard); err != nil { + t.Fatalf("EnsureSrc: %v", err) + } + + for _, tc := range []struct{ name, ref string }{ + {"branch", "main"}, + {"tag", "v1.2.3"}, + {"full sha", head}, + {"short sha", head[:8]}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := ResolveRef(src, tc.ref) + if err != nil { + t.Fatalf("ResolveRef(%s): %v", tc.ref, err) + } + if got != head { + t.Errorf("ResolveRef(%s) = %s, want %s", tc.ref, got, head) + } + }) + } + + t.Run("unknown ref", func(t *testing.T) { + if got, err := ResolveRef(src, "no/such/branch"); err == nil { + t.Errorf("ResolveRef = %s, want an error", got) + } + }) + + t.Run("no clone at all", func(t *testing.T) { + // Not merely "git fails here": without the .git guard, git would search + // upwards and answer out of whatever repository contains the path. + if got, err := ResolveRef(filepath.Join(src, "cmd"), "main"); err == nil { + t.Errorf("ResolveRef = %s, want an error naming the missing clone", got) + } + }) +} diff --git a/scripts/ingest.sh b/scripts/ingest.sh index bc5a00f..4faefde 100755 --- a/scripts/ingest.sh +++ b/scripts/ingest.sh @@ -39,17 +39,19 @@ # --dataset-kind synthetic --local # # CI NOTE -# The future CI ingest workflow (.github/workflows/ingest.yml) is expected to -# call this same script once bucket credentials / OIDC (the dev-hubble WIF -# setup) exist — the script is the single source of truth for the fetch → -# convert → PR flow, invoked locally today and from CI later. +# .github/workflows/ingest.yml calls this script in full mode — it passes the +# dispatched gs:// path and --dataset-kind and does nothing else itself. So +# this script is the single source of truth for the fetch → convert → PR flow +# on both sides, and a CI-ingested run is identical to a locally ingested one. +# That workflow still can't run until bucket credentials / OIDC (the +# dev-hubble WIF setup) exist; it fails early and loudly until then. set -euo pipefail # ------------------------------------------------------------------ helpers PROG="$(basename "$0")" die() { echo "$PROG: error: $*" >&2; exit 1; } -usage() { sed -n '3,45p' "$0" | sed 's/^# \{0,1\}//'; } +usage() { sed -n '3,47p' "$0" | sed 's/^# \{0,1\}//'; } TMPDIRS=() cleanup() {