diff --git a/Cargo.lock b/Cargo.lock index 943c86de..63e07620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,6 +78,10 @@ dependencies = [ "tempfile", ] +[[package]] +name = "cargo-ox-check" +version = "0.1.0" + [[package]] name = "cargo-platform" version = "0.1.9" diff --git a/crates/cargo_ox_check/Cargo.toml b/crates/cargo_ox_check/Cargo.toml new file mode 100644 index 00000000..61b019c0 --- /dev/null +++ b/crates/cargo_ox_check/Cargo.toml @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[package] +name = "cargo-ox-check" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors = ["The Oxidizer Project Authors"] +description = "Opinionated, unified Rust build/CI scaffolding for GitHub Actions and Azure DevOps" +readme = "README.md" +repository = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo_ox_check" +homepage = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo_ox_check" +keywords = ["cargo", "subcommand", "ci", "just", "github-actions"] +categories = ["development-tools::cargo-plugins"] + +[[bin]] +name = "cargo-ox-check" +path = "src/main.rs" + +[dependencies] + +[lints] +workspace = true diff --git a/crates/cargo_ox_check/docs/design/ado.md b/crates/cargo_ox_check/docs/design/ado.md new file mode 100644 index 00000000..53c25acf --- /dev/null +++ b/crates/cargo_ox_check/docs/design/ado.md @@ -0,0 +1,435 @@ +# Azure DevOps Pipelines Integration + +This document describes what `cargo ox-check update --backend ado` emits for Azure DevOps +Pipelines, and how a repo wires those files into its own CI. + +ox-check emits three layers, all owned by ox-check with the standard owned-file flow (edit → +dirty → `.ox-check-proposed` sibling on next update). The split is by what users actually +need to change: + +1. **Root pipelines** (`ox-check-pr.yml`, `ox-check-nightly.yml` at `.pipelines/`). Triggers, + runner pool, secret variable groups, and the optional `extends:` to a compliance + template (1ESPT/SubstratePT/CloudBuild) live here. ox-check ships an opinionated default; + users who need to customize edit in place and accept the proposal-on-update flow. + ox-check's emitted root pipelines contain **no** references to compliance harnesses — + wrapping with 1ESPT is purely a user-side edit. +2. **Stages templates** (`ox-check/pr.yml`, `ox-check/nightly.yml`), containing the impact job + and the per-group jobs with all the dependency / output-variable plumbing. These + change when ox-check's groups or impact wiring evolve; most users won't ever edit them. +3. **Per-group step templates** (`ox-check/steps/*.yml`). Each is a multi-step template that + runs setup + the matching `just ox-check--` recipe. + +See also: + +- [design.md §6](./design.md#6-repo-layout) for the file-category model. +- [checks.md](./checks.md) for what each group runs. +- [local.md](./local.md) for the `just` recipes the templates invoke. +- [github.md](./github.md) for the GitHub Actions counterpart. + +## 1. Why three layers + +- **Frequently-changing wiring** (group set, impact computation, fan-out, output-variable + plumbing) lives in the stages template. Updates apply automatically; users don't have + to merge changes. +- **Per-repo customization** (triggers, runner pool, compliance harness, secrets) lives + in the root pipeline. Users who customize it accept the cost of merging the + `.ox-check-proposed` sibling when the ox-check defaults evolve — which is rare, since the + root pipeline is intentionally minimal. +- **Compliance composition** is purely a user concern. ox-check's stages template is plain + ADO YAML; 1ESPT/SubstratePT/CloudBuild composition happens in the user's root pipeline + by way of `extends:` and `parameters.stages`. + +## 2. Emitted artifacts + +```text +.pipelines/ +├── ox-check-pr.yml owned (root PR pipeline) +├── ox-check-nightly.yml owned (root nightly pipeline) +└── ox-check/ + ├── pr.yml owned (PR-tier stages template) + ├── nightly.yml owned (nightly-tier stages template) + └── steps/ + ├── setup.yml owned (install just + catalog tools) + ├── impact.yml owned (cargo-delta impact step; omitted if .delta.toml disabled) + ├── pr-fast.yml owned (one step template per group) + ├── pr-test.yml owned + ├── pr-mutants.yml owned + ├── nightly-test.yml owned + ├── nightly-advisories.yml owned + ├── nightly-runtime.yml owned + └── nightly-exhaustive.yml owned +``` + +All files are regular owned files (carry an `ox-check-checksum` first line, governed by +[updates.md §5](./updates.md#5-the-decision-algorithm)). Users who customize the root +pipeline take ownership through the standard dirty-file flow. + +## 3. Root pipelines + +The default `ox-check-pr.yml` ox-check emits is the minimum needed to run ox-check's stages +template: + +```yaml +# .pipelines/ox-check-pr.yml +trigger: none # PR validation only; configure in branch policies +pr: + branches: + include: [main] + +stages: +- template: ox-check/pr.yml + parameters: + linuxPool: { vmImage: ubuntu-latest } + windowsPool: { vmImage: windows-latest } +``` + +The nightly root pipeline adds a schedule: + +```yaml +# .pipelines/ox-check-nightly.yml +schedules: +- cron: "0 6 * * *" + displayName: ox-check nightly + branches: + include: [main] + always: true + +stages: +- template: ox-check/nightly.yml + parameters: + linuxPool: { vmImage: ubuntu-latest } + windowsPool: { vmImage: windows-latest } +``` + +For an internal/compliance pipeline, the user replaces their root pipeline with one that +extends 1ESPT/SubstratePT and passes ox-check's stages template as the stages parameter, +overriding the pools with the team's 1ESPT pools: + +```yaml +# .pipelines/ox-check-pr.yml (user-edited for 1ESPT) +trigger: none +pr: { branches: { include: [main] } } + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: { name: } + stages: + - template: /.pipelines/ox-check/pr.yml@self + parameters: + linuxPool: { name: } + windowsPool: { name: } +``` + +The `extends:` keyword, the resources block, and the pool definitions are entirely the +user's business. ox-check's `pr.yml` is a plain stages template that drops in unchanged. To +trim the matrix (run on Linux only), the user sets `windowsPool` to an empty object +`{}` — `pr.yml` treats an empty pool as "skip this OS." Same shape for adding macOS via +a future `macosPool` parameter (deferred to v2; ADO macOS adoption is rare in the +surveyed repos). + +## 4. Owned stages templates + +The `pr.yml` stages template is where the wiring lives. Every per-group step template +takes the same three impact-exclude parameters unconditionally; which ones a group's +checks actually consume is the catalog's concern, not the wiring layer's. This means +moving a check between groups (e.g. `clippy` from `pr-fast` to `nightly-advisories`) +never changes the stages template. + +Approximate shape (ox-check writes this verbatim; users never edit it): + +```yaml +# .pipelines/ox-check/pr.yml (owned by cargo-ox-check) +parameters: +- name: linuxPool + type: object + default: { vmImage: ubuntu-latest } +- name: windowsPool + type: object + default: { vmImage: windows-latest } + +stages: +- stage: OX_CHECK_pr + jobs: + - job: impact + pool: ${{ parameters.linuxPool }} + steps: + - template: steps/impact.yml + parameters: + baseRef: $(System.PullRequest.TargetBranch) + - ${{ each group in ['pr_fast', 'pr_test_linux', 'pr_mutants'] }}: + - job: ${{ group }} + dependsOn: impact + pool: ${{ parameters.linuxPool }} + variables: + excludeNotModified: $[ dependencies.impact.outputs['delta.exclude_not_modified'] ] + excludeNotAffected: $[ dependencies.impact.outputs['delta.exclude_not_affected'] ] + excludeNotRequired: $[ dependencies.impact.outputs['delta.exclude_not_required'] ] + impactSkip: $[ dependencies.impact.outputs['delta.skip'] ] + steps: + - template: steps/${{ replace(group, '_', '-') }}.yml # pseudo-syntax; real emitter unrolls + parameters: + excludeNotModified: $(excludeNotModified) + excludeNotAffected: $(excludeNotAffected) + excludeNotRequired: $(excludeNotRequired) + impactSkip: $(impactSkip) + - ${{ if ne(length(parameters.windowsPool), 0) }}: + - job: pr_test_windows + dependsOn: impact + pool: ${{ parameters.windowsPool }} + variables: + excludeNotModified: $[ dependencies.impact.outputs['delta.exclude_not_modified'] ] + excludeNotAffected: $[ dependencies.impact.outputs['delta.exclude_not_affected'] ] + excludeNotRequired: $[ dependencies.impact.outputs['delta.exclude_not_required'] ] + impactSkip: $[ dependencies.impact.outputs['delta.skip'] ] + steps: + - template: steps/pr-test.yml + parameters: + excludeNotModified: $(excludeNotModified) + excludeNotAffected: $(excludeNotAffected) + excludeNotRequired: $(excludeNotRequired) + impactSkip: $(impactSkip) +``` + +The wiring never short-circuits jobs on `skip=true`. Each group always runs; the +recipes inside the group decide whether a given check can no-op. This matters because +several PR-tier checks (`fmt`, `deny`, `audit`, `aprz`, `pr-title`, `spellcheck`) don't +scope to workspace members and must run on every PR, including docs-only PRs where +nothing in the workspace is "affected." See +[local.md §4](./local.md#4-impact-scoping-pass-through-env-vars) for the recipe-side +contract. + +The real emitter unrolls the `${{ each group }}` block at template-compile time into +explicit jobs (ADO's `each` is compile-time so this works, but the syntax is fiddly — +the snippet above shows the intent, not the verbatim YAML). + +ADO's `strategy.matrix` doesn't compose with output-variable expressions cleanly (the +expansion happens at compile time but the values aren't available until impact has run), +so ox-check unrolls the OS axis into two explicit jobs (`pr_test_linux` and +`pr_test_windows`) at template-compile time using the `${{ if … }}` conditional. Setting +`windowsPool: {}` in the user's root pipeline elides `pr_test_windows` entirely. + +The nightly stages template is simpler — it omits the `impact` job and runs each group +full-workspace, with the same `linuxPool` / `windowsPool` parameter shape. The exclude +parameters are still threaded through (defaulted to empty) so step templates have a +uniform interface across tiers: + +```yaml +# .pipelines/ox-check/nightly.yml (owned by cargo-ox-check) +parameters: +- name: linuxPool + type: object + default: { vmImage: ubuntu-latest } +- name: windowsPool + type: object + default: { vmImage: windows-latest } + +stages: +- stage: OX_CHECK_nightly + jobs: + - job: nightly_test_linux + pool: ${{ parameters.linuxPool }} + steps: [ { template: steps/nightly-test.yml } ] + - ${{ if ne(length(parameters.windowsPool), 0) }}: + - job: nightly_test_windows + pool: ${{ parameters.windowsPool }} + steps: [ { template: steps/nightly-test.yml } ] + - job: advisories + pool: ${{ parameters.linuxPool }} + steps: [ { template: steps/nightly-advisories.yml } ] + - job: runtime + pool: ${{ parameters.linuxPool }} + steps: [ { template: steps/nightly-runtime.yml } ] + - job: exhaustive + pool: ${{ parameters.linuxPool }} + steps: [ { template: steps/nightly-exhaustive.yml } ] +``` + +If `.delta.toml`'s managed region is disabled +([updates.md §opt-out](./updates.md#6-opting-out-in-file-stubs)), `pr.yml` is regenerated +**without** the `impact` job: each group job becomes unconditional and the `exclude*` vars +remain empty, so every group runs full-workspace. `steps/impact.yml` is not emitted in +that mode. + +## 5. Per-group step templates + +Each per-group step template has the **same** uniform parameter surface — the three +impact-exclude variables plus a per-template handful of PR-context strings. This means +the stages template doesn't need to know which excludes a group's checks consume; it +just threads all three to every group. Moving a check between groups is a pure catalog +change. + +```yaml +# .pipelines/ox-check/steps/pr-fast.yml (owned by cargo-ox-check) +parameters: +- name: prTitle + type: string + default: $(System.PullRequest.Title) +- name: excludeNotModified + type: string + default: "" +- name: excludeNotAffected + type: string + default: "" +- name: excludeNotRequired + type: string + default: "" +- name: impactSkip + type: string + default: "false" +steps: +- template: setup.yml +- script: just ox-check-pr-fast + displayName: ox-check pr-fast + env: + PR_TITLE: ${{ parameters.prTitle }} + OX_CHECK_EXCLUDE_NOT_MODIFIED: ${{ parameters.excludeNotModified }} + OX_CHECK_EXCLUDE_NOT_AFFECTED: ${{ parameters.excludeNotAffected }} + OX_CHECK_EXCLUDE_NOT_REQUIRED: ${{ parameters.excludeNotRequired }} + OX_CHECK_IMPACT_SKIP: ${{ parameters.impactSkip }} +``` + +Uniform parameter set on every per-group template: + +| Parameter | Default | Notes | +|-----------------------|--------------|----------------------------------------------------| +| `excludeNotModified` | `""` | Forwarded as `OX_CHECK_EXCLUDE_NOT_MODIFIED`. | +| `excludeNotAffected` | `""` | Forwarded as `OX_CHECK_EXCLUDE_NOT_AFFECTED`. | +| `excludeNotRequired` | `""` | Forwarded as `OX_CHECK_EXCLUDE_NOT_REQUIRED`. | +| `impactSkip` | `"false"` | Forwarded as `OX_CHECK_IMPACT_SKIP`. Recipes that consume the excludes may early-return when this is `"true"`; non-scoping recipes (fmt, deny, audit, …) ignore it. See [local.md §4](./local.md#4-impact-scoping-pass-through-env-vars). | + +Per-group additions (only where the group consumes PR-context strings the recipe needs): + +| Template | Extra parameters | +|---------------------------|-------------------------------------------------------------------------| +| `pr-fast.yml` | `prTitle` (default `$(System.PullRequest.Title)`) | +| `pr-mutants.yml` | `prBaseRef` (default `$(System.PullRequest.TargetBranch)`) | +| `pr-test.yml` | — | +| `nightly-*.yml` | — | + +`$(System.PullRequest.*)` are auto-populated by ADO on PR build-validation runs. No +manual web-UI wiring is needed. + +The recipes themselves consume only the env vars they need; the catalog records the +mapping (see [checks.md §5](./checks.md#5-impact-scoping-check--env-var-mapping)). +Threading all three to every template costs a few lines per step template but is the +right separation: wiring is about "which jobs depend on impact and feed it forward", not +about "which check needs which env var." + +These templates are consumed primarily by ox-check's own stages template. Users who want to +plug individual groups into an unrelated pipeline can `template:` them directly without +passing any exclude parameters — they default to empty (full workspace) — and only +override what they want to scope. + +### `setup.yml` and `impact.yml` + +`setup.yml` installs `just` (`cargo install just --locked --version >=`) and runs +`just ox-check-tools-install-missing`. Does not install Rust; expects `cargo` on PATH — +provided by the user's msrustup step in 1ESPT pipelines or by a previous step in OSS +pipelines (see §6). + +`impact.yml` takes one parameter `baseRef` and runs the cargo-delta logic described in +[github.md §6](./github.md#6-impact-scoping). The four results are exported as ADO output +variables via `##vso[task.setvariable variable=…;isOutput=true]`: + +- `delta.exclude_not_modified` +- `delta.exclude_not_affected` +- `delta.exclude_not_required` +- `delta.skip` (advisory: `true` when no workspace member is in any tier) + +Downstream jobs reference them via `dependencies.impact.outputs['delta.']` inside +the runtime macro `$[ … ]` (rather than the compile-time `${{ … }}` macro) because output +variables aren't resolved until the producing job has finished. The stages template +handles all that — users don't write it. + +When `System.PullRequest.TargetBranch` is empty (non-PR triggered run on `main`), the +impact step short-circuits with `delta.skip=false` and empty `exclude*` values, so the +downstream groups all run full-workspace. This makes the same stages template valid for +CI runs that aren't PR validations. + +The check → tier mapping is in +[checks.md §5](./checks.md#5-impact-scoping-check--env-var-mapping). The recipe-side +mechanics are in [local.md §4](./local.md#4-impact-scoping-pass-through-env-vars). + +## 6. Rust toolchain + +ox-check does not install Rust on ADO. The step templates assume `cargo` is on PATH. The +user's root pipeline (or compliance template) installs Rust before the ox-check stages run. + +Why ox-check doesn't ship a Rust install step: + +- **1ESPT compliance.** Compliance pipelines install Rust via msrustup + (Microsoft-internal). The standard `RustInstaller` ADO task is not used. ox-check must + emit nothing that conflicts with that. +- **Toolchain choice is a repo decision.** msrustup channels (`ms-prod-1.93`, etc.) are + repo-policy questions ox-check has no business making. + +In the OSS / non-1ESPT case, the user adds a `RustInstaller@1` task (or a rustup +shell script) to their root pipeline before the ox-check stages template runs. A typical +placement: a setup stage that `dependsOn`s nothing and runs first, followed by the ox-check +stages. + +`_ox-check-require` (invoked by every check recipe) validates the installed `rustc` against +the catalog minimum at recipe time; missing or below-minimum `rustc` produces a clean +failure message. For nightly-requiring checks (miri, careful, udeps), the failure message +suggests asking the team's pipeline owner to add `nightly` to msrustup. + +## 7. Caching + +`setup.yml` computes a cache key from: OS, rustc version (read from +`rust-toolchain.toml`), `Cargo.lock`, `.cargo/config.toml`, and the binary's embedded +catalog hash. Uses the ADO pipeline workspace cache (`Cache@2` task). `CARGO_HOME` is +pinned to a workspace-scratch location to keep cache scoping predictable. + +The cache covers: + +- The `cargo install`-ed tools from `ox-check-tools-install-missing`. +- The `target/` directory (per ox-check recipe; a per-recipe cache scope means a `pr-test` + cache hit doesn't have to wait on a `pr-fast` cache miss). + +Cache scoping inside 1ESPT-compliant pipelines is bounded by the template's allowed cache +namespaces; the emitted cache step uses the project-scoped namespace by default and the +user can override via a parameter on `setup.yml` if their compliance policy requires a +different one. + +## 8. Security + +The step templates do nothing privileged on their own — they just install tools and +invoke `just`. The user's root pipeline controls service-connection scoping, secret +variable groups, and approval gates. + +Recommended user-pipeline shape: + +- PR pipelines and nightly pipelines are separate root files (so they can have separate + triggers, separate variable groups, and different `extends:` if needed). +- Nightly variable groups (with any external-service credentials) are referenced only by + the nightly pipeline. +- All cargo-tool installs done by `setup.yml` use `--locked`. No `cargo-binstall`. + +## 9. Incremental adoption + +For repos with an existing 1ESPT-extending pipeline, adopting ox-check is incremental: + +1. Run `cargo ox-check update --backend ado` to emit owned templates and root pipelines. +2. Either delete the emitted root pipelines (`.pipelines/ox-check-{pr,nightly}.yml`) if they + conflict with the repo's existing ones, or edit the existing pipelines to call out to + `ox-check/pr.yml` / `ox-check/nightly.yml`. +3. In the repo's existing pipeline, add a stage that does + `template: /.pipelines/ox-check/pr.yml@self` under `parameters.stages` of the 1ESPT + `extends:` block. +4. Verify the stage runs green on a PR. +5. Optionally split into individual group stages by hand if the compliance template + requires it. + +ox-check's owned templates compose cleanly with the 1ESPT `enableStages` flag system: each +group is its own job inside the `OX_CHECK_pr` stage, so 1ESPT can gate or split them as +needed. The pre-existing repo-specific compliance steps (msrustup, NuGet pushes, signing, +…) keep running alongside the ox-check stage. ox-check does not own the pipeline's shape — +it just contributes a stage. diff --git a/crates/cargo_ox_check/docs/design/checks.md b/crates/cargo_ox_check/docs/design/checks.md new file mode 100644 index 00000000..c6291f77 --- /dev/null +++ b/crates/cargo_ox_check/docs/design/checks.md @@ -0,0 +1,226 @@ +# Check Catalog + +This document defines the opinionated default profile: which checks ship, how they're +grouped, which tier they belong to, and how the tool-version policy works. It is the +canonical source for "what does ox-check actually run?" + +See also: + +- [design.md](./design.md) for the overall principles and CLI shape. +- [local.md](./local.md) for how the catalog is exposed as `just` recipes. +- [github.md](./github.md) / [ado.md](./ado.md) for how groups map to CI building blocks. + +## 1. Groups and tiers + +The check catalog is hardcoded in the binary. Each check belongs to one or more *groups*, and +each group belongs to exactly one *tier*. Groups are the unit of CI parallelization (one CI +job per group) and the unit of local invocation through `just` (one `just` recipe per group). +A user (or CI) never has to enumerate individual checks — they operate at the group level. + +The **single-tier-per-group** rule is deliberate: if you see `just ox-check-pr-fast` in CI logs, +you know it is a PR-tier check; if you see `just ox-check-nightly-runtime`, you know it is +nightly-only. This makes "what gets executed" trivially answerable from the group name. + +A consequence is that some checks must appear in two groups — one PR group and one nightly +group — when the check should run in both tiers. The two invocations may differ (e.g. +`mutants` runs diff-scoped in PR and full-workspace in nightly) or be identical (e.g. `tests` +runs the same way in both, but the nightly run catches flakes/environmental drift on `main`). + +Group recipes follow the pattern `ox-check--` (e.g. `ox-check-pr-fast`, +`ox-check-nightly-runtime`). The tier prefix removes the need to pick distinct names for groups +in different tiers and makes the tier of any failing job obvious from its name alone. + +### PR tier (3 groups) + +| Group | OS scope | Purpose | +|--------------------|--------------------|----------------------------------------------------------------------------------------------------------------------| +| `pr-fast` | Linux only | All static analysis: nothing here compiles user tests or examples through to execution. Fast feedback, fail-fast. | +| `pr-test` | Linux + Windows | Code execution: tests (instrumented for coverage), doctests, examples. Coverage reporting is folded in via `cargo llvm-cov nextest`. | +| `pr-mutants` | Linux only | Diff-scoped mutation testing on the change in this PR. | + +### Nightly tier (4 groups) + +| Group | OS scope | Purpose | +|----------------------|-----------------|----------------------------------------------------------------------------------------------------------------------------------------| +| `nightly-test` | Linux + Windows | Re-runs the test suite on `main` (with coverage instrumentation) to catch flakes/environment-dependent failures and to publish a full coverage snapshot of the current `main`. | +| `nightly-advisories` | Linux only | Re-runs every check whose outcome can change without a commit to this repo: `deny`, `audit`, `aprz` (external databases), `clippy` (lint set evolves with toolchain), `udeps` (uses `cargo +nightly`, which evolves). | +| `nightly-runtime` | Linux only | Tests under stricter runtimes that catch UB and timing/threading bugs: `miri`, `careful`. (Both tools are Linux-only.) | +| `nightly-exhaustive` | Linux only | The expensive whole-workspace permutations that don't fit the PR budget: full `cargo mutants`, `cargo-hack --feature-powerset`, and `cargo bench --no-run` plus a single-iteration smoke run per bench target. | + +OS-scope is an opinion ox-check ships and the user overrides per-repo through the +backend-specific knobs ([github.md §4](./github.md#4-owned-reusable-workflows) for +`test_os`, [ado.md §4](./ado.md#4-owned-stages-templates) for `linuxPool`/`windowsPool`). +Locally there is no OS matrix; `just ox-check-pr-test` runs against whatever OS the +developer is on. See [design.md §8.3](./design.md#83-cross-os-test-matrices) for the +overall rationale. + +The `nightly-exhaustive` group's checks are independent and could in principle live in three +parallel jobs; they're folded into one group because each individually is just one check, +and nightly tolerates the longer wall-clock that serial execution within one job implies. +Repos that want to parallelize them can split the recipe into three group recipes locally. + +## 2. Checks by group + +The cell format is `cargo invocation (short rationale)`. "Source" cites the surveyed repo +that provided the strongest version of the check. + +### `pr-fast` + +| Check | Invocation | Source | +|--------------------------------|-----------------------------------------------------------|--------| +| `fmt` | `cargo fmt --all --check` | all | +| `clippy` | `cargo clippy --workspace --all-targets --all-features --locked -- -D warnings` | all | +| `cargo-sort` | `cargo sort --workspace --check` | oxidizer-github | +| `license-headers` | `cargo heather --workspace` | oxidizer (`heather`), oxidizer-github | +| `ensure-no-cyclic-deps` | `cargo ensure-no-cyclic-deps --workspace` | oxidizer-github (sibling crate in `ox-tools-gh`) | +| `ensure-no-default-features` | `cargo ensure-no-default-features --workspace` | oxidizer-github | +| `doc-build` | `RUSTDOCFLAGS='-D warnings' cargo doc --workspace --all-features --no-deps` | oxidizer-github | +| `readme-check` | `cargo doc2readme --check` for each crate that opts in (presence of a `[package.metadata.doc2readme]` table) | oxidizer-github | +| `spellcheck` | `cargo spellcheck check --code 1` | oxidizer-github | +| `pr-title` | Conventional-Commits regex applied to the title in the `PR_TITLE` env var, with a fallback to `git log -1 --pretty=%s HEAD` when unset. Written as a `[script("pwsh")]` recipe (the one check that needs scripting; see [design.md §8.3](./design.md#83-cross-os-test-matrices)). The CI emitter sets `PR_TITLE` on the pr-fast job: GitHub Actions reads `${{ github.event.pull_request.title }}`; ADO reads `$(System.PullRequest.Title)`. Local `just ox-check-pr-fast` works without setup via the git fallback. | oxidizer-github | +| `deny` | `cargo deny check` | all | +| `audit` | `cargo audit` | oxidizer | +| `udeps` | `cargo +nightly udeps --workspace --all-targets --all-features` | oxidizer, oxidizer-github | +| `semver-check` | `cargo semver-checks --workspace` | oxidizer-github | +| `external-types` | `cargo check-external-types --workspace` | oxidizer-github | +| `aprz` | `cargo aprz check` — third-party risk analysis published on crates.io | oxidizer | + +### `pr-test` + +| Check | Invocation | Source | +|--------------|-----------------------------------------------------------------------------|--------| +| `llvm-cov` | `cargo llvm-cov nextest --workspace --all-features --locked --lcov --output-path target/coverage/lcov.info` ＋ HTML report ＋ enforced minimum threshold. The instrumented `nextest` run produces both the test pass/fail signal and the coverage artifacts in a single pass. | oxidizer, oxidizer-github | +| `doc-test` | `cargo test --doc --workspace --all-features --locked` (nextest does not run doctests, so this is a separate cargo-test invocation) | oxidizer, oxidizer-github | +| `examples` | `cargo run --example ` for each example target | oxidizer, oxidizer-github | + +### `pr-mutants` + +| Check | Invocation | Source | +|-----------|----------------------------------------------------------------------------|--------| +| `mutants` | `cargo mutants --in-diff …HEAD --no-shuffle --jobs 0` (diff-scoped) | oxidizer-github | + +The PR mode requires a base ref. Locally, the recipe defaults to `origin/main` (or `master`) +and can be overridden via a `BASE_REF` env var; in GitHub Actions the workflow passes +`${{ github.event.pull_request.base.sha }}`; in ADO the template parameter `prBaseRef` is +wired to `System.PullRequest.TargetBranch`. + +### `nightly-test` + +| Check | Invocation | Source | +|--------------|-----------------------------------------------------------------------------|--------| +| `llvm-cov` | `cargo llvm-cov nextest --workspace --all-features --locked --lcov --output-path target/llvm-cov/nightly.lcov` | oxidizer, oxidizer-github | +| `doc-test` | `cargo test --doc --workspace --all-features --locked` | oxidizer, oxidizer-github | +| `examples` | `cargo run --example ` for each example target | oxidizer, oxidizer-github | + +The same checks as `pr-test`, run on `main`. Two purposes: catch flakes/environmental +sensitivities that didn't trip in PR, and publish a full coverage snapshot for the current +state of `main` (the PR `llvm-cov` upload only reflects diffed code; this one reflects the +whole codebase). The CI emitter wires the lcov artifact upload step in the nightly workflow +only. + +### `nightly-advisories` + +| Check | Invocation | Source | +|----------|---------------------------------------------------------------------|--------| +| `deny` | `cargo deny check` | all | +| `audit` | `cargo audit` | oxidizer | +| `aprz` | `cargo aprz check` | oxidizer | +| `clippy` | `cargo clippy --workspace --all-targets --all-features --locked -- -D warnings` | all | +| `udeps` | `cargo +nightly udeps --workspace --all-targets --all-features` | oxidizer, oxidizer-github | + +These checks share a property: their outcome can change without a commit to this repo. +`deny`/`audit`/`aprz` consult external databases (RustSec advisory DB, license registries, +Azure risk indices). `clippy` reflects whatever lint set ships with the currently-installed +toolchain — even when `rust-toolchain.toml` is pinned, repos using floating channels +(`stable`, or msrustup channel pointers like `ms-prod-1.93`) can pick up new lints when the +pointer is bumped upstream. `udeps` runs on `cargo +nightly` and reflects whatever nightly +is installed on the runner. Re-running these nightly turns "something landed upstream +yesterday" into a tracked failure rather than an invisible regression discovered next time +someone opens an unrelated PR. + +### `nightly-runtime` + +| Check | Invocation | Source | +|-----------|-------------------------------------------------------------------------------------|--------| +| `miri` | `cargo +nightly miri nextest run --workspace` | oxidizer, oxidizer-github | +| `careful` | `cargo +nightly careful test --workspace --all-features --locked` | oxidizer-github | + +### `nightly-exhaustive` + +| Check | Invocation | Source | +|----------------------|--------------------------------------------------------------------------------------------------------------|--------| +| `mutants-full` | `cargo mutants --workspace --no-shuffle --jobs 0` | oxidizer-github | +| `cargo-hack` powerset| `cargo hack --workspace --feature-powerset --depth 2 check` | oxidizer, oxidizer-github | +| `bench` | `cargo bench --workspace --all-features --no-run` ＋ a single-iteration smoke benchmark for each bench target | oxidizer | + +## 3. Per-check vs grouped CI execution + +Each *group* is one CI job. Within a job, the checks belonging to the group run sequentially +as the `just` recipe defines them. A failure in any check fails the group; the per-check log +lines are visible in the job log but the CI surface (the green/red pill in the PR view) is +per-group. + +This is the deliberate middle ground between "one giant CI step running `just ox-check-pr`" +(loses all per-check structure, one red X for any failure) and "twenty-five individual CI +steps" (unmaintainable YAML, fragile, and the tool would have to re-emit the workflow file +every time the catalog changes). Groups are stable units of meaning the user can talk about; +checks are implementation details that can churn. + +## 4. What nightly does and does not re-run + +The rule is simple: **a check belongs in nightly iff its outcome can change without a +commit to this repo.** Re-running everything else nightly would just burn CI time +duplicating PR signal. + +What that means concretely: + +- **Re-run in nightly** (in addition to PR): + - `llvm-cov`, `doc-test`, `examples` (in `nightly-test`) — non-determinism, environment + sensitivity, runner drift can produce flakes that the PR run missed. + - `deny`, `audit`, `aprz`, `clippy`, `udeps` (in `nightly-advisories`) — see §2. +- **Run only in PR** — checks whose outcome is fully determined by the source tree and + the pinned tool versions, so re-running on the same `main` commit can't surface anything + new: `fmt`, `cargo-sort`, `license-headers`, `ensure-no-cyclic-deps`, + `ensure-no-default-features`, `doc-build`, `readme-check`, `spellcheck`, `pr-title`, + `semver-check`, `external-types`, diff-scoped `mutants`. +- **Run only in nightly** — the expensive whole-workspace work that can't fit a PR + budget: `miri`, `careful` (in `nightly-runtime`); full `mutants`, + `cargo-hack --feature-powerset`, `bench` (in `nightly-exhaustive`). + +The single-tier-per-group rule still holds: when a check appears in both tiers it lives in +two different groups (one PR group, one nightly group). Repos that want a +belt-and-suspenders cron run of `just ox-check-pr` on `main` can wire one up in their own +workflow/pipeline file alongside the ox-check composite actions / step templates. + +## 5. Impact-scoping check → env-var mapping + +The tool uses [`cargo-delta`](https://crates.io/crates/cargo-delta) to skip checks for +unaffected workspace members on PR runs. cargo-delta computes three concentric impact tiers +(`required ⊇ affected ⊇ modified`) and emits each as a string of `--exclude X --exclude Y …` +flags (the workspace complement of the relevant tier), which composes cleanly with `cargo +--workspace`. Each catalog check is tagged with the tier it consumes: + +| Env var | cargo-delta source | Checks that consume it | +|-------------------------------|--------------------------------------------------------|------------------------------------------------------| +| `OX_CHECK_EXCLUDE_NOT_MODIFIED` | `cargo delta impact -f cargo-excludes --modified` | clippy, udeps | +| `OX_CHECK_EXCLUDE_NOT_AFFECTED` | `cargo delta impact -f cargo-excludes --affected` | llvm-cov, doc-test, examples, miri, careful, semver-check, mutants (diff and full), cargo-hack powerset, bench | +| `OX_CHECK_EXCLUDE_NOT_REQUIRED` | `cargo delta impact -f cargo-excludes --required` | doc-build, readme-check, external-types | + +Checks with no per-crate scope ignore the vars: `fmt` (always all files), `pr-title`, +`spellcheck`, `deny`, `audit`, `aprz`, `cargo-sort`, `license-headers`, +`ensure-no-cyclic-deps`, `ensure-no-default-features`. The mapping is hardcoded in the +catalog alongside each check's invocation. + +The recipe-side mechanics are in [local.md §4](./local.md#4-impact-scoping-pass-through-env-vars) +(including the `OX_CHECK_IMPACT_SKIP` early-return hint). The CI-side wiring (the +`ox-check-impact` building block, how downstream jobs consume the excludes) is in +[github.md](./github.md#impact-scoping) and [ado.md](./ado.md#impact-scoping). + +Trade-off acknowledged: the risk cargo-delta introduces is that a misconfigured analysis +silently skips checks that should have run, leaving "all green" on a PR that actually broke +something. The design mitigates this with: (1) trip-wire patterns in `.delta.toml` that +bias toward full runs whenever config changes; (2) the `skip` flag is advisory only and +the CI wiring never gates whole jobs on it — non-scoping checks (`fmt`, `deny`, `audit`, +`aprz`, `pr-title`, `spellcheck`) always run regardless of impact analysis; (3) nightly +always runs full-workspace, catching anything the PR-scoping missed within 24 hours; and +(4) any repo can disable scoping wholesale by emptying `.delta.toml`'s region. diff --git a/crates/cargo_ox_check/docs/design/design.md b/crates/cargo_ox_check/docs/design/design.md new file mode 100644 index 00000000..7310ec18 --- /dev/null +++ b/crates/cargo_ox_check/docs/design/design.md @@ -0,0 +1,378 @@ +# cargo-ox-check — Design + +> Status: **Draft**. +> Crate name: `cargo-ox-check`. +> Home: `github.com/microsoft/ox-tools`, published to crates.io. + +This is the top-level design document. It captures the why, the principles, and the +user-visible shape of the tool. Detail lives in companion documents: + +- [checks.md](./checks.md) — the opinionated check catalog, the group/tier structure +- [local.md](./local.md) — the `justfiles/ox-check/` layout, recipe surface, and customization. +- [updates.md](./updates.md) — the drift-detection and update algorithm; opt-out semantics. +- [github.md](./github.md) — GitHub Actions emission, example workflows, impact wiring. +- [ado.md](./ado.md) — Azure DevOps Pipelines emission, 1ESPT/msrustup composition. +- [../verification.md](../verification.md) — continuous-validation strategy: dogfooding, + fixture tests, schema validation. + +## 1. Problem + +Across the surveyed Rust repos (`oxidizer`, `oxidizer-github`, `ox-tools`, `ox-tools-gh`, +`assistants-oxide`, `ox-docs`) the build/test/CI infrastructure is conceptually similar but +implemented six different ways: + +| Repo | CI | Justfile shape | Toolchain | Notable specifics | +|------|----|----------------|-----------|-------------------| +| `oxidizer` | ADO 1ESPT (`SubstratePT`) | 500-line monolith + `just_mutants.just` | `ms-prod-1.93` | Stage flags (`enableStages`), `cargo-aprz`, stable-API checks | +| `oxidizer-github` | GitHub Actions | Modular `justfiles/{basic,coverage,format,setup,spelling}.just` + `constants.env` | `1.93` | `cargo-delta` impact-scoped builds, sticky semver comments, composite `setup` action | +| `ox-tools` | ADO (CloudBuild + classic) | none in worktree | `ms-prod-1.92` | NuGet/MSBuild scaffolding, internal templates | +| `ox-tools-gh` | GitHub Actions | Same modular shape as `oxidizer-github` | `1.93` | Mirror surface to OSS oxidizer | +| `assistants-oxide` | ADO 1ESPT (custom `rust/`) | Monolith + `.just/tds.just` | `ms-prod-1.93` | Symcrypt setup steps, NuGet publish stage | +| `ox-docs` | ADO classic | Monolith | `ms-prod-1.88` | Mixed C#/.NET + Rust, mdbook/docfx | + +The same logical checks (clippy, fmt, deny, miri, mutants, coverage, hack feature-powerset, udeps, +semver, spellcheck, license headers, doc/doctest, careful, audit, ensure-no-cyclic-deps, +ensure-no-default-features, doc2readme, …) are spelled in subtly different ways in each repo, with +different argument sets, different tool versions, and different opinions about which tier (PR vs. +nightly) a check belongs to. + +Maintaining six artisanal copies is expensive: improvements made in one repo (e.g. `cargo-delta` +impact scoping in `oxidizer-github`) take months to propagate, security/policy upgrades are +missed, and onboarding new Rust repos requires copying-and-praying. + +## 2. Goals + +1. **One opinionated build profile** for Rust repos, with sane defaults distilled from the + strongest patterns observed across the existing repos. +2. **Two tiers**: `pr` (blocking on every pull request) and `nightly` (slow, scheduled). +3. **Both CI backends** — GitHub Actions and Azure DevOps Pipelines — generated from the same + source of truth. The user picks one or both per repo via a CLI flag. +4. **Compliance preservation**: ADO pipelines that must `extends:` 1ESPT/SubstratePT continue to + do so. The tool's emitted templates contain no references to those harnesses; the user's + root pipeline does the wrapping. See [ado.md](./ado.md). +5. **Local/CI parity at every level**: every individual check, every group of checks, and the + full tier are all reproducible locally with a single `just` invocation, using the exact same + arguments CI uses. The three commands `just ox-check-pr`, `just ox-check-nightly`, and + `just ox-check-full` (= pr + nightly) are first-class local entry points. +6. **Plain-cargo fallback**: a developer with only `cargo` installed (no `just`, no + `cargo-ox-check`) can still build and run tests. +7. **Friendly updates**: the tool detects, per file and per managed region, whether the user has + modified it, and updates only the unmodified bits. +8. **Open source**: the crate ships from `github.com/microsoft/ox-tools` and publishes to + crates.io. The binary contains no Microsoft-internal dependencies; everything it can install + on the user's behalf comes from crates.io. + +## 3. Non-Goals + +- Replacing 1ESPT, SubstratePT, CloudBuild, or any other compliance/release pipeline. ox-check's + emitted templates contain no references to those harnesses; users wrap ox-check's stages + template in their compliance-extending pipeline themselves. See [ado.md](./ado.md). +- Building a general-purpose CI compiler/IR. We share **check semantics**, not CI features. +- Owning `.cargo/config.toml`, `rust-toolchain.toml`, or workspace layout in `Cargo.toml`. +- Installing the Rust toolchain. msrustup owns it on 1ESPT; the runner image owns it on + GitHub-hosted runners; the user owns it locally. The tool validates `rustc` version at + recipe time and produces a clean failure when it doesn't meet the catalog minimum. +- Managing exact tool versions on the user's behalf — we enforce minimums only. See + [local.md §3](./local.md#3-tool-versions-and-installation). +- Hosting a service. The tool is a CLI binary; updates ship via crates.io. +- Acting as a runtime: the tool emits `just` recipes and CI building blocks, then exits. + It is **not** invoked at build/test/CI time. `just` is the runtime. +- Destructive operations: `cargo ox-check update` never deletes files. Removing a previously + configured CI backend is a manual `rm -rf` by the user. + +## 4. Guiding Principle + +> **`cargo-ox-check` writes files. `just` runs them. The repo composes everything.** + +Corollaries that drive every section below: + +- The tool's only job is to author and update files. It is not on the local-build hot path or in + the CI graph at runtime. +- The local daily-driver is `just ox-check` (and friends). Those recipes call `cargo …` directly. CI + jobs invoke the same `just` recipes. Local and CI are bit-identical because they share one + implementation in the imported `.just` files. +- Drift detection lives inside the files themselves (per-file checksums and per-managed-region + checksums). There is no parallel metadata file. See [updates.md](./updates.md). +- The tool inserts managed sections into the user's `Justfile` and into a small set of shared + config files (`deny.toml`, `[workspace.lints]` in the workspace `Cargo.toml`, and `[lints]` + in each crate's `Cargo.toml`, plus `.delta.toml` and `rustfmt.toml`). Outside those sections, + the user's content is preserved verbatim. Everything else is in tool-owned files under + `justfiles/ox-check/` and the backend-specific CI directories. + +## 5. User Experience + +### 5.1 Installation (maintainer) + +```sh +cargo install --locked cargo-ox-check +``` + +Only the repo maintainer who runs updates needs the binary installed. Everyone else uses +`just` (or plain `cargo`). + +### 5.2 The single command + +```text +cargo ox-check update [--backend ]... [--no-backends] [--dry-run] +``` + +That is the entire CLI surface. There is intentionally no `init`, `migrate`, `check`, `run`, +`doctor`, `diff`, `explain`, `disable`, `enable`, or `versions` subcommand. + +The algorithm is uniform — there is no distinction between "first run" and "subsequent run." +The full per-item decision table lives in [updates.md](./updates.md). + +`--dry-run` performs the same analysis but writes nothing. Exit code 0 means "everything is in +sync with the binary's current templates and all managed content matched, ignoring disabled +items"; exit code 1 means "something is out of date or user-modified." + +`--backend ` is a repeatable flag controlling which CI backend(s) get emitted. Valid +backend names today are `github` and `ado`; the flag is repeatable (`--backend github +--backend ado`) so that adding a third backend in the future doesn't require new CLI +syntax. If `--backend` is omitted, the tool autodetects from the `origin` git remote URL +(`github.com` → `github`; `dev.azure.com` / `*.visualstudio.com` → `ado`). `--no-backends` +is valid and useful for repos that want only the local `just` setup with no CI files. +`update` never deletes files; to stop using a backend the user removes its directory by +hand and reruns without that backend. + +### 5.3 Daily driver + +The local UX is plain `just`: + +```text +$ just ox-check +[just] running ox-check-tools-check +[just] running ox-check-pr-fast +[just] running ox-check-pr-test +[just] running ox-check-pr-mutants +ox-check OK +``` + +`ox-check` is an alias for `ox-check-pr`. Both are plain `just` recipes (not wrappers around +`cargo ox-check`). The PR tier is made up of a small set of *check groups* — each group is a +`just` recipe that runs the individual checks belonging to it. Groups are the level at which +CI parallelizes. See [checks.md](./checks.md) for the group → check mapping and +[local.md](./local.md) for the recipe tree. + +Other tier entry points: + +- `just ox-check-pr` — fast checks suitable for every PR. +- `just ox-check-nightly` — slow checks: miri, full mutants, feature-powerset, bench, etc. +- `just ox-check-full` — both tiers, run sequentially. + +A user with only `just` installed (no `cargo-ox-check`) can run any check, any group, or any tier +without ever invoking the tool. `cargo-ox-check` is only required by the maintainer who wants to +update the recipes or CI building blocks. + +### 5.4 No-tooling fallback + +A user with only `cargo` (no `just`, no `cargo-ox-check`) can still run the basics: + +```sh +cargo test --workspace --all-targets --all-features --locked +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +cargo fmt --check +``` + +The same commands appear as the body of the corresponding `just` recipes in +`justfiles/ox-check/checks.just`, so they are discoverable by reading that file. The fallback +covers core hygiene only — coverage, miri, mutants, etc. still require their respective tools. + +## 6. Repo Layout + +The tool produces a small set of files. They fall into three categories: + +- **owned** — the tool fully writes the file. There is no in-file checksum line; ox-check + tracks ownership and last-rendered content in a sidecar manifest at the repo root + (`.ox-check.lock`). An advisory one-line `# Managed by cargo-ox-check` comment may appear + at the top of each owned file, but it carries no metadata. Updates apply + automatically when the user hasn't touched the file. If the user edits the file, the + next `update` writes a `.ox-check-proposed` sibling **only if the template has changed + since the last render** — claiming a file with no upstream churn produces zero + noise. +- **managed-region** — a user-composed file with one or more tool-managed sections + bracketed by sentinel comments. The sentinel pair (`# >>> ox-check-managed: ` … + `# <<< ox-check-managed: `) delimits the region body and identifies it by stable ID; + the manifest tracks the last-rendered checksum per `(host, id)`. Outside the + sentinels, the user's content is preserved byte-for-byte. +- **user-authored** — files the user owns; the tool only reads them. + `rust-toolchain.toml` and `.cargo/config.toml` fall in this category. + +Opt-out is expressed inline by **emptiness**: an empty managed-region body (just the +sentinels, no content between them) disables a region; an empty owned file disables +that owned item. See [updates.md §6](./updates.md#6-opting-out-in-file-stubs). + +```text +repo/ +├── .ox-check.lock sidecar manifest tracking last-rendered checksums (see updates.md) +├── Justfile managed-region: ox-check-imports +├── justfiles/ox-check/ owned (see local.md) +├── Cargo.toml managed-region: ox-check-workspace-lints (or ox-check-lints in single-crate) +├── crates//Cargo.toml managed-region: ox-check-lints (one per workspace member) +├── deny.toml managed-region: ox-check-deny +├── rustfmt.toml managed-region: ox-check-rustfmt (opt out with empty stub) +├── .delta.toml managed-region: ox-check-delta (opt out disables impact scoping) +├── rust-toolchain.toml user-authored (read only) +├── .cargo/config.toml user-authored (read only) +│ +├── .github/ only if --backend github (or autodetected) — see github.md +│ ├── actions/ox-check-*/ owned (per-group composite actions) +│ ├── workflows/ox-check-pr-impl.yml owned (reusable workflow doing the wiring) +│ ├── workflows/ox-check-nightly-impl.yml owned +│ ├── workflows/ox-check-pr.yml owned (root workflow: triggers/permissions/runner) +│ └── workflows/ox-check-nightly.yml owned +│ +└── .pipelines/ only if --backend ado (or autodetected) — see ado.md + ├── ox-check/pr.yml owned (stages template doing the wiring) + ├── ox-check/nightly.yml owned + ├── ox-check/steps/*.yml owned (per-group step templates) + ├── ox-check-pr.yml owned (root pipeline: triggers/pool/optional extends:) + └── ox-check-nightly.yml owned +``` + +Detail on each host: + +- **`Justfile` and `justfiles/ox-check/*.just`** — see [local.md](./local.md). +- **`Cargo.toml` lints regions** — workspace `Cargo.toml` carries the + `ox-check-workspace-lints` region containing a single `[workspace.lints]` table whose + rust/clippy/rustdoc entries are written in dotted-key form + (`rust.unsafe_op_in_unsafe_fn = "deny"`, `clippy.unwrap_used = "deny"`, etc.). This + form is chosen because TOML forbids re-declaring a table header — if ox-check wrote + `[workspace.lints.clippy]` inside the region, users couldn't add another + `[workspace.lints.clippy]` block elsewhere in the file. With dotted keys, users + append new lints in the same scope right after the closing sentinel; see §7. Each + member `Cargo.toml` carries an `ox-check-lints` region with exactly + `[lints]\nworkspace = true`. The emitter uses `toml-edit` for round-trip-safe + manipulation. In a single-crate repo (no `[workspace]` table), the workspace region + becomes `ox-check-lints` and contains a single `[lints]` table with the same + dotted-key layout. +- **`deny.toml`** — managed region at the end of the file with the tool's baseline + license/advisory rules. Users add their own keys outside the region. Created if absent. + Content detailed in [checks.md](./checks.md). +- **`rustfmt.toml`** — created with the opinionated baseline if absent; managed region at the + end of the file. The most contested opinion in the catalog; users who want to keep their own + formatting opt the file out via the empty-stub mechanism in [updates.md](./updates.md). +- **`.delta.toml`** — cargo-delta configuration that drives impact-scoped CI runs. Created if + absent. Region at the end of the file. Disabling the region opts the repo out of impact + scoping entirely. See [checks.md](./checks.md#impact-scoping) and the per-backend wiring in + [github.md](./github.md) / [ado.md](./ado.md). +- **`rust-toolchain.toml`** and **`.cargo/config.toml`** — never touched. Read-only inputs + used by `_ox-check-require` to validate the user's `rustc` version against the catalog + minimum. The CI building blocks do not install Rust; that is the user's pipeline's job + (msrustup in 1ESPT, rustup on GH runners). + +The tool's persistent state lives in `.ox-check.lock` at the repo root — the sidecar +manifest tracking last-rendered checksums per owned file and per managed region. See +[updates.md §1](./updates.md#1-the-manifest). All other state — including opt-outs — +lives in the affected file itself; see [updates.md](./updates.md). + +## 7. Customization + +Four escape valves, in increasing severity: + +1. **Compose around the tool**: add your own `.just` files and import them from your `Justfile` + alongside the `ox-check/*` imports. Add your own `.github/workflows/*.yml` files (anything not + prefixed `ox-check-` is left alone). Add your own `.pipelines/` templates and root pipelines. + The path of least resistance and the recommended approach for project-specific checks. +2. **Edit a managed-region host file outside the sentinels**: extra recipes in your + `Justfile`, extra rules in `deny.toml` outside the managed region, extra clippy + lints written in dotted-key form after the closing sentinel (e.g. `clippy.pedantic = "warn"` + in the `[workspace.lints]` scope). The tool preserves everything outside the + sentinels verbatim. Note that TOML forbids redeclaring a table header (`[workspace.lints.clippy]` + etc.), so user extensions must use dotted-key form or sit in a different parent + table; overriding an individual key already set inside the region requires editing + inside it, which triggers the dirty-file flow (see [updates.md §5](./updates.md#5-the-decision-algorithm)). +3. **Opt out by emptying.** Empty a managed region (leave only the sentinels) or empty + an owned file. The tool will skip the item on every future `update` and only emit a + `.ox-check-proposed` sibling when the template actually changes. See + [updates.md §6](./updates.md#6-opting-out-in-file-stubs). +4. **Take ownership of an owned file or managed region by editing it.** The next + `update` detects the dirt (via checksum comparison against the manifest), leaves your + file alone, and writes a `.ox-check-proposed` sibling only if the template changed since + the last render. Re-bless by deleting your file (or region) and rerunning `update`. + Suitable for one-off divergence; for permanent divergence prefer the opt-out stub. + +What the tool deliberately does **not** do: + +- Modify `Cargo.toml` outside the `ox-check-workspace-lints` / `ox-check-lints` managed regions. +- Modify `.cargo/config.toml` or `rust-toolchain.toml`. +- Replace existing workflows, root pipelines, or any file it didn't create. + +The intentional consequence: there is exactly one place to look for "what does this repo do +differently from the default?" — the working tree itself, plus the `--dry-run` summary listing +outstanding proposed updates. + +## 8. Cross-Cutting Concerns + +### 8.1 Security + +- Generated GH composite actions and ADO step templates do nothing privileged on their own; + they just invoke `just` recipes. The user's workflow / pipeline file controls permissions + and secrets. +- All cargo-tool installs done by the setup building blocks use `--locked`. No + `cargo-binstall`. +- The tool never sources or executes content from any user-edited file at runtime; + everything executable in the repo is plain `just` recipes the user can read. +- Recommended user-workflow shape: `permissions: contents: read` on PR workflows; grant + `pull-requests: read` only on the pr-fast job (for PR title). Nightly secrets, if any, live + on the nightly workflow only — never on the PR workflow. See the snippets in + [github.md](./github.md) and [ado.md](./ado.md). + +### 8.2 Monorepo / multi-workspace + +Out of scope for v1. `ox-check-*` recipes always operate on `--workspace` from the repo root. +Repos with multiple workspaces (uncommon in the surveyed set) compose by having a separate +ox-check tree per workspace root, each with its own `cargo ox-check update`. Revisit after first +adopters report friction. + +### 8.3 Cross-OS test matrices + +CI fans out a small set of groups across operating systems. Default matrix: + +| Group | OS scope | Rationale | +|-------------------------------|-----------------|----------------------------------------------------------| +| `pr-test`, `nightly-test` | Linux + Windows | Where compile-time and runtime OS bugs actually surface. | +| All other groups | Linux only | Source-only checks (`pr-fast`), advisory DBs, miri/careful (Linux-only tooling), mutants & feature-powerset (already expensive). | + +macOS is not in the default matrix — adopters who need it add it via the root workflow's +`test_os` input (GH) or root pipeline's `testPools` parameter (ADO). Both knobs are +documented in [github.md §4](./github.md#4-owned-reusable-workflows) for `test_os` and +[ado.md §4](./ado.md#4-owned-stages-templates) for `linuxPool`/`windowsPool`. + +Locally there is no matrix — `just ox-check-pr-test` runs against whatever OS the developer +is on. CI fan-out lives entirely in the owned wiring layer (the reusable workflow / stages +template), so users don't write per-OS jobs. + +cargo-delta impact runs once on Linux and feeds the same excludes to every matrix leg — +the source tree is OS-invariant. Caching keys already include OS (see +[github.md §8](./github.md#8-caching) and [ado.md §7](./ado.md#7-caching)). + +**Helper scripts use PowerShell Core (`pwsh`) on every platform.** Almost every check +recipe is a single-line `cargo …` invocation that works unmodified on Windows — +including `license-headers` (which calls `cargo heather`), `ensure-no-cyclic-deps` +(`cargo ensure-no-cyclic-deps`), and `ensure-no-default-features` +(`cargo ensure-no-default-features`), all of which are plain cargo subcommands from the +ox-tools family. The one current exception is `pr-title`, which does a regex match +against `$PR_TITLE` (no equivalent cargo subcommand and `just` itself has no +boolean-regex primitive). That check is written as a `[script("pwsh")]` block. `pwsh` +is preinstalled on GH-hosted runners (`ubuntu-latest` included) and Microsoft-hosted +ADO Linux agents; Linux/macOS developers install it from + as a one-time prerequisite. The +`_ox-check-require pwsh` recipe enforces this with a clean failure message and a per-OS +install hint. The dependency is kept (rather than dropped to remove the one script) +so future additions that don't fit cleanly as cargo subcommands have an established +escape hatch. + +### 8.4 Internal vs OSS + +The crate ships from `github.com/microsoft/ox-tools` (alongside the existing tools published +from that repo) and from crates.io. The binary contains: + +- The full check catalog (see [checks.md](./checks.md)), including `cargo aprz`, which is + itself published to crates.io. +- All emitters (GH, ADO). + +There is no overlay system, no internal-only check, and no proprietary content. ADO +templates are plain ADO templates — they happen to be shaped to compose cleanly with +SubstratePT/1ESPT, but they are freely usable in any ADO environment. + diff --git a/crates/cargo_ox_check/docs/design/github.md b/crates/cargo_ox_check/docs/design/github.md new file mode 100644 index 00000000..99158d80 --- /dev/null +++ b/crates/cargo_ox_check/docs/design/github.md @@ -0,0 +1,421 @@ +# GitHub Actions Integration + +This document describes what `cargo ox-check update --backend github` emits for GitHub +Actions, and how a repo wires those files into its own CI. + +ox-check emits three layers, all owned by ox-check with the standard owned-file flow (edit → +dirty → `.ox-check-proposed` sibling on next update). The split is by what users actually +need to change: + +1. **Root workflows** (`ox-check-pr.yml`, `ox-check-nightly.yml` at `.github/workflows/`). + Triggers, `permissions`, runner choice, any secret pass-through. ox-check ships an + opinionated default; users who need to customize edit in place and accept the + proposal-on-update flow. +2. **Reusable workflows** (`ox-check-pr-impl.yml`, `ox-check-nightly-impl.yml`), containing the + impact job and the per-group jobs with all the `needs.impact.outputs.*` plumbing. + These change when ox-check's groups or impact wiring evolve; most users won't ever edit + them. +3. **Per-group composite actions** (`.github/actions/ox-check-*/`). Each is a multi-step + composite that runs setup + the matching `just ox-check--` recipe. + +See also: + +- [design.md §6](./design.md#6-repo-layout) for the file-category model. +- [checks.md](./checks.md) for what each group runs. +- [local.md](./local.md) for the `just` recipes the composite actions invoke. +- [ado.md](./ado.md) for the ADO counterpart. + +## 1. Why three layers + +- **Frequently-changing wiring** (group set, impact computation, fan-out, `needs:` graph) + lives in the reusable workflows. Updates apply automatically; users don't have to merge + changes. +- **Per-repo customization** (triggers, permissions, runner pool, secret scoping) lives + in the root workflows. Users who customize them accept the cost of merging the + `.ox-check-proposed` sibling when the ox-check defaults evolve — which is rare, since the + root workflow is intentionally minimal. +- The reusable-workflow seam ([`workflow_call`][1]) is GitHub's first-class mechanism for + exactly this: a workflow can call another workflow in the same repo, passing inputs and + secrets. We use it so the root workflow stays ~10 lines. + +[1]: https://docs.github.com/en/actions/sharing-automations/reusing-workflows + +## 2. Emitted artifacts + +```text +.github/ +├── actions/ +│ ├── ox-check-setup/action.yml owned (install just + catalog tools) +│ ├── ox-check-impact/action.yml owned (cargo-delta; omitted if .delta.toml disabled) +│ ├── ox-check-pr-fast/action.yml owned (one composite action per group) +│ ├── ox-check-pr-test/action.yml owned +│ ├── ox-check-pr-mutants/action.yml owned +│ ├── ox-check-nightly-test/action.yml owned +│ ├── ox-check-nightly-advisories/action.yml owned +│ ├── ox-check-nightly-runtime/action.yml owned +│ └── ox-check-nightly-exhaustive/action.yml owned +└── workflows/ + ├── ox-check-pr-impl.yml owned (reusable workflow doing the wiring) + ├── ox-check-nightly-impl.yml owned (reusable workflow for nightly) + ├── ox-check-pr.yml owned (root workflow; triggers/permissions/runner) + └── ox-check-nightly.yml owned +``` + +All files are regular owned files (carry an `ox-check-checksum` first line, governed by +[updates.md §5](./updates.md#5-the-decision-algorithm)). Users who customize the root +workflow take ownership through the standard dirty-file flow. + +## 3. Root workflows + +The default `ox-check-pr.yml` ox-check emits is the minimum needed to call the reusable +workflow: + +```yaml +# .github/workflows/ox-check-pr.yml +name: ox-check-pr +on: + pull_request: {} + merge_group: {} +permissions: + contents: read +jobs: + ox-check: + uses: ./.github/workflows/ox-check-pr-impl.yml +``` + +The nightly root workflow adds a schedule and `workflow_dispatch`: + +```yaml +# .github/workflows/ox-check-nightly.yml +name: ox-check-nightly +on: + schedule: [{ cron: '0 6 * * *' }] + workflow_dispatch: {} +permissions: + contents: read +jobs: + ox-check: + uses: ./.github/workflows/ox-check-nightly-impl.yml +``` + +Common edits users make to the root workflow (these flip the file to "dirty" and produce +a `.ox-check-proposed` sibling on the next `update` — see +[updates.md §5](./updates.md#5-the-decision-algorithm)): + +- **Self-hosted runners**: pass `with: { runs_on: 'self-hosted-rust' }` to the reusable + workflow. +- **Trim or expand the test matrix**: pass `with: { test_os: '["ubuntu-latest"]' }` to + run tests on Linux only, or `'["ubuntu-latest","windows-latest","macos-latest"]'` to + add macOS. See §4 for the input contract. +- **Required secrets**: add `secrets: inherit` (or specific secrets) under the `ox-check:` + job. +- **Different schedule** for nightly. +- **Concurrency groups**: add a `concurrency:` block. +- **Path filters** to skip the workflow on docs-only PRs (though ox-check's own + `.delta.toml` trip-wire patterns already do impact-scoped skipping). + +## 4. Owned reusable workflows + +`ox-check-pr-impl.yml` is where the wiring lives. Every per-group composite action takes +the same three impact-exclude inputs unconditionally; which ones a group's checks +actually consume is the catalog's concern, not the wiring layer's. Moving a check +between groups never changes the reusable workflow. + +Approximate shape (ox-check writes this verbatim; users never edit it): + +```yaml +# .github/workflows/ox-check-pr-impl.yml (owned by cargo-ox-check) +on: + workflow_call: + inputs: + runs_on: + type: string + default: ubuntu-latest + description: Runner for single-OS jobs (impact, pr-fast, pr-mutants). + test_os: + type: string + default: '["ubuntu-latest","windows-latest"]' + description: JSON array of runners for the cross-OS pr-test matrix. + +jobs: + impact: + runs-on: ${{ inputs.runs_on }} + outputs: + exclude_not_modified: ${{ steps.impact.outputs.exclude_not_modified }} + exclude_not_affected: ${{ steps.impact.outputs.exclude_not_affected }} + exclude_not_required: ${{ steps.impact.outputs.exclude_not_required }} + skip: ${{ steps.impact.outputs.skip }} + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 0 } + - id: impact + uses: ./.github/actions/ox-check-impact + with: + base_ref: ${{ github.event.pull_request.base.sha }} + + pr-fast: + needs: impact + runs-on: ${{ inputs.runs_on }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/ox-check-pr-fast + with: + pr_title: ${{ github.event.pull_request.title }} + exclude_not_modified: ${{ needs.impact.outputs.exclude_not_modified }} + exclude_not_affected: ${{ needs.impact.outputs.exclude_not_affected }} + exclude_not_required: ${{ needs.impact.outputs.exclude_not_required }} + impact_skip: ${{ needs.impact.outputs.skip }} + + pr-test: + needs: impact + strategy: + fail-fast: false + matrix: + os: ${{ fromJSON(inputs.test_os) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/ox-check-pr-test + with: + exclude_not_modified: ${{ needs.impact.outputs.exclude_not_modified }} + exclude_not_affected: ${{ needs.impact.outputs.exclude_not_affected }} + exclude_not_required: ${{ needs.impact.outputs.exclude_not_required }} + impact_skip: ${{ needs.impact.outputs.skip }} + + pr-mutants: + needs: impact + runs-on: ${{ inputs.runs_on }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/ox-check-pr-mutants + with: + base_ref: origin/${{ github.event.pull_request.base.ref }} + exclude_not_modified: ${{ needs.impact.outputs.exclude_not_modified }} + exclude_not_affected: ${{ needs.impact.outputs.exclude_not_affected }} + exclude_not_required: ${{ needs.impact.outputs.exclude_not_required }} + impact_skip: ${{ needs.impact.outputs.skip }} +``` + +The wiring never short-circuits jobs on `skip=true`. Each group always runs; the +recipes inside the group decide whether a given check can no-op. This matters because +several PR-tier checks (`fmt`, `deny`, `audit`, `aprz`, `pr-title`, `spellcheck`) don't +scope to workspace members and must run on every PR, including docs-only PRs where +nothing in the workspace is "affected." See +[local.md §4](./local.md#4-impact-scoping-pass-through-env-vars) for the recipe-side +contract. + +The nightly reusable workflow is simpler — it omits the `impact` job and runs each group +full-workspace. The exclude inputs are still passed (defaulted to empty) so the composite +actions have a uniform interface across tiers: + +```yaml +# .github/workflows/ox-check-nightly-impl.yml (owned) +on: + workflow_call: + inputs: + runs_on: + type: string + default: ubuntu-latest + test_os: + type: string + default: '["ubuntu-latest","windows-latest"]' +jobs: + test: + strategy: + fail-fast: false + matrix: { os: ${{ fromJSON(inputs.test_os) }} } + runs-on: ${{ matrix.os }} + steps: [ { uses: actions/checkout@v4 }, { uses: ./.github/actions/ox-check-nightly-test } ] + advisories: { runs-on: ${{ inputs.runs_on }}, steps: [ { uses: actions/checkout@v4 }, { uses: ./.github/actions/ox-check-nightly-advisories } ] } + runtime: { runs-on: ${{ inputs.runs_on }}, steps: [ { uses: actions/checkout@v4 }, { uses: ./.github/actions/ox-check-nightly-runtime } ] } + exhaustive: { runs-on: ${{ inputs.runs_on }}, steps: [ { uses: actions/checkout@v4 }, { uses: ./.github/actions/ox-check-nightly-exhaustive } ] } +``` + +Nightly composite actions don't receive `exclude_*` at all — their inputs default to +empty (full workspace) and the reusable workflow omits the passthrough. Threading them +through is purely a PR-tier optimization; nightly never benefits. + +If `.delta.toml`'s managed region is disabled +([updates.md §opt-out](./updates.md#6-opting-out-in-file-stubs)), +`ox-check-pr-impl.yml` is regenerated **without** the `impact` job: each group job becomes +unconditional and the `exclude_*` inputs default to empty, so every group runs +full-workspace. `.github/actions/ox-check-impact/` is not emitted in that mode. + +The reusable workflow declares a small input set so the root workflow can pass overrides: + +| Input | Type | Default | Meaning | +|-----------|--------|----------------------------------------|--------------------------------------------------------| +| `runs_on` | string | `ubuntu-latest` | Runner for single-OS jobs (impact, pr-fast, pr-mutants). | +| `test_os` | string | `'["ubuntu-latest","windows-latest"]'` | JSON array of runners for the cross-OS `pr-test` matrix. Override to drop Windows for OSS-only repos, or to add `macos-latest`, `windows-2022`, a self-hosted label, etc. | + +The nightly reusable workflow has the same two inputs; its `nightly-test` job uses +`test_os` and every other job uses `runs_on`. + +We deliberately keep this input surface minimal. Anything more elaborate (e.g. +per-job runner overrides) lives in the user's own workflow, which can compose its own +`uses:`-of-reusable-workflow shape. + +## 5. Per-group composite actions + +Each per-group composite action has the **same** uniform input surface — the three +impact-exclude variables plus a per-action handful of PR-context strings. This means +the reusable workflow doesn't need to know which excludes a group's checks consume; it +threads all three to every action. Moving a check between groups is a pure catalog +change. + +```yaml +# .github/actions/ox-check-pr-fast/action.yml (owned) +name: ox-check-pr-fast +description: ox-check PR fast group +inputs: + pr_title: + description: PR title for the pr-title check. + required: false + default: "" + exclude_not_modified: + description: cargo-excludes string from ox-check-impact (--modified). Empty = full workspace. + required: false + default: "" + exclude_not_affected: + description: cargo-excludes string from ox-check-impact (--affected). Empty = full workspace. + required: false + default: "" + exclude_not_required: + description: cargo-excludes string from ox-check-impact (--required). Empty = full workspace. + required: false + default: "" + impact_skip: + description: '"true" when no workspace member is in any impact tier. Recipes that scope to workspace members may early-return; non-scoping recipes ignore this.' + required: false + default: "false" +runs: + using: composite + steps: + - uses: ./.github/actions/ox-check-setup + - shell: bash + env: + PR_TITLE: ${{ inputs.pr_title }} + OX_CHECK_EXCLUDE_NOT_MODIFIED: ${{ inputs.exclude_not_modified }} + OX_CHECK_EXCLUDE_NOT_AFFECTED: ${{ inputs.exclude_not_affected }} + OX_CHECK_EXCLUDE_NOT_REQUIRED: ${{ inputs.exclude_not_required }} + OX_CHECK_IMPACT_SKIP: ${{ inputs.impact_skip }} + run: just ox-check-pr-fast +``` + +Uniform input set on every per-group composite action: + +| Input | Default | Notes | +|---------------------------|-------------|----------------------------------------------------| +| `exclude_not_modified` | `""` | Forwarded as `OX_CHECK_EXCLUDE_NOT_MODIFIED`. | +| `exclude_not_affected` | `""` | Forwarded as `OX_CHECK_EXCLUDE_NOT_AFFECTED`. | +| `exclude_not_required` | `""` | Forwarded as `OX_CHECK_EXCLUDE_NOT_REQUIRED`. | +| `impact_skip` | `"false"` | Forwarded as `OX_CHECK_IMPACT_SKIP`. Recipes that consume the excludes may early-return when `"true"`; non-scoping recipes (fmt, deny, audit, …) ignore it. See [local.md §4](./local.md#4-impact-scoping-pass-through-env-vars). | + +Per-action additions (only where the action consumes PR-context strings the recipe needs): + +| Action | Extra inputs | +|------------------------------|-------------------------------------------------------------------------| +| `ox-check-pr-fast` | `pr_title` | +| `ox-check-pr-mutants` | `base_ref` | +| `ox-check-pr-test` | — | +| `ox-check-nightly-*` | — | + +The recipes themselves consume only the env vars they need; the catalog records the +mapping (see [checks.md §5](./checks.md#5-impact-scoping-check--env-var-mapping)). +Threading all three to every action costs a few lines per composite but is the right +separation: wiring is about "which jobs depend on impact and feed it forward", not about +"which check needs which env var." + +These actions are consumed primarily by ox-check's own reusable workflow. Users who want to +plug individual groups into an unrelated workflow can `uses:` them directly. + +### `ox-check-setup` + +`ox-check-setup` installs `just` (`cargo install just --locked --version >=`) and runs +`just ox-check-tools-install-missing`. Does not install Rust; expects `cargo` on PATH (see +§7). `ox-check-impact` is described in §6 below. + +## 6. Impact scoping + +`.github/actions/ox-check-impact/action.yml` is a composite action with input `base_ref`. It +runs: + +1. `git checkout $base_ref` and `cargo delta -c .delta.toml snapshot > baseline.json`. +2. `git checkout $head` and `cargo delta -c .delta.toml snapshot > current.json`. +3. `cargo delta impact -c .delta.toml --baseline baseline.json --current current.json -f + cargo-excludes --modified|--affected|--required` once per tier. + +Outputs: + +| Output | Meaning | +|-------------------------|----------------------------------------------------------------------------------------------------------| +| `exclude_not_modified` | `--exclude X --exclude Y …` string for the complement of cargo-delta's `modified` tier. | +| `exclude_not_affected` | Same, for the `affected` tier. | +| `exclude_not_required` | Same, for the `required` tier. | +| `skip` | `"true"` when no workspace member is in any tier (no PR-relevant change). Propagated via `impact_skip` to every composite action; recipes that scope to workspace members may use it to early-return, but the wiring never gates whole jobs on it (see §4). | + +The reusable workflow handles consumption — users never wire these outputs themselves. + +The check → tier mapping is in +[checks.md §5](./checks.md#5-impact-scoping-check--env-var-mapping). The recipe-side +mechanics are in [local.md §4](./local.md#4-impact-scoping-pass-through-env-vars). + +## 7. Rust toolchain + +ox-check does not install Rust on GitHub. The composite actions assume `cargo` is on PATH. +GH-hosted runners ship with a recent stable Rust and `rustup` pre-installed; if your +`rust-toolchain.toml` pins a different channel, the first `cargo` invocation in a job +triggers `rustup` to download the pinned toolchain. For a published stable channel this +typically takes 10–30 seconds on Linux (somewhat longer on Windows and longer still for +nightly with components). The auto-install runs once per job and is not cached across +jobs by ox-check — `~/.rustup` has high invalidation churn and the install cost is small +relative to the cached cargo registry / `target/` paths (§8). Repos that want to skip +even this per-job overhead can add their own toolchain-install step (e.g. +`dtolnay/rust-toolchain@stable`) before the ox-check composite action runs. + +On self-hosted runners or pre-baked images without rustup, the user adds a Rust install +step to their root workflow before the `uses:` of the reusable workflow: + +```yaml +jobs: + ox-check: + uses: ./.github/workflows/ox-check-pr-impl.yml + # Self-hosted? Add a setup workflow that runs first and uploads + # toolchain to a shared cache, then reference it here. +``` + +Since reusable workflows can't accept "previous step" handoff, self-hosted users usually +forgo the reusable-workflow shape and write a single workflow that calls the composite +actions directly. ox-check's composite actions are exposed for that use case. + +`_ox-check-require` (invoked by every check recipe) validates the installed `rustc` against +the catalog minimum at recipe time; missing or below-minimum `rustc` produces a clean +failure message. + +## 8. Caching + +The `ox-check-setup` composite action computes a cache key from: OS, rustc version (read +from `rust-toolchain.toml`), `Cargo.lock`, `.cargo/config.toml`, and the binary's +embedded catalog hash. Uses `actions/cache` natively. `CARGO_HOME` is pinned to a +workspace-scratch location to keep cache scoping predictable. + +The cache covers: + +- The `cargo install`-ed tools from `ox-check-tools-install-missing`. +- The `target/` directory (per ox-check recipe; a per-recipe cache scope means a `pr-test` + cache hit doesn't have to wait on a `pr-fast` cache miss). + +## 9. Security + +The composite actions do nothing privileged on their own — they just install tools and +invoke `just`. The reusable workflow propagates only what the root workflow passes (and +only the inputs explicitly declared). + +Recommended root workflow shape: + +- `permissions: contents: read` at the workflow level. ox-check's default ships with + this. +- No `pull-requests: write` (the PR-title check only needs the title from the event + payload, which is already in `${{ github.event.pull_request.title }}`). +- Nightly secrets, if any, live on `ox-check-nightly.yml` only — never on `ox-check-pr.yml`. +- All cargo-tool installs done by `ox-check-setup` use `--locked`. No `cargo-binstall`. diff --git a/crates/cargo_ox_check/docs/design/local.md b/crates/cargo_ox_check/docs/design/local.md new file mode 100644 index 00000000..7eb82425 --- /dev/null +++ b/crates/cargo_ox_check/docs/design/local.md @@ -0,0 +1,301 @@ +# Local Recipe Surface + +This document describes the `justfiles/ox-check/` tree that ox-check writes into a repo, how the +recipes are organized, and how local invocations differ from CI invocations (spoiler: they +don't — that's the design). + +See also: + +- [design.md](./design.md) for the overall principles. +- [checks.md](./checks.md) for the catalog the recipes implement. +- [updates.md](./updates.md) for how these files are tracked / regenerated. + +## 1. File layout + +```text +repo/ +├── Justfile managed-region: ox-check-imports +│ # >>> ox-check-managed: ox-check-imports +│ # checksum: sha256:… rendered-by: cargo-ox-check 0.4.1 +│ import 'justfiles/ox-check/checks.just' +│ import 'justfiles/ox-check/groups.just' +│ import 'justfiles/ox-check/tiers.just' +│ import 'justfiles/ox-check/tools.just' +│ alias ox-check := ox-check-pr +│ # <<< ox-check-managed: ox-check-imports +│ …user content… +│ +└── justfiles/ox-check/ owned (one checksum per file) + ├── checks.just per-check recipes (ox-check-fmt, ox-check-clippy, ox-check-llvm-cov, …) + ├── groups.just group recipes (ox-check-pr-fast, ox-check-pr-test, ox-check-nightly-test, …) + ├── tiers.just tier aggregators (ox-check-pr, ox-check-nightly, ox-check-full) + └── tools.just ox-check-tools-check + ox-check-tools-install + helpers +``` + +The Justfile region is the only file ox-check adds to that the user co-owns. The four files +under `justfiles/ox-check/` are tool-owned (full file checksums). If the user wants to add +project-specific recipes, they add them to the top-level `Justfile` outside the managed +region, or to their own additional imported `.just` files. + +## 2. Recipe layers + +`justfiles/ox-check/` is structured to make all three levels (check, group, tier) addressable +from the command line. + +### checks.just + +One recipe per individual check, each named `ox-check-`. Recipes are usually a single +`cargo …` line; a handful (license-headers, ensure-no-cyclic-deps, +ensure-no-default-features, pr-title, the bench smoke loop) are short `[script]` blocks. +Every check recipe is prefixed with a quick version-gate dependency: + +```just +ox-check-clippy: (_ox-check-require "cargo-clippy") + cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +``` + +`_ox-check-require` is a private recipe in `tools.just` that calls `cargo install --list` to +verify the tool meets the catalog's declared minimum version; missing or below-minimum +tools fail with a one-line `cargo install` hint. The cost is one cheap `cargo install +--list` invocation per check, well under a second on a warm cache. + +### groups.just + +One recipe per group, named `ox-check--`. The check-recipe and group-recipe +namespaces are kept disjoint by naming choice: no check is named `-` for +any tier × group combination (e.g. the coverage-instrumented test check is named +`llvm-cov`, not `test`, so that `ox-check-pr-test` unambiguously refers to the PR-tier +test group). The `pr-mutants` group runs the diff-scoped recipe; `nightly-exhaustive` +runs the full-workspace recipe: + +```just +ox-check-pr-fast: ox-check-fmt ox-check-clippy ox-check-cargo-sort ox-check-license-headers \ + ox-check-ensure-no-cyclic-deps ox-check-ensure-no-default-features \ + ox-check-doc-build ox-check-readme-check ox-check-spellcheck ox-check-pr-title \ + ox-check-deny ox-check-audit ox-check-udeps ox-check-semver-check \ + ox-check-external-types ox-check-aprz + +ox-check-pr-test: ox-check-llvm-cov ox-check-doc-test ox-check-examples +ox-check-pr-mutants: ox-check-mutants-diff + +ox-check-nightly-test: ox-check-llvm-cov ox-check-doc-test ox-check-examples +ox-check-nightly-advisories: ox-check-deny ox-check-audit ox-check-aprz ox-check-clippy ox-check-udeps +ox-check-nightly-runtime: ox-check-miri ox-check-careful +ox-check-nightly-exhaustive: ox-check-mutants-full ox-check-cargo-hack ox-check-bench-only +``` + +### tiers.just + +Three tier aggregators. Each tier is a recipe that depends on the appropriate set of groups +in a deterministic order: + +```just +ox-check-pr: ox-check-tools-check ox-check-pr-fast ox-check-pr-test ox-check-pr-mutants +ox-check-nightly: ox-check-tools-check ox-check-nightly-test ox-check-nightly-advisories \ + ox-check-nightly-runtime ox-check-nightly-exhaustive +ox-check-full: ox-check-pr ox-check-nightly +``` + +### tools.just + +- `ox-check-tools-check` — print a status table of every tool's installed version vs. minimum. +- `ox-check-tools-install` — install every catalog tool at the minimum version (or skip if + already satisfied). Used as a one-shot in CI setup and locally on first use. +- `ox-check-tools-install-missing` — install only the tools that are missing or below minimum. +- `_ox-check-require ` — internal helper called by each check. + +The full tool-version policy these recipes implement is detailed in §3 below. + +## 3. Tool versions and installation + +### 3.1 Policy + +The tool **never pins exact versions** for the user. The catalog records, for each tool, a +*minimum required version* (e.g. `cargo-nextest >= 0.9.122`). Users are free to install +newer versions, use `mise`/`asdf`, install via package manager, etc. + +### 3.2 Detecting installed versions + +`_ox-check-require ` (a private `just` recipe in `tools.just`) uses +`cargo install --list` to enumerate currently-installed cargo subcommands and their +versions, then compares against the catalog minimum. This avoids the problem of tools +without a stable `--version` flag, is fast, and works uniformly for everything the tool +cares about (all the cargo-* checks). For the small number of non-cargo dependencies +(`just` itself and `pwsh`), the recipe falls back to `tool --version` and a known parser. + +### 3.3 Installing tools + +`ox-check-tools-install` and `ox-check-tools-install-missing` are plain `just` recipes that loop +over the catalog and run `cargo install --locked --version >=`. They are the +*only* mechanism the tool uses to install cargo-managed tools — there is no separate code +path for CI. CI setup just calls the recipes. Locally, the user runs the recipes once +when `ox-check-tools-check` complains. + +Two prerequisites are not cargo-installable and must be present before the recipes can +run: + +- **`just`** itself — bootstrap with `cargo install just --locked` once, or use a system + package. Every backend's setup composite/template installs it via cargo as a one-shot. +- **`pwsh`** (PowerShell Core) — used by the `pr-title` `[script]` recipe (regex + match on `$PR_TITLE`; no equivalent cargo subcommand and `just` lacks a + boolean-regex primitive). The other historically-scripted checks + (`license-headers`, `ensure-no-cyclic-deps`, `ensure-no-default-features`) are now + plain cargo subcommands from the ox-tools family and don't need pwsh. `pwsh` is + preinstalled on every relevant CI runner (GH-hosted Linux/Windows/macOS, + Microsoft-hosted ADO agents). On a developer machine without pwsh, + `_ox-check-require pwsh` fails with a per-OS install hint pointing at + . The dependency is kept (rather than + dropped to remove the one script) so future additions that don't fit cleanly as + cargo subcommands have an established escape hatch. + +Trade-off acknowledged: `cargo install --locked` is slow on a cold cache (several minutes +for the full catalog). It is also the most reliable mechanism in restricted networks. +Caching (via the GH cache action and the ADO pipeline workspace cache) is configured by the +setup action/template to key on `Cargo.lock`, the toolchain channel, and the binary's +catalog hash. See [github.md](./github.md#caching) and [ado.md](./ado.md#caching). + +### 3.4 Per-check warnings + +Every check recipe depends on `_ox-check-require ` so even ad-hoc invocations like +`just ox-check-miri` warn loudly if the installed tool predates the catalog minimum. The full +tier invocations additionally print a one-line tools summary at the top. + +### 3.5 The Rust toolchain + +`rust-toolchain.toml` is read but never written, and ox-check never installs a Rust toolchain +itself. Per-backend rationale lives in [github.md](./github.md#rust-toolchain) and +[ado.md](./ado.md#rust-toolchain); short version: msrustup owns it on ADO/1ESPT, the runner +image owns it on GH, the user owns it locally. + +`_ox-check-require` validates the installed `rustc` against the catalog's minimum at recipe +time; missing or below-minimum `rustc` produces a clean failure message naming the version +mismatch. Per-check toolchain requirements (e.g. miri, careful, udeps need nightly) are +also enforced by `_ox-check-require`, which suggests the user-environment-appropriate install +command in the failure message (`rustup install nightly` or "ask your team's pipeline +owner to add `nightly` to msrustup"). + +## 4. Impact-scoping pass-through env vars + +Every check recipe whose work is per-crate accepts three optional pass-through env vars +and forwards them verbatim as `--workspace`-compatible exclude flags. Empty (the local +default) means full workspace; CI populates them from the `ox-check-impact` building block: + +```just +ox-check-clippy: (_ox-check-require "cargo-clippy") + cargo clippy --workspace ${OX_CHECK_EXCLUDE_NOT_MODIFIED:-} --all-targets --all-features --locked -- -D warnings + +ox-check-llvm-cov: (_ox-check-require "cargo-llvm-cov") (_ox-check-require "cargo-nextest") + cargo llvm-cov nextest --workspace ${OX_CHECK_EXCLUDE_NOT_AFFECTED:-} --all-features --locked --lcov --output-path target/coverage/lcov.info +``` + +The mapping from check to env var is fixed in the catalog (see +[checks.md §5](./checks.md#5-impact-scoping-check--env-var-mapping)). Checks with no +per-crate scope (`pr-title`, `aprz`, `deny`, `audit`, `spellcheck`, `fmt`, `cargo-sort`, +`license-headers`, `ensure-no-cyclic-deps`, `ensure-no-default-features`) ignore the +vars. Group recipes do not interpolate the vars themselves — each underlying check +recipe reads what it needs, so a group recipe is just a dependency list and nothing +changes when scoping is disabled. + +### 4.1 `OX_CHECK_IMPACT_SKIP` early-return hint + +A fourth env var, `OX_CHECK_IMPACT_SKIP`, is set to `"true"` by the CI wiring when +cargo-delta reports that no workspace member is in any impact tier (typically a +docs-only PR or a PR touching only files cargo-delta's `file_exclude_patterns` ignore). +It is **advisory**, not a kill switch: + +- The CI wiring **never** uses it to skip whole jobs. Every group runs on every PR. +- Recipes that scope to workspace members **may** check it and early-return — for + example, `ox-check-clippy` skips the cargo invocation when `OX_CHECK_IMPACT_SKIP=true`, + saving the cargo-delta-computed exclude list from being parsed and the workspace from + being touched. +- Recipes that don't scope to workspace members **ignore** it. `fmt` still runs (the + source tree may have non-Rust files affected by the PR), `deny`/`audit`/`aprz` still + run (their outcome doesn't depend on what was changed), `pr-title` still runs. + +This separation is what makes the wiring layer durably structural. "Which checks can +no-op when nothing in the workspace is affected?" is a per-check property and lives in +the catalog/recipe, not in the wiring layer. Moving a check between groups never +requires touching the stages template / reusable workflow. + +A typical skip-aware recipe looks like: + +```just +ox-check-clippy: (_ox-check-require "cargo-clippy") + @[ "${OX_CHECK_IMPACT_SKIP:-false}" = "true" ] && echo 'no affected crates; skipping clippy' && exit 0; \ + cargo clippy --workspace ${OX_CHECK_EXCLUDE_NOT_MODIFIED:-} --all-targets --all-features --locked -- -D warnings +``` + +(On Windows, with `set shell := ["pwsh", "-NoProfile", "-Command"]`, the equivalent +short-circuit uses `if ($env:OX_CHECK_IMPACT_SKIP -eq 'true') { exit 0 }`.) + +### 4.2 Local impact-scoped runs + +Not the default. To preview what CI would skip, run cargo-delta manually and export the +env vars: + +```sh +git stash; git checkout origin/main +cargo delta -c .delta.toml snapshot > /tmp/base.json +git stash pop +cargo delta -c .delta.toml snapshot > /tmp/head.json +export OX_CHECK_EXCLUDE_NOT_AFFECTED="$(cargo delta -c .delta.toml impact \ + --baseline /tmp/base.json --current /tmp/head.json -f cargo-excludes --affected)" +just ox-check-pr-test +``` + +A wrapper recipe (`ox-check-impact-set base=origin/main`) is left to v2: it has subtle +git-state interactions and the manual flow is good enough for the rare case a developer +actually wants to reproduce CI scoping locally. + +## 5. Daily driver + +```text +$ just ox-check +[just] running ox-check-tools-check +[just] running ox-check-pr-fast +[just] running ox-check-pr-test +[just] running ox-check-pr-mutants +ox-check OK +``` + +`ox-check` is an alias for `ox-check-pr` (set in the managed `Justfile` region). All three tiers +(`ox-check-pr`, `ox-check-nightly`, `ox-check-full`) are first-class — locally reproducible with +exactly the same arguments CI uses, because CI invokes the same `just` recipes. + +## 6. No-tooling fallback + +A user with only `cargo` (no `just`, no `cargo-ox-check`) can still run the basics: + +```sh +cargo test --workspace --all-targets --all-features --locked +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +cargo fmt --check +``` + +The same commands appear as the body of the corresponding `just` recipes in +`justfiles/ox-check/checks.just`, so they are discoverable by reading that file. The fallback +covers core hygiene only — coverage, miri, mutants, etc. still require their respective +tools. + +## 7. Customization at the recipe level + +Per the four customization tiers in [design.md §7](./design.md#7-customization): + +- **Add your own recipes** to the top-level `Justfile` outside the managed region. The + Justfile's managed region only contains `import` lines and an alias — your recipes never + collide with it. +- **Add your own `.just` files** and `import` them after the managed region's closing + sentinel. +- **Override a single ox-check recipe**: the `just` import-and-override rules make this awkward + (just doesn't have a "the most specific definition wins" rule). The recommended way is to + copy the recipe you want to change into your top-level Justfile with a different name + (e.g. `my-clippy`) and reference *that* from your own group/tier recipes. Don't fight the + ox-check-* names; just compose around them. +- **Disable a recipe wholesale**: opt out of the managed `Justfile` region per + [updates.md §opt-out](./updates.md#6-opting-out-in-file-stubs). This stops the imports from + happening at all, so all `ox-check-*` recipes vanish. Use this only when ox-check is no longer + the right tool for your repo. + +Customizing the *contents* of `justfiles/ox-check/*.just` is supported — they're owned files, +so editing them flips them to "dirty" and the next `update` writes a `.ox-check-proposed` +sibling instead of overwriting. See [updates.md](./updates.md) for the lifecycle. diff --git a/crates/cargo_ox_check/docs/design/updates.md b/crates/cargo_ox_check/docs/design/updates.md new file mode 100644 index 00000000..3624703e --- /dev/null +++ b/crates/cargo_ox_check/docs/design/updates.md @@ -0,0 +1,398 @@ +# Updates, Drift Detection, and Opt-Out + +This document defines how `cargo ox-check update` decides what to write, what to leave alone, +and what to flag. The mechanism is built around a single sidecar manifest file that +captures what ox-check last wrote, plus per-region sentinel comments that let the tool find +its content again on subsequent runs. + +See also: + +- [design.md](./design.md) for the overall principles and the CLI shape. +- [local.md](./local.md) for the just-recipe files this algorithm manages. +- [github.md](./github.md) / [ado.md](./ado.md) for the CI building-block files. + +## 1. The manifest + +`.ox-check.lock` at the repo root. TOML, committed to the repo. Tracks, for every owned +file and every managed region, the checksum of what ox-check most recently rendered there. + +```toml +# .ox-check.lock — generated by cargo-ox-check. Do not edit by hand. +version = 1 +rendered_by = "cargo-ox-check 0.4.1" + +[[file]] +path = "justfiles/ox-check/checks.just" +checksum = "sha256:8f3a…" + +[[file]] +path = ".github/actions/ox-check-pr-fast/action.yml" +checksum = "sha256:b91c…" + +[[file]] +path = ".github/workflows/ox-check-pr.yml" +checksum = "sha256:c4d2…" + +[[region]] +host = "Justfile" +id = "ox-check-imports" +checksum = "sha256:1e5b…" + +[[region]] +host = "Cargo.toml" +id = "ox-check-workspace-lints" +checksum = "sha256:7d22…" + +[[region]] +host = "crates/foo/Cargo.toml" +id = "ox-check-lints" +checksum = "sha256:0a13…" +``` + +The manifest is the **single source of truth** for "what did ox-check last write." Every +decision the tool makes about whether to overwrite, propose, or leave alone is driven by +comparing three things: + +- `last_rendered_checksum` — what the manifest says ox-check wrote last time. +- `current_disk_checksum` — what is on disk right now. +- `current_template_checksum` — what ox-check's current catalog would write. + +There is no per-template version tracking and no in-file checksum line. The manifest is +small, deterministic (sorted, fixed-format), and human-readable; diffs are minimal across +ox-check version bumps. + +### When the manifest is read and written + +- `cargo ox-check update` reads the manifest at startup. Missing or unreadable manifest is + treated as a first run (every item is "never seen"). +- `cargo ox-check update` rewrites the manifest at the end of a successful run, reflecting + what was actually written (or what proposal contents were recorded). +- `cargo ox-check update --dry-run` reads the manifest but never writes. + +### Format and stability + +- TOML schema version is `version = 1`. The tool refuses to run against a newer schema + and migrates automatically from an older one. +- File entries are keyed by `path` (slash-separated, relative to repo root). +- Region entries are keyed by `(host, id)`. The `id` is the same identifier carried by + the in-file sentinel comments (`# >>> ox-check-managed: `); it's globally unique + within the ox-check catalog. See §3. +- Entries are written in a deterministic order: files alphabetically by path; regions + alphabetically by `(host, id)`. This makes `.ox-check.lock` diff-friendly across ox-check + version bumps that don't change content. + +## 2. Owned files + +Identified by **path**. The catalog (compiled into the binary) specifies the full set of +paths the tool owns. Examples: `justfiles/ox-check/checks.just`, +`.github/actions/ox-check-pr-fast/action.yml`, `.pipelines/ox-check/pr.yml`. + +There is no in-file checksum line. Owned files carry at most a single advisory comment +on line 1 (where the file's syntax allows), naming the file as ox-check-managed and +pointing readers at the manifest: + +```just +# Managed by cargo-ox-check. See .ox-check.lock at repo root. Do not edit by hand. +``` + +This warning is informational only. The manifest is the actual source of truth — a +reader can run `cargo ox-check update --dry-run` to see exactly what would happen if they +edited the file. + +## 3. Managed regions + +Co-owned files with one or more tool-managed sections delimited by sentinel comments. +Sentinels persist; the inner `# checksum:` line that earlier designs carried no longer +exists. + +```just +# >>> ox-check-managed: ox-check-imports +import 'justfiles/ox-check/checks.just' +import 'justfiles/ox-check/groups.just' +import 'justfiles/ox-check/tiers.just' +import 'justfiles/ox-check/tools.just' +alias ox-check := ox-check-pr +# <<< ox-check-managed: ox-check-imports +``` + +The sentinel pair serves two purposes: + +1. **Stable identification.** The `` is the same string ox-check's catalog uses and + matches the manifest's `[[region]].id` field. The tool finds the region by scanning + for the opening sentinel — line-number-independent, robust against user edits to + surrounding content. +2. **Body delimitation.** Everything between the sentinels (exclusive) is "the region's + body." Lines outside the sentinels are user-owned and preserved verbatim. + +For TOML hosts the sentinels are TOML line comments around the affected content. To +work with TOML's no-duplicate-table rule, ox-check writes a single parent-table header +inside each region and expresses all nested values via dotted keys; this lets users +append further entries outside the region in the same parent scope without redeclaring +any sub-table header: + +```toml +# >>> ox-check-managed: ox-check-workspace-lints +[workspace.lints] +rust.unsafe_op_in_unsafe_fn = "deny" +clippy.unwrap_used = "deny" +clippy.expect_used = "deny" +rustdoc.broken_intra_doc_links = "deny" +# <<< ox-check-managed: ox-check-workspace-lints +# User-added lints continue [workspace.lints] via dotted keys — valid TOML, ox-check +# preserves them verbatim: +clippy.pedantic = "warn" +rust.missing_docs = "warn" +``` + +Whitespace and comments inside the region are preserved verbatim by the rewrite — the +tool computes the checksum over the body bytes literally. + +### User-extension limits for TOML regions + +Three constraints follow from TOML's no-duplicate-tables rule and ox-check's chosen +dotted-key layout: + +- **Adding new keys** to the parent table from below the region works and is the + recommended extension pattern (the example above). +- **Overriding a key set inside the region** is *not* possible from outside — + repeating `clippy.unwrap_used = "warn"` below the region while ox-check wrote + `clippy.unwrap_used = "deny"` inside it is a duplicate-key TOML parse error. To + override, edit the value inside the region; the dirty-file flow (§5) takes over from + there, and ox-check will leave the user's edit alone going forward. +- **Extending a TOML array set inside the region** (e.g. `licenses.allow = [...]` in + `deny.toml`) is not possible from outside — TOML has no array-merge syntax. The + user edits inside the region (dirty flow) or empties the region and writes their own + config alongside. + +For non-TOML hosts (`Justfile`), no such constraints apply: the user can write any +additional recipes above or below the imports region. + +## 4. Per-host insertion anchors + +If a host file exists but does not contain the expected region's sentinels (and the +region is not disabled per §6), the tool inserts the region at a deterministic per-host +anchor: + +| Host | Region | Anchor for insertion | +|---------------------------------------|-----------------------------------------|---------------------------------------------------------------| +| `Justfile` | `ox-check-imports` | After leading `set …` / shebang lines, before the first user recipe. | +| Workspace `Cargo.toml` (workspace) | `ox-check-workspace-lints` | End of file (after the last existing table). | +| Workspace `Cargo.toml` (single-crate) | `ox-check-lints` | End of file. | +| Per-crate `Cargo.toml` | `ox-check-lints` | End of file. | +| `deny.toml` | `ox-check-deny` | End of file. | +| `rustfmt.toml` | `ox-check-rustfmt` | End of file. | +| `.delta.toml` | `ox-check-delta` | End of file. | + +If the host file is missing entirely (and the region is not disabled by a pre-created +stub), the tool creates it containing only the managed region. + +## 5. The decision algorithm + +For every item the catalog manages (each owned file path; each `(host, region-id)` +pair), the tool runs the same three-checksum comparison. There is no distinction between +"first run" and "subsequent run" — the absence of a manifest entry is just one possible +state — and there is no separate disable-detection pass: emptying a file or region +just produces the standard dirty-file behavior (see §6). + +Let: + +- `D` = current disk content's checksum (or *absent* if the file/region isn't there). +- `L` = `last_rendered_checksum` from `.ox-check.lock` (or *absent* if no entry). +- `T` = checksum of what the current catalog would render. + +| `D` | `L` | `T` vs `L` | Action | +|-----|-----|------------|-------------------------------------------------------------------------------------------------------------------| +| absent | absent | n/a | Render, write, record. Set `L = T`. | +| absent | present | n/a | "User deleted." Re-render and write — deletion is the re-bless gesture. Set `L = T`. | +| present | absent | n/a | "User got there first." Leave the on-disk content alone; write `.proposed` with the rendered template. Set `L = T`. The on-disk file is dirty from inception; future runs will see `D != L`. | +| present | present | `T == L` and `D == L` | Clean and up to date. No-op. | +| present | present | `T != L` and `D == L` | Clean; template changed. Overwrite the file/region with the freshly rendered content. Set `L = T`. | +| present | present | `T == L` and `D != L` | Dirty; template unchanged. Leave the user's edit alone. **No proposal** — this is the case that the old design got wrong. | +| present | present | `T != L` and `D != L` | Dirty and template changed. Leave the user's edit alone; write `.proposed`. Set `L = T`. | + +The fourth-last row is the heart of the new model: a user-claimed file with no upstream +churn produces zero noise. They claimed it; ox-check stays out of the way until ox-check's +template actually changes. + +### `.ox-check-proposed` side files + +Every proposal — for an owned file or for a managed region — is written as +`.ox-check-proposed` next to the host file. A proposal is always a **full, +ready-to-use file**, not a fragment: + +- **Owned file proposal**: the freshly rendered file content. +- **Managed-region proposal**: the host file with the affected region's body replaced + by the freshly rendered body. All content outside the sentinels is preserved + byte-for-byte from the host's current on-disk state. Other managed regions in the + same host that are not pending an update keep their current on-disk content. +- **Multiple pending regions in one host**: bundled into a single proposal file + showing what the host would look like if every pending region update were accepted. + (Common case is one region per host; bundling is the conservative fallback.) + +Concretely the user can: + +- `diff .ox-check-proposed` to see exactly what would change. +- `mv .ox-check-proposed ` to accept all pending changes wholesale. +- Open the proposal in their editor and merge selectively into `` by hand. +- Delete the proposal to reject. + +There is no separate naming scheme for region proposals — `.ox-check-proposed` is +the only file the user has to know about. The previous `..proposed` +naming was dropped because (a) it produced unhelpful fragmentary content that didn't +diff cleanly against the host, and (b) the same flat proposal-per-host convention +already worked for owned files. + +After a proposal is written, the manifest's `L` is bumped to the new `T`. This means: + +- Deleting the proposal without doing anything else dismisses the nag for the current + template revision. The next ox-check version that changes this template will write a + fresh proposal. +- Leaving the proposal in place also dismisses the nag. The proposal file just sits in + `git status` as a visible reminder; ox-check won't write another one until `T` changes + again. +- The user's copy on disk is never touched. + +**Rejection flow.** A user who wants to keep their custom version and ignore an +upstream change does nothing — or deletes the proposal — and the tool stops surfacing +this revision. The next time ox-check's template changes, a fresh proposal will appear so +the user can re-evaluate. The user never has to learn a "reject" gesture; rejection is +the default behavior of doing nothing. + +**Acceptance flow.** A user who wants to take the upstream change either: + +- Merges the proposal content into their file by hand (then deletes the proposal). + After merging, `D == T` so the next run sees clean state. +- Or deletes their copy (`rm path/to/file` or empty the region between sentinels and + then delete the sentinels). On the next run, the file/region is absent → tool + re-renders with the current template → `L = T`, `D = T`, clean. + +`.ox-check-proposed` files are **not** added to `.gitignore`. Showing up in `git status` +and diffs is the point — a proposal you can't see is a proposal you'll forget about. + +### Exit codes + +- `--dry-run` exit code 0: this run would write no proposal (clean, or dirty but + template unchanged since last render). +- `--dry-run` exit code 1: at least one item is dirty *and* the template has changed + since last render — a `.ox-check-proposed` sibling would be written. + +The same partitioning is printed at the end of every non-`--dry-run` `update`. + +## 6. Opting out: empty the file or region + +There is no separate "disable" mechanism. The dirty-file path from §5 already produces +the right behavior, so the way to opt out is to make ox-check see the file or region as +"dirty with no further content to add": empty it. + +- **Owned file**: make the file empty (zero bytes, whitespace-only, whatever — ox-check + computes a checksum either way and it won't match `L`). +- **Managed region**: leave the sentinel pair in place but empty the body: + + ```just + # >>> ox-check-managed: ox-check-imports + # <<< ox-check-managed: ox-check-imports + ``` + +The state machine handles the rest: + +- After ox-check first renders the item, `L` is set to the template's checksum. +- The user empties it. Now `D` (empty checksum) ≠ `L`. The item is "dirty." +- On the next `update`, ox-check sees `D != L`: + - `T == L` (template unchanged): no-op, no proposal. Quiet. + - `T != L` (template changed since last render): write `.proposed`, set `L = T`. + Quiet again on subsequent runs until template changes again. + +### Disabling an item the tool hasn't yet rendered + +Pre-create the empty file (zero bytes) or empty stub (region sentinels with no body) +before the first `update`. ox-check sees `D` present, `L` absent → "user got there first": +the on-disk emptiness is preserved and a one-shot `.ox-check-proposed` is written so the +user can see what they're opting out of. Delete the proposal to dismiss; thereafter +silent unless the template changes. + +### Re-enabling + +- A region: delete both sentinel lines; the next `update` re-inserts the region at the + anchor (§4) with fresh content. +- An owned file: delete the file; the next `update` recreates it. + +### No special cases + +Opting out of `ox-check-imports` will break the `just ox-check-*` recipes; that is the user's +choice. The tool does not refuse to honor an empty file or region for any ID or path. + +### Bulk handling + +There is no bulk-disable command. Empty-ing a file is a one-keystroke operation per +item, and disabling more than two or three regions is a strong signal that ox-check is +not the right fit for the repo (in which case the user should remove ox-check entirely). + +## 7. Why a manifest file + +Two alternatives were considered and rejected: + +- **Per-file checksums in line-1 comments.** Earlier design. Two problems: (a) every + managed file gets a noisy comment line; (b) claiming a file by removing the checksum + loses information about which template version was last rendered, so the tool can't + distinguish "user claimed and there's no upstream change" from "user claimed and + there is an upstream change." Both cases looked identical and forced a permanent + `.ox-check-proposed` nag. + +- **Template-version markers without checksums.** Putting only the ox-check version (e.g. + `# ox-check-version: 0.4.1`) in-file. Compact, but the tool would either need to ship + every historical template version (heavy) or only compare against the current one + (losing claiming information again). + +The sidecar manifest: + +- Captures both the **content** the tool last wrote (via checksum) and, implicitly, the + ox-check version that wrote it. +- Eliminates the claiming ambiguity: the tool always knows what it last produced, + independent of what the user has done since. +- Removes all in-file checksum noise; only a single advisory comment (and the + region-sentinel pair) remain. +- Is small, deterministic, and diff-friendly — meaningful changes in the manifest + reflect meaningful changes in what ox-check manages. + +The trade-off is one additional file in the repo. The user's experience is otherwise +unchanged: they still don't have to know about the manifest most of the time, and the +opt-out mechanism remains entirely in-file. + +## 8. Backend selection during update + +`--backend ` is a repeatable flag; valid names today are `github` and `ado`. If +omitted, the tool autodetects from the `origin` git remote URL: + +- `github.com` → `github` +- `dev.azure.com` or `*.visualstudio.com` → `ado` +- anything else, or if `origin` is missing → the tool errors out asking for an explicit + `--backend` flag. + +`--no-backends` is valid and useful for repos that want only the local `just` setup +with no CI files; it is mutually exclusive with `--backend`. Autodetection runs every +time `--backend` and `--no-backends` are both absent; there is no "first run" special +case. `update` never deletes files. To stop using a backend, the user removes the +corresponding directory by hand and trims the stale entries from `.ox-check.lock` (or +just deletes them and reruns without that backend; the tool purges manifest entries +whose catalog item is no longer enabled by the current backend set). + +## 9. Dry-run UX + +`cargo ox-check update --dry-run` performs the full analysis but writes nothing (neither +the manifest nor any file). Output is grouped by category: + +- **Will create**: items that don't exist in the manifest or on disk. +- **Will update**: clean items whose template content has changed. +- **Will leave alone (silent)**: dirty items whose template hasn't changed since last + render. No proposal will be written. +- **Will propose**: dirty items whose template *has* changed; a `.ox-check-proposed` + sibling will be written. Includes both user-edited and emptied/opted-out items — + emptiness has no special status in the algorithm. +- **Unchanged**: clean items whose template is identical to last render. +- **Stale manifest entries**: paths or `(host, id)` pairs in `.ox-check.lock` that the + current catalog no longer manages (e.g. removed by ox-check version bump, or by + changing the enabled backend set). These are purged on the next non-dry-run. + +Exit code 0 if nothing is dirty-and-template-changed; exit code 1 otherwise. The same +output format is printed at the end of a non-dry-run `update`, summarizing what +actually happened. diff --git a/crates/cargo_ox_check/docs/verification.md b/crates/cargo_ox_check/docs/verification.md new file mode 100644 index 00000000..a965b62b --- /dev/null +++ b/crates/cargo_ox_check/docs/verification.md @@ -0,0 +1,256 @@ +# Continuous Validation Strategy + +This document defines how `cargo-ox-check` is kept correct over time. The headline mechanism +is dogfooding — the `microsoft/ox-tools` repo, where cargo-ox-check itself lives, uses +`cargo ox-check update` to manage its own CI. Every PR that touches the catalog or the +emitters produces a visible diff in `.github/` and `justfiles/ox-check/`, then runs through +the regenerated CI on the same commit. A broken emitter or catalog fails the PR's own +checks immediately. + +See also: + +- [design/](./design/) — the tool's design. +- [design/updates.md](./design/updates.md) — the state machine validated by fixture tests. +- [design/checks.md](./design/checks.md) — the catalog dogfooded by ox-tools. + +## 1. Goals + +- **Detect regressions on the PR that introduces them.** No "this broke a downstream repo" + surprises after a release. +- **Validate the whole pipeline**, not just unit-level behavior: catalog → templates → + manifest → emitted CI → CI actually running. +- **Cover the state machine in [updates.md §5](./design/updates.md#5-the-decision-algorithm) + exhaustively** — every row of the decision table is exercised by some test. +- **Keep validation cheap** — most of it runs in the PR pipeline; nothing requires a + bespoke test environment. + +## 2. Layers + +### 2.1 Self-hosting (primary) + +`microsoft/ox-tools` is the canonical adopter of `cargo-ox-check`. Its `.github/workflows/`, +`.github/actions/`, `justfiles/ox-check/`, `[workspace.lints]` region in `Cargo.toml`, etc. +are all emitted by `cargo ox-check update` against the in-repo version of the binary. There +is no manual maintenance of these files after the initial migration. + +Every PR runs (via a small bootstrap workflow described in §3): + +1. `cargo build --locked -p cargo-ox-check` — build the binary from source. +2. `target/debug/cargo-ox-check ox-check update` — regenerate every owned file and managed region. +3. `git diff --exit-code` — fail with a clear message if regeneration produced changes the + PR didn't commit. +4. Continue into the normal `ox-check-pr` workflow, which is itself the freshly regenerated + workflow file. + +What this validates end-to-end: + +- The catalog renders to valid YAML / TOML / `just`. +- The manifest's three-checksum state machine produces idempotent output (rerunning + `update` with no changes is a no-op). +- Every emitted CI building block actually runs — broken composite actions, broken + reusable workflows, broken step templates surface immediately. +- The full default check catalog is exercised on every PR. ox-tools deliberately enables + every catalog check (no opt-out stubs) and the default cross-OS matrix (Linux + + Windows for test groups). + +What this doesn't catch — see §2.4. + +### 2.2 Fixture-based integration tests + +Under `crates/cargo_ox_check/tests/fixtures/`, a small set of fixture repos covers shapes +ox-tools doesn't have. Each fixture is a directory tree plus an `expected/` snapshot. +The test runner copies the fixture to a tmpdir, runs `cargo ox-check update`, and asserts +byte-equal output against `expected/`. + +Initial fixture set: + +| Fixture | Purpose | +|--------------------|-------------------------------------------------------------------------------------------| +| `fresh/` | Empty repo. Exercises the "first run, no manifest" creation paths. | +| `single-crate/` | Non-workspace repo. Validates the `[lints]` (vs `[workspace.lints]`) branch. | +| `simple-workspace/`| Two-member workspace. The everyday case mirroring ox-tools at small scale. | +| `opt-outs/` | Empty owned files, empty region bodies. Validates the "no proposal until template change" rule. | +| `customized/` | Dirty owned files and dirty regions. Validates the dirty-flow including `.ox-check-proposed` emission. | +| `migration/` | A repo with an old manifest schema and pre-existing emitted content. Validates the on-load migration logic. | + +Each fixture is exercised by at least three independent assertions: + +- **Idempotence** — running `update` twice in a row produces zero diff on the second run. +- **Determinism** — running `update` against two identical clones produces identical + output (no time-, env-, or path-dependence). +- **Manifest consistency** — after a clean run, every entry in `.ox-check.lock` matches the + checksum of the corresponding on-disk content. + +The fixture corpus grows when a bug is fixed: the bug's repro becomes a fixture before +the fix lands. + +### 2.3 Schema validation + +Run as part of `ox-check-pr-fast` against ox-tools's emitted output (and as part of each +fixture's assertion suite): + +- **`actionlint`** on every emitted `.github/workflows/*.yml` and + `.github/actions/*/action.yml`. Catches GitHub-Actions-specific errors that plain + YAML validation misses. +- **`just --summary --unstable`** on every `justfiles/ox-check/*.just`. Verifies recipes + parse and dependency graph is well-formed. +- **`taplo check`** on every TOML file ox-check writes to. Verifies the post-edit file is + still parsable TOML and conforms to the cargo schema where applicable. +- **ADO YAML**: no widely-available local validator. The fixture snapshots are the + contract; the manual release checklist (§2.4) covers semantic verification against + real ADO. We accept this gap because ox-tools cannot dogfood ADO emission anyway. + +### 2.4 Manual release verification + +Three things ox-tools dogfooding doesn't catch, addressed by a pre-release checklist +maintained in `docs/release-checklist.md`: + +- **Compliance-extending ADO pipelines** (1ESPT, SubstratePT, CloudBuild). Validated + manually by running `cargo ox-check update --dry-run` against `oxidizer`, + `assistants-oxide`, and `ox-docs` (internal mirrors) and inspecting the diff. If the + diff looks right, queue a buddy build to confirm the regenerated pipeline still + passes. +- **Cross-repo migrations**. Each release that bumps the manifest schema or renames a + catalog item runs `update --dry-run` against every surveyed repo and confirms the + proposed migration is correct. +- **Self-hosted runners and non-default matrices**. Spot-checked against `oxidizer`'s + Microsoft-pool builds before each release. + +The release checklist is a literal markdown file in the repo; checking it off is part +of the publish PR. + +## 3. The PR-time workflow + +ox-tools's `.github/workflows/ox-check-pr.yml` (post-migration) wraps the regenerated +`ox-check-pr-impl.yml` with a small self-validation gate. Sketch: + +```yaml +name: ox-check-pr +on: + pull_request: {} + merge_group: {} +permissions: { contents: read } +jobs: + regenerate-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - run: cargo build --locked -p cargo-ox-check + - name: Regenerate emitted files + run: ./target/debug/cargo-ox-check ox-check update + - name: Assert no drift + run: | + if ! git diff --exit-code; then + echo "::error::cargo-ox-check changed files. Run 'cargo ox-check update' locally and commit the diff." + exit 1 + fi + + ox-check: + needs: regenerate-check + uses: ./.github/workflows/ox-check-pr-impl.yml +``` + +The `regenerate-check` job runs first. If a PR changes the catalog or emitter without +also committing the regenerated output, this fails with an actionable message. After +that, the standard `ox-check-pr-impl.yml` reusable workflow runs every group, exactly as +in any consumer repo. + +The wrapper workflow above is the **one** hand-written workflow in ox-tools — it +bootstraps the dogfood loop. Every other CI artifact is regenerated. + +## 4. Bootstrap and breaking changes + +### Initial bootstrap + +The very first cargo-ox-check PR cannot dogfood itself — the binary doesn't exist yet. +Bootstrap plan: + +1. PR #1: lands the binary's skeleton (current state, no `update` logic yet) plus + hand-written workflows for the unit and integration tests. ox-tools's CI is still + hand-written. +2. PR #N (first usable `update`): lands the emitter implementation. Run + `cargo ox-check update` locally, commit the diff, push. From this point forward + ox-tools is self-hosted. +3. PR #N+1 onward: every PR runs the regenerate-check gate. + +### Breaking changes inside cargo-ox-check + +Two flavors require care: + +- **Manifest-schema bumps.** Migration logic must be in place before any release that + needs it. The `migration/` fixture exercises old-schema-to-new-schema upgrades. + Release notes call out the bump. +- **Renames in the emitted CI surface** (e.g., `ox-check-pr-fast` → `ox-check-pr-static`). + Treated as major-version bumps. The PR introducing the rename is split into two + commits: (a) implement, (b) `cargo ox-check update` to regenerate. Downstream repos do + the same two-step on adoption. + +Both flavors are caught by ox-tools's own regenerate-check: a missing migration or a +rename that doesn't round-trip will produce drift on the second run, failing the gate. + +### Recovering from a self-inflicted breakage + +If a PR lands that breaks the regenerate-check (because reviewers missed it), the +breakage is **not stuck** — every PR builds cargo-ox-check from source on its own branch, +so the fix PR is free to either revert the offending change or land the missing +regenerated output, and its own CI will pass cleanly. + +What is affected: unrelated PRs that branched off the broken commit will fail their +regenerate-check, because they inherit the drift through the merge base. They recover +by rebasing past the fix. + +Procedure: + +1. Open a fix PR. Either: (a) revert the offending commit, or (b) commit the missing + `cargo ox-check update` output. Either flavor builds the binary from the fix branch + and produces a clean `git diff` against its own tree, so CI passes. +2. Merge. +3. In-flight PRs rebase to pick up the fix; their regenerate-check passes once their + merge base is past the fix commit. + +ox-tools never depends on the published crates.io version of cargo-ox-check for its own +checks — it always builds from source. So a broken release on crates.io doesn't +cascade into ox-tools's CI; only a broken `main` does, and only for unrelated +in-flight PRs. + +## 5. Coverage gaps + +Acknowledged limits of this strategy: + +- **1ESPT/SubstratePT/CloudBuild composition** — ox-tools is OSS; it cannot dogfood + internal compliance harnesses. Manual release checklist covers this. +- **Self-hosted runner pools** — ox-tools uses GH-hosted; the `runs_on` input is set + to defaults. Self-hosted shapes are documented but exercised only by the manual + release checklist. +- **macOS** — not in the default matrix (see [design.md §8.3](./design/design.md#83-cross-os-test-matrices)); + not dogfooded. +- **Very deep workspaces or unusual layouts** — covered only by fixtures, not by + real-world traffic. New layouts that adopters surface become new fixtures. +- **Long-lived divergence** — a repo that's been on an old cargo-ox-check version for + many releases is only validated by the cross-repo migration step in §2.4. + +## 6. Files and locations + +| Path | Purpose | +|-----------------------------------------------------|-------------------------------------------------------------------------| +| `crates/cargo_ox_check/tests/fixtures/` | Integration test fixtures (one directory per shape). | +| `crates/cargo_ox_check/tests/update.rs` | Test runner: per-fixture idempotence/determinism/consistency assertions. | +| `crates/cargo_ox_check/tests/schema.rs` | actionlint / taplo / just-parse wrappers run against ox-tools and fixtures. | +| `.github/workflows/ox-check-pr.yml` | Hand-written self-validation wrapper (the one bootstrap file). | +| `.github/workflows/ox-check-pr-impl.yml` (and friends) | Regenerated by `cargo ox-check update`. Subject to the regenerate-check. | +| `justfiles/ox-check/*.just` | Regenerated. Subject to the regenerate-check. | +| `Cargo.toml` (ox-check-workspace-lints region) | Regenerated. Subject to the regenerate-check. | +| `.ox-check.lock` | The manifest itself. Diffed on every PR. | +| `docs/release-checklist.md` | Pre-publish checks for things dogfooding misses. | + +## 7. Future work + +- A dedicated `cargo ox-check verify` subcommand that runs `update --dry-run` + all + schema validators + the manifest-consistency check in one step, for local + pre-commit use. +- A small set of "downstream canary" repos in `microsoft/` org that pin cargo-ox-check + to `main` (not to a release) and report failures via issues. Catches regressions + earlier than the manual release checklist. +- Pre-release smoke runs of every adopter's CI against a release candidate, gated by + a draft pre-release tag. diff --git a/crates/cargo_ox_check/src/main.rs b/crates/cargo_ox_check/src/main.rs new file mode 100644 index 00000000..3d4016ca --- /dev/null +++ b/crates/cargo_ox_check/src/main.rs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! `cargo ox-check` — opinionated, unified Rust build/CI scaffolding. +//! +//! See `docs/design/design.md` (and its sibling files) for the full design. + +fn main() { + eprintln!("cargo-ox-check: not yet implemented; see docs/design/design.md"); + std::process::exit(1); +}