diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 7752d3039..c35ecb4de 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -4,6 +4,8 @@ examine_globs = ["crates/**"] exclude_globs = [ "crates/cargo-aprz-lib/**", + # Remove when cargo-gamma runs on itself; until then neither mutation engine covers these crates. + "crates/cargo-gamma*/**", "crates/testing_aids/**", ] all_features = true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e271fffb7..3465cdcd2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -253,7 +253,8 @@ jobs: enable-sccache: true rust-toolchain: RUST_NIGHTLY rust-components: llvm-tools-preview - cargo-tools: CARGO_LLVM_COV_VERSION + # Some coverage tests shell out to nextest and require it to be installed. + cargo-tools: CARGO_LLVM_COV_VERSION, CARGO_NEXTEST_VERSION # execute - name: Generate Coverage (all-features) diff --git a/.spelling b/.spelling index b1af03763..146a83b5e 100644 --- a/.spelling +++ b/.spelling @@ -12,6 +12,9 @@ 1ms ACLs accessor +accessors +bitset +catchable acyclic addrs ado @@ -36,6 +39,9 @@ autodetect autodetected autodetection autodetects +memoizes +Tarjan +uncached B6 backend backends @@ -92,6 +98,7 @@ clonable cobertura codebase coverable +codebase's codebases Codecov Codecov's @@ -330,6 +337,8 @@ setsockopt shareable skimmable SLAs +skippable +handoff smallvec spawner SPDX @@ -521,3 +530,355 @@ deprecations parallelization remediate recency +100% +20% +313 +40% +5% +60% +API's +Absolutising +CDN +CLI +Cartography +Const +De +Destructuring +E0381 +Fortran +GitLab +Harrold +Hinnant's +Honouring +Insta +Memoization +Metadata +Nextest +Nextest's +OOM +Offutt +POSIX +Recognised +SARIF +Schemata +Serialises +Signalled +Stryker +Symlinks +TOML +UI's +Unbuilt +Unrecognised +Untch +Untracked +abandonable +absolutising +allowlist +arity +askable +behaviour +behavioural +booleans +callee's +canonicalising +canonicalization +cartography +cgroup +cgroup's +cgroups +checkable +codenames +combinator +combinator's +compilable +constructible +creatable +deduplicated +defence +degradations +dereference +dereferencing +destructure +detections +dev +diagnostic's +diff's +discoverability +else's +erroring +evaluator +executables +fallbacks +favour +filterset +fixpoint +flavour +getenv +getter +git's +gradeable +greppable +honour +honoured +honouring +hunked +implementor +initialised +initialiser +initialisers +initialising +initializers +installable +instrumenter +judgement +judgements +killable +labelled +libtest's +linkable +makefile +misparse +mispredicts +mistyped +modelled +multithreaded +musl +mutator +mutators +natively +neighbours +nextest's +normalisation +normalised +npm +observationally +other's +outlier +outsized +parsers +pgid +pid +pids +pointee +positionally +postcondition +prebuilt +preflight +prepass +proc +reachability +reborrow +reborrowed +reborrows +recognisable +recognise +recognised +recursing +reentrant +refactor +reflinks +reflows +regenerable +reimplementing +reinitialization +revalidated +reviewable +reviewably +rewrapped +rlibs +runtime's +rustc's +rustflags +scheduler's +schema's +schemata +scorable +scrollbars +serde +serializer's +sharded +sharding +signalled +signalling +styler +subpattern +subtree's +subtrees +superset +suppressions +symlink +symlink's +syntaxes +sysroot +systemd +tokenise +toolchain's +tty +tuple's +turbofish +unaddable +unanchored +unary +unattributable +unattributably +unbuildable +unbuilt +uncompilable +uncompiled +undecisive +undelegated +unescaping +unexecuted +unfalsifiable +ungradeable +uninitialised +uninstrumented +unkillable +unlinked +unmetered +unmutatable +unmutated +unoffered +unopenable +unparsable +unreached +unrecognised +unrequested +unresolvable +unsharded +unstyled +unsuppressed +unterminated +untracked +untyped +unviability +unviable +unwritable +v1 +validator +vendored +vendoring +wakeup +worktree +§6.0's +— +% +0% +0o600 +0o644 +11% +150% +50% +7% +80% +96% += +Absolutizes +AST +backtick +behaviours +BOM +btrfs +callees +camelCase +canonicalised +CIFS +colour +condvar +converger +counterfactual +decrement +Dekker's +dependents' +destructor +dialogs +DLL +doctests +doctest +enqueueing +escapable +ext4 +F2 +F4's +FFI +filesystem's +filtersets +fireable +glibc +honours +initializer +inode +judgeable +len +loopback +lossily +lossy +maintainer's +misclassify +monopolising +mtime +mtimes +mutator's +NFS +nondeterministic +NTFS's +NUL +onwards +optimisation +oversized +parser's +postfix +precheck +preflight's +readably +recurse +recurses +relabelled +rescales +rescans +resizes +rustflag +SIGKILL +spawner's +subprocess +symlinks +syn's +teardown +tokenising +tokenized +tradeoff +unannotated +unapplied +unbuffered +unconfirmable +Uncontained +unescape +unescaped +unhandled +unhinted +unioned +unix +Unix's +unjudged +unmeasurable +unoptimized +unprinted +unpublishable +unreaped +unrepresentable +unreviewable +unrun +unsatisfiable +unvisited +unwatched +VCS +wasm +weakenings +worklist +XFS +~ +Baselining +cryptographic +groupable +memoization +triaging diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a7bd45be..eb79af8c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,4 +9,5 @@ Please see each crate's change log below: - [`cargo-each`](./crates/cargo-each/CHANGELOG.md) - [`cargo-ensure-no-cyclic-deps`](./crates/cargo_ensure_no_cyclic_deps/CHANGELOG.md) - [`cargo-ensure-no-default-features`](./crates/cargo-ensure-no-default-features/CHANGELOG.md) +- [`cargo-gamma`](./crates/cargo-gamma/CHANGELOG.md) - [`cargo-heather`](./crates/cargo-heather/CHANGELOG.md) diff --git a/Cargo.lock b/Cargo.lock index 1f04e50b8..d88661cc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -101,9 +101,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" @@ -237,9 +237,22 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] [[package]] name = "block-buffer" @@ -250,6 +263,96 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bolero" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff44d278fc0062c95327087ed96b3d256906d1d8f579e534a3de8d6b386913a" +dependencies = [ + "bolero-afl", + "bolero-engine", + "bolero-generator", + "bolero-honggfuzz", + "bolero-kani", + "bolero-libfuzzer", + "cfg-if", + "rand 0.9.5", +] + +[[package]] +name = "bolero-afl" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9bf4cbd0bacf9356d3c7e5d9d088480f2076ba3c595c15ee9a6a378cdd7b297" +dependencies = [ + "bolero-engine", + "cc", +] + +[[package]] +name = "bolero-engine" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca199170a7c92c669c1019f9219a316b66bcdcfa4b36cac5a460a4c1a851aba" +dependencies = [ + "anyhow", + "bolero-generator", + "lazy_static", + "pretty-hex", + "rand 0.9.5", + "rand_xoshiro", +] + +[[package]] +name = "bolero-generator" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98a5782f2650f80d533f58ec339c6dce4cc5428f9c2755894f98156f52af81f2" +dependencies = [ + "bolero-generator-derive", + "rand_core 0.9.5", +] + +[[package]] +name = "bolero-generator-derive" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a21a3b022507b9edd2050caf370d945e398c1a7c8455531220fa3968c45d29e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "bolero-honggfuzz" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a118ef27295eddefadc6a99728ee698d1b18d2e80dc4777d21bee3385096ffd" +dependencies = [ + "bolero-engine", +] + +[[package]] +name = "bolero-kani" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852ea5784a9f3e68bfd302ca80b8b863bce140593eb5770fee6ab110899c28fc" +dependencies = [ + "bolero-engine", +] + +[[package]] +name = "bolero-libfuzzer" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "858dc57c11725c52662501fa79fdbc6f7050339a05ca1bf1e587add0fed40d62" +dependencies = [ + "bolero-engine", + "cc", +] + [[package]] name = "borsh" version = "1.8.0" @@ -262,13 +365,13 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", "regex-automata", - "serde", + "serde_core", ] [[package]] @@ -311,7 +414,7 @@ dependencies = [ "serial_test", "sha2", "tempfile", - "toml_edit", + "toml_edit 0.25.13+spec-1.1.0", "tracing", "tracing-subscriber", "walkdir", @@ -442,6 +545,106 @@ dependencies = [ "toml", ] +[[package]] +name = "cargo-gamma" +version = "0.1.0" +dependencies = [ + "cargo-gamma-lib", + "clap", + "mimalloc", + "terminal_size", +] + +[[package]] +name = "cargo-gamma-attrs" +version = "0.1.0" +dependencies = [ + "cargo-gamma-attrs-impl", +] + +[[package]] +name = "cargo-gamma-attrs-impl" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "cargo-gamma-engine" +version = "0.1.0" +dependencies = [ + "blake3", + "bolero", + "camino", + "compact_str", + "proc-macro2", + "rustc-hash 2.1.3", + "rustc_lexer", + "serde", + "strsim", + "syn 2.0.119", + "walkdir", +] + +[[package]] +name = "cargo-gamma-lib" +version = "0.1.0" +dependencies = [ + "blake3", + "bolero", + "camino", + "cargo-gamma-attrs-impl", + "cargo-gamma-engine", + "cargo-gamma-process", + "cargo-gamma-rt", + "cargo-gamma-unsafe", + "cargo_metadata", + "clap", + "clap_complete", + "compact_str", + "ignore", + "libc", + "loom", + "owo-colors", + "proc-macro2", + "reflink-copy", + "rustc-hash 2.1.3", + "serde", + "serde_json", + "syn 2.0.119", + "tempfile", + "toml", + "toml_edit 0.25.13+spec-1.1.0", + "walkdir", +] + +[[package]] +name = "cargo-gamma-process" +version = "0.1.0" +dependencies = [ + "camino", + "cargo-gamma-unsafe", + "tempfile", +] + +[[package]] +name = "cargo-gamma-rt" +version = "0.1.0" +dependencies = [ + "loom", +] + +[[package]] +name = "cargo-gamma-unsafe" +version = "0.1.0" +dependencies = [ + "libc", + "loom", + "tempfile", + "windows-sys 0.61.2", +] + [[package]] name = "cargo-heather" version = "0.3.0" @@ -488,7 +691,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -502,9 +705,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.2" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ "find-msvc-tools", "jobserver", @@ -556,7 +759,7 @@ checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -573,9 +776,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -583,9 +786,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -594,16 +797,25 @@ dependencies = [ "terminal_size", ] +[[package]] +name = "clap_complete" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be2ad0423bdbbb0e25bc89add796f3559706d4a95e1bc98e4d9662a957b6a19" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -688,6 +900,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.10.1" @@ -730,9 +948,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] @@ -857,7 +1075,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -914,7 +1132,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -943,9 +1161,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "encode_unicode" @@ -1022,9 +1240,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "filetime" @@ -1038,9 +1256,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "fixedbitset" @@ -1104,9 +1322,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -1119,9 +1337,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -1129,15 +1347,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -1146,38 +1364,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -1190,6 +1408,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1215,17 +1448,27 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", - "r-efi", - "rand_core", + "r-efi 5.3.0", "wasip2", - "wasip3", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasm-bindgen", ] @@ -1281,7 +1524,7 @@ dependencies = [ "gix-worktree-stream", "nonempty", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1308,7 +1551,7 @@ dependencies = [ "gix-trace", "kstring", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", "unicode-bom", ] @@ -1371,7 +1614,7 @@ dependencies = [ "gix-ref", "gix-sec", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", "unicode-bom", ] @@ -1381,11 +1624,11 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "bstr", "gix-path", "libc", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1403,7 +1646,7 @@ dependencies = [ "gix-sec", "gix-trace", "gix-url", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1427,7 +1670,7 @@ dependencies = [ "bstr", "gix-hash", "gix-object", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1442,7 +1685,7 @@ dependencies = [ "gix-path", "gix-ref", "gix-sec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1470,7 +1713,7 @@ dependencies = [ "once_cell", "parking_lot", "prodash", - "thiserror 2.0.18", + "thiserror 2.0.20", "walkdir", "zlib-rs", ] @@ -1493,7 +1736,7 @@ dependencies = [ "gix-trace", "gix-utils", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1507,7 +1750,7 @@ dependencies = [ "gix-features", "gix-path", "gix-utils", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1516,7 +1759,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "bstr", "gix-features", "gix-path", @@ -1531,7 +1774,7 @@ dependencies = [ "faster-hex", "gix-features", "sha1-checked", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1564,7 +1807,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "bstr", "filetime", "fnv", @@ -1583,7 +1826,7 @@ dependencies = [ "memmap2", "rustix", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1594,7 +1837,7 @@ checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" dependencies = [ "gix-tempfile", "gix-utils", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1603,7 +1846,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "gix-commitgraph", "gix-date", "gix-hash", @@ -1627,7 +1870,7 @@ dependencies = [ "gix-validate", "itoa", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1648,7 +1891,7 @@ dependencies = [ "memmap2", "parking_lot", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1669,7 +1912,7 @@ dependencies = [ "memmap2", "parking_lot", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", "uluru", ] @@ -1682,19 +1925,19 @@ dependencies = [ "bstr", "faster-hex", "gix-trace", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] name = "gix-path" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751d6bd162106f8c1e7e9aaccb5bbdd605267e91a930a17a4560c46e33a9100c" +checksum = "2b075e730586bba7341304d6fc1b4efc1d10cf64532622521c0e07f30e661046" dependencies = [ "bstr", "gix-trace", "gix-validate", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1703,13 +1946,13 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "bstr", "gix-attributes", "gix-config-value", "gix-glob", "gix-path", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1722,7 +1965,7 @@ dependencies = [ "gix-config-value", "parking_lot", "rustix", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1748,7 +1991,7 @@ dependencies = [ "gix-utils", "maybe-async", "nonempty", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1779,7 +2022,7 @@ dependencies = [ "gix-utils", "gix-validate", "memmap2", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1795,7 +2038,7 @@ dependencies = [ "gix-revision", "gix-validate", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1804,7 +2047,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "bstr", "gix-commitgraph", "gix-date", @@ -1830,7 +2073,7 @@ dependencies = [ "gix-hashtable", "gix-object", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1839,7 +2082,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af4fe6c152c1d50aea36f299825702cd37e303307832fec1d0fdd5844e47ce2f" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "gix-path", "libc", "windows-sys 0.61.2", @@ -1855,7 +2098,7 @@ dependencies = [ "gix-hash", "gix-lock", "nonempty", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1870,7 +2113,7 @@ dependencies = [ "gix-pathspec", "gix-refspec", "gix-url", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1907,7 +2150,7 @@ dependencies = [ "gix-sec", "gix-url", "reqwest", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1916,7 +2159,7 @@ version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "gix-commitgraph", "gix-date", "gix-hash", @@ -1924,7 +2167,7 @@ dependencies = [ "gix-object", "gix-revwalk", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1936,25 +2179,25 @@ dependencies = [ "bstr", "gix-path", "percent-encoding", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] name = "gix-utils" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1795bd2a970ca8b2185318c2abb97d955c71992f1cf28de73ad3b593a9f3ce8" +checksum = "0da1c46491b49458a446cc76f0085860f8164c2290742e0aa8c653ce67240a97" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "unicode-normalization", ] [[package]] name = "gix-validate" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a034e84d1e04e1b1f20f51f12491da230b6ac8b925d0c8e1b89bcd87a7c5ccc" +checksum = "4dae8780f63ed8a803b8bdabbd7aa5f5c5d74592c8b50eed875c1bb4f6545a6a" dependencies = [ "bstr", ] @@ -1992,7 +2235,7 @@ dependencies = [ "gix-path", "gix-worktree", "io-close", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2013,11 +2256,24 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "globset" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "h2" -version = "0.4.16" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -2131,9 +2387,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -2256,9 +2512,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -2270,9 +2526,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -2283,9 +2539,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2297,16 +2553,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2317,15 +2574,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -2336,12 +2593,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "idna" version = "1.1.0" @@ -2363,6 +2614,22 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -2371,8 +2638,6 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", - "serde", - "serde_core", ] [[package]] @@ -2388,13 +2653,14 @@ dependencies = [ [[package]] name = "insta" -version = "1.47.2" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "once_cell", "regex", "similar", + "strip-ansi-escapes", "tempfile", ] @@ -2500,7 +2766,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -2543,7 +2809,7 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "libc", ] @@ -2594,20 +2860,14 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dab4f1d99e0dd8008da0d3452bb71ce08e1645465c2accc124bcb599c6b3143" dependencies = [ - "thiserror 2.0.18", + "thiserror 2.0.20", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libmimalloc-sys" @@ -2620,9 +2880,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" dependencies = [ "libc", ] @@ -2646,9 +2906,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -2661,9 +2921,22 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] [[package]] name = "lru-slab" @@ -2680,6 +2953,15 @@ dependencies = [ "libc", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "maybe-async" version = "0.2.11" @@ -2693,9 +2975,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" @@ -2772,13 +3054,13 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ecce9d566cb9234ae3db9e249c8b55665feaaf32b0859ff1e27e310d2beb3d8" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "combine", "libc", "mach2", "nix", "sysctl", - "thiserror 2.0.18", + "thiserror 2.0.20", "widestring", "windows 0.48.0", ] @@ -2804,7 +3086,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -2826,6 +3108,15 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2869,7 +3160,7 @@ checksum = "565bae93429e274bfbb568535b43dd6458538908014e44faec44dddfafcb5fd3" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2972,9 +3263,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "platforms" @@ -2987,9 +3278,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" @@ -3002,9 +3293,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -3015,6 +3306,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "predicates" version = "3.1.4" @@ -3043,20 +3343,26 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.37" +name = "pretty-hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a65843dfefbafd3c879c683306959a6de478443ffe9c9adf02f5976432402d7" + +[[package]] +name = "proc-macro-crate" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24" dependencies = [ - "proc-macro2", - "syn 2.0.119", + "toml_datetime 0.6.3", + "toml_edit 0.20.2", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -3084,7 +3390,7 @@ dependencies = [ "rustc-hash 2.1.3", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -3092,22 +3398,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.4.2", + "getrandom 0.4.3", "lru-slab", - "rand", + "rand 0.10.2", "rand_pcg", "ring", "rustc-hash 2.1.3", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3129,13 +3435,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -3208,6 +3520,16 @@ dependencies = [ "triomphe", ] +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.2" @@ -3215,8 +3537,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", - "getrandom 0.4.2", - "rand_core", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -3231,7 +3572,16 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", ] [[package]] @@ -3266,7 +3616,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", ] [[package]] @@ -3277,14 +3627,26 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 2.0.20", +] + +[[package]] +name = "reflink-copy" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9dd7ab4af0363d5ccfd2838d782a28196cf32a5cc2e4fe3c5dc83f2be588b8b" +dependencies = [ + "cfg-if", + "libc", + "rustix", + "windows 0.62.2", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -3294,9 +3656,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -3305,9 +3667,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -3432,6 +3794,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "781442f29170c5c93b7185ad559492601acdc71d5bb0706f5868094f45cfcd08" +[[package]] +name = "rustc_lexer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86aae0c77166108c01305ee1a36a1e77289d7dc6ca0a3cd91ff4992de2d16a5" +dependencies = [ + "unicode-xid", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -3527,7 +3898,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -3599,9 +3970,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -3625,7 +3996,7 @@ dependencies = [ "semver", "serde", "tame-index", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", "toml", "url", @@ -3661,6 +4032,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -3686,7 +4063,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "core-foundation", "core-foundation-sys", "libc", @@ -3715,9 +4092,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -3725,29 +4102,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -3784,7 +4161,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -3938,6 +4315,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3984,9 +4370,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -4019,7 +4405,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.1", "byteorder", "enum-as-inner", "libc", @@ -4045,7 +4431,7 @@ dependencies = [ "serde", "serde_json", "smol_str", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "toml-span", "twox-hash", @@ -4068,7 +4454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.52.0", @@ -4107,11 +4493,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -4127,13 +4513,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -4144,9 +4530,9 @@ checksum = "9e1d62dcaf3eec4b3d50494a244af37d0e169c37075ccfe02144ad98e86c6901" [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -4195,9 +4581,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -4242,7 +4628,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -4271,17 +4657,17 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.2+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow 1.0.3", + "winnow 1.0.4", ] [[package]] @@ -4293,6 +4679,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -4302,6 +4694,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" +dependencies = [ + "indexmap", + "toml_datetime 0.6.3", + "winnow 0.5.40", +] + [[package]] name = "toml_edit" version = "0.25.13+spec-1.1.0" @@ -4309,19 +4712,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow 1.0.3", + "winnow 1.0.4", ] [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow 1.0.3", + "winnow 1.0.4", ] [[package]] @@ -4352,7 +4755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags 2.11.1", + "bitflags 2.13.1", "bytes", "futures-core", "futures-util", @@ -4397,6 +4800,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", ] [[package]] @@ -4405,9 +4820,16 @@ version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", "sharded-slab", + "smallvec", "thread_local", + "tracing", "tracing-core", + "tracing-log", ] [[package]] @@ -4533,14 +4955,20 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -4559,6 +4987,15 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ab49d5c3685992922df5fa16c8f5fcb9fc252baf373f0bf0f37c2121f119c2d" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "wait-timeout" version = "0.2.1" @@ -4595,20 +5032,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -4666,28 +5094,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.5.0" @@ -4701,18 +5107,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.11.1", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "web-sys" version = "0.3.104" @@ -5111,6 +5505,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.15" @@ -5119,9 +5522,9 @@ checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ "memchr", ] @@ -5149,15 +5552,6 @@ dependencies = [ "url", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" @@ -5165,111 +5559,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "wit-bindgen-core" -version = "0.51.0" +name = "writeable" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] -name = "wit-bindgen-rust" -version = "0.51.0" +name = "yoke" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn 2.0.119", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", + "stable_deref_trait", + "yoke-derive", + "zerofrom", ] [[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" +name = "yoke-derive" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ - "anyhow", - "prettyplease", "proc-macro2", "quote", "syn 2.0.119", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.11.1", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", + "synstructure", ] [[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.3" +name = "zerocopy" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", + "zerocopy-derive", ] [[package]] -name = "yoke-derive" -version = "0.8.2" +name = "zerocopy-derive" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", ] [[package]] @@ -5301,9 +5636,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -5312,9 +5647,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -5323,13 +5658,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -5354,9 +5689,9 @@ checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index bc5fab389..67ee14a99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,14 +25,25 @@ homepage = "https://github.com/microsoft/ox-tools" # external dependencies anyhow = { version = "1.0.100", default-features = false } assert_cmd = { version = "2.2.0", default-features = false } +async-once-cell = { version = "0.5", default-features = false } +blake3 = { version = "1.8.5", default-features = false } +bolero = { version = "0.13.4", default-features = false } bytes = { version = "1.11.1", default-features = false } camino = { version = "1.2.5", default-features = false } # local dependencies cargo-aprz-lib = { path = "crates/cargo-aprz-lib", default-features = false, version = "1.1.0" } +cargo-gamma-attrs-impl = { path = "crates/cargo-gamma-attrs-impl", default-features = false, version = "0.1.0" } +cargo-gamma-engine = { path = "crates/cargo-gamma-engine", default-features = false, version = "0.1.0" } +cargo-gamma-lib = { path = "crates/cargo-gamma-lib", default-features = false, version = "0.1.0" } +cargo-gamma-process = { path = "crates/cargo-gamma-process", default-features = false, version = "0.1.0" } +cargo-gamma-rt = { path = "crates/cargo-gamma-rt", default-features = false, version = "0.1.0" } +cargo-gamma-unsafe = { path = "crates/cargo-gamma-unsafe", default-features = false, version = "0.1.0" } +cargo-heather = { path = "crates/cargo-heather", default-features = false, version = "0.2.1" } cargo_metadata = { version = "0.23.1", default-features = false } cel-interpreter = { version = "0.10.0", default-features = false } chrono = { version = "0.4.40", default-features = false } clap = { version = "4.5.60", default-features = false } +clap_complete = { version = "4.6.8", default-features = false } compact_str = { version = "0.10.0", default-features = false } csv = { version = "1.4.0", default-features = false } directories = { version = "6.0.0", default-features = false } @@ -42,11 +53,15 @@ flate2 = { version = "1.1.9", default-features = false } futures = { version = "0.3.31", default-features = false } futures-util = { version = "0.3.31", default-features = false } humantime-serde = { version = "1.1.1", default-features = false } +ignore = { version = "0.4.33", default-features = false } indicatif = { version = "0.18.6", default-features = false } insta = { version = "1.44.1", default-features = false } layered = { version = "0.3.6", default-features = false } lcov = { version = "0.8.2", default-features = false } +libc = { version = "0.2.178", default-features = false } log = { version = "0.4.33", default-features = false } +loom = { version = "0.7.2", default-features = false } +many_cpus = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1.52", default-features = false } mmap-rs = { version = "0.7.0", default-features = false } mutants = { version = "0.0.3", default-features = false } @@ -56,12 +71,16 @@ pastey = { version = "0.2.3", default-features = false } percent-encoding = { version = "2.3.2", default-features = false } petgraph = { version = "0.8.3", default-features = false } predicates = { version = "3.1.4", default-features = false } +proc-macro2 = { version = "1.0.103", default-features = false } ra_ap_syntax = { version = "0.0.331", default-features = false } +rapidhash = { version = "4.1.1", default-features = false } +reflink-copy = { version = "0.1.30", default-features = false } regex = { version = "1.12.2", default-features = false } reqwest = { version = "0.13.4", default-features = false } rmp-serde = { version = "1.3.1", default-features = false } rust_xlsxwriter = { version = "0.98.2", default-features = false } rustc-hash = { version = "2.1.0", default-features = false } +rustc_lexer = { version = "0.1.0", default-features = false } rustdoc-types-v50 = { package = "rustdoc-types", version = "0.50.0", default-features = false } rustdoc-types-v51 = { package = "rustdoc-types", version = "0.51.0", default-features = false } rustdoc-types-v52 = { package = "rustdoc-types", version = "0.52.0", default-features = false } @@ -79,6 +98,7 @@ serial_test = { version = "4.0.1", default-features = false } sha2 = { version = "0.10.9", default-features = false } strsim = { version = "0.11.1", default-features = false } strum = { version = "0.28.0", default-features = false } +syn = { version = "2.0.111", default-features = false } tar = { version = "0.4.46", default-features = false } tempfile = { version = "3.27.0", default-features = false } terminal_size = { version = "0.4.4", default-features = false } @@ -92,6 +112,7 @@ url = { version = "2.5.8", default-features = false } vlen = { version = "0.4.6", default-features = false } walkdir = { version = "2.5.0", default-features = false } windows = { version = "0.62.2", default-features = false } +windows-sys = { version = "0.61.2", default-features = false } wiremock = { version = "0.6.5", default-features = false } zstd = { version = "0.13.3", default-features = false } diff --git a/README.md b/README.md index ae8afa3bb..598c4c74f 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ These are the crates built out of this repo: - [`cargo-each`](./crates/cargo-each/README.md) - A cargo subcommand that runs a command over a cargo-style selection of workspace members - [`cargo-ensure-no-cyclic-deps`](./crates/cargo_ensure_no_cyclic_deps/README.md) - A cargo subcommand to detect cyclic dependencies in workspace crates - [`cargo-ensure-no-default-features`](./crates/cargo-ensure-no-default-features/README.md) - A cargo subcommand that ensures dependencies are declared with default-features = false +- [`cargo-gamma`](./crates/cargo-gamma/README.md) - Fast mutation testing for Rust - [`cargo-heather`](./crates/cargo-heather/README.md) - A cargo subcommand to validate license headers in Rust, TOML, PowerShell, Just, and env source files ## About this Repo diff --git a/crates/cargo-anvil/docs/design/checks.md b/crates/cargo-anvil/docs/design/checks.md index 0c60bfa69..9e5f916e3 100644 --- a/crates/cargo-anvil/docs/design/checks.md +++ b/crates/cargo-anvil/docs/design/checks.md @@ -170,7 +170,7 @@ that provided the strongest version of the check. | `deny` | `cargo deny check` | all | | `audit` | `cargo audit` | oxidizer | | `udeps` | `cargo + udeps --workspace --all-features` run **twice** — once with default targets (lib + bins) and once with `--all-targets`. cargo-udeps only analyzes the targets it's told to, and each run catches a variant the other masks: the default-targets run surfaces a dep in `[dependencies]` referenced only by tests/benches/examples (it should be a dev-dep; `--all-targets` would see it as "used"), while the `--all-targets` run surfaces unused `[dev-dependencies]` (never compiled by the default-targets run). Together they cover unused deps, unused dev-deps, and deps that should be dev-deps. | oxidizer, oxidizer-github | -| `semver-check` | `cargo semver-checks --baseline-rev ` per affected library crate. The PR target is the baseline. Exit 100 is a completed check with deny-level findings; exit 101 or another nonzero status means the comparison was inconclusive. Both outcomes write `target/anvil/comments/semver.md` and remain advisory, matching the repository's native `semver` job (`continue-on-error: true`). Proven rename and bin→lib transitions with no comparable baseline are skipped without a comment. Anvil preflight failures such as invalid current-workspace metadata or an unavailable baseline ref still fail because the recipe cannot establish what to compare. | oxidizer-github | +| `semver-check` | `cargo semver-checks --baseline-rev ` per affected library crate. The PR target is the baseline. Exit 100 is a completed check with deny-level findings; exit 101 or another nonzero status means the comparison was inconclusive. Both outcomes write `target/anvil/comments/semver.md` and remain advisory, matching the repository's native `semver` job (`continue-on-error: true`). Proven rename and bin→lib transitions with no comparable baseline, and dependencies proven to be yanked only in the checked-out baseline tree, are skipped without a comment. Anvil preflight failures such as invalid current-workspace metadata or an unavailable baseline ref still fail because the recipe cannot establish what to compare. | oxidizer-github | | `external-types` | `cargo + check-external-types --manifest-path` per library crate (per-manifest because the tool has no `--workspace`/`--package`; bin-only crates have no public API surface and are skipped). Setup installs the catalog version but validation accepts newer installed tools. The selected nightly is tested with the catalog version; an incompatible newer tool fails closed with a tool/nightly compatibility diagnostic rather than silently selecting a different schema. | oxidizer-github | ### `pr-slow` @@ -392,7 +392,8 @@ When cargo-delta reports a manifest directory leaf instead of a package or libra name, Anvil accepts it only if it uniquely identifies one workspace package; missing or ambiguous aliases fail rather than silently dropping affected work. Advisory checks may report policy findings without failing, but failure to execute -the advisory tool is still an operational error. +the advisory tool is still an operational error unless the baseline itself has become +unusable because one of its dependency versions was subsequently yanked. The recipe-side mechanics are in [local.md §4](./local.md#4-impact-scoping-via-the-anvil-impact-recipe). The cloud workflow-side wiring (the diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/semver-check.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/semver-check.just index 4f7112b6e..df65eaf65 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/semver-check.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/semver-check.just @@ -158,16 +158,19 @@ anvil-semver-check: anvil-semver-check-validate-prereqs anvil-impact $findings.Add('```') | Out-Null $findings.Add('') | Out-Null } elseif ($semverExit -eq 101) { - # Exit 101 means the check could not complete. Two transitions - # intentionally have no comparable branch baseline: a package - # renamed in place and a baseline bin becoming a library. + # Exit 101 means the check could not complete. Some conditions + # intentionally have no usable branch baseline: a package renamed + # in place, a baseline bin becoming a library, or a dependency + # version in the baseline becoming yanked after that commit landed. $packageMissing = $output.Contains(('package `{0}` not found in' -f $p)) $manifestParseFailed = $output.Contains('possibly due to errors:') $renamedPackage = $packageMissing -and -not $manifestParseFailed $baselineLibraryMissing = $output.Contains('has no lib target') -or $output.Contains('no library targets found') - if ($renamedPackage -or $baselineLibraryMissing) { - Write-Host " $p has no comparable branch baseline; skipping (likely renamed or bin->lib transition)" -ForegroundColor Yellow + $baselineDependencyYanked = $output.Contains('is yanked') -and + ($output -match 'target[\\/]+semver-checks[\\/]+git-') + if ($renamedPackage -or $baselineLibraryMissing -or $baselineDependencyYanked) { + Write-Host " $p has no usable branch baseline; skipping (renamed, bin->lib, or yanked baseline dependency)" -ForegroundColor Yellow } else { Write-Warning "anvil-semver-check: $p could not be checked (cargo-semver-checks exit 101):`n$output" $inconclusive.Add('#### `' + $p + '` (exit 101)') | Out-Null diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index 311903589..408f45c64 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -443,7 +443,11 @@ fn semver_exit_code_contract_is_executed() { ); assert!(!tmp.path().join("target/anvil/comments/semver.md").exists()); - for output in ["has no lib target", "no library targets found"] { + for output in [ + "has no lib target", + "no library targets found", + "version 1.0.0 is yanked: target/semver-checks/git-origin_main/crates/fixture", + ] { let bin_to_lib = run_just( tmp.path(), &["anvil-semver-check"], diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index c00d04a3e..c0ab6168d 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -4374,16 +4374,19 @@ anvil-semver-check: anvil-semver-check-validate-prereqs anvil-impact $findings.Add('```') | Out-Null $findings.Add('') | Out-Null } elseif ($semverExit -eq 101) { - # Exit 101 means the check could not complete. Two transitions - # intentionally have no comparable branch baseline: a package - # renamed in place and a baseline bin becoming a library. + # Exit 101 means the check could not complete. Some conditions + # intentionally have no usable branch baseline: a package renamed + # in place, a baseline bin becoming a library, or a dependency + # version in the baseline becoming yanked after that commit landed. $packageMissing = $output.Contains(('package `{0}` not found in' -f $p)) $manifestParseFailed = $output.Contains('possibly due to errors:') $renamedPackage = $packageMissing -and -not $manifestParseFailed $baselineLibraryMissing = $output.Contains('has no lib target') -or $output.Contains('no library targets found') - if ($renamedPackage -or $baselineLibraryMissing) { - Write-Host " $p has no comparable branch baseline; skipping (likely renamed or bin->lib transition)" -ForegroundColor Yellow + $baselineDependencyYanked = $output.Contains('is yanked') -and + ($output -match 'target[\\/]+semver-checks[\\/]+git-') + if ($renamedPackage -or $baselineLibraryMissing -or $baselineDependencyYanked) { + Write-Host " $p has no usable branch baseline; skipping (renamed, bin->lib, or yanked baseline dependency)" -ForegroundColor Yellow } else { Write-Warning "anvil-semver-check: $p could not be checked (cargo-semver-checks exit 101):`n$output" $inconclusive.Add('#### `' + $p + '` (exit 101)') | Out-Null diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index f20c14d05..b05e44c0d 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -4253,16 +4253,19 @@ anvil-semver-check: anvil-semver-check-validate-prereqs anvil-impact $findings.Add('```') | Out-Null $findings.Add('') | Out-Null } elseif ($semverExit -eq 101) { - # Exit 101 means the check could not complete. Two transitions - # intentionally have no comparable branch baseline: a package - # renamed in place and a baseline bin becoming a library. + # Exit 101 means the check could not complete. Some conditions + # intentionally have no usable branch baseline: a package renamed + # in place, a baseline bin becoming a library, or a dependency + # version in the baseline becoming yanked after that commit landed. $packageMissing = $output.Contains(('package `{0}` not found in' -f $p)) $manifestParseFailed = $output.Contains('possibly due to errors:') $renamedPackage = $packageMissing -and -not $manifestParseFailed $baselineLibraryMissing = $output.Contains('has no lib target') -or $output.Contains('no library targets found') - if ($renamedPackage -or $baselineLibraryMissing) { - Write-Host " $p has no comparable branch baseline; skipping (likely renamed or bin->lib transition)" -ForegroundColor Yellow + $baselineDependencyYanked = $output.Contains('is yanked') -and + ($output -match 'target[\\/]+semver-checks[\\/]+git-') + if ($renamedPackage -or $baselineLibraryMissing -or $baselineDependencyYanked) { + Write-Host " $p has no usable branch baseline; skipping (renamed, bin->lib, or yanked baseline dependency)" -ForegroundColor Yellow } else { Write-Warning "anvil-semver-check: $p could not be checked (cargo-semver-checks exit 101):`n$output" $inconclusive.Add('#### `' + $p + '` (exit 101)') | Out-Null diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index 20556135d..dc0bc2986 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -3126,16 +3126,19 @@ anvil-semver-check: anvil-semver-check-validate-prereqs anvil-impact $findings.Add('```') | Out-Null $findings.Add('') | Out-Null } elseif ($semverExit -eq 101) { - # Exit 101 means the check could not complete. Two transitions - # intentionally have no comparable branch baseline: a package - # renamed in place and a baseline bin becoming a library. + # Exit 101 means the check could not complete. Some conditions + # intentionally have no usable branch baseline: a package renamed + # in place, a baseline bin becoming a library, or a dependency + # version in the baseline becoming yanked after that commit landed. $packageMissing = $output.Contains(('package `{0}` not found in' -f $p)) $manifestParseFailed = $output.Contains('possibly due to errors:') $renamedPackage = $packageMissing -and -not $manifestParseFailed $baselineLibraryMissing = $output.Contains('has no lib target') -or $output.Contains('no library targets found') - if ($renamedPackage -or $baselineLibraryMissing) { - Write-Host " $p has no comparable branch baseline; skipping (likely renamed or bin->lib transition)" -ForegroundColor Yellow + $baselineDependencyYanked = $output.Contains('is yanked') -and + ($output -match 'target[\\/]+semver-checks[\\/]+git-') + if ($renamedPackage -or $baselineLibraryMissing -or $baselineDependencyYanked) { + Write-Host " $p has no usable branch baseline; skipping (renamed, bin->lib, or yanked baseline dependency)" -ForegroundColor Yellow } else { Write-Warning "anvil-semver-check: $p could not be checked (cargo-semver-checks exit 101):`n$output" $inconclusive.Add('#### `' + $p + '` (exit 101)') | Out-Null diff --git a/crates/cargo-each/Cargo.toml b/crates/cargo-each/Cargo.toml index 68fbf69bd..a160afa86 100644 --- a/crates/cargo-each/Cargo.toml +++ b/crates/cargo-each/Cargo.toml @@ -16,11 +16,6 @@ license.workspace = true homepage.workspace = true repository = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo-each" -[package.metadata.ox-gen-readme] -# Binary-only crate: no library rustdoc to generate a README from, so the -# README is hand-maintained rather than produced by cargo-doc2readme. -disable = true - [dependencies] cargo_metadata = { workspace = true } clap = { workspace = true, features = ["derive", "std", "help", "usage", "error-context"] } diff --git a/crates/cargo-each/README.md b/crates/cargo-each/README.md index b5887bbd2..a5596b469 100644 --- a/crates/cargo-each/README.md +++ b/crates/cargo-each/README.md @@ -4,6 +4,7 @@ # Cargo-Each [![crates.io](https://img.shields.io/crates/v/cargo-each.svg)](https://crates.io/crates/cargo-each) +[![docs.rs](https://docs.rs/cargo-each/badge.svg)](https://docs.rs/cargo-each) [![MSRV](https://img.shields.io/crates/msrv/cargo-each)](https://crates.io/crates/cargo-each) [![CI](https://github.com/microsoft/ox-tools/actions/workflows/main.yml/badge.svg?event=push)](https://github.com/microsoft/ox-tools/actions/workflows/main.yml) [![Coverage](https://codecov.io/gh/microsoft/ox-tools/graph/badge.svg?token=FCUG0EL5TI)](https://codecov.io/gh/microsoft/ox-tools) @@ -123,6 +124,7 @@ cargo each -p crate-a -p crate-b --once -- \ cargo clippy {packages} --all-targets -- -D warnings ``` +
This crate was developed as part of The Oxidizer Project. Browse this crate's source code. diff --git a/crates/cargo-each/src/main.rs b/crates/cargo-each/src/main.rs index 4e83b5b0e..1efed6a42 100644 --- a/crates/cargo-each/src/main.rs +++ b/crates/cargo-each/src/main.rs @@ -11,12 +11,17 @@ //! It exists to replace hand-rolled for-each-package shell loops with a //! single cargo-native, cross-platform command. //! -//! This crate ships **only** the `cargo-each` executable; it exposes no -//! library API (every module below is crate-internal). Consume it as a cargo -//! subcommand, not as a dependency. +//! `cargo-each` ships as an executable only; it is a cargo subcommand, not a +//! library dependency. #![doc(html_logo_url = "https://media.githubusercontent.com/media/microsoft/ox-tools/refs/heads/main/crates/cargo-each/logo.png")] #![doc(html_favicon_url = "https://media.githubusercontent.com/media/microsoft/ox-tools/refs/heads/main/crates/cargo-each/favicon.ico")] //! +//! # Installation +//! +//! ```text +//! cargo install cargo-each +//! ``` +//! //! # Usage //! //! ```text diff --git a/crates/cargo-gamma-attrs-impl/Cargo.toml b/crates/cargo-gamma-attrs-impl/Cargo.toml new file mode 100644 index 000000000..24a038a24 --- /dev/null +++ b/crates/cargo-gamma-attrs-impl/Cargo.toml @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[package] +name = "cargo-gamma-attrs-impl" +description = "Implementation of the inert attribute macros exposed by cargo-gamma-attrs" +version = "0.1.0" +readme = "README.md" +keywords = ["cargo", "mutation-testing", "testing"] +categories = ["development-tools::testing"] + +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo-gamma-attrs-impl" + +[package.metadata.docs.rs] +all-features = true + +[package.metadata.cargo_check_external_types] +# This implementation crate exists to provide proc-macro entry points, so its +# public API necessarily accepts and returns proc-macro token streams. +allowed_external_types = ["proc_macro2::TokenStream"] + +# HARD CONSTRAINT: this crate must not become a proc-macro crate. +# A proc macro's code runs only inside rustc while some *other* crate is being compiled, which +# puts it out of reach of both the coverage harness and the mutation runtime. Keeping the logic +# in an ordinary library is the whole reason this crate exists — see the crate docs. + +[dependencies] +proc-macro2 = { workspace = true, features = ["span-locations", "proc-macro"] } +# `#[gamma::value()]` promises its argument is one valid Rust expression, and only a real +# expression parser can keep that promise. Checking it with tokens alone would accept things the +# compiler later rejects at the expansion site, which is the diagnostic this crate exists to avoid. +syn = { workspace = true, features = ["full", "parsing", "printing", "visit", "extra-traits", "clone-impls", "proc-macro"] } + +# >>> anvil-managed: anvil-lints +[lints] +workspace = true +# <<< anvil-managed: anvil-lints diff --git a/crates/cargo-gamma-attrs-impl/README.md b/crates/cargo-gamma-attrs-impl/README.md new file mode 100644 index 000000000..04f216a68 --- /dev/null +++ b/crates/cargo-gamma-attrs-impl/README.md @@ -0,0 +1,61 @@ +
+ Cargo-Gamma-Attrs-Impl Logo + +# Cargo-Gamma-Attrs-Impl + +[![crates.io](https://img.shields.io/crates/v/cargo-gamma-attrs-impl.svg)](https://crates.io/crates/cargo-gamma-attrs-impl) +[![docs.rs](https://docs.rs/cargo-gamma-attrs-impl/badge.svg)](https://docs.rs/cargo-gamma-attrs-impl) +[![MSRV](https://img.shields.io/crates/msrv/cargo-gamma-attrs-impl)](https://crates.io/crates/cargo-gamma-attrs-impl) +[![CI](https://github.com/microsoft/ox-tools/actions/workflows/main.yml/badge.svg?event=push)](https://github.com/microsoft/ox-tools/actions/workflows/main.yml) +[![Coverage](https://codecov.io/gh/microsoft/ox-tools/graph/badge.svg?token=FCUG0EL5TI)](https://codecov.io/gh/microsoft/ox-tools) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](../../LICENSE) +This crate was developed as part of the Oxidizer project + +
+ +The implementation behind [`cargo-gamma-attrs`][__link0], +which is where the inert `#[gamma::skip]`, `#[gamma::expect_survived]` and +`#[gamma::expect_killed]` attributes are actually exposed. + +You almost certainly want that crate instead. This one is a normal library rather than a +proc-macro crate so its logic can be called by ordinary tests, covered, and mutation tested. +What remains in the proc-macro crate is a shim thin enough to read at a glance. + +## Why this crate exists + +`cargo-gamma-attrs` is a proc-macro crate, and a proc macro’s code runs only inside `rustc`, +while some *other* crate is being compiled. That puts it beyond the reach of both measurements +this project cares about: + +* A coverage harness collects counters from test binaries. A proc macro increments its counters + inside the compiler, which writes no profile the harness sees. +* A mutation run selects one mutant per test process at run time. A proc macro has already + finished by then, so none of its mutants can be active while a test is watching. + +Splitting the logic into an ordinary library makes it reachable by coverage and mutation tests. +The proc-macro crate remains a thin shim. + +## What the macros accept + +See the [`cargo-gamma-attrs`][__link1] documentation for the +user-facing description. In brief: a comma-separated selector list, optionally followed by +`reason = "..."` and `tag = "..."`, both of which must be string literals. + +`#[gamma::value()]` instead takes an expression. It is checked by [`value`][__link2], because its +argument is spliced into the user’s crate as a mutant and must be exactly one expression. + +## Stability + +This crate is an implementation detail of `cargo-gamma-attrs` and carries no stability +guarantee of its own. Depend on `cargo-gamma-attrs`. + + +
+ +This crate was developed as part of The Oxidizer Project. Browse this crate's source code. + + + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjJhdIQbFhzZ8rzWNNYbuRaDSGWynFgbH4PMdoT7GNcbVwNPtPjAhvFhYvRhcoQbH5RUmmY8e-sbYyqmHPyeK9obgdLJAJ7T65AbUAUW0Y4uz2thZIGDdmNhcmdvLWdhbW1hLWF0dHJzLWltcGxlMC4xLjB2Y2FyZ29fZ2FtbWFfYXR0cnNfaW1wbA + [__link0]: https://crates.io/crates/cargo-gamma-attrs + [__link1]: https://docs.rs/cargo-gamma-attrs + [__link2]: https://docs.rs/cargo-gamma-attrs-impl/0.1.0/cargo_gamma_attrs_impl/?search=value diff --git a/crates/cargo-gamma-attrs-impl/docs/DESIGN.md b/crates/cargo-gamma-attrs-impl/docs/DESIGN.md new file mode 100644 index 000000000..61270db45 --- /dev/null +++ b/crates/cargo-gamma-attrs-impl/docs/DESIGN.md @@ -0,0 +1,23 @@ +# cargo-gamma-attrs-impl — Design + +> Status: **Implemented**. +> Crate name: `cargo-gamma-attrs-impl`. + +## Purpose + +This ordinary library implements parsing and validation for the inert +attributes exported by `cargo-gamma-attrs`. + +## Boundaries + +- This crate must remain a normal library, not a proc-macro crate. Keeping the + logic outside rustc makes it directly testable and mutation-testable. +- It accepts exactly one Rust expression where an attribute promises an + expression and rejects unsupported keys or malformed selectors. +- It returns the original item unchanged after validation. + +## Stability + +The crate is published only to support `cargo-gamma-attrs`. Its Rust API is an +implementation detail; the diagnostics and accepted attribute syntax are the +user-visible contract. diff --git a/crates/cargo-gamma-attrs-impl/favicon.ico b/crates/cargo-gamma-attrs-impl/favicon.ico new file mode 100644 index 000000000..18acb282f --- /dev/null +++ b/crates/cargo-gamma-attrs-impl/favicon.ico @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82caca17fd4d08a23a9d76f5c895029ff6bb0d13b0e2693ef229a4ea691ffccd +size 46496 diff --git a/crates/cargo-gamma-attrs-impl/logo.png b/crates/cargo-gamma-attrs-impl/logo.png new file mode 100644 index 000000000..20aaf26cf --- /dev/null +++ b/crates/cargo-gamma-attrs-impl/logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc17b32e4b6b8c4f0c1d443b680a25b68e0cd8913954dfd047a7659a6bd0baff +size 131471 diff --git a/crates/cargo-gamma-attrs-impl/src/implementation.rs b/crates/cargo-gamma-attrs-impl/src/implementation.rs new file mode 100644 index 000000000..a213c7a3d --- /dev/null +++ b/crates/cargo-gamma-attrs-impl/src/implementation.rs @@ -0,0 +1,1184 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use proc_macro2::{Delimiter, Literal, TokenStream, TokenTree}; +use syn::parse::Parser as _; +use syn::punctuated::Punctuated; +use syn::{Expr, ImplItemFn, ItemFn, Token, TraitItemFn}; + +/// Validates the argument list of `#[gamma::]` and returns the item untouched. +/// +/// A well-formed attribute expands to exactly the item it was written on: these macros exist so +/// that `cargo-gamma` can see a suppression in the source, not to rewrite anything. A malformed +/// one expands to the item preceded by a `compile_error!`, so the item itself still parses and the +/// user gets one clear diagnostic rather than a cascade of follow-on errors. +#[must_use] +pub fn inert(name: &str, attr: TokenStream, item: TokenStream) -> TokenStream { + match validate(attr) { + Ok(()) => item, + Err(message) => { + let mut out = error(&format!("#[gamma::{name}]: {message}")); + + out.extend(item); + out + } + } +} + +/// Validates the argument list of `#[gamma::test_timeout_multiplier(...)]` and returns the item untouched. +#[must_use] +pub fn inert_timeout(name: &str, attr: &TokenStream, item: TokenStream) -> TokenStream { + match validate_timeout_multiplier(attr) { + Ok(()) => item, + Err(message) => { + let mut out = error(&format!("#[gamma::{name}]: {message}")); + + out.extend(item); + out + } + } +} + +/// Validates the argument of `#[gamma::value()]` and returns the item untouched. +/// +/// The attribute states the expression `cargo-gamma` substitutes for the annotated function's body +/// when it mutates its return value, in place of the one it would otherwise guess from the return +/// type. Like the suppression macros this expands to the item alone; unlike them, what it validates +/// is not a shape but a language — the argument has to be one Rust expression, because that text +/// ends up spliced into the user's crate. +/// +/// Four things are rejected, each because the alternative is worse than a compile error: +/// +/// - Nothing at all. `#[gamma::value()]` states no value, and reading it as "use the guess" would +/// make an attribute that looks like it did something do nothing. +/// - More than one expression. `#[gamma::value(0, 1)]` looks like a list of candidates, and this +/// is not one: a site states one value, so accepting the first and dropping the rest would be a +/// silent loss. +/// - Anything that is not an expression. The tool cannot repair `1 +`, and splicing it would move +/// the error to a mutant nobody wrote. +/// - A second `#[gamma::value(...)]` on the same item. Two stated values would leave which one +/// wins to the order the compiler happened to expand them in, and last-wins is a rule nobody can +/// see in the source. +/// +/// It also insists the item is a function or method. An attribute on an `impl` block or a module +/// could only mean "every function beneath this states this value", and one expression essentially +/// never type-checks as the body of more than one signature, so the inheriting reading would be a +/// promise the compiler breaks at every use. +#[must_use] +pub fn value(attr: TokenStream, item: TokenStream) -> TokenStream { + match validate_value(attr, &item) { + Ok(()) => item, + Err(message) => { + let mut out = error(&format!("#[gamma::value]: {message}")); + + out.extend(item); + out + } + } +} + +/// The deepest delimiter nesting a token stream may have and still be parsed. +/// +/// Matches the limit the library side applies in `cargo_gamma_lib::parse::nesting`. Handing a +/// token stream nested deeper than this to `syn`'s recursive descent parser would exhaust the +/// stack and abort the compiler without diagnostics. +/// +/// Exposed (hidden from docs) so `cargo-gamma-lib`'s agreement test can pin this copy against the +/// library's own `NESTING_LIMIT`, which is the only thing that keeps the two in step. +#[doc(hidden)] +pub const NESTING_LIMIT: usize = 64; + +/// How many postfix links are allowed per delimiter nesting level. +/// +/// This stays in step with `cargo_gamma_lib::parse::nesting`: a run of calls or indexes is a +/// recursive expression tree even though each delimiter closes before the next one opens; field, +/// method, and try links add the same recursive shape. +/// +/// Exposed (hidden from docs) so `cargo-gamma-lib`'s agreement test can pin this copy against the +/// library's own `CHAIN_FACTOR`. +#[doc(hidden)] +pub const CHAIN_FACTOR: usize = 4; + +/// Whether the preceding token can end an expression. +#[derive(Clone, Copy, Eq, PartialEq)] +enum Previous { + Other, + Expression, +} + +/// One token-stream level waiting to be walked. +struct Frame { + iter: proc_macro2::token_stream::IntoIter, + depth: usize, + postfix: usize, + casts: usize, + operators: usize, + previous: Previous, +} + +/// Returns whether a token stream exceeds its delimiter or postfix-expression limits. +/// +/// Nested groups are walked with an explicit stack rather than recursion, because this code runs +/// inside `rustc` while compiling user code: a proc macro that exhausts the stack takes the +/// compiler with it. +fn exceeds_nesting_limit(stream: &TokenStream, limit: usize) -> bool { + let postfix_limit = limit.saturating_mul(CHAIN_FACTOR); + let mut frames = vec![Frame { + iter: stream.clone().into_iter(), + depth: 0, + postfix: 0, + casts: 0, + operators: 0, + previous: Previous::Other, + }]; + + while let Some(mut frame) = frames.pop() { + while let Some(tree) = frame.iter.next() { + match tree { + TokenTree::Group(group) => { + let postfix = + matches!(group.delimiter(), Delimiter::Parenthesis | Delimiter::Bracket) && frame.previous == Previous::Expression; + + if postfix { + frame.postfix += 1; + + if frame.postfix > postfix_limit { + return true; + } + } else { + frame.postfix = 0; + } + + let next_depth = frame.depth + 1; + + if next_depth > limit { + return true; + } + + // A complete group can be the receiver of the next call or index. The child + // gets a fresh postfix chain because only adjacent links share one expression. + frame.previous = Previous::Expression; + frames.push(frame); + frames.push(Frame { + iter: group.stream().into_iter(), + depth: next_depth, + postfix: 0, + casts: 0, + operators: 0, + previous: Previous::Other, + }); + break; + } + + TokenTree::Ident(ident) if ident == "as" => { + frame.postfix = 0; + frame.casts += 1; + + if frame.casts > postfix_limit { + return true; + } + + frame.previous = Previous::Other; + } + + TokenTree::Ident(_) | TokenTree::Literal(_) => frame.previous = Previous::Expression, + + TokenTree::Punct(punct) => { + if matches!( + punct.as_char(), + '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' | '!' | '<' | '>' | '=' + ) { + frame.operators += 1; + + if frame.operators > postfix_limit { + return true; + } + } + + let postfix = matches!(punct.as_char(), '.' | '?') && frame.previous == Previous::Expression; + + if postfix { + frame.postfix += 1; + + if frame.postfix > postfix_limit { + return true; + } + } else { + frame.postfix = 0; + } + + if matches!(punct.as_char(), ',' | ';') { + frame.casts = 0; + frame.operators = 0; + } + + frame.previous = if matches!(punct.as_char(), '?' | '>') { + Previous::Expression + } else { + Previous::Other + }; + } + } + } + } + + false +} + +/// Checks that an argument list is one expression, on a function that states no other value. +fn validate_value(attr: TokenStream, item: &TokenStream) -> Result<(), String> { + if attr.is_empty() { + return Err("expected one expression, as in `#[gamma::value(0)]`".to_owned()); + } + + if exceeds_nesting_limit(&attr, NESTING_LIMIT) { + return Err("expression nests too deeply to be safely parsed".to_owned()); + } + + if exceeds_nesting_limit(item, NESTING_LIMIT) { + return Err("item nests too deeply to be safely parsed".to_owned()); + } + + let written = attr.to_string(); + + if let Err(reported) = syn::parse2::(attr.clone()) { + let parser = Punctuated::::parse_terminated; + + return match parser.parse2(attr) { + Ok(list) if list.len() > 1 => Err(format!( + "expected one expression, but `{written}` is {}; a site states one value", + list.len() + )), + _ => Err(format!("`{written}` is not a Rust expression: {reported}")), + }; + } + + if !is_function(item) { + return Err("expected a function or method; a value stated on an `impl` block or a module would have to type-check as the body of every function beneath it".to_owned()); + } + + if !has_a_body(item) { + return Err( + "expected a function with a body; a declaration has none to replace, and a value is not inherited by the implementations of a trait method" + .to_owned(), + ); + } + + if states_a_value(item) { + return Err("an item may state one value; two would leave which of them applies to the order they were expanded in".to_owned()); + } + + Ok(()) +} + +/// Returns whether an item is a function, a method, or a trait method. +/// +/// All three are asked, because the same attribute is written in all three positions and only the +/// grammar differs: a free function is an `ItemFn`, a method an `ImplItemFn`, and a trait method a +/// `TraitItemFn` whose body may be absent entirely. +fn is_function(item: &TokenStream) -> bool { + // #[gamma::skip(logical.or_to_and, reason = "syn accepts every function-with-body form through the adjacent item and method parsers alike; the only distinct form is a bodyless trait declaration, which reaches the final parser unchanged")] + syn::parse2::(item.clone()).is_ok() + || syn::parse2::(item.clone()).is_ok() + || syn::parse2::(item.clone()).is_ok() +} + +/// Returns whether a function has a body to replace. +/// +/// A trait method may be a declaration ending in `;`, and a value stated there would substitute +/// nothing anywhere: the value is not inherited by the implementations, for the same reason it is +/// not inherited from an `impl` block. Saying so is better than expanding to an attribute that +/// reads as a working hint and generates no mutant. +fn has_a_body(item: &TokenStream) -> bool { + // #[gamma::skip(literal.bool_flip, reason = "validate_value calls this only after is_function; every function form with a body parses as a TraitItemFn, so the parse-error default is unreachable for supported input")] + syn::parse2::(item.clone()).map_or(true, |method| method.default.is_some()) +} + +/// Returns whether an item still carries a `#[gamma::value(...)]` attribute of its own. +/// +/// Attribute macros expand outermost first, and the item handed to one still carries every +/// attribute below it. So the first of two `#[gamma::value(...)]` attributes sees the second here, +/// which is what makes the duplicate a diagnostic rather than a coin toss. +/// +/// Only the item's own attributes are examined — the ones before its body — so a nested function +/// inside the body stating its own value is left alone. It is a different site, and stating a value +/// there is exactly as legitimate. +fn states_a_value(item: &TokenStream) -> bool { + let mut trees = item.clone().into_iter().peekable(); + + while let Some(tree) = trees.next() { + match tree { + TokenTree::Punct(punct) if punct.as_char() == '#' => { + let Some(TokenTree::Group(group)) = trees.peek() else { + continue; + }; + + if group.delimiter() == Delimiter::Bracket && names_value(&group.stream()) { + return true; + } + } + + // The first token that is not part of an attribute is the start of the signature, and + // everything past it belongs to the function rather than to what is written on it. + TokenTree::Punct(_) | TokenTree::Group(_) => {} + TokenTree::Ident(_) | TokenTree::Literal(_) => break, + } + } + + false +} + +/// Returns whether the body of an attribute names `gamma::value`. +fn names_value(inner: &TokenStream) -> bool { + let mut trees = inner.clone().into_iter(); + + let Some(TokenTree::Ident(namespace)) = trees.next() else { + return false; + }; + + if namespace != "gamma" { + return false; + } + + // Two `:` tokens rather than one `::`, because a path separator reaches a proc macro as a pair + // of joint puncts and never as a single tree. + for _separator in 0..2 { + match trees.next() { + Some(TokenTree::Punct(punct)) if punct.as_char() == ':' => {} + _other => return false, + } + } + + matches!(trees.next(), Some(TokenTree::Ident(name)) if name == "value") +} + +/// Returns whether a literal's source text is a string literal that yields a `&str`. +/// +/// Rust writes a string literal in four shapes: `"x"`, `r"x"`, `r#"x"#`, and any number of hashes +/// beyond that. All of them are strings, and a `reason` or `tag` written as a raw string is a +/// perfectly reasonable thing to want — it is how one embeds a quote or a backslash without +/// escaping. Asking only whether the text opens with a quote rejected every raw form. +/// +/// The prefix is what carries the type, so it is what this looks at. Stripping an optional `r` and +/// then any run of hashes must leave a quote. A byte string opens with `b` and a C string with `c`, +/// neither of which survives that strip, so both stay rejected: `b"x"` is a `&[u8]` and `c"x"` is a +/// `&CStr`, and neither is the `&str` the attribute promises. `br"x"` is rejected for the same +/// reason, since the `b` comes first and the strip never reaches its `r`. +fn is_string_literal(text: &str) -> bool { + let body = text.strip_prefix('r').unwrap_or(text); + + body.trim_start_matches('#').starts_with('"') +} + +/// The largest multiplier worth accepting, mirroring `bounds::factor` in `cargo-gamma-lib`. +/// +/// The two cannot share a constant: this is a proc-macro crate and the tool does not depend on it +/// in the direction that would allow it. They must nevertheless agree, because disagreeing means +/// `cargo build` accepts a multiplier that `cargo gamma` then refuses, and the user is told about +/// their typo by whichever happens to run second. +/// +/// Exposed (hidden from docs) so `cargo-gamma-lib`'s agreement test can pin this copy against the +/// library's own `MOST_FACTOR`. +#[doc(hidden)] +pub const MOST_FACTOR: f64 = 1e6; + +/// Checks a multiplier is positive, finite, and small enough to scale a baseline without overflow. +fn is_bounded_multiplier(value: f64) -> bool { + value > 0.0 && value.is_finite() && value <= MOST_FACTOR +} + +/// Checks that a timeout multiplier attribute has at least one argument and validates its contents. +fn validate_timeout_multiplier(attr: &TokenStream) -> Result<(), String> { + if attr.is_empty() { + return Err("expected a timeout multiplier, as in `#[gamma::test_timeout_multiplier(2.0)]`".to_owned()); + } + + let positional = match attr.clone().into_iter().next() { + Some(TokenTree::Literal(_)) => true, + Some(TokenTree::Punct(punct)) => matches!(punct.as_char(), '+' | '-'), + // `inf`, `nan`, and `infinity` tokenize as identifiers, not literals, so without this arm + // they slip past the positional check and fall through to `validate`, which treats a bare + // identifier as a selector and accepts it. They parse as non-finite floats and are never + // valid selectors, so a lone one is a malformed multiplier that must be refused here — + // otherwise `#[gamma::test_timeout_multiplier(inf)]` compiles clean and the tool's scanner + // rejects it later, or an unbounded factor reaches `Duration::mul_f64`. + Some(TokenTree::Ident(ident)) => ident.to_string().parse::().is_ok(), + _ => false, + }; + + if !positional { + return validate(attr.clone()); + } + + match attr.to_string().parse::() { + Ok(value) if is_bounded_multiplier(value) => Ok(()), + _ => Err(format!( + "timeout multiplier must be a positive number no greater than {MOST_FACTOR}" + )), + } +} + +/// Checks the structural shape of an argument list. +/// +/// Selector *names* are not checked here: the registry lives in the tool, and duplicating it in a +/// proc macro would mean two lists that drift apart. `cargo gamma` reports unknown selectors as +/// hard errors with a spelling suggestion. What this does check is the shape that a human is +/// likely to get wrong without noticing — a `reason` or `tag` that is not a string, or a timeout +/// multiplier that is not a positive number. +/// +/// Nested groups are walked with an explicit stack rather than by recursion. The nesting is +/// whatever the user wrote inside an attribute, and this code runs inside `rustc` while their +/// crate is being compiled: a proc macro that exhausts the stack takes the compiler with it, and +/// presents as a crash nobody would think to blame on a parenthesis in an attribute argument. +/// Depth on the heap has no such cliff, and the traversal order is unchanged, so a file with more +/// than one malformed argument still reports the first one. +fn validate(attr: TokenStream) -> Result<(), String> { + let mut frames: Vec<(Vec, usize)> = vec![(attr.into_iter().collect(), 0)]; + + 'frames: while let Some((trees, mut index)) = frames.pop() { + while index < trees.len() { + if let TokenTree::Ident(ident) = &trees[index] { + let key = ident.to_string(); + + if matches!(key.as_str(), "reason" | "tag") { + let is_equals = matches!(trees.get(index + 1), Some(TokenTree::Punct(punct)) if punct.as_char() == '='); + + if !is_equals { + return Err(format!("`{key}` must be written as `{key} = \"...\"`")); + } + + match trees.get(index + 2) { + Some(TokenTree::Literal(literal)) if is_string_literal(&literal.to_string()) => {} + _ => return Err(format!("`{key}` must be a string literal")), + } + + if trees + .get(index + 3) + .is_some_and(|tree| !matches!(tree, TokenTree::Punct(punct) if punct.as_char() == ',')) + { + return Err(format!("`{key}` must not have trailing tokens after its value")); + } + + // #[gamma::skip(literal.int_decrement, literal.int_to_one, reason = "the two tokens this steps over are the `=` and the string literal, and neither branch of this loop acts on either, so a shorter step reaches the same next meaningful token")] + index += 3; + continue; + } + + if matches!( + key.as_str(), + "test_timeout_multiplier" | "timeout_multiplier" | "factor" | "multiplier" + ) { + let is_equals = matches!(trees.get(index + 1), Some(TokenTree::Punct(punct)) if punct.as_char() == '='); + + if !is_equals { + return Err(format!("`{key}` must be written as `{key} = `")); + } + + match trees.get(index + 2) { + Some(TokenTree::Literal(literal)) if literal.to_string().parse::().is_ok_and(is_bounded_multiplier) => {} + _ => return Err(format!("`{key}` must be a positive number no greater than {MOST_FACTOR}")), + } + + if trees + .get(index + 3) + .is_some_and(|tree| !matches!(tree, TokenTree::Punct(punct) if punct.as_char() == ',')) + { + return Err(format!("`{key}` must not have trailing tokens after its value")); + } + + index += 3; + continue; + } + } + + if let TokenTree::Group(group) = &trees[index] + && group.delimiter() == Delimiter::Parenthesis + { + let inner: Vec = group.stream().into_iter().collect(); + + // The rest of this list is resumed once the group is done — the point a recursive + // walk would return to. + frames.push((trees, index + 1)); + frames.push((inner, 0)); + + continue 'frames; + } + + // #[gamma::skip(assign.add_to_sub, stmt.delete_assign, literal.int_decrement, reason = "subtracting, removing, or zeroing this advancement intentionally makes the token walk loop forever")] + index += 1; + } + } + + Ok(()) +} + +/// Builds a `compile_error!` invocation carrying `message`. +/// +/// The message is rendered by [`Literal::string`] rather than by escaping the text by hand, which +/// is what makes the result correct for every input: a message containing a quote, a backslash or +/// a newline needs no special handling here, and there is consequently no way for this to produce +/// something that does not parse. +fn error(message: &str) -> TokenStream { + let rendered = format!("::core::compile_error!({});", Literal::string(message)); + + rendered.parse().expect("a `Literal::string` always parses as a string literal") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Parses test input the way `rustc` would hand it to a proc macro. + fn stream(source: &str) -> TokenStream { + source.parse().expect("test input is valid Rust tokens") + } + + /// The overwhelmingly common case: a well-formed attribute must expand to the item and to + /// nothing else. Anything extra would show up in every annotated crate in the world. + #[test] + fn a_valid_attribute_expands_to_the_item_alone() { + let out = inert("skip", stream("arith, reason = \"checked by proptest\""), stream("fn f() {}")); + + assert_eq!(out.to_string(), "fn f () { }"); + } + + /// No arguments at all means every mutator, and is the shortest thing a user can write, so it + /// must not be mistaken for a malformed list. + #[test] + fn an_empty_argument_list_is_accepted() { + let out = inert("skip", stream(""), stream("fn f() {}")); + + assert_eq!(out.to_string(), "fn f () { }"); + } + + /// The whole selector vocabulary has to survive validation untouched, since none of it is + /// checked here. This pins the shapes: bare names, dotted names, families, presets, academic + /// aliases and negation. + #[test] + fn the_whole_selector_vocabulary_is_accepted() { + for selectors in [ + "arith.add_to_sub", + "arith", + "@arithmetic", + "ROR", + "arith, !arith.add_to_sub", + "literal, @boundary, !stmt", + ] { + let out = inert("skip", stream(selectors), stream("fn f() {}")); + + assert_eq!(out.to_string(), "fn f () { }", "`{selectors}` should have been accepted"); + } + } + + /// A malformed attribute must still leave the item behind. Swallowing it would turn one + /// diagnostic about the attribute into a pile of diagnostics about the missing function. + #[test] + fn a_malformed_attribute_keeps_the_item_and_adds_one_error() { + let out = inert("skip", stream("reason = performance"), stream("fn f() {}")).to_string(); + + assert!(out.contains("compile_error"), "expected a compile_error in `{out}`"); + assert!(out.ends_with("fn f () { }"), "expected the item to survive in `{out}`"); + } + + /// The message names the macro that was misused, because a file can carry all three and the + /// user needs to know which line to look at. + #[test] + fn the_message_names_the_macro_that_was_misused() { + for name in ["skip", "expect_survived", "expect_killed"] { + let out = inert(name, stream("reason = performance"), stream("fn f() {}")).to_string(); + + assert!( + out.contains(&format!("#[gamma::{name}]")), + "expected `{name}` to be named in `{out}`" + ); + } + } + + /// `reason = performance` is the mistake this validation exists for: it looks like it works, + /// and a regex-based tool would accept it. The exact wording is asserted because the wording + /// is the entire value of the diagnostic. + #[test] + fn a_bare_word_value_is_rejected_as_not_a_string() { + assert_eq!( + validate(stream("reason = performance")), + Err("`reason` must be a string literal".to_owned()) + ); + assert_eq!( + validate(stream("tag = telemetry")), + Err("`tag` must be a string literal".to_owned()) + ); + } + + /// A number is a literal but not a string literal, so [`is_string_literal`] is what separates + /// them. Without it `reason = 42` would be accepted. + #[test] + fn a_non_string_literal_value_is_rejected() { + assert_eq!(validate(stream("reason = 42")), Err("`reason` must be a string literal".to_owned())); + assert_eq!(validate(stream("tag = 1.5")), Err("`tag` must be a string literal".to_owned())); + assert_eq!( + validate(stream("reason = 'c'")), + Err("`reason` must be a string literal".to_owned()) + ); + } + + /// `reason("...")` is the other natural way to get it wrong, borrowed from attribute grammars + /// that do use that form. It has to be reported as a spelling problem rather than accepted. + #[test] + fn the_call_form_is_rejected_with_the_spelling_it_should_have() { + assert_eq!( + validate(stream("reason(\"x\")")), + Err("`reason` must be written as `reason = \"...\"`".to_owned()) + ); + assert_eq!( + validate(stream("tag(\"x\")")), + Err("`tag` must be written as `tag = \"...\"`".to_owned()) + ); + } + + /// Nesting deeper than any stack can hold is walked without one. + /// + /// The argument list of an attribute is user input, and this code runs inside `rustc` while + /// the user's crate is compiled. A recursive walk turns a machine-generated argument into a + /// compiler crash — no line, no attribute named, and no reason anyone would connect it to the + /// parentheses they wrote. The depth here is far past what a recursive walk survives, and the + /// argument is well-formed, so the only thing being asked is whether the walk completes. + #[test] + fn nesting_deeper_than_the_stack_is_still_walked_to_the_end() { + let depth = 50_000; + let source = format!("{}arith{}", "(".repeat(depth), ")".repeat(depth)); + + assert_eq!(validate(stream(&source)), Ok(())); + } + + /// A malformed argument buried under nesting is still found, and still reported first. + #[test] + fn a_malformed_argument_inside_a_nested_group_is_still_reported() { + assert_eq!( + validate(stream("arith, (((reason = performance))), tag = \"x\"")), + Err("`reason` must be a string literal".to_owned()) + ); + } + + /// A key with nothing after it at all runs off the end of the token list, which must be a + /// diagnostic rather than an out-of-bounds index. + #[test] + fn a_key_with_nothing_after_it_is_rejected() { + assert_eq!( + validate(stream("reason")), + Err("`reason` must be written as `reason = \"...\"`".to_owned()) + ); + assert_eq!(validate(stream("reason =")), Err("`reason` must be a string literal".to_owned())); + } + + /// A punctuation mark that is not `=` must not be mistaken for one. + #[test] + fn a_key_followed_by_the_wrong_punctuation_is_rejected() { + assert_eq!( + validate(stream("reason : \"x\"")), + Err("`reason` must be written as `reason = \"...\"`".to_owned()) + ); + } + + /// Both named arguments are optional and may appear together, in either order, after the + /// selectors. Skipping three tokens per key is what lets the second one be found. + #[test] + fn both_named_arguments_are_accepted_together_in_either_order() { + assert_eq!(validate(stream("stmt, reason = \"why\", tag = \"group\"")), Ok(())); + assert_eq!(validate(stream("stmt, tag = \"group\", reason = \"why\"")), Ok(())); + } + + /// Skipping past a validated key must not skip past the value's own tokens in a way that + /// hides a later mistake. This puts a bad key immediately after a good one. + #[test] + fn a_mistake_after_a_valid_named_argument_is_still_found() { + assert_eq!( + validate(stream("reason = \"why\", tag = broken")), + Err("`tag` must be a string literal".to_owned()) + ); + } + + /// A word that merely contains a key name is not a key, or `reasoning` would be treated as + /// `reason` and rejected for having no `=` after it. + #[test] + fn an_identifier_that_only_resembles_a_key_is_left_alone() { + assert_eq!(validate(stream("reasoning, tagged")), Ok(())); + } + + /// Arguments nested in parentheses are validated too, so a mistake cannot be hidden one level + /// down. + #[test] + fn a_mistake_nested_in_parentheses_is_found() { + assert_eq!( + validate(stream("outer(reason = performance)")), + Err("`reason` must be a string literal".to_owned()) + ); + assert_eq!(validate(stream("a(b(tag = 7))")), Err("`tag` must be a string literal".to_owned())); + } + + /// A valid nested list is still valid, which is what stops the recursion from being a way to + /// reject things. + #[test] + fn a_valid_nested_list_is_accepted() { + assert_eq!(validate(stream("outer(reason = \"why\")")), Ok(())); + } + + /// Only parentheses are argument lists. Brackets and braces belong to another grammar, and + /// descending into them would invent errors in token trees this macro does not own. + #[test] + fn other_delimiters_are_not_treated_as_argument_lists() { + assert_eq!(validate(stream("outer[reason = performance]")), Ok(())); + assert_eq!(validate(stream("outer{reason = performance}")), Ok(())); + } + + /// A byte string is not a string literal — it is a `&[u8]` — so it must be rejected. This is + /// also what pins the check to the *start* of the literal's text: `b"x"` ends with a quote just + /// as a real string does, and only the opening character tells them apart. `br"x"` is the case + /// that pins the strip to a *leading* `r` rather than any `r`: its `r` must not be reached. + #[test] + fn a_byte_string_value_is_rejected() { + assert_eq!( + validate(stream("reason = b\"x\"")), + Err("`reason` must be a string literal".to_owned()) + ); + assert_eq!(validate(stream("tag = b\"x\"")), Err("`tag` must be a string literal".to_owned())); + assert_eq!( + validate(stream("reason = br\"x\"")), + Err("`reason` must be a string literal".to_owned()) + ); + assert_eq!( + validate(stream("tag = br#\"x\"#")), + Err("`tag` must be a string literal".to_owned()) + ); + } + + /// A raw string is a string, and it is how one writes a reason containing a quote or a + /// backslash without escaping it. Every hash count must be accepted, since `r##"x"##` differs + /// from `r#"x"#` only in how much of the text can appear verbatim. + #[test] + fn a_raw_string_value_is_accepted_at_every_hash_count() { + assert_eq!(validate(stream("reason = r\"x\"")), Ok(())); + assert_eq!(validate(stream("reason = r#\"x\"#")), Ok(())); + assert_eq!(validate(stream("tag = r##\"x\"##")), Ok(())); + assert_eq!(validate(stream("arith.add_to_sub, reason = r#\"say \"no\"\"#")), Ok(())); + } + + /// A C string is a `&CStr`, not a `&str`, so it is rejected for the same reason a byte string + /// is. It is asserted separately because its prefix letter differs, and a check written to + /// name only `b` would let it through. + #[test] + fn a_c_string_value_is_rejected() { + assert_eq!( + validate(stream("reason = c\"x\"")), + Err("`reason` must be a string literal".to_owned()) + ); + assert_eq!(validate(stream("tag = cr\"x\"")), Err("`tag` must be a string literal".to_owned())); + } + + /// The hash strip must not be all that stands between a non-string and acceptance. A lifetime + /// or an identifier reaching this point has no quote after its hashes — of which it has none — + /// so the closing test on the quote is what rejects it, and this pins that test. + #[test] + fn a_literal_whose_prefix_strips_to_no_quote_is_rejected() { + assert!(!is_string_literal("42")); + assert!(!is_string_literal("r42")); + assert!(!is_string_literal("#")); + assert!(!is_string_literal("")); + assert!(is_string_literal("\"x\"")); + } + + /// A named value ends at its literal. Anything before the separating comma is malformed, + /// rather than another selector or a nested list that could hide a typo. + #[test] + fn trailing_tokens_after_a_named_value_are_rejected() { + assert_eq!( + validate(stream("reason = \"x\" (tag = broken)")), + Err("`reason` must not have trailing tokens after its value".to_owned()) + ); + assert_eq!( + validate(stream("test_timeout_multiplier = 2.0 + unexpected")), + Err("`test_timeout_multiplier` must not have trailing tokens after its value".to_owned()) + ); + } + + /// This has to be a compiler error, not merely a rejected token stream: otherwise a malformed + /// selector-free directive reaches the source scanner and widens into a suppression of all + /// mutations in scope. + #[test] + fn trailing_named_value_tokens_expand_to_a_compile_error() { + let out = inert("skip", stream("reason = \"checked\" + unexpected"), stream("fn f() {}")).to_string(); + + assert!(out.contains("compile_error"), "expected a compile_error in `{out}`"); + assert!(out.contains("trailing tokens"), "expected the reason in `{out}`"); + assert!(out.ends_with("fn f () { }"), "expected the item to survive in `{out}`"); + } + + /// The message ends up inside a string literal, so a message containing a quote or a backslash + /// has to come back out intact rather than ending the literal early. This is the case that the + /// previous hand-rolled escaping existed to handle. + #[test] + fn a_message_containing_quotes_and_backslashes_still_parses() { + let out = error(r#"he said "hi" and \ then left"#).to_string(); + + assert!(out.starts_with(":: core :: compile_error !"), "unexpected shape: `{out}`"); + assert!(out.contains(r#"\"hi\""#), "quotes should be escaped in `{out}`"); + assert!(out.contains(r"\\"), "backslashes should be escaped in `{out}`"); + } + + /// The invocation is fully qualified and terminated, so that it works in a crate that has + /// shadowed `core` and in statement position alike. + #[test] + fn the_error_is_a_fully_qualified_terminated_invocation() { + assert_eq!(error("boom").to_string(), ":: core :: compile_error ! (\"boom\") ;"); + } + + /// The case every annotated function is: one expression, on a function, expanding to that + /// function and nothing else. Anything extra would appear in every crate that states a value. + #[test] + fn a_stated_value_expands_to_the_item_alone() { + let out = value(stream("u32::MAX"), stream("fn f() -> u32 { 1 }")); + + assert_eq!(out.to_string(), "fn f () -> u32 { 1 }"); + } + + /// Every shape of expression a user might reach for has to survive, since the point of the + /// attribute is to name a value the tool could not guess — which is rarely a bare literal. + #[test] + fn the_expressions_worth_stating_are_accepted() { + for expression in [ + "0", + "-1", + "\"xyzzy\"", + "Box::new(File)", + "Some(Config { size: 8 })", + "vec![1, 2, 3]", + "(1, 2)", + "if cfg!(unix) { 1 } else { 2 }", + "Vec::::new()", + "core::iter::once(1).collect()", + ] { + assert_eq!( + validate_value(stream(expression), &stream("fn f() {}")), + Ok(()), + "`{expression}` should have been accepted" + ); + } + } + + /// An empty list states nothing. Reading it as "use the guess" would make an attribute that + /// looks deliberate do nothing at all, which is the failure this crate exists to prevent. + #[test] + fn stating_nothing_is_rejected() { + assert_eq!( + validate_value(stream(""), &stream("fn f() {}")), + Err("expected one expression, as in `#[gamma::value(0)]`".to_owned()) + ); + } + + /// A comma-separated list looks like a set of candidate values, and it is not one. Taking the + /// first and dropping the rest would lose the others without saying so. + #[test] + fn stating_more_than_one_expression_is_rejected_by_count() { + assert_eq!( + validate_value(stream("0, 1"), &stream("fn f() {}")), + Err("expected one expression, but `0 , 1` is 2; a site states one value".to_owned()) + ); + } + + /// A trailing comma still contains only one expression. It must take the parser-error path, + /// not the diagnostic reserved for lists with two or more candidates. + #[test] + fn one_expression_with_a_trailing_comma_is_not_reported_as_multiple_values() { + let rejected = validate_value(stream("0,"), &stream("fn f() {}")).expect_err("a trailing comma is not one expression"); + + assert!(rejected.starts_with("`0 ,` is not a Rust expression: "), "{rejected}"); + } + + /// Text that is not an expression cannot be repaired, and splicing it would move the error to + /// a mutant the user never wrote. The parser's own words are carried through, because they are + /// better than anything this could invent. + #[test] + fn text_that_is_not_an_expression_is_rejected() { + let rejected = validate_value(stream("1 +"), &stream("fn f() {}")); + + assert!( + rejected + .as_ref() + .is_err_and(|message| message.starts_with("`1 +` is not a Rust expression: ")), + "{rejected:?}" + ); + } + + /// A statement is not an expression, and `let` is the one users are most likely to try. + #[test] + fn a_statement_is_rejected() { + let rejected = validate_value(stream("let x = 1;"), &stream("fn f() {}")); + + assert!(rejected.is_err(), "{rejected:?}"); + } + + /// Two stated values would leave which one applies to the order the compiler expanded them in, + /// which is not a rule anyone can see in the source. The first to expand sees the second, + /// because an attribute macro is handed the item with the attributes below it still attached. + #[test] + fn a_second_stated_value_on_the_same_item_is_rejected() { + let item = stream("#[gamma::value(1)] fn f() -> u32 { 2 }"); + + assert_eq!( + validate_value(stream("0"), &item), + Err("an item may state one value; two would leave which of them applies to the order they were expanded in".to_owned()) + ); + } + + /// The duplicate check must not reach past the signature into the body, or a nested function + /// stating its own value — a different site, and an entirely legitimate one — would be read as + /// a second value on the outer function. + #[test] + fn a_value_stated_by_a_nested_function_is_not_a_duplicate() { + let item = stream("fn f() -> u32 { #[gamma::value(1)] fn g() -> u32 { 2 } g() }"); + + assert_eq!(validate_value(stream("0"), &item), Ok(())); + } + + /// Other attributes on the same item are not stated values, however many of them there are. + #[test] + fn other_attributes_on_the_item_are_not_duplicates() { + let item = stream("#[doc = \"why\"] #[inline] #[must_use] fn f() -> u32 { 2 }"); + + assert_eq!(validate_value(stream("0"), &item), Ok(())); + } + + /// A path that merely begins like the one being looked for is not it, or `#[gamma::skip]` on + /// the same function would be reported as a second stated value. + #[test] + fn another_attribute_in_the_same_namespace_is_not_a_duplicate() { + for other in [ + "#[gamma::skip(arith)]", + "#[gamma::expect_killed]", + "#[mutants::skip]", + "#[value(1)]", + ] { + let item = stream(&format!("{other} fn f() -> u32 {{ 2 }}")); + + assert_eq!(validate_value(stream("0"), &item), Ok(()), "`{other}` is not a stated value"); + } + } + + /// The three positions the attribute is written in have three different grammars, and all + /// three have to be accepted — including a trait method, which may have no body at all. + #[test] + fn a_function_a_method_and_a_trait_method_are_all_functions() { + for item in [ + "fn f() -> u32 { 1 }", + "pub async fn f() -> u32 { 1 }", + "unsafe fn f() -> u32 { 1 }", + "fn f(t: T) -> T where T: Send { t }", + "fn f(&self) -> u32 { self.n }", + "fn f(&self) -> u32;", + ] { + assert!(is_function(&stream(item)), "`{item}` is a function"); + } + + let declaration = stream("fn f(&self) -> u32;"); + let _ = syn::parse2::(declaration.clone()).unwrap_err(); + let _ = syn::parse2::(declaration.clone()).unwrap_err(); + let _ = syn::parse2::(declaration).unwrap(); + } + + /// A value stated on an `impl` block or a module would have to mean "every function beneath + /// this returns this", and one expression essentially never type-checks as more than one + /// signature's body. Rejecting it is what keeps inheritance from being invented by accident. + #[test] + fn a_value_stated_on_something_that_is_not_a_function_is_rejected() { + for item in [ + "impl Cursor { fn at(&self) -> usize { self.at } }", + "mod m { pub fn at() -> usize { 0 } }", + "struct S { n: u8 }", + "trait T { fn at(&self) -> usize; }", + "const N: u8 = 1;", + ] { + assert!( + validate_value(stream("0"), &stream(item)).is_err(), + "`{item}` is not a function and must be rejected" + ); + } + } + + /// The exact diagnostic matters here: it explains why the attribute is not inherited rather + /// than merely saying that parsing failed. + #[test] + fn a_non_function_reports_the_inheritance_problem() { + assert_eq!( + validate_value(stream("0"), &stream("mod m {}")), + Err("expected a function or method; a value stated on an `impl` block or a module would have to type-check as the body of every function beneath it".to_owned()) + ); + } + + /// Attribute recognition is deliberately exact. Near misses exercise each token boundary so + /// punctuation elsewhere in the item cannot be mistaken for `#[gamma::value]`. + #[test] + fn malformed_value_attribute_paths_are_not_recognized() { + assert!(!states_a_value(&stream("! [gamma::value(1)] fn f() {}"))); + assert!(!states_a_value(&stream("fn #[gamma::value(1)]"))); + + // A `#` not immediately followed by a bracketed group is not the start of an attribute, + // whether something else follows it or it is the stream's last token; both must be passed + // over rather than mistaken for `#[gamma::value(...)]`. + assert!(!states_a_value(&stream("# fn f() {}"))); + assert!(!states_a_value(&stream("#"))); + + for inner in ["", "other::value", "gamma=value", "gamma==value", "gamma::skip"] { + assert!(!names_value(&stream(inner)), "`{inner}` is not gamma::value"); + } + assert!(names_value(&stream("gamma::value"))); + } + + /// A declared trait method is a function with nothing to replace, and its implementations do + /// not inherit the value — so an attribute there would be a hint that generates no mutant + /// anywhere. It is the one function-shaped position a value cannot be stated on. + #[test] + fn a_value_stated_on_a_function_with_no_body_is_rejected() { + assert!(has_a_body(&stream("fn f() -> u32 { 1 }"))); + assert!(!has_a_body(&stream("fn f(&self) -> u32;"))); + + let rejected = validate_value(stream("0"), &stream("fn f(&self) -> u32;")).expect_err("a declaration has no body to replace"); + + assert!(rejected.contains("a declaration has none"), "{rejected}"); + } + + /// A rejected value still leaves the item behind, for the same reason a malformed suppression + /// does: one diagnostic about the attribute beats a pile about the missing function. + #[test] + fn a_rejected_value_keeps_the_item_and_names_the_macro() { + let out = value(stream(""), stream("fn f() -> u32 { 1 }")).to_string(); + + assert!(out.contains("#[gamma::value]"), "expected the macro to be named in `{out}`"); + assert!(out.ends_with("fn f () -> u32 { 1 }"), "expected the item to survive in `{out}`"); + } + + /// An accepted timeout multiplier expands to the item alone, with no `compile_error!` added — + /// the timeout family's counterpart to + /// [`a_valid_attribute_expands_to_the_item_alone`]. A regression that returned the + /// `compile_error!`-carrying item from the `Ok` arm would break every valid + /// `#[gamma::test_timeout_multiplier(...)]` in the wild. + #[test] + fn an_accepted_timeout_multiplier_expands_to_the_item_alone() { + let out = inert_timeout("test_timeout_multiplier", &stream("2.0"), stream("fn f() {}")); + + assert_eq!(out.to_string(), "fn f () { }"); + } + + /// A rejected timeout multiplier leaves the item behind and names the macro, exactly as a + /// rejected suppression or value does — the timeout family's counterpart. A regression that + /// returned the item unchanged from the `Err` arm (dropping the `compile_error!`) would let a + /// malformed multiplier compile clean. + #[test] + fn a_rejected_timeout_multiplier_keeps_the_item_and_names_the_macro() { + let out = inert_timeout("test_timeout_multiplier", &stream("inf"), stream("fn f() {}")).to_string(); + + assert!(out.contains("compile_error"), "expected a compile_error in `{out}`"); + assert!( + out.contains("#[gamma::test_timeout_multiplier]"), + "expected the macro to be named in `{out}`" + ); + assert!(out.ends_with("fn f () { }"), "expected the item to survive in `{out}`"); + } + + /// A timeout multiplier must be a positive, bounded number when written with `key = value`. + #[test] + fn timeout_multiplier_validation() { + let refused = Err("`test_timeout_multiplier` must be a positive number no greater than 1000000".to_owned()); + + assert_eq!(validate(stream("test_timeout_multiplier = 2.5")), Ok(())); + assert_eq!(validate(stream("timeout_multiplier = 3.0")), Ok(())); + assert_eq!(validate(stream("test_timeout_multiplier = 2")), Ok(())); + assert_eq!(validate(stream("test_timeout_multiplier = \"fast\"")), refused); + assert_eq!(validate(stream("test_timeout_multiplier = -1.0")), refused); + + // The bound must agree with `bounds::factor` in the tool, so that a value the tool refuses + // is not silently accepted while the user's crate compiles. + assert_eq!(validate(stream("test_timeout_multiplier = 1e300")), refused); + assert_eq!(validate(stream("test_timeout_multiplier = 1000001")), refused); + assert_eq!(validate(stream("test_timeout_multiplier = 1000000")), Ok(())); + + assert_eq!( + validate(stream("test_timeout_multiplier(2.0)")), + Err("`test_timeout_multiplier` must be written as `test_timeout_multiplier = `".to_owned()) + ); + assert_eq!(validate_timeout_multiplier(&stream("2.5")), Ok(())); + assert_eq!( + validate_timeout_multiplier(&stream("arith, test_timeout_multiplier = 2.5, reason = \"complex math\"")), + Ok(()) + ); + // A leading group (here a parenthesized list) is neither a literal, a sign, nor an + // identifier, so it falls through the positional check's wildcard arm exactly as a + // non-numeric leading identifier does, and the whole attribute is handed to the general + // structural validator instead. That validator only inspects identifier keys and bare + // groups it recurses into, so a lone parenthesized literal — matching none of the + // recognized keys — is accepted. + assert_eq!(validate_timeout_multiplier(&stream("(2.5)")), Ok(())); + // `inf`, `nan`, and `infinity` arrive as identifiers rather than literals and parse as + // non-finite floats, so they slip past the literal check; they must be refused like any + // other out-of-range multiplier rather than mistaken for a bare selector and accepted. + for value in ["0", "-1", "1e300", "\"fast\"", "inf", "nan", "infinity", "NaN"] { + assert!(validate_timeout_multiplier(&stream(value)).is_err(), "`{value}` was accepted"); + } + assert_eq!( + validate_timeout_multiplier(&stream("")), + Err("expected a timeout multiplier, as in `#[gamma::test_timeout_multiplier(2.0)]`".to_owned()) + ); + } + + /// Deeply nested expressions or items are rejected before syn's recursive descent parser can + /// overflow the compiler's stack. + #[test] + fn deeply_nested_tokens_are_rejected_safely() { + let depth = 100; + let deep_expr = format!("{}1{}", "(".repeat(depth), ")".repeat(depth)); + let err = validate_value(stream(&deep_expr), &stream("fn f() -> u32 { 1 }")).expect_err("nested expr rejected"); + assert_eq!(err, "expression nests too deeply to be safely parsed"); + + let deep_item = format!("fn f() -> u32 {{ {}1{} }}", "(".repeat(depth), ")".repeat(depth)); + let err = validate_value(stream("1"), &stream(&deep_item)).expect_err("nested item rejected"); + assert_eq!(err, "item nests too deeply to be safely parsed"); + } + + #[test] + fn deeply_chained_postfix_expressions_are_rejected_safely() { + let chains = [ + format!("call{}", "()".repeat(NESTING_LIMIT * CHAIN_FACTOR + 1)), + format!("value{}", "[0]".repeat(NESTING_LIMIT * CHAIN_FACTOR + 1)), + format!("value{}", ".call()".repeat(NESTING_LIMIT * CHAIN_FACTOR + 1)), + ]; + + for expression in chains { + let err = validate_value(stream(&expression), &stream("fn f() -> u32 { 1 }")).expect_err("postfix chain rejected"); + + assert_eq!(err, "expression nests too deeply to be safely parsed"); + } + } + + #[test] + fn deeply_chained_cast_expressions_are_rejected_safely() { + let expression = format!("1{}", " as u64".repeat(NESTING_LIMIT * CHAIN_FACTOR + 1)); + let err = validate_value(stream(&expression), &stream("fn f() -> u32 { 1 }")).expect_err("cast chain rejected"); + + assert_eq!(err, "expression nests too deeply to be safely parsed"); + } + + #[test] + fn a_long_unary_chain_expands_to_a_guard_diagnostic() { + let expression = format!("{}1", "-".repeat(NESTING_LIMIT * CHAIN_FACTOR + 1)); + let output = value(stream(&expression), stream("fn f() -> i32 { 1 }")).to_string(); + + assert!(output.contains("compile_error"), "{output}"); + assert!(output.contains("expression nests too deeply"), "{output}"); + } + + #[test] + fn a_long_binary_chain_is_rejected_by_the_guard() { + let expression = format!("true{}", " || true".repeat(NESTING_LIMIT * CHAIN_FACTOR + 1)); + let error = validate_value(stream(&expression), &stream("fn f() -> bool { true }")).expect_err("binary chain must be rejected"); + + assert_eq!(error, "expression nests too deeply to be safely parsed"); + } +} diff --git a/crates/cargo-gamma-attrs-impl/src/lib.rs b/crates/cargo-gamma-attrs-impl/src/lib.rs new file mode 100644 index 000000000..61b274e78 --- /dev/null +++ b/crates/cargo-gamma-attrs-impl/src/lib.rs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![cfg(not(all(test, miri)))] +#![forbid( + unsafe_code, + reason = "every raw platform call in this workspace lives in `cargo-gamma-unsafe`, behind a safe interface" +)] + +//! The implementation behind [`cargo-gamma-attrs`](https://crates.io/crates/cargo-gamma-attrs), +//! which is where the inert `#[gamma::skip]`, `#[gamma::expect_survived]` and +//! `#[gamma::expect_killed]` attributes are actually exposed. +//! +//! You almost certainly want that crate instead. This one is a normal library rather than a +//! proc-macro crate so its logic can be called by ordinary tests, covered, and mutation tested. +//! What remains in the proc-macro crate is a shim thin enough to read at a glance. +//! +//! # Why this crate exists +//! +//! `cargo-gamma-attrs` is a proc-macro crate, and a proc macro's code runs only inside `rustc`, +//! while some *other* crate is being compiled. That puts it beyond the reach of both measurements +//! this project cares about: +//! +//! - A coverage harness collects counters from test binaries. A proc macro increments its counters +//! inside the compiler, which writes no profile the harness sees. +//! - A mutation run selects one mutant per test process at run time. A proc macro has already +//! finished by then, so none of its mutants can be active while a test is watching. +//! +//! Splitting the logic into an ordinary library makes it reachable by coverage and mutation tests. +//! The proc-macro crate remains a thin shim. +//! +//! # What the macros accept +//! +//! See the [`cargo-gamma-attrs`](https://docs.rs/cargo-gamma-attrs) documentation for the +//! user-facing description. In brief: a comma-separated selector list, optionally followed by +//! `reason = "..."` and `tag = "..."`, both of which must be string literals. +//! +//! `#[gamma::value()]` instead takes an expression. It is checked by [`value`], because its +//! argument is spliced into the user's crate as a mutant and must be exactly one expression. +//! +//! # Stability +//! +//! This crate is an implementation detail of `cargo-gamma-attrs` and carries no stability +//! guarantee of its own. Depend on `cargo-gamma-attrs`. + +mod implementation; + +pub use implementation::{CHAIN_FACTOR, MOST_FACTOR, NESTING_LIMIT, inert, inert_timeout, value}; diff --git a/crates/cargo-gamma-attrs/Cargo.toml b/crates/cargo-gamma-attrs/Cargo.toml new file mode 100644 index 000000000..a0d02aac5 --- /dev/null +++ b/crates/cargo-gamma-attrs/Cargo.toml @@ -0,0 +1,38 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[package] +name = "cargo-gamma-attrs" +description = "Inert attribute macros for suppressing cargo-gamma mutations" +version = "0.1.0" +readme = "README.md" +keywords = ["cargo", "mutation-testing", "testing"] +categories = ["development-tools::testing"] + +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo-gamma-attrs" + +[package.metadata.docs.rs] +all-features = true + +# This crate only re-exports inert proc macros and has no attributable runtime lines. +[package.metadata.coverage-gate] +min-lines-percent = 0.0 + +# Users write `#[gamma::skip]`, which is an attribute *path*, not a package name, so the +# lib name is deliberately short: it appears in every instrumented crate. +[lib] +name = "gamma" +proc-macro = true + +[dependencies] +cargo-gamma-attrs-impl = { workspace = true } + +# >>> anvil-managed: anvil-lints +[lints] +workspace = true +# <<< anvil-managed: anvil-lints diff --git a/crates/cargo-gamma-attrs/README.md b/crates/cargo-gamma-attrs/README.md new file mode 100644 index 000000000..e4c18d165 --- /dev/null +++ b/crates/cargo-gamma-attrs/README.md @@ -0,0 +1,340 @@ +
+ Cargo-Gamma-Attrs Logo + +# Cargo-Gamma-Attrs + +[![crates.io](https://img.shields.io/crates/v/cargo-gamma-attrs.svg)](https://crates.io/crates/cargo-gamma-attrs) +[![docs.rs](https://docs.rs/cargo-gamma-attrs/badge.svg)](https://docs.rs/cargo-gamma-attrs) +[![MSRV](https://img.shields.io/crates/msrv/cargo-gamma-attrs)](https://crates.io/crates/cargo-gamma-attrs) +[![CI](https://github.com/microsoft/ox-tools/actions/workflows/main.yml/badge.svg?event=push)](https://github.com/microsoft/ox-tools/actions/workflows/main.yml) +[![Coverage](https://codecov.io/gh/microsoft/ox-tools/graph/badge.svg?token=FCUG0EL5TI)](https://codecov.io/gh/microsoft/ox-tools) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](../../LICENSE) +This crate was developed as part of the Oxidizer project + +
+ +Inert attribute macros for controlling and suppressing mutations generated by +[`cargo-gamma`][__link0]. + +```rust +#[gamma::skip] +fn hand_verified(x: u32) -> u32 { + x + 1 +} + +#[gamma::skip(arith, reason = "fixed-point math, checked by proptest")] +fn scaled(a: i64, b: i64) -> i64 { + a * b / 1000 +} + +#[gamma::test_timeout_multiplier(3.0)] +fn slow_algorithm(data: &[u8]) -> usize { + data.len() * 2 +} +``` + +Every macro in this crate expands to the annotated item unchanged. The attributes exist so +that `cargo-gamma` can see them, and so that a typo in a selector list or invalid argument is a +compile error in your crate rather than a silently ineffective directive. + +## Depending on it + +```toml +[dependencies] +cargo-gamma-attrs = "0.1" +``` + +The package is `cargo-gamma-attrs` but its library is named `gamma`, which is what lets the +attributes be written as `#[gamma::skip]` with no `use` and no rename in the manifest. That +spelling matters more than it looks: it is the same text as the comment form, so the two +channels stay character-for-character identical and one can become the other by deleting two +slashes. + +An ordinary dependency rather than a dev-dependency, because these attributes are written on the +code being measured rather than on the tests measuring it, and `cargo build` compiles a library +target without its dev-dependencies. The crates are tiny and the attributes expand to nothing, +so what ships is a build-time dependency and no code. A crate that would rather not carry one at +all has the comment form, which needs no dependency whatever: + +```text +// #[gamma::skip(arith)] +fn scaled(a: i64, b: i64) -> i64 { a * b / 1000 } +``` + +## The vocabulary + +Arguments are the same selector language the command line and the config file use, so one name +learned is one name that works everywhere. A selector is a mutator name (`arith.add_to_sub`), a +family prefix (`arith`), a profile (`@arithmetic`), or an academic alias (`ROR`); `!` in front +removes one from the set, and selectors apply left to right, so `arith, !arith.add_to_sub` +means what it reads as. No arguments at all means every mutator. + +```rust +// Every mutator. +#[gamma::skip] +fn everything() {} + +// One mutator, by its full name. +#[gamma::skip(relational.lt_to_le)] +fn one_mutator(a: u32, b: u32) -> bool { + a < b +} + +// A whole family. +#[gamma::skip(literal)] +fn a_family() -> &'static str { + "not asserted on" +} + +// A profile. +#[gamma::skip(@arithmetic)] +fn a_profile(a: i32, b: i32) -> i32 { + a * b + 1 +} + +// An academic alias, matched case-insensitively. +#[gamma::skip(ROR)] +fn an_alias(a: u32, b: u32) -> bool { + a >= b +} + +// A family, minus one of its members. +#[gamma::skip(arith, !arith.add_to_sub)] +fn all_but_one(a: i32, b: i32) -> i32 { + a * b - 1 +} +``` + +Two optional named arguments may follow the selectors: + +* `reason = "..."` — why the suppression exists. It is carried into the report, so a reviewer + meets the justification beside the number rather than having to go and find the line. +* `tag = "..."` — a label for grouping suppressions, so a project can audit “everything we + skipped for performance” as a set. + +```rust +#[gamma::skip(stmt, reason = "instrumentation only", tag = "telemetry")] +fn record(counter: &mut u64) { + *counter += 1; +} +``` + +Both must be string literals. Writing `reason = performance` or `reason("...")` is a compile +error, which is the point: a suppression that does not parse should fail loudly rather than +quietly cover nothing. + +These next four examples are compiled and required to *fail*, because the claim above is the +whole reason this crate exists and asserting it anywhere else would only prove that a token +stream containing `compile_error!` was produced — not that a user’s build stops. The failure +reaches the user’s build or it does not, and only compiling one can tell. + +```rust +// `reason` is not a string literal. +#[gamma::skip(reason = performance)] +fn scaled(a: i64) -> i64 { + a * 2 +} +``` + +```rust +// `reason` is called rather than assigned. +#[gamma::skip(reason("hand verified"))] +fn scaled(a: i64) -> i64 { + a * 2 +} +``` + +```rust +// `tag` is a number. +#[gamma::expect_survived(literal, tag = 7)] +fn describe(n: usize) -> String { + format!("{n} items") +} +``` + +```rust +// The same rule applies however deeply the argument is nested. +#[gamma::expect_killed(a(b(tag = 7)))] +fn checksum(bytes: &[u8]) -> usize { + bytes.len() +} +``` + +```rust +// A named value ends at the literal, so a selector-free typo cannot become `skip` for all. +#[gamma::skip(reason = "hand verified" + unexpected)] +fn scaled(a: i64) -> i64 { + a * 2 +} +``` + +The item itself still parses in every one of those cases: the macro expands to the item +*preceded* by the `compile_error!`, so the build reports the malformed attribute and nothing +else. An expansion that swallowed the item would bury one honest diagnostic under every use of +the function it annotated. + +```rust +#[gamma::skip(reason = performance)] +fn scaled(a: i64) -> i64 { + a * 2 +} + +// Resolves, which it could not if the attribute had eaten the item it was written on. The only +// error this example produces is the one about `reason`. +fn caller() -> i64 { + scaled(21) +} +``` + +## The suppression macros + +[`macro@skip`][__link1] withdraws mutants. [`macro@expect_survived`][__link2] and [`macro@expect_killed`][__link3] assert +the opposite things about them and fail the run when the assertion breaks: the first is a +ratchet for a known gap, so that closing it is deliberate and visible rather than accidental, +and the second pins a test whose value someone wants protected from erosion. + +```rust +// Records a gap. If someone later writes a test that kills these mutants, the run fails until +// the attribute is removed — so the gap closes on purpose, with the record cleaned up. +#[gamma::expect_survived(literal, reason = "log text is deliberately not asserted on")] +fn describe(n: usize) -> String { + format!("processed {n} items") +} + +// Pins a guarantee. If a refactor weakens the suite until one of these mutants survives, the +// run fails immediately rather than after the score has slowly drifted. +#[gamma::expect_killed] +fn checksum(bytes: &[u8]) -> u32 { + bytes.iter().fold(0u32, |acc, b| { + acc.wrapping_mul(31).wrapping_add(u32::from(*b)) + }) +} + +// Both take the same selectors, so an expectation can be as narrow as a suppression. +#[gamma::expect_killed(relational, reason = "the boundary tests exist to catch exactly these")] +fn in_range(x: i32, low: i32, high: i32) -> bool { + x >= low && x <= high +} +``` + +## Stating a value + +[`macro@value`][__link4] is the one macro that is not about suppression. It states the expression +`cargo-gamma` substitutes for a function’s body when it mutates the value that function +returns, in place of the one it would otherwise guess from the return type: + +```rust +// The tool cannot name a value of `dyn Reader`, so without this the site is never mutated and +// nothing ever asks whether the returned reader is checked. +#[gamma::value(Box::new(File))] +fn reader() -> Box { + Box::new(File) +} + +// Here it replaces a guess rather than creating a mutant that did not exist. +#[gamma::value(u32::MAX)] +fn budget() -> u32 { + 512 +} +``` + +It takes one expression rather than selectors, and it can only ever *add* a mutant or change +what an existing one substitutes — it is not a suppression channel, and the site it is written +on keeps being mutated whether the stated value compiles or not. A value that does not +type-check becomes an unviable mutant and is withdrawn on the round that discovers it, exactly +as a bad guess would have been. + +Write exactly one, directly on the function or method it applies to. Two on the same item is a +compile error rather than a rule about which one wins, and it is not inherited from an `impl` +block or a module, because one expression essentially never type-checks as the body of every +function beneath one. + +```rust +// Two stated values, which would otherwise leave the answer to expansion order. +#[gamma::value(0)] +#[gamma::value(1)] +fn budget() -> u32 { + 512 +} +``` + +## Where they can go + +The suppression macros go on any item, and cover everything beneath it — so an attribute on a +module covers the module, and one on an `impl` covers every method in it: + +```rust +#[gamma::skip(arith, reason = "generated fixed-point tables")] +mod tables { + pub fn scale(x: i64) -> i64 { + x * 4096 + } +} + +struct Cursor { + at: usize, +} + +#[gamma::skip(reason = "exercised only through the fuzzer")] +impl Cursor { + fn advance(&mut self) { + self.at += 1; + } + + fn at_start(&self) -> bool { + self.at == 0 + } +} +``` + +[`macro@value`][__link5] is the exception, and deliberately so: it goes on a function or a method and +nowhere else, because the expression it states has to type-check as that one signature’s body. + +## Relationship to the comment form + +Attributes in statement and expression position are still unstable, so the primary channel for +surgical suppression is a comment that is *character-for-character* the attribute form: + +```text +// #[gamma::skip(arith)] +let total = a * scale + offset; +``` + +When expression attributes stabilize, deleting the two slashes turns every such comment into +real Rust. This crate is what those comments become. + +The comment form takes exactly the same arguments, so nothing has to be relearned to move from +one to the other: + +```text +// #[gamma::skip(relational, reason = "the bound is checked one frame later")] +if index < self.len { + // #[gamma::skip(stmt, tag = "telemetry")] + self.hits += 1; +} + +// #[gamma::expect_killed(@boundary)] +let clamped = value.min(limit); +``` + +The comment form is not validated by the compiler, since it is a comment — `cargo-gamma` +diagnoses a misspelled selector there instead, and refuses to treat an unrecognized directive +as a suppression that quietly did nothing. + +Choose the attribute when the unit being excused is a whole item, and the comment when it is one +statement or one expression inside an item that is otherwise worth mutating. A third channel, +`gamma.toml`, is the right place for a policy that is about the project rather than about +any particular line. + + +
+ +This crate was developed as part of The Oxidizer Project. Browse this crate's source code. + + + [__link0]: https://crates.io/crates/cargo-gamma + [__link1]: `macro@skip` + [__link2]: `macro@expect_survived` + [__link3]: `macro@expect_killed` + [__link4]: `macro@value` + [__link5]: `macro@value` diff --git a/crates/cargo-gamma-attrs/docs/DESIGN.md b/crates/cargo-gamma-attrs/docs/DESIGN.md new file mode 100644 index 000000000..699db15ba --- /dev/null +++ b/crates/cargo-gamma-attrs/docs/DESIGN.md @@ -0,0 +1,24 @@ +# cargo-gamma-attrs — Design + +> Status: **Implemented**. +> Crate name: `cargo-gamma-attrs`. + +## Purpose + +This proc-macro crate exposes the user-facing `gamma` attribute namespace used +to suppress mutations and state expected outcomes. Each attribute validates +its arguments and otherwise returns the annotated item unchanged. + +## Boundaries + +- The library target is deliberately named `gamma`, so users write + `#[gamma::skip]`. +- Parsing and validation live in `cargo-gamma-attrs-impl`; this proc-macro + crate remains a thin compiler-hosted shim. +- The macros must not instrument code or add runtime behavior. + +## Public contract + +The supported attributes, selector grammar, and diagnostics are part of +cargo-gamma's source-level configuration contract. Invalid directives fail at +compile time instead of becoming silent no-ops. diff --git a/crates/cargo-gamma-attrs/favicon.ico b/crates/cargo-gamma-attrs/favicon.ico new file mode 100644 index 000000000..18acb282f --- /dev/null +++ b/crates/cargo-gamma-attrs/favicon.ico @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82caca17fd4d08a23a9d76f5c895029ff6bb0d13b0e2693ef229a4ea691ffccd +size 46496 diff --git a/crates/cargo-gamma-attrs/logo.png b/crates/cargo-gamma-attrs/logo.png new file mode 100644 index 000000000..20aaf26cf --- /dev/null +++ b/crates/cargo-gamma-attrs/logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc17b32e4b6b8c4f0c1d443b680a25b68e0cd8913954dfd047a7659a6bd0baff +size 131471 diff --git a/crates/cargo-gamma-attrs/src/lib.rs b/crates/cargo-gamma-attrs/src/lib.rs new file mode 100644 index 000000000..dcf652466 --- /dev/null +++ b/crates/cargo-gamma-attrs/src/lib.rs @@ -0,0 +1,599 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![cfg(not(all(test, miri)))] +#![forbid( + unsafe_code, + reason = "every raw platform call in this workspace lives in `cargo-gamma-unsafe`, behind a safe interface; enforcing that here is what makes it a property the compiler checks rather than a convention people remember" +)] + +//! Inert attribute macros for controlling and suppressing mutations generated by +//! [`cargo-gamma`](https://crates.io/crates/cargo-gamma). +//! +//! ```rust +//! #[gamma::skip] +//! fn hand_verified(x: u32) -> u32 { +//! x + 1 +//! } +//! +//! #[gamma::skip(arith, reason = "fixed-point math, checked by proptest")] +//! fn scaled(a: i64, b: i64) -> i64 { +//! a * b / 1000 +//! } +//! +//! #[gamma::test_timeout_multiplier(3.0)] +//! fn slow_algorithm(data: &[u8]) -> usize { +//! data.len() * 2 +//! } +//! ``` +//! +//! Every macro in this crate expands to the annotated item unchanged. The attributes exist so +//! that `cargo-gamma` can see them, and so that a typo in a selector list or invalid argument is a +//! compile error in your crate rather than a silently ineffective directive. +//! +//! # Depending on it +//! +//! ```toml +//! [dependencies] +//! cargo-gamma-attrs = "0.1" +//! ``` +//! +//! The package is `cargo-gamma-attrs` but its library is named `gamma`, which is what lets the +//! attributes be written as `#[gamma::skip]` with no `use` and no rename in the manifest. That +//! spelling matters more than it looks: it is the same text as the comment form, so the two +//! channels stay character-for-character identical and one can become the other by deleting two +//! slashes. +//! +//! An ordinary dependency rather than a dev-dependency, because these attributes are written on the +//! code being measured rather than on the tests measuring it, and `cargo build` compiles a library +//! target without its dev-dependencies. The crates are tiny and the attributes expand to nothing, +//! so what ships is a build-time dependency and no code. A crate that would rather not carry one at +//! all has the comment form, which needs no dependency whatever: +//! +//! ```text +//! // #[gamma::skip(arith)] +//! fn scaled(a: i64, b: i64) -> i64 { a * b / 1000 } +//! ``` +//! +//! # The vocabulary +//! +//! Arguments are the same selector language the command line and the config file use, so one name +//! learned is one name that works everywhere. A selector is a mutator name (`arith.add_to_sub`), a +//! family prefix (`arith`), a profile (`@arithmetic`), or an academic alias (`ROR`); `!` in front +//! removes one from the set, and selectors apply left to right, so `arith, !arith.add_to_sub` +//! means what it reads as. No arguments at all means every mutator. +//! +//! ```rust +//! // Every mutator. +//! #[gamma::skip] +//! fn everything() {} +//! +//! // One mutator, by its full name. +//! #[gamma::skip(relational.lt_to_le)] +//! fn one_mutator(a: u32, b: u32) -> bool { +//! a < b +//! } +//! +//! // A whole family. +//! #[gamma::skip(literal)] +//! fn a_family() -> &'static str { +//! "not asserted on" +//! } +//! +//! // A profile. +//! #[gamma::skip(@arithmetic)] +//! fn a_profile(a: i32, b: i32) -> i32 { +//! a * b + 1 +//! } +//! +//! // An academic alias, matched case-insensitively. +//! #[gamma::skip(ROR)] +//! fn an_alias(a: u32, b: u32) -> bool { +//! a >= b +//! } +//! +//! // A family, minus one of its members. +//! #[gamma::skip(arith, !arith.add_to_sub)] +//! fn all_but_one(a: i32, b: i32) -> i32 { +//! a * b - 1 +//! } +//! ``` +//! +//! Two optional named arguments may follow the selectors: +//! +//! - `reason = "..."` — why the suppression exists. It is carried into the report, so a reviewer +//! meets the justification beside the number rather than having to go and find the line. +//! - `tag = "..."` — a label for grouping suppressions, so a project can audit "everything we +//! skipped for performance" as a set. +//! +//! ```rust +//! #[gamma::skip(stmt, reason = "instrumentation only", tag = "telemetry")] +//! fn record(counter: &mut u64) { +//! *counter += 1; +//! } +//! ``` +//! +//! Both must be string literals. Writing `reason = performance` or `reason("...")` is a compile +//! error, which is the point: a suppression that does not parse should fail loudly rather than +//! quietly cover nothing. +//! +//! These next four examples are compiled and required to *fail*, because the claim above is the +//! whole reason this crate exists and asserting it anywhere else would only prove that a token +//! stream containing `compile_error!` was produced — not that a user's build stops. The failure +//! reaches the user's build or it does not, and only compiling one can tell. +//! +//! ```compile_fail +//! // `reason` is not a string literal. +//! #[gamma::skip(reason = performance)] +//! fn scaled(a: i64) -> i64 { +//! a * 2 +//! } +//! ``` +//! +//! ```compile_fail +//! // `reason` is called rather than assigned. +//! #[gamma::skip(reason("hand verified"))] +//! fn scaled(a: i64) -> i64 { +//! a * 2 +//! } +//! ``` +//! +//! ```compile_fail +//! // `tag` is a number. +//! #[gamma::expect_survived(literal, tag = 7)] +//! fn describe(n: usize) -> String { +//! format!("{n} items") +//! } +//! ``` +//! +//! ```compile_fail +//! // The same rule applies however deeply the argument is nested. +//! #[gamma::expect_killed(a(b(tag = 7)))] +//! fn checksum(bytes: &[u8]) -> usize { +//! bytes.len() +//! } +//! ``` +//! +//! ```compile_fail +//! // A named value ends at the literal, so a selector-free typo cannot become `skip` for all. +//! #[gamma::skip(reason = "hand verified" + unexpected)] +//! fn scaled(a: i64) -> i64 { +//! a * 2 +//! } +//! ``` +//! +//! The item itself still parses in every one of those cases: the macro expands to the item +//! *preceded* by the `compile_error!`, so the build reports the malformed attribute and nothing +//! else. An expansion that swallowed the item would bury one honest diagnostic under every use of +//! the function it annotated. +//! +//! ```compile_fail +//! #[gamma::skip(reason = performance)] +//! fn scaled(a: i64) -> i64 { +//! a * 2 +//! } +//! +//! // Resolves, which it could not if the attribute had eaten the item it was written on. The only +//! // error this example produces is the one about `reason`. +//! fn caller() -> i64 { +//! scaled(21) +//! } +//! ``` +//! +//! # The suppression macros +//! +//! [`macro@skip`] withdraws mutants. [`macro@expect_survived`] and [`macro@expect_killed`] assert +//! the opposite things about them and fail the run when the assertion breaks: the first is a +//! ratchet for a known gap, so that closing it is deliberate and visible rather than accidental, +//! and the second pins a test whose value someone wants protected from erosion. +//! +//! ```rust +//! // Records a gap. If someone later writes a test that kills these mutants, the run fails until +//! // the attribute is removed — so the gap closes on purpose, with the record cleaned up. +//! #[gamma::expect_survived(literal, reason = "log text is deliberately not asserted on")] +//! fn describe(n: usize) -> String { +//! format!("processed {n} items") +//! } +//! +//! // Pins a guarantee. If a refactor weakens the suite until one of these mutants survives, the +//! // run fails immediately rather than after the score has slowly drifted. +//! #[gamma::expect_killed] +//! fn checksum(bytes: &[u8]) -> u32 { +//! bytes.iter().fold(0u32, |acc, b| { +//! acc.wrapping_mul(31).wrapping_add(u32::from(*b)) +//! }) +//! } +//! +//! // Both take the same selectors, so an expectation can be as narrow as a suppression. +//! #[gamma::expect_killed(relational, reason = "the boundary tests exist to catch exactly these")] +//! fn in_range(x: i32, low: i32, high: i32) -> bool { +//! x >= low && x <= high +//! } +//! ``` +//! +//! # Stating a value +//! +//! [`macro@value`] is the one macro that is not about suppression. It states the expression +//! `cargo-gamma` substitutes for a function's body when it mutates the value that function +//! returns, in place of the one it would otherwise guess from the return type: +//! +//! ```rust +//! # trait Reader { fn read(&self) -> u8; } +//! # struct File; +//! # impl Reader for File { fn read(&self) -> u8 { 0 } } +//! // The tool cannot name a value of `dyn Reader`, so without this the site is never mutated and +//! // nothing ever asks whether the returned reader is checked. +//! #[gamma::value(Box::new(File))] +//! fn reader() -> Box { +//! Box::new(File) +//! } +//! +//! // Here it replaces a guess rather than creating a mutant that did not exist. +//! #[gamma::value(u32::MAX)] +//! fn budget() -> u32 { +//! 512 +//! } +//! ``` +//! +//! It takes one expression rather than selectors, and it can only ever *add* a mutant or change +//! what an existing one substitutes — it is not a suppression channel, and the site it is written +//! on keeps being mutated whether the stated value compiles or not. A value that does not +//! type-check becomes an unviable mutant and is withdrawn on the round that discovers it, exactly +//! as a bad guess would have been. +//! +//! Write exactly one, directly on the function or method it applies to. Two on the same item is a +//! compile error rather than a rule about which one wins, and it is not inherited from an `impl` +//! block or a module, because one expression essentially never type-checks as the body of every +//! function beneath one. +//! +//! ```compile_fail +//! // Two stated values, which would otherwise leave the answer to expansion order. +//! #[gamma::value(0)] +//! #[gamma::value(1)] +//! fn budget() -> u32 { +//! 512 +//! } +//! ``` +//! +//! # Where they can go +//! +//! The suppression macros go on any item, and cover everything beneath it — so an attribute on a +//! module covers the module, and one on an `impl` covers every method in it: +//! +//! ```rust +//! #[gamma::skip(arith, reason = "generated fixed-point tables")] +//! mod tables { +//! pub fn scale(x: i64) -> i64 { +//! x * 4096 +//! } +//! } +//! +//! struct Cursor { +//! at: usize, +//! } +//! +//! #[gamma::skip(reason = "exercised only through the fuzzer")] +//! impl Cursor { +//! fn advance(&mut self) { +//! self.at += 1; +//! } +//! +//! fn at_start(&self) -> bool { +//! self.at == 0 +//! } +//! } +//! ``` +//! +//! [`macro@value`] is the exception, and deliberately so: it goes on a function or a method and +//! nowhere else, because the expression it states has to type-check as that one signature's body. +//! +//! # Relationship to the comment form +//! +//! Attributes in statement and expression position are still unstable, so the primary channel for +//! surgical suppression is a comment that is *character-for-character* the attribute form: +//! +//! ```text +//! // #[gamma::skip(arith)] +//! let total = a * scale + offset; +//! ``` +//! +//! When expression attributes stabilize, deleting the two slashes turns every such comment into +//! real Rust. This crate is what those comments become. +//! +//! The comment form takes exactly the same arguments, so nothing has to be relearned to move from +//! one to the other: +//! +//! ```text +//! // #[gamma::skip(relational, reason = "the bound is checked one frame later")] +//! if index < self.len { +//! // #[gamma::skip(stmt, tag = "telemetry")] +//! self.hits += 1; +//! } +//! +//! // #[gamma::expect_killed(@boundary)] +//! let clamped = value.min(limit); +//! ``` +//! +//! The comment form is not validated by the compiler, since it is a comment — `cargo-gamma` +//! diagnoses a misspelled selector there instead, and refuses to treat an unrecognized directive +//! as a suppression that quietly did nothing. +//! +//! Choose the attribute when the unit being excused is a whole item, and the comment when it is one +//! statement or one expression inside an item that is otherwise worth mutating. A third channel, +//! `gamma.toml`, is the right place for a policy that is about the project rather than about +//! any particular line. + +#![cfg_attr(coverage_nightly, feature(coverage_attribute))] +// Nothing in this crate can be reached by a coverage run, or killed by a mutation run. Its only +// caller is `rustc`, during the expansion of some *other* crate: the counters a proc macro increments +// inside the compiler are never written to a profile the harness collects, and a mutation run has +// finished building long before it selects a mutant, so no mutant of this code is ever active while +// a test is watching. +// +// That is why the substance lives in `cargo-gamma-attrs-impl`, an ordinary library that ordinary +// tests can call. What is left here is a shim thin enough to read at a glance, and marking it off +// is the difference between an honest 100% and a figure with an unexplained hole in it. +#![cfg_attr(coverage_nightly, coverage(off))] +// A proc-macro crate's exported functions are `TokenStream -> TokenStream`; documenting each +// individually would be noise given the crate-level docs above. +#![allow(clippy::missing_const_for_fn, reason = "proc-macro entry points cannot be const")] + +use proc_macro::TokenStream; + +/// Suppresses mutations on the annotated item and everything beneath it. +/// +/// With no arguments, every mutator is suppressed. With arguments, only the listed selectors are: +/// a mutator name (`arith.add_to_sub`), a family prefix (`arith`), a profile (`@removal`), or an +/// academic alias (`ROR`). +/// +/// # Examples +/// +/// ```rust +/// // Nothing in here is mutated at all. +/// #[gamma::skip] +/// fn hand_verified(x: u32) -> u32 { +/// x + 1 +/// } +/// +/// // Only arithmetic is left alone; the relational mutants on `>` are still generated. +/// #[gamma::skip(arith, reason = "fixed-point, covered by proptest")] +/// fn positive_scaled(a: i64, b: i64) -> bool { +/// a * b / 1000 > 0 +/// } +/// +/// // A profile, minus one member, with a tag for auditing. +/// #[gamma::skip(@arithmetic, !arith.add_to_sub, tag = "hot-loop")] +/// fn blend(a: i32, b: i32) -> i32 { +/// a * b + (a - b) +/// } +/// ``` +#[proc_macro_attribute] +pub fn skip(attr: TokenStream, item: TokenStream) -> TokenStream { + cargo_gamma_attrs_impl::inert("skip", attr.into(), item.into()).into() +} + +/// Asserts that the selected mutants *survive*, and fails the run if any is killed. +/// +/// This is a ratchet for known gaps: it records the gap so that closing it is a deliberate, +/// visible act rather than an accident. Unlike [`macro@skip`], the mutants are still generated and +/// still tested — what changes is which outcome is treated as a failure. +/// +/// # Examples +/// +/// ```rust +/// // Nobody asserts on this text today. When somebody starts to, the run fails until this +/// // attribute is deleted, so the record never outlives the gap it describes. +/// #[gamma::expect_survived(literal, reason = "log text is deliberately not asserted on")] +/// fn describe(n: usize) -> String { +/// format!("processed {n} items") +/// } +/// ``` +#[proc_macro_attribute] +pub fn expect_survived(attr: TokenStream, item: TokenStream) -> TokenStream { + cargo_gamma_attrs_impl::inert("expect_survived", attr.into(), item.into()).into() +} + +/// Asserts that the selected mutants are *killed*, and fails the run if any survives. +/// +/// This pins a guarantee. A refactor that quietly weakens the suite fails the run at the line that +/// lost its cover, rather than showing up later as a slow drift in the overall score. +/// +/// # Examples +/// +/// ```rust +/// // Every mutant of this function must be caught by some test. +/// #[gamma::expect_killed] +/// fn checksum(bytes: &[u8]) -> u32 { +/// bytes.iter().fold(0u32, |acc, b| { +/// acc.wrapping_mul(31).wrapping_add(u32::from(*b)) +/// }) +/// } +/// +/// // Or just the boundary ones, because those are what the boundary tests exist for. +/// #[gamma::expect_killed(@boundary, reason = "guarded by the range tests")] +/// fn in_range(x: i32, low: i32, high: i32) -> bool { +/// x >= low && x <= high +/// } +/// ``` +#[proc_macro_attribute] +pub fn expect_killed(attr: TokenStream, item: TokenStream) -> TokenStream { + cargo_gamma_attrs_impl::inert("expect_killed", attr.into(), item.into()).into() +} + +/// States the expression a function's return-value mutant substitutes for its body. +/// +/// `cargo-gamma` reads return types syntactically, so it guesses. Where the guess is wrong the +/// mutant does not compile and is withdrawn, and where no guess could hold — a bare type parameter, +/// an associated type, a `Box`, a non-iterator `impl Trait` — no mutant is offered at +/// all and the site goes unmeasured. This says what to substitute, which replaces the guess in the +/// first case and creates the mutant in the second. +/// +/// Write exactly one, directly on the function or method whose value it states. It is not inherited +/// from an `impl` block or a module, and stating one on either is a compile error: a single +/// expression essentially never type-checks as the body of every function beneath it. For the same +/// reason it cannot be stated on a trait method that is only declared — there is no body to replace, +/// and the implementations do not inherit it. `const fn` bodies and empty bodies are never mutated, +/// so a value stated on one is honoured by nothing. +/// +/// Nothing is taken on trust. The stated expression becomes an ordinary mutant, and if it does not +/// type-check it is withdrawn exactly as any other unviable mutant is — one rollback round, no +/// special handling, and never a suppression. That is what makes this the safest hint there is: the +/// compiler checks it. +/// +/// # Examples +/// +/// ```rust +/// # trait Reader { fn read(&self) -> u8; } +/// # struct File; +/// # impl Reader for File { fn read(&self) -> u8 { 0 } } +/// // No guess could hold here: `dyn Reader` names a capability, not a type with a `Default`, so +/// // without this the site is never mutated at all. +/// #[gamma::value(Box::new(File))] +/// fn reader() -> Box { +/// Box::new(File) +/// } +/// +/// // A stated value replaces what the tool would have guessed for the return type. +/// #[gamma::value(u32::MAX)] +/// fn budget() -> u32 { +/// 512 +/// } +/// ``` +/// +/// These fail to compile, which is the point of validating the argument here rather than leaving a +/// malformed hint to be quietly ignored by the tool: +/// +/// ```compile_fail +/// // Nothing is stated. +/// #[gamma::value()] +/// fn budget() -> u32 { +/// 512 +/// } +/// ``` +/// +/// ```compile_fail +/// // Two expressions; a site states one value. +/// #[gamma::value(0, 1)] +/// fn budget() -> u32 { +/// 512 +/// } +/// ``` +/// +/// ```compile_fail +/// // Not an expression at all. +/// #[gamma::value(1 +)] +/// fn budget() -> u32 { +/// 512 +/// } +/// ``` +/// +/// ```compile_fail +/// // Two stated values, which would leave the answer to expansion order. +/// #[gamma::value(0)] +/// #[gamma::value(1)] +/// fn budget() -> u32 { +/// 512 +/// } +/// ``` +/// +/// ```compile_fail +/// // An `impl` block cannot state one value for every method in it. +/// struct Cursor { at: usize } +/// +/// #[gamma::value(0)] +/// impl Cursor { +/// fn at(&self) -> usize { +/// self.at +/// } +/// } +/// ``` +/// +/// ```compile_fail +/// // A declaration has no body to replace, and its implementations do not inherit the value. +/// trait Cursor { +/// #[gamma::value(0)] +/// fn at(&self) -> usize; +/// } +/// ``` +#[proc_macro_attribute] +pub fn value(attr: TokenStream, item: TokenStream) -> TokenStream { + cargo_gamma_attrs_impl::value(attr.into(), item.into()).into() +} + +/// Sets or overrides the timeout multiplier for test execution of mutants in the annotated item. +/// +/// When a mutant in the annotated item is tested, the timeout multiplier specified here overrides +/// the project default from `gamma.toml` or the `--test-timeout-multiplier` command-line argument. +/// +/// # Examples +/// +/// ```rust +/// // Gives mutants in this slow or variable test target 3x the baseline timeout. +/// #[gamma::test_timeout_multiplier(3.0)] +/// fn heavy_computation(data: &[u8]) -> Vec { +/// data.iter().map(|b| b.wrapping_add(1)).collect() +/// } +/// +/// // Named form with reasons or specific mutators. +/// #[gamma::test_timeout_multiplier(arith, test_timeout_multiplier = 2.5, reason = "complex math")] +/// fn compute_hash(seed: u64) -> u64 { +/// seed.wrapping_mul(6364136223846793005).wrapping_add(1) +/// } +/// ``` +/// +/// A malformed multiplier is a compile error, so a mistyped budget fails loudly at build time +/// rather than being quietly rejected later by the tool's directive scanner — or worse, reaching +/// `Duration::mul_f64` unbounded. As with the suppression and value macros, these examples are +/// compiled and required to *fail*: only compiling one proves the failure reaches a user's build. +/// +/// ```compile_fail +/// // Not a number at all. +/// #[gamma::test_timeout_multiplier("fast")] +/// fn heavy(data: &[u8]) -> usize { +/// data.len() +/// } +/// ``` +/// +/// ```compile_fail +/// // `inf` parses as a float but is not finite, so it can never bound a timeout. It arrives as an +/// // identifier rather than a literal, which is exactly the case a weaker guard lets through. +/// #[gamma::test_timeout_multiplier(inf)] +/// fn heavy(data: &[u8]) -> usize { +/// data.len() +/// } +/// ``` +/// +/// ```compile_fail +/// // Negative, so smaller than any baseline it would scale. +/// #[gamma::test_timeout_multiplier(-1.0)] +/// fn heavy(data: &[u8]) -> usize { +/// data.len() +/// } +/// ``` +#[proc_macro_attribute] +pub fn test_timeout_multiplier(attr: TokenStream, item: TokenStream) -> TokenStream { + cargo_gamma_attrs_impl::inert_timeout("test_timeout_multiplier", &attr.into(), item.into()).into() +} + +/// Alias for [`macro@test_timeout_multiplier`]. +#[proc_macro_attribute] +pub fn timeout_multiplier(attr: TokenStream, item: TokenStream) -> TokenStream { + cargo_gamma_attrs_impl::inert_timeout("timeout_multiplier", &attr.into(), item.into()).into() +} + +/// Generic `gamma` attribute for mutant control, including timeout multiplier overrides. +/// +/// # Examples +/// +/// ```rust +/// use gamma::gamma; +/// +/// #[gamma(test_timeout_multiplier = 2.5)] +/// fn slow_algorithm(n: usize) -> usize { +/// n * 2 +/// } +/// ``` +#[proc_macro_attribute] +pub fn gamma(attr: TokenStream, item: TokenStream) -> TokenStream { + cargo_gamma_attrs_impl::inert_timeout("gamma", &attr.into(), item.into()).into() +} diff --git a/crates/cargo-gamma-engine/Cargo.toml b/crates/cargo-gamma-engine/Cargo.toml new file mode 100644 index 000000000..1c2ac0556 --- /dev/null +++ b/crates/cargo-gamma-engine/Cargo.toml @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[package] +name = "cargo-gamma-engine" +description = "Internal Rust source mutation engine for cargo-gamma" +version = "0.1.0" +readme = "README.md" +keywords = ["cargo", "mutation-testing", "testing"] +categories = ["development-tools::testing"] + +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo-gamma-engine" + +[package.metadata.docs.rs] +all-features = true + +# Establish the cross-platform CI baseline for this new crate. Raise it as coverage improves. +[package.metadata.coverage-gate] +min-lines-percent = 99.2 + +[dependencies] +blake3 = { workspace = true, features = ["std"] } +camino = { workspace = true } +compact_str = { workspace = true, features = ["serde", "std"] } +proc-macro2 = { workspace = true, features = ["span-locations", "proc-macro"] } +rustc-hash = { workspace = true, features = ["std"] } +rustc_lexer = { workspace = true } +serde = { workspace = true, features = ["derive", "rc", "std"] } +strsim = { workspace = true } +syn = { workspace = true, features = ["full", "parsing", "printing", "visit", "extra-traits", "clone-impls", "proc-macro"] } + +[dev-dependencies] +bolero = { workspace = true } +walkdir = { workspace = true } + +# >>> anvil-managed: anvil-lints +[lints] +workspace = true +# <<< anvil-managed: anvil-lints diff --git a/crates/cargo-gamma-engine/README.md b/crates/cargo-gamma-engine/README.md new file mode 100644 index 000000000..48f0a32dc --- /dev/null +++ b/crates/cargo-gamma-engine/README.md @@ -0,0 +1,29 @@ +
+ Cargo-Gamma-Engine Logo + +# Cargo-Gamma-Engine + +[![crates.io](https://img.shields.io/crates/v/cargo-gamma-engine.svg)](https://crates.io/crates/cargo-gamma-engine) +[![docs.rs](https://docs.rs/cargo-gamma-engine/badge.svg)](https://docs.rs/cargo-gamma-engine) +[![MSRV](https://img.shields.io/crates/msrv/cargo-gamma-engine)](https://crates.io/crates/cargo-gamma-engine) +[![CI](https://github.com/microsoft/ox-tools/actions/workflows/main.yml/badge.svg?event=push)](https://github.com/microsoft/ox-tools/actions/workflows/main.yml) +[![Coverage](https://codecov.io/gh/microsoft/ox-tools/graph/badge.svg?token=FCUG0EL5TI)](https://codecov.io/gh/microsoft/ox-tools) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](../../LICENSE) +This crate was developed as part of the Oxidizer project + +
+ +This crate is an internal implementation detail of +[`cargo-gamma`][__link0]. It contains the Rust source parsing, +mutation collection, stable identity, and schema instrumentation pipeline. + +Do not depend on it directly. Its API may change incompatibly without notice; it is published +only so that `cargo-gamma` can be installed through crates.io. + + +
+ +This crate was developed as part of The Oxidizer Project. Browse this crate's source code. + + + [__link0]: https://crates.io/crates/cargo-gamma diff --git a/crates/cargo-gamma-engine/docs/DESIGN.md b/crates/cargo-gamma-engine/docs/DESIGN.md new file mode 100644 index 000000000..0914e0767 --- /dev/null +++ b/crates/cargo-gamma-engine/docs/DESIGN.md @@ -0,0 +1,28 @@ +# cargo-gamma-engine — Design + +> Status: **Implemented**. +> Crate name: `cargo-gamma-engine`. + +## Purpose + +This crate owns Rust source parsing, mutation-site discovery, stable mutation +identity, mutator selection, and mutant-schema instrumentation. + +## Boundaries + +- Input is Rust source plus a mutator selection; output is deterministic + mutation metadata and instrumented source. +- The engine does not invoke Cargo, run tests, supervise processes, or decide + campaign verdicts. +- Stable identities are content-derived so reports and incremental knowledge + remain meaningful as unrelated source changes. +- Mutation discovery uses source-visible type evidence to avoid replacements + known not to compile. This includes concrete `Self` and associated types in + implementations, standard time types without `Default`, and standard + `fmt::Result` aliases. +- The crate forbids unsafe code. + +## Stability + +The crate is published so `cargo-gamma` can be installed from crates.io. Its +Rust API is internal and carries no independent compatibility guarantee. diff --git a/crates/cargo-gamma-engine/favicon.ico b/crates/cargo-gamma-engine/favicon.ico new file mode 100644 index 000000000..18acb282f --- /dev/null +++ b/crates/cargo-gamma-engine/favicon.ico @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82caca17fd4d08a23a9d76f5c895029ff6bb0d13b0e2693ef229a4ea691ffccd +size 46496 diff --git a/crates/cargo-gamma-engine/logo.png b/crates/cargo-gamma-engine/logo.png new file mode 100644 index 000000000..20aaf26cf --- /dev/null +++ b/crates/cargo-gamma-engine/logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc17b32e4b6b8c4f0c1d443b680a25b68e0cd8913954dfd047a7659a6bd0baff +size 131471 diff --git a/crates/cargo-gamma-engine/src/cfg.rs b/crates/cargo-gamma-engine/src/cfg.rs new file mode 100644 index 000000000..e59b777fb --- /dev/null +++ b/crates/cargo-gamma-engine/src/cfg.rs @@ -0,0 +1,1048 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The configuration predicates that hold for one build, and what they say about an attribute. + +use syn::punctuated::Punctuated; +use syn::{Attribute, Expr, Lit, Meta, Token}; + +use crate::HashSet; + +/// The configuration predicates that hold for a build. +/// +/// Construct one with [`CfgSet::parse`] from captured `rustc` output or +/// [`CfgSet::unconditional`] for a context where nothing should be stripped. +#[derive(Clone, Debug, Default)] +pub struct CfgSet { + /// Bare names, such as `unix` or a `--cfg loom` the build passes. + names: HashSet, + + /// Key/value pairs, such as `target_arch="x86_64"` or `feature="std"`. + pairs: HashSet<(String, String)>, + + /// Names that are in force under a condition this cannot evaluate. + /// + /// Answering one of these either way would settle a predicate on a guess, so they are + /// unanswerable and the code they gate stays mutable. + undecided: HashSet, + + /// Whether predicates are checked at all. + /// + /// An unresolved set holds everything, so a caller with no cfg information behaves exactly as + /// this tool did before cfg evaluation existed. + enforced: bool, +} + +impl CfgSet { + /// Returns a set under which every predicate holds, so nothing is ever stripped. + #[must_use] + pub fn unconditional() -> Self { + Self::default() + } + + /// Reads a set out of the lines `rustc --print cfg` prints. + /// + /// Each line is either a bare name or `key="value"`. Anything else is skipped rather than + /// guessed at. + /// + /// ```rust + /// # use cargo_gamma_engine::cfg::CfgSet; + /// let set = CfgSet::parse("unix\ntarget_os=\"linux\"\n"); + /// + /// assert!(set.holds_str("unix")); + /// assert!(set.holds_str("target_os = \"linux\"")); + /// ``` + #[must_use] + pub fn parse(printed: &str) -> Self { + let mut set = Self { + enforced: true, + ..Self::default() + }; + + for line in printed.lines() { + let line = line.trim(); + + if line.is_empty() { + continue; + } + + if let Some((key, value)) = line.split_once('=') { + let key = key.trim(); + let quoted = value.trim(); + + // The pair is dropped when the line names no value at all — `nonsense=` is not + // something `rustc` prints and guessing what it meant would invent a predicate. + // A value that is *written* and empty is another matter: `target_abi=""` on most + // targets and `target_env=""` on Apple and wasm are real answers, and + // `#[cfg(target_env = "")]` is how source asks for exactly them. Refusing that + // pair would not leave the predicate unanswerable, since the key is not marked + // undecided either, so the lookup would answer `No` to something the compiler + // says is true and take the code it gates out of the population. + if !key.is_empty() && !quoted.is_empty() { + let _added = set.pairs.insert((key.to_owned(), quoted.trim_matches('"').to_owned())); + } + } else { + let _added = set.names.insert(line.to_owned()); + } + } + + set + } + + /// Adds the Cargo features enabled for the package this set will be used on. + /// + /// Features are per package, so one set per package is built from one shared `rustc` answer. + #[must_use] + pub fn with_features(mut self, features: impl IntoIterator) -> Self { + for feature in features { + let _added = self.pairs.insert(("feature".to_owned(), feature)); + } + + self + } + + /// Marks names whose truth this set cannot decide. + /// + /// A name here answers neither yes nor no, so a predicate that turns on it is unanswerable and + /// the code it gates stays mutable. That is the direction every uncertainty in this module + /// resolves in: a mutant that should not exist is visible, while one silently missing from the + /// population is a hole in the measurement nobody can see. + #[must_use] + pub fn with_undecided(mut self, names: impl IntoIterator) -> Self { + for name in names { + let _added = self.undecided.insert(name); + } + + self + } + + /// Adds the bare `test` predicate, for a set describing a `cargo test` build. + /// + /// The instrumented build is `cargo test --no-run`, so `rustc` compiles each library's + /// unit-test target with `--cfg test` — but `rustc --print cfg`, which is asked about the + /// target rather than about any particular crate, never mentions it. Without this, an item + /// gated `#[cfg(any(feature = "x", test))]` with the feature off looks like code the compiler + /// never sees, and is silently dropped from the population although the unit tests exercise it. + /// + /// ```rust + /// # use cargo_gamma_engine::cfg::CfgSet; + /// let set = CfgSet::parse("unix\n").with_test(); + /// + /// // Compiled into the unit-test target, so it is real code that real tests run. + /// assert!(set.holds_str("any(feature = \"absent\", test)")); + /// + /// // And `#[cfg(not(test))]` is ordinary production code, which stays mutable. + /// assert!(set.holds_str("not(test)")); + /// ``` + #[must_use] + pub fn with_test(mut self) -> Self { + let _added = self.names.insert("test".to_owned()); + + self + } + + /// Returns whether every effective `#[cfg(...)]` among `attrs` holds. + /// + /// Active `cfg_attr` attributes are expanded before their `cfg` children are consulted. An + /// undecidable `cfg_attr` condition is left unapplied: stripping code because an attribute + /// might have appeared would be the same unsupported guess this module avoids elsewhere. + #[must_use] + pub fn holds_for(&self, attrs: &[Attribute]) -> bool { + // #[gamma::skip(cond.always_false, reason = "`decide` also returns `Unknown` whenever enforcement is off, so this is only an early return and removing it cannot change an answer")] + if !self.enforced { + return true; + } + + self.effective(attrs).iter().all(|attribute| { + // An attribute this module cannot parse says nothing about whether the code is built, + // so the code stays mutable. + cfg_predicate(attribute).is_none_or(|predicate| !is_test_only(&predicate) && self.holds(&predicate)) + }) + } + + /// Returns whether effective attributes confine an item to test code. + /// + /// `cfg_attr` can add either `cfg(test)` or a test attribute itself. Both have to be read + /// after its condition is evaluated, or the collector can mutate a test helper that rustc + /// treats as a test, while a false condition can hide ordinary production code on a guess. + #[must_use] + pub fn test_gated(&self, attrs: &[Attribute]) -> bool { + self.effective(attrs) + .iter() + .any(|attribute| cfg_predicate(attribute).is_some_and(|predicate| is_test_only(&predicate)) || is_test_attribute(attribute)) + || attrs.iter().any(|attribute| { + cfg_attr(&attribute.meta).is_some_and(|(condition, nested)| { + self.decide(&condition, true) == Verdict::Yes + && nested.iter().any(|attribute| { + cfg_predicate(attribute).is_some_and(|predicate| is_test_only(&predicate)) || is_test_attribute(attribute) + }) + }) + }) + } + + /// Returns whether a predicate written as source text holds. + /// + /// An unparsable predicate holds, for the same reason an unparsable attribute does. + /// + /// The tool itself always has real attributes to hand and so calls [`Self::holds_for`]; this is + /// the spelling the doctests and the unit tests use, because writing a predicate as text is the + /// only way to state one readably. + #[must_use] + pub fn holds_str(&self, predicate: &str) -> bool { + if crate::parse::exceeds_nesting_limit(predicate) { + return true; + } + + syn::parse_str::(predicate).map_or(true, |meta| self.holds(&meta)) + } + + /// Decides a predicate written as source text, without collapsing an unknown answer. + /// + /// [`Self::holds_str`] answers the question source code asks — "does this item survive" — and + /// so reads an unknown as "keep it". A caller deciding whether a *configuration table* is in + /// force needs the three answers apart: applying a table on an unknown would put names in + /// force that the build may not set, which strips the code their negation gates. + #[must_use] + pub fn decide_str(&self, predicate: &str) -> Verdict { + if crate::parse::exceeds_nesting_limit(predicate) { + return Verdict::Unknown; + } + + syn::parse_str::(predicate).map_or(Verdict::Unknown, |meta| self.verdict(&meta)) + } + + /// Decides a parsed predicate without collapsing an unknown answer. + #[must_use] + pub fn decide_meta(&self, predicate: &Meta) -> Verdict { + self.verdict(predicate) + } + + /// Returns whether one parsed predicate holds, treating an unknown answer as holding. + fn holds(&self, meta: &Meta) -> bool { + !matches!(self.verdict(meta), Verdict::No) + } + + /// Expands every `cfg_attr` whose condition is definitely true. + /// + /// The worklist rather than recursion matters for nested `cfg_attr`s: source is input, and + /// the expansion must not add another unbounded walk beside the parser's nesting guard. + fn effective(&self, attrs: &[Attribute]) -> Vec { + let mut effective: Vec = attrs.iter().map(|attribute| attribute.meta.clone()).collect(); + let mut at = 0; + + while at < effective.len() { + let Some((condition, nested)) = cfg_attr(&effective[at]) else { + at += 1; + continue; + }; + + let _removed = effective.remove(at); + + if self.verdict(&condition) == Verdict::Yes { + for attribute in nested.into_iter().rev() { + effective.insert(at, attribute); + } + } + } + + effective + } + + /// Returns whether this set describes a build that compiles some target with `--cfg test`. + fn is_test_build(&self) -> bool { + self.names.contains("test") + } + + /// Decides a predicate under both halves of a `cargo test` build. + /// + /// `cargo test --no-run` compiles a library twice: once as its own unit-test target, where + /// `--cfg test` is set, and once as the plain library that the integration tests and binaries + /// link, where it is not. A predicate mentioning `test` therefore has two answers at once, and + /// code that *either* of them keeps is code the build really contains. + /// + /// So the two halves are evaluated separately and disagreement comes out as `Unknown`, which + /// keeps the mutant. That is what makes `any(feature = "x", test)` mutable when the feature is + /// off, and what stops `not(test)` — ordinary production code, compiled into every target + /// except the unit-test one — from being deleted from the population. + fn verdict(&self, meta: &Meta) -> Verdict { + // #[gamma::skip(cond.always_false, reason = "when this is not a test build, `decide` ignores its `test` argument, so evaluating both halves produces the same verdict as this fast path")] + if !self.is_test_build() { + // #[gamma::skip(literal.bool_flip, reason = "outside a test build `decide` never consults its `test` argument, so true and false produce the identical verdict")] + return self.decide(meta, false); + } + + let unit_test = self.decide(meta, true); + let library = self.decide(meta, false); + + if unit_test == library { unit_test } else { Verdict::Unknown } + } + + /// Decides one parsed predicate, which may be unanswerable. + /// + /// The three-valued answer is not pedantry. `not(version("1.80"))` has to come out *unknown* + /// rather than false, because negating a predicate this module cannot evaluate would remove + /// code from the population on the strength of a guess. + /// + /// `test` says which half of a `cargo test` build is being decided; see [`Self::verdict`]. + fn decide(&self, meta: &Meta, test: bool) -> Verdict { + if !self.enforced { + return Verdict::Unknown; + } + + match meta { + // A bare name: `unix`, `loom`, `test`. `rustc` lists every name that is on, so a name + // that is absent is genuinely off rather than merely unrecognised — unless the build + // description marked it unanswerable. `test` is the one name whose answer depends on + // which target of the build is being compiled. + Meta::Path(path) => path.get_ident().map_or(Verdict::Unknown, |name| { + let name = name.to_string(); + + if name == "test" && self.is_test_build() { + return Verdict::from(test); + } + + if self.undecided.contains(&name) { + return Verdict::Unknown; + } + + Verdict::from(self.names.contains(&name)) + }), + + // `key = "value"`: `target_arch = "x86_64"`, `feature = "std"`. + Meta::NameValue(pair) => { + let (Some(key), Expr::Lit(literal)) = (pair.path.get_ident(), &pair.value) else { + return Verdict::Unknown; + }; + + let Lit::Str(value) = &literal.lit else { + return Verdict::Unknown; + }; + + let key = key.to_string(); + + if self.undecided.contains(&key) { + return Verdict::Unknown; + } + + Verdict::from(self.pairs.contains(&(key, value.value()))) + } + + Meta::List(list) => self.decide_list(list, test), + } + } + + /// Decides an `all(..)`, `any(..)` or `not(..)` predicate. + fn decide_list(&self, list: &syn::MetaList, test: bool) -> Verdict { + let Some(name) = list.path.get_ident().map(ToString::to_string) else { + return Verdict::Unknown; + }; + + // `version(..)`, and whatever the language adds next, is not modelled here, and an + // unmodelled predicate must not remove code from the population. + if !matches!(name.as_str(), "all" | "any" | "not") { + return Verdict::Unknown; + } + + let parser = Punctuated::::parse_terminated; + + let Ok(inner) = list.parse_args_with(parser) else { + return Verdict::Unknown; + }; + + let mut answers = inner.iter().map(|meta| self.decide(meta, test)); + + match name.as_str() { + // One `No` settles an `all`, and one `Yes` settles an `any`, however unknown the rest + // is. Otherwise an unknown among them leaves the whole thing unknown. + "all" => combine(&mut answers, Verdict::No, Verdict::Yes), + "any" => combine(&mut answers, Verdict::Yes, Verdict::No), + + // `not` takes exactly one predicate. Anything else is malformed, and a malformed + // predicate is as unanswerable as an unmodelled one. + // #[gamma::skip(iter.first_to_last, reason = "this branch accepts exactly one operand, for which `first` and `last` are the same; with any other length the guard rejects it")] + _ => match inner.first() { + Some(only) if inner.len() == 1 => self.decide(only, test).negated(), + _ => Verdict::Unknown, + }, + } + } +} + +/// Returns the predicate inside one `cfg` attribute. +/// +/// A malformed attribute is deliberately not interpreted: source rustc cannot understand is not +/// evidence that code should be omitted from the population. +fn cfg_predicate(attribute: &Meta) -> Option { + let Meta::List(list) = attribute else { + return None; + }; + + if !attribute.path().is_ident("cfg") { + return None; + } + + list.parse_args::().ok() +} + +/// Returns a `cfg_attr` condition and the attributes it conditionally applies. +/// +/// `cfg_attr` requires at least a predicate and one attribute. A malformed spelling is ignored +/// just like an unparsable `cfg`: it must not shrink the population on a guess. +fn cfg_attr(attribute: &Meta) -> Option<(Meta, Vec)> { + let Meta::List(list) = attribute else { + return None; + }; + + if !attribute.path().is_ident("cfg_attr") { + return None; + } + + let parser = Punctuated::::parse_terminated; + let arguments = list.parse_args_with(parser).ok()?; + let condition = arguments.first()?.clone(); + + (arguments.len() > 1).then(|| (condition, arguments.into_iter().skip(1).collect())) +} + +/// Returns whether an effective attribute is a test function attribute. +/// +/// The collector has always excluded `#[test]` and framework-qualified variants such as +/// `#[tokio::test]`; `cfg_attr` makes the same rule conditional rather than different. +fn is_test_attribute(attribute: &Meta) -> bool { + attribute.path().segments.last().is_some_and(|segment| segment.ident == "test") +} + +/// Returns whether direct configuration gates confine an item to a unit-test build. +/// +/// This compatibility helper intentionally has no selected configuration, so it cannot apply +/// `cfg_attr`. Discovery calls [`test_gated_for`] with its target-specific [`CfgSet`]; the +/// collector calls [`CfgSet::test_gated`] for item attributes. +#[must_use] +pub fn test_gated(attrs: &[Attribute]) -> bool { + attrs + .iter() + .any(|attribute| cfg_predicate(&attribute.meta).is_some_and(|predicate| is_test_only(&predicate))) +} + +/// Returns whether effective configuration gates confine an item to a unit-test build under `cfg`. +/// +/// Module discovery needs this narrower answer: `#[test]` belongs to a function, while only +/// `cfg(test)` controls whether a separately declared module file is test scaffolding. +#[must_use] +pub fn test_gated_for(cfg: &CfgSet, attrs: &[Attribute]) -> bool { + cfg.effective(attrs) + .iter() + .any(|attribute| cfg_predicate(attribute).is_some_and(|predicate| is_test_only(&predicate))) +} + +/// Returns whether a predicate can only hold in a unit-test target. +/// +/// `#[cfg(test)]` marks code that exists only to test other code, and mutating it measures the +/// tests' tests, which nobody has. The same is true of `all(test, feature = "x")`: a conjunction +/// requires every operand, so the item is unreachable outside the unit-test target however the +/// rest of the predicate turns out. +/// +/// Only `all` is descended into. Under `any`, `test` is an alternative rather than a requirement — +/// `any(feature = "x", test)` also describes the code the feature builds — and under `not` it says +/// the opposite of test-only. Either would take live production code out of the population. +fn is_test_only(meta: &Meta) -> bool { + match meta { + Meta::Path(path) => path.is_ident("test"), + + Meta::List(list) if list.path.is_ident("all") => { + let parser = Punctuated::::parse_terminated; + + list.parse_args_with(parser).is_ok_and(|inner| inner.iter().any(is_test_only)) + } + + _ => false, + } +} + +/// What this module can say about a predicate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Verdict { + /// The predicate holds, so the code is in the build. + Yes, + + /// The predicate does not hold, so the compiler strips the code. + No, + + /// This module cannot tell, so the code is left mutable. + Unknown, +} + +impl From for Verdict { + fn from(held: bool) -> Self { + if held { Self::Yes } else { Self::No } + } +} + +impl Verdict { + /// Returns the answer to the negation of whatever produced this one. + const fn negated(self) -> Self { + match self { + Self::Yes => Self::No, + Self::No => Self::Yes, + Self::Unknown => Self::Unknown, + } + } +} + +/// Folds the answers of a combinator's operands. +/// +/// `settles` is the answer that decides the whole combinator on its own — `No` for `all`, `Yes` +/// for `any` — and `otherwise` is what an operand-free or entirely undecisive list comes to. +fn combine(answers: &mut dyn Iterator, settles: Verdict, otherwise: Verdict) -> Verdict { + let mut unknown = false; + + for answer in answers { + if answer == settles { + return settles; + } + + unknown |= answer == Verdict::Unknown; + } + + if unknown { Verdict::Unknown } else { otherwise } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set() -> CfgSet { + CfgSet::parse("unix\ntarget_arch=\"x86_64\"\ntarget_os=\"linux\"\npanic=\"unwind\"\n") + .with_features(["std".to_owned(), "serde".to_owned()]) + } + + fn attribute(text: &str) -> Vec { + let item: syn::ItemFn = syn::parse_str(&format!("{text} fn f() {{}}")).expect("the fixture parses"); + + item.attrs + } + + #[test] + fn a_bare_name_is_looked_up() { + assert!(set().holds_str("unix")); + assert!(!set().holds_str("windows")); + assert!(!set().holds_str("loom"), "a custom cfg nobody passed is off"); + } + + #[test] + fn a_key_value_pair_is_looked_up() { + assert!(set().holds_str("target_arch = \"x86_64\"")); + assert!(!set().holds_str("target_arch = \"aarch64\"")); + assert!(set().holds_str("panic = \"unwind\"")); + } + + /// `""` is a value rustc really prints — `target_abi` on most targets, `target_env` on Apple + /// and on wasm — and a predicate testing for it is the idiomatic way to say "no particular + /// one". Dropping the pair would not leave the predicate unanswerable, because the key is not + /// undecided either: the lookup would answer `No` for something rustc says is true, and the + /// code it gates would leave the population. + #[test] + fn an_empty_value_is_a_value_like_any_other() { + let set = CfgSet::parse("target_env=\"\"\ntarget_abi=\"\"\ntarget_os=\"macos\"\n"); + + assert!(set.holds_str("target_env = \"\"")); + assert!(set.holds_str("target_abi = \"\"")); + assert!(!set.holds_str("target_env = \"gnu\"")); + assert!(set.holds_str("not(target_env = \"gnu\")")); + } + + /// A name marked undecided answers neither yes nor no: `holds_str` keeps the code (an unknown + /// holds), while `decide_str` reports the unknown rather than collapsing it. + #[test] + fn an_undecided_name_is_unknown_rather_than_looked_up() { + let undecided = set().with_undecided(["loom".to_owned()]); + + assert!(undecided.holds_str("loom"), "an unknown predicate must not strip the code it gates"); + assert_eq!(undecided.decide_str("loom"), Verdict::Unknown); + + // A name not marked undecided is looked up exactly as before. + assert!(!undecided.holds_str("windows")); + } + + /// The same three-valued treatment applies to a key/value pair whose key is undecided. + #[test] + fn an_undecided_key_value_pair_is_unknown_rather_than_looked_up() { + let undecided = set().with_undecided(["target_arch".to_owned()]); + + assert!(undecided.holds_str("target_arch = \"x86_64\"")); + assert_eq!(undecided.decide_str("target_arch = \"x86_64\""), Verdict::Unknown); + + // An unrelated pair is decided normally. + assert!(undecided.holds_str("panic = \"unwind\"")); + assert_eq!(undecided.decide_str("panic = \"unwind\""), Verdict::Yes); + } + + #[test] + fn a_feature_is_a_pair_like_any_other() { + assert!(set().holds_str("feature = \"std\"")); + assert!(!set().holds_str("feature = \"stats\"")); + } + + #[test] + fn the_combinators_compose() { + assert!(set().holds_str("all(unix, target_arch = \"x86_64\")")); + assert!(!set().holds_str("all(unix, windows)")); + assert!(set().holds_str("any(windows, unix)")); + assert!(!set().holds_str("any(windows, feature = \"stats\")")); + assert!(set().holds_str("not(windows)")); + assert!(!set().holds_str("not(unix)")); + assert!(set().holds_str("not(all(unix, windows))")); + assert!(!set().holds_str("any()"), "an empty `any` holds for nothing"); + assert!(set().holds_str("all()"), "an empty `all` holds vacuously"); + } + + #[test] + fn an_unmodelled_predicate_holds() { + // Removing code because this module has not heard of a predicate would silently shrink the + // population, which is the one failure mode nobody can see in a report. + assert!(set().holds_str("version(\"1.80\")")); + assert!(set().holds_str("version(unix)")); + assert!(set().holds_str("not(version(\"1.80\"))")); + assert!(set().holds_str("this is not valid syntax at all")); + } + + #[test] + fn an_unknown_operand_leaves_a_combinator_unknown() { + // `not` of something unanswerable is unanswerable, not false. Getting this wrong would + // strip code on the strength of a predicate this module has never heard of. + assert!(set().holds_str("not(version(\"1.80\"))")); + assert!(set().holds_str("all(unix, version(\"1.80\"))")); + assert!(set().holds_str("any(windows, version(\"1.80\"))")); + + // A decisive operand still settles it, however unknown its neighbours are. + assert!(!set().holds_str("all(windows, version(\"1.80\"))")); + assert!(set().holds_str("any(unix, version(\"1.80\"))")); + assert!(!set().holds_str("not(any(unix, version(\"1.80\")))")); + } + + #[test] + fn a_malformed_not_holds() { + assert!(set().holds_str("not(unix, windows)"), "a `not` of two things is not a `not`"); + assert!(set().holds_str("not()")); + } + + /// `all(..)`/`any(..)`/`not(..)` take a comma-separated list of predicates, and a token stream + /// that does not parse that way — two predicates with no comma between them — is malformed in + /// a way the compiler itself would also reject; treating it as false would strip code on the + /// strength of a predicate this module could not actually read. + #[test] + fn a_combinator_whose_arguments_do_not_parse_as_a_meta_list_holds() { + assert!(set().holds_str("all(unix windows)")); + } + + /// `key = value` with anything but a string literal on the right — a number, a boolean, a + /// path — is not a shape `rustc --print cfg` ever produces, so guessing at it would risk + /// stripping code on a predicate this module cannot actually evaluate. + #[test] + fn a_name_value_predicate_whose_literal_is_not_a_string_holds() { + assert!(set().holds_str("target_pointer_width = 64")); + } + + /// A combinator whose head is a path rather than a bare identifier — `a::b(unix)` — is not one + /// of `all`, `any` or `not` however it is spelled, and treating it as false would remove code + /// this module simply does not recognise. + #[test] + fn a_combinator_whose_head_is_not_a_bare_identifier_holds() { + assert!(set().holds_str("a::b(unix)")); + } + + #[test] + fn an_unresolved_set_holds_everything() { + let set = CfgSet::unconditional(); + + assert!(set.holds_str("windows")); + assert!(set.holds_str("feature = \"nothing-like-this\"")); + assert!(set.holds_str("not(unix)")); + assert!(set.holds_for(&attribute("#[cfg(windows)]"))); + } + + /// `decide_str` is the three-valued counterpart of `holds_str`: a decisive predicate reports + /// its actual verdict rather than collapsing straight to a boolean. + #[test] + fn decide_str_reports_the_undecided_verdict() { + assert_eq!(set().decide_str("unix"), Verdict::Yes); + assert_eq!(set().decide_str("windows"), Verdict::No); + assert_eq!(set().decide_str("version(\"1.80\")"), Verdict::Unknown); + } + + /// A predicate written deeply enough to overflow a recursive-descent parser is refused by the + /// same nesting guard the source parser uses, before `syn` ever sees it — for both the + /// boolean-collapsing and three-valued entry points. + #[test] + fn a_predicate_nested_deeper_than_the_limit_is_refused_rather_than_parsed() { + let depth = 4_096; + let predicate = format!("{}unix{}", "not(".repeat(depth), ")".repeat(depth)); + + assert!( + set().holds_str(&predicate), + "an unparsed predicate must not strip the code it gates" + ); + assert_eq!(set().decide_str(&predicate), Verdict::Unknown); + } + + #[test] + fn printed_lines_that_are_not_settings_are_skipped() { + let set = CfgSet::parse("unix\n\n \ntarget_os=\"linux\"\nnonsense=\n=orphan\n"); + + assert!(set.holds_str("unix")); + assert!(set.holds_str("target_os = \"linux\"")); + assert!(!set.holds_str("nonsense = \"\"")); + assert!(!set.holds_str("orphan")); + assert!(!set.names.contains("")); + } + + #[test] + fn only_cfg_attributes_are_consulted() { + assert!(set().holds_for(&attribute("#[inline]"))); + assert!(set().holds_for(&attribute("#[doc = \"windows\"]"))); + assert!(set().holds_for(&attribute("#[allow(windows)]"))); + + // An inactive `cfg_attr` adds nothing. + assert!(set().holds_for(&attribute("#[cfg_attr(windows, inline)]"))); + } + + #[test] + fn an_active_cfg_attr_applies_its_cfg_attribute() { + let active = attribute("#[cfg_attr(unix, cfg(windows))]"); + let inactive = attribute("#[cfg_attr(windows, cfg(windows))]"); + let nested = attribute("#[cfg_attr(unix, cfg_attr(unix, cfg(windows)))]"); + + assert!(!set().holds_for(&active), "the active inner cfg removes the item"); + assert!(set().holds_for(&inactive), "an inactive cfg_attr contributes no cfg"); + assert!( + !set().holds_for(&nested), + "active cfg_attrs expand until the effective attribute is reached" + ); + } + + #[test] + fn every_cfg_attribute_has_to_hold() { + assert!(set().holds_for(&attribute("#[cfg(unix)]"))); + assert!(!set().holds_for(&attribute("#[cfg(windows)]"))); + assert!(set().holds_for(&attribute("#[cfg(unix)]\n#[cfg(target_os = \"linux\")]"))); + assert!(!set().holds_for(&attribute("#[cfg(unix)]\n#[cfg(windows)]"))); + } + + #[test] + fn an_unparsable_cfg_attribute_holds() { + // A bare literal is not a `Meta`, so this is the shape that fails to parse. + assert!(set().holds_for(&attribute("#[cfg(\"windows\")]"))); + } + + #[test] + fn a_qualified_predicate_holds() { + // A path with more than one segment is not something `cfg` accepts, so nothing is known + // about it and the code stays mutable. + assert!(set().holds_str("some::thing")); + assert!(set().holds_str("some::thing(unix)")); + assert!(set().holds_str("some::thing = \"x\"")); + } + + /// The set the collector uses describes `cargo test --no-run`, which compiles a library both + /// with and without `--cfg test`. + fn test_build() -> CfgSet { + set().with_test() + } + + /// `any(feature = "x", test)` with the feature off is compiled into the unit-test target, so + /// the unit tests run it and a mutant in it is killable. Leaving it out of the population was + /// issue #22. + #[test] + fn an_item_a_disjunction_admits_into_the_test_target_is_mutable() { + assert!(test_build().holds_for(&attribute("#[cfg(any(feature = \"absent\", test))]"))); + assert!(test_build().holds_for(&attribute("#[cfg(any(windows, test))]"))); + + // Nothing about `test` rescues a disjunction that is false in both halves of the build. + assert!(!test_build().holds_for(&attribute("#[cfg(any(windows, feature = \"absent\"))]"))); + } + + /// A conjunction with `test` in it cannot hold outside the unit-test target, so the item is + /// test code — the tests' own scaffolding — whatever the rest of the predicate says. + #[test] + fn an_item_a_conjunction_confines_to_the_test_target_is_not_mutable() { + assert!(!test_build().holds_for(&attribute("#[cfg(all(test, feature = \"std\"))]"))); + assert!(!test_build().holds_for(&attribute("#[cfg(all(unix, all(test, unix)))]"))); + + // A conjunction that never mentions `test` is decided exactly as before. + assert!(test_build().holds_for(&attribute("#[cfg(all(unix, feature = \"std\"))]"))); + } + + /// `#[cfg(not(test))]` is ordinary production code: every target of the build except the + /// unit-test one compiles it, and the integration tests link and can kill mutants in it. + #[test] + fn an_item_kept_out_of_the_test_target_stays_mutable() { + assert!(test_build().holds_for(&attribute("#[cfg(not(test))]"))); + assert!(test_build().holds_for(&attribute("#[cfg(all(not(test), unix))]"))); + + // `not` says the opposite of test-only, so it must not be read as a test gate. + assert!(!is_test_only(&syn::parse_str::("not(test)").expect("the fixture parses"))); + } + + /// Plain `#[cfg(test)]` is the unit-test module itself. It is in the build, so the predicate + /// is not false, but it is test code and nothing in it is mutated. + #[test] + fn a_bare_test_gate_is_in_the_build_but_not_mutated() { + assert!(test_build().holds_str("test"), "the unit-test target compiles it"); + assert!(!test_build().holds_for(&attribute("#[cfg(test)]"))); + + // A set that knows nothing of `--cfg test` still answers as it always did. + assert!(!set().holds_str("test")); + } + + #[test] + fn an_active_cfg_attr_can_make_an_item_test_only() { + let cfg = test_build(); + let gated = attribute("#[cfg_attr(unix, cfg(test))]"); + let test_attribute = attribute("#[cfg_attr(unix, test)]"); + let inactive = attribute("#[cfg_attr(windows, cfg(test))]"); + + assert!(!cfg.holds_for(&gated), "the effective cfg(test) excludes test scaffolding"); + assert!(cfg.test_gated(&gated)); + assert!(cfg.test_gated(&test_attribute), "an effective #[test] is test code too"); + assert!(!cfg.test_gated(&inactive), "an inactive cfg_attr adds no test gate"); + } + + /// Module discovery and item collection use the same recursive test-gate rule. Each once read + /// only the top-level path of the attribute, saw `all` rather than the `test` inside it, and + /// mutated the tests' own helpers. + #[test] + fn the_shared_classifier_reads_a_compound_gate_recursively() { + assert!(test_gated(&attribute("#[cfg(test)]"))); + assert!(test_gated(&attribute("#[cfg(all(test, unix))]"))); + assert!(test_gated(&attribute("#[cfg(all(unix, all(test, feature = \"x\")))]"))); + + // `test` as one alternative among several describes code the other alternative compiles, + // so it is production code and stays in the population. + assert!(!test_gated(&attribute("#[cfg(any(test, feature = \"runtime\"))]"))); + assert!(!test_gated(&attribute("#[cfg(not(test))]"))); + assert!(!test_gated(&attribute("#[cfg(unix)]"))); + assert!(!test_gated(&attribute("#[test]")), "only a cfg gate says what is compiled"); + + // An attribute that does not parse says nothing, so the item stays mutable. + assert!(!test_gated(&attribute("#[cfg(\"test\")]"))); + } + + #[test] + fn any_test_gate_marks_an_item_among_other_attributes() { + assert!(test_gated(&attribute("#[cfg(unix)]\n#[cfg(test)]"))); + assert!(!test_gated(&attribute("#[cfg(unix)]\n#[cfg(windows)]"))); + } + + /// `test_gated_for` is the narrower, `cfg`-only sibling of `CfgSet::test_gated` that module + /// discovery uses: it reads through active `cfg_attr`s using the given set, exactly as + /// `CfgSet::holds_for` does, but does not treat a bare `#[test]` attribute as a gate — only + /// `cfg(test)`, direct or expanded, marks a module file as test scaffolding. + #[test] + fn test_gated_for_reads_through_an_active_cfg_attr() { + let cfg = test_build(); + + assert!(test_gated_for(&cfg, &attribute("#[cfg(test)]"))); + assert!(!test_gated_for(&cfg, &attribute("#[cfg(unix)]"))); + + // The condition of the `cfg_attr` is decided against `cfg` before the nested `cfg(test)` + // is inspected, so only a `cfg_attr` whose own condition holds contributes its gate. + assert!(test_gated_for(&cfg, &attribute("#[cfg_attr(unix, cfg(test))]"))); + assert!(!test_gated_for(&cfg, &attribute("#[cfg_attr(windows, cfg(test))]"))); + + // A bare `#[test]` is not a `cfg` at all, so this narrower classifier leaves it alone — + // unlike `CfgSet::test_gated`, which module discovery does not call. + assert!(!test_gated_for(&cfg, &attribute("#[test]"))); + } + + /// The alphabet the law tests below build their predicate trees out of. + /// + /// One leaf of each kind the evaluator distinguishes, and — the point of the exercise — one + /// that it cannot answer. A law checked only over answerable leaves would be satisfied by a + /// two-valued evaluator, which is exactly the evaluator this module must not become. + const LEAVES: [&str; 6] = [ + "unix", // a name that is on + "windows", // a name that is off + "target_arch = \"x86_64\"", // a pair that holds + "target_arch = \"aarch64\"", // a pair that does not + "version(\"1.80\")", // unmodelled, and so unanswerable + "all()", // vacuously true, and the identity of `all` + ]; + + /// Every predicate of the shape the laws are stated over, smallest first. + /// + /// Enumerated rather than sampled. The interesting domain here is tiny — six leaves, three + /// combinators, two levels — so it can be covered completely, and a complete answer is worth + /// more than a random one: a law that holds for every tree of this shape holds, full stop, + /// with no seed to reproduce and no shrinking to interpret. + fn trees() -> Vec { + let mut out: Vec = LEAVES.iter().map(|leaf| (*leaf).to_owned()).collect(); + + for a in LEAVES { + out.push(format!("not({a})")); + + for b in LEAVES { + out.push(format!("all({a}, {b})")); + out.push(format!("any({a}, {b})")); + } + } + + out + } + + /// Evaluates a predicate to its three-valued verdict, which `holds_str` flattens away. + /// + /// The laws are about all three answers. Asserting them through `holds_str` would collapse + /// `Yes` and `Unknown` together and let an evaluator that answered `Yes` to everything pass. + fn verdict_of(set: &CfgSet, predicate: &str) -> Verdict { + let meta = syn::parse_str::(predicate).expect("every generated predicate parses"); + + set.verdict(&meta) + } + + /// Negating twice returns the original answer, including when that answer is `Unknown`. + /// + /// The `Unknown` half is the one that matters: an evaluator that resolved unknowns to `No` + /// under negation would satisfy this law on the four answerable leaves and break it here, and + /// breaking it means code that might be in the build being deleted from the population. + #[test] + fn double_negation_is_the_identity_on_all_three_answers() { + let set = set(); + + for tree in trees() { + assert_eq!( + verdict_of(&set, &format!("not(not({tree}))")), + verdict_of(&set, &tree), + "not(not({tree}))" + ); + } + } + + /// Negating something unanswerable leaves it unanswerable. + #[test] + fn negation_never_turns_an_unknown_into_an_answer() { + let set = set(); + + for tree in trees() { + if verdict_of(&set, &tree) == Verdict::Unknown { + assert_eq!( + verdict_of(&set, &format!("not({tree})")), + Verdict::Unknown, + "not({tree}) must stay unknown" + ); + } + } + } + + /// `all` and `any` do not care what order their operands are written in. + #[test] + fn the_combinators_are_commutative() { + let set = set(); + + for a in LEAVES { + for b in LEAVES { + for name in ["all", "any"] { + assert_eq!( + verdict_of(&set, &format!("{name}({a}, {b})")), + verdict_of(&set, &format!("{name}({b}, {a})")), + "{name}({a}, {b})" + ); + } + } + } + } + + /// Nor how they are bracketed. + #[test] + fn the_combinators_are_associative() { + let set = set(); + + for a in LEAVES { + for b in LEAVES { + for c in LEAVES { + for name in ["all", "any"] { + assert_eq!( + verdict_of(&set, &format!("{name}({name}({a}, {b}), {c})")), + verdict_of(&set, &format!("{name}({a}, {name}({b}, {c}))")), + "{name} over ({a}, {b}, {c})" + ); + } + } + } + } + } + + /// De Morgan's laws hold in all three values, which is what pins `all` and `any` to each other. + /// + /// Without this, an arm added later could make `any` short-circuit on an unknown while `all` + /// did not, and each combinator's own tests would still pass. + #[test] + fn de_morgan_holds_including_where_the_answer_is_unknown() { + let set = set(); + + for a in LEAVES { + for b in LEAVES { + assert_eq!( + verdict_of(&set, &format!("not(all({a}, {b}))")), + verdict_of(&set, &format!("any(not({a}), not({b}))")), + "not(all({a}, {b}))" + ); + assert_eq!( + verdict_of(&set, &format!("not(any({a}, {b}))")), + verdict_of(&set, &format!("all(not({a}), not({b}))")), + "not(any({a}, {b}))" + ); + } + } + } + + /// Each combinator's identity leaves its operand alone, and its annihilator settles it. + /// + /// `all(x, all())` is `x` and `all(x, any())` is `No`, whatever `x` is — including unknown. + /// The annihilator half is where a three-valued evaluator earns its keep: one `No` settles an + /// `all` however unanswerable the rest of it is, and refusing to settle there would leave + /// dead code in the population instead of taking it out. + #[test] + fn the_identities_and_annihilators_of_the_combinators_hold() { + let set = set(); + + for tree in trees() { + assert_eq!(verdict_of(&set, &format!("all({tree}, all())")), verdict_of(&set, &tree)); + assert_eq!(verdict_of(&set, &format!("any({tree}, any())")), verdict_of(&set, &tree)); + assert_eq!(verdict_of(&set, &format!("all({tree}, any())")), Verdict::No, "all({tree}, any())"); + assert_eq!(verdict_of(&set, &format!("any({tree}, all())")), Verdict::Yes, "any({tree}, all())"); + } + } + + /// No predicate this module builds is ever answered by guessing. + /// + /// The safety argument of the whole module is that an unanswerable predicate keeps the code, + /// so the one thing no law may permit is an `Unknown` operand producing an answer that is not + /// forced by the other operands. `all` may answer `No` and `any` may answer `Yes` with an + /// unknown present, because one operand settles those outright; nothing else is allowed. + #[test] + fn an_unknown_operand_only_settles_a_combinator_the_other_operand_had_already_settled() { + let set = set(); + + for a in LEAVES { + for b in LEAVES { + if verdict_of(&set, a) != Verdict::Unknown && verdict_of(&set, b) != Verdict::Unknown { + continue; + } + + let (all, any) = ( + verdict_of(&set, &format!("all({a}, {b})")), + verdict_of(&set, &format!("any({a}, {b})")), + ); + + assert!( + all == Verdict::Unknown || (all == Verdict::No && [a, b].iter().any(|one| verdict_of(&set, one) == Verdict::No)), + "all({a}, {b}) answered {all:?} with an unknown operand and nothing to force it" + ); + assert!( + any == Verdict::Unknown || (any == Verdict::Yes && [a, b].iter().any(|one| verdict_of(&set, one) == Verdict::Yes)), + "any({a}, {b}) answered {any:?} with an unknown operand and nothing to force it" + ); + } + } + } +} diff --git a/crates/cargo-gamma-engine/src/error.rs b/crates/cargo-gamma-engine/src/error.rs new file mode 100644 index 000000000..0816cdbaa --- /dev/null +++ b/crates/cargo-gamma-engine/src/error.rs @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::error::Error as StdError; +use core::fmt::{self, Display, Formatter}; +use std::io; + +/// An engine error with the coordinator-facing classification preserved. +#[derive(Debug)] +pub struct Error { + message: String, + cause: Option>, + usage: bool, + skippable: bool, +} + +impl Error { + #[must_use] + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + cause: None, + usage: false, + skippable: false, + } + } + + #[must_use] + pub const fn usage(mut self) -> Self { + self.usage = true; + self + } + + #[must_use] + pub const fn is_usage(&self) -> bool { + self.usage + } + + #[must_use] + pub const fn skippable(mut self) -> Self { + self.skippable = true; + self + } + + #[must_use] + pub const fn is_skippable(&self) -> bool { + self.skippable + } + + #[must_use] + pub fn caused_by(mut self, cause: impl StdError + Send + Sync + 'static) -> Self { + self.cause = Some(Box::new(cause)); + self + } + + #[must_use] + pub fn into_parts(self) -> (String, Option>, bool, bool) { + (self.message, self.cause, self.usage, self.skippable) + } +} + +impl Display for Error { + #[expect(clippy::renamed_function_params, reason = "`f` is less clear than `formatter`")] + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.message)?; + + if let Some(cause) = &self.cause { + write!(formatter, ": {cause}")?; + } + + Ok(()) + } +} + +impl StdError for Error { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + self.cause.as_ref().map(|cause| &**cause as &(dyn StdError + 'static)) + } +} + +impl From for Error { + fn from(value: io::Error) -> Self { + Self::new("I/O error").caused_by(value) + } +} + +macro_rules! error { + ($($arg:tt)*) => { $crate::error::Error::new(format!($($arg)*)) }; +} + +pub(crate) use error; + +#[cfg(test)] +mod tests { + use super::*; + + /// A freshly built error carries the message given to it and starts out neither a usage error + /// nor skippable, with no cause attached. + #[test] + fn a_new_error_starts_plain() { + let error = Error::new("something went wrong"); + + assert!(!error.is_usage()); + assert!(!error.is_skippable()); + assert_eq!(error.to_string(), "something went wrong"); + assert!(error.source().is_none()); + } + + /// `usage` and `skippable` each flip their own flag and leave the other alone, in either + /// application order. + #[test] + fn usage_and_skippable_are_independent_flags() { + let usage_only = Error::new("bad flags").usage(); + + assert!(usage_only.is_usage()); + assert!(!usage_only.is_skippable()); + + let skippable_only = Error::new("missing file").skippable(); + + assert!(!skippable_only.is_usage()); + assert!(skippable_only.is_skippable()); + + let both = Error::new("both").usage().skippable(); + + assert!(both.is_usage()); + assert!(both.is_skippable()); + } + + /// A cause attached with `caused_by` is appended to the display text and surfaces as the + /// `source` the standard error trait exposes. + #[test] + fn a_cause_is_displayed_and_surfaced_as_the_source() { + let cause = io::Error::other("disk exploded"); + let error = Error::new("could not read file").caused_by(cause); + + assert_eq!(error.to_string(), "could not read file: disk exploded"); + assert_eq!(error.source().expect("a cause was attached").to_string(), "disk exploded"); + } + + /// `into_parts` hands back exactly the state built up on the error, in the documented order. + #[test] + fn into_parts_returns_the_built_up_state() { + let error = Error::new("partial write") + .usage() + .skippable() + .caused_by(io::Error::other("truncated")); + + let (message, cause, usage, skippable) = error.into_parts(); + + assert_eq!(message, "partial write"); + assert_eq!(cause.expect("a cause was attached").to_string(), "truncated"); + assert!(usage); + assert!(skippable); + } + + /// An I/O error converts into an engine error labeled generically, with the original error + /// preserved as the cause so its detail is not lost. + #[test] + fn an_io_error_converts_with_its_detail_preserved_as_the_cause() { + let io_error = io::Error::other("permission denied"); + let error = Error::from(io_error); + + assert_eq!(error.to_string(), "I/O error: permission denied"); + assert_eq!(error.source().expect("a cause was attached").to_string(), "permission denied"); + } + + /// The `error!` macro formats its arguments the way `format!` would, producing a plain error + /// with that text as its message. + #[test] + fn the_error_macro_formats_its_arguments() { + let built = error!("{} of {}", 2, 3); + + assert_eq!(built.to_string(), "2 of 3"); + assert!(!built.is_usage()); + assert!(!built.is_skippable()); + } +} diff --git a/crates/cargo-gamma-engine/src/lib.rs b/crates/cargo-gamma-engine/src/lib.rs new file mode 100644 index 000000000..c921bc901 --- /dev/null +++ b/crates/cargo-gamma-engine/src/lib.rs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![cfg(not(all(test, miri)))] +#![doc(hidden)] +#![forbid( + unsafe_code, + reason = "raw platform calls stay in `cargo-gamma-unsafe`; the source engine has no reason to use them" +)] +#![cfg_attr(coverage_nightly, feature(coverage_attribute))] +#![allow( + clippy::redundant_pub_crate, + reason = "this non-published crate exposes implementation types only so workspace crates can compose the engine" +)] + +//! This crate is an internal implementation detail of +//! [`cargo-gamma`](https://crates.io/crates/cargo-gamma). It contains the Rust source parsing, +//! mutation collection, stable identity, and schema instrumentation pipeline. +//! +//! Do not depend on it directly. Its API may change incompatibly without notice; it is published +//! only so that `cargo-gamma` can be installed through crates.io. + +use rustc_hash::{FxHashMap, FxHashSet}; + +pub(crate) type HashMap = FxHashMap; +pub(crate) type HashSet = FxHashSet; + +/// The engine's error result. +pub type Result = core::result::Result; + +pub mod cfg; +mod error; +pub mod model; +pub mod ops; +pub mod parse; +pub mod schema; + +pub use error::Error; diff --git a/crates/cargo-gamma-engine/src/model/identity.rs b/crates/cargo-gamma-engine/src/model/identity.rs new file mode 100644 index 000000000..74d3a06b8 --- /dev/null +++ b/crates/cargo-gamma-engine/src/model/identity.rs @@ -0,0 +1,268 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The stable, content-addressed identity of a mutant and its site. + +use blake3::Hasher; +use camino::Utf8Path; +use compact_str::CompactString; + +/// A mutant's compact, content-addressed identity. +pub type MutantId = CompactString; + +/// The identity normalization contract emitted by this version. +/// +/// Version 4 adds caller-supplied error replacement text to those mutants' identities. Every +/// identity whose replacement comes from the registry remains byte-identical. +pub const MUTANT_ID_VERSION: u32 = 4; + +/// Computes the stable, content-addressed identity of a mutant. +/// +/// Deliberately *not* keyed on line and column. Inserting a line at the top of a file would +/// renumber every mutant below it, which would reshuffle every shard, orphan every cached verdict +/// and silently detach every configured expectation. The enclosing item path provides the same +/// disambiguation while surviving both reformatting and code motion within a file, and the +/// occurrence index handles two textually identical sites in one function. Trait implementation +/// paths include both the self type and the implemented trait, so same-named methods from two +/// traits never depend on source order for that disambiguation. +#[must_use] +pub fn mutant_id( + file: &Utf8Path, + item_path: &str, + mutator: &str, + normalized_site_text: &str, + occurrence: u32, + replacement_index: u32, +) -> MutantId { + mutant_id_with_discriminator(file, item_path, mutator, normalized_site_text, occurrence, replacement_index, None) +} + +/// Computes an identity with additional caller-supplied replacement content. +#[must_use] +pub(crate) fn mutant_id_with_discriminator( + file: &Utf8Path, + item_path: &str, + mutator: &str, + normalized_site_text: &str, + occurrence: u32, + replacement_index: u32, + discriminator: Option<&str>, +) -> MutantId { + let mut hasher = Hasher::new(); + + // Length-prefix every field so that no two different field splits can hash alike. + for field in [file.as_str(), item_path, mutator, normalized_site_text] { + let _ = hasher.update(&(field.len() as u64).to_le_bytes()); + let _ = hasher.update(field.as_bytes()); + } + + let _ = hasher.update(&occurrence.to_le_bytes()); + let _ = hasher.update(&replacement_index.to_le_bytes()); + if let Some(discriminator) = discriminator { + let _ = hasher.update(&(discriminator.len() as u64).to_le_bytes()); + let _ = hasher.update(discriminator.as_bytes()); + } + + let digest = hasher.finalize(); + let bytes = digest.as_bytes(); + let mut out = MutantId::with_capacity(MUTANT_ID_HEX_LEN); + + for byte in bytes.iter().take(MUTANT_ID_BYTES) { + out.push(HEX[usize::from(byte >> 4)]); + out.push(HEX[usize::from(byte & 0x0f)]); + } + + out +} + +/// How much of the digest a mutant identifier keeps. +/// +/// Six bytes is forty-eight bits, which by the birthday bound gives roughly even odds of one +/// collision only once a run holds about `2^24` — sixteen million — distinct mutants. This +/// workspace generates on the order of ten thousand, so the margin is three orders of magnitude, +/// and at a hundred thousand mutants the chance of any collision is still under one in three +/// thousand. Widening this is cheap and safe; narrowing it is not, because the identifier is what +/// cached verdicts, shard assignments and configured expectations are keyed on, so a collision +/// silently attaches one mutant's history to another. +const MUTANT_ID_BYTES: usize = 6; + +/// The width of a rendered mutant identifier: two lowercase hex characters per kept digest byte. +pub const MUTANT_ID_HEX_LEN: usize = MUTANT_ID_BYTES * 2; + +/// The lowercase hex alphabet, indexed by nibble. +const HEX: [char; 16] = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; + +/// Identifies a mutation site for the purpose of counting repeats of it within one item. +/// +/// A digest rather than the text itself, because the counter has to be kept for every distinct +/// site in a file and holding the item path and the normalized source of each one costs two owned +/// strings per mutant that nothing ever reads back. At 128 bits a collision between two real sites +/// is not a thing that happens. +#[must_use] +pub fn site_key(item_path: &str, mutator: &str, normalized_site_text: &str) -> u128 { + let mut hasher = Hasher::new(); + + for field in [item_path, mutator, normalized_site_text] { + let _ = hasher.update(&(field.len() as u64).to_le_bytes()); + let _ = hasher.update(field.as_bytes()); + } + + let mut key = [0_u8; 16]; + + key.copy_from_slice(hasher.finalize().as_bytes().get(..16).unwrap_or(&[0; 16])); + u128::from_le_bytes(key) +} + +/// Normalizes the source text of a site for hashing. +/// +/// Whitespace runs collapse to a single space and comments disappear; everything else is +/// preserved verbatim, including identifiers, literal values, literal suffixes and integer bases. +/// Preserving too little would let a `cargo fmt` run reshuffle the whole population; preserving +/// too little meaning would let a genuine edit keep its old identity and silently reattach a stale +/// verdict to code whose behavior changed. When in doubt, this preserves. +#[must_use] +pub fn normalize_site_text(text: &str) -> CompactString { + let mut out = CompactString::with_capacity(text.len()); + let comments = crate::parse::comment_spans(text); + let mut comments = comments.iter().peekable(); + let mut offset = 0; + let mut pending_space = false; + + while offset < text.len() { + if let Some(comment) = comments.peek() + && comment.start == offset + { + offset = comment.end; + let _ = comments.next(); + pending_space = !out.is_empty(); + continue; + } + + if let Some(end) = crate::parse::literal_end(text, offset) { + if pending_space { + out.push(' '); + pending_space = false; + } + + out.push_str(text.get(offset..end).unwrap_or("")); + offset = end; + continue; + } + + let character = text + .get(offset..) + .and_then(|rest| rest.chars().next()) + .expect("the loop keeps the UTF-8 boundary below the text length"); + offset += character.len_utf8(); + + if character.is_whitespace() { + pending_space = !out.is_empty(); + continue; + } + + if pending_space { + out.push(' '); + pending_space = false; + } + + out.push(character); + } + + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The public `mutant_id` entry point is the discriminator-less case of + /// `mutant_id_with_discriminator`, and must produce exactly the same identity as calling that + /// function directly with `None`. + #[test] + fn mutant_id_delegates_to_the_discriminated_form_with_no_discriminator() { + let file = Utf8Path::new("src/lib.rs"); + + let direct = mutant_id(file, "subject::f", "arith.add_to_sub", "1 + 1", 0, 0); + let via_discriminator = mutant_id_with_discriminator(file, "subject::f", "arith.add_to_sub", "1 + 1", 0, 0, None); + + assert_eq!(direct, via_discriminator); + assert_eq!(direct.len(), MUTANT_ID_HEX_LEN); + assert!(direct.chars().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase())); + } + + /// Changing any one field of the identity — including whether a discriminator is present — + /// changes the identity, and an absent discriminator is not the same as an empty one. + #[test] + fn each_field_and_the_discriminator_affect_the_identity() { + let file = Utf8Path::new("src/lib.rs"); + let baseline = mutant_id_with_discriminator(file, "subject::f", "arith.add_to_sub", "1 + 1", 0, 0, None); + + let other_file = mutant_id_with_discriminator(Utf8Path::new("src/other.rs"), "subject::f", "arith.add_to_sub", "1 + 1", 0, 0, None); + let other_occurrence = mutant_id_with_discriminator(file, "subject::f", "arith.add_to_sub", "1 + 1", 1, 0, None); + let other_replacement_index = mutant_id_with_discriminator(file, "subject::f", "arith.add_to_sub", "1 + 1", 0, 1, None); + let with_discriminator = mutant_id_with_discriminator(file, "subject::f", "arith.add_to_sub", "1 + 1", 0, 0, Some("panic")); + let with_empty_discriminator = mutant_id_with_discriminator(file, "subject::f", "arith.add_to_sub", "1 + 1", 0, 0, Some("")); + let with_other_discriminator = + mutant_id_with_discriminator(file, "subject::f", "arith.add_to_sub", "1 + 1", 0, 0, Some("overflow")); + + for other in [other_file, other_occurrence, other_replacement_index, with_discriminator.clone()] { + assert_ne!(baseline, other, "a field change must not collide with the baseline identity"); + } + + assert_ne!(with_discriminator, with_other_discriminator); + assert_ne!(baseline, with_empty_discriminator); + } + + /// Whitespace runs collapse to one space, and a run entirely at the start of the text + /// disappears rather than producing a leading space. + #[test] + fn whitespace_runs_collapse_to_a_single_space() { + assert_eq!(normalize_site_text(" let x = 1 ; "), "let x = 1 ;"); + assert_eq!(normalize_site_text("\t\nfoo"), "foo"); + } + + /// A line comment disappears entirely, and the code around it is joined by exactly one space + /// whether the comment sits between two tokens or trails the last one. + #[test] + fn line_comments_are_dropped_and_replaced_by_a_single_space() { + assert_eq!(normalize_site_text("let x = 1; // set x\nlet y = 2;"), "let x = 1; let y = 2;"); + assert_eq!(normalize_site_text("// leading\nlet x = 1;"), "let x = 1;"); + assert_eq!(normalize_site_text("let x = 1; // trailing"), "let x = 1;"); + } + + /// A block comment disappears the same way a line comment does, including one that sits + /// directly before a string literal so the literal branch is reached with a pending space. + #[test] + fn block_comments_are_dropped_and_replaced_by_a_single_space() { + assert_eq!(normalize_site_text("let x = /* the answer */ 42;"), "let x = 42;"); + assert_eq!(normalize_site_text("let s = /* comment */ \"value\";"), "let s = \"value\";"); + } + + /// String, raw-string and char literals are copied verbatim, including whitespace and + /// comment-shaped text inside them, rather than being normalized like ordinary code. + #[test] + fn literal_contents_are_preserved_verbatim() { + assert_eq!(normalize_site_text("\"a b // not a comment\""), "\"a b // not a comment\""); + assert_eq!(normalize_site_text("r\"raw /* text */\""), "r\"raw /* text */\""); + assert_eq!(normalize_site_text("'x'"), "'x'"); + } + + /// Non-ASCII identifier characters outside any literal are copied one character at a time + /// through the byte-length-aware fallback path, rather than being mistaken for a literal or a + /// single-byte character. + #[test] + fn multibyte_characters_outside_literals_are_preserved() { + assert_eq!(normalize_site_text("café + 1"), "café + 1"); + } + + /// A site key changes whenever any of its inputs changes, and is stable for identical inputs. + #[test] + fn site_key_is_stable_and_sensitive_to_its_inputs() { + let baseline = site_key("subject::f", "arith.add_to_sub", "1 + 1"); + + assert_eq!(baseline, site_key("subject::f", "arith.add_to_sub", "1 + 1")); + assert_ne!(baseline, site_key("subject::g", "arith.add_to_sub", "1 + 1")); + assert_ne!(baseline, site_key("subject::f", "arith.add_to_mul", "1 + 1")); + assert_ne!(baseline, site_key("subject::f", "arith.add_to_sub", "2 + 2")); + } +} diff --git a/crates/cargo-gamma-engine/src/model/interner.rs b/crates/cargo-gamma-engine/src/model/interner.rs new file mode 100644 index 000000000..cee5e0549 --- /dev/null +++ b/crates/cargo-gamma-engine/src/model/interner.rs @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::sync::Arc; + +use camino::{Utf8Path, Utf8PathBuf}; + +use super::MutantDefinition; +#[cfg(test)] +use super::MutationSite; +use crate::HashMap; + +/// Hands out one shared copy of each distinct string a population repeats. +/// +/// Three source fields are drawn from small closed sets — the file, the mutator name and the +/// enclosing item path — while a run produces hundreds of thousands of mutations. Owned per +/// mutation, those three are hundreds of thousands of heap copies of a few thousand distinct +/// values; shared, they are one allocation each and a pointer per mutant. +/// +/// Sharing happens in two places because it can only ever collapse what the holder can see. A file +/// shares its own strings as it produces its mutants, and [`Interner::share`] collapses what +/// repeats *between* files once the whole population exists. +#[derive(Debug, Default)] +pub struct Interner { + /// The shared copy of each distinct string handed out so far. + /// + /// Keyed through the crate's `FxHashMap`, not the standard `SipHash` map: the keys are the + /// tool's own mutator names and item paths — bounded and non-adversarial — so the + /// DoS-resistant hash the standard map defaults to buys nothing on a per-mutant path. + texts: HashMap>, + + /// The shared copy of each distinct path handed out so far, keyed the same way and for the same + /// reason as [`Self::texts`]. + paths: HashMap>, +} + +impl Interner { + /// The shared copy of a string, creating it the first time the value is seen. + pub fn text(&mut self, value: &str) -> Arc { + if let Some(shared) = self.texts.get(value) { + return Arc::clone(shared); + } + + let shared: Arc = Arc::from(value); + let _stored = self.texts.insert(value.to_owned(), Arc::clone(&shared)); + + shared + } + + /// The shared copy of a path, creating it the first time the value is seen. + pub fn path(&mut self, value: &Utf8Path) -> Arc { + if let Some(shared) = self.paths.get(value) { + return Arc::clone(shared); + } + + let shared: Arc = Arc::from(value); + let _stored = self.paths.insert(value.to_owned(), Arc::clone(&shared)); + + shared + } + + /// Collapses a population's repeated strings onto one shared copy each. + /// + /// Used after decoding a report, where every mutant arrived with its own copies. The values are + /// unchanged — only how many allocations hold them. + pub fn share(&mut self, mutations: &mut [MutantDefinition]) { + for mutation in mutations { + mutation.file = self.path(&mutation.file); + mutation.mutator = self.text(&mutation.mutator); + mutation.item_path = self.text(&mutation.item_path); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::MutantId; + use crate::ops::collect::Shape; + + /// Two mutants naming the same file must end up pointing at one string, not two equal ones. + #[test] + fn a_repeated_value_is_handed_out_as_the_same_allocation() { + let mut interner = Interner::default(); + + let first = interner.text("arith.add_to_sub"); + let second = interner.text("arith.add_to_sub"); + let other = interner.text("arith.add_to_mul"); + + assert!(Arc::ptr_eq(&first, &second), "the same value came back as a second allocation"); + assert!(!Arc::ptr_eq(&first, &other)); + assert_eq!(&*other, "arith.add_to_mul"); + + let path = interner.path(Utf8Path::new("src/lib.rs")); + let again = interner.path(Utf8Path::new("src/lib.rs")); + + assert!(Arc::ptr_eq(&path, &again)); + assert_eq!(path.as_str(), "src/lib.rs"); + } + + /// A decoded report arrives with a copy per mutant, which is exactly what sharing undoes. + #[test] + fn sharing_a_decoded_population_leaves_its_values_alone() { + let sample = |name: &str| MutantDefinition { + id: MutantId::new("deadbeefcafe"), + file: Arc::from(Utf8Path::new(name)), + site: Arc::new(MutationSite { + span: 0..1, + line: 1, + end_line: 1, + column: 1, + original: "true".to_owned().into(), + }), + mutator: Arc::from("lit.true_to_false"), + item_path: Arc::from("subject::f"), + occurrence: 0, + replacement_index: 0, + replacement: "false".to_owned().into(), + shape: Shape::Expr, + }; + + let mut mutations = vec![sample("a.rs"), sample("a.rs")]; + + assert!( + !Arc::ptr_eq(&mutations[0].file, &mutations[1].file), + "the fixture must start unshared" + ); + + let before: Vec = mutations.iter().map(|m| format!("{} {}", m.file, m.mutator)).collect(); + + Interner::default().share(&mut mutations); + + let after: Vec = mutations.iter().map(|m| format!("{} {}", m.file, m.mutator)).collect(); + + assert_eq!(before, after, "sharing changed a value"); + assert!(Arc::ptr_eq(&mutations[0].file, &mutations[1].file)); + assert!(Arc::ptr_eq(&mutations[0].mutator, &mutations[1].mutator)); + } +} diff --git a/crates/cargo-gamma-engine/src/model/mod.rs b/crates/cargo-gamma-engine/src/model/mod.rs new file mode 100644 index 000000000..e455fdc2c --- /dev/null +++ b/crates/cargo-gamma-engine/src/model/mod.rs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Source-level mutation identity and storage. + +mod identity; +mod interner; +mod mutant_definition; +mod mutation_site; + +pub(crate) use identity::mutant_id_with_discriminator; +pub use identity::{MUTANT_ID_HEX_LEN, MUTANT_ID_VERSION, MutantId, mutant_id, normalize_site_text, site_key}; +pub use interner::Interner; +pub use mutant_definition::MutantDefinition; +pub use mutation_site::MutationSite; diff --git a/crates/cargo-gamma-engine/src/model/mutant_definition.rs b/crates/cargo-gamma-engine/src/model/mutant_definition.rs new file mode 100644 index 000000000..775c3ae20 --- /dev/null +++ b/crates/cargo-gamma-engine/src/model/mutant_definition.rs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::ops::Range; +use std::sync::Arc; + +use camino::Utf8Path; +use compact_str::CompactString; + +use super::{MutantId, MutationSite}; +use crate::ops::collect::Shape; + +/// One source-level mutation, before Cargo/run policy and execution state are attached. +#[derive(Debug, Clone)] +pub struct MutantDefinition { + pub id: MutantId, + pub file: Arc, + /// Shared site data (span, location, original text) for all replacements at this span. + pub site: Arc, + pub mutator: Arc, + pub item_path: Arc, + pub occurrence: u32, + pub replacement_index: u32, + pub replacement: CompactString, + pub shape: Shape, +} + +impl MutantDefinition { + /// Byte range of the construct in the original file. + #[inline] + #[must_use] + pub fn span(&self) -> &Range { + &self.site.span + } + + /// One-based start line. + #[inline] + #[must_use] + pub fn line(&self) -> usize { + self.site.line + } + + /// One-based end line. + #[inline] + #[must_use] + pub fn end_line(&self) -> usize { + self.site.end_line + } + + /// One-based start column. + #[inline] + #[must_use] + pub fn column(&self) -> usize { + self.site.column + } + + /// The original source text of the construct. + #[inline] + #[must_use] + pub fn original(&self) -> &CompactString { + &self.site.original + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Builds a definition whose accessors are each expected to return a distinguishable value, + /// so a mistake such as reading the wrong field cannot pass unnoticed. + fn sample() -> MutantDefinition { + MutantDefinition { + id: MutantId::new("deadbeefcafe"), + file: Arc::from(Utf8Path::new("src/lib.rs")), + site: Arc::new(MutationSite { + span: 12..19, + line: 3, + end_line: 4, + column: 5, + original: "1 + 1".to_owned().into(), + }), + mutator: Arc::from("arith.add_to_sub"), + item_path: Arc::from("subject::f"), + occurrence: 0, + replacement_index: 0, + replacement: "1 - 1".to_owned().into(), + shape: Shape::Expr, + } + } + + /// Every accessor reads through to the field on the shared `MutationSite` it delegates to, + /// rather than to some other field or a stale copy. + #[test] + fn accessors_read_through_to_the_shared_site() { + let definition = sample(); + + assert_eq!(definition.span(), &(12..19)); + assert_eq!(definition.line(), 3); + assert_eq!(definition.end_line(), 4); + assert_eq!(definition.column(), 5); + assert_eq!(definition.original(), "1 + 1"); + } +} diff --git a/crates/cargo-gamma-engine/src/model/mutation_site.rs b/crates/cargo-gamma-engine/src/model/mutation_site.rs new file mode 100644 index 000000000..653965dbb --- /dev/null +++ b/crates/cargo-gamma-engine/src/model/mutation_site.rs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::ops::Range; + +use compact_str::CompactString; + +/// Per-span data shared by every replacement that targets the same source construct. +/// +/// Multiple mutators (or multiple replacements from one mutator) can target the same byte range. +/// Storing the location and original text once per span, rather than once per definition, removes +/// repeated allocations proportional to the replacement count at each site. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MutationSite { + /// Byte range of the construct in the original file. + pub span: Range, + + /// One-based line of the start of the span. + pub line: usize, + + /// One-based line of the last source line the span covers. + pub end_line: usize, + + /// One-based column of the start of the span. + pub column: usize, + + /// The original source text of the construct. + pub original: CompactString, +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/candidate.rs b/crates/cargo-gamma-engine/src/ops/collect/candidate.rs new file mode 100644 index 000000000..be4e820b0 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/candidate.rs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::ops::Range; +use std::sync::Arc; + +use compact_str::CompactString; + +use super::Shape; + +/// One mutation opportunity, before it is given a run-wide ordinal. +#[derive(Debug, Clone)] +pub struct Candidate { + /// Registry name of the mutator. + pub mutator: &'static str, + + /// Byte range of the construct being replaced. + pub span: Range, + + /// The text that replaces the *whole* span. + /// + /// Whole-span, not just the changed token: every mutant is then a uniform (range, text) pair, + /// so instrumentation never has to know what family produced it, and the human-readable form + /// shows the real before and after rather than a bare operator with no context. + pub replacement: CompactString, + + /// Index among the replacements this mutator offers at this site. + pub replacement_index: u32, + + /// Path of the enclosing item. + /// + /// Shared rather than owned: a file opens a few hundred scopes and emits tens of thousands of + /// candidates, so one allocation per scope and a pointer per candidate is the whole difference. + pub item_path: Arc, + + /// How the site must be guarded. + pub shape: Shape, +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector.rs b/crates/cargo-gamma-engine/src/ops/collect/collector.rs new file mode 100644 index 000000000..88ce0b36e --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector.rs @@ -0,0 +1,1858 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::fmt::Display; +use std::sync::Arc; + +use compact_str::{CompactString, format_compact}; +use proc_macro2::Span; +use syn::punctuated::Punctuated; +use syn::spanned::Spanned as _; +use syn::token::Comma; +use syn::visit::{self, Visit}; +use syn::{ + Attribute, BinOp, Block, Expr, ExprBinary, ExprBreak, ExprCall, ExprContinue, ExprForLoop, ExprIf, ExprIndex, ExprLit, ExprLoop, + ExprMatch, ExprMethodCall, ExprRange, ExprReference, ExprRepeat, ExprReturn, ExprStruct, ExprUnary, ExprWhile, FnArg, GenericArgument, + Generics, ImplItem, ImplItemConst, ImplItemFn, ItemConst, ItemFn, ItemImpl, ItemMod, ItemStatic, ItemTrait, Lit, Local, Macro, Member, + Pat, RangeLimits, ReturnType, Signature, Stmt, TraitItemConst, TraitItemFn, Type, UnOp, Variant, +}; + +use super::stated::stated_range; +use super::{Candidate, Defaults, Shape}; +use crate::cfg::CfgSet; +use crate::ops::registry::Selection; +use crate::parse::SourceFile; +use crate::{HashMap, HashSet}; + +mod indexes; +mod noop; +pub(super) mod phase_one; +mod predicates; +mod tables; +mod types; +mod values; + +#[cfg(test)] +mod tests; + +use indexes::{Indexes, NumericUses, indexes}; +use noop::is_noop; +use predicates::{ + binds_a_pattern, boolean_literal, callee_name, callee_type, declared_name, diverges, expr_attrs, is_assign_op, is_capacity_call, + is_capacity_result, is_catch_all, is_constant_case, is_default_call, is_diagnostic_message, is_integer_zero_literal, + is_numeric_binding, is_numeric_return, is_numeric_type, is_promotable, is_textual, loop_produces_value, returns_numeric, + returns_result, stmt_attrs, type_name, +}; +use tables::{binary_replacements, in_place_reorder, method_renames}; +use types::{Types, returns_undefaultable_error, undefaulted_parameters}; +use values::{Kind, resolve_type, return_values}; + +use super::defaults::{DefaultPaths, standard_defaulted_parameters}; + +/// The marker string every string-valued mutant is replaced by. +/// +/// It is deliberately implausible: a test that happens to accept it was almost certainly not +/// asserting on the string at all. +const XYZZY: &str = "xyzzy"; + +/// One reversible change to the block-scoped binding evidence. +/// +/// Remembered so a lexical block can undo exactly what it wrote — its own insertions and shadows — +/// rather than restoring a saved copy of the whole in-scope set. Replaying every entry recorded +/// since a block was entered, in reverse, returns `bindings` and `deferred` to the state they held +/// on entry even when a name was touched more than once. +enum Undo { + /// Restores `bindings[name]` to the value it held before the block wrote it: `Some(was)` puts + /// the prior type evidence back, `None` removes a name the block introduced. + Binding(String, Option), + + /// Restores whether `deferred` contained `name` before the block changed it. + Deferred(String, bool), +} + +struct LoopContext { + label: Option, + produces_value: bool, +} + +/// The traversal state. +pub(super) struct Collector<'a> { + file: &'a SourceFile, + selection: &'a Selection, + + /// The enclosing item path at each nesting level, each entry already fully joined. + /// + /// Storing the joined form rather than the segments means emitting a candidate rebuilds nothing, + /// and a large tree emits far more candidates than it opens scopes. Shared rather than owned so + /// that emitting is a pointer copy: every candidate under one scope names the same path. + scope: Vec>, + + /// The path of a candidate outside any item, shared for the same reason. + outermost: Arc, + + candidates: Vec, + + /// Depth of nesting inside a context where mutation is not possible or not useful. + /// + /// Const and static initializers are the important case: the encoding wraps the original + /// expression in an `if` over a function call, which is not permitted in a const context. A + /// mutant that cannot compile is not a weak test, it is noise, so these are never generated + /// rather than generated and then rolled back. + inert_depth: usize, + + /// Whether the `impl` block being traversed implements `Default`. + /// + /// Replacing a function body with `Default::default()` is the fallback for a type this tool + /// cannot name a value of, and inside `impl Default for T` that fallback names the very + /// function being replaced. The mutant is unbounded recursion, so it neither compiles into + /// something meaningful nor fails fast: it exhausts the stack, which costs a full timeout to + /// discover and says nothing about the tests. + in_default_impl: bool, + + /// The concrete type named by `Self` inside the enclosing `impl` block. + impl_self_type: Option, + + /// Concrete associated types declared by the enclosing `impl` block. + impl_self_associated: HashMap, + + /// The names this file uses for the standard `Default` trait and its fallback spelling. + /// + /// The fallback values this collector emits use `Default::default()`, but inherent methods and + /// custom traits can have the same spelling. This resolver keeps no-op elimination tied to + /// the standard trait while recognizing a bare shadow that the fallback text would recurse + /// through. + default_paths: DefaultPaths, + + /// Caller-supplied `Err(...)` payloads, from `--error`. + errors: &'a [String], + + /// Whether the function being traversed returns a number. + /// + /// Perturbing a returned value only makes sense when the value is one that can be off by one. + /// Without this the family offered `(Vec::new()) + 1` for every function returning a + /// collection, and each of those costs a rollback round to discover it never compiled. + numeric_return: bool, + + /// Whether the enclosing function returns a `Result` whose error type comes from another crate. + /// + /// `Ok(v)` becoming `Err(Default::default())` needs an error value, and the error type is fixed + /// by the signature rather than visible at the call site. Recording it on the way in is what + /// lets the site be screened. + foreign_error_return: bool, + + /// Whether each named binding in the enclosing function holds a number. + /// + /// The perturbation family adds one to an expression, and the commonest thing it is offered is + /// a bare identifier — for which nothing in the syntax says whether it is an integer or a + /// `String`. Function parameters and annotated `let`s are the two places a local type is + /// written down in the source, and recording them converts most of those guesses into an + /// answer. A binding that is not here was never written down, and is left alone rather than + /// assumed either way. + /// + /// `true` means the source says the name holds a number. Absent and `false` both mean the + /// source does not say so, and are treated alike: the family is only offered on positive + /// evidence. + bindings: HashMap, + + /// Whether each field name declared anywhere in this file holds a number. + /// + /// Collected in one pass before traversal, because a field is very often read above the + /// `struct` that declares it and an in-order record would miss exactly those uses. A name two + /// structs declare with disagreeing types is recorded as unknown: without type resolution + /// there is no way to tell which one a given `x.count` refers to. + fields: HashMap, + + /// Names the file uses somewhere in a way only a number can be used. + /// + /// Read as evidence of last resort, after annotations and initialisers have had their say. + numeric_uses: NumericUses, + + /// Whether each constant and static declared anywhere in this file holds a number. + /// + /// A screaming-case name is otherwise taken for a number, which is right for `MAX` and + /// `DEFAULT_SIZE` and wrong for every `const PREFIX: &str` and `const CAP: Duration` — and + /// adding one to those is the single largest source of mutants that cannot compile. The + /// declaration says which is which, in the file, in writing, so it is read rather than guessed. + /// Collected in one pass ahead of traversal for the same reason the field index is: a constant + /// is very often used above the item that declares it. + constants: HashMap, + + /// Names in the enclosing function declared by a `let` that supplies no initialiser. + /// + /// A `let scanned;` whose value is settled later by a plain assignment makes that assignment + /// load-bearing: delete it and the binding is still uninitialised at its first use. That is a + /// compile error rather than a mutant, and an unusually awkward one, because rustc reports + /// E0381 at the *use*, which is a different statement from the one that was mutated. Withdrawal + /// attributes a diagnostic to the mutant occupying its span, finds no mutant there, and gives + /// up on the whole run rather than on the one bad mutant. Recording the deferred names lets + /// their initialising assignments be passed over before any of that can happen. + deferred: HashSet, + + /// The undo log a lexical block replays on the way out, so leaving a block restores the binding + /// evidence it changed without copying the whole in-scope set. + /// + /// A block can see its enclosing scope's `bindings` and `deferred`, so it inherits them rather + /// than starting fresh; what it must not do is let its own insertions and shadows outlive it. + /// Each entry records one touched name's prior state, and [`Collector::in_scope`] reverses its + /// own suffix in reverse order on exit — undoing exactly what the block added, at a cost that + /// grows with the names the block touches rather than with every name in scope. A nested + /// function ([`Collector::in_function`]) swaps the maps out wholesale instead, so it discards + /// its own suffix rather than replaying it. + undo: Vec, + + /// Type parameters in scope that are not known to implement `Default`. + /// + /// The `fn_value` family reaches for `Default::default()` whenever it cannot name a value of a + /// type, and for an abstract type that is a guess rather than a fact: nothing says a caller's + /// `E` or a trait's `Self::Value` has a `Default`, and on a serde-shaped API almost none of + /// them do. Holding the names lets the guess be withheld exactly where it is unfounded. + generics: Vec, + + /// Type parameters in scope that are known to implement the standard `Default` trait. + /// + /// This is intentionally separate from `generics`: one controls whether a fallback value can + /// be emitted, while this one says whether `T::default()` is exactly the standard fallback + /// already present in a body. + defaulted: Vec, + + /// The configuration predicates that hold for the build this file will be part of. + /// + /// Code behind a predicate that does not hold is stripped by the compiler, so a guard there is + /// never compiled, no test can activate it, and the mutant would be reported as a survivor no + /// test could ever have caught. + cfg: &'a CfgSet, + + /// The module path each name this file imports was brought in from. + /// + /// Read once before traversal, like the field and numeric indexes, because a type is used above + /// its `use` as often as below it and an in-order record would miss exactly those uses. + /// + /// `None` marks a name two `use` items disagree about, which is as unknown as never having been + /// imported. + imports: HashMap>>, + + /// What the rest of the workspace implements `Default` for. + defaults: &'a Defaults, + + /// The spans and replacement texts already recorded, so no two candidates can be the same edit. + /// + /// Mutators are written independently and several of them converge on the same small integers: + /// `int_increment` and `int_to_one` both turn a literal `0` into `1`, and `int_decrement` and + /// `int_to_zero` both turn a `1` into `0`. Two names for one edit is still one edit — the same + /// build, the same tests, the same verdict — so the second is pure cost, and it also weights + /// its site twice in the score. + /// Keyed by span, holding the indices into `candidates` already emitted there. + /// + /// Indices rather than the replacement text: the text is already owned by the candidate, and + /// copying it into a key would be one heap allocation per site for a value thrown away at the + /// end of collection. A span carries one or two candidates in almost every case, so the scan is + /// shorter than hashing the string would be. + seen: HashMap<(usize, usize), Vec>, + + /// Enclosing loops, used to reject a valueless `break` when its target requires a value. + loops: Vec, +} + +impl<'a> Collector<'a> { + /// Creates a collector rooted at the file's top-level scope. + pub(super) fn new( + file: &'a SourceFile, + selection: &'a Selection, + errors: &'a [String], + cfg: &'a CfgSet, + defaults: &'a Defaults, + ) -> Self { + Self::with_indexes(file, selection, errors, cfg, defaults, indexes(&file.ast, selection)) + } + + /// Creates a collector from indexes a caller already built. + /// + /// Lets the fused phase-one pass (`collector::phase_one`) hand over the indexes it computed in + /// its own single walk, so the collector's traversal does not pay for computing them a second + /// time the way [`Collector::new`] otherwise would. + pub(super) fn with_indexes( + file: &'a SourceFile, + selection: &'a Selection, + errors: &'a [String], + cfg: &'a CfgSet, + defaults: &'a Defaults, + indexes: Indexes, + ) -> Self { + let default_paths = DefaultPaths::of(&file.ast); + + Self { + file, + selection, + scope: Vec::new(), + outermost: Arc::from(""), + candidates: Vec::new(), + inert_depth: 0, + in_default_impl: false, + impl_self_type: None, + impl_self_associated: HashMap::default(), + default_paths, + numeric_return: false, + foreign_error_return: false, + bindings: HashMap::default(), + fields: indexes.fields, + imports: indexes.imports, + defaults, + numeric_uses: indexes.numeric_uses, + constants: indexes.constants, + deferred: HashSet::default(), + undo: Vec::new(), + generics: Vec::new(), + defaulted: Vec::new(), + errors, + cfg, + seen: HashMap::default(), + loops: Vec::new(), + } + } + + /// Consumes the collector, returning what it found. + pub(super) fn finish(self) -> Vec { + self.candidates + } + + /// Records an expression-shaped candidate if its mutator is selected. + fn emit(&mut self, mutator: &'static str, span: Span, replacement: impl Into, replacement_index: u32) { + self.emit_shaped(mutator, span, replacement, replacement_index, Shape::Expr); + } + + /// Records a candidate of a given shape if its mutator is selected. + fn emit_shaped( + &mut self, + mutator: &'static str, + span: Span, + replacement: impl Into, + replacement_index: u32, + shape: Shape, + ) { + if !self.wants(mutator) { + return; + } + + self.emit_at(mutator, span.byte_range(), replacement, replacement_index, shape); + } + + /// Records a candidate over an explicit byte range. + /// + /// Most sites come from a single syntax node and can use its span, but a site can also span + /// several nodes — the elements of a `vec!` from the first to the last — with no one node + /// covering exactly the text being replaced. + fn emit_at( + &mut self, + mutator: &'static str, + range: core::ops::Range, + replacement: impl Into, + replacement_index: u32, + shape: Shape, + ) { + if !self.wants(mutator) { + return; + } + + // A range outside the file text means the node came from a macro expansion, where byte + // offsets do not correspond to anything we can splice. + if range.start >= range.end || range.end > self.file.text.len() { + return; + } + + let replacement = replacement.into(); + + // A mutant that reproduces the code it replaces cannot be caught by any test, because + // there is nothing to catch. Reporting it would accuse the suite of a gap that does not + // exist, and testing it would spend a whole build and test cycle to learn nothing. + if is_noop( + &replacement, + self.file.text.get(range.clone()).unwrap_or_default(), + shape, + &self.default_paths, + &self.defaulted, + ) { + return; + } + + // Whichever mutator reached this edit first keeps it. Selection is consulted before this + // point, so a run that asks for only one of a colliding pair still gets the mutant. + let span_key = (range.start, range.end); + + if self + .seen + .get(&span_key) + .is_some_and(|emitted| emitted.iter().any(|&at| self.replacement_at(at) == replacement)) + { + return; + } + + let at = u32::try_from(self.candidates.len()) + .expect("a candidate vector cannot exceed u32::MAX entries before exhausting address space"); + + self.candidates.push(Candidate { + mutator, + span: range, + replacement, + replacement_index, + item_path: self.scope.last().map_or_else(|| Arc::clone(&self.outermost), Arc::clone), + shape, + }); + + self.seen.entry(span_key).or_default().push(at); + } + + /// The replacement text of an already-emitted candidate, for the dedup scan. + fn replacement_at(&self, at: u32) -> &str { + self.candidates + .get(at as usize) + .map_or("", |candidate| candidate.replacement.as_str()) + } + + /// Returns whether a mutator would produce anything here. + /// + /// Checked before building any replacement text, so a tree scanned with a mutator switched off + /// pays nothing for the text it would have spliced. + fn wants(&self, mutator: &str) -> bool { + self.inert_depth == 0 && self.selection.contains(mutator) + } + + /// Returns the source text a span covers, or an empty string if the span is not in the file. + /// + /// Borrowed from the file rather than the collector, so a caller can hold the result across a + /// mutating call and no copy is made for a span that turns out not to be wanted. + fn text_of(&self, span: Span) -> &'a str { + self.file.text.get(span.byte_range()).unwrap_or("") + } + + /// Returns the negation of the expression a span covers. + /// + /// The parentheses are not optional. `!` binds tighter than every binary operator, so negating + /// `a == b` without them yields `!a == b`, which is a different expression and usually does + /// not even type-check. + fn negation_of(&self, span: Span) -> CompactString { + format_compact!("!({})", self.text_of(span)) + } + + /// Runs a closure with a name pushed onto the scope stack. + fn scoped(&mut self, name: impl Display, body: impl FnOnce(&mut Self) -> T) -> T { + let path = self + .scope + .last() + .map_or_else(|| name.to_string(), |parent| format!("{parent}::{name}")); + + self.scope.push(Arc::from(path)); + + let result = body(self); + let _ = self.scope.pop(); + + result + } + + /// Returns the stable scope name of one inherent or trait implementation. + fn impl_scope(&self, node: &ItemImpl) -> String { + let self_type = type_name(&node.self_ty); + + let Some((_bang, trait_path, _for)) = &node.trait_ else { + return self_type; + }; + + let trait_path = compact_path(self.text_of(trait_path.span())); + + if trait_path.is_empty() { + return self_type; + } + + format!("<{self_type} as {trait_path}>") + } + + /// Emits the function-value mutants for a function with the given signature and body. + /// + /// The bluntest question that can be asked of a test suite: replacing a whole function body + /// with a plausible constant asks whether the suite looks at the answer at all. + /// + /// The attributes are read as well as the signature, because a site may state the expression to + /// substitute rather than leave it to be guessed from the return type. See [`stated_range`]. + fn function(&mut self, attrs: &[Attribute], sig: &Signature, body: &Block) { + // The whole body of a `const fn` is a const context, so nothing in it can call the guard + // predicate. `visit_const_fn` keeps the subtree inert; this only guards the body value. + if sig.constness.is_some() { + return; + } + + // An empty body already produces the unit value, so replacing it with one changes nothing. + // A mutant that cannot alter behavior can never be caught, and reporting it as a survivor + // would be an accusation against the test suite for something no test could detect. + if body.stmts.is_empty() { + return; + } + + let span = body.span(); + + // Read before anything is emitted, because it decides whether the guessed values below are + // offered at all. `None` covers both "nothing was stated" and "what was stated cannot be + // read", and the second is already on its way to stopping the run — see `stated::check`. + // + // A stated value the run will not emit is no reason to withhold the guesses. The attribute + // may change what a site substitutes; it may never take the site's only `fn_value` mutant + // away, and a selection naming a sibling mutator but not `fn_value.stated` would otherwise + // do exactly that — silently, since a site that emits nothing is a site nothing reports. + let stated = stated_range(attrs).filter(|_range| self.wants("fn_value.stated")); + + // A method's own type parameters join the ones its `impl` block declares; both are in + // scope in the signature being read here. + let mut abstracts = self.generics.clone(); + + abstracts.extend(undefaulted_parameters(&sig.generics, &self.default_paths)); + + let values = return_values( + &sig.output, + &Types { + abstracts: &abstracts, + imports: &self.imports, + defaults: self.defaults, + self_type: self.impl_self_type.as_ref(), + self_associated: Some(&self.impl_self_associated), + }, + ); + + // `Default::default()` inside `impl Default` is a call to this very function, so the + // mutant is unbounded recursion rather than a different answer. It cannot be killed by a + // test noticing a wrong value, only by the stack running out, which costs a full timeout + // to reach and reports the slowest verdict there is for the least information. + let recursive = self.in_default_impl && sig.ident == "default"; + + // An `impl Iterator` return needs both arms of the guard wrapped so they share a type, + // which is a different splice from every other return. See `Shape::IterBlock`. + let shape = match &sig.output { + ReturnType::Type(_, ty) if resolve_type(ty) == Kind::Iterator => Shape::IterBlock, + _ => Shape::Block, + }; + + // The value a function ends on is the one its caller reasons about, so it is one of the + // positions where being wrong by one is a real fault rather than a compile error — but + // only when the value is a number, which the signature already says. + if is_numeric_return(&sig.output) + && let Some(Stmt::Expr(trailing, None)) = body.stmts.last() + && !matches!(trailing, Expr::Lit(ExprLit { lit: Lit::Int(_), .. })) + { + self.perturb_proven(trailing); + } + + let value_count = values.len(); + + for (index, (mutator, value)) in values.into_iter().enumerate() { + let index = u32::try_from(index).unwrap_or(u32::MAX); + + if recursive && mutator == "fn_value.default" { + continue; + } + + // A site that states its own value has answered the question these guesses exist to + // guess at, so the stated expression is emitted in their place rather than beside them. + // Offering both would ask the same question twice at one site, and the second answer + // would be the one the author had already said was wrong. + if stated.is_some() { + continue; + } + + self.emit_shaped(mutator, span, value, index, shape); + } + + // Emitted at its own mutator name rather than under whichever guess it displaced, so that + // its identity is its own: a verdict cached against a guess that could not compile must not + // be inherited by the expression written to replace it, which is the whole reason the + // attribute was reached for. + if let Some(range) = stated { + let expression = self.file.text.get(range).unwrap_or_default(); + + self.emit_shaped("fn_value.stated", span, expression, 0, shape); + } + + // Caller-supplied error values, which reach the error types `Err(Default::default())` + // cannot. Their indices continue the static list's so that adding one does not renumber + // the mutants already generated at this site. + if returns_result(&sig.output) && self.wants("fn_value.err_with") { + for (offset, error) in self.errors.iter().enumerate() { + let index = u32::try_from(value_count.saturating_add(offset)).unwrap_or(u32::MAX); + let replacement = format_compact!("Err({error})"); + + self.emit_shaped("fn_value.err_with", span, replacement, index, Shape::Block); + } + } + } + + /// Emits the statement-deletion mutants for one statement. + /// + /// Only statements whose value is discarded are eligible. Deleting a `let` would leave every + /// later use of the binding unresolved, which is a compile error rather than a mutant, and + /// deleting a block's trailing expression would change the block's type. + fn statement(&mut self, statement: &Stmt) { + let Stmt::Expr(expression, Some(_)) = statement else { + return; + }; + + let mutator = match expression { + // A call whose result is thrown away is being run for its effect, which is exactly the + // thing a test that only checks return values will not notice going missing. + Expr::Call(_) | Expr::MethodCall(_) => in_place_reorder(expression).unwrap_or("stmt.delete_call"), + Expr::Assign(assign) if self.initializes_deferred(&assign.left) => return, + Expr::Assign(_) => "stmt.delete_assign", + Expr::Binary(binary) if is_assign_op(&binary.op) => "stmt.delete_assign", + + // A `break` carrying a value decides the type of the loop it leaves, so deleting it + // can change that type rather than the program's behaviour. + Expr::Break(brk) if brk.expr.is_none() => "loop.delete_break", + Expr::Continue(_) => "loop.delete_continue", + + _ => return, + }; + + self.emit_shaped(mutator, statement.span(), "", 0, Shape::Stmt); + } + + /// Returns whether the target of an assignment is a binding that was declared without a value. + /// + /// Only a bare name can be the thing that first gives a deferred `let` its value; assigning + /// through a field, an index or a dereference presupposes that the binding already holds one. + fn initializes_deferred(&self, target: &Expr) -> bool { + let Expr::Path(path) = target else { + return false; + }; + + path.path + .get_ident() + .is_some_and(|ident| self.deferred.contains(&ident.to_string())) + } + + /// Emits the guard mutants for one boolean condition. + /// + /// Shared by `if`, `while` and match arms, which ask the same question in three syntaxes and + /// should not answer it three different ways. + fn condition(&mut self, negate: &'static str, always_true: &'static str, always_false: &'static str, cond: &Expr) { + // A condition that binds a pattern cannot be negated or replaced by a boolean. In a let + // chain the binding may sit anywhere in the `&&` spine, not just at the top. + if binds_a_pattern(cond) { + return; + } + + let span = cond.span(); + + if self.wants(negate) { + let negated = self.negation_of(span); + + self.emit(negate, span, negated, 0); + } + + // A condition that is already the literal it would be replaced by yields a mutant that + // compiles to the original program, so it can never be caught and would be scored as a + // survivor forever. + let literal = boolean_literal(cond); + + if literal != Some(true) { + self.emit(always_true, span, "true", 1); + } + + if literal != Some(false) { + self.emit(always_false, span, "false", 2); + } + } + + /// Emits the mutants for the arms of one `match`. + /// + /// Two unrelated families meet here. An arm with a guard is a condition like any other and is + /// mutated as one. An arm without a guard can instead be made to stop matching, but only when + /// a later wildcard is there to receive what falls through — the compiler does not count a + /// guarded arm towards exhaustiveness, so adding a guard to the last arm that can match a + /// value turns the mutant into a compile error rather than a question about the tests. + fn match_arms(&mut self, node: &ExprMatch) { + // The first catch-all, and only an unguarded one that is actually compiled: a guarded `_` + // catches nothing in particular and leaves the match relying on the arms above it, and one + // behind a predicate that does not hold is not in the program at all, so relying on it + // would produce a mutant whose match is no longer exhaustive. Suppression is deliberately + // not consulted — a suppressed arm is still compiled and still keeps the match exhaustive. + let wildcard = node + .arms + .iter() + .position(|arm| is_catch_all(&arm.pat) && arm.guard.is_none() && self.cfg.holds_for(&arm.attrs)); + + for (index, arm) in node.arms.iter().enumerate() { + if self.skipped(&arm.attrs) { + continue; + } + + if let Some((_if, guard)) = arm.guard.as_ref() { + self.condition("match_guard.negate", "match_guard.always_true", "match_guard.always_false", guard); + + // An arm that already has a guard is disabled by forcing that guard false, which + // `match_guard.always_false` above already offers. Emitting a second mutant that + // does the same thing would pay twice for one question. + continue; + } + + if wildcard.is_some_and(|at| index < at) { + self.emit_shaped("match_arm.never_matches", arm.pat.span(), "", 0, Shape::Arm); + } + } + } + + /// Emits the field-omission mutants for one struct literal. + /// + /// Only a literal with a base expression is eligible, because the base is what keeps the + /// result well formed once a field is taken out. Each mutant asks whether any test can tell + /// the written value from the one the base would have supplied — which, for a field that is + /// being set to its default anyway, nothing can. + fn struct_fields(&mut self, node: &ExprStruct) { + if !self.wants("struct_field.omit") { + return; + } + + // The `..` token rather than the expression after it, since the field being removed runs + // up to the token and the whitespace between them belongs to neither. + let Some(rest) = node.dot2_token.as_ref().map(syn::spanned::Spanned::span) else { + return; + }; + + let whole = node.span().byte_range(); + + if whole.start >= whole.end || whole.end > self.file.text.len() { + return; + } + + for (index, field) in node.fields.iter().enumerate() { + // A field behind a predicate that does not hold is not in the compiled literal, so + // omitting it changes nothing that could be observed. + if self.skipped(&field.attrs) { + continue; + } + + let from = field.span().byte_range().start; + let to = node + .fields + .iter() + .nth(index.saturating_add(1)) + .map_or_else(|| rest.byte_range().start, |next| next.span().byte_range().start); + + // Everything between the two is the field, its comma and the space after it. + if from < whole.start || to > whole.end || from >= to { + continue; + } + + let head = self + .file + .text + .get(whole.start..from) + .expect("field bounds were checked above and parser spans are UTF-8 boundaries"); + let tail = self + .file + .text + .get(to..whole.end) + .expect("field bounds were checked above and parser spans are UTF-8 boundaries"); + + let replacement = format_compact!("{head}{tail}"); + let ordinal = u32::try_from(index).unwrap_or(u32::MAX); + + self.emit_shaped("struct_field.omit", node.span(), replacement, ordinal, Shape::Expr); + } + } + + /// Offers `+ 1` and `- 1` for one expression, in a position where a boundary is being decided. + /// + /// Deliberately not applied to every expression. Doing so would double the population of a + /// large project, duplicate the literal and arithmetic families wherever they already apply, + /// and produce type errors anywhere the expression is generic or not numeric at all. The + /// positions it is applied to are the ones that carry a postcondition somebody could get + /// wrong by one: what a function is handed, what it gives back, what is indexed, and where a + /// range stops. + /// Offers a mutant for each element of a `vec!` literal, with that element removed. + /// + /// The removal sweeps up the separating comma along with the element, so the list that is left + /// is still well formed wherever the element sat in it. + fn omit_elements(&mut self, node: &Macro, elements: &Punctuated) { + let spans: Vec<_> = elements.iter().map(|element| element.span().byte_range()).collect(); + + let (Some(first), Some(last)) = (spans.first(), spans.last()) else { + return; + }; + + // The site has to be the whole `vec![..]`, not just the elements inside it. A guarded + // mutant is one arm of an `if`, and `1, 2` is a list rather than an expression, so + // narrowing the site to the element range would emit code that does not parse at all. + let whole = node.span().byte_range(); + let items = first.start..last.end; + + let (Some(text), Some(head), Some(tail)) = ( + self.file.text.get(items.clone()), + self.file.text.get(whole.start..items.start), + self.file.text.get(items.end..whole.end), + ) else { + return; + }; + + let (text, head, tail) = (text.to_owned(), head.to_owned(), tail.to_owned()); + + for (index, span) in spans.iter().enumerate() { + // Everything up to this element, and everything from the next one on. For the final + // element there is no next, so the cut runs to the end and takes the preceding comma + // with it. + let (from, to) = spans.get(index.saturating_add(1)).map_or_else( + || { + ( + spans.get(index.wrapping_sub(1)).map_or(span.start, |previous| previous.end), + items.end, + ) + }, + |next| (span.start, next.start), + ); + + let (Some(before), Some(after)) = ( + text.get(..from.saturating_sub(items.start)), + text.get(to.saturating_sub(items.start)..), + ) else { + continue; + }; + + let replacement = format_compact!("{head}{before}{after}{tail}"); + let ordinal = u32::try_from(index).unwrap_or(u32::MAX); + + self.emit_at("collection.omit_element", whole.clone(), replacement, ordinal, Shape::Expr); + } + } + + /// Offers the curated same-shape renames of a standard-library method. + /// + /// The whole call expression is the site, not the method name, because a mutant is spliced in + /// as `if guard { .. } else { .. }` and an `if` is not a legal method name. Rewriting the name + /// inside a copy of the call's own text keeps any turbofish and every argument exactly as + /// written, which reconstructing the call from its parts would not. + fn rename_method(&mut self, node: &ExprMethodCall, method: &str) { + let Some(swaps) = method_renames(method, node.args.len()) else { + return; + }; + + let whole = node.span().byte_range(); + let name = node.method.span().byte_range(); + + // A method call whose receiver spans a macro expansion can put the name outside the call, + // in which case there is nothing meaningful to splice. + if name.start < whole.start || name.end > whole.end { + return; + } + + let text = &self.file.text; + let (Some(before), Some(after)) = (text.get(whole.start..name.start), text.get(name.end..whole.end)) else { + return; + }; + + let (before, after) = (before.to_owned(), after.to_owned()); + + for (index, (mutator, replacement)) in swaps.iter().enumerate() { + let index = u32::try_from(index).unwrap_or(u32::MAX); + + self.emit(mutator, node.span(), format_compact!("{before}{replacement}{after}"), index); + } + } + + fn perturb(&mut self, expression: &Expr) { + // The veto outranks the proof. Both are read off the source, but only one of them can be + // wrong in a way that costs a rollback round: nothing in the allowlist proves an expression + // is a number so firmly that a `format!` beside it should be ignored. + if self.is_known_numeric(expression) && !is_textual(expression) { + self.perturb_proven(expression); + } + } + + /// Offers the perturbations for an expression the surrounding code has already typed. + /// + /// A `return` inside a function whose signature says it yields a number needs no inference: + /// the signature settles the question, and settles it better than this file could. + fn perturb_proven(&mut self, expression: &Expr) { + if !self.wants("expr.increment") && !self.wants("expr.decrement") { + return; + } + + // A divergent expression -- `return`, a never-returning macro, a breakless `loop`, an + // all-diverging `if` or `match` -- has type `!` and so passes the signature-only numeric + // proof, but its value is never reached: `(return 5) + 1` returns before the `+ 1` runs, so + // both perturbations behave exactly like the original and survive every test. See `diverges`. + if diverges(expression) { + return; + } + + if is_capacity_result(expression) { + return; + } + + let span = expression.span(); + let text = self.text_of(span); + + if text.is_empty() { + return; + } + + // Parenthesised because the expression may bind more loosely than the addition, and + // because the result is spliced into whatever position the original held. + let incremented = format_compact!("({text}) + 1"); + let decremented = format_compact!("({text}) - 1"); + + self.emit("expr.increment", span, incremented, 0); + self.emit("expr.decrement", span, decremented, 1); + } + + /// Returns whether the source positively says an expression is a number. + /// + /// Without type resolution this can only read what the source wrote down, so it is an + /// allowlist rather than an attempt to rule out everything that is not a number. The default + /// answer is no. That default is measured rather than assumed: guessing yes wherever nothing + /// contradicts it makes three of every four perturbed mutants fail to compile, and those two + /// operators alone then account for more than three quarters of the unviable population. Every + /// withdrawal costs a share of a rollback round, which is a full rebuild of the instrumented + /// tree. + /// + /// The trade is real in both directions, which is why the inference beneath this is worth its + /// weight: a perturbation withheld on a value that was a number is a question never asked. The + /// answer is to widen what the source is read for -- parameters, annotated and inferred + /// locals, struct fields, casts, arithmetic, and methods whose names fix their return type -- + /// rather than to keep guessing where it says nothing at all. + /// + /// A literal answers no despite plainly being a number, because the literal family already + /// perturbs it and offering both would pay twice for one question. + fn is_known_numeric(&self, expression: &Expr) -> bool { + match expression { + // `x as usize` writes the type at the use site, which is as good as an annotation. + Expr::Cast(cast) => is_numeric_binding(&cast.ty), + + // `-x` is a number. `!x` is a bool or a bitwise complement, and `*x` may be anything. + Expr::Unary(unary) => matches!(unary.op, UnOp::Neg(_)), + + Expr::Binary(binary) => match binary.op { + // Nothing in wide use subtracts, multiplies, divides or takes a remainder of + // anything but a number. + BinOp::Sub(_) | BinOp::Mul(_) | BinOp::Div(_) | BinOp::Rem(_) => true, + + // `String + &str` is addition too, so this one operator has to look at what it is + // adding. Either side being a number settles it, because the two must agree. + BinOp::Add(_) => self.is_known_numeric(&binary.left) || self.is_known_numeric(&binary.right), + + _ => false, + }, + + Expr::Path(path) if path.qself.is_none() => { + // A constant is one of the most worthwhile things this family has to offer, and + // the screaming case tells `MAX` and `DEFAULT_SIZE` apart from `PhantomData` and + // `Ordering::Relaxed`, which are a unit struct and a variant. + let last = path.path.segments.last().is_some_and(|segment| { + let name = segment.ident.to_string(); + + // What the file declares outranks how the name is spelled, in both directions: + // a `const LIMIT: usize` is a number however it is reached, and a + // `const PREFIX: &str` is not one however loudly it is spelled. + self.constants.get(&name).copied().unwrap_or_else(|| is_constant_case(&name)) + }); + + last || path.path.get_ident().is_some_and(|ident| { + let name = ident.to_string(); + + self.bindings + .get(&name) + .copied() + .unwrap_or_else(|| self.numeric_uses.names.contains(&name)) + }) + } + + // A method whose name fixes its return type across the ecosystem says what it yields + // more reliably than any inference here could. + Expr::MethodCall(call) => returns_numeric(&call.method.to_string()), + + // `usize::from(..)`, `u64::try_from(..).unwrap()`: the type is written at the call + // site, so there is nothing to guess. + Expr::Call(call) => callee_type(&call.func).is_some_and(|name| is_numeric_type(&name)), + + // A field's type is written in the `struct` that declares it, which the pre-pass read. + Expr::Field(field) => match &field.member { + Member::Named(name) => { + let name = name.to_string(); + + self.fields + .get(&name) + .copied() + .unwrap_or_else(|| self.numeric_uses.fields.contains(&name)) + } + Member::Unnamed(_) => false, + }, + + Expr::Paren(paren) => self.is_known_numeric(&paren.expr), + + _ => false, + } + } + + /// Offers the perturbations for every argument of a call, unless the callee is one whose + /// arguments are a performance decision rather than a behavioural one. + fn perturb_arguments(&mut self, callee: Option<&str>, args: &Punctuated) { + if callee.is_some_and(is_capacity_call) { + return; + } + + for argument in args { + self.perturb(argument); + } + } + + /// Returns whether an item's attributes take it out of the population entirely. + /// + /// Two unrelated reasons land here: the item is test code, or it is behind a configuration + /// predicate that does not hold for this build. Both mean the same thing to the collector — + /// do not descend — so they are asked together at every place that can be entered. + fn skipped(&self, attrs: &[Attribute]) -> bool { + is_excluded(self.cfg, attrs) || !self.cfg.holds_for(attrs) + } + + /// Runs `body`, treating everything it visits as inert when `constant` holds. + /// + /// A guard is a function call, which const contexts disallow; mutants generated there would + /// never compile and end up withdrawn as noise instead of measuring anything. + fn in_const(&mut self, constant: bool, body: impl FnOnce(&mut Self) -> T) -> T { + if constant { self.inert(body) } else { body(self) } + } + + /// Runs `body` with type parameters that explicitly implement standard `Default` in scope. + fn with_defaulted_parameters(&mut self, generics: &Generics, body: impl FnOnce(&mut Self) -> T) -> T { + let depth = self.defaulted.len(); + let names = standard_defaulted_parameters(generics, &self.default_paths); + + self.defaulted.extend(names); + + let result = body(self); + + self.defaulted.truncate(depth); + result + } + + /// Runs `body` with the enclosing function's return type recorded. + /// + /// Restored rather than cleared afterwards, because a nested function inside a body must not + /// leave the outer one's `return` expressions looking like its own. + fn in_function(&mut self, sig: &Signature, body: impl FnOnce(&mut Self) -> T) -> T { + let outer = self.numeric_return; + let outer_error = self.foreign_error_return; + + self.numeric_return = is_numeric_return(&sig.output); + self.foreign_error_return = returns_undefaultable_error( + &sig.output, + &Types { + abstracts: &[], + imports: &self.imports, + defaults: self.defaults, + self_type: self.impl_self_type.as_ref(), + self_associated: Some(&self.impl_self_associated), + }, + ); + + // Saved and restored rather than cleared, because a function defined inside another one + // cannot see the outer function's locals and must not be allowed to reason from them. The + // maps are swapped out whole, so the undo entries the parameters and the body record refer + // to a map that is thrown away — they are discarded on the way out rather than replayed. + let mark = self.undo.len(); + let outer_bindings = core::mem::take(&mut self.bindings); + let outer_deferred = core::mem::take(&mut self.deferred); + + for input in &sig.inputs { + let FnArg::Typed(typed) = input else { + continue; + }; + + if let Pat::Ident(ident) = &*typed.pat { + self.bind(ident.ident.to_string(), is_numeric_binding(&typed.ty)); + } + } + + let result = body(self); + + self.bindings = outer_bindings; + self.deferred = outer_deferred; + self.undo.truncate(mark); + self.numeric_return = outer; + self.foreign_error_return = outer_error; + result + } + + /// Notes whether a `let` leaves its binding empty for a later assignment to settle. + /// + /// Declarations are seen in source order, so re-declaring a name with a value lifts an earlier + /// deferral, which is what shadowing a deferred binding means. + fn record_declaration(&mut self, local: &Local) { + let Some(name) = declared_name(&local.pat) else { + return; + }; + + // `insert`/`remove` report whether the name was already there, which is exactly the prior + // state the enclosing block has to be able to put back. + let was_present = if local.init.is_none() { + !self.deferred.insert(name.clone()) + } else { + self.deferred.remove(&name) + }; + + self.undo.push(Undo::Deferred(name, was_present)); + } + + /// Records a name's type evidence, remembering the value it displaces so the enclosing block + /// can put it back on the way out. + /// + /// The one place `bindings` is written during traversal, so routing every write through here is + /// what keeps the block's undo log complete. `true` means the source says the name holds a + /// number; absent and `false` are treated alike, so an overwrite of one by the other still has + /// to be recorded to be reversible. + fn bind(&mut self, name: String, numeric: bool) { + let prior = self.bindings.insert(name.clone(), numeric); + + self.undo.push(Undo::Binding(name, prior)); + } + + /// Runs `body` with the binding evidence scoped to one lexical block. + /// + /// A block *can* see its enclosing scope's locals — inheriting them is why the maps are not + /// cleared on entry — what it must not do is let its own outlive it. Without that a `let` + /// inside `{ … }` overwrote the evidence for a shadowed outer name and the overwrite survived + /// the block, so a later use of the outer binding was judged against the inner one's type. That + /// decides whether the perturbation mutants are viable, so a wrong answer either withholds + /// valid mutants or emits ones that cannot compile. + /// + /// `deferred` is scoped the same way and for the same reason. Note that it must be *inherited*: + /// the common shape is a `let scanned;` in an outer block settled by an assignment inside an + /// `if`, and the assignment is only passed over because the nested block can still see the + /// deferral its parent recorded. + /// + /// Rather than copy both maps on entry and move them back on exit, the block marks the undo log + /// and replays everything it recorded after that mark, in reverse. Reverse order is what makes a + /// name touched more than once come back to the value it held on entry rather than to an + /// intermediate one. + fn in_scope(&mut self, body: impl FnOnce(&mut Self) -> T) -> T { + let mark = self.undo.len(); + + let result = body(self); + + self.unwind(mark); + result + } + + /// Reverses every binding change recorded since `mark`, newest first, and forgets it. + fn unwind(&mut self, mark: usize) { + while self.undo.len() > mark { + match self + .undo + .pop() + .expect("the undo log length was checked before removing its final entry") + { + Undo::Binding(name, Some(prior)) => { + let _shadowed = self.bindings.insert(name, prior); + } + Undo::Binding(name, None) => { + let _dropped = self.bindings.remove(&name); + } + Undo::Deferred(name, true) => { + let _restored = self.deferred.insert(name); + } + Undo::Deferred(name, false) => { + let _dropped = self.deferred.remove(&name); + } + } + } + } + + fn inert(&mut self, body: impl FnOnce(&mut Self) -> T) -> T { + self.inert_depth += 1; + + let result = body(self); + + self.inert_depth -= 1; + result + } +} + +/// Returns whether any attribute suppresses mutation of the whole item. +fn is_excluded(cfg: &CfgSet, attrs: &[Attribute]) -> bool { + // `#[cfg(test)]` — and any compound gate that implies it, such as `all(test, unix)` — marks + // code that exists only to test other code. Mutating it measures the tests' tests, which + // nobody has. Read by the cfg subsystem's own classifier, so conditional `cfg_attr` gates are + // evaluated under the same target as ordinary `cfg` gates. + cfg.test_gated(attrs) +} + +/// Removes trivia from a trait path without changing literal contents. +/// +/// An item path feeds a stable mutant identity, so formatting `impl Trait < T > for S` must not +/// change it. A simple whitespace filter would also rewrite a literal in a const generic, so +/// literals and comments are stepped over with the source parser's lexer instead. +fn compact_path(text: &str) -> String { + let comments = crate::parse::comment_spans(text); + let mut comments = comments.iter().peekable(); + let mut at = 0; + let mut compact = String::with_capacity(text.len()); + + while at < text.len() { + if let Some(comment) = comments.peek() + && comment.start == at + { + at = comment.end; + let _next = comments.next(); + continue; + } + + if let Some(end) = crate::parse::literal_end(text, at) { + compact.push_str(text.get(at..end).unwrap_or_default()); + at = end; + continue; + } + + let character = text + .get(at..) + .and_then(|rest| rest.chars().next()) + .expect("the loop keeps the UTF-8 boundary below the text length"); + at += character.len_utf8(); + + if !character.is_whitespace() { + compact.push(character); + } + } + + compact +} + +#[expect( + clippy::renamed_function_params, + reason = "syn names every visitor parameter `i`, which says nothing about what it is" +)] +impl<'ast> Visit<'ast> for Collector<'_> { + /// Records the type of an annotated `let`, so that later uses of the name can be judged. + /// + /// Statements are visited in source order, so a name resolves to the most recent binding that + /// precedes the use, which is what shadowing means. A `let` with no annotation is left off the + /// record rather than guessed at. + fn visit_local(&mut self, node: &'ast Local) { + // A binding that is not in the build says nothing about the types later statements see, and + // recording it would shadow the one that is. + if self.skipped(&node.attrs) { + return; + } + + match &node.pat { + Pat::Type(typed) => { + if let Pat::Ident(ident) = &*typed.pat { + self.bind(ident.ident.to_string(), is_numeric_binding(&typed.ty)); + } + } + + // Most locals carry no annotation, so reading only the annotated ones left the great + // majority of names unknown. An initialiser this collector can already type answers + // the same question the annotation would have, and answers it for `let count = + // items.len();`, which is the shape the perturbation family meets most often. + Pat::Ident(ident) => { + if let Some(init) = node.init.as_ref().filter(|init| init.diverge.is_none()) + && self.is_known_numeric(&init.expr) + { + self.bind(ident.ident.to_string(), true); + } + } + + _ => {} + } + + visit::visit_local(self, node); + } + + fn visit_item_fn(&mut self, node: &'ast ItemFn) { + if self.skipped(&node.attrs) { + return; + } + + self.scoped(&node.sig.ident, |collector| { + collector.with_defaulted_parameters(&node.sig.generics, |collector| { + collector.function(&node.attrs, &node.sig, &node.block); + collector.in_function(&node.sig, |collector| { + collector.in_const(node.sig.constness.is_some(), |collector| { + visit::visit_item_fn(collector, node); + }); + }); + }); + }); + } + + fn visit_impl_item_fn(&mut self, node: &'ast ImplItemFn) { + if self.skipped(&node.attrs) { + return; + } + + self.scoped(&node.sig.ident, |collector| { + collector.with_defaulted_parameters(&node.sig.generics, |collector| { + collector.function(&node.attrs, &node.sig, &node.block); + collector.in_function(&node.sig, |collector| { + collector.in_const(node.sig.constness.is_some(), |collector| { + visit::visit_impl_item_fn(collector, node); + }); + }); + }); + }); + } + + fn visit_item_mod(&mut self, node: &'ast ItemMod) { + if self.skipped(&node.attrs) { + return; + } + + self.scoped(&node.ident, |collector| visit::visit_item_mod(collector, node)); + } + + fn visit_item_trait(&mut self, node: &'ast ItemTrait) { + if self.skipped(&node.attrs) { + return; + } + + self.scoped(&node.ident, |collector| { + collector.with_defaulted_parameters(&node.generics, |collector| visit::visit_item_trait(collector, node)); + }); + } + + fn visit_item_impl(&mut self, node: &'ast ItemImpl) { + if self.skipped(&node.attrs) { + return; + } + + let depth = self.generics.len(); + let defaulted_depth = self.defaulted.len(); + let outer = self.in_default_impl; + let outer_self_type = self.impl_self_type.replace((*node.self_ty).clone()); + let outer_associated = core::mem::replace( + &mut self.impl_self_associated, + node.items + .iter() + .filter_map(|item| match item { + ImplItem::Type(associated) => Some((associated.ident.to_string(), associated.ty.clone())), + _ => None, + }) + .collect(), + ); + + self.in_default_impl = node + .trait_ + .as_ref() + .is_some_and(|(_bang, path, _for)| self.default_paths.is_fallback_trait(path)); + + self.generics.extend(undefaulted_parameters(&node.generics, &self.default_paths)); + self.defaulted + .extend(standard_defaulted_parameters(&node.generics, &self.default_paths)); + let scope = self.impl_scope(node); + + self.scoped(scope, |collector| visit::visit_item_impl(collector, node)); + self.generics.truncate(depth); + self.defaulted.truncate(defaulted_depth); + self.in_default_impl = outer; + self.impl_self_type = outer_self_type; + self.impl_self_associated = outer_associated; + } + + fn visit_item_const(&mut self, node: &'ast ItemConst) { + self.inert(|collector| visit::visit_item_const(collector, node)); + } + + fn visit_item_static(&mut self, node: &'ast ItemStatic) { + self.inert(|collector| visit::visit_item_static(collector, node)); + } + + fn visit_impl_item_const(&mut self, node: &'ast ImplItemConst) { + self.inert(|collector| visit::visit_impl_item_const(collector, node)); + } + + fn visit_trait_item_const(&mut self, node: &'ast TraitItemConst) { + self.inert(|collector| visit::visit_trait_item_const(collector, node)); + } + + fn visit_macro(&mut self, node: &'ast Macro) { + // The expansion is not visible here and its spans do not map back onto the source, so + // nothing inside a macro is traversed. `vec![a, b, c]` is the exception worth making: the + // elements are written literally at the call site, so their spans are ordinary source + // spans and removing one is an ordinary splice. + // + // `vec![value; count]` is excluded for free, because it does not parse as a comma- + // separated list. Arrays are excluded deliberately: an array's length is part of its type, + // so dropping an element changes the type rather than the behavior. + if !node.path.is_ident("vec") { + return; + } + + let Ok(elements) = node.parse_body_with(Punctuated::::parse_terminated) else { + return; + }; + + // One element and the list would become empty, which is a different question — whether the + // collection is needed at all — and one that `Vec::new()` already asks of the function. + if elements.len() < 2 { + return; + } + + self.omit_elements(node, &elements); + } + + fn visit_attribute(&mut self, _node: &'ast Attribute) { + // Attributes are metadata, not behavior. This matters more than it sounds: a doc comment + // is desugared into `#[doc = "..."]`, so without this every line of documentation in the + // tree would present itself as a mutable string literal. + } + + fn visit_pat(&mut self, _node: &'ast Pat) { + // A pattern is matched against, not evaluated, so nothing in one can be guarded: a guard + // is an `if` expression and no expression is legal in pattern position. This matters + // because `syn` models a literal pattern as an `ExprLit`, so without this every `"skip" + // =>` match arm would offer itself as a mutable literal and produce a mutant that cannot + // compile. + } + + fn visit_expr_binary(&mut self, node: &'ast ExprBinary) { + let span = node.span(); + let assigns = is_assign_op(&node.op); + let mut operands = None; + + // A `&&` that is part of a let-chain is not an ordinary boolean operator: turning it into + // `||` is rejected by the parser, because a binding cannot escape one arm of an `or`. The + // binding may be on either side, as the chain associates to the left and a `let` commonly + // comes last. + let chains_a_binding = matches!(node.op, BinOp::And(_)) && (binds_a_pattern(&node.left) || binds_a_pattern(&node.right)); + + for (index, (mutator, operator)) in binary_replacements(&node.op).iter().enumerate() { + let index = u32::try_from(index).unwrap_or(u32::MAX); + + // Most binary expressions in a tree have no selected mutator, and the operand text is + // only needed to build a replacement, so it is read once and only on demand. + if chains_a_binding || !self.wants(mutator) { + continue; + } + + let (left, right) = *operands.get_or_insert_with(|| (self.text_of(node.left.span()), self.text_of(node.right.span()))); + + // The operands are parenthesized because the replacement is spliced in as a unit and + // must not renegotiate precedence with whatever encloses it: rewriting the `*` in + // `a + b * c` to `+` has to keep `b + c` grouped. The left side of a compound + // assignment is left alone, since it is a place expression and reads better bare. + let replacement = if assigns { + format_compact!("{left} {operator} ({right})") + } else { + format_compact!("({left}) {operator} ({right})") + }; + + self.emit(mutator, span, replacement, index); + } + + visit::visit_expr_binary(self, node); + } + + fn visit_expr_repeat(&mut self, node: &'ast ExprRepeat) { + // The length of `[0u8; 32]` is a const expression, so it cannot hold a guard, but the + // element expression can. + self.visit_expr(&node.expr); + self.inert(|collector| collector.visit_expr(&node.len)); + } + + fn visit_expr_reference(&mut self, node: &'ast ExprReference) { + // `fn f() -> &'static [&'static str] { &["a", "b"] }` compiles only because the borrowed + // array is a constant, which lets it be promoted to static storage. A guard is a function + // call, so instrumenting anything inside one stops it being constant, the array becomes an + // ordinary temporary, and the borrow no longer outlives the function. The result is a + // borrow-check error over the whole enclosing expression rather than at the mutated site. + self.in_const(is_promotable(&node.expr), |collector| { + visit::visit_expr_reference(collector, node); + }); + } + + fn visit_variant(&mut self, node: &'ast Variant) { + // An enum discriminant is a const expression. + self.inert(|collector| visit::visit_variant(collector, node)); + } + + fn visit_type(&mut self, node: &'ast Type) { + // Every expression reachable from inside a type is a const expression: the length in + // `[u8; 200]`, the argument in `Matrix<3, 3>`, and the same two nested arbitrarily deep in + // a field, a return type or a `where` clause. `visit_expr_repeat` covers `[0u8; 32]`, the + // *value*, and it is easy to assume that is the same thing — it is not, and the difference + // is a mutant that cannot compile in a position the rollback rounds then have to discover + // by building the whole tree. + self.inert(|collector| visit::visit_type(collector, node)); + } + + fn visit_generic_argument(&mut self, node: &'ast GenericArgument) { + // A const generic argument is a const expression wherever it stands, and `visit_type` + // only reaches the ones a type encloses. The turbofish on a path in expression position — + // `Foo::<{ N + 1 }>::BAR`, `g::<{ N + 1 }>()` — is not inside any type, so without this + // it is mutated, and a guard is a function call that no const context will evaluate. + if matches!(node, GenericArgument::Const(_)) { + self.inert(|collector| visit::visit_generic_argument(collector, node)); + return; + } + + visit::visit_generic_argument(self, node); + } + + fn visit_trait_item_fn(&mut self, node: &'ast TraitItemFn) { + if self.skipped(&node.attrs) { + return; + } + + self.scoped(&node.sig.ident, |collector| { + collector.with_defaulted_parameters(&node.sig.generics, |collector| { + if let Some(body) = node.default.as_ref() { + collector.function(&node.attrs, &node.sig, body); + } + + collector.in_function(&node.sig, |collector| { + collector.in_const(node.sig.constness.is_some(), |collector| { + visit::visit_trait_item_fn(collector, node); + }); + }); + }); + }); + } + + fn visit_block(&mut self, node: &'ast Block) { + self.in_scope(|collector| { + for statement in &node.stmts { + // A statement behind a predicate that does not hold is discarded after parsing, so + // it is not in the build being measured. Mutating it emits changes to text the + // compiler throws away: the crate builds, every test passes, and the mutant is + // scored as a survivor — telling the reader their tests miss a line that is not in + // their program, and inflating the denominator with mutants nothing could kill. + if collector.skipped(stmt_attrs(statement)) { + continue; + } + + collector.statement(statement); + + // Recorded as the block is judged rather than when `visit_local` later fires, because + // every statement here is examined before the traversal descends into any of them. A + // deferral noted during the descent would arrive after the assignment that settles it + // had already been accepted as a candidate. + if let Stmt::Local(local) = statement { + collector.record_declaration(local); + } + } + + // Descended into statement by statement rather than through `visit_block`, so that the + // configured-out ones are not entered either. + for statement in &node.stmts { + if collector.skipped(stmt_attrs(statement)) { + continue; + } + + visit::visit_stmt(collector, statement); + } + }); + } + + /// Leaves an expression the build does not contain unvisited. + /// + /// Statements are handled in `visit_block`, which never reaches this. This covers the expression + /// positions rustc admits an attribute in, and the ones it does not admit yet, so that enabling + /// `stmt_expr_attributes` in a crate under measurement does not silently reopen the gap. + fn visit_expr(&mut self, node: &'ast Expr) { + if self.skipped(expr_attrs(node)) { + return; + } + + visit::visit_expr(self, node); + } + + fn visit_expr_unary(&mut self, node: &'ast ExprUnary) { + // `*` is by far the most common unary operator and has no mutant, so the operand text is + // never read for one. + match node.op { + // Negating zero yields zero, so removing the negation changes nothing. + UnOp::Neg(_) if is_integer_zero_literal(&node.expr) => {} + UnOp::Neg(_) => self.emit("unary.remove_neg", node.span(), self.text_of(node.expr.span()), 0), + UnOp::Not(_) => self.emit("unary.remove_not", node.span(), self.text_of(node.expr.span()), 0), + _ => {} + } + + visit::visit_expr_unary(self, node); + } + + fn visit_expr_if(&mut self, node: &'ast ExprIf) { + self.condition("cond.negate", "cond.always_true", "cond.always_false", &node.cond); + + visit::visit_expr_if(self, node); + } + + fn visit_expr_while(&mut self, node: &'ast ExprWhile) { + // Only the negation, because a `while` forced to `true` never terminates and one forced to + // `false` is the loop deleted — the first costs a full timeout to reach a verdict already + // available more cheaply, and the second is what statement deletion asks. + if !binds_a_pattern(&node.cond) && self.wants("cond.negate") { + let span = node.cond.span(); + let negated = self.negation_of(span); + + self.emit("cond.negate", span, negated, 0); + } + + self.loops.push(LoopContext { + label: node.label.as_ref().map(|label| label.name.ident.to_string()), + produces_value: false, + }); + visit::visit_expr_while(self, node); + let _ = self.loops.pop(); + } + + fn visit_expr_for_loop(&mut self, node: &'ast ExprForLoop) { + self.loops.push(LoopContext { + label: node.label.as_ref().map(|label| label.name.ident.to_string()), + produces_value: false, + }); + visit::visit_expr_for_loop(self, node); + let _ = self.loops.pop(); + } + + fn visit_expr_loop(&mut self, node: &'ast ExprLoop) { + self.loops.push(LoopContext { + label: node.label.as_ref().map(|label| label.name.ident.to_string()), + produces_value: loop_produces_value(node), + }); + visit::visit_expr_loop(self, node); + let _ = self.loops.pop(); + } + + fn visit_expr_match(&mut self, node: &'ast ExprMatch) { + self.match_arms(node); + + visit::visit_expr_match(self, node); + } + + fn visit_expr_struct(&mut self, node: &'ast ExprStruct) { + self.struct_fields(node); + + visit::visit_expr_struct(self, node); + } + + fn visit_expr_range(&mut self, node: &'ast ExprRange) { + // A range with no end has no boundary to move, and one with no start is still a boundary + // worth moving, so only the end is required. + if let Some(end) = node.end.as_ref() { + let start = node.start.as_ref().map_or_else(CompactString::default, |start| { + let text = self.text_of(start.span()); + + format_compact!("({text})") + }); + + let end_text = self.text_of(end.span()); + + // The change is expressed by moving the endpoint rather than by swapping `..` for + // `..=`, even though swapping is what the mutation means. Every mutant here is run by + // wrapping the site as `if guard { mutant } else { original }`, and the two arms of an + // `if` must have the same type. `Range` and `RangeInclusive` are different types, so a + // literal swap cannot compile — not occasionally, but every single time, which would + // make the whole family a guaranteed build round spent to withdraw itself. + // + // `a..b + 1` covers exactly what `a..=b` covers and `a..=b - 1` covers exactly what + // `a..b` covers, so the question put to the suite is unchanged. On an unsigned + // endpoint that is already zero the subtraction overflows and the mutant is caught by + // the panic, which is the right answer for the wrong reason but still the right answer. + // + // Parenthesised for the same reason every other replacement here is: the operands are + // spliced back into an expression whose precedence we do not control. + match node.limits { + RangeLimits::HalfOpen(_) => { + self.emit( + "range.exclusive_to_inclusive", + node.span(), + format_compact!("{start}..(({end_text}) + 1)"), + 0, + ); + } + RangeLimits::Closed(_) => { + self.emit( + "range.inclusive_to_exclusive", + node.span(), + format_compact!("{start}..=(({end_text}) - 1)"), + 0, + ); + } + } + + self.perturb(end); + } + + if let Some(start) = node.start.as_ref() { + self.perturb(start); + } + + visit::visit_expr_range(self, node); + } + + fn visit_expr_break(&mut self, node: &'ast ExprBreak) { + // A labelled `break` may be leaving a labelled block rather than a loop, and `continue` + // cannot leave a block. A `break` carrying a value decides the type of its loop, which + // `continue` cannot supply. + if node.expr.is_none() && node.label.is_none() { + self.emit_shaped("loop.break_to_continue", node.span(), "continue", 0, Shape::Break); + } + + visit::visit_expr_break(self, node); + } + + fn visit_expr_continue(&mut self, node: &'ast ExprContinue) { + let target = node.label.as_ref().map(|label| label.ident.to_string()); + let requires_value = target.as_ref().map_or_else( + || self.loops.last().is_some_and(|context| context.produces_value), + |target| { + self.loops + .iter() + .rev() + .find(|context| context.label.as_ref() == Some(target)) + .is_some_and(|context| context.produces_value) + }, + ); + + if requires_value { + visit::visit_expr_continue(self, node); + return; + } + + // A label on a `continue` can only name a loop, so the same label is always valid on a + // `break`. + let replacement = node + .label + .as_ref() + .map_or_else(|| CompactString::new("break"), |label| format_compact!("break {label}")); + + self.emit_shaped("loop.continue_to_break", node.span(), replacement, 0, Shape::Continue); + + visit::visit_expr_continue(self, node); + } + + fn visit_expr_call(&mut self, node: &'ast ExprCall) { + self.perturb_arguments(callee_name(&node.func).as_deref(), &node.args); + + // `Some(v)`, `Ok(v)` and `Err(v)` decide, at the point it is decided, whether a value is + // present and whether an operation succeeded. Replacing a whole function can only ask that + // question a function at a time; this asks it at the site. + if node.args.len() == 1 { + match callee_name(&node.func).as_deref() { + Some("Some") => self.emit("option.some_to_none", node.span(), "None", 0), + Some("Ok") if !self.foreign_error_return => self.emit("result.ok_to_err", node.span(), "Err(Default::default())", 0), + Some("Err") => self.emit("result.err_to_ok", node.span(), "Ok(Default::default())", 0), + _ => {} + } + } + + visit::visit_expr_call(self, node); + } + + fn visit_expr_path(&mut self, node: &'ast syn::ExprPath) { + // A bare `None` in expression position. Patterns never reach here, because `visit_pat` + // stops the traversal before a pattern's interior is examined at all. + if node.path.is_ident("None") { + self.emit("option.none_to_some", node.span(), "Some(Default::default())", 0); + } + + visit::visit_expr_path(self, node); + } + + fn visit_expr_assign(&mut self, node: &'ast syn::ExprAssign) { + // What is assigned is what the rest of the function reads, so replacing it with the type's + // default asks whether anything downstream depends on the value rather than the write. + if !is_default_call(&node.right, &self.default_paths, &self.defaulted) { + self.emit("assign_value.default", node.right.span(), "Default::default()", 0); + } + + visit::visit_expr_assign(self, node); + } + + fn visit_expr_method_call(&mut self, node: &'ast ExprMethodCall) { + let method = node.method.to_string(); + + self.perturb_arguments(Some(method.as_str()), &node.args); + self.rename_method(node, &method); + + // The receiver is ordinary code and is traversed; the message is not descended into at all. + // Renaming the call itself is still offered above, because `expect` and `unwrap_or_default` + // differ in what the program does rather than in what it says on the way down. + if is_diagnostic_message(&method, node.args.len()) { + self.visit_expr(&node.receiver); + return; + } + + visit::visit_expr_method_call(self, node); + } + + fn visit_expr_index(&mut self, node: &'ast ExprIndex) { + self.perturb(&node.index); + + visit::visit_expr_index(self, node); + } + + fn visit_expr_return(&mut self, node: &'ast ExprReturn) { + // An integer-literal return value is left to the literal family, which already offers the + // same `+ 1` / `- 1` neighbours (and more) for it; perturbing it here as well would only + // duplicate those mutants. A non-literal value -- a variable, a call -- is not covered + // elsewhere, so it is still perturbed here. + if self.numeric_return + && let Some(value) = node.expr.as_ref() + && !matches!(&**value, Expr::Lit(ExprLit { lit: Lit::Int(_), .. })) + { + self.perturb_proven(value); + } + + visit::visit_expr_return(self, node); + } + + fn visit_expr_lit(&mut self, node: &'ast ExprLit) { + let span = node.span(); + + match &node.lit { + Lit::Int(value) => { + let digits = value.base10_digits(); + + // The perturbations go first so that where they collide with the value family on a + // small literal — `0` becoming `1` is both an increment and a "to one" — the name + // that survives is the one a reader checking a boundary is looking for. + if let Ok(parsed) = digits.parse::() { + // Checked, because the literal may be `i64::MAX`: wrapping would offer + // `i64::MIN` as an "increment", and the unchecked form panics in a debug build. + if let Some(incremented) = parsed.checked_add(1) { + self.emit("literal.int_increment", span, incremented.to_string(), 2); + } + + if !(digits == "0" && value.suffix().starts_with('u')) + && let Some(decremented) = parsed.checked_sub(1) + { + self.emit("literal.int_decrement", span, decremented.to_string(), 3); + } + } + + if digits != "0" { + self.emit("literal.int_to_zero", span, "0", 0); + } + + if digits != "1" { + self.emit("literal.int_to_one", span, "1", 1); + } + } + + Lit::Bool(value) => { + self.emit("literal.bool_flip", span, (!value.value).to_string(), 0); + } + + Lit::Str(value) => { + let text = value.value(); + + if !text.is_empty() { + self.emit("literal.str_to_empty", span, "\"\"", 0); + } + + // Replacing `"xyzzy"` with `"xyzzy"` is the original program. + if text != XYZZY { + self.emit("literal.str_to_xyzzy", span, "\"xyzzy\"", 1); + } + } + + _ => {} + } + + visit::visit_expr_lit(self, node); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/indexes.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/indexes.rs new file mode 100644 index 000000000..59e0df6b8 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/indexes.rs @@ -0,0 +1,501 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The per-file pre-pass the collector reads before it can judge an expression. + +use syn::visit::{self, Visit}; +use syn::{ + BinOp, Expr, ExprBinary, ExprForLoop, ExprIndex, ExprMethodCall, File, ImplItemConst, ItemConst, ItemStatic, ItemStruct, ItemUse, + Member, Pat, TraitItemConst, Type, UseTree, +}; + +use super::predicates::{is_int_literal, is_numeric_binding, is_numeric_receiver}; +use crate::ops::registry::Selection; +use crate::{HashMap, HashSet}; + +/// The names a file uses in a way only a number can be used. +#[derive(Default)] +pub(super) struct NumericUses { + /// Bare identifiers: locals, parameters, loop indices. + pub(super) names: HashSet, + + /// Field names, which stand in for the declarations the per-file pre-pass cannot reach. + pub(super) fields: HashSet, +} + +/// The per-file indexes the collector reads before it can judge an expression. +/// +/// Four questions, one descent. Each is answered by looking at a different kind of item — a +/// `struct`'s fields, a `use`, a constant's declared type, an expression that only a number could +/// appear in — so none of them depends on another's answer, and building them separately was four +/// walks of a syntax tree to learn things one walk can learn at once. +/// +/// Read before traversal rather than during it, because every one of these is used above the item +/// that establishes it at least as often as below: a field is read before the `struct` is declared, +/// a type before its `use`, a constant before its `const`. +pub(in crate::ops::collect) struct Indexes { + /// Whether each field name declared anywhere in this file holds a number. + pub(super) fields: HashMap, + + /// The module path each name this file imports was brought in from. + /// + /// `None` marks a name two `use` items disagree about, which is as unknown as never having + /// been imported. + pub(super) imports: HashMap>>, + + /// Names the file uses somewhere in a way only a number can be used. + pub(super) numeric_uses: NumericUses, + + /// Whether each constant and static declared anywhere in this file holds a number. + pub(super) constants: HashMap, +} + +/// Builds the indexes a selection actually consults, and skips the walk entirely when it consults +/// none of them. +/// +/// Three of the four exist only to decide whether an expression is a number, which only the +/// perturbation family asks; the fourth exists only to recognise a type that has no `Default`, +/// which only the `fn_value` family asks. A run narrowed to, say, the relational mutators asks +/// neither, and paying for the answers anyway is the whole of this cost. +/// Fills whichever indexes were asked for, ignoring scope. +pub(super) struct Walk { + indexes: Indexes, + + /// Whether the numeric evidence — fields, constants, uses — is wanted. + numeric: bool, + + /// Whether the import paths are wanted. + imports: bool, +} + +impl Walk { + /// Notes a name, when the expression is the bare identifier that proves it. + pub(super) fn note(&mut self, expression: &Expr) { + match expression { + Expr::Path(path) if path.qself.is_none() => { + if let Some(ident) = path.path.get_ident() { + let _added = self.indexes.numeric_uses.names.insert(ident.to_string()); + } + } + + // A field's own `struct` is very often in another file, which the pre-pass cannot + // see. How the field is used here is the only evidence available for those. + Expr::Field(field) => { + if let Member::Named(name) = &field.member { + let _added = self.indexes.numeric_uses.fields.insert(name.to_string()); + } + } + + Expr::Paren(paren) => self.note(&paren.expr), + Expr::Reference(reference) => self.note(&reference.expr), + _ => {} + } + } + + /// Records one constant's declaration, demoting a name two declarations disagree about. + pub(super) fn declared(&mut self, name: &str, ty: &Type) { + if !self.numeric { + return; + } + + let numeric = is_numeric_binding(ty); + + let _known = self + .indexes + .constants + .entry(name.to_owned()) + .and_modify(|known| *known = *known && numeric) + .or_insert(numeric); + } + + /// Records every name one `use` tree brings into scope, and where each came from. + pub(super) fn descend(&mut self, prefix: &mut Vec, tree: &UseTree) { + match tree { + UseTree::Path(path) => { + prefix.push(path.ident.to_string()); + self.descend(prefix, &path.tree); + let _popped = prefix.pop(); + } + + UseTree::Name(name) if name.ident == "self" => { + if let Some((binding, parent)) = prefix.split_last() { + self.imported(binding.clone(), parent); + } + } + + UseTree::Name(name) => self.imported(name.ident.to_string(), prefix), + + UseTree::Rename(rename) => self.imported(rename.rename.to_string(), prefix), + + UseTree::Group(group) => { + for item in &group.items { + self.descend(prefix, item); + } + } + + UseTree::Glob(_) => {} + } + } + + /// Records where one imported name came from, demoting a name two `use` items disagree about. + /// + /// The index is keyed by the bare name and spans the whole file, but a file may hold several + /// modules, and `use crate::Error` in one says nothing about `use std::io::Error` in another. + /// Letting the later `use` win made one module's answer depend on another's: a bare `Error` + /// resolved to whichever happened to be written last, so the wrong module either emitted + /// `Err(Default::default())` mutants that cannot compile or silently withheld valid ones. + /// + /// Demoting to `None` rather than picking a winner leaves the name exactly as unknown as one + /// that was never imported. That is a weaker answer than module scoping would give, and a + /// deliberately cheap one: the resulting guess is re-checked by the compiler, so being wrong + /// costs a withdrawn mutant rather than a wrong score. Importing the same path twice is not a + /// disagreement and does not demote. + fn imported(&mut self, name: String, prefix: &[String]) { + let _known = self + .indexes + .imports + .entry(name) + .and_modify(|known| { + if known.as_deref() != Some(prefix) { + *known = None; + } + }) + .or_insert_with(|| Some(prefix.to_vec())); + } + + /// The local update `visit_item_struct` makes, without its recursive continuation. + /// + /// Exposed with no continuation of its own so the fused phase-one pass (see + /// `collector::phase_one`) can drive this exact per-node logic from its own single traversal. + pub(super) fn on_item_struct(&mut self, node: &ItemStruct) { + if self.numeric { + for field in &node.fields { + let Some(name) = field.ident.as_ref() else { + continue; + }; + + let numeric = is_numeric_binding(&field.ty); + + // Two structs disagreeing about a name means neither answer can be trusted for + // a bare `x.count`, so the name is demoted to unknown rather than won by + // whichever was seen last. + let _known = self + .indexes + .fields + .entry(name.to_string()) + .and_modify(|known| *known = *known && numeric) + .or_insert(numeric); + } + } + } + + /// The local update `visit_item_use` makes, without its recursive continuation. + pub(super) fn on_item_use(&mut self, node: &ItemUse) { + if self.imports { + self.descend(&mut Vec::new(), &node.tree); + } + } + + /// The local update `visit_expr_binary` makes, without its recursive continuation. + pub(super) fn on_expr_binary(&mut self, node: &ExprBinary) { + if self.numeric { + match node.op { + // Nothing else in wide use subtracts, multiplies, divides or takes a remainder. + BinOp::Sub(_) + | BinOp::Mul(_) + | BinOp::Div(_) + | BinOp::Rem(_) + | BinOp::SubAssign(_) + | BinOp::MulAssign(_) + | BinOp::DivAssign(_) + | BinOp::RemAssign(_) => { + self.note(&node.left); + self.note(&node.right); + } + + // `String + &str` and `Ordering` comparisons make these two ambiguous on their + // own, so they count only against an integer literal, which fixes both sides. + BinOp::Add(_) | BinOp::AddAssign(_) | BinOp::Lt(_) | BinOp::Gt(_) | BinOp::Le(_) | BinOp::Ge(_) => { + if is_int_literal(&node.right) { + self.note(&node.left); + } + + if is_int_literal(&node.left) { + self.note(&node.right); + } + } + + _ => {} + } + } + } + + /// The local update `visit_expr_index` makes, without its recursive continuation. + pub(super) fn on_expr_index(&mut self, node: &ExprIndex) { + if self.numeric { + self.note(&node.index); + } + } + + /// The local update `visit_expr_method_call` makes, without its recursive continuation. + pub(super) fn on_expr_method_call(&mut self, node: &ExprMethodCall) { + if self.numeric && is_numeric_receiver(&node.method.to_string()) { + self.note(&node.receiver); + } + } + + /// The local update `visit_expr_for_loop` makes, without its recursive continuation. + pub(super) fn on_expr_for_loop(&mut self, node: &ExprForLoop) { + if self.numeric + && matches!(&*node.expr, Expr::Range(_)) + && let Pat::Ident(ident) = &*node.pat + { + let _added = self.indexes.numeric_uses.names.insert(ident.ident.to_string()); + } + } +} + +#[expect( + clippy::renamed_function_params, + reason = "syn names every visitor parameter `i`, which says nothing about what it is" +)] +impl<'ast> Visit<'ast> for Walk { + fn visit_item_struct(&mut self, node: &'ast ItemStruct) { + if self.numeric { + for field in &node.fields { + let Some(name) = field.ident.as_ref() else { + continue; + }; + + let numeric = is_numeric_binding(&field.ty); + + // Two structs disagreeing about a name means neither answer can be trusted for + // a bare `x.count`, so the name is demoted to unknown rather than won by + // whichever was seen last. + let _known = self + .indexes + .fields + .entry(name.to_string()) + .and_modify(|known| *known = *known && numeric) + .or_insert(numeric); + } + } + + visit::visit_item_struct(self, node); + } + + fn visit_item_use(&mut self, node: &'ast ItemUse) { + if self.imports { + self.descend(&mut Vec::new(), &node.tree); + } + + visit::visit_item_use(self, node); + } + + fn visit_item_const(&mut self, node: &'ast ItemConst) { + self.declared(&node.ident.to_string(), &node.ty); + + visit::visit_item_const(self, node); + } + + fn visit_item_static(&mut self, node: &'ast ItemStatic) { + self.declared(&node.ident.to_string(), &node.ty); + + visit::visit_item_static(self, node); + } + + fn visit_impl_item_const(&mut self, node: &'ast ImplItemConst) { + self.declared(&node.ident.to_string(), &node.ty); + + visit::visit_impl_item_const(self, node); + } + + fn visit_trait_item_const(&mut self, node: &'ast TraitItemConst) { + self.declared(&node.ident.to_string(), &node.ty); + + visit::visit_trait_item_const(self, node); + } + + fn visit_expr_binary(&mut self, node: &'ast ExprBinary) { + if self.numeric { + match node.op { + // Nothing else in wide use subtracts, multiplies, divides or takes a remainder. + BinOp::Sub(_) + | BinOp::Mul(_) + | BinOp::Div(_) + | BinOp::Rem(_) + | BinOp::SubAssign(_) + | BinOp::MulAssign(_) + | BinOp::DivAssign(_) + | BinOp::RemAssign(_) => { + self.note(&node.left); + self.note(&node.right); + } + + // `String + &str` and `Ordering` comparisons make these two ambiguous on their + // own, so they count only against an integer literal, which fixes both sides. + BinOp::Add(_) | BinOp::AddAssign(_) | BinOp::Lt(_) | BinOp::Gt(_) | BinOp::Le(_) | BinOp::Ge(_) => { + if is_int_literal(&node.right) { + self.note(&node.left); + } + + if is_int_literal(&node.left) { + self.note(&node.right); + } + } + + _ => {} + } + } + + visit::visit_expr_binary(self, node); + } + + fn visit_expr_index(&mut self, node: &'ast ExprIndex) { + if self.numeric { + self.note(&node.index); + } + + visit::visit_expr_index(self, node); + } + + fn visit_expr_method_call(&mut self, node: &'ast ExprMethodCall) { + if self.numeric && is_numeric_receiver(&node.method.to_string()) { + self.note(&node.receiver); + } + + visit::visit_expr_method_call(self, node); + } + + fn visit_expr_for_loop(&mut self, node: &'ast ExprForLoop) { + if self.numeric + && matches!(&*node.expr, Expr::Range(_)) + && let Pat::Ident(ident) = &*node.pat + { + let _added = self.indexes.numeric_uses.names.insert(ident.ident.to_string()); + } + + visit::visit_expr_for_loop(self, node); + } +} + +pub(super) fn indexes(file: &File, selection: &Selection) -> Indexes { + let mut walk = Walk::new(selection); + + if !walk.numeric && !walk.imports { + return walk.indexes; + } + + walk.visit_file(file); + walk.indexes +} + +impl Walk { + /// Builds an empty index set, gated exactly as [`indexes`] gates its own walk. + /// + /// Exposed so the fused phase-one pass (`collector::phase_one`) can build the same starting + /// state `indexes` would, drive it through one combined traversal instead of `indexes`'s own, + /// and read the result back out with [`Walk::into_indexes`]. + pub(super) fn new(selection: &Selection) -> Self { + Self { + indexes: Indexes { + fields: HashMap::default(), + imports: HashMap::default(), + numeric_uses: NumericUses::default(), + constants: HashMap::default(), + }, + numeric: selection.contains("expr.increment") || selection.contains("expr.decrement"), + + // Two families read this one, not one: `fn_value` to know whether a return type has a + // `Default` to reach for, and `result.ok_to_err` to know whether the `Err` it would + // write could be built at all. Missing the second cost five mutants when this gate was + // first written, which is what a gate on an index has to be checked against. + imports: selection.any_in_family("fn_value") || selection.contains("result.ok_to_err"), + } + } + + /// Consumes the walk, returning what it found. + pub(super) fn into_indexes(self) -> Indexes { + self.indexes + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn walk(numeric: bool, imports: bool) -> Walk { + Walk { + indexes: Indexes { + fields: HashMap::default(), + imports: HashMap::default(), + numeric_uses: NumericUses::default(), + constants: HashMap::default(), + }, + numeric, + imports, + } + } + + #[test] + fn note_tracks_named_fields_behind_references() { + let mut walk = walk(true, false); + let expression = syn::parse_str::("&record.count").expect("the field expression parses"); + + walk.note(&expression); + + assert!(walk.indexes.numeric_uses.fields.contains("count")); + } + + #[test] + fn declared_ignores_constants_when_numeric_index_is_disabled() { + let mut walk = walk(false, false); + let ty = syn::parse_str::("usize").expect("the numeric type parses"); + + walk.declared("COUNT", &ty); + + assert!(walk.indexes.constants.is_empty()); + } + + #[test] + fn descend_handles_groups_renames_and_globs() { + let mut walk = walk(false, true); + let item = syn::parse_str::("use crate::{Thing as Alias, inner::Item, *};").expect("the use item parses"); + + walk.descend(&mut Vec::new(), &item.tree); + + assert_eq!(walk.indexes.imports.get("Alias"), Some(&Some(vec!["crate".to_owned()]))); + assert_eq!( + walk.indexes.imports.get("Item"), + Some(&Some(vec!["crate".to_owned(), "inner".to_owned()])) + ); + assert_eq!(walk.indexes.imports.len(), 2); + } + + #[test] + fn indexes_collect_static_and_trait_constants_without_named_tuple_fields() { + let file = syn::parse_file( + r" + trait Limits { + const TRAIT_LIMIT: usize; + } + + static STATIC_LIMIT: usize = 1; + + struct Pair(usize, usize); + + fn note(limit: usize) { + let _ = 1 < limit; + } + ", + ) + .expect("the file parses"); + let selection = Selection::parse("expr.increment").expect("the numeric selector resolves"); + let indexes = indexes(&file, &selection); + + assert_eq!(indexes.constants.get("STATIC_LIMIT"), Some(&true)); + assert_eq!(indexes.constants.get("TRAIT_LIMIT"), Some(&true)); + assert!(indexes.fields.is_empty()); + assert!(indexes.numeric_uses.names.contains("limit")); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/noop.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/noop.rs new file mode 100644 index 000000000..389844290 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/noop.rs @@ -0,0 +1,428 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Whether a proposed replacement would leave the program as it was. + +use rustc_lexer::{LiteralKind, TokenKind}; + +use super::super::defaults::DefaultPaths; +use crate::ops::collect::Shape; + +/// Returns whether a replacement reproduces the code it would replace. +/// +/// Such a mutant is not a weak one, it is not a mutant at all: the compiled program is unchanged, +/// so no test can distinguish it and it survives every suite that will ever be written. Left in, it +/// is charged for like any other mutant and then reported as a survivor, which is an accusation +/// against tests that had nothing to answer for. +/// +/// It arises whenever a function already returns one of the values the `fn_value` family offers — +/// `fn ready() -> bool { true }` is the everyday case, and `{ 0 }`, `{ None }` and `{ "" }` are the +/// others. The comparison is made on tokens rather than text so that layout, comments and the +/// braces around a body do not decide the answer; a body is stripped of its braces first, since +/// what replaces it is an expression rather than a block. +/// +/// Both sides must parse for the answer to be yes. A replacement this tool cannot itself tokenise +/// is not one it can claim to have recognised as a no-op. +/// +/// One shape is compared more closely than tokens allow. The reference family wraps its values in +/// `Box::leak(Box::new(...))`, and a body already written that way differs from the replacement +/// only in how the value inside spells its default: a type parameter explicitly bounded by the +/// standard `Default` trait can write `T::default()` instead of `Default::default()`. Those are +/// the same call, so the mutant is the original program under another name. +pub(super) fn is_noop(replacement: &str, original: &str, shape: Shape, defaults: &DefaultPaths, defaulted_types: &[String]) -> bool { + let original = if shape == Shape::Block || shape == Shape::IterBlock { + let trimmed = original.trim(); + + trimmed + .strip_prefix('{') + .and_then(|rest| rest.strip_suffix('}')) + .unwrap_or(original) + } else { + original + }; + + if same_tokens(replacement, original) { + return true; + } + + is_same_leak(replacement, original, defaults, defaulted_types) +} + +/// Returns whether two expressions are the same `Box::leak(Box::new(...))`, up to how the value +/// inside names its `Default`. +/// +/// Deliberately narrow: it answers for this one shape and nothing else, rather than pretending to +/// decide equivalence in general. +pub(super) fn is_same_leak(replacement: &str, original: &str, defaults: &DefaultPaths, defaulted_types: &[String]) -> bool { + let (Some(replacement), Some(original)) = (leaked_value(replacement), leaked_value(original)) else { + return false; + }; + + if is_default_call(replacement, defaults, defaulted_types) && is_default_call(original, defaults, defaulted_types) { + return true; + } + + same_tokens(replacement, original) +} + +/// The value a `Box::leak(Box::new(value))` leaks, or `None` for any other expression. +/// +/// A leading `&*` is stripped first. A shared reference is offered as `&*Box::leak(..)`, and +/// without this the reborrow would hide the shape from the no-op check — so a body that already +/// leaks a default would be handed a mutant that is the same program, and it would survive every +/// suite that will ever be written. +pub(super) fn leaked_value(text: &str) -> Option<&str> { + let tokens = lexemes(text)?; + let mut expression = strip_parentheses(&tokens); + + if expression.first()?.kind == TokenKind::And && expression.get(1)?.kind == TokenKind::Star { + expression = strip_parentheses(expression.get(2..)?); + } + + let leaked = call_argument(text, expression, "Box", "leak")?; + let tokens = lexemes(leaked)?; + + call_argument(leaked, strip_parentheses(&tokens), "Box", "new") +} + +#[derive(Clone, Copy, Debug)] +struct Lexeme<'a> { + kind: TokenKind, + text: &'a str, + start: usize, + end: usize, +} + +fn call_argument<'a>(text: &'a str, expression: &[Lexeme<'a>], qualifier: &str, name: &str) -> Option<&'a str> { + let expression = strip_parentheses(expression); + let open = expression.iter().position(|token| token.kind == TokenKind::OpenParen)?; + let (callee, call) = expression.split_at(open); + + if !path_ends_with(callee, qualifier, name) || call.last()?.kind != TokenKind::CloseParen { + return None; + } + + let mut depth = 0_usize; + for (index, token) in call.iter().enumerate() { + match token.kind { + TokenKind::OpenParen => depth = depth.checked_add(1)?, + TokenKind::CloseParen => { + depth = depth.checked_sub(1)?; + if depth == 0 && index + 1 != call.len() { + return None; + } + } + TokenKind::Comma if depth == 1 => return None, + _ => {} + } + } + + if depth != 0 || call.len() < 3 { + return None; + } + + let argument = call.get(1..call.len() - 1)?; + text.get(argument.first()?.start..argument.last()?.end) +} + +fn path_ends_with(path: &[Lexeme<'_>], qualifier: &str, name: &str) -> bool { + let Some(segments) = path_segments(path) else { + return false; + }; + + matches!(segments.as_slice(), [.., found_qualifier, found_name] if *found_qualifier == qualifier && *found_name == name) +} + +fn path_segments<'a>(path: &[Lexeme<'a>]) -> Option> { + let path = strip_parentheses(path); + let mut segments = Vec::new(); + let mut index = usize::from( + path.first().is_some_and(|token| token.kind == TokenKind::Colon) && path.get(1).is_some_and(|token| token.kind == TokenKind::Colon), + ) * 2; + + loop { + let token = path.get(index)?; + if !matches!(token.kind, TokenKind::Ident | TokenKind::RawIdent) { + return None; + } + segments.push(token.text); + index += 1; + + if index == path.len() { + return Some(segments); + } + if path.get(index)?.kind != TokenKind::Colon || path.get(index + 1)?.kind != TokenKind::Colon { + return None; + } + index += 2; + } +} + +fn strip_parentheses<'slice, 'text>(mut expression: &'slice [Lexeme<'text>]) -> &'slice [Lexeme<'text>] { + loop { + if expression.first().is_none_or(|token| token.kind != TokenKind::OpenParen) + || expression.last().is_none_or(|token| token.kind != TokenKind::CloseParen) + { + return expression; + } + + let mut depth = 0_usize; + let mut closes_at_end = false; + for (index, token) in expression.iter().enumerate() { + match token.kind { + TokenKind::OpenParen => depth += 1, + TokenKind::CloseParen => { + depth = depth.saturating_sub(1); + if depth == 0 { + closes_at_end = index + 1 == expression.len(); + break; + } + } + _ => {} + } + } + if !closes_at_end { + return expression; + } + expression = &expression[1..expression.len() - 1]; + } +} + +fn is_default_call(text: &str, defaults: &DefaultPaths, defaulted_types: &[String]) -> bool { + let Some(tokens) = lexemes(text) else { + return false; + }; + let expression = strip_parentheses(&tokens); + let Some(open) = expression.iter().position(|token| token.kind == TokenKind::OpenParen) else { + return false; + }; + let (callee, call) = expression.split_at(open); + if !matches!(call, [open, close] if open.kind == TokenKind::OpenParen && close.kind == TokenKind::CloseParen) { + return false; + } + let Some(segments) = path_segments(callee) else { + return false; + }; + let segments: Vec = segments.into_iter().map(str::to_owned).collect(); + + defaults.is_standard_default_segments(&segments) + || matches!(segments.as_slice(), [qualifier, method] if method == "default" && defaulted_types.contains(qualifier)) +} + +fn same_tokens(left: &str, right: &str) -> bool { + let (Some(left), Some(right)) = (lexemes(left), lexemes(right)) else { + return false; + }; + + if left.len() != right.len() + || !left + .iter() + .zip(&right) + .all(|(left, right)| left.kind == right.kind && left.text == right.text) + { + return false; + } + + left.windows(2).zip(right.windows(2)).all(|(left, right)| { + let left_first = &left[0]; + let left_second = &left[1]; + let right_first = &right[0]; + let right_second = &right[1]; + let spacing_matters = spacing_matters(left_first.kind, left_second.kind); + + !spacing_matters || (left_first.end == left_second.start) == (right_first.end == right_second.start) + }) +} + +fn lexemes(text: &str) -> Option>> { + let mut at = 0; + let mut lexemes = Vec::new(); + + for token in rustc_lexer::tokenize(text) { + let start = at; + at += token.len; + + match token.kind { + TokenKind::Whitespace | TokenKind::LineComment | TokenKind::BlockComment { terminated: true } => {} + kind if valid(kind) => lexemes.push(Lexeme { + kind, + text: text.get(start..at)?, + start, + end: at, + }), + _invalid => return None, + } + } + + Some(lexemes) +} + +const fn valid(kind: TokenKind) -> bool { + match kind { + TokenKind::Unknown | TokenKind::BlockComment { terminated: false } | TokenKind::Lifetime { starts_with_number: true } => false, + TokenKind::Literal { kind, .. } => valid_literal(kind), + _ => true, + } +} + +const fn valid_literal(kind: LiteralKind) -> bool { + matches!( + kind, + LiteralKind::Int { empty_int: false, .. } + | LiteralKind::Float { empty_exponent: false, .. } + | LiteralKind::Char { terminated: true } + | LiteralKind::Byte { terminated: true } + | LiteralKind::Str { terminated: true } + | LiteralKind::ByteStr { terminated: true } + | LiteralKind::RawStr { + started: true, + terminated: true, + .. + } + | LiteralKind::RawByteStr { + started: true, + terminated: true, + .. + } + ) +} + +const fn is_punctuation(kind: TokenKind) -> bool { + matches!( + kind, + TokenKind::Semi + | TokenKind::Comma + | TokenKind::Dot + | TokenKind::At + | TokenKind::Pound + | TokenKind::Tilde + | TokenKind::Question + | TokenKind::Colon + | TokenKind::Dollar + | TokenKind::Eq + | TokenKind::Not + | TokenKind::Lt + | TokenKind::Gt + | TokenKind::Minus + | TokenKind::And + | TokenKind::Or + | TokenKind::Plus + | TokenKind::Star + | TokenKind::Slash + | TokenKind::Caret + | TokenKind::Percent + ) +} + +const fn spacing_matters(left: TokenKind, right: TokenKind) -> bool { + if is_punctuation(left) && is_punctuation(right) { + return true; + } + + is_word(left) && is_word(right) +} + +const fn is_word(kind: TokenKind) -> bool { + matches!( + kind, + TokenKind::Ident | TokenKind::RawIdent | TokenKind::Literal { .. } | TokenKind::Lifetime { .. } + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn defaults() -> DefaultPaths { + DefaultPaths::of(&syn::parse_file("").expect("an empty file parses")) + } + + #[test] + fn layout_and_comments_do_not_change_token_equality() { + assert!(same_tokens("Some(1)", "Some /* note */ ( 1 )")); + } + + #[test] + fn punctuation_jointness_remains_semantic() { + assert!(!same_tokens("a >= b", "a > = b")); + assert!(!same_tokens("c\"text\"", "c \"text\"")); + } + + #[test] + fn repeated_noop_checks_do_not_advance_the_proc_macro_source_map() { + let marker = || { + "source_map_marker" + .parse::() + .expect("the marker tokenizes") + .into_iter() + .next() + .expect("the marker has one token") + .span() + .byte_range() + .start + }; + let first = marker(); + let second = marker(); + let expected_growth = second.saturating_sub(first); + let defaults = DefaultPaths::of(&syn::parse_file("").expect("an empty file parses")); + let defaulted = vec!["T".to_owned()]; + + for _candidate in 0..10_000 { + assert!(is_noop( + "&*Box::leak(Box::new(Default::default()))", + "{ Box::leak(Box::new(T::default())) }", + Shape::Block, + &defaults, + &defaulted, + )); + } + + assert_eq!(marker().saturating_sub(second), expected_growth); + } + + #[test] + fn leaked_values_and_call_arguments_reject_non_plain_calls() { + let leaked = "&*(Box::leak(Box::new((value))))"; + + assert_eq!(leaked_value(leaked), Some("(value)")); + + let extra = "Box::new(value)()"; + let extra_tokens = lexemes(extra).expect("the extra call tokenizes"); + assert_eq!(call_argument(extra, &extra_tokens, "Box", "new"), None); + + let multiple = "Box::new(value, other)"; + let multiple_tokens = lexemes(multiple).expect("the multi-argument call tokenizes"); + assert_eq!(call_argument(multiple, &multiple_tokens, "Box", "new"), None); + + let empty = "Box::new()"; + let empty_tokens = lexemes(empty).expect("the empty call tokenizes"); + assert_eq!(call_argument(empty, &empty_tokens, "Box", "new"), None); + } + + #[test] + fn default_call_checks_reject_invalid_shapes() { + let defaults = defaults(); + + assert!(!is_default_call("/*", &defaults, &[])); + assert!(!is_default_call("Default::default", &defaults, &[])); + assert!(!is_default_call("Default::default(value)", &defaults, &[])); + assert!(!is_default_call("1()", &defaults, &[])); + } + + #[test] + fn invalid_tokens_are_rejected_while_literals_remain_valid() { + assert!(!same_tokens("/*", "/*")); + assert!(lexemes("/*").is_none()); + + let invalid = rustc_lexer::tokenize("/*") + .next() + .expect("the unterminated block comment yields one token") + .kind; + assert!(!valid(invalid)); + + #[rustfmt::skip] + let literal_kind = rustc_lexer::tokenize("1").find_map(|token| match token.kind { TokenKind::Literal { kind, .. } => Some(kind), _ => None }).expect("an integer tokenizes as a literal"); + let valid_literal_fn: fn(LiteralKind) -> bool = valid_literal; + assert!(valid_literal_fn(literal_kind)); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/phase_one.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/phase_one.rs new file mode 100644 index 000000000..2da01463a --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/phase_one.rs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Fusing the stated-value audit and the numeric/import indexes into one syntax-tree walk. +//! +//! [`stated::check`](super::super::stated::check) and [`indexes`](super::indexes::indexes) each +//! drive their own [`syn::visit::Visit`] over the same file, and the [`Collector`](super::Collector) +//! that follows them drives a third. The two pre-passes visit completely disjoint sets of node +//! kinds — `Audit` reads attributes and function-like items, `Walk` reads structs, `use`s, +//! constants and a handful of numeric-looking expressions — so nothing about combining them into +//! one walk changes what either one sees or in what order it sees it: every visit method below is +//! exactly the local update the corresponding standalone type already made, run in the same +//! recursive descent, just not paying for that descent twice. +//! +//! [`super::defaults::DefaultPaths`] deliberately stays out of this fusion. It never was a +//! recursive [`syn::visit::Visit`] walk — [`DefaultPaths::of`](super::defaults::DefaultPaths::of) +//! is a single pass over `file.items` — so folding it in here would not remove a traversal, only +//! move an already-cheap one. + +use syn::visit::{self, Visit}; +use syn::{ + Attribute, ExprBinary, ExprForLoop, ExprIndex, ExprMethodCall, ImplItemConst, ImplItemFn, ItemConst, ItemFn, ItemStatic, ItemStruct, + ItemUse, TraitItemConst, TraitItemFn, +}; + +use super::super::stated::{self, Audit}; +use super::indexes::{Indexes, Walk}; +use crate::Result; +use crate::ops::registry::Selection; +use crate::parse::SourceFile; + +/// Runs the stated-value audit and the numeric/import indexes in the same walk over a file, then +/// reports the audit's fault exactly as [`stated::check`](super::super::stated::check) would. +/// +/// Returns the indexes only when there is no fault to report, matching the order the two passes +/// already ran in at their one call site: the stated-value check has always run, and had to fail +/// the whole file, before the indexes it built were of any use to a collector that would never run. +pub(in crate::ops::collect) fn run(file: &SourceFile, selection: &Selection) -> Result { + let mut combined = PhaseOne { + audit: Audit::default(), + walk: Walk::new(selection), + }; + + combined.visit_file(&file.ast); + + stated::fault(file, combined.audit)?; + + Ok(combined.walk.into_indexes()) +} + +/// The combined visitor: one [`Audit`] and one [`Walk`], driven by a single recursive descent. +/// +/// Holds both sub-visitors' state rather than merging their fields into one type, so each keeps +/// exactly the fields, invariants and standalone tests it already had; only the traversal itself is +/// shared. +struct PhaseOne { + audit: Audit, + walk: Walk, +} + +#[expect( + clippy::renamed_function_params, + reason = "syn names every visitor parameter `i`, which says nothing about what it is" +)] +impl<'ast> Visit<'ast> for PhaseOne { + fn visit_attribute(&mut self, node: &'ast Attribute) { + self.audit.on_attribute(node); + visit::visit_attribute(self, node); + } + + fn visit_item_fn(&mut self, node: &'ast ItemFn) { + self.audit.on_item_fn(node); + visit::visit_item_fn(self, node); + } + + fn visit_impl_item_fn(&mut self, node: &'ast ImplItemFn) { + self.audit.on_impl_item_fn(node); + visit::visit_impl_item_fn(self, node); + } + + fn visit_trait_item_fn(&mut self, node: &'ast TraitItemFn) { + self.audit.on_trait_item_fn(node); + visit::visit_trait_item_fn(self, node); + } + + fn visit_item_struct(&mut self, node: &'ast ItemStruct) { + self.walk.on_item_struct(node); + visit::visit_item_struct(self, node); + } + + fn visit_item_use(&mut self, node: &'ast ItemUse) { + self.walk.on_item_use(node); + visit::visit_item_use(self, node); + } + + fn visit_item_const(&mut self, node: &'ast ItemConst) { + self.walk.declared(&node.ident.to_string(), &node.ty); + visit::visit_item_const(self, node); + } + + fn visit_item_static(&mut self, node: &'ast ItemStatic) { + self.walk.declared(&node.ident.to_string(), &node.ty); + visit::visit_item_static(self, node); + } + + fn visit_impl_item_const(&mut self, node: &'ast ImplItemConst) { + self.walk.declared(&node.ident.to_string(), &node.ty); + visit::visit_impl_item_const(self, node); + } + + fn visit_trait_item_const(&mut self, node: &'ast TraitItemConst) { + self.walk.declared(&node.ident.to_string(), &node.ty); + visit::visit_trait_item_const(self, node); + } + + fn visit_expr_binary(&mut self, node: &'ast ExprBinary) { + self.walk.on_expr_binary(node); + visit::visit_expr_binary(self, node); + } + + fn visit_expr_index(&mut self, node: &'ast ExprIndex) { + self.walk.on_expr_index(node); + visit::visit_expr_index(self, node); + } + + fn visit_expr_method_call(&mut self, node: &'ast ExprMethodCall) { + self.walk.on_expr_method_call(node); + visit::visit_expr_method_call(self, node); + } + + fn visit_expr_for_loop(&mut self, node: &'ast ExprForLoop) { + self.walk.on_expr_for_loop(node); + visit::visit_expr_for_loop(self, node); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/predicates.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/predicates.rs new file mode 100644 index 000000000..580a117bf --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/predicates.rs @@ -0,0 +1,802 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Syntactic questions the collector asks about one node. + +use syn::visit::Visit; +use syn::{ + Attribute, BinOp, Block, Expr, ExprBlock, ExprBreak, ExprForLoop, ExprGroup, ExprLit, ExprLoop, ExprParen, ExprUnsafe, ExprWhile, + GenericArgument, Label, Lit, Macro, Pat, Path, PathArguments, ReturnType, Stmt, Type, UnOp, +}; + +use super::super::defaults::DefaultPaths; +use super::values::{Kind, resolve_type, strip}; + +/// Returns a printable name for a type, used to build the enclosing item path for `impl` blocks. +pub(super) fn type_name(ty: &Type) -> String { + match ty { + Type::Path(path) => path + .path + .segments + .last() + .map_or_else(|| "_".to_owned(), |segment| segment.ident.to_string()), + + Type::Reference(reference) => type_name(&reference.elem), + _ => "_".to_owned(), + } +} + +/// Returns whether borrowing an expression relies on it being promoted to static storage. +/// +/// Promotion is a property of const-evaluability, which cannot be decided from syntax alone. What +/// is decidable is the shape that makes it plausible — an aggregate built only from literals — and +/// that is the shape whose promotion a guard would silently take away. +pub(super) fn is_promotable(expression: &Expr) -> bool { + match expression { + // A path is either a constant, which is promotable, or a local, which is not a temporary + // in the first place. Neither holds a mutation site, so the answer costs nothing either way. + Expr::Lit(_) | Expr::Path(_) => true, + + Expr::Array(array) => array.elems.iter().all(is_promotable), + Expr::Tuple(tuple) => tuple.elems.iter().all(is_promotable), + Expr::Repeat(repeat) => is_promotable(&repeat.expr), + Expr::Reference(reference) => is_promotable(&reference.expr), + Expr::Unary(unary) => matches!(unary.op, UnOp::Neg(_)) && is_promotable(&unary.expr), + Expr::Paren(paren) => is_promotable(&paren.expr), + + _ => false, + } +} + +/// Returns whether a condition binds a pattern, either directly or as part of a `&&` chain. +/// +/// A `let` in condition position is not an expression that can be negated, replaced by a boolean, +/// or have its `&&` turned into `||`: all three are rejected by the parser or leave the bindings +/// the rest of the condition and the body depend on unbound. +pub(super) fn binds_a_pattern(condition: &Expr) -> bool { + match condition { + Expr::Let(_) => true, + Expr::Binary(binary) if matches!(binary.op, BinOp::And(_)) => binds_a_pattern(&binary.left) || binds_a_pattern(&binary.right), + Expr::Paren(paren) => binds_a_pattern(&paren.expr), + _ => false, + } +} + +/// Returns the value of a condition that is written as a boolean literal, seeing through grouping. +/// +/// Replacing `if true` with `if true` reproduces the original program, so the mutant can never be +/// killed and would sit in every report as a permanent survivor. +pub(super) fn boolean_literal(condition: &Expr) -> Option { + match condition { + Expr::Lit(ExprLit { lit: Lit::Bool(value), .. }) => Some(value.value), + Expr::Paren(paren) => boolean_literal(&paren.expr), + _ => None, + } +} + +/// Returns whether an expression is an integer literal equal to zero, seeing through grouping. +/// +/// Integer zero is its own negation, so dropping the `-` from `-0` leaves the program unchanged. +/// Floating-point zero is excluded because its sign is observable. +pub(super) fn is_integer_zero_literal(expression: &Expr) -> bool { + match expression { + Expr::Lit(ExprLit { lit: Lit::Int(value), .. }) => value.base10_digits() == "0", + Expr::Paren(paren) => is_integer_zero_literal(&paren.expr), + _ => false, + } +} + +/// Returns whether a `loop` has positive evidence that it produces a value. +/// +/// An unlabelled value-carrying `break` belongs to the innermost loop. A labelled one may cross +/// nested loops, so it is counted when it names this loop and no nested loop shadows the label. +/// Closures and nested items are not descended into because control flow cannot cross them. +pub(super) fn loop_produces_value(node: &ExprLoop) -> bool { + let mut visitor = ValueBreak { + label: node.label.as_ref(), + nested_loops: 0, + shadowed: 0, + found: false, + }; + + visitor.visit_block(&node.body); + visitor.found +} + +struct ValueBreak<'a> { + label: Option<&'a Label>, + nested_loops: usize, + shadowed: usize, + found: bool, +} + +impl ValueBreak<'_> { + fn enters(&mut self, label: Option<&Label>, body: impl FnOnce(&mut Self)) { + let shadows = self.label.is_some_and(|root| label.is_some_and(|nested| nested.name == root.name)); + + self.nested_loops += 1; + self.shadowed += usize::from(shadows); + body(self); + self.shadowed -= usize::from(shadows); + self.nested_loops -= 1; + } +} + +#[expect( + clippy::renamed_function_params, + reason = "syn names every visitor parameter `i`, which says nothing about what it is" +)] +impl<'ast> Visit<'ast> for ValueBreak<'_> { + fn visit_expr_break(&mut self, node: &'ast ExprBreak) { + if node.expr.is_some() + && (node.label.is_none() && self.nested_loops == 0 + || self.shadowed == 0 + && self + .label + .is_some_and(|root| node.label.as_ref().is_some_and(|label| label.ident == root.name.ident))) + { + self.found = true; + } + } + + fn visit_expr_loop(&mut self, node: &'ast ExprLoop) { + self.enters(node.label.as_ref(), |visitor| visitor.visit_block(&node.body)); + } + + fn visit_expr_for_loop(&mut self, node: &'ast ExprForLoop) { + self.enters(node.label.as_ref(), |visitor| visitor.visit_block(&node.body)); + } + + fn visit_expr_while(&mut self, node: &'ast ExprWhile) { + self.enters(node.label.as_ref(), |visitor| visitor.visit_block(&node.body)); + } + + fn visit_expr_closure(&mut self, _node: &'ast syn::ExprClosure) {} + + fn visit_item(&mut self, _node: &'ast syn::Item) {} +} + +/// Returns whether the source positively says an expression is text rather than a number. +/// +/// The mirror of `is_known_numeric`, and it exists because that one has to answer from evidence +/// that is sometimes circumstantial — a screaming-case name, a field name some other struct +/// declared, a name used as an index once elsewhere in the file. Text is the case where the source +/// says so outright, and `+ 1` against it is `E0369` every time: a build that is thrown away, a +/// share of a rollback round paid, and nothing measured. +/// +/// An allowlist again, and deliberately a short one. A plain path whose type is a caller's struct +/// is not decidable here and is not attempted; what is decidable is a value the source constructed +/// as text on the spot. +pub(super) fn is_textual(expression: &Expr) -> bool { + match expression { + Expr::Lit(literal) => matches!(literal.lit, Lit::Str(_) | Lit::ByteStr(_) | Lit::Char(_) | Lit::Byte(_)), + + // `format!`, `concat!` and `stringify!` have exactly one result type between them. + Expr::Macro(macro_call) => macro_call + .mac + .path + .segments + .last() + .is_some_and(|segment| matches!(segment.ident.to_string().as_str(), "format" | "concat" | "stringify")), + + // `min`, `max` and `clamp` are `Ord` methods, so they return whatever the receiver was — + // which is the one way a method call reaches here still holding text after + // `is_known_numeric` accepted it for the name alone. + Expr::MethodCall(call) => { + let method = call.method.to_string(); + + returns_text(&method) || (matches!(method.as_str(), "min" | "max" | "clamp") && is_textual(&call.receiver)) + } + + // `String::from(..)`, `str::to_owned(..)`: the type is written at the call site. + Expr::Call(call) => callee_type(&call.func).is_some_and(|name| matches!(name.as_str(), "String" | "str" | "OsString" | "PathBuf")), + + // `String + &str` is addition, so one textual side makes the whole sum textual — which is + // the case `is_known_numeric` cannot see, since it accepts a sum either side of which + // looked like a number. + Expr::Binary(binary) => matches!(binary.op, BinOp::Add(_)) && (is_textual(&binary.left) || is_textual(&binary.right)), + + Expr::Paren(paren) => is_textual(&paren.expr), + Expr::Reference(reference) => is_textual(&reference.expr), + + _ => false, + } +} + +/// Returns whether a method's name fixes its return type as text across the ecosystem. +/// +/// `to_string` is the one that matters and the rest are its neighbours: each is a method whose name +/// is so strongly associated with producing a string that a receiver of some other type would be a +/// surprise. Nothing here is a method a number has. +pub(super) fn returns_text(method: &str) -> bool { + matches!( + method, + "to_string" + | "to_owned" + | "to_uppercase" + | "to_lowercase" + | "to_ascii_uppercase" + | "to_ascii_lowercase" + | "to_string_lossy" + | "into_string" + | "concat" + | "join" + | "repeat" + | "escape_debug" + | "escape_default" + ) +} + +/// Returns whether an expression is an integer literal, which fixes the type of whatever it meets. +pub(super) fn is_int_literal(expression: &Expr) -> bool { + match expression { + Expr::Lit(literal) => matches!(literal.lit, Lit::Int(_)), + Expr::Paren(paren) => is_int_literal(&paren.expr), + _ => false, + } +} + +/// Returns whether a method exists only on numbers, so that having a receiver proves it is one. +/// +/// `min`, `max` and `clamp` are deliberately absent even though they return a number: they are +/// `Ord` methods, so any two comparable values have them and a receiver proves nothing. +pub(super) fn is_numeric_receiver(method: &str) -> bool { + matches!( + method, + "saturating_add" + | "saturating_sub" + | "saturating_mul" + | "wrapping_add" + | "wrapping_sub" + | "wrapping_mul" + | "checked_add" + | "checked_sub" + | "checked_mul" + | "checked_div" + | "abs" + | "signum" + | "pow" + | "rem_euclid" + | "div_euclid" + | "count_ones" + | "count_zeros" + | "leading_zeros" + | "trailing_zeros" + | "is_power_of_two" + | "next_power_of_two" + | "to_le_bytes" + | "to_be_bytes" + ) +} + +pub(super) fn is_constant_case(name: &str) -> bool { + name.chars().any(|character| character.is_ascii_uppercase()) && !name.chars().any(char::is_lowercase) +} + +/// Returns whether a type's name says its associated functions produce a number. +/// +/// Written out rather than derived, because these are the only names for which `usize::from(..)` +/// and its kind can be read off the call site without resolving anything. +pub(super) fn is_numeric_type(name: &str) -> bool { + matches!( + name, + "u8" | "u16" + | "u32" + | "u64" + | "u128" + | "usize" + | "i8" + | "i16" + | "i32" + | "i64" + | "i128" + | "isize" + | "f32" + | "f64" + | "NonZeroU8" + | "NonZeroU16" + | "NonZeroU32" + | "NonZeroU64" + | "NonZeroUsize" + ) +} + +/// Returns whether a method's name is enough on its own to know it yields a number. +/// +/// Only names whose meaning is fixed across the ecosystem are listed, and only ones that yield a +/// bare number rather than an `Option` or a `Result` wrapping one -- `checked_add` is absent for +/// that reason, and `clone` because what it yields depends entirely on its receiver. +pub(super) fn returns_numeric(method: &str) -> bool { + matches!( + method, + "len" + | "count" + | "capacity" + | "abs" + | "signum" + | "pow" + | "min" + | "max" + | "clamp" + | "saturating_add" + | "saturating_sub" + | "saturating_mul" + | "wrapping_add" + | "wrapping_sub" + | "wrapping_mul" + | "as_millis" + | "as_micros" + | "as_nanos" + | "as_secs" + | "subsec_millis" + | "subsec_nanos" + | "elapsed_secs" + | "leading_zeros" + | "trailing_zeros" + | "count_ones" + | "count_zeros" + ) +} + +/// Returns whether a callee's arguments describe how much room to set aside rather than what the +/// program should do. +/// +/// Perturbing one of these produces a mutant that changes only an allocation strategy. A test +/// suite that caught it would be a test suite pinning an implementation detail, so reporting it as +/// a survivor accuses the tests of a gap they should not be asked to fill. +pub(super) fn is_capacity_call(name: &str) -> bool { + matches!( + name, + "with_capacity" | "with_capacity_in" | "reserve" | "reserve_exact" | "try_reserve" | "try_reserve_exact" | "shrink_to" + ) +} + +/// Returns whether a call's argument is a message a person reads after the program has failed. +/// +/// `expect` and `expect_err` take one argument and it is never behavior: it is what the panic says, +/// read only once the program has already given up. Rewriting it changes what a crash prints, not +/// what the program does, so the mutant is unkillable by any test worth writing — killing one means +/// asserting the exact wording of a panic message, which pins phrasing that should stay free to +/// improve and turns a typo fix into a failing suite. +/// +/// The panicking macros need no such rule: nothing inside a macro is traversed at all, because the +/// expansion's spans do not map back onto the source. So `assert!(x, "...")` and `panic!("...")` +/// are already exempt, and this is what is left. +/// +/// Everything the argument is built from is exempt with it. A message assembled by a `format!` or +/// a helper call is still a message, and the reasoning does not change with its shape. +pub(super) fn is_diagnostic_message(name: &str, arity: usize) -> bool { + arity == 1 && matches!(name, "expect" | "expect_err") +} + +/// Returns whether a pattern accepts every value the arms above it left over. +/// +/// `_` is the obvious spelling. A bare binding — `other => ...` — is the other one: it names the +/// value instead of discarding it, but it accepts every value just the same, so a match ending in +/// one stays exhaustive when an earlier arm is stopped from matching. +/// +/// A binding is only recognised when it is spelled the way a binding is: one identifier in value +/// case, with no subpattern. `None`, `MAX` and every other unit variant or constant parse as the +/// same node and match exactly one value, so taking those for catch-alls would leave the match +/// non-exhaustive and the mutant unable to compile. +pub(super) fn is_catch_all(pat: &Pat) -> bool { + match pat { + Pat::Wild(_) => true, + Pat::Ident(binding) => binding.subpat.is_none() && is_binding_case(&binding.ident.to_string()), + Pat::Paren(paren) => is_catch_all(&paren.pat), + _ => false, + } +} + +/// Returns whether an identifier is spelled the way a binding is. +/// +/// Bindings are `snake_case` and constants and unit variants are not, so the leading character +/// tells them apart. Anything that does not start lower — `None`, `MAX`, `Ordering` — is taken for +/// a pattern that matches one value rather than all of them, which is the safe way to be wrong: +/// the mutant is withheld instead of failing to compile. +pub(super) fn is_binding_case(name: &str) -> bool { + name.starts_with(|first: char| first == '_' || first.is_lowercase()) +} + +/// Returns whether an expression is already a standard `Default::default()` call. +/// +/// Replacing it with itself would be a mutant no test could ever detect, which would be reported +/// as a survivor and read as an accusation against the suite for something it cannot do. A final +/// segment named `default` is not enough: inherent methods and custom traits can use that name, +/// and their calls are real behavior a default-value mutant must still test. +pub(super) fn is_default_call(expression: &Expr, defaults: &DefaultPaths, defaulted_types: &[String]) -> bool { + match expression { + Expr::Call(call) if call.args.is_empty() => { + callee_path(&call.func).is_some_and(|path| is_standard_default_callee(path, defaults, defaulted_types)) + } + Expr::Paren(paren) => is_default_call(&paren.expr, defaults, defaulted_types), + _ => false, + } +} + +/// Returns the path directly called by an expression, seeing through redundant parentheses. +fn callee_path(callee: &Expr) -> Option<&Path> { + match callee { + Expr::Path(path) => Some(&path.path), + Expr::Paren(paren) => callee_path(&paren.expr), + _ => None, + } +} + +/// Returns whether a `default` method path selects the standard trait. +fn is_standard_default_callee(path: &Path, defaults: &DefaultPaths, defaulted_types: &[String]) -> bool { + let Some(method) = path.segments.last() else { + return false; + }; + + if method.ident != "default" || path.segments.len() < 2 { + return false; + } + + if defaults.is_standard_default_callee(path) { + return true; + } + + if path.segments.len() == 2 { + let qualifier = path.segments.first().expect("the path length was checked to contain two segments"); + let qualifier = qualifier.ident.to_string(); + + return defaulted_types.iter().any(|name| name == &qualifier); + } + + false +} + +/// Returns whether an expression is a call to one of those functions. +/// +/// Their arguments are excluded because perturbing them says nothing; their *results* are excluded +/// because a call that reserves room returns the collection, never a number. +pub(super) fn is_capacity_result(expression: &Expr) -> bool { + match expression { + Expr::Call(call) => callee_name(&call.func).is_some_and(|name| is_capacity_call(&name)), + Expr::MethodCall(call) => is_capacity_call(&call.method.to_string()), + Expr::Paren(paren) => is_capacity_result(&paren.expr), + _ => false, + } +} + +/// Returns whether a signature promises a number, which is the only thing worth perturbing by one. +pub(super) fn is_numeric_return(output: &ReturnType) -> bool { + let ReturnType::Type(_arrow, ty) = output else { + return false; + }; + + matches!(resolve_type(ty), Kind::Signed | Kind::Unsigned | Kind::Float) +} + +/// Returns whether an expression's value is unreachable because control leaves before it is used. +/// +/// A divergent expression has type `!`, which coerces to the numeric return type and so slips past +/// the signature-only proof that a tail is a number. Perturbing it is pointless: `(return 5) + 1` +/// returns before the `+ 1` ever runs, so the mutant behaves exactly like the original and survives +/// every test. Only the cases the source states outright are reported. This never tries to prove a +/// value *is* reachable, so anything it is unsure about answers `false` and is perturbed as before. +pub(super) fn diverges(expression: &Expr) -> bool { + match expression { + // Control leaves the function, or the enclosing loop, outright. + Expr::Return(_) | Expr::Break(_) | Expr::Continue(_) => true, + + // The standard never-returning macros: a `panic!`, `unreachable!`, `todo!` or + // `unimplemented!` in tail position is the body's value only in the type system's bookkeeping. + Expr::Macro(node) => is_diverging_macro(&node.mac), + + // A `loop` with no `break` runs forever; one with a `break` may yield a value. + Expr::Loop(node) => !has_break(&node.body), + + // Wrappers that add no control flow carry the divergence of what they hold. + Expr::Paren(ExprParen { expr, .. }) | Expr::Group(ExprGroup { expr, .. }) => diverges(expr), + Expr::Block(ExprBlock { block, .. }) | Expr::Unsafe(ExprUnsafe { block, .. }) => block_diverges(block), + + // An `if` diverges only when it cannot fall through: it needs an `else`, and both the `then` + // block and the `else` branch must diverge in turn. + Expr::If(node) => node + .else_branch + .as_ref() + .is_some_and(|(_else, otherwise)| block_diverges(&node.then_branch) && diverges(otherwise)), + + // A `match` diverges when it has arms and every one of them does. A match on an empty type + // has no arm to reach here as a numeric tail, so the emptiness check only guards the logic. + Expr::Match(node) => !node.arms.is_empty() && node.arms.iter().all(|arm| diverges(&arm.body)), + + _ => false, + } +} + +/// Returns whether a block's value is unreachable because its final statement diverges. +fn block_diverges(block: &Block) -> bool { + match block.stmts.last() { + Some(Stmt::Expr(expr, _semi)) => diverges(expr), + Some(Stmt::Macro(node)) => is_diverging_macro(&node.mac), + _ => false, + } +} + +/// Returns whether a macro is one of the standard never-returning ones. +fn is_diverging_macro(mac: &Macro) -> bool { + mac.path.segments.last().is_some_and(|segment| { + matches!( + segment.ident.to_string().as_str(), + "panic" | "unreachable" | "todo" | "unimplemented" + ) + }) +} + +/// Returns whether a loop body contains any `break`, anywhere within it. +/// +/// Deliberately conservative: a `break` that belongs to a nested loop still counts, so a loop is +/// only called breakless when it truly has none. Erring this way keeps the divergence test from +/// ever rejecting a loop that might produce a value. +fn has_break(block: &Block) -> bool { + let mut finder = BreakFinder { found: false }; + + finder.visit_block(block); + finder.found +} + +/// Records whether any `break` was seen while walking a loop body. +struct BreakFinder { + found: bool, +} + +impl<'ast> Visit<'ast> for BreakFinder { + fn visit_expr_break(&mut self, _node: &'ast ExprBreak) { + self.found = true; + } +} + +/// Returns a type's `index`th generic argument, so `Result` can be asked for either side. +pub(super) fn payload(ty: &Type, index: usize) -> Option<&Type> { + let Type::Path(path) = strip(ty) else { + return None; + }; + + let PathArguments::AngleBracketed(args) = &path.path.segments.last()?.arguments else { + return None; + }; + + args.args + .iter() + .filter_map(|arg| match arg { + GenericArgument::Type(ty) => Some(ty), + _ => None, + }) + .nth(index) +} + +/// Returns the single name a `let` pattern introduces, if it introduces exactly one. +/// +/// A `let` with no initialiser can only bind a bare name, optionally typed — a tuple or struct +/// pattern has nothing to destructure until a value arrives, so the compiler rejects it. Looking +/// through `Pat::Type` is therefore enough, and anything more elaborate is not a deferred binding. +pub(super) fn declared_name(pat: &Pat) -> Option { + match pat { + Pat::Type(typed) => declared_name(&typed.pat), + Pat::Ident(ident) if ident.subpat.is_none() => Some(ident.ident.to_string()), + _ => None, + } +} + +/// Returns whether a written-down type is one that `+ 1` applies to. +/// +/// References are peeled first: `&usize + 1` compiles, so treating a reference as a non-number +/// would throw away a mutant that builds and runs. +pub(super) fn is_numeric_binding(ty: &Type) -> bool { + match ty { + Type::Reference(reference) => is_numeric_binding(&reference.elem), + + _ => matches!(resolve_type(ty), Kind::Signed | Kind::Unsigned | Kind::Float), + } +} + +/// Returns the type a called path is qualified by, which is the segment before the function name. +/// +/// `Vec::new` is qualified by `Vec` and `usize::from` by `usize`; a bare `helper()` is qualified by +/// nothing, and there is nothing to read off it. +pub(super) fn callee_type(callee: &Expr) -> Option { + match callee { + Expr::Path(path) => { + let mut segments = path.path.segments.iter().rev(); + let _last = segments.next()?; + + segments.next().map(|segment| segment.ident.to_string()) + } + + Expr::Paren(paren) => callee_type(&paren.expr), + _ => None, + } +} + +/// Returns the final path segment of a called expression, which is the function's own name. +/// +/// `Vec::with_capacity` and a bare `with_capacity` should be recognised as the same thing, and the +/// path in between says nothing the skip list needs. +pub(super) fn callee_name(callee: &Expr) -> Option { + match callee { + Expr::Path(path) => path.path.segments.last().map(|segment| segment.ident.to_string()), + Expr::Paren(paren) => callee_name(&paren.expr), + _ => None, + } +} + +/// Returns whether an operator is a compound assignment, whose left side is a place expression. +pub(super) const fn is_assign_op(op: &BinOp) -> bool { + matches!( + op, + BinOp::AddAssign(_) + | BinOp::SubAssign(_) + | BinOp::MulAssign(_) + | BinOp::DivAssign(_) + | BinOp::RemAssign(_) + | BinOp::BitAndAssign(_) + | BinOp::BitOrAssign(_) + | BinOp::BitXorAssign(_) + | BinOp::ShlAssign(_) + | BinOp::ShrAssign(_) + ) +} + +/// Returns whether a function's return type is syntactically a `Result`. +pub(super) fn returns_result(output: &ReturnType) -> bool { + matches!(output, ReturnType::Type(_, ty) if resolve_type(ty) == Kind::Result) +} + +/// The attributes written on one expression. +/// +/// `syn` gives every expression variant an `attrs` field but no way to reach it without naming the +/// variant, and the collector has to ask the question of an `Expr` it has not matched on. The +/// fallthrough is the empty slice, which reads as "nothing configured it out" — the fail-open answer +/// every other predicate here takes for a shape it cannot identify. +pub(super) fn expr_attrs(expression: &Expr) -> &[Attribute] { + macro_rules! attrs_of { + ($($variant:ident),+ $(,)?) => { + match expression { + $(Expr::$variant(node) => &node.attrs,)+ + _ => &[], + } + }; + } + + attrs_of!( + Array, Assign, Async, Await, Binary, Block, Break, Call, Cast, Closure, Const, Continue, Field, ForLoop, Group, If, Index, Infer, + Let, Lit, Loop, Macro, Match, MethodCall, Paren, Path, Range, Reference, Repeat, Return, Struct, Try, TryBlock, Tuple, Unary, + Unsafe, While, Yield, + ) +} + +/// The attributes written on one statement. +/// +/// A statement's attributes live on whatever it wraps, so `#[cfg(unix)] f();` and `#[cfg(unix)] let +/// x = 1;` put them in different places. Both mean the same thing about the statement. +pub(super) fn stmt_attrs(statement: &Stmt) -> &[Attribute] { + match statement { + Stmt::Local(local) => &local.attrs, + Stmt::Macro(mac) => &mac.attrs, + Stmt::Expr(expression, _semi) => expr_attrs(expression), + + // An item's own visitor asks the question already, and asking it here as well would need + // the same match over every item kind to no additional effect. + Stmt::Item(_) => &[], + } +} + +#[cfg(test)] +mod tests { + use syn::parse_quote; + + use super::*; + + fn default_paths(source: &str) -> DefaultPaths { + DefaultPaths::of(&syn::parse_file(source).expect("test source should parse")) + } + + /// The value-break visitor must walk nested loop forms but ignore closures and nested items, + /// because only the loop forms can carry a `break` back to the loop being classified. + #[test] + fn nested_loop_forms_are_visited_when_deciding_whether_a_loop_yields_a_value() { + let node: ExprLoop = parse_quote!('outer: loop { + loop { + break; + } + + while false {} + + for _ in 0..1 { + break 'outer 7; + } + + let _ignored = || 1; + + fn helper() {} + }); + + assert!(loop_produces_value(&node)); + } + + /// Textuality is decided only from syntax, so every syntax form that states "this is text" + /// needs a direct unit test rather than relying on wider collector behavior to reach it. + #[test] + fn literal_macro_method_and_reference_forms_can_all_be_textual() { + let literal: Expr = parse_quote!("hello"); + let macro_call: Expr = parse_quote!(format!("hello {}", 1)); + let text_method: Expr = parse_quote!("hello".to_string()); + let ord_method: Expr = parse_quote!(String::from("a").min(String::from("b"))); + let reference: Expr = parse_quote!(&"hello"); + + assert!(is_textual(&literal)); + assert!(is_textual(¯o_call)); + assert!(is_textual(&text_method)); + assert!(is_textual(&ord_method)); + assert!(is_textual(&reference)); + } + + #[test] + fn only_whitelisted_method_names_are_treated_as_returning_text() { + assert!(returns_text("escape_default")); + assert!(!returns_text("checked_add")); + } + + /// Standard `Default::default` detection must fail closed for malformed and non-default paths, + /// while still accepting explicit type-qualified defaults for the caller's own type parameter. + #[test] + fn default_callee_detection_rejects_non_paths_and_non_default_methods() { + let defaults = default_paths("use std::default::Default as StdDefault;"); + let defaulted_types = vec!["Thing".to_owned()]; + let wrong_callee: Expr = parse_quote!(value + 1); + let wrong_method: Path = parse_quote!(Thing::new); + let bare_defaulted: Path = parse_quote!(Thing::default); + let aliased_standard: Path = parse_quote!(StdDefault::default); + let empty = Path { + leading_colon: None, + segments: syn::punctuated::Punctuated::default(), + }; + + assert!(callee_path(&parse_quote!((StdDefault::default))).is_some()); + assert!(callee_path(&wrong_callee).is_none()); + assert!(!is_standard_default_callee(&empty, &defaults, &defaulted_types)); + assert!(!is_standard_default_callee(&wrong_method, &defaults, &defaulted_types)); + assert!(is_standard_default_callee(&aliased_standard, &defaults, &defaulted_types)); + assert!(is_standard_default_callee(&bare_defaulted, &defaults, &defaulted_types)); + } + + #[test] + fn capacity_and_callee_helpers_fall_back_for_non_call_expressions() { + let associated_call: Expr = parse_quote!(Vec::with_capacity(8)); + let method_call: Expr = parse_quote!(buffer.reserve(8)); + let path: Expr = parse_quote!(Vec::with_capacity); + let non_path: Expr = parse_quote!(buffer + 1); + + assert!(is_capacity_result(&associated_call)); + assert!(is_capacity_result(&method_call)); + assert_eq!(callee_type(&path), Some("Vec".to_owned())); + assert_eq!(callee_type(&non_path), None); + assert_eq!(callee_name(&path), Some("with_capacity".to_owned())); + assert_eq!(callee_name(&non_path), None); + } + + /// Divergence depends on the *last* statement only, so a macro statement tail and a non-tail + /// expression-less block must be distinguished directly. + #[test] + fn macro_statement_tails_diverge_but_non_expression_tails_do_not() { + let macro_block: Block = parse_quote!({ + panic!(); + }); + let local_tail: Block = parse_quote!({ + let _value = 1; + }); + + assert!(block_diverges(¯o_block)); + assert!(!block_diverges(&local_tail)); + } + + #[test] + fn payload_requires_a_path_type() { + let reference: Type = parse_quote!(&Result); + + assert!(payload(&reference, 0).is_none()); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/tables.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/tables.rs new file mode 100644 index 000000000..def7358ad --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/tables.rs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The curated substitutions a mutator chooses from. + +use syn::{BinOp, Expr}; + +/// The curated renames for a standard-library method, keyed by name and argument count. +/// +/// The argument count is not decoration. Without type resolution the only evidence that a `take` +/// is `Iterator::take` rather than `Option::take` or `Cell::take` is that it was given a count, +/// and swapping the second kind for `skip` would be applying a transformation nobody advertised. +/// Every entry here is a pair of standard-library methods with the same receiver, arity and +/// result *type*, which is a stricter requirement than it sounds. `take` and `skip` ask a genuine +/// question about a chain, but `Take` and `Skip` are different types, and a mutant shares an +/// `if` with the code it replaces, so that swap could never compile. The same rules out +/// `take_while`/`skip_while`. What is left are the methods whose two spellings agree on a type: +/// `bool`, `Option`, `String` and `&str`. +pub(super) fn method_renames(method: &str, arity: usize) -> Option<&'static [(&'static str, &'static str)]> { + let swaps: &'static [(&'static str, &'static str)] = match (method, arity) { + ("any", 1) => &[("iter.any_to_all", "all")], + ("all", 1) => &[("iter.all_to_any", "any")], + + // Zero arguments is `Iterator::min`; one is `Ord::min`. Both are a choice between the + // extremes, so both are worth swapping. + ("min", 0 | 1) => &[("iter.min_to_max", "max")], + ("max", 0 | 1) => &[("iter.max_to_min", "min")], + + ("first", 0) => &[("iter.first_to_last", "last")], + ("last", 0) => &[("iter.last_to_first", "first")], + + ("starts_with", 1) => &[("string.starts_with_to_ends_with", "ends_with")], + ("ends_with", 1) => &[("string.ends_with_to_starts_with", "starts_with")], + + ("to_lowercase", 0) => &[("string.lower_to_upper", "to_uppercase")], + ("to_uppercase", 0) => &[("string.upper_to_lower", "to_lowercase")], + ("to_ascii_lowercase", 0) => &[("string.lower_to_upper", "to_ascii_uppercase")], + ("to_ascii_uppercase", 0) => &[("string.upper_to_lower", "to_ascii_lowercase")], + + ("trim_start", 0) => &[("string.trim_start_to_trim_end", "trim_end")], + ("trim_end", 0) => &[("string.trim_end_to_trim_start", "trim_start")], + + _ => return None, + }; + + Some(swaps) +} + +/// The mutator for deleting an in-place ordering or deduplication call. +/// +/// These are the counterpart to the adapters above: because they return `()`, the only way to +/// remove one is to delete the whole statement, and the question they ask — does anything observe +/// that this collection was ordered? — is worth asking under its own name rather than folding it +/// into generic statement deletion. +pub(super) fn in_place_reorder(expression: &Expr) -> Option<&'static str> { + let Expr::MethodCall(call) = expression else { + return None; + }; + + match call.method.to_string().as_str() { + "sort" | "sort_by" | "sort_by_key" | "sort_unstable" | "sort_unstable_by" | "sort_unstable_by_key" => Some("iter.remove_sort"), + "dedup" | "dedup_by" | "dedup_by_key" => Some("iter.remove_dedup"), + _ => None, + } +} + +/// The mutators and replacement operators available for a binary operator. +pub(super) const fn binary_replacements(op: &BinOp) -> &'static [(&'static str, &'static str)] { + match op { + BinOp::Lt(_) => &[("relational.lt_to_le", "<="), ("relational.lt_to_gt", ">")], + BinOp::Le(_) => &[("relational.le_to_lt", "<"), ("relational.le_to_ge", ">=")], + BinOp::Gt(_) => &[("relational.gt_to_ge", ">="), ("relational.gt_to_lt", "<")], + BinOp::Ge(_) => &[("relational.ge_to_gt", ">"), ("relational.ge_to_le", "<=")], + BinOp::Eq(_) => &[("relational.eq_to_ne", "!=")], + BinOp::Ne(_) => &[("relational.ne_to_eq", "==")], + + BinOp::Add(_) => &[("arith.add_to_sub", "-"), ("arith.add_to_mul", "*")], + BinOp::Sub(_) => &[("arith.sub_to_add", "+"), ("arith.sub_to_div", "/")], + BinOp::Mul(_) => &[("arith.mul_to_div", "/"), ("arith.mul_to_add", "+")], + BinOp::Div(_) => &[("arith.div_to_mul", "*"), ("arith.div_to_rem", "%")], + BinOp::Rem(_) => &[("arith.rem_to_div", "/"), ("arith.rem_to_mul", "*")], + + BinOp::BitAnd(_) => &[("bitwise.and_to_or", "|"), ("bitwise.and_to_xor", "^")], + BinOp::BitOr(_) => &[("bitwise.or_to_and", "&")], + BinOp::BitXor(_) => &[("bitwise.xor_to_and", "&")], + BinOp::Shl(_) => &[("shift.shl_to_shr", ">>")], + BinOp::Shr(_) => &[("shift.shr_to_shl", "<<")], + + BinOp::And(_) => &[("logical.and_to_or", "||")], + BinOp::Or(_) => &[("logical.or_to_and", "&&")], + + BinOp::AddAssign(_) => &[("assign.add_to_sub", "-=")], + BinOp::SubAssign(_) => &[("assign.sub_to_add", "+=")], + BinOp::MulAssign(_) => &[("assign.mul_to_div", "/=")], + BinOp::DivAssign(_) => &[("assign.div_to_mul", "*=")], + BinOp::RemAssign(_) => &[("assign.rem_to_div", "/=")], + BinOp::BitAndAssign(_) => &[("assign.and_to_or", "|=")], + BinOp::BitOrAssign(_) => &[("assign.or_to_and", "&=")], + BinOp::BitXorAssign(_) => &[("assign.xor_to_and", "&=")], + BinOp::ShlAssign(_) => &[("assign.shl_to_shr", ">>=")], + BinOp::ShrAssign(_) => &[("assign.shr_to_shl", "<<=")], + + _ => &[], + } +} + +#[cfg(test)] +mod tests { + use syn::parse_quote; + + use super::*; + + #[test] + fn method_renames_depend_on_both_name_and_arity() { + assert_eq!(method_renames("min", 0), Some(&[("iter.min_to_max", "max")][..])); + assert_eq!(method_renames("min", 2), None); + assert_eq!( + method_renames("to_ascii_uppercase", 0), + Some(&[("string.upper_to_lower", "to_ascii_lowercase")][..]) + ); + } + + #[test] + fn in_place_reorder_distinguishes_sort_dedup_and_other_calls() { + let sort: Expr = parse_quote!(values.sort_unstable_by_key(key)); + let dedup: Expr = parse_quote!(values.dedup_by_key(key)); + let other: Expr = parse_quote!(values.reserve(8)); + let not_a_call: Expr = parse_quote!(values[0]); + + assert_eq!(in_place_reorder(&sort), Some("iter.remove_sort")); + assert_eq!(in_place_reorder(&dedup), Some("iter.remove_dedup")); + assert_eq!(in_place_reorder(&other), None); + assert_eq!(in_place_reorder(¬_a_call), None); + } + + #[test] + fn binary_replacements_cover_supported_operator_families() { + let relational: syn::ExprBinary = parse_quote!(left < right); + let arithmetic: syn::ExprBinary = parse_quote!(left + right); + let logical: syn::ExprBinary = parse_quote!(left && right); + let shift_assign: syn::ExprBinary = parse_quote!(left <<= right); + + assert_eq!( + binary_replacements(&relational.op), + &[("relational.lt_to_le", "<="), ("relational.lt_to_gt", ">")] + ); + assert_eq!( + binary_replacements(&arithmetic.op), + &[("arith.add_to_sub", "-"), ("arith.add_to_mul", "*")] + ); + assert_eq!(binary_replacements(&logical.op), &[("logical.and_to_or", "||")]); + assert_eq!(binary_replacements(&shift_assign.op), &[("assign.shl_to_shr", ">>=")]); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/tests.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/tests.rs new file mode 100644 index 000000000..d6445c5f9 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/tests.rs @@ -0,0 +1,1095 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use syn::punctuated::Punctuated; +use syn::visit::Visit; +use syn::{Expr, Item, Member, Stmt}; + +use super::super::{Defaults, collect_in}; +use super::{Collector, compact_path}; +use crate::cfg::CfgSet; +use crate::ops::registry::Selection; +use crate::parse::SourceFile; + +/// The mutators a source admits once its spans no longer index into its text. +/// +/// Every splice in this module refuses a node whose byte range reaches past the file, because +/// such a range means the node came from somewhere other than the text being edited and +/// splicing it would corrupt the source. Truncating the text after parsing puts the collector +/// in exactly that state for every span past the cut, which is the only way to reach those +/// guards from a unit test. +fn mutators_past_the_end(source: &str, keep: usize, ops: &str) -> Vec<&'static str> { + let mut file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse(ops).unwrap(); + + file.text.truncate(keep); + + collect_in(&file, &selection, &CfgSet::unconditional()) + .into_iter() + .map(|candidate| candidate.mutator) + .collect() +} + +/// The mutators a source admits under a selection, with the given predicates holding. +fn mutators(source: &str, ops: &str, cfg: &CfgSet) -> Vec<&'static str> { + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse(ops).unwrap(); + + collect_in(&file, &selection, cfg) + .into_iter() + .map(|candidate| candidate.mutator) + .collect() +} + +#[test] +fn a_wildcard_arm_that_is_configured_out_is_not_a_wildcard() { + // The arm is not in the program the compiler builds, so an earlier arm stopped from + // matching has nothing to fall through to and the mutant fails to compile as a + // non-exhaustive match, taking the whole run down with it. + let source = "fn f(x: bool) -> i32 { match x { true => 1, false => 2, #[cfg(not(unix))] _ => 3 } }"; + let found = mutators(source, "match_arm.never_matches", &CfgSet::parse("unix")); + + assert!(found.is_empty(), "{found:?}"); +} + +/// A statement the build does not contain is not mutated, and neither is anything inside it. +/// +/// The text is discarded after parsing, so a mutant there compiles away: the crate builds, every +/// test passes, and the mutant is scored as a survivor. The reader is told their tests miss a line +/// that is not in their program, and the denominator carries mutants nothing could ever kill. +#[test] +fn a_statement_that_is_configured_out_is_not_mutated() { + let source = "fn f() { #[cfg(not(unix))] g(1 + 2); }"; + let found = mutators(source, "stmt.delete_call,arith.add_to_sub", &CfgSet::parse("unix")); + + assert!(found.is_empty(), "{found:?}"); +} + +/// The same statement, in the build this time, is mutated as it always was. +#[test] +fn a_statement_that_is_configured_in_still_is_mutated() { + let source = "fn f() { #[cfg(unix)] g(1 + 2); }"; + let found = mutators(source, "stmt.delete_call,arith.add_to_sub", &CfgSet::parse("unix")); + + assert_eq!(found, vec!["stmt.delete_call", "arith.add_to_sub"]); +} + +#[test] +fn an_active_cfg_attr_that_removes_an_item_is_not_mutated() { + let source = "#[cfg_attr(unix, cfg(windows))] fn f() -> i32 { 1 + 2 }"; + let found = mutators(source, "arith.add_to_sub", &CfgSet::parse("unix").with_test()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn an_active_cfg_attr_that_adds_a_test_gate_is_not_mutated() { + let source = "#[cfg_attr(unix, cfg(test))] fn helper() -> i32 { 1 + 2 }"; + let found = mutators(source, "arith.add_to_sub", &CfgSet::parse("unix").with_test()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_test_cfg_attr_is_not_mutated_when_test_cfg_differs_between_targets() { + let source = "#[cfg_attr(test, test)] fn helper() -> i32 { 1 + 2 }"; + let found = mutators(source, "arith.add_to_sub", &CfgSet::parse("unix").with_test()); + + assert!(found.is_empty(), "{found:?}"); +} + +/// A `let` the build does not contain is not mutated, and does not shadow the binding that is. +#[test] +fn a_local_that_is_configured_out_is_not_mutated() { + let source = "fn f() { #[cfg(not(unix))] let x = 1 + 2; }"; + let found = mutators(source, "arith.add_to_sub", &CfgSet::parse("unix")); + + assert!(found.is_empty(), "{found:?}"); +} + +/// The whole of a configured-out statement goes, including the blocks nested inside it. +#[test] +fn a_configured_out_statement_takes_its_nested_blocks_with_it() { + let source = "fn f(p: bool) { #[cfg(not(unix))] if p { g(1 + 2); } }"; + let found = mutators(source, "arith.add_to_sub,cond.negate,stmt.delete_call", &CfgSet::parse("unix")); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_wildcard_arm_that_is_configured_in_still_is_one() { + let source = "fn f(x: bool) -> i32 { match x { true => 1, false => 2, #[cfg(unix)] _ => 3 } }"; + let found = mutators(source, "match_arm.never_matches", &CfgSet::parse("unix")); + + assert_eq!(found, vec!["match_arm.never_matches", "match_arm.never_matches"]); +} + +#[test] +fn a_suppressed_wildcard_arm_still_counts_as_present() { + // Suppression withholds a mutant, it does not delete code: the arm is compiled and the + // match is exhaustive because of it, so the arms above it stay mutable. + let source = "fn f(x: bool) -> i32 { match x { true => 1, #[gamma::skip(match_arm.never_matches)] _ => 3 } }"; + let found = mutators(source, "match_arm.never_matches", &CfgSet::parse("unix")); + + assert_eq!(found, vec!["match_arm.never_matches"]); +} + +#[test] +fn a_binding_catch_all_receives_what_falls_through() { + let source = "fn f(x: i32) -> i32 { match x { 1 => 10, other => other } }"; + let found = mutators(source, "match_arm.never_matches", &CfgSet::unconditional()); + + assert_eq!(found, vec!["match_arm.never_matches"]); +} + +#[test] +fn a_unit_variant_arm_is_not_a_catch_all() { + // `None` parses as a binding but matches one value, so relying on it would leave the + // mutated match non-exhaustive. + let source = "fn f(x: Option) -> i32 { match x { Some(n) => n, None => 0 } }"; + let found = mutators(source, "match_arm.never_matches", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_leaked_default_is_not_offered_to_a_body_that_already_leaks_one() { + // `Box::leak(Box::new(T::default()))` and the replacement's + // `Box::leak(Box::new(Default::default()))` are the same call written two ways, so the + // mutant is the original program and no test could ever kill it. + let source = "fn f() -> &'static T { Box::leak(Box::new(T::default())) }"; + let found = mutators(source, "fn_value", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_leaked_value_is_still_offered_to_a_body_that_leaks_something_else() { + let source = "fn f() -> &'static T { Box::leak(Box::new(make())) }"; + let found = mutators(source, "fn_value", &CfgSet::unconditional()); + + assert!(!found.is_empty(), "{found:?}"); +} + +#[test] +fn a_parenthesized_condition_literal_is_still_the_literal_it_would_become() { + // `if (true)` replaced by `true` is the original program, so offering it would plant a + // survivor no test could ever kill. Seeing through the parentheses is what prevents that. + let source = "fn f() -> i32 { if (true) { 1 } else { 2 } }"; + let found = mutators(source, "cond.always_true,cond.always_false", &CfgSet::unconditional()); + + assert_eq!(found, vec!["cond.always_false"]); +} + +#[test] +fn a_parenthesized_zero_is_still_zero_when_negated() { + // Negating zero yields zero, so dropping the `-` changes nothing. + let source = "fn f() -> i32 { -(0) }"; + let found = mutators(source, "unary.remove_neg", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_parenthesized_default_call_is_still_a_default_call() { + let source = "fn f(x: &mut i32) { *x = (Default::default()); }"; + let found = mutators(source, "assign_value.default", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_parenthesized_callee_still_names_the_function_it_calls() { + let source = "fn f(x: &mut i32) { *x = (Default::default)(); }"; + let found = mutators(source, "assign_value.default", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn only_standard_zero_argument_default_calls_are_noops() { + let standard = [ + "fn f(mut n: i32) { n = core::default::Default::default(); }", + "use std::default::Default as StdDefault; fn f(mut n: i32) { n = StdDefault::default(); }", + "use std::default as defaults; fn f(mut n: i32) { n = defaults::Default::default(); }", + ]; + let bare_custom = "trait Default { fn default() -> i32; } struct Thing; impl Default for Thing { fn default() -> i32 { 0 } } fn f(mut n: i32) { n = Default::default(); }"; + let nonstandard = [ + "struct Default; impl Default { fn default() -> i32 { 0 } } fn f(mut n: i32) { n = (Default::default)(); }", + "struct Thing; impl Thing { fn default(_: i32) -> i32 { 0 } } fn f(mut n: i32) { n = Thing::default(1); }", + "mod custom { pub trait Default { fn default() -> i32; } } struct Thing; impl custom::Default for Thing { fn default() -> i32 { 0 } } fn f(mut n: i32) { n = custom::Default::default(); }", + "mod custom { pub trait Default { fn default() -> i32; } } use custom::Default as Alias; struct Thing; impl Alias for Thing { fn default() -> i32 { 0 } } fn f(mut n: i32) { n = Alias::default(); }", + ]; + + for source in standard { + let found = mutators(source, "assign_value.default", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{source}: {found:?}"); + } + + assert!( + mutators(bare_custom, "assign_value.default", &CfgSet::unconditional()).is_empty(), + "the replacement text is the same local Default call" + ); + + for source in nonstandard { + let found = mutators(source, "assign_value.default", &CfgSet::unconditional()); + + assert_eq!(found, vec!["assign_value.default"], "{source}: {found:?}"); + } +} + +#[test] +fn only_standard_default_implementations_suppress_recursive_replacements() { + let standard = "use core::default::Default as StdDefault; struct S; impl StdDefault for S { fn default() -> Self { S } }"; + let bare_custom = "trait Default { fn default() -> Self; } struct S; impl Default for S { fn default() -> Self { S } }"; + let custom = + "mod custom { pub trait Default { fn default() -> Self; } } struct S; impl custom::Default for S { fn default() -> Self { S } }"; + let aliased_custom = "mod custom { pub trait Default { fn default() -> Self; } } use custom::Default as Alias; struct S; impl Alias for S { fn default() -> Self { S } }"; + let aliased_standard_with_a_bare_custom_trait = "trait Default { fn default() -> Self; } use core::default::Default as StdDefault; struct S; impl Default for S { fn default() -> Self { S } } impl StdDefault for S { fn default() -> Self { S } }"; + + assert!(mutators(standard, "fn_value.default", &CfgSet::unconditional()).is_empty()); + assert!( + mutators(bare_custom, "fn_value.default", &CfgSet::unconditional()).is_empty(), + "the fallback's bare spelling would recurse through the local trait" + ); + + for source in [custom, aliased_custom] { + let found = mutators(source, "fn_value.default", &CfgSet::unconditional()); + + assert_eq!(found, vec!["fn_value.default"], "{source}: {found:?}"); + } + + assert_eq!( + mutators( + aliased_standard_with_a_bare_custom_trait, + "fn_value.default", + &CfgSet::unconditional() + ), + vec!["fn_value.default"] + ); +} + +#[test] +fn a_parenthesized_binding_pattern_still_catches_what_falls_through() { + let source = "fn f(x: i32) -> i32 { match x { 1 => 10, (other) => other } }"; + let found = mutators(source, "match_arm.never_matches", &CfgSet::unconditional()); + + assert_eq!(found, vec!["match_arm.never_matches"]); +} + +#[test] +fn a_parenthesized_return_type_still_resolves_to_the_type_inside() { + let source = "fn f() -> (i32) { g() }"; + let found = mutators(source, "fn_value", &CfgSet::unconditional()); + + assert_eq!(found, vec!["fn_value.minus_one", "fn_value.one", "fn_value.zero"]); +} + +#[test] +fn a_parenthesized_tuple_return_type_is_still_a_tuple() { + let source = "fn f() -> ((i32, u8)) { g() }"; + let found = mutators(source, "fn_value.tuple", &CfgSet::unconditional()); + + assert!(!found.is_empty(), "{found:?}"); +} + +#[test] +fn a_parenthesized_iterator_return_type_is_still_an_iterator() { + let source = "fn f() -> (impl Iterator) { core::iter::empty() }"; + let found = mutators(source, "fn_value", &CfgSet::unconditional()); + + assert!(!found.is_empty(), "{found:?}"); +} + +#[test] +fn a_parenthesized_reference_return_type_is_still_a_reference() { + let source = "fn f() -> (&'static mut T) { g() }"; + let found = mutators(source, "fn_value", &CfgSet::unconditional()); + + assert_eq!(found, vec!["fn_value.default"]); +} + +#[test] +fn only_standard_default_bounds_make_generic_fallbacks_available() { + let standard = "use core::default::Default as StdDefault; fn f() -> &'static mut T { g() }"; + let custom = "mod custom { pub trait Default {} } fn f() -> &'static mut T { g() }"; + let aliased_custom = "mod custom { pub trait Default {} } use custom::Default as Alias; fn f() -> &'static mut T { g() }"; + + assert_eq!(mutators(standard, "fn_value", &CfgSet::unconditional()), vec!["fn_value.default"]); + assert!(mutators(custom, "fn_value", &CfgSet::unconditional()).is_empty()); + assert!(mutators(aliased_custom, "fn_value", &CfgSet::unconditional()).is_empty()); +} + +#[test] +fn a_parenthesized_abstract_payload_is_still_abstract() { + // `impl Clone` has no value this tool can write down, so the `Some` case must stay + // withheld however many parentheses stand between the option and the trait. + let source = "fn f() -> Option<(impl Clone)> { g() }"; + let found = mutators(source, "fn_value", &CfgSet::unconditional()); + + assert_eq!(found, vec!["fn_value.none"]); +} + +#[test] +fn a_parenthesized_leak_is_still_the_leak_it_wraps() { + // Same reasoning as the unparenthesized case: the replacement would be the original. + let source = "fn f() -> &'static T { (Box::leak((Box::new(T::default())))) }"; + let found = mutators(source, "fn_value", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_parenthesized_name_is_still_the_name_that_proves_it_numeric() { + // The pre-pass reads the whole file for evidence that a name holds a number, and + // `(a) + 1` is the same evidence as `a + 1`. + let source = "fn f() { let a = g(); let _ = (a) + 1; h(a); }"; + let found = mutators(source, "expr.increment", &CfgSet::unconditional()); + + assert_eq!(found, vec!["expr.increment"]); +} + +#[test] +fn a_parenthesized_integer_literal_still_fixes_the_other_side() { + // `+` is ambiguous until one side is an integer literal, which `(1)` still is. + let source = "fn f() { let a = g(); let _ = a + (1); h(a); }"; + let found = mutators(source, "expr.increment", &CfgSet::unconditional()); + + assert_eq!(found, vec!["expr.increment"]); +} + +#[test] +fn a_parenthesized_callee_still_names_the_type_it_returns() { + // `(i32::from)(x)` yields a number, so the call is perturbed as well as its argument. + let source = "fn f(x: u8) { h((i32::from)(x)); }"; + let found = mutators(source, "expr.increment", &CfgSet::unconditional()); + + assert_eq!(found.len(), 2, "{found:?}"); +} + +#[test] +fn a_parenthesized_capacity_call_is_still_a_capacity_call() { + // A capacity is a request, not an answer: growing or shrinking one changes no result a + // test could read, so the perturbation is withheld however it is parenthesized. The + // signature is taken at its word here, which is what puts the call in numeric position. + let source = "fn f(n: usize) -> usize { (Vec::with_capacity(n)) }"; + let found = mutators(source, "expr.increment,expr.decrement", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_parenthesized_let_condition_still_binds_a_pattern() { + // A condition that binds cannot be replaced by a bare `true`, `false`, or negation + // without leaving the body without its binding, so no condition mutant is offered. + let source = "fn f(o: Option) { if (let Some(v) = o) { h(v); } }"; + let found = mutators(source, "cond", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +/// A rename mutant is named after the swap it performs, so the name and the edit have to agree. +/// +/// Asserting the mutator name alone would let the table pair `any` with `count` and still pass +/// — shipping a mutant whose name says one thing and whose edit does another. That is worse +/// than a missing mutant, because the user reads the name in the report and concludes something +/// about a test gap that does not exist. Each case therefore pins the exact replacement text, +/// which is the standard the `fn_value` family already meets. +#[test] +fn every_method_rename_replaces_the_call_its_name_promises() { + // The table pairs methods that agree on receiver, arity and result type. Each pair is + // listed here so that dropping an entry is a failure rather than a silent loss of reach. + let cases: &[(&str, &str, &str)] = &[ + ("fn f(v: V) -> bool { v.any(|x| x) }", "iter.any_to_all", "v.all(|x| x)"), + ("fn f(v: V) -> bool { v.all(|x| x) }", "iter.all_to_any", "v.any(|x| x)"), + ("fn f(v: V) -> O { v.min() }", "iter.min_to_max", "v.max()"), + ("fn f(a: A, b: A) -> A { a.min(b) }", "iter.min_to_max", "a.max(b)"), + ("fn f(v: V) -> O { v.max() }", "iter.max_to_min", "v.min()"), + ("fn f(a: A, b: A) -> A { a.max(b) }", "iter.max_to_min", "a.min(b)"), + ("fn f(v: V) -> O { v.first() }", "iter.first_to_last", "v.last()"), + ("fn f(v: V) -> O { v.last() }", "iter.last_to_first", "v.first()"), + ( + "fn f(s: S) -> bool { s.starts_with(\"a\") }", + "string.starts_with_to_ends_with", + "s.ends_with(\"a\")", + ), + ( + "fn f(s: S) -> bool { s.ends_with(\"a\") }", + "string.ends_with_to_starts_with", + "s.starts_with(\"a\")", + ), + ("fn f(s: S) -> S { s.to_lowercase() }", "string.lower_to_upper", "s.to_uppercase()"), + ("fn f(s: S) -> S { s.to_uppercase() }", "string.upper_to_lower", "s.to_lowercase()"), + ( + "fn f(s: S) -> S { s.to_ascii_lowercase() }", + "string.lower_to_upper", + "s.to_ascii_uppercase()", + ), + ( + "fn f(s: S) -> S { s.to_ascii_uppercase() }", + "string.upper_to_lower", + "s.to_ascii_lowercase()", + ), + ( + "fn f(s: S) -> S { s.trim_start() }", + "string.trim_start_to_trim_end", + "s.trim_end()", + ), + ( + "fn f(s: S) -> S { s.trim_end() }", + "string.trim_end_to_trim_start", + "s.trim_start()", + ), + ]; + + for (source, expected, replacement) in cases { + let file = SourceFile::parse("test.rs", (*source).to_owned()).expect("parses"); + let selection = Selection::parse(expected).expect("a selection"); + let found: Vec<(&str, String)> = collect_in(&file, &selection, &CfgSet::unconditional()) + .into_iter() + .map(|candidate| (candidate.mutator, candidate.replacement.to_string())) + .collect(); + + assert_eq!(found, vec![(*expected, (*replacement).to_owned())], "{source}"); + } +} + +#[test] +fn every_in_place_reorder_is_offered_for_the_call_it_names() { + let cases: &[(&str, &str)] = &[ + ("fn f(v: &mut V) { v.sort(); }", "iter.remove_sort"), + ("fn f(v: &mut V) { v.sort_by(g); }", "iter.remove_sort"), + ("fn f(v: &mut V) { v.sort_by_key(g); }", "iter.remove_sort"), + ("fn f(v: &mut V) { v.sort_unstable(); }", "iter.remove_sort"), + ("fn f(v: &mut V) { v.sort_unstable_by(g); }", "iter.remove_sort"), + ("fn f(v: &mut V) { v.sort_unstable_by_key(g); }", "iter.remove_sort"), + ("fn f(v: &mut V) { v.dedup(); }", "iter.remove_dedup"), + ("fn f(v: &mut V) { v.dedup_by(g); }", "iter.remove_dedup"), + ("fn f(v: &mut V) { v.dedup_by_key(g); }", "iter.remove_dedup"), + ]; + + for (source, expected) in cases { + let found = mutators(source, expected, &CfgSet::unconditional()); + + assert_eq!(found, vec![*expected], "{source}"); + } +} + +/// The synthesized values for a return type, as `mutator=replacement`, in a stable order. +fn values_for_return(ty: &str) -> Vec { + let source = format!("fn f() -> {ty} {{ g() }}"); + let file = SourceFile::parse("test.rs", source).unwrap(); + let selection = Selection::parse("fn_value").unwrap(); + + let mut values: Vec = collect_in(&file, &selection, &CfgSet::unconditional()) + .into_iter() + .map(|candidate| format!("{}={}", candidate.mutator, candidate.replacement)) + .collect(); + + values.sort(); + values +} + +#[test] +fn every_return_type_is_served_the_values_that_belong_to_it() { + // One row per kind the resolver can name. Both halves matter: the mutator says what + // question is being asked, and the replacement says whether the type was understood -- + // `&[u8]` and `&u8` agree on the former and disagree on the latter. + let cases: &[(&str, &[&str])] = &[ + ("()", &["fn_value.unit=()"]), + ("bool", &["fn_value.bool_false=false", "fn_value.bool_true=true"]), + ("i32", &["fn_value.minus_one=-1", "fn_value.one=1", "fn_value.zero=0"]), + ("u32", &["fn_value.one=1", "fn_value.zero=0"]), + ("f32", &["fn_value.minus_one=-1.0", "fn_value.one=1.0", "fn_value.zero=0.0"]), + ("f64", &["fn_value.minus_one=-1.0", "fn_value.one=1.0", "fn_value.zero=0.0"]), + ("&'static str", &["fn_value.empty_string=\"\"", "fn_value.xyzzy_string=\"xyzzy\""]), + // A string literal is `&'static str` and will not type-check where a mutable slice was + // promised, so a leaked boxed `str` stands in for it. + ( + "&'static mut str", + &[ + "fn_value.empty_string=Box::leak(String::new().into_boxed_str())", + "fn_value.xyzzy_string=Box::leak(String::from(\"xyzzy\").into_boxed_str())", + ], + ), + ( + "&mut str", + &[ + "fn_value.empty_string=Box::leak(String::new().into_boxed_str())", + "fn_value.xyzzy_string=Box::leak(String::from(\"xyzzy\").into_boxed_str())", + ], + ), + ( + "String", + &["fn_value.empty_string=String::new()", "fn_value.xyzzy_string=\"xyzzy\".to_owned()"], + ), + ( + "NonZeroU32", + &[ + "fn_value.one=NonZeroU32::new(1).unwrap()", + "fn_value.two=NonZeroU32::new(2).unwrap()", + ], + ), + // A slice reference has a `Default` of its own, so it is not leaked into being. + ("&'static [u8]", &["fn_value.default=Default::default()"]), + ( + "&'static u8", + &["fn_value.one=&*Box::leak(Box::new(1))", "fn_value.zero=&*Box::leak(Box::new(0))"], + ), + ( + "&'static mut u8", + &["fn_value.one=Box::leak(Box::new(1))", "fn_value.zero=Box::leak(Box::new(0))"], + ), + ( + "Vec", + &[ + "fn_value.empty_collection=Vec::new()", + "fn_value.one_element=core::iter::once(0).collect()", + "fn_value.one_element=core::iter::once(1).collect()", + ], + ), + ("Box", &["fn_value.one=Box::new(1)", "fn_value.zero=Box::new(0)"]), + ("Rc", &["fn_value.one=Rc::new(1)", "fn_value.zero=Rc::new(0)"]), + ("Arc", &["fn_value.one=Arc::new(1)", "fn_value.zero=Arc::new(0)"]), + ("Cow<'static, str>", &["fn_value.default=Cow::Owned(Default::default())"]), + ( + "Option", + &["fn_value.none=None", "fn_value.some=Some(0)", "fn_value.some=Some(1)"], + ), + ( + "Result", + &[ + "fn_value.err_default=Err(Default::default())", + "fn_value.ok=Ok(0)", + "fn_value.ok=Ok(1)", + ], + ), + ( + "HashMap", + &[ + "fn_value.empty_collection=HashMap::new()", + "fn_value.one_element=core::iter::once((0, 0)).collect()", + ], + ), + ( + "(i32, u8)", + &[ + "fn_value.tuple=(-1, 0)", + "fn_value.tuple=(-1, 1)", + "fn_value.tuple=(0, 0)", + "fn_value.tuple=(0, 1)", + "fn_value.tuple=(1, 0)", + "fn_value.tuple=(1, 1)", + ], + ), + ("(u32,)", &["fn_value.tuple=(0,)", "fn_value.tuple=(1,)"]), + ]; + + for (ty, expected) in cases { + assert_eq!(values_for_return(ty), *expected, "{ty}"); + } +} + +#[test] +fn every_iterator_trait_that_either_satisfies_is_served_iterator_values() { + // `Either` implements exactly these four, which is what makes them safe to mutate; a + // trait outside the set has no value this tool could write down. + for ty in [ + "impl Iterator", + "impl DoubleEndedIterator", + "impl ExactSizeIterator", + "impl FusedIterator", + ] { + assert_eq!( + values_for_return(ty), + [ + "fn_value.empty_collection=core::iter::empty()", + "fn_value.one_element=core::iter::once(0)", + "fn_value.one_element=core::iter::once(1)", + ], + "{ty}" + ); + } + + assert!(values_for_return("impl Clone").is_empty()); +} + +#[test] +fn every_expression_kind_is_descended_into() { + // Each source nests one mutable site inside one kind of expression. A visitor that + // stopped recursing there would still collect everything else, so nothing but a case + // per kind notices the loss. + let cases: &[(&str, &str)] = &[ + ("binary", "fn f() { let _ = (a + b) * c; }"), + ("unary", "fn f() { let _ = -(a + b); }"), + ("if", "fn f() { if c { let _ = a + b; } }"), + ("while", "fn f() { while c { let _ = a + b; } }"), + ("match", "fn f(x: T) { match x { _ => { let _ = a + b; } } }"), + ("struct", "fn f() { let _ = S { v: a + b }; }"), + ("range", "fn f() { let _ = (a + b)..c; }"), + ("break", "fn f() { loop { break a + b; } }"), + ("call", "fn f() { g(a + b); }"), + ("assign", "fn f() { x = a + b; }"), + ("method_call", "fn f() { x.g(a + b); }"), + ("index", "fn f() { let _ = x[a + b]; }"), + ("return", "fn f() -> T { return a + b; }"), + ("for_loop", "fn f() { for i in y { let _ = a + b; } }"), + ]; + + for (kind, source) in cases { + let found = mutators(source, "arith.add_to_sub", &CfgSet::unconditional()); + + assert_eq!(found, vec!["arith.add_to_sub"], "{kind}: {source}"); + } +} + +#[test] +fn every_expression_kind_is_descended_into_while_looking_for_numeric_names() { + // The pre-pass that decides which names hold numbers has its own visitor, and each of + // these nests the evidence one level inside the kind being overridden, so only recursion + // reaches it. Perturbing `a` where it is merely passed along is what proves the name was + // learned; the arithmetic holding the evidence is perturbed either way. + let cases: &[(&str, &str)] = &[ + ("binary", "fn f() { let a = g(); let _ = (a * b) + z; h(a); }"), + ("index", "fn f() { let a = g(); let _ = v[a * b]; h(a); }"), + ("method_call", "fn f() { let a = g(); let _ = x.q(a * b); h(a); }"), + ("for_loop", "fn f() { let a = g(); for i in y { let _ = a * b; } h(a); }"), + ]; + + for (kind, source) in cases { + let file = SourceFile::parse("test.rs", (*source).to_owned()).unwrap(); + let selection = Selection::parse("expr.increment").unwrap(); + + let found: Vec = collect_in(&file, &selection, &CfgSet::unconditional()) + .into_iter() + .map(|candidate| candidate.replacement.to_string()) + .collect(); + + assert!(found.contains(&"(a) + 1".to_owned()), "{kind}: {found:?}"); + } +} + +#[test] +fn a_const_generic_argument_in_expression_position_is_never_mutated() { + // A guard is a function call, and no const context will evaluate one, so a mutant here + // could only ever be withdrawn as unviable after the whole tree had been built for it. + for source in [ + "fn f() { let _ = Foo::<{ a + b }>::BAR; }", + "fn f() { g::<{ a + b }>(); }", + "fn f() { let _ = >::C; }", + ] { + let found = mutators(source, "arith", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{source}: {found:?}"); + } +} + +#[test] +fn a_generic_argument_that_is_not_const_is_still_reached() { + // Only the const case is inert; a closure body written as an associated-type binding is + // ordinary code that a guard can sit inside. + let source = "fn f() { let _ = g::<[u8; 4]>(a + b); }"; + let found = mutators(source, "arith.add_to_sub", &CfgSet::unconditional()); + + assert_eq!(found, vec!["arith.add_to_sub"], "{found:?}"); +} + +/// A span that reaches past the file text is refused rather than spliced. +/// +/// This is the macro-expansion case: the node is real, but its offsets do not describe the +/// bytes this file holds, so every edit derived from them would land in the wrong place. +#[test] +fn a_site_whose_span_runs_past_the_text_offers_nothing() { + let source = "fn f(a: i32, b: i32) -> bool { a < b }\n"; + let whole = mutators(source, "relational", &CfgSet::unconditional()); + let truncated = mutators_past_the_end(source, 10, "relational"); + + assert!(!whole.is_empty(), "the source offers nothing even intact"); + assert!(truncated.is_empty(), "{truncated:?}"); +} + +/// The struct-field omission refuses a literal whose span it cannot index. +#[test] +fn a_struct_literal_past_the_text_omits_no_field() { + let source = "fn f(base: S) -> S { S { a: 1, b: 2, ..base } }\n"; + let whole = mutators(source, "struct_field.omit", &CfgSet::unconditional()); + let truncated = mutators_past_the_end(source, 22, "struct_field.omit"); + + assert_eq!(whole, vec!["struct_field.omit", "struct_field.omit"]); + assert!(truncated.is_empty(), "{truncated:?}"); +} + +/// A literal with no base expression has nothing to fall back on, so no field can be dropped. +#[test] +fn a_struct_literal_without_a_base_omits_no_field() { + let source = "fn f() -> S { S { a: 1, b: 2 } }\n"; + let found = mutators(source, "struct_field.omit", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +/// The `vec!` element omission refuses a macro whose span it cannot index. +#[test] +fn a_vec_literal_past_the_text_omits_no_element() { + let source = "fn f() -> Vec { vec![1, 2, 3] }\n"; + let whole = mutators(source, "collection.omit_element", &CfgSet::unconditional()); + let truncated = mutators_past_the_end(source, 24, "collection.omit_element"); + + assert_eq!(whole.len(), 3, "{whole:?}"); + assert!(truncated.is_empty(), "{truncated:?}"); +} + +/// A method with no rename in the table is left alone rather than renamed to nothing. +#[test] +fn a_method_the_table_does_not_know_is_not_renamed() { + let source = "fn f(x: Thing) -> i32 { x.frobnicate() }\n"; + let found = mutators(source, "iter,string", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +/// A perturbation needs the text of the expression, which a span past the file cannot give. +#[test] +fn a_perturbed_expression_past_the_text_offers_nothing() { + let source = "fn f(a: i32) -> i32 { g(a) }\n"; + let whole = mutators(source, "expr.increment,expr.decrement", &CfgSet::unconditional()); + let truncated = mutators_past_the_end(source, 22, "expr.increment,expr.decrement"); + + assert!(!whole.is_empty(), "the source offers nothing even intact"); + assert!(truncated.is_empty(), "{truncated:?}"); +} + +/// A block is a lexical scope, and its `let` must not outlive it. +/// +/// A `let` inside `{ … }` must not overwrite the numeric evidence for a shadowed outer name beyond +/// the block. Both directions are wrong and in opposite ways: losing the evidence withholds valid +/// mutants and inflates the score, and gaining it emits mutants that cannot compile. +#[test] +fn a_block_local_binding_does_not_outlive_its_block() { + // The outer `n` is numeric again once the block has closed, so its use is still perturbed. + let kept = mutators( + "fn f(n: usize) { { let n: String = g(); } h(n); }", + "expr.increment", + &CfgSet::unconditional(), + ); + + assert_eq!(kept, vec!["expr.increment"], "the inner `let` must not erase the outer evidence"); + + // And the reverse: the inner `let` must not lend its numeric type to the outer `String`, which + // would offer `(n) + 1` on a name that cannot be added to. + let refused = mutators( + "fn f(n: String) { { let n: usize = g(); } h(n); }", + "expr.increment", + &CfgSet::unconditional(), + ); + + assert!(refused.is_empty(), "the inner `let` must not leak numeric evidence: {refused:?}"); +} + +/// Scoping the evidence must not break the case it exists to serve. +/// +/// A `let scanned;` settled by an assignment inside a nested block is the common shape, and it only +/// works because the nested block inherits the deferral its parent recorded. Clearing rather than +/// inheriting would offer the initialising assignment as a deletion candidate, whose mutant fails +/// to compile as E0381 at a *different* statement — the diagnostic this suppression exists to +/// avoid. +#[test] +fn a_deferred_binding_is_still_seen_from_inside_a_nested_block() { + let source = "fn f(c: bool) -> i32 { let scanned; if c { scanned = 1; } else { scanned = 2; } scanned }"; + let found = mutators(source, "stmt.delete_assign", &CfgSet::unconditional()); + + assert!( + found.is_empty(), + "an assignment that first initialises a deferred `let` must not be deletable: {found:?}" + ); +} + +/// One module's `use` must not decide another module's question. +/// +/// The import index is keyed by the bare name and spans the whole file, so a later `use` must not +/// overwrite an earlier one and leave a bare `Error` resolving to whichever was written last. That +/// would make the answer depend on the order the modules happen to appear in: reading `std::io` +/// first offers `Err(Default::default())` for both, reading `crate` first withholds it for both, +/// and in each case one of the two modules gets the other's answer. +#[test] +fn one_modules_import_does_not_decide_anothers() { + let io_first = concat!( + "mod a { use std::io::Error; pub fn f() -> Result { Ok(0) } }\n", + "mod b { use crate::Error; pub fn g() -> Result { Ok(0) } }", + ); + let crate_first = concat!( + "mod b { use crate::Error; pub fn g() -> Result { Ok(0) } }\n", + "mod a { use std::io::Error; pub fn f() -> Result { Ok(0) } }", + ); + + let first = mutators(io_first, "fn_value.err_default", &CfgSet::unconditional()); + let second = mutators(crate_first, "fn_value.err_default", &CfgSet::unconditional()); + + assert_eq!( + first, second, + "the decision must not depend on which `use` the file happens to write last" + ); +} + +/// Demoting a contested name must not cost the answer for a name nothing contests. +#[test] +fn an_uncontested_import_still_resolves_to_where_it_came_from() { + // Foreign, so it has no `Default` and the mutant is withheld. + let foreign = mutators( + "mod a { use std::io::Error; pub fn f() -> Result { Ok(0) } }", + "fn_value.err_default", + &CfgSet::unconditional(), + ); + + assert!(foreign.is_empty(), "a foreign error type has no `Default`: {foreign:?}"); + + // Local, so `Default::default()` is a guess worth making. + let local = mutators( + "mod b { use crate::Error; pub fn g() -> Result { Ok(0) } }", + "fn_value.err_default", + &CfgSet::unconditional(), + ); + + assert_eq!(local, vec!["fn_value.err_default"], "a workspace error type may have a `Default`"); +} + +/// Two modules importing the *same* path do not disagree, so neither loses its answer. +#[test] +fn importing_one_path_twice_is_not_a_disagreement() { + let source = concat!( + "mod a { use std::io::Error; pub fn f() -> Result { Ok(0) } }\n", + "mod c { use std::io::Error; pub fn h() -> Result { Ok(0) } }", + ); + let found = mutators(source, "fn_value.err_default", &CfgSet::unconditional()); + + assert!( + found.is_empty(), + "a repeated identical import must not demote the name to unknown: {found:?}" + ); +} + +/// A cast writes the type at the use site, so a cast argument is perturbed like any other number. +#[test] +fn a_cast_argument_is_perturbed_as_a_number() { + let source = "fn f(x: usize) { g(x as i32); }\n"; + let found = mutators(source, "expr.increment,expr.decrement", &CfgSet::unconditional()); + + assert_eq!(found, vec!["expr.decrement", "expr.increment"], "{found:?}"); +} + +/// Negation is a number; `!`/`*` are excluded elsewhere because they may not be. +#[test] +fn a_negated_argument_is_perturbed_as_a_number() { + let source = "fn f(x: i32) { g(-x); }\n"; + let found = mutators(source, "expr.increment,expr.decrement", &CfgSet::unconditional()); + + assert_eq!(found, vec!["expr.decrement", "expr.increment"], "{found:?}"); +} + +/// `String + &str` is addition too, so `+` asks each side rather than assuming both are numbers. +/// A numeric left side settles it without needing the literal on the right to answer as well. +#[test] +fn an_addition_is_numeric_when_either_side_is() { + let source = "fn f(a: i32) { g(a + 1); }\n"; + let found = mutators(source, "expr.increment,expr.decrement", &CfgSet::unconditional()); + + assert_eq!(found, vec!["expr.decrement", "expr.increment"], "{found:?}"); +} + +/// A tuple field's type is not recorded anywhere this file reads, so it never answers "number". +#[test] +fn a_tuple_field_access_is_never_known_numeric() { + let source = "fn f(t: (i32, i32)) { g(t.0); }\n"; + let found = mutators(source, "expr.increment,expr.decrement", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +/// A trait path past the text cannot be read, so the scope falls back to naming itself after the +/// `Self` type alone rather than reporting `` from an unreadable trait. +#[test] +fn a_trait_path_past_the_text_names_the_scope_for_the_self_type_alone() { + let source = "struct S; trait Trait { fn f(&self) -> i32; } impl Trait for S { fn f(&self) -> i32 { 1 + 2 } }\n"; + let truncated = mutators_past_the_end(source, 51, "arith.add_to_sub"); + + assert!(truncated.is_empty(), "{truncated:?}"); +} + +/// A method call past the text is refused rather than renamed with a phantom suffix. +#[test] +fn a_method_call_past_the_text_is_not_renamed() { + let source = "fn f(v: Vec) -> Option<&i32> { v.first() }\n"; + let whole = mutators(source, "iter.first_to_last", &CfgSet::unconditional()); + let truncated = mutators_past_the_end(source, 44, "iter.first_to_last"); + + assert_eq!(whole, vec!["iter.first_to_last"], "{whole:?}"); + assert!(truncated.is_empty(), "{truncated:?}"); +} + +/// A trait definition the build does not contain offers nothing from any of its default bodies. +#[test] +fn a_trait_that_is_configured_out_is_not_mutated() { + let source = "#[cfg(not(unix))] trait Hidden { fn f(&self) -> i32 { 1 + 2 } }\n"; + let found = mutators(source, "arith.add_to_sub", &CfgSet::parse("unix")); + + assert!(found.is_empty(), "{found:?}"); +} + +/// An expression the build does not contain is refused even when it sits inside a call argument +/// rather than as a whole statement, which is the position `visit_block` cannot filter for it. +#[test] +fn an_argument_expression_that_is_configured_out_is_not_mutated() { + let source = "fn f() -> i32 { g(#[cfg(not(unix))] (1 + 2)) }\n"; + let found = mutators(source, "arith.add_to_sub", &CfgSet::parse("unix")); + + assert!(found.is_empty(), "{found:?}"); +} + +/// `omit_elements` refuses a macro with fewer than two elements, but its own caller already +/// filters that case before ever calling it. Calling it directly is the only way to reach the +/// guard that keeps the function safe on its own terms. +#[test] +fn omit_elements_refuses_a_macro_with_no_elements() { + let file = SourceFile::parse("test.rs", "fn f() -> Vec { vec![] }".to_owned()).unwrap(); + let selection = Selection::parse("collection.omit_element").unwrap(); + let cfg = CfgSet::unconditional(); + let defaults = Defaults::default(); + let mut collector = Collector::new(&file, &selection, selection.errors(), &cfg, &defaults); + let macro_node: syn::Macro = syn::parse_str("vec![]").unwrap(); + let elements = Punctuated::new(); + + collector.omit_elements(¯o_node, &elements); + + assert!(collector.finish().is_empty()); +} + +#[test] +fn omit_elements_refuses_element_spans_outside_the_macro() { + let file = SourceFile::parse("test.rs", "fn f() { vec![1, 2]; }".to_owned()).unwrap(); + let Item::Fn(function) = &file.ast.items[0] else { + panic!("test source should contain a function"); + }; + let Stmt::Macro(statement) = &function.block.stmts[0] else { + panic!("test source should contain a macro statement"); + }; + let node = statement.mac.clone(); + let mut elements = node.parse_body_with(Punctuated::::parse_terminated).unwrap(); + elements[0] = syn::parse_quote!(outside); + let selection = Selection::parse("collection.omit_element").unwrap(); + let cfg = CfgSet::unconditional(); + let defaults = Defaults::default(); + let mut collector = Collector::new(&file, &selection, selection.errors(), &cfg, &defaults); + + collector.omit_elements(&node, &elements); + + assert!(collector.finish().is_empty()); +} + +#[test] +fn omit_update_refuses_field_spans_outside_the_struct() { + let mut file = SourceFile::parse("test.rs", "fn f() { S { field: 1, ..base }; }".to_owned()).unwrap(); + let Item::Fn(function) = &mut file.ast.items[0] else { + panic!("test source should contain a function"); + }; + let Stmt::Expr(Expr::Struct(expression), _) = &mut function.block.stmts[0] else { + panic!("test source should contain a struct expression"); + }; + expression.fields[0].member = Member::Named(syn::Ident::new("outside", proc_macro2::Span::call_site())); + let node = expression.clone(); + let selection = Selection::parse("struct_field.omit").unwrap(); + let cfg = CfgSet::unconditional(); + let defaults = Defaults::default(); + let mut collector = Collector::new(&file, &selection, selection.errors(), &cfg, &defaults); + + collector.struct_fields(&node); + + assert!(collector.finish().is_empty()); +} + +#[test] +fn rename_method_refuses_a_name_span_outside_the_call() { + let mut file = SourceFile::parse("test.rs", "fn f(values: &[u8]) { values.first(); }".to_owned()).unwrap(); + let Item::Fn(function) = &mut file.ast.items[0] else { + panic!("test source should contain a function"); + }; + let Stmt::Expr(Expr::MethodCall(expression), _) = &mut function.block.stmts[0] else { + panic!("test source should contain a method call"); + }; + expression.method = syn::Ident::new("first", proc_macro2::Span::call_site()); + let node = expression.clone(); + let selection = Selection::parse("iter.first_to_last").unwrap(); + let cfg = CfgSet::unconditional(); + let defaults = Defaults::default(); + let mut collector = Collector::new(&file, &selection, selection.errors(), &cfg, &defaults); + + collector.rename_method(&node, "first"); + + assert!(collector.finish().is_empty()); +} + +#[test] +fn direct_local_visit_refuses_a_configured_out_binding() { + let file = SourceFile::parse("test.rs", "fn f() { #[cfg(not(unix))] let value: i32 = 1; }".to_owned()).unwrap(); + let Item::Fn(function) = &file.ast.items[0] else { + panic!("test source should contain a function"); + }; + let Stmt::Local(local) = &function.block.stmts[0] else { + panic!("test source should contain a local"); + }; + let selection = Selection::parse("expr.increment").unwrap(); + let cfg = CfgSet::parse("unix"); + let defaults = Defaults::default(); + let mut collector = Collector::new(&file, &selection, selection.errors(), &cfg, &defaults); + + collector.visit_local(local); + + assert!(collector.bindings.is_empty()); + assert!(collector.finish().is_empty()); +} + +/// `==` and the other comparisons are not known to be numbers on either side, so a comparison +/// argument is never perturbed as one. +#[test] +fn a_comparison_argument_is_never_known_numeric() { + let source = "fn f(a: i32, b: i32) { g(a == b); }\n"; + let found = mutators(source, "expr.increment,expr.decrement", &CfgSet::unconditional()); + + assert!(found.is_empty(), "{found:?}"); +} + +/// A comment inside a qualified trait path is trivia, not part of the trait's identity, so the +/// scope name built from it must not carry the comment along. +#[test] +fn a_comment_inside_a_trait_path_does_not_reach_the_item_path() { + let source = "struct S; impl some/*noise*/::Trait for S { fn f(&self) -> i32 { 1 + 2 } }\n"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("arith.add_to_sub").unwrap(); + let candidates = collect_in(&file, &selection, &CfgSet::unconditional()); + + assert_eq!(candidates.len(), 1, "{candidates:?}"); + assert_eq!(&*candidates[0].item_path, "::f"); +} + +/// A literal inside a trait path -- a const generic argument -- is stepped over by the source +/// parser's lexer rather than by the whitespace filter, so it survives intact. +#[test] +fn a_literal_inside_a_trait_path_survives_compaction() { + let source = "struct S; impl Trait<5> for S { fn f(&self) -> i32 { 1 + 2 } }\n"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("arith.add_to_sub").unwrap(); + let candidates = collect_in(&file, &selection, &CfgSet::unconditional()); + + assert_eq!(candidates.len(), 1, "{candidates:?}"); + assert_eq!(&*candidates[0].item_path, ">::f"); +} + +/// A string literal is recognized by the source parser's lexer, not by the whitespace filter, so +/// its contents -- including a comment-shaped sequence and internal spaces -- pass through +/// `compact_path` untouched even though everything around it is compacted. +#[test] +fn compact_path_steps_over_a_string_literal_intact() { + let compacted = compact_path(r#"Trait < "a /* not a comment */ b" >"#); + + assert_eq!(compacted, r#"Trait<"a /* not a comment */ b">"#); +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/types.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/types.rs new file mode 100644 index 000000000..270b387f9 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/types.rs @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! What the file's own declarations say about the types a function mentions. + +use syn::{GenericParam, Generics, ReturnType, Type}; + +use super::super::defaults::{DefaultPaths, standard_defaulted_parameters}; +use super::predicates::payload; +use super::values::{Kind, resolve_type, strip}; +use crate::HashMap; +use crate::ops::collect::Defaults; + +/// What the value-choosing functions know about the file they are reasoning inside. +/// +/// The two facts travel together through the whole recursion and neither is ever used without the +/// other, so they are carried as one thing rather than as a growing parameter list. +pub(super) struct Types<'a> { + /// Type names in scope that cannot be constructed: type parameters without a `Default` bound, + /// associated types, and trait objects. + pub(super) abstracts: &'a [String], + + /// The module path each imported name came from, so a bare type name can be traced to its crate. + pub(super) imports: &'a HashMap>>, + + /// What the workspace's own sources say about which of their types implement `Default`. + pub(super) defaults: &'a Defaults, + + /// The concrete type named by `Self` inside an `impl` block. + pub(super) self_type: Option<&'a Type>, + + /// Concrete associated types declared by the enclosing `impl`. + pub(super) self_associated: Option<&'a HashMap>, +} + +impl Types<'_> { + /// Returns whether a type has no `Default` to reach for. + /// + /// Two independent readings say so, and either is enough: an error type from another crate, + /// which effectively never has one, or a type this workspace defines and gives none. + pub(super) fn lacks_default(&self, ty: &Type) -> bool { + let concrete = self + .concrete_self_type(ty) + .or_else(|| self.concrete_self_associated_type(ty)) + .unwrap_or(ty); + + is_foreign_error(concrete, self.imports) + || Self::is_standard_time_without_default(concrete, self.imports) + || self.defaults.lacks_default(concrete) + } + + fn is_standard_time_without_default(ty: &Type, imports: &HashMap>>) -> bool { + let Type::Path(path) = strip(ty) else { + return false; + }; + let Some(last) = path.path.segments.last() else { + return false; + }; + let name = last.ident.to_string(); + + if !matches!(name.as_str(), "Instant" | "SystemTime") { + return false; + } + + let prefix = if path.path.segments.len() > 1 { + path.path + .segments + .iter() + .take(path.path.segments.len() - 1) + .map(|segment| segment.ident.to_string()) + .collect::>() + } else { + let Some(Some(prefix)) = imports.get(&name) else { + return false; + }; + prefix.clone() + }; + + matches!(prefix.as_slice(), [root, module] if root == "std" && module == "time") + } + + /// Returns whether this is the standard library's zero-argument `fmt::Result` alias. + pub(super) fn is_fmt_result(&self, ty: &Type) -> bool { + let Type::Path(path) = strip(ty) else { + return false; + }; + let segments = path + .path + .segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect::>(); + + match segments.as_slice() { + [root, module, result] => matches!(root.as_str(), "std" | "core") && module == "fmt" && result == "Result", + [module, result] if module == "fmt" && result == "Result" => self + .imports + .get(module) + .and_then(Option::as_ref) + .is_some_and(|path| matches!(path.as_slice(), [root] if matches!(root.as_str(), "std" | "core"))), + _ => false, + } + } + + fn concrete_self_type<'a>(&'a self, ty: &Type) -> Option<&'a Type> { + let Type::Path(path) = strip(ty) else { + return None; + }; + + (path.qself.is_none() && path.path.is_ident("Self")) + .then_some(self.self_type) + .flatten() + } + + fn concrete_self_associated_type<'a>(&'a self, ty: &Type) -> Option<&'a Type> { + let Type::Path(path) = strip(ty) else { + return None; + }; + let mut segments = path.path.segments.iter(); + let (Some(root), Some(name), None) = (segments.next(), segments.next(), segments.next()) else { + return None; + }; + + (path.qself.is_none() && root.ident == "Self") + .then(|| self.self_associated?.get(&name.ident.to_string())) + .flatten() + } +} + +/// The one type outside this workspace whose name ends in `Error` and that does implement `Default`. +/// +/// `core::fmt::Error` is a unit struct standing for "formatting failed", and it derives everything. +/// Every other error type in `std` that was checked does not implement `Default`, so it is cheaper +/// to name the exception than to enumerate the rule. +pub(super) const DEFAULTABLE_ERROR: &str = "fmt"; + +/// Returns whether a type is an error type from outside this workspace, which will have no `Default`. +/// +/// Error types are the largest single source of mutants that cannot compile: `Err(Default::default())` +/// wants an error value, and `std::io::Error`, `anyhow::Error`, `serde_json::Error` and their kind +/// have no `Default` and are not going to acquire one. Withholding the mutant there costs no signal, +/// because there was never a mutant to lose. +/// +/// The rule is deliberately confined to types from *other* crates. A workspace error type may well +/// be an enum with a `#[default]` variant, and the collector cannot see the definition to find out, +/// so `crate::`, `self::`, `super::` and any name this file does not import stay optimistic. +pub(super) fn is_foreign_error(ty: &Type, imports: &HashMap>>) -> bool { + let Type::Path(path) = strip(ty) else { + return false; + }; + + let Some(last) = path.path.segments.last() else { + return false; + }; + + if !last.ident.to_string().ends_with("Error") { + return false; + } + + // Written out in full, so where it comes from is right there. A single-segment path is a bare + // name instead, and only the file's imports can say what it was. + let owned; + let prefix: &[String] = if path.path.segments.len() > 1 { + owned = path + .path + .segments + .iter() + .rev() + .skip(1) + .rev() + .map(|segment| segment.ident.to_string()) + .collect::>(); + &owned + } else { + match imports.get(&last.ident.to_string()) { + // A name two `use` items disagree about is as unknown as one never imported, so it + // takes the same answer rather than the last writer's. + Some(Some(path)) => path, + Some(None) | None => return false, + } + }; + + let (Some(root), Some(qualifier)) = (prefix.first(), prefix.last()) else { + return false; + }; + + if matches!(root.as_str(), "crate" | "self" | "super") { + return false; + } + + qualifier != DEFAULTABLE_ERROR +} + +/// Returns whether a signature's `Result` fixes its error type to one with no `Default`. +/// +/// `Ok(v)` becoming `Err(Default::default())` needs a value of the error type, and the call site +/// does not name it — the signature does. When the error type is written out, it is read straight +/// off the return type. When the signature uses a crate-wide `type Result` alias instead, which +/// is close to universal in real Rust, the alias is resolved through the workspace index. +pub(super) fn returns_undefaultable_error(output: &ReturnType, types: &Types<'_>) -> bool { + let ReturnType::Type(_arrow, ty) = output else { + return false; + }; + + if resolve_type(ty) != Kind::Result { + return false; + } + + if let Some(error) = payload(ty, 1) { + return types.lacks_default(error); + } + + // No second argument, so the error type is whatever an alias fixed it to. The alias is named by + // the return type's last segment, and the index answers for the name it resolved to. + let Type::Path(path) = strip(ty) else { + unreachable!("a type resolved as Result must be a path after wrappers are stripped"); + }; + let alias = path + .path + .segments + .last() + .expect("a type resolved as Result must have a final path segment"); + + types + .defaults + .aliased_error(&alias.ident.to_string()) + .is_some_and(|error| types.defaults.lacks_error_default(error)) +} + +/// Returns whether a type is one no concrete `Default` can be assumed for. +/// +/// `Default::default()` is the fallback for an unknown concrete type, but a caller's type +/// parameter, an associated type projected from one, and a trait object or `impl Trait` name no +/// constructible type unless the signature supplies a `Default` bound. +/// +/// `Self::Value` is deliberately excluded. Inside an `impl` block it resolves to the type chosen +/// by that block and may implement `Default`. +pub(super) fn is_abstract_type(ty: &Type, abstracts: &[String]) -> bool { + match ty { + Type::TraitObject(_) | Type::ImplTrait(_) => true, + + Type::Paren(paren) => is_abstract_type(&paren.elem, abstracts), + + Type::Path(path) if path.qself.is_none() => { + let Some(last) = path.path.segments.last() else { + return false; + }; + + if path.path.segments.len() > 1 { + return abstracts.contains(&path.path.segments[0].ident.to_string()); + } + + // `Box` is exactly as unconstructable as the `dyn Reader` inside it. + if last.ident == "Box" { + return payload(ty, 0).is_some_and(|inner| is_abstract_type(inner, abstracts)); + } + + abstracts.contains(&last.ident.to_string()) + } + + _ => false, + } +} + +/// Names every type parameter a generics list declares without a `Default` bound. +/// +/// A parameter written `T: Default` is excluded, because there the promise this is looking for was +/// made explicitly and the mutant it would otherwise withhold compiles. +pub(super) fn undefaulted_parameters(generics: &Generics, defaults: &DefaultPaths) -> Vec { + let defaulted = standard_defaulted_parameters(generics, defaults); + + generics + .params + .iter() + .filter_map(|param| match param { + GenericParam::Type(ty) => (!defaulted.iter().any(|name| name == &ty.ident.to_string())).then(|| ty.ident.to_string()), + + _ => None, + }) + .collect() +} + +#[cfg(test)] +mod tests { + use syn::punctuated::Punctuated; + use syn::{Path, PathSegment, TypePath, parse_quote}; + + use super::*; + + fn defaults(source: &str) -> Defaults { + Defaults::of(&syn::parse_file(source).expect("test source should parse")) + } + + fn empty_path_type() -> Type { + Type::Path(TypePath { + qself: None, + path: Path { + leading_colon: None, + segments: Punctuated::::new(), + }, + }) + } + + #[test] + fn foreign_error_detection_rejects_empty_paths_and_empty_import_prefixes() { + let mut imports = HashMap::default(); + let imported_error: Type = parse_quote!(TheirError); + + let _old = imports.insert("TheirError".to_owned(), Some(Vec::new())); + + assert!(!is_foreign_error(&empty_path_type(), &HashMap::default())); + assert!(!is_foreign_error(&imported_error, &imports)); + } + + #[test] + fn result_aliases_and_direct_errors_can_both_be_screened() { + let defaults = defaults( + " + struct Error; + type Result = core::result::Result; + ", + ); + let abstracts = Vec::new(); + let imports = HashMap::default(); + let types = Types { + abstracts: &abstracts, + imports: &imports, + defaults: &defaults, + self_type: None, + self_associated: None, + }; + let aliased: ReturnType = parse_quote!(-> Result); + let direct: ReturnType = parse_quote!(-> Result); + + assert!(returns_undefaultable_error(&aliased, &types)); + assert!(returns_undefaultable_error(&direct, &types)); + } + + #[test] + fn abstract_type_detection_handles_empty_paths_and_boxed_trait_objects() { + assert!(!is_abstract_type(&empty_path_type(), &[])); + assert!(is_abstract_type(&parse_quote!(Box), &[])); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/collector/values.rs b/crates/cargo-gamma-engine/src/ops/collect/collector/values.rs new file mode 100644 index 000000000..cd15ea27e --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/collector/values.rs @@ -0,0 +1,699 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The replacement values a function's return type admits. + +use compact_str::{CompactString, format_compact}; +use syn::{GenericArgument, PathArguments, PathSegment, ReturnType, Type, TypeParamBound}; + +use super::predicates::payload; +use super::types::{Types, is_abstract_type}; + +/// How deep the recursion through nested return types is allowed to go. +/// +/// A tuple of options of results nests as far as the author cared to write, and each level +/// multiplies the number of values below it. Three levels reaches `Result, E>`, which +/// is the shape this exists for, and stops well before a type whose values would dominate the +/// population of the whole file. +pub(super) const RETURN_DEPTH: usize = 3; + +/// The most replacement values any single return type may contribute. +/// +/// The bound is on the product, not on any one level, because it is the product that decides how +/// many mutants a function costs. A tuple of four booleans is sixteen combinations, and every one +/// of them is a separate build round's worth of test time. +pub(super) const RETURN_WIDTH: usize = 8; + +type ReplacementValue = (&'static str, CompactString); + +fn some_value((_name, text): ReplacementValue) -> ReplacementValue { + ("fn_value.some", format_compact!("Some({text})")) +} + +/// The replacement values worth trying for a function's return type. +/// +/// Each entry is a mutator name and the text of a value of that type. The list is generated rather +/// than looked up so that nested types compose: a `Result, E>` is a `Result` whose +/// success values are the `Option` values, which are in turn the `bool` values. +/// +/// The type is read syntactically, so an alias, a generic parameter or an associated type falls +/// through to `Default::default()`, which may not compile — an acceptable trade, since a bad guess +/// costs one rollback round rather than losing the mutant entirely. +pub(super) fn return_values(output: &ReturnType, types: &Types<'_>) -> Vec { + let ReturnType::Type(_arrow, ty) = output else { + return vec![("fn_value.unit", "()".into())]; + }; + + values_for(ty, RETURN_DEPTH, types) +} + +/// The replacement values for one type, recursing through its parameters. +/// +/// `depth` bounds the recursion; at zero the type contributes `Default::default()` rather than +/// nothing, because a value that type-checks is still worth trying even when its shape is unknown. +#[expect(clippy::too_many_lines, reason = "the match exhaustively maps every return-type kind")] +pub(super) fn values_for(ty: &Type, depth: usize, types: &Types<'_>) -> Vec { + let kind = if types.is_fmt_result(ty) { Kind::Result } else { resolve_type(ty) }; + + // An abstract type contributes nothing rather than a guess. `Default::default()` is what this + // family reaches for when it cannot name a value, and for a caller's type parameter or a + // trait's associated type nothing promises there is one to reach for. + // + // An `impl Iterator` is the one exception, and only because it is not really a guess: every + // iterator can be named through `gamma_rt::Either`, whatever concrete type the body chose. + if kind != Kind::Iterator && is_abstract_type(ty, types.abstracts) { + return Vec::new(); + } + + // An error type from another crate contributes nothing for the same reason, but with a stronger + // warrant: `Default::default()` is not a guess that might be wrong here, it is one that has been + // measured wrong over and over. This is the largest single cause of mutants that cannot compile. + if kind == Kind::Unknown && types.lacks_default(ty) { + return Vec::new(); + } + + if depth == 0 { + return vec![("fn_value.default", "Default::default()".into())]; + } + + match kind { + // The kinds whose values are written out rather than built from another type's. + kind @ (Kind::Unit + | Kind::Bool + | Kind::Signed + | Kind::Unsigned + | Kind::Float + | Kind::StaticStr + | Kind::MutStr + | Kind::String + | Kind::NonZero) => literal_values(kind, ty), + + // The empty case is universal; the one-element case needs a value to put in it, which is + // what the recursion supplies. + Kind::Option => { + let mut values = vec![("fn_value.none", "None".into())]; + let inner = inner_values(ty, 0, depth, types); + + if inner.is_empty() { + // Nothing is known about the payload, either because it is a type this file cannot + // resolve or because it is a reference. `Default::default()` is the one expression + // that stands for a value of any type it fits, so it is what is left to try. It + // will not always compile, and that is accepted: the alternative is to emit only + // `None` and never ask whether the present case is tested at all. + if !payload(ty, 0).is_some_and(|inner| is_abstract_type(inner, types.abstracts) || types.lacks_default(inner)) { + values.push(("fn_value.some_default", "Some(Default::default())".into())); + } + } else { + values.extend(inner.into_iter().map(some_value)); + } + + cap(values) + } + + Kind::Result => { + let inner = inner_values(ty, 0, depth, types); + let mut values = if inner.is_empty() { + if payload(ty, 0).is_some_and(|inner| is_abstract_type(inner, types.abstracts) || types.lacks_default(inner)) { + Vec::new() + } else { + vec![("fn_value.ok_default", "Ok(Default::default())".into())] + } + } else { + inner + .into_iter() + .map(|(_name, text)| ("fn_value.ok", format_compact!("Ok({text})"))) + .collect() + }; + + // The error type is only named when the path spells it. `type Result = + // core::result::Result` is everywhere in real crates, and there the error + // is whatever the alias fixed it to — almost never something with a `Default`. Offering + // `Err(Default::default())` on a guess buys one mutant that usually cannot compile, so + // it is offered only when the second argument is present and not abstract. + if payload(ty, 1).is_some_and(|inner| !is_abstract_type(inner, types.abstracts) && !types.lacks_default(inner)) { + values.push(("fn_value.err_default", "Err(Default::default())".into())); + } + + cap(values) + } + + // Every one of these builds from an iterator of its element type, so one construction + // covers all of them and the element values come from the recursion. + Kind::Collection => { + let empty = format_compact!("{}::new()", collection_ctor(ty)); + let mut values = vec![("fn_value.empty_collection", empty)]; + + values.extend( + inner_values(ty, 0, depth, types) + .into_iter() + .map(|(_name, text)| ("fn_value.one_element", format_compact!("core::iter::once({text}).collect()"))), + ); + + cap(values) + } + + // A map's element is a pair, so its one-element form needs both parameters rather than the + // first alone. + Kind::Map => { + let empty = format_compact!("{}::new()", collection_ctor(ty)); + let mut values = vec![("fn_value.empty_collection", empty)]; + + let keys = inner_values(ty, 0, depth, types); + let vals = inner_values(ty, 1, depth, types); + + if let (Some((_kn, key)), Some((_vn, value))) = (keys.first(), vals.first()) { + values.push(( + "fn_value.one_element", + format_compact!("core::iter::once(({key}, {value})).collect()"), + )); + } + + cap(values) + } + + // A smart pointer is transparent to the caller's reasoning, so the values worth trying are + // its contents wrapped back up. + Kind::Wrapper => { + let ctor = wrapper_ctor(ty); + + cap(inner_values(ty, 0, depth, types) + .into_iter() + .map(|(name, text)| (name, format_compact!("{ctor}({text})"))) + .collect()) + } + + // `Cow` is a wrapper whose constructor is a variant rather than a function, and `Owned` + // is the variant that does not borrow from anything in scope. + // The written path is reused rather than `std::borrow::Cow`, which would name a different + // type from the one the function returns whenever the author meant somebody else's. + Kind::Cow => { + let ctor = collection_ctor(ty); + + cap(inner_values(ty, 0, depth, types) + .into_iter() + .map(|(name, text)| (name, format_compact!("{ctor}::Owned({text})"))) + .collect()) + } + + // An `impl Iterator` return is one concrete type chosen by the body, so `Empty`, + // `Once` and whatever the author wrote are three types that cannot be arms of one `if` + // on their own. `Shape::IterBlock` wraps each arm so that they can be, which is why values + // are offered here rather than withheld. + // + // `empty()` needs no item type at all, since the wrapper infers it from the other arm. + // `once(v)` needs a value, so it is offered only when the signature wrote `Item = T` and + // `T` is a type this tool can name a value of. + Kind::Iterator => { + let mut values = vec![("fn_value.empty_collection", "core::iter::empty()".into())]; + + if let Some(item) = iterator_item(ty) { + values.extend( + values_for(item, depth.saturating_sub(1), types) + .into_iter() + .map(|(_name, text)| ("fn_value.one_element", format_compact!("core::iter::once({text})"))), + ); + } + + cap(values) + } + Kind::Reference => reference_values(ty, depth, types), + + // Every combination of the elements' values, which is where the product bound earns its + // keep: three fields with three values each is twenty-seven mutants for one function. + Kind::Tuple => tuple_values(ty, depth, types), + + Kind::Unknown => vec![("fn_value.default", "Default::default()".into())], + } +} + +/// Every combination of a tuple's element values, which is where the width bound earns its keep: +/// three fields with three values each is twenty-seven mutants for a single function, and the +/// user has to read every one of them. +pub(super) fn tuple_values(ty: &Type, depth: usize, types: &Types<'_>) -> Vec { + let Type::Tuple(tuple) = strip(ty) else { + return vec![("fn_value.default", "Default::default()".into())]; + }; + + let mut combinations: Vec> = vec![Vec::new()]; + + for element in &tuple.elems { + let choices = values_for(element, depth.saturating_sub(1), types); + let mut next = Vec::new(); + + for existing in &combinations { + for (_name, text) in &choices { + if next.len() >= RETURN_WIDTH { + break; + } + + let mut combination = existing.clone(); + + combination.push(text.clone()); + next.push(combination); + } + } + + combinations = next; + } + + combinations + .into_iter() + .map(|parts| { + let text = if parts.len() == 1 { + format_compact!("({},)", parts[0]) + } else { + format_compact!("({})", parts.join(", ")) + }; + + ("fn_value.tuple", text) + }) + .collect() +} + +/// The replacement values for a type that has a fixed list of them. +/// +/// These are the kinds whose values can be written down directly, as opposed to the containers and +/// wrappers whose values are built by recursing into a parameter. Kinds outside that group +/// contribute nothing here, because they are handled by the caller. +pub(super) fn literal_values(kind: Kind, ty: &Type) -> Vec { + match kind { + Kind::Unit => vec![("fn_value.unit", "()".into())], + + Kind::Bool => vec![("fn_value.bool_true", "true".into()), ("fn_value.bool_false", "false".into())], + + Kind::Signed => vec![ + ("fn_value.zero", "0".into()), + ("fn_value.one", "1".into()), + ("fn_value.minus_one", "-1".into()), + ], + + Kind::Unsigned => vec![("fn_value.zero", "0".into()), ("fn_value.one", "1".into())], + + Kind::Float => vec![ + ("fn_value.zero", "0.0".into()), + ("fn_value.one", "1.0".into()), + ("fn_value.minus_one", "-1.0".into()), + ], + + Kind::StaticStr => vec![ + ("fn_value.empty_string", "\"\"".into()), + ("fn_value.xyzzy_string", "\"xyzzy\"".into()), + ], + + // A literal will not do here — it is `&'static str`, and the signature asked for a mutable + // slice. Leaking a boxed `str` yields the `&'static mut str` that will actually type-check, + // using the same `Box::leak` idiom the reference values are built with. + Kind::MutStr => vec![ + ("fn_value.empty_string", "Box::leak(String::new().into_boxed_str())".into()), + ( + "fn_value.xyzzy_string", + "Box::leak(String::from(\"xyzzy\").into_boxed_str())".into(), + ), + ], + + Kind::String => vec![ + ("fn_value.empty_string", "String::new()".into()), + ("fn_value.xyzzy_string", "\"xyzzy\".to_owned()".into()), + ], + + // A `NonZero` cannot hold the zero every other numeric type offers, so the interesting + // values are the smallest it can hold and one that is merely different. + Kind::NonZero => vec![ + ("fn_value.one", format_compact!("{}::new(1).unwrap()", type_text(ty))), + ("fn_value.two", format_compact!("{}::new(2).unwrap()", type_text(ty))), + ], + + _ => Vec::new(), + } +} + +/// The values for a reference return, produced by leaking a box. +/// +/// A reference has to point at something that outlives the call, and the obvious spellings do not: +/// `&Default::default()` borrows a temporary that dies at the end of the expression, so the mutant +/// fails to compile rather than answering anything. `Box::leak` is what makes the family reach +/// these returns at all — it yields a `&'static mut T`, which coerces to a reference of any shorter +/// lifetime and to a shared one, so `&T`, `&'a T` and `&mut T` are all served by the same text. +/// +/// This matters more than it sounds. A getter handing back `&String` or `&[T]` behind a reference +/// is one of the commonest shapes in Rust, and until this existed every one of them was passed over +/// in silence — not reported as unmutatable, simply absent, so a suite that never checked what a +/// getter returned still scored perfectly. +/// +/// The values are the element type's own, so `&Vec` offers an empty vector and a one-element +/// one exactly as `Vec` would. The leak is deliberate and its cost is bounded by the mutant's +/// own lifetime: the process running the tests exits shortly afterwards. A mutant leaking on a hot +/// path can exhaust the memory limit and be reported as `OUTOFMEM`, which is a kill — the mutant +/// did change observable behaviour — though not the one the tests were asked about. That is the +/// boundary of the policy: the leak's own allocation is allowed to decide a verdict, but only ever +/// as a documented kill, never as a survivor a suite could be blamed for. +pub(super) fn reference_values(ty: &Type, depth: usize, types: &Types<'_>) -> Vec { + let Some(elem) = reference_elem(ty) else { + return Vec::new(); + }; + + // `Box::leak` hands back `&mut T`. Where the signature asked for `&T` that is usually invisible, + // because a return position reborrows it silently — but not always. As an `impl Iterator`'s + // item it is the thing the item type is *inferred from*, so `&mut String` is inferred where + // `&String` was promised and the mutant is withdrawn as unviable. Reborrowing here says what + // was meant in every position instead of relying on one of them being forgiving. + let shared = matches!(strip(ty), Type::Reference(reference) if reference.mutability.is_none()); + let prefix = if shared { "&*" } else { "" }; + + cap(values_for(elem, depth.saturating_sub(1), types) + .into_iter() + .map(|(name, text)| (name, format_compact!("{prefix}Box::leak(Box::new({text}))"))) + .collect()) +} + +/// The type a reference points at, seeing through parentheses and invisible grouping. +pub(super) fn reference_elem(ty: &Type) -> Option<&Type> { + match ty { + Type::Reference(reference) => Some(&reference.elem), + Type::Paren(paren) => reference_elem(&paren.elem), + _ => None, + } +} + +/// Truncates a value list to the width bound. +pub(super) fn cap(mut values: Vec) -> Vec { + values.truncate(RETURN_WIDTH); + values +} + +/// The values of a generic type's `index`th type parameter. +/// +/// Lifetime and const parameters are skipped, so `Cow<'a, str>` finds `str` at index zero the way +/// `Option` finds `T`. +pub(super) fn inner_values(ty: &Type, index: usize, depth: usize, types: &Types<'_>) -> Vec { + type_argument(ty, index).map_or_else( + || vec![("fn_value.default", "Default::default()".into())], + |inner| values_for(inner, depth.saturating_sub(1), types), + ) +} + +/// The `Item` type an `impl Iterator` signature binds, when it wrote one. +/// +/// A bare `impl Iterator`, or one whose item is itself opaque, gives nothing to build a value +/// from. That costs only the one-element mutant: the empty one needs no item type, because the +/// wrapper infers it from the arm holding the original. +pub(super) fn iterator_item(ty: &Type) -> Option<&Type> { + let Type::ImplTrait(imp) = strip(ty) else { + return None; + }; + + imp.bounds.iter().find_map(|bound| { + let TypeParamBound::Trait(tr) = bound else { + return None; + }; + + let PathArguments::AngleBracketed(args) = &tr.path.segments.last()?.arguments else { + return None; + }; + + args.args.iter().find_map(|arg| match arg { + GenericArgument::AssocType(assoc) if assoc.ident == "Item" => Some(&assoc.ty), + _ => None, + }) + }) +} + +/// The `index`th type argument of a path type, ignoring lifetimes and const generics. +pub(super) fn type_argument(ty: &Type, index: usize) -> Option<&Type> { + let Type::Path(path) = strip(ty) else { + return None; + }; + + let PathArguments::AngleBracketed(args) = &path.path.segments.last()?.arguments else { + return None; + }; + + args.args + .iter() + .filter_map(|arg| match arg { + GenericArgument::Type(inner) => Some(inner), + _ => None, + }) + .nth(index) +} + +/// The path text of a type, so that an associated function can be called on it. +pub(super) fn type_text(ty: &Type) -> String { + match strip(ty) { + Type::Path(path) => path + .path + .segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect::>() + .join("::"), + _ => "Default".to_owned(), + } +} + +/// The number of type arguments a path segment carries. +/// +/// Lifetimes and const arguments are not counted, because they say nothing about which type is +/// being named: `Cow<'a, str>` names one type, not two. +pub(super) fn type_arguments(segment: &PathSegment) -> usize { + let PathArguments::AngleBracketed(args) = &segment.arguments else { + return 0; + }; + + args.args.iter().filter(|arg| matches!(arg, GenericArgument::Type(_))).count() +} + +/// The constructor path for a collection type, keeping any qualification the author wrote. +pub(super) fn collection_ctor(ty: &Type) -> String { + let Type::Path(path) = strip(ty) else { + return "Vec".to_owned(); + }; + + path.path + .segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect::>() + .join("::") +} + +/// The constructor for a smart-pointer type. +pub(super) fn wrapper_ctor(ty: &Type) -> String { + format!("{}::new", collection_ctor(ty)) +} + +/// Sees through parentheses to the type underneath. +pub(super) fn strip(ty: &Type) -> &Type { + match ty { + Type::Paren(paren) => strip(&paren.elem), + other => other, + } +} + +/// The coarse classification of a return type that decides which values are worth trying. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Kind { + Unit, + Bool, + Signed, + Unsigned, + Float, + StaticStr, + /// A mutable string slice, whose values cannot be string literals. + MutStr, + String, + NonZero, + Option, + Result, + /// Anything built from an iterator of a single element type: `Vec`, `VecDeque`, sets, heaps. + Collection, + /// Anything built from an iterator of key-value pairs. + Map, + /// A smart pointer constructed by `new`: `Box`, `Rc`, `Arc`. + Wrapper, + Cow, + Iterator, + Tuple, + Reference, + Unknown, +} + +/// Classifies a return type syntactically. +/// +/// Only the last segment of a path is compared, because a standard type may be written bare, fully +/// qualified, or re-exported, and there is no name resolution here to tell those apart. That alone +/// would take any type whose final segment reads `Vec` for the standard one, so the count of type +/// arguments is checked as well: a `Vec` that takes none is somebody else's `Vec`, and treating it +/// as the standard one produces a mutant that cannot compile. A name that carries the wrong number +/// of type arguments is therefore classified as unknown, which still offers `Default::default()` +/// and so keeps a guess without pretending to know the shape. +/// +/// A local type that shadows a standard name *and* matches its arity — a bare `struct String` — +/// remains indistinguishable and will yield a mutant that does not compile. That mutant is +/// classified as unviable and reported as such, which is the accepted cost of not resolving names. +pub(super) fn resolve_type(ty: &Type) -> Kind { + match ty { + Type::Tuple(tuple) if tuple.elems.is_empty() => Kind::Unit, + + Type::Reference(reference) => match &*reference.elem { + // Mutability is load-bearing, not decoration: `StaticStr`'s values are string literals, + // which are `&'static str` and cannot be returned where `&mut str` was promised. A + // mutable one therefore gets its own kind rather than being folded in here. + Type::Path(path) if path.path.is_ident("str") => { + if reference.mutability.is_some() { + Kind::MutStr + } else { + Kind::StaticStr + } + } + + // `&[T]` has a `Default`, unlike references in general, so it keeps the mutant that + // depends on one. + Type::Slice(_) => Kind::Unknown, + + _ => Kind::Reference, + }, + + Type::Tuple(_) => Kind::Tuple, + + Type::Paren(paren) => resolve_type(&paren.elem), + + // The traits whose `impl Trait` returns this tool can synthesize a value for. All four are + // satisfied by `gamma_rt::Either` whenever both of its sides satisfy them, and both + // `core::iter::empty()` and `core::iter::once(v)` satisfy all four. + // + // Any other `impl Trait` has no expression this tool can name that is guaranteed to + // satisfy it. + Type::ImplTrait(imp) => { + let iterator = imp.bounds.iter().any(|bound| { + matches!(bound, TypeParamBound::Trait(tr) + if tr.path.segments.last().is_some_and(|segment| { + matches!(segment.ident.to_string().as_str(), + "Iterator" | "DoubleEndedIterator" | "ExactSizeIterator" | "FusedIterator") + })) + }); + + if iterator { Kind::Iterator } else { Kind::Unknown } + } + + Type::Path(path) => { + let Some(segment) = path.path.segments.last() else { + return Kind::Unknown; + }; + + let name = segment.ident.to_string(); + let arity = type_arguments(segment); + + if name.starts_with("NonZero") && name != "NonZero" { + return if arity == 0 { Kind::NonZero } else { Kind::Unknown }; + } + + match (name.as_str(), arity) { + ("bool", 0) => Kind::Bool, + ("i8" | "i16" | "i32" | "i64" | "i128" | "isize", 0) => Kind::Signed, + ("u8" | "u16" | "u32" | "u64" | "u128" | "usize", 0) => Kind::Unsigned, + ("f32" | "f64", 0) => Kind::Float, + ("String", 0) => Kind::String, + ("Option", 1..) => Kind::Option, + ("Result", 1..) => Kind::Result, + ("Vec" | "VecDeque" | "HashSet" | "BTreeSet" | "BinaryHeap" | "LinkedList", 1..) => Kind::Collection, + ("HashMap" | "BTreeMap", 2..) => Kind::Map, + ("Box" | "Rc" | "Arc", 1..) => Kind::Wrapper, + ("Cow", 1..) => Kind::Cow, + _ => Kind::Unknown, + } + } + + _ => Kind::Unknown, + } +} + +#[cfg(test)] +mod tests { + use syn::punctuated::Punctuated; + use syn::{Path, TypeImplTrait, TypePath, parse_quote}; + + use super::*; + use crate::HashMap; + use crate::ops::collect::Defaults; + + fn empty_path_type() -> Type { + Type::Path(TypePath { + qself: None, + path: Path { + leading_colon: None, + segments: Punctuated::::new(), + }, + }) + } + + fn test_types<'a>(abstracts: &'a [String], imports: &'a HashMap>>, defaults: &'a Defaults) -> Types<'a> { + Types { + abstracts, + imports, + defaults, + self_type: None, + self_associated: None, + } + } + + #[test] + fn unknown_undefaultable_types_and_non_tuples_contribute_fallbacks() { + let abstracts = Vec::new(); + let imports = HashMap::default(); + let defaults = Defaults::default(); + let types = test_types(&abstracts, &imports, &defaults); + let foreign_error: Type = parse_quote!(std::io::Error); + let plain: Type = parse_quote!(String); + + assert!(values_for(&foreign_error, RETURN_DEPTH, &types).is_empty()); + assert_eq!( + tuple_values(&plain, RETURN_DEPTH, &types), + vec![("fn_value.default", "Default::default()".into())] + ); + assert!(literal_values(Kind::Option, &parse_quote!(Option)).is_empty()); + } + + #[test] + fn reference_and_type_argument_helpers_reject_non_matching_syntax() { + let abstracts = Vec::new(); + let imports = HashMap::default(); + let defaults = Defaults::default(); + let types = test_types(&abstracts, &imports, &defaults); + let plain: Type = parse_quote!(String); + let bare_collection: Type = parse_quote!(Vec); + let tuple: Type = parse_quote!((bool,)); + let reference: Type = parse_quote!(&str); + + assert!(reference_values(&plain, RETURN_DEPTH, &types).is_empty()); + assert_eq!(reference_elem(&plain), None); + assert_eq!(type_argument(&reference, 0), None); + assert_eq!(type_argument(&bare_collection, 0), None); + assert_eq!( + inner_values(&bare_collection, 0, RETURN_DEPTH, &types), + vec![("fn_value.default", "Default::default()".into())] + ); + assert_eq!(type_text(&tuple), "Default"); + assert_eq!(collection_ctor(&tuple), "Vec"); + } + + #[test] + fn iterator_item_requires_an_impl_trait_with_item_binding() { + let plain: Type = parse_quote!(Vec); + let imp = Type::ImplTrait(TypeImplTrait { + impl_token: syn::token::Impl::default(), + bounds: Punctuated::from_iter([ + TypeParamBound::Lifetime(parse_quote!('static)), + TypeParamBound::Trait(parse_quote!(Iterator)), + ]), + }); + + assert_eq!(iterator_item(&plain), None); + assert_eq!(iterator_item(&imp), None); + } + + #[test] + fn resolve_type_treats_an_empty_path_as_unknown() { + assert_eq!(resolve_type(&empty_path_type()), Kind::Unknown); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/defaults.rs b/crates/cargo-gamma-engine/src/ops/collect/defaults.rs new file mode 100644 index 000000000..7e24b1233 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/defaults.rs @@ -0,0 +1,1063 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::hash_map::Entry; + +use syn::visit::{self, Visit}; +use syn::{ + File, GenericParam, Generics, ItemEnum, ItemImpl, ItemStruct, ItemType, ItemUnion, Path, Type, TypeParamBound, UseTree, WherePredicate, +}; + +use crate::cfg::CfgSet; +use crate::{HashMap, HashSet}; + +/// The spellings in one file that name the standard-library `Default` trait. +/// +/// Rust's prelude makes bare `Default` normally mean `core::default::Default`, but local items +/// and imports can shadow it in different namespaces. The collector cannot run name resolution, +/// so it records explicit standard aliases and explicit shadows before it decides whether a call, +/// bound, derive, or impl refers to the standard trait. +#[derive(Debug, Default)] +pub(super) struct DefaultPaths { + aliases: HashSet, + module_aliases: HashMap>, + shadows: HashSet, + derive_shadows: HashSet, +} + +impl DefaultPaths { + /// Collects standard-trait aliases and local shadows from a parsed file. + pub(super) fn of(file: &File) -> Self { + Self::of_in(file, &CfgSet::unconditional()) + } + + /// Collects bindings visible in this file's outer lexical scope. + /// + /// A child module owns its imports and declarations: letting its `Default` shadow the parent + /// makes an unrelated outer `impl Default` look custom. Conditional items likewise introduce + /// no binding when the selected build strips them. + pub(super) fn of_in(file: &File, cfg: &CfgSet) -> Self { + let mut paths = Self::default(); + + for item in &file.items { + if !cfg.holds_for(item_attrs(item)) { + continue; + } + + match item { + syn::Item::Use(node) => paths.descend_use(&mut Vec::new(), &node.tree), + syn::Item::Trait(node) if node.ident == "Default" => { + let _added = paths.shadows.insert("Default".to_owned()); + } + syn::Item::Struct(node) => paths.note_type_shadow(&node.ident), + syn::Item::Enum(node) => paths.note_type_shadow(&node.ident), + syn::Item::Union(node) => paths.note_type_shadow(&node.ident), + syn::Item::Type(node) => paths.note_type_shadow(&node.ident), + syn::Item::Mod(node) => paths.note_type_shadow(&node.ident), + _ => {} + } + } + + paths + } + + /// Returns whether an implementation path names the standard `Default` trait. + pub(super) fn is_standard_trait(&self, path: &Path) -> bool { + let segments: Vec = path.segments.iter().map(|segment| segment.ident.to_string()).collect(); + + self.is_standard_trait_segments(&segments) + } + + /// Returns whether the fallback spelling `Default::default()` resolves to this trait. + /// + /// A bare custom trait named `Default` shadows the prelude, so it is not the standard trait, + /// but the fallback text would still recurse inside its own `default` method. A standard trait + /// written through an alias recurses only when bare `Default` still resolves to that standard + /// trait; qualified and differently aliased custom traits stay mutable. + pub(super) fn is_fallback_trait(&self, path: &Path) -> bool { + if path.segments.len() == 1 && path.segments.first().is_some_and(|segment| segment.ident == "Default") { + return true; + } + + self.is_standard_trait(path) && !self.shadows.contains("Default") + } + + /// Returns whether a bare name resolves to the standard `Default` trait. + fn is_standard_trait_name(&self, name: &str) -> bool { + self.aliases.contains(name) || (name == "Default" && !self.shadows.contains(name)) + } + + /// Returns whether a complete associated-function path calls the standard default trait. + pub(super) fn is_standard_default_callee(&self, path: &Path) -> bool { + let mut segments: Vec = path.segments.iter().map(|segment| segment.ident.to_string()).collect(); + + if segments.pop().as_deref() != Some("default") { + return false; + } + + self.is_standard_callee_segments(&segments) + } + + pub(super) fn is_standard_default_segments(&self, segments: &[String]) -> bool { + let Some((method, qualifier)) = segments.split_last() else { + return false; + }; + + method == "default" && self.is_standard_callee_segments(qualifier) + } + + /// Returns whether a derive path invokes the built-in standard `Default` derive. + fn is_standard_derive(&self, path: &Path) -> bool { + path.is_ident("Default") && !self.derive_shadows.contains("Default") + } + + /// Records one imported binding and the complete path it came from. + fn imported(&mut self, binding: String, prefix: &[String], source: String) { + let mut path = prefix.to_vec(); + + if source != "self" { + path.push(source); + } + + let path = self.expand_module_alias(&path); + + if is_standard_trait_path(&path) { + let _added = self.aliases.insert(binding); + } else if is_standard_module_path(&path) { + let _replaced = self.module_aliases.insert(binding.clone(), path); + + if binding == "Default" { + let _added = self.shadows.insert(binding); + } + } else if binding == "Default" { + let _added = self.shadows.insert(binding.clone()); + let _added = self.derive_shadows.insert(binding); + } + } + + /// Records every binding one `use` tree introduces. + fn descend_use(&mut self, prefix: &mut Vec, tree: &UseTree) { + match tree { + UseTree::Path(path) => { + prefix.push(path.ident.to_string()); + self.descend_use(prefix, &path.tree); + let _popped = prefix.pop(); + } + + UseTree::Name(name) => self.imported(name.ident.to_string(), prefix, name.ident.to_string()), + UseTree::Rename(rename) => self.imported(rename.rename.to_string(), prefix, rename.ident.to_string()), + + UseTree::Group(group) => { + for item in &group.items { + self.descend_use(prefix, item); + } + } + + UseTree::Glob(_) => {} + } + } + + /// Resolves a direct or imported spelling of the standard trait. + fn is_standard_trait_segments(&self, segments: &[String]) -> bool { + if let [name] = segments + && self.is_standard_trait_name(name) + { + return true; + } + + is_standard_trait_path(&self.expand_module_alias(segments)) + } + + /// Resolves the qualifier of an associated call through the type namespace. + fn is_standard_callee_segments(&self, segments: &[String]) -> bool { + if let [name] = segments + && (self.aliases.contains(name) || (name == "Default" && !self.shadows.contains(name))) + { + return true; + } + + is_standard_trait_path(&self.expand_module_alias(segments)) + } + + /// Resolves the first segment of a path through one or more imported standard modules. + fn expand_module_alias(&self, segments: &[String]) -> Vec { + let mut expanded = segments.to_vec(); + + for _ in 0..segments.len() { + let first = expanded + .first() + .expect("alias expansion cannot consume more segments than the original path contains"); + let Some(prefix) = self.module_aliases.get(first) else { + break; + }; + + let _spliced = expanded.splice(..1, prefix.iter().cloned()); + } + + expanded + } +} + +/// Returns whether a path spells the standard `Default` trait without imports. +fn is_standard_trait_path(path: &[String]) -> bool { + matches!( + path, + [root, module, name] + if matches!(root.as_str(), "std" | "core") && module == "default" && name == "Default" + ) +} + +/// Returns whether a path names a standard module that can be imported under an alias. +fn is_standard_module_path(path: &[String]) -> bool { + matches!(path, [root] if matches!(root.as_str(), "std" | "core")) + || matches!(path, [root, module] if matches!(root.as_str(), "std" | "core") && module == "default") +} + +impl DefaultPaths { + /// Records a source declaration that occupies the bare type/module name `Default`. + fn note_type_shadow(&mut self, ident: &syn::Ident) { + if ident == "Default" { + let _added = self.shadows.insert("Default".to_owned()); + } + } +} + +/// Returns the outer attributes of an item. +fn item_attrs(item: &syn::Item) -> &[syn::Attribute] { + match item { + syn::Item::Const(node) => &node.attrs, + syn::Item::Enum(node) => &node.attrs, + syn::Item::ExternCrate(node) => &node.attrs, + syn::Item::Fn(node) => &node.attrs, + syn::Item::ForeignMod(node) => &node.attrs, + syn::Item::Impl(node) => &node.attrs, + syn::Item::Macro(node) => &node.attrs, + syn::Item::Mod(node) => &node.attrs, + syn::Item::Static(node) => &node.attrs, + syn::Item::Struct(node) => &node.attrs, + syn::Item::Trait(node) => &node.attrs, + syn::Item::TraitAlias(node) => &node.attrs, + syn::Item::Type(node) => &node.attrs, + syn::Item::Union(node) => &node.attrs, + syn::Item::Use(node) => &node.attrs, + _ => &[], + } +} + +/// Names type parameters whose bounds explicitly promise the standard `Default` trait. +pub(super) fn standard_defaulted_parameters(generics: &Generics, defaults: &DefaultPaths) -> Vec { + let mut names = Vec::new(); + + for parameter in &generics.params { + let GenericParam::Type(parameter) = parameter else { + continue; + }; + + if parameter.bounds.iter().any(|bound| standard_default_bound(bound, defaults)) { + names.push(parameter.ident.to_string()); + } + } + + if let Some(where_clause) = &generics.where_clause { + for predicate in &where_clause.predicates { + let WherePredicate::Type(predicate) = predicate else { + continue; + }; + let Type::Path(path) = &predicate.bounded_ty else { + continue; + }; + let Some(name) = path.path.get_ident() else { + continue; + }; + + if predicate.bounds.iter().any(|bound| standard_default_bound(bound, defaults)) { + let name = name.to_string(); + + if !names.contains(&name) { + names.push(name); + } + } + } + } + + names +} + +/// Returns whether one type bound names the standard `Default` trait. +fn standard_default_bound(bound: &TypeParamBound, defaults: &DefaultPaths) -> bool { + matches!(bound, TypeParamBound::Trait(bound) if defaults.is_standard_trait(&bound.path)) +} + +/// What the workspace's own sources say about which of their types implement `Default`. +/// +/// The `fn_value` family and its relatives reach for `Default::default()` whenever they cannot name +/// a value of a type. For a type this workspace defines, that guess can be checked without any type +/// resolution: the definition is in a file that was parsed anyway, and either it derives the +/// standard `Default`, something writes a standard `impl Default for` it, or it has none. +/// +/// The index is evidence of *presence* and never proof of absence for anything it cannot see whole. +/// A type it has no definition for stays optimistic, because it may come from a dependency that +/// does implement `Default`. A definition it can see but whose `Default` a macro generates is the +/// one way it can be wrong, and it errs in the safe direction: a mutant is withheld that would have +/// compiled, which costs a little signal rather than producing a wrong verdict. +#[derive(Debug, Default)] +pub struct Defaults { + /// The local bindings that decide whether `Default` means the standard trait. + paths: DefaultPaths, + + /// The `struct`, `enum` and `union` names this workspace defines. + /// + /// Type aliases are deliberately absent: an alias names someone else's type, so its `Default` + /// is that type's business and nothing here can settle it. + defined: HashSet, + + /// Those of them that derive or implement the standard `Default` trait. + defaulted: HashSet, + + /// For each `type Alias = Result`, the name of that default error type. + /// + /// A crate-wide `Result` alias is close to universal in real Rust, and it hides the error type + /// from every signature that uses it. Recording what the alias fixed it to is what lets a + /// one-argument `Result` be reasoned about at all. + /// + /// The names are unqualified, and essentially every crate calls its alias `Result`, so a + /// workspace of several crates routinely disagrees about what one key means. `None` records + /// exactly that: the alias was seen naming more than one error type, so nothing can be + /// concluded from it. Keeping the disagreement is what makes the index independent of the + /// order the files happened to be parsed in. + result_error: HashMap>, + + /// The selected configuration, used to ignore declarations that introduce no binding in this + /// build. + cfg: CfgSet, +} + +/// Folds one alias into the map, demoting a key two files disagree about. +/// +/// The last writer must not win. The partials are built by worker threads and folded in completion +/// order, so a map that overwrote on collision would give a different answer on every run over an +/// unchanged workspace — and the answer decides whether the `result.ok_to_err` family is emitted at +/// all, which is the largest screened family there is. +/// +/// Demotion errs toward emitting, which is the safe direction: a mutant that turns out not to +/// compile costs a rollback round, whereas one that is withheld leaves a real gap in the suite +/// looking like a better score. +fn merge_alias(into: &mut HashMap>, alias: String, error: Option) { + match into.entry(alias) { + Entry::Occupied(mut seen) => { + if *seen.get() != error { + *seen.get_mut() = None; + } + } + Entry::Vacant(empty) => { + let _inserted = empty.insert(error); + } + } +} + +impl Defaults { + /// Builds the index over one parsed file. + /// + /// A workspace's index is the [`absorb`](Self::absorb) of one of these per file, which is what + /// lets the files be read on whichever thread happens to claim them. + #[must_use] + pub fn of(file: &File) -> Self { + Self::of_in(file, &CfgSet::unconditional()) + } + + /// Builds the index over one parsed file for the selected configuration. + #[must_use] + pub fn of_in(file: &File, cfg: &CfgSet) -> Self { + let mut index = Self { + paths: DefaultPaths::of_in(file, cfg), + cfg: cfg.clone(), + ..Self::default() + }; + + index.visit_file(file); + index + } + + /// Returns whether the workspace defines this type and gives it no `Default`. + /// + /// Names are compared unqualified, so two crates in one workspace can both define a `Config`. + /// Presence wins that collision: if either of them has a `Default`, neither is screened. The + /// alternative would withhold a mutant that compiles, and this index exists to be conservative. + #[must_use] + pub fn lacks_default(&self, ty: &Type) -> bool { + let Some(name) = name_of(ty) else { + return false; + }; + + self.lacks_error_default(&name) + } + + /// Folds another index into this one. + /// + /// Used to put together what several threads each learned from the files they parsed, in + /// whatever order they finished. Every field resolves a collision without reference to that + /// order, because the order is not reproducible and the result decides which mutants exist. + /// + /// `defined` and `defaulted` are unions, which is what makes presence win: a name one crate + /// defines without a `Default` and another defines with one ends up in both sets, and is not + /// screened. `result_error` cannot union, because its values are single names rather than + /// membership, so a key two files disagree about is demoted to "unknown" instead — see + /// [`merge_alias`]. + pub fn absorb(&mut self, other: Self) { + self.defined.extend(other.defined); + self.defaulted.extend(other.defaulted); + + for (alias, error) in other.result_error { + merge_alias(&mut self.result_error, alias, error); + } + } + + /// Returns whether a type named by the index has no `Default`, given only its name. + /// + /// The by-name form exists for an error type reached through an alias, where what was recorded + /// is a name rather than a syntax node. + #[must_use] + pub fn lacks_error_default(&self, name: &str) -> bool { + self.defined.contains(name) && !self.defaulted.contains(name) + } + + /// Returns the error type a `Result` alias fixed, given the alias's name. + /// + /// `None` covers both "no such alias" and "the workspace disagrees about this one", which are + /// the same answer to the only question asked of it: nothing may be screened on this name. + #[must_use] + pub fn aliased_error(&self, alias: &str) -> Option<&str> { + self.result_error.get(alias)?.as_deref() + } + + fn note_derive(&mut self, name: &str, attributes: &[syn::Attribute]) { + let _inserted = self.defined.insert(name.to_owned()); + + if derives_default(attributes, &self.paths) { + let _inserted = self.defaulted.insert(name.to_owned()); + } + } +} + +#[expect( + clippy::renamed_function_params, + reason = "syn names every visitor parameter `i`, which says nothing about what it is" +)] +impl<'ast> Visit<'ast> for Defaults { + fn visit_item_struct(&mut self, node: &'ast ItemStruct) { + if !self.cfg.holds_for(&node.attrs) { + return; + } + + self.note_derive(&node.ident.to_string(), &node.attrs); + + visit::visit_item_struct(self, node); + } + + fn visit_item_enum(&mut self, node: &'ast ItemEnum) { + if !self.cfg.holds_for(&node.attrs) { + return; + } + + self.note_derive(&node.ident.to_string(), &node.attrs); + + visit::visit_item_enum(self, node); + } + + fn visit_item_union(&mut self, node: &'ast ItemUnion) { + if !self.cfg.holds_for(&node.attrs) { + return; + } + + self.note_derive(&node.ident.to_string(), &node.attrs); + + visit::visit_item_union(self, node); + } + + fn visit_item_impl(&mut self, node: &'ast ItemImpl) { + if !self.cfg.holds_for(&node.attrs) { + return; + } + + if let Some((_bang, path, _for)) = &node.trait_ + && self.paths.is_standard_trait(path) + && let Some(name) = name_of(&node.self_ty) + { + let _inserted = self.defaulted.insert(name); + } + + visit::visit_item_impl(self, node); + } + + fn visit_item_type(&mut self, node: &'ast ItemType) { + if !self.cfg.holds_for(&node.attrs) { + return; + } + + // Only an alias for `Result` matters, and only when it supplies its own error type as a + // parameter default. `type Result = core::result::Result` — with the error + // written into the right-hand side rather than defaulted — is handled by the same read, + // because the alias's target is what names the error either way. + if name_of(&node.ty).as_deref() == Some("Result") { + let defaulted = node.generics.params.iter().find_map(|param| match param { + GenericParam::Type(ty) => ty.default.as_ref().and_then(name_of), + _ => None, + }); + + let written = payload_name(&node.ty, 1); + + if let Some(error) = defaulted.or(written) { + // One file can hold two aliases of the same name, in separate modules, just as two + // files can. Both go through the same demotion. + merge_alias(&mut self.result_error, node.ident.to_string(), Some(error)); + } + } + + visit::visit_item_type(self, node); + } +} + +/// Returns whether a `#[derive(...)]` on an item lists `Default`. +fn derives_default(attributes: &[syn::Attribute], paths: &DefaultPaths) -> bool { + let mut found = false; + + for attribute in attributes { + if !attribute.path().is_ident("derive") { + continue; + } + + // Ignored rather than propagated: an attribute this cannot parse is one whose contents are + // unknown, and the index's whole contract is that not knowing means staying optimistic. + let _ignored = attribute.parse_nested_meta(|meta| { + if paths.is_standard_derive(&meta.path) { + found = true; + } + + Ok(()) + }); + } + + found +} + +/// The last segment of a type's path, which is the name the index is keyed by. +fn name_of(ty: &Type) -> Option { + match ty { + Type::Path(path) => path.path.segments.last().map(|segment| segment.ident.to_string()), + Type::Paren(paren) => name_of(&paren.elem), + Type::Group(group) => name_of(&group.elem), + _ => None, + } +} + +/// The name of a type's `index`th generic argument. +fn payload_name(ty: &Type, index: usize) -> Option { + let Type::Path(path) = ty else { + return None; + }; + + let syn::PathArguments::AngleBracketed(args) = &path.path.segments.last()?.arguments else { + return None; + }; + + args.args + .iter() + .filter_map(|arg| match arg { + syn::GenericArgument::Type(ty) => Some(ty), + _ => None, + }) + .nth(index) + .and_then(name_of) +} + +#[cfg(test)] +mod tests { + use syn::parse_quote; + + use super::*; + + fn index(sources: &[&str]) -> Defaults { + let mut defaults = Defaults::default(); + + for source in sources { + defaults.absorb(Defaults::of(&syn::parse_file(source).unwrap())); + } + + defaults + } + + #[test] + fn a_type_the_workspace_defines_without_a_default_is_reported() { + let defaults = index(&["pub struct Error { code: u8 }"]); + let ty: Type = parse_quote!(Error); + + assert!(defaults.lacks_default(&ty)); + } + + #[test] + fn a_derived_default_is_seen() { + let defaults = index(&["#[derive(Debug, Default)] pub struct Error;"]); + let ty: Type = parse_quote!(Error); + + assert!(!defaults.lacks_default(&ty)); + } + + /// An `enum` is indexed the same way a `struct` is -- its own derive is read, and the type is + /// no longer merely optimistic about having a `Default`. + #[test] + fn an_enum_with_a_derived_default_is_seen() { + let defaults = index(&["#[derive(Default)] pub enum Choice { #[default] A, B }"]); + let ty: Type = parse_quote!(Choice); + + assert!(!defaults.lacks_default(&ty)); + } + + #[test] + fn a_written_default_impl_is_seen_from_another_file() { + let defaults = index(&["pub struct Error;", "impl Default for Error { fn default() -> Self { Self } }"]); + let ty: Type = parse_quote!(Error); + + assert!(!defaults.lacks_default(&ty)); + } + + #[test] + fn only_standard_default_implementations_and_derives_are_indexed() { + let ty: Type = parse_quote!(Error); + let standard_alias = index(&[ + "pub struct Error;", + "use core::default::Default as StdDefault; impl StdDefault for Error { fn default() -> Self { Self } }", + ]); + let standard_module_alias = index(&[ + "pub struct Error;", + "use core::default as defaults; impl defaults::Default for Error { fn default() -> Self { Self } }", + ]); + let standard_derive = index(&["struct Default;", "#[derive(Default)] pub struct Error;"]); + let custom = index(&[ + "pub struct Error;", + "mod custom { pub trait Default { fn default() -> Self; } } impl custom::Default for Error { fn default() -> Self { Self } }", + ]); + let custom_alias = index(&[ + "pub struct Error;", + "mod custom { pub trait Default { fn default() -> Self; } } use custom::Default as Alias; impl Alias for Error { fn default() -> Self { Self } }", + ]); + let custom_derive = index(&["#[derive(custom::Default)] pub struct Error;"]); + + assert!(!standard_alias.lacks_default(&ty)); + assert!(!standard_module_alias.lacks_default(&ty)); + assert!(!standard_derive.lacks_default(&ty)); + assert!(custom.lacks_default(&ty)); + assert!(custom_alias.lacks_default(&ty)); + assert!(custom_derive.lacks_default(&ty)); + } + + #[test] + fn nested_and_disabled_default_shadows_do_not_poison_an_outer_implementation() { + let source = " + struct Error; + mod nested { + trait Default {} + use custom::Default; + } + #[cfg(any())] + trait Default {} + impl Default for Error { + fn default() -> Self { Self } + } + "; + let defaults = Defaults::of_in(&syn::parse_file(source).unwrap(), &CfgSet::parse("unix")); + let ty: Type = parse_quote!(Error); + + assert!(!defaults.lacks_default(&ty)); + } + + /// A `union` named `Default` shadows the prelude exactly like a `struct`, `enum`, `type` or + /// `mod` of that name does — `note_type_shadow` treats every kind of type-namespace + /// declaration alike. + #[test] + fn a_union_named_default_shadows_the_prelude() { + let source = " + struct Error; + union Default { flag: u8 } + impl Default for Error { + fn default() -> Self { Self } + } + "; + let defaults = Defaults::of_in(&syn::parse_file(source).unwrap(), &CfgSet::unconditional()); + let ty: Type = parse_quote!(Error); + + assert!(defaults.lacks_default(&ty), "a custom `Default` union shadows the standard trait"); + } + + /// A grouped `use` such as `use std::{fmt, default::Default}` introduces every item in the + /// group, not only the first, and a glob import introduces no binding this module can + /// interpret — but must not panic doing nothing with it. + #[test] + fn grouped_and_glob_use_trees_are_both_handled() { + let defaults = index(&["pub struct Error; + use std::{fmt, default::Default as StdDefault}; + impl StdDefault for Error { fn default() -> Self { Self } } + use std::collections::*;"]); + let ty: Type = parse_quote!(Error); + + assert!( + !defaults.lacks_default(&ty), + "the aliased Default inside the group must still resolve" + ); + } + + /// Importing a custom, non-standard item under the bare name `Default` shadows the prelude — + /// the same outcome a locally *declared* `Default` produces, reached through the import path + /// instead of a declaration. + #[test] + fn importing_a_custom_default_directly_shadows_the_prelude() { + let source = " + struct Error; + mod custom { pub trait Default { fn default() -> Self; } } + use custom::Default; + impl Default for Error { + fn default() -> Self { Self } + } + "; + let defaults = Defaults::of_in(&syn::parse_file(source).unwrap(), &CfgSet::unconditional()); + let ty: Type = parse_quote!(Error); + + assert!(defaults.lacks_default(&ty), "importing a custom Default directly is still a shadow"); + } + + /// Aliasing the `core::default` module itself to the bare name `Default` — an unusual but + /// legal import — makes `Default` mean a module rather than the trait, and so must shadow the + /// prelude exactly as a locally declared or imported one does. + #[test] + fn aliasing_the_default_module_to_the_bare_name_shadows_the_prelude() { + let source = " + struct Error; + use core::default as Default; + impl Default for Error { + fn default() -> Self { Self } + } + "; + let defaults = Defaults::of_in(&syn::parse_file(source).unwrap(), &CfgSet::unconditional()); + let ty: Type = parse_quote!(Error); + + assert!( + defaults.lacks_default(&ty), + "aliasing the default module to `Default` shadows the bare trait name" + ); + } + + /// A single-segment alias of `std` or `core` itself (rather than of `std::default`) is + /// recorded as a standard module too, and a path built through it resolves exactly as one + /// built directly on `std`/`core` would. + #[test] + fn a_single_segment_standard_module_alias_still_resolves() { + let defaults = index(&["pub struct Error; + use std as MyStd; + impl MyStd::default::Default for Error { fn default() -> Self { Self } }"]); + let ty: Type = parse_quote!(Error); + + assert!(!defaults.lacks_default(&ty)); + } + + #[test] + fn a_type_the_workspace_does_not_define_stays_optimistic() { + let defaults = index(&["pub struct Error;"]); + let ty: Type = parse_quote!(Utf8Error); + + assert!(!defaults.lacks_default(&ty)); + } + + #[test] + fn a_qualified_path_is_keyed_by_its_last_segment() { + let defaults = index(&["pub struct Error;"]); + let ty: Type = parse_quote!(crate::error::Error); + + assert!(defaults.lacks_default(&ty)); + } + + #[test] + fn a_name_two_crates_disagree_about_stays_optimistic() { + let defaults = index(&["pub struct Config { a: u8 }", "#[derive(Default)] pub struct Config;"]); + let ty: Type = parse_quote!(Config); + + assert!(!defaults.lacks_default(&ty)); + } + + #[test] + fn an_alias_is_not_a_definition() { + let defaults = index(&["pub type Handle = std::fs::File;"]); + let ty: Type = parse_quote!(Handle); + + assert!(!defaults.lacks_default(&ty)); + } + + #[test] + fn a_result_alias_records_the_error_it_defaults_to() { + let defaults = index(&["pub type Result = core::result::Result;"]); + + assert_eq!(defaults.aliased_error("Result"), Some("Error")); + } + + #[test] + fn a_result_alias_records_the_error_written_into_its_target() { + let defaults = index(&["pub type Result = core::result::Result;"]); + + assert_eq!(defaults.aliased_error("Result"), Some("Error")); + } + + #[test] + fn an_alias_for_something_other_than_a_result_is_not_recorded() { + let defaults = index(&["pub type Pairs = Vec<(u8, u8)>;"]); + + assert_eq!(defaults.aliased_error("Pairs"), None); + } + + /// Two crates that each call their alias `Result` leave the name meaning nothing, whichever + /// order their files were folded in. + /// + /// The partials are built by worker threads and folded as they finish, so an index that let + /// the last writer win would give a different population — and a different score — on two runs + /// over an unchanged workspace. + #[test] + fn two_crates_disagreeing_about_result_demote_the_alias_in_either_order() { + const ONE: &str = "pub type Result = core::result::Result;"; + const TWO: &str = "pub type Result = core::result::Result;"; + + assert_eq!(index(&[ONE, TWO]).aliased_error("Result"), None); + assert_eq!(index(&[TWO, ONE]).aliased_error("Result"), None); + } + + /// Two crates that agree keep the answer, since there is nothing to disagree about. + #[test] + fn two_crates_agreeing_about_result_keep_the_error_they_agree_on() { + const SAME: &str = "pub type Result = core::result::Result;"; + + assert_eq!(index(&[SAME, SAME]).aliased_error("Result"), Some("Error")); + } + + /// A demoted alias stays demoted, rather than being revived by a later partial that happens to + /// name one of the two errors. + #[test] + fn an_alias_already_demoted_is_not_revived_by_a_later_agreement() { + let defaults = index(&[ + "pub type Result = core::result::Result;", + "pub type Result = core::result::Result;", + "pub type Result = core::result::Result;", + ]); + + assert_eq!(defaults.aliased_error("Result"), None); + } + + /// One file holding two aliases of the same name in separate modules is the same collision. + #[test] + fn one_file_disagreeing_with_itself_demotes_the_alias_too() { + let defaults = index(&["mod one { pub type Result = core::result::Result; } + mod two { pub type Result = core::result::Result; }"]); + + assert_eq!(defaults.aliased_error("Result"), None); + } + + /// `is_standard_default_callee` is only ever true for a path whose last segment is literally + /// `default`; anything else — including a path that otherwise looks like it names the trait — + /// is refused before the qualifier is even inspected. + #[test] + fn a_callee_path_not_ending_in_default_is_refused() { + let paths = DefaultPaths::default(); + let path: Path = parse_quote!(Default::new); + + assert!(!paths.is_standard_default_callee(&path)); + } + + /// `is_standard_default_segments` mirrors `is_standard_default_callee` for a segment list + /// that has already been split off a method name; an empty list has no last segment to + /// split and so is refused outright, and a non-`default` final segment is refused exactly as + /// the path form is. + #[test] + fn empty_or_non_default_segments_are_refused() { + let paths = DefaultPaths::default(); + + assert!(!paths.is_standard_default_segments(&[])); + assert!(!paths.is_standard_default_segments(&["Default".to_owned(), "new".to_owned()])); + assert!(paths.is_standard_default_segments(&["Default".to_owned(), "default".to_owned()])); + } + + /// `item_attrs` reads the attributes of every item kind it recognizes, including the ones no + /// other test in this module happens to declare, and falls back to an empty slice for a kind + /// it does not (a `Verbatim` item, which `syn` produces for tokens it does not interpret). + #[test] + fn item_attrs_reads_every_recognized_item_kind_and_falls_back_for_the_rest() { + let labeled = |item: syn::Item| { + assert_eq!( + item_attrs(&item).len(), + 1, + "expected exactly the one `#[allow(dead_code)]` attribute" + ); + }; + + labeled(parse_quote!( + #[allow(dead_code)] + extern crate core; + )); + labeled(parse_quote!( + #[allow(dead_code)] + extern "C" {} + )); + labeled(parse_quote!( + #[allow(dead_code)] + macro_rules! m { + () => {}; + } + )); + labeled(parse_quote!( + #[allow(dead_code)] + trait Alias = Clone; + )); + labeled(parse_quote!( + #[allow(dead_code)] + union U { + a: u8, + } + )); + + let verbatim = syn::Item::Verbatim(proc_macro2::TokenStream::new()); + + assert!( + item_attrs(&verbatim).is_empty(), + "an unrecognized item kind has no attributes to read" + ); + } + + /// A `where` clause is read the same way inline bounds are: a plain type bound reports its + /// parameter, and every other predicate shape -- a lifetime bound, a bound on a type this index + /// cannot key by a single name, and a bound on a multi-segment path -- is passed over rather than + /// mistaken for one. + #[test] + fn a_where_clause_bound_also_names_its_parameter_as_defaulted() { + let defaults = DefaultPaths::default(); + let item: syn::ItemFn = parse_quote! { + fn f<'a, T, U, V>() where T: Default, 'a: 'static, (U,): Default, some::U: Default, V: Clone {} + }; + + let names = standard_defaulted_parameters(&item.sig.generics, &defaults); + + assert_eq!(names, vec!["T".to_owned()]); + } + + /// The same parameter named twice -- once inline and once in the `where` clause -- is reported + /// only once. + #[test] + fn a_where_clause_repeating_an_inline_bound_is_not_reported_twice() { + let defaults = DefaultPaths::default(); + let item: syn::ItemFn = parse_quote! { + fn f() where T: Default {} + }; + + let names = standard_defaulted_parameters(&item.sig.generics, &defaults); + + assert_eq!(names, vec!["T".to_owned()]); + } + + /// Every item kind `Defaults` indexes is skipped, along with its own derive, when a predicate + /// controlling it does not hold -- matching the same rule every other item-level index in this + /// crate applies. + #[test] + fn every_indexed_item_kind_is_skipped_when_its_predicate_does_not_hold() { + let source = " + #[cfg(not(unix))] + #[derive(Debug)] + struct AStruct; + + #[cfg(not(unix))] + #[derive(Debug)] + enum AnEnum { Variant } + + #[cfg(not(unix))] + #[derive(Debug)] + union AUnion { flag: u8 } + + struct Excluded; + + #[cfg(not(unix))] + impl Default for Excluded { + fn default() -> Self { Self } + } + + #[cfg(not(unix))] + type ExcludedResult = core::result::Result; + "; + let defaults = Defaults::of_in(&syn::parse_file(source).unwrap(), &CfgSet::parse("unix")); + + assert!( + !defaults.lacks_default(&parse_quote!(AStruct)), + "an excluded struct stays unknown, rather than being screened as missing a default" + ); + assert!(!defaults.lacks_default(&parse_quote!(AnEnum)), "an excluded enum stays unknown too"); + assert!( + !defaults.lacks_default(&parse_quote!(AUnion)), + "an excluded union stays unknown too" + ); + assert!( + defaults.lacks_default(&parse_quote!(Excluded)), + "the excluded impl must not register a `Default` this type does not really have here" + ); + assert_eq!( + defaults.aliased_error("ExcludedResult"), + None, + "the excluded alias must not be registered" + ); + } + + /// A type alias's generic parameter can be a lifetime or a `const`, which carries no default + /// type to read, rather than the type parameter the search is looking for. + #[test] + fn a_type_alias_with_no_type_parameter_reads_the_alias_target_instead() { + let source = "struct Error; type MyResult<'a, const N: usize> = core::result::Result;"; + let defaults = Defaults::of_in(&syn::parse_file(source).unwrap(), &CfgSet::unconditional()); + + assert!(defaults.lacks_default(&parse_quote!(Error))); + } + + /// An attribute that is not `#[derive(...)]` is passed over, so a `#[derive(Default)]` sitting + /// beside an unrelated attribute is still found. + #[test] + fn a_non_derive_attribute_beside_a_real_derive_does_not_hide_it() { + let source = "#[allow(dead_code)] #[derive(Default)] struct S;"; + let defaults = Defaults::of_in(&syn::parse_file(source).unwrap(), &CfgSet::unconditional()); + + assert!(!defaults.lacks_default(&parse_quote!(S))); + } + + /// `name_of` steps through a parenthesized type to the name inside, the same way it already + /// does for any other wrapper it might be asked about. + #[test] + fn name_of_reads_through_a_parenthesized_type() { + let ty: Type = parse_quote!((Error)); + + assert_eq!(name_of(&ty), Some("Error".to_owned())); + } + + /// `name_of` also steps through the invisible grouping `macro_rules!` hygiene can introduce, + /// which never appears in ordinary source but is still one token away from a parenthesized type. + #[test] + fn name_of_reads_through_an_invisible_group() { + let ty = Type::Group(syn::TypeGroup { + group_token: syn::token::Group::default(), + elem: Box::new(parse_quote!(Error)), + }); + + assert_eq!(name_of(&ty), Some("Error".to_owned())); + } + + /// `payload_name` refuses a type it cannot key by a path, a path with no generic arguments at + /// all, and it steps over a non-type argument -- such as a lifetime -- to reach the type that + /// follows it. + #[test] + fn payload_name_handles_non_path_types_bare_paths_and_non_type_arguments() { + let tuple: Type = parse_quote!((u8, Error)); + let bare: Type = parse_quote!(Error); + let with_lifetime: Type = parse_quote!(Result<'a, Error>); + + assert_eq!(payload_name(&tuple, 0), None); + assert_eq!(payload_name(&bare, 0), None); + assert_eq!(payload_name(&with_lifetime, 0), Some("Error".to_owned())); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/definitions.rs b/crates/cargo-gamma-engine/src/ops/collect/definitions.rs new file mode 100644 index 000000000..986277b04 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/definitions.rs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Turning the candidates a file admits into mutant definitions with stable identities. + +use std::sync::Arc; + +use camino::Utf8Path; +use compact_str::CompactString; + +use super::Candidate; +use crate::HashMap; +use crate::model::{Interner, MutantDefinition, MutationSite, mutant_id_with_discriminator, normalize_site_text, site_key}; +use crate::parse::SourceFile; + +/// Turns candidates into source-level mutant definitions, assigning stable ids. +#[must_use] +pub fn into_definitions(file: &SourceFile, candidates: Vec) -> Vec { + let mut occurrences: HashMap = HashMap::default(); + let mut site_occurrences: HashMap<(u128, core::ops::Range), u32> = HashMap::default(); + + // One copy of the path for the whole file, rather than one per mutation. + let path: Arc = Arc::from(Utf8Path::new(file.path.as_str())); + + // The mutator names and item paths repeat within a file too — a few dozen distinct values + // across every mutant it produces — so they are shared as they are met. Sharing them across + // files as well is the survey's job, once it has the whole population. + let mut interner = Interner::default(); + + // Per-span site table: candidates that target the same byte range share one MutationSite, and + // its normalized text, computed once per span rather than once per candidate — every mutator + // offered at a site re-derives the same identity component from the same bytes otherwise, and a + // site with several replacements pays for the normalization as many times as it has mutants. + let mut sites: HashMap, (Arc, Arc)> = HashMap::default(); + + let mut definitions = Vec::with_capacity(candidates.len()); + + for candidate in candidates { + let (site, normalized) = sites.entry(candidate.span.clone()).or_insert_with(|| { + let original = CompactString::new(file.slice(&candidate.span)); + let (line, column) = file.location(candidate.span.start); + let end_line = file.location(candidate.span.end).0; + let normalized = normalize_site_text(&original); + ( + Arc::new(MutationSite { + span: candidate.span.clone(), + line, + end_line, + column, + original, + }), + Arc::new(normalized), + ) + }); + let site = Arc::clone(site); + let normalized = Arc::clone(normalized); + + let key = site_key(&candidate.item_path, candidate.mutator, &normalized); + let site_key = (key, candidate.span.clone()); + let index = if let Some(index) = site_occurrences.get(&site_key) { + *index + } else { + let occurrence = occurrences.entry(key).or_insert(0); + let index = *occurrence; + *occurrence += 1; + let _previous = site_occurrences.insert(site_key, index); + index + }; + + definitions.push(MutantDefinition { + id: mutant_id_with_discriminator( + &file.path, + &candidate.item_path, + candidate.mutator, + &normalized, + index, + candidate.replacement_index, + (candidate.mutator == "fn_value.err_with").then_some(candidate.replacement.as_str()), + ), + file: Arc::clone(&path), + site, + mutator: interner.text(candidate.mutator), + item_path: interner.text(&candidate.item_path), + occurrence: index, + replacement_index: candidate.replacement_index, + replacement: candidate.replacement, + shape: candidate.shape, + }); + } + + definitions +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/mod.rs b/crates/cargo-gamma-engine/src/ops/collect/mod.rs new file mode 100644 index 000000000..566011305 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/mod.rs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Walking a syntax tree and producing the mutants it admits. +//! +//! The traversal tracks the enclosing item path and how many identical sites for a mutator have +//! already been seen, giving each mutant an identity that survives reformatting and code motion. + +mod candidate; +mod collector; +mod defaults; +mod definitions; +mod shape; +mod stated; +mod traversal; + +pub use candidate::Candidate; +pub use defaults::Defaults; +pub use definitions::into_definitions; +pub use shape::Shape; +pub use stated::check as check_stated; +pub use traversal::{check_stated_and_collect_with, collect, collect_in, collect_with}; + +#[cfg(test)] +mod tests; diff --git a/crates/cargo-gamma-engine/src/ops/collect/shape.rs b/crates/cargo-gamma-engine/src/ops/collect/shape.rs new file mode 100644 index 000000000..b7bf6b002 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/shape.rs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use serde::{Deserialize, Serialize}; + +/// What kind of construct a mutation site is, which decides how a guard can be wrapped around it. +/// +/// The three cases are not stylistic. Rust will not accept the same guard text in all three +/// positions, so the shape has to travel with the mutant all the way to instrumentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Shape { + /// An expression. Guarded by a parenthesized `if`/`else` yielding one of two values. + /// + /// The parentheses matter: a bare block or `if` in condition position (`if { .. } { .. }`) + /// is rejected, and without them the guard would also rebind precedence against whatever + /// operator encloses the site. + Expr, + + /// A block that must stay a block, such as a function body. Guarded by an `if`/`else` whose + /// `else` arm is the original block, wrapped in braces so the result is still a block. + Block, + + /// A function body returning `impl Iterator`, where both arms are wrapped in a variant of + /// `gamma_rt::Either` so that they share one type. + /// + /// This is [`Shape::Block`] with one extra step, and the extra step is forced. The two arms of + /// an `if` must agree on a type, but `impl Iterator` is a single concrete type chosen by the + /// body, so a synthesized iterator and the original are never the same type. Naming a shared + /// type is not possible either: the item type may be unwritten, or itself opaque. Wrapping + /// each arm in a different variant of one two-parameter enum sidesteps both problems, because + /// the compiler infers both parameters and carries `Send`, `Sync` and `Clone` across for free. + IterBlock, + + /// A `continue` replaced with `break`. + /// + /// A generic expression guard makes the containing block appear to return `()`, even when the + /// original `continue` occupied a diverging tail position. Instrumentation keeps the original + /// `continue` as the block's tail and conditionally executes the replacement before it. + Continue, + + /// A `break` replaced with `continue`. + /// + /// Symmetric with [`Shape::Continue`]: the original `break` remains the tail expression so the + /// schema has the same type as the source. + Break, + + /// A whole statement, which the mutant deletes. Guarded by a negated `if` that runs the + /// original only when the mutant is inactive. + Stmt, + + /// A match arm's pattern, which the mutant stops from matching. + /// + /// Deleting an arm outright is not something a runtime guard can do, because which arms exist + /// is fixed when the code is compiled. Adding a guard achieves the same behaviour: an arm + /// whose guard is false does not match, and control falls through to whatever follows. This is + /// why the collector only offers the mutant when a later wildcard arm is there to catch it. + /// + /// It also costs a constant amount of text. The obvious alternative — replacing the whole + /// `match` with a copy that lacks the arm — grows with the square of the arm count, so a + /// hundred-arm dispatch would emit a hundred copies of itself and price the family out of any + /// codebase large enough to want it. + Arm, +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/stated.rs b/crates/cargo-gamma-engine/src/ops/collect/stated.rs new file mode 100644 index 000000000..8bda69a88 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/stated.rs @@ -0,0 +1,417 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The value a site states for its own return-value mutant. +//! +//! Return types are read syntactically rather than resolved, which costs twice. An alias is not +//! seen through, so the guess falls back to `Default::default()` and a build round is spent +//! discovering that the mutant does not compile. Where no guess could hold at all — a bare type +//! parameter, an associated type, a `Box`, a non-iterator `impl Trait` — no mutant is +//! offered and the site goes unmeasured by the family whose whole question is "does anything check +//! what this function returns?". +//! +//! `#[gamma::value()]` answers both. It states what to substitute, which replaces the guess +//! in the first case and creates the mutant in the second. +//! +//! It can only ever add a mutant or change what one substitutes. There is no spelling of it that +//! removes a site or reaches another family, which is what keeps it from becoming a suppression +//! channel that evades review. Nothing about it is taken on trust either: the stated expression +//! becomes an ordinary mutant, and one that does not type-check is withdrawn by the same rollback +//! that withdraws a bad guess. + +use core::ops::Range; + +use proc_macro2::TokenStream; +use syn::spanned::Spanned as _; +use syn::visit::{self, Visit}; +use syn::{Attribute, Expr, ImplItemFn, ItemFn, Meta, TraitItemFn}; + +use crate::error::Error; +use crate::parse::SourceFile; +use crate::{HashSet, Result}; + +/// The two path segments a stated value is written under. +const PATH: [&str; 2] = ["gamma", "value"]; + +/// What to say about an argument list that is not one expression. +const MALFORMED: &str = "a site states one value, so `#[gamma::value(...)]` takes one Rust expression, as in `#[gamma::value(0)]`"; + +/// What to say about a stated value written anywhere but on a function. +const MISPLACED: &str = "`#[gamma::value(...)]` states the value one function returns, so it goes on a function or a method; \ + one expression cannot be the body of every function beneath an `impl` block or a module"; + +/// What to say about two stated values on one item. +const DUPLICATED: &str = "an item may state one value; two would leave which of them applies to the order they were written in"; + +/// What to say about a stated value on a function that has no body. +const BODILESS: &str = "`#[gamma::value(...)]` replaces the body of the function it is written on, and a declaration has none; \ + a trait method's implementations do not inherit it"; + +/// Returns the byte range of the expression an item's attributes state, if they state one. +/// +/// A range rather than the tokens, because what the mutant substitutes is the text the user wrote: +/// re-rendering a token stream spaces it as `Box :: new (File)`, which compiles but reads as +/// something nobody typed, and every report, diff and patch would carry that spelling instead of +/// theirs. +/// +/// A malformed argument yields `None`. It is diagnosed once, by [`check`], and that diagnostic +/// stops the run — so nothing downstream has to decide what half an attribute means. +pub(super) fn stated_range(attrs: &[Attribute]) -> Option> { + let arguments = attrs.iter().find_map(arguments)?; + + if syn::parse2::(arguments.clone()).is_err() { + return None; + } + + let mut tokens = arguments.into_iter(); + let first = tokens.next()?.span().byte_range(); + let last = tokens.last().map_or_else(|| first.clone(), |tree| tree.span().byte_range()); + + (first.start < last.end).then_some(first.start..last.end) +} + +/// Checks every stated value a file carries. +/// +/// # Errors +/// +/// Returns an error if a stated value is malformed, duplicated, written on something that is not a +/// function, or written on a function with no body to replace. The proc macro rejects all four at +/// compile time, so a crate that builds cannot reach them — but this tool reads source rather than +/// build output, and `gamma list mutants` runs against trees that have never been compiled. +/// Ignoring one there would leave a hint that reads as if it works and does nothing, which is the +/// failure mode the whole channel exists to avoid. +/// +/// Fatal rather than a warning, for the same reason a suppression naming no mutator is: the run +/// that swallows it reports a score computed from a population the author did not ask for. +pub fn check(file: &SourceFile) -> Result<()> { + let mut audit = Audit::default(); + + audit.visit_file(&file.ast); + + fault(file, audit) +} + +/// Turns a completed audit into the same result [`check`] has always returned. +/// +/// Split out so the fused phase-one pass (see `collector::phase_one`) can run this exact check +/// against an `Audit` it filled during its own single walk, rather than paying for a second, +/// standalone one just to reach this diagnostic. +pub(super) fn fault(file: &SourceFile, mut audit: Audit) -> Result<()> { + for span in audit.stated { + if !audit.on_functions.contains(&span.start) { + audit.faults.push((span, MISPLACED.to_owned())); + } + } + + // By position rather than by the order the walk happened to reach them, so a file with two + // mistakes always reports the same one first. + audit.faults.sort_by_key(|(at, _message)| at.start); + + let Some((at, message)) = audit.faults.first() else { + return Ok(()); + }; + + Err(Error::new(format!("{}:{}: {message}", file.path, file.line_of(at.start))).usage()) +} + +/// Every stated value in a file, and everything wrong with the ones that are wrong. +/// +/// The two are collected in one walk because the misplacement rule is the difference between them: +/// an attribute is misplaced exactly when it was seen and no function claimed it. Deciding that by +/// enumerating the item kinds it must *not* appear on would have to be revised every time the +/// language grows another one. +/// +/// `pub(super)`, and each visited node kind has a matching `on_*` method with no recursive +/// continuation of its own, so the fused phase-one pass can drive this exact per-node logic from +/// its own single traversal instead of running this visitor's `visit_file` a second time. +#[derive(Debug, Default)] +pub(super) struct Audit { + /// Where every `#[gamma::value(...)]` in the file sits. + stated: Vec>, + + /// Where the ones written on a function or a method sit, keyed by start offset. + on_functions: HashSet, + + /// What is wrong, and where. + faults: Vec<(Range, String)>, +} + +impl Audit { + /// Records what an item's own attributes get wrong, and which of them a function claimed. + fn item(&mut self, attrs: &[Attribute]) { + let stated: Vec<&Attribute> = attrs.iter().filter(|attribute| is_stated_value(attribute)).collect(); + + if let Some(second) = stated.get(1) { + self.faults.push((second.span().byte_range(), DUPLICATED.to_owned())); + } + + for attribute in stated { + let _claimed = self.on_functions.insert(attribute.span().byte_range().start); + + let malformed = arguments(attribute).is_none_or(|tokens| syn::parse2::(tokens).is_err()); + + if malformed { + self.faults.push((attribute.span().byte_range(), MALFORMED.to_owned())); + } + } + } + + /// The local update `visit_attribute` makes, without its recursive continuation. + pub(super) fn on_attribute(&mut self, node: &Attribute) { + if is_stated_value(node) { + self.stated.push(node.span().byte_range()); + } + } + + /// The local update `visit_item_fn` makes, without its recursive continuation. + pub(super) fn on_item_fn(&mut self, node: &ItemFn) { + self.item(&node.attrs); + } + + /// The local update `visit_impl_item_fn` makes, without its recursive continuation. + pub(super) fn on_impl_item_fn(&mut self, node: &ImplItemFn) { + self.item(&node.attrs); + } + + /// The local update `visit_trait_item_fn` makes, without its recursive continuation. + pub(super) fn on_trait_item_fn(&mut self, node: &TraitItemFn) { + // A declaration has no body to replace, and a stated value is not inherited by the + // implementations any more than it is inherited from an `impl` block. Left unreported, it + // would read as a hint that works and generate nothing anywhere. + if node.default.is_none() { + for attribute in node.attrs.iter().filter(|attribute| is_stated_value(attribute)) { + self.faults.push((attribute.span().byte_range(), BODILESS.to_owned())); + let _claimed = self.on_functions.insert(attribute.span().byte_range().start); + } + } else { + self.item(&node.attrs); + } + } +} + +#[expect( + clippy::renamed_function_params, + reason = "syn names every visitor parameter `i`, which says nothing about what it is" +)] +impl<'ast> Visit<'ast> for Audit { + fn visit_attribute(&mut self, node: &'ast Attribute) { + self.on_attribute(node); + visit::visit_attribute(self, node); + } + + fn visit_item_fn(&mut self, node: &'ast ItemFn) { + self.on_item_fn(node); + visit::visit_item_fn(self, node); + } + + fn visit_impl_item_fn(&mut self, node: &'ast ImplItemFn) { + self.on_impl_item_fn(node); + visit::visit_impl_item_fn(self, node); + } + + fn visit_trait_item_fn(&mut self, node: &'ast TraitItemFn) { + self.on_trait_item_fn(node); + + visit::visit_trait_item_fn(self, node); + } +} + +/// Returns whether an attribute is a stated value, whatever it happens to state. +/// +/// Asked where the answer decides a diagnostic as well as where it decides a mutant, so an +/// attribute nobody can read still counts: that is exactly the one worth reporting. +fn is_stated_value(attribute: &Attribute) -> bool { + let mut segments = attribute.path().segments.iter(); + + PATH.iter() + .all(|expected| segments.next().is_some_and(|segment| segment.ident == expected)) + && segments.next().is_none() +} + +/// The argument list of an attribute that states a value. +/// +/// A bare `#[gamma::value]` has none, and neither does `#[gamma::value = 1]`. Both state nothing an +/// expression could be read out of, so both are reported rather than guessed at. +fn arguments(attribute: &Attribute) -> Option { + if !is_stated_value(attribute) { + return None; + } + + match &attribute.meta { + Meta::List(list) if !list.tokens.is_empty() => Some(list.tokens.clone()), + _other => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(source: &str) -> SourceFile { + SourceFile::parse("test.rs", source.to_owned()).expect("the fixture must parse") + } + + fn function_item(parsed: &SourceFile) -> &ItemFn { + parsed + .ast + .items + .iter() + .find_map(|item| if let syn::Item::Fn(item) = item { Some(item) } else { None }) + .expect("the fixture contains a function") + } + + fn rejection(source: &str) -> String { + check(&file(source)) + .expect_err("the fixture states a value that cannot be honoured") + .to_string() + } + + /// The expression is taken from the source verbatim, because that spelling is what every + /// report, diff and applied patch shows the user afterwards. Re-rendering the tokens instead + /// would hand them back `Some (Config { size : 8 })`, which compiles and which nobody wrote. + #[test] + fn the_stated_expression_is_the_text_the_user_wrote() { + let parsed = file("#[gamma::value(Some(Config { size: 8 }))]\nfn f() -> Option { g() }"); + let item = function_item(&parsed); + + let range = stated_range(&item.attrs).expect("the fixture states a value"); + + assert_eq!(parsed.slice(&range), "Some(Config { size: 8 })"); + } + + /// A single-token expression has one span rather than a first and a last, and is the shortest + /// thing anybody states. + #[test] + fn a_one_token_expression_is_read_whole() { + let parsed = file("#[gamma::value(7)]\nfn f() -> u32 { 1 }"); + let item = function_item(&parsed); + + let range = stated_range(&item.attrs).expect("the fixture states a value"); + + assert_eq!(parsed.slice(&range), "7"); + } + + /// An item with no stated value has none, which is what leaves the guessing rules in charge of + /// every site that says nothing. + #[test] + fn an_item_that_states_nothing_has_no_range() { + let parsed = file("#[inline]\n#[gamma::skip(arith)]\nfn f() -> u32 { 1 }"); + let item = function_item(&parsed); + + assert_eq!(stated_range(&item.attrs), None); + } + + #[test] + fn a_malformed_stated_value_has_no_range() { + let parsed = file("#[gamma::value(0, 1)]\nfn f() -> u32 { 1 }"); + let item = function_item(&parsed); + + assert_eq!(stated_range(&item.attrs), None); + } + + /// A file that states nothing is not a file with a problem, however many other attributes it + /// carries. + #[test] + fn a_file_with_no_stated_values_is_accepted() { + check(&file("#[gamma::skip(arith)]\nfn f() -> u32 { 1 }\n#[test]\nfn t() {}")) + .expect("a file with no stated values has nothing to report"); + } + + /// The three positions a function is written in are all functions, and a stated value is at + /// home in each. + #[test] + fn a_value_stated_on_a_function_a_method_or_a_trait_method_is_accepted() { + let sources = [ + "#[gamma::value(0)]\nfn f() -> u32 { 1 }", + "struct S;\nimpl S {\n#[gamma::value(0)]\nfn f(&self) -> u32 { 1 }\n}", + "trait T {\n#[gamma::value(0)]\nfn f(&self) -> u32 { 1 }\n}", + ]; + + for source in sources { + check(&file(source)).unwrap_or_else(|error| panic!("`{source}` states a value on a function: {error}")); + } + } + + /// Two values on one item would be settled by which was written first, and a rule that subtle + /// is one nobody reads. The line reported is the second one, which is the one to delete. + #[test] + fn two_stated_values_on_one_item_are_reported() { + let rejected = rejection("#[gamma::value(0)]\n#[gamma::value(1)]\nfn f() -> u32 { 2 }"); + + assert!(rejected.contains("test.rs:2: "), "{rejected}"); + assert!(rejected.contains("an item may state one value"), "{rejected}"); + } + + /// Everything that is not one expression is reported the same way, because the fix is the same + /// in every case: write one. + #[test] + fn an_argument_that_is_not_one_expression_is_reported() { + for arguments in ["", "0, 1", "1 +", "let x = 1;"] { + let rejected = rejection(&format!("#[gamma::value({arguments})]\nfn f() -> u32 {{ 2 }}")); + + assert!(rejected.contains("test.rs:1: "), "`{arguments}`: {rejected}"); + assert!( + rejected.contains("states one value") || rejected.contains("is not a single Rust expression"), + "`{arguments}`: {rejected}" + ); + } + } + + /// A bare path and a name-value pair carry no argument list at all, and neither states a value. + #[test] + fn a_stated_value_with_no_argument_list_is_reported() { + for attribute in ["#[gamma::value]", "#[gamma::value = 1]"] { + let rejected = rejection(&format!("{attribute}\nfn f() -> u32 {{ 2 }}")); + + assert!(rejected.contains("states one value"), "`{attribute}`: {rejected}"); + } + } + + /// Inheritance is not invented: an `impl` block or a module would have to state one expression + /// that type-checks as the body of every function beneath it, which essentially never holds. + #[test] + fn a_value_stated_on_anything_but_a_function_is_reported() { + let sources = [ + "struct S;\n#[gamma::value(0)]\nimpl S { fn f(&self) -> u32 { 1 } }", + "#[gamma::value(0)]\nmod m { pub fn f() -> u32 { 1 } }", + "#[gamma::value(0)]\nstruct S { n: u8 }", + "#[gamma::value(0)]\nconst N: u8 = 1;", + ]; + + for source in sources { + let rejected = check(&file(source)).expect_err("the fixture misplaces a stated value").to_string(); + + assert!(rejected.contains("goes on a function or a method"), "`{source}`: {rejected}"); + } + } + + /// A trait method that is only declared has no body, so there is nothing for a stated value to + /// replace — and the implementations do not inherit it, for the same reason an `impl` block + /// does not hand one down. Silence here would be a hint that reads as working and does nothing + /// anywhere in the tree. + #[test] + fn a_value_stated_on_a_declaration_is_reported() { + let rejected = rejection("trait T {\n#[gamma::value(0)]\nfn f(&self) -> u32;\n}"); + + assert!(rejected.contains("test.rs:2: "), "{rejected}"); + assert!(rejected.contains("a declaration has none"), "{rejected}"); + } + + /// A nested function states its own value, at its own site, and is not the enclosing function + /// stating a second one. + #[test] + fn a_function_inside_a_function_may_state_its_own_value() { + let source = "#[gamma::value(0)]\nfn f() -> u32 { #[gamma::value(1)] fn g() -> u32 { 2 } g() }"; + + check(&file(source)).expect("both values are stated on a function of their own"); + } + + /// A file with two mistakes reports the earlier one, so the message does not depend on the + /// order the walk reached them in. + #[test] + fn the_first_fault_in_the_file_is_the_one_reported() { + let rejected = rejection("#[gamma::value(0, 1)]\nfn f() -> u32 { 2 }\n#[gamma::value()]\nfn g() -> u32 { 3 }"); + + assert!(rejected.contains("test.rs:1: "), "{rejected}"); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/tests.rs b/crates/cargo-gamma-engine/src/ops/collect/tests.rs new file mode 100644 index 000000000..50b642317 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/tests.rs @@ -0,0 +1,3300 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::ops::Range; + +use compact_str::CompactString; + +use super::*; +use crate::cfg::CfgSet; +use crate::ops::registry::{REGISTRY, Selection}; +use crate::parse::SourceFile; + +fn candidates(source: &str, ops: &str) -> Vec { + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse(ops).unwrap(); + + collect(&file, &selection) +} + +fn mutators(source: &str, ops: &str) -> Vec<&'static str> { + candidates(source, ops).into_iter().map(|c| c.mutator).collect() +} + +fn with_errors(source: &str, errors: &[&str]) -> Vec { + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let mut selection = Selection::empty(); + + selection.set_errors(errors.iter().map(|e| (*e).to_owned()).collect()); + collect(&file, &selection) +} + +/// Two replacements offered at one site must be told apart by their identifiers. +/// +/// `replacement_index` is the only thing distinguishing them: same file, same item, same +/// mutator, same site text, same occurrence. It reaches `mutant_id`, and an id is not a +/// cosmetic label — suppressions name mutants by id, and incremental execution decides what it has +/// already resolved by id. Collapsing the index makes two genuinely different mutants share +/// one identity, so a suppression aimed at either silences both and an incremental run credits a +/// verdict to a mutant that never earned it. +#[test] +fn two_replacements_at_one_site_get_distinct_identifiers() { + let found = with_errors("fn f() -> Result { Ok(1) }", &["MyError::Io", "MyError::Eof"]); + + assert_eq!(found.len(), 2, "{found:?}"); + assert_eq!(found[0].span, found[1].span, "the premise is that only the index differs"); + assert_eq!(found[0].mutator, found[1].mutator); + assert_ne!(found[0].replacement_index, found[1].replacement_index); + + let file = SourceFile::parse("test.rs", "fn f() -> Result { Ok(1) }".to_owned()).unwrap(); + let mutants = into_definitions(&file, found); + + assert_ne!(mutants[0].id, mutants[1].id, "two mutants sharing an id cannot be suppressed apart"); +} + +/// Multiple replacements targeting the same source span share one `MutationSite` allocation. +#[test] +fn replacements_at_same_span_share_a_site_allocation() { + let found = with_errors("fn f() -> Result { Ok(1) }", &["MyError::Io", "MyError::Eof"]); + + assert!(found.len() >= 2); + assert_eq!(found[0].span, found[1].span, "premise: same span"); + + let file = SourceFile::parse("test.rs", "fn f() -> Result { Ok(1) }".to_owned()).unwrap(); + let mutants = into_definitions(&file, found); + + assert!( + std::sync::Arc::ptr_eq(&mutants[0].site, &mutants[1].site), + "two definitions at the same span must share one MutationSite" + ); + assert_eq!(mutants[0].site.original, mutants[1].site.original); + assert_eq!(mutants[0].site.line, mutants[1].site.line); +} + +/// The number of replacements at one site does not consume occurrence numbers belonging to later +/// identical sites. +#[test] +fn occurrence_counts_sites_instead_of_replacements() { + let item = "fn f() -> Result { Ok(1) }"; + let source = format!("{item}\n{item}"); + let file = SourceFile::parse("test.rs", source).unwrap(); + let first = with_errors(item, &["MyError::Io", "MyError::Eof"]); + let mut second = first.clone(); + + for candidate in &mut second { + candidate.span = candidate.span.start + item.len() + 1..candidate.span.end + item.len() + 1; + } + + let mutants = into_definitions(&file, first.into_iter().chain(second).collect()); + let occurrences: Vec = mutants.iter().map(|mutant| mutant.occurrence).collect(); + + assert_eq!(occurrences, vec![0, 0, 1, 1]); +} + +/// The cached per-span normalized text must produce exactly the identity a fresh, uncached +/// normalization of the same site's bytes would — for every mutant the site offers, not just the +/// first one to reach the cache. +#[test] +fn cached_normalization_matches_a_direct_recomputation_for_every_replacement_at_a_site() { + let source = "fn f() -> Result { Ok(1) }"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let found = with_errors(source, &["MyError::Io", "MyError::Eof", "MyError::Closed"]); + + // Independently recomputed, from the same source bytes, with no cache in the loop. + let expected: Vec = found + .iter() + .map(|candidate| { + let text = file.slice(&candidate.span); + let normalized = crate::model::normalize_site_text(text); + + crate::model::mutant_id_with_discriminator( + &file.path, + &candidate.item_path, + candidate.mutator, + &normalized, + 0, + candidate.replacement_index, + (candidate.mutator == "fn_value.err_with").then_some(candidate.replacement.as_str()), + ) + }) + .collect(); + + let actual: Vec = into_definitions(&file, found).into_iter().map(|mutant| mutant.id).collect(); + + assert_eq!(actual, expected); +} + +#[test] +fn named_error_identity_changes_with_text_and_order() { + let source = "fn f() -> Result { Ok(1) }"; + let identify = |errors: &[&str]| { + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + + into_definitions(&file, with_errors(source, errors)) + .into_iter() + .map(|mutant| (mutant.replacement.to_string(), mutant.id)) + .collect::>() + }; + let original = identify(&["MyError::Io", "MyError::Eof"]); + let changed = identify(&["MyError::Io", "MyError::Closed"]); + let reordered = identify(&["MyError::Eof", "MyError::Io"]); + + assert_ne!(original["Err(MyError::Eof)"], changed["Err(MyError::Closed)"]); + assert_ne!(original["Err(MyError::Io)"], reordered["Err(MyError::Io)"]); + assert_ne!(original["Err(MyError::Eof)"], reordered["Err(MyError::Eof)"]); +} + +#[test] +fn each_named_error_value_becomes_its_own_mutant() { + let found = with_errors("fn f() -> Result { Ok(1) }", &["MyError::Io", "MyError::Eof"]); + + let replacements: Vec<&str> = found.iter().map(|c| c.replacement.as_str()).collect(); + + assert_eq!(replacements, vec!["Err(MyError::Io)", "Err(MyError::Eof)"]); + assert!(found.iter().all(|c| c.mutator == "fn_value.err_with")); +} + +#[test] +fn named_error_values_only_reach_functions_returning_result() { + let found = with_errors("fn f() -> i32 { 1 }", &["MyError::Io"]); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn naming_no_error_values_produces_no_error_mutants() { + let found = with_errors("fn f() -> Result { Ok(1) }", &[]); + + assert!(found.is_empty(), "{found:?}"); +} + +/// One operator's source, the selection that reaches it, and the exact replacements it must emit. +type OperatorCase = (&'static str, &'static str, &'static [(&'static str, &'static str)]); + +/// Every binary and compound-assignment operator, paired with the exact `(mutator, replacement)` +/// set it must produce. +/// +/// The replacement is the text of the *whole* expression, which is why the operands appear: a +/// mutant is a `(span, text)` pair over the entire construct, not a patched operator token. +const OPERATOR_ORACLE: &[OperatorCase] = &[ + ( + "fn f(a: i32, b: i32) -> bool { a < b }", + "relational", + &[("relational.lt_to_le", "(a) <= (b)"), ("relational.lt_to_gt", "(a) > (b)")], + ), + ( + "fn f(a: i32, b: i32) -> bool { a <= b }", + "relational", + &[("relational.le_to_lt", "(a) < (b)"), ("relational.le_to_ge", "(a) >= (b)")], + ), + ( + "fn f(a: i32, b: i32) -> bool { a > b }", + "relational", + &[("relational.gt_to_ge", "(a) >= (b)"), ("relational.gt_to_lt", "(a) < (b)")], + ), + ( + "fn f(a: i32, b: i32) -> bool { a >= b }", + "relational", + &[("relational.ge_to_gt", "(a) > (b)"), ("relational.ge_to_le", "(a) <= (b)")], + ), + ( + "fn f(a: i32, b: i32) -> bool { a == b }", + "relational", + &[("relational.eq_to_ne", "(a) != (b)")], + ), + ( + "fn f(a: i32, b: i32) -> bool { a != b }", + "relational", + &[("relational.ne_to_eq", "(a) == (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a + b }", + "arith", + &[("arith.add_to_sub", "(a) - (b)"), ("arith.add_to_mul", "(a) * (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a - b }", + "arith", + &[("arith.sub_to_add", "(a) + (b)"), ("arith.sub_to_div", "(a) / (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a * b }", + "arith", + &[("arith.mul_to_div", "(a) / (b)"), ("arith.mul_to_add", "(a) + (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a / b }", + "arith", + &[("arith.div_to_mul", "(a) * (b)"), ("arith.div_to_rem", "(a) % (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a % b }", + "arith", + &[("arith.rem_to_div", "(a) / (b)"), ("arith.rem_to_mul", "(a) * (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a & b }", + "bitwise", + &[("bitwise.and_to_or", "(a) | (b)"), ("bitwise.and_to_xor", "(a) ^ (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a | b }", + "bitwise", + &[("bitwise.or_to_and", "(a) & (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a ^ b }", + "bitwise", + &[("bitwise.xor_to_and", "(a) & (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a << b }", + "shift", + &[("shift.shl_to_shr", "(a) >> (b)")], + ), + ( + "fn f(a: i32, b: i32) -> i32 { a >> b }", + "shift", + &[("shift.shr_to_shl", "(a) << (b)")], + ), + ( + "fn f(x: bool, y: bool) -> bool { x && y }", + "logical", + &[("logical.and_to_or", "(x) || (y)")], + ), + ( + "fn f(x: bool, y: bool) -> bool { x || y }", + "logical", + &[("logical.or_to_and", "(x) && (y)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a += b; }", + "assign", + &[("assign.add_to_sub", "*a -= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a -= b; }", + "assign", + &[("assign.sub_to_add", "*a += (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a *= b; }", + "assign", + &[("assign.mul_to_div", "*a /= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a /= b; }", + "assign", + &[("assign.div_to_mul", "*a *= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a %= b; }", + "assign", + &[("assign.rem_to_div", "*a /= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a &= b; }", + "assign", + &[("assign.and_to_or", "*a |= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a |= b; }", + "assign", + &[("assign.or_to_and", "*a &= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a ^= b; }", + "assign", + &[("assign.xor_to_and", "*a &= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a <<= b; }", + "assign", + &[("assign.shl_to_shr", "*a >>= (b)")], + ), + ( + "fn f(a: &mut i32, b: i32) { *a >>= b; }", + "assign", + &[("assign.shr_to_shl", "*a <<= (b)")], + ), +]; + +/// The mutation operators are the product, so the token each one emits is part of its contract and +/// not an implementation detail. +/// +/// Checking these families by mutator *name* alone — which is all that happened before — lets a +/// wrong or swapped replacement token through untouched: the tool would emit a `<=` mutant, label +/// it `lt_to_gt`, and report a survivor against an edit it never made. Worse, a swap can collapse +/// two entries onto the same token, silently turning two mutants into one duplicate. The oracle is +/// therefore the exact `(mutator, replacement)` set, asserted for every entry in the table. +#[test] +fn every_binary_and_compound_assignment_operator_emits_the_replacement_its_name_promises() { + for (source, ops, expected) in OPERATOR_ORACLE { + let mut found: Vec<(&str, String)> = candidates(source, ops) + .into_iter() + .map(|candidate| (candidate.mutator, candidate.replacement.to_string())) + .collect(); + + let mut expected: Vec<(&str, String)> = expected + .iter() + .map(|(mutator, replacement)| (*mutator, (*replacement).to_owned())) + .collect(); + + found.sort_unstable(); + expected.sort_unstable(); + + assert_eq!(found, expected, "{source}"); + } +} + +/// The table is only worth pinning if it is the whole table: an entry deleted from +/// `binary_replacements` would otherwise vanish from both the code and its oracle at once. +#[test] +fn the_operator_oracle_covers_every_replacement_the_tables_offer() { + let pairs: usize = OPERATOR_ORACLE.iter().map(|(_, _, expected)| expected.len()).sum(); + + assert_eq!(OPERATOR_ORACLE.len(), 28, "one row per binary and compound-assignment operator"); + assert_eq!(pairs, 38, "one assertion per `binary_replacements` entry"); +} + +#[test] +fn unselected_mutators_produce_nothing() { + assert!(mutators("fn f(a: i32, b: i32) -> bool { a < b }", "arith").is_empty()); +} + +#[test] +fn spans_cover_the_whole_binary_expression() { + let source = "fn f(a: i32, b: i32) -> bool { a < b }"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let found = collect(&file, &Selection::parse("relational").unwrap()); + + assert_eq!(file.slice(&found[0].span), "a < b"); +} + +#[test] +fn nested_binary_expressions_are_all_found() { + let found = mutators("fn f(a: i32, b: i32, c: i32) -> i32 { a + b * c }", "arith"); + + assert!(found.contains(&"arith.add_to_sub")); + assert!(found.contains(&"arith.mul_to_div")); +} + +#[test] +fn candidates_come_back_in_source_order() { + let source = "fn f(a: i32, b: i32) -> i32 { let x = a - b; x * a }"; + let found = candidates(source, "arith"); + + for pair in found.windows(2) { + assert!(pair.len() > 1); + assert!(pair[0].span.start <= pair[1].span.start); + } +} + +#[test] +fn the_largest_integer_literal_does_not_overflow() { + // `i64::MAX` has no increment. Computing one unchecked panics in a debug build and wraps to + // `i64::MIN` in a release build, which would offer a "+1" mutant that is smaller than the + // literal it replaces. + let found = mutators("fn f() -> i64 { 9223372036854775807 }", "literal"); + + assert!(found.contains(&"literal.int_decrement"), "{found:?}"); + assert!(!found.contains(&"literal.int_increment"), "{found:?}"); +} + +#[test] +fn item_paths_include_the_enclosing_function() { + let found = candidates("fn outer(a: i32) -> i32 { a + 1 }", "arith.add_to_sub"); + + assert_eq!(&*found[0].item_path, "outer"); +} + +#[test] +fn item_paths_include_module_impl_and_method() { + let source = "mod m { struct S; impl S { fn go(&self, a: i32) -> i32 { a + 1 } } }"; + let found = candidates(source, "arith.add_to_sub"); + + assert_eq!(&*found[0].item_path, "m::S::go"); +} + +#[test] +fn item_paths_distinguish_trait_defaults_from_each_other_and_free_functions() { + let source = "fn f() -> i32 { 1 + 2 } + trait First { fn f() -> i32 { 1 + 2 } } + trait Second { fn f() -> i32 { 1 + 2 } }"; + let found = candidates(source, "arith.add_to_sub"); + let mut paths: Vec<&str> = found.iter().map(|candidate| &*candidate.item_path).collect(); + + paths.sort_unstable(); + + assert_eq!(paths, vec!["First::f", "Second::f", "f"]); +} + +#[test] +fn impl_paths_strip_references_and_generics() { + let source = "struct S(T); impl S { fn go(&self, a: i32) -> i32 { a + 1 } }"; + let found = candidates(source, "arith.add_to_sub"); + + assert_eq!(&*found[0].item_path, "S::go"); +} + +#[test] +fn trait_implementation_paths_keep_identical_methods_stable_when_reordered() { + let first = "struct S; + trait First { fn f(&self) -> i32; } + trait Second { fn f(&self) -> i32; } + impl S { fn f(&self) -> i32 { 1 + 2 } } + impl First for S { fn f(&self) -> i32 { 1 + 2 } } + impl Second for S { fn f(&self) -> i32 { 1 + 2 } }"; + let second = "struct S; + trait First { fn f(&self) -> i32; } + trait Second { fn f(&self) -> i32; } + impl S { fn f(&self) -> i32 { 1 + 2 } } + impl Second for S { fn f(&self) -> i32 { 1 + 2 } } + impl First for S { fn f(&self) -> i32 { 1 + 2 } }"; + + let identities = |source: &str| { + let file = SourceFile::parse("test.rs", source.to_owned()).expect("the fixture parses"); + let selection = Selection::parse("arith.add_to_sub").expect("the mutator exists"); + let mut ids: Vec<(String, String)> = into_definitions(&file, collect(&file, &selection)) + .into_iter() + .map(|mutant| (mutant.item_path.to_string(), mutant.id.to_string())) + .collect(); + + ids.sort_unstable(); + ids + }; + + let before = identities(first); + let after = identities(second); + let paths: Vec<&str> = before.iter().map(|(path, _id)| path.as_str()).collect(); + + assert_eq!(before, after, "each trait method keeps its id when implementation order changes"); + assert!(paths.contains(&"S::f"), "{paths:?}"); + assert!(paths.contains(&"::f"), "{paths:?}"); + assert!(paths.contains(&"::f"), "{paths:?}"); +} + +#[test] +fn test_functions_are_not_mutated() { + let source = "#[test] fn t() { assert_eq!(1 + 1, 2); }"; + + assert!(candidates(source, "arith").is_empty()); +} + +#[test] +fn test_impl_methods_are_not_mutated() { + let source = "struct S; impl S { #[test] fn t(&self) { let _ = 1 + 1; } }"; + + assert!(candidates(source, "arith").is_empty()); +} + +#[test] +fn cfg_test_modules_are_not_mutated() { + let source = "#[cfg(test)] mod tests { fn helper(a: i32) -> i32 { a + 1 } }"; + + assert!(candidates(source, "arith").is_empty()); +} + +#[test] +fn non_test_cfg_modules_are_mutated() { + let source = "#[cfg(unix)] mod platform { fn helper(a: i32) -> i32 { a + 1 } }"; + + assert_eq!(candidates(source, "arith.add_to_sub").len(), 1); +} + +#[test] +fn a_compound_gate_that_only_a_test_build_satisfies_is_not_mutated() { + // The parser this replaced looked only at the top level of the predicate, so `all(test, unix)` + // read as an ordinary platform gate and the test helper below it entered the population. + for source in [ + "#[cfg(all(test, unix))] mod tests { fn helper(a: i32) -> i32 { a + 1 } }", + "#[cfg(all(unix, all(test, feature = \"x\")))] mod tests { fn helper(a: i32) -> i32 { a + 1 } }", + "#[cfg(not(feature = \"x\"))] #[cfg(test)] mod tests { fn helper(a: i32) -> i32 { a + 1 } }", + ] { + assert!(candidates(source, "arith").is_empty(), "{source}"); + } +} + +#[test] +fn a_gate_a_production_build_can_also_satisfy_is_still_mutated() { + // `any(test, feature = "…")` holds whenever the feature does, so this code is compiled into the + // library the run measures. Reading the bare `test` as decisive would drop every mutant in it. + let source = "#[cfg(any(test, feature = \"runtime\"))] mod support { fn helper(a: i32) -> i32 { a + 1 } }"; + + assert_eq!(candidates(source, "arith.add_to_sub").len(), 1); +} + +#[test] +fn tokio_test_functions_are_not_mutated() { + let source = "#[tokio::test] async fn t() { let _ = 1 + 1; }"; + + assert!(candidates(source, "arith").is_empty()); +} + +#[test] +fn const_initializers_are_not_mutated() { + // The encoding wraps expressions in an `if` over a function call, which const contexts + // reject, so generating these would produce guaranteed compile failures. + assert!(candidates("const N: i32 = 1 + 2;", "arith").is_empty()); + assert!(candidates("static N: i32 = 1 + 2;", "arith").is_empty()); +} + +#[test] +fn const_fn_bodies_are_not_mutated() { + // Every expression inside a `const fn` is in a const context, not just the body value, so + // the whole subtree has to stay inert. Mutating one of these compiles nowhere. + assert!(candidates("const fn f(a: i32, b: i32) -> i32 { a + b }", "arith").is_empty()); + assert!(candidates("const fn f(a: usize, b: &[u8]) -> bool { a < b.len() }", "relational").is_empty()); + + // A non-const function in the same file must still be mutated. + let source = "const fn f(a: i32) -> i32 { a + 1 }\nfn g(a: i32) -> i32 { a + 2 }"; + + assert!(!candidates(source, "arith").is_empty()); +} + +#[test] +fn array_lengths_in_types_are_not_mutated() { + // `[u8; 200]` in a type is a const context, and the guard is a function call. This is not + // the same position as the length in the *value* `[0u8; 32]`, which was already inert, and + // the difference cost a real crate a build that could not compile and could not be blamed + // on any one mutant. + assert!(candidates("struct Pairs([u8; 200]);", "literal").is_empty()); + assert!(candidates("struct Pairs([u8; 100 * 2]);", "arith").is_empty()); + assert!(candidates("fn f() -> [u8; 4] { todo!() }", "literal").is_empty()); + assert!(candidates("fn f(a: [u8; 4]) -> usize { a.len() }", "literal").is_empty()); + + // The element of an array *value* is an ordinary expression and must stay mutable; it is + // only the length beside it that cannot hold a guard. + assert!(!candidates("fn f() -> [u8; 4] { [7; 4] }", "literal").is_empty()); + assert!(candidates("type Row = [u8; 16];", "literal").is_empty()); +} + +#[test] +fn const_generic_arguments_are_not_mutated() { + // Same reason as an array length: the argument is a const expression, and it can sit + // arbitrarily deep inside a type. + assert!(candidates("struct Grid(Matrix<3>);", "literal").is_empty()); + assert!(candidates("fn f() -> Wrapper> { todo!() }", "literal").is_empty()); +} + +#[test] +fn a_value_beside_an_inert_type_is_still_mutated() { + // Making types inert must not swallow the function they belong to; the array length is a + // const context but the body around it is ordinary code. + let source = "fn f(a: [u8; 4], b: i32) -> i32 { b + 1 }"; + + assert!(!candidates(source, "arith").is_empty()); +} + +#[test] +fn macro_interiors_are_not_mutated() { + let source = "fn f() { println!(\"{}\", 1 + 2); }"; + + assert!(candidates(source, "arith").is_empty()); +} + +#[test] +fn if_conditions_can_be_negated() { + let source = "fn f(a: bool) -> i32 { if a { 1 } else { 2 } }"; + let found = mutators(source, "cond.negate"); + + assert_eq!(found, vec!["cond.negate"]); +} + +#[test] +fn a_negated_condition_is_parenthesized() { + // `!` binds tighter than any binary operator, so `!a == b` is a different expression that + // usually does not even type-check. + let source = "fn f(a: i32, b: i32) -> i32 { if a == b { 1 } else { 2 } }"; + let found = candidates(source, "cond.negate"); + + assert_eq!(found[0].replacement, "!(a == b)"); +} + +#[test] +fn removing_a_unary_operator_leaves_the_operand() { + let found = candidates("fn f(a: i32) -> i32 { -a }", "unary.remove_neg"); + + assert_eq!(found[0].replacement, "a"); +} + +#[test] +fn if_let_conditions_are_left_alone() { + let source = "fn f(a: Option) -> i32 { if let Some(x) = a { x } else { 2 } }"; + + assert!(candidates(source, "cond").is_empty()); +} + +#[test] +fn while_conditions_can_be_negated() { + let source = "fn f(mut a: i32) { while a > 0 { a -= 1; } }"; + let found = mutators(source, "cond.negate"); + + assert_eq!(found, vec!["cond.negate"]); +} + +#[test] +fn integer_literals_yield_boundary_replacements() { + let found = mutators("fn f() -> i32 { 5 }", "literal"); + + assert!(found.contains(&"literal.int_to_zero")); + assert!(found.contains(&"literal.int_to_one")); + assert!(found.contains(&"literal.int_increment")); + assert!(found.contains(&"literal.int_decrement")); +} + +#[test] +fn a_literal_zero_is_not_replaced_by_zero() { + let found = mutators("fn f() -> i32 { 0 }", "literal"); + + assert!(!found.contains(&"literal.int_to_zero")); +} + +#[test] +fn a_literal_one_is_not_replaced_by_one() { + let found = mutators("fn f() -> i32 { 1 }", "literal"); + + assert!(!found.contains(&"literal.int_to_one")); +} + +#[test] +fn an_error_type_from_another_crate_is_not_given_a_default() { + for source in [ + "fn f() -> Result { todo!() }", + "use std::io; fn f() -> Result { todo!() }", + "use std::io::Error; fn f() -> Result { todo!() }", + "use anyhow::Error; fn f() -> Result { todo!() }", + ] { + let found = candidates(source, "fn_value"); + let texts: Vec<&str> = found.iter().map(|c| c.replacement.as_str()).collect(); + + assert!(!texts.contains(&"Err(Default::default())"), "{source}: {texts:?}"); + } +} + +#[test] +fn an_ok_is_not_flipped_to_an_error_the_signature_cannot_default() { + let found = mutators("use std::io; fn f() -> Result { Ok(1) }", "result"); + + assert!(!found.contains(&"result.ok_to_err"), "{found:?}"); +} + +#[test] +fn an_ok_is_still_flipped_when_the_error_type_is_the_workspace_s_own() { + let found = mutators("fn f() -> Result { Ok(1) }", "result"); + + assert!(found.contains(&"result.ok_to_err"), "{found:?}"); +} + +#[test] +fn an_ok_inside_an_aliased_result_stays_optimistic() { + let found = mutators("fn f() -> Result { Ok(1) }", "result"); + + assert!(found.contains(&"result.ok_to_err"), "{found:?}"); +} + +#[test] +fn a_workspace_error_type_is_still_given_a_default() { + for source in [ + "fn f() -> Result { todo!() }", + "fn f() -> Result { todo!() }", + "fn f() -> Result { todo!() }", + ] { + let found = candidates(source, "fn_value"); + let texts: Vec<&str> = found.iter().map(|c| c.replacement.as_str()).collect(); + + assert!(texts.contains(&"Err(Default::default())"), "{source}: {texts:?}"); + } +} + +#[test] +fn the_one_formatting_error_that_does_have_a_default_keeps_its_mutant() { + for source in [ + "fn f() -> Result { todo!() }", + "use core::fmt; fn f() -> Result { todo!() }", + "use core::fmt::Error; fn f() -> Result { todo!() }", + ] { + let found = candidates(source, "fn_value"); + let texts: Vec<&str> = found.iter().map(|c| c.replacement.as_str()).collect(); + + assert!(texts.contains(&"Err(Default::default())"), "{source}: {texts:?}"); + } +} + +#[test] +fn a_small_literal_is_not_mutated_to_the_same_value_twice() { + for source in ["fn f() -> i32 { 0 }", "fn f() -> i32 { 1 }", "fn f() -> i32 { 2 }"] { + let found = candidates(source, "literal"); + let mut edits: Vec<(usize, usize, &str)> = found.iter().map(|c| (c.span.start, c.span.end, c.replacement.as_str())).collect(); + let count = edits.len(); + + edits.sort_unstable(); + edits.dedup(); + assert_eq!(edits.len(), count, "{source} produced a duplicate edit"); + } +} + +#[test] +fn a_collision_keeps_the_perturbation_rather_than_the_value() { + let found = mutators("fn f() -> i32 { 0 }", "literal"); + + assert!(found.contains(&"literal.int_increment")); + assert!(!found.contains(&"literal.int_to_one")); +} + +#[test] +fn a_collision_still_yields_a_mutant_when_only_the_value_mutator_is_selected() { + let found = mutators("fn f() -> i32 { 0 }", "literal.int_to_one"); + + assert_eq!(found, vec!["literal.int_to_one"]); +} + +#[test] +fn increment_replacements_are_the_neighbouring_values() { + let found = candidates("fn f() -> i32 { 5 }", "literal.int_increment,literal.int_decrement"); + let mut replacements: Vec<&str> = found.iter().map(|c| c.replacement.as_str()).collect(); + + replacements.sort_unstable(); + assert_eq!(replacements, vec!["4", "6"]); +} + +#[test] +fn an_explicitly_unsigned_zero_is_not_decremented_below_its_range() { + assert!(candidates("fn f() -> u32 { 0u32 }", "literal.int_decrement").is_empty()); +} + +#[test] +fn an_unsuffixed_zero_keeps_its_decrement_candidate() { + let found = candidates("fn f() -> i32 { 0 }", "literal.int_decrement"); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].replacement, "-1"); +} + +#[test] +fn a_borrowed_literal_array_is_left_alone_so_it_can_still_be_promoted() { + let source = "fn f(k: &str) -> Option<&'static [&'static str]> { Some(match k { \"a\" => &[\"id\", \"name\"], _ => return None }) }"; + let found = mutators(source, "literal"); + + assert!(found.is_empty(), "promotable borrow was instrumented: {found:?}"); +} + +#[test] +fn a_borrowed_array_of_computed_values_is_still_mutated() { + let found = mutators("fn f(n: i32) -> i32 { let v = &[n + 1, n * 2]; v[0] }", "arith"); + + assert!(found.contains(&"arith.add_to_sub")); +} + +#[test] +fn a_let_chain_condition_is_not_negated_or_replaced() { + let source = "fn f(x: Option, y: bool) -> i32 { if let Some(n) = x && y { n } else { 0 } }"; + let found = mutators(source, "cond,logical"); + + assert!(found.is_empty(), "let-chain condition was mutated: {found:?}"); +} + +#[test] +fn a_binding_at_the_end_of_a_let_chain_is_also_recognized() { + let source = "fn f(x: Option, y: bool) -> i32 { if y && let Some(n) = x { n } else { 0 } }"; + let found = mutators(source, "cond,logical"); + + assert!(found.is_empty(), "trailing let-chain binding was mutated: {found:?}"); +} + +#[test] +fn a_while_let_chain_condition_is_not_negated() { + let source = "fn f(mut x: Option, y: bool) -> i32 { while let Some(_n) = x && y { x = None; } 0 }"; + let found = mutators(source, "cond"); + + assert!(!found.contains(&"cond.negate")); +} + +#[test] +fn an_ordinary_compound_condition_is_still_mutated() { + let found = mutators( + "fn f(a: bool, b: bool) -> bool { if a && b { true } else { false } }", + "cond,logical", + ); + + assert!(found.contains(&"cond.negate")); + assert!(found.contains(&"logical.and_to_or")); +} + +#[test] +fn an_empty_string_literal_is_not_replaced_by_an_empty_string() { + let found = mutators("fn f() -> &'static str { \"\" }", "literal"); + + assert!(!found.contains(&"literal.str_to_empty")); + assert!(found.contains(&"literal.str_to_xyzzy")); +} + +#[test] +fn the_marker_string_is_not_replaced_by_itself() { + // The mutant would be the original program, so it could never be killed and would be + // reported as a survivor on every run. + let found = mutators("fn f() -> &'static str { \"xyzzy\" }", "literal"); + + assert!(!found.contains(&"literal.str_to_xyzzy"), "{found:?}"); + assert!(found.contains(&"literal.str_to_empty"), "{found:?}"); +} + +#[test] +fn a_panic_message_is_not_rewritten() { + let found = mutators("fn f(x: Option) -> i32 { x.expect(\"x must be present\") }", "literal"); + + assert!(!found.contains(&"literal.str_to_empty"), "{found:?}"); + assert!(!found.contains(&"literal.str_to_xyzzy"), "{found:?}"); +} + +#[test] +fn an_expected_error_message_is_not_rewritten() { + let found = mutators("fn f(x: Result) -> String { x.expect_err(\"must fail\") }", "literal"); + + assert!(!found.contains(&"literal.str_to_xyzzy"), "{found:?}"); +} + +#[test] +fn a_value_a_panic_message_is_asked_of_is_still_mutated() { + // Only the message is exempt. The receiver is ordinary code and stays in the population. + let found = mutators( + "fn f(x: &str) -> i32 { x.strip_prefix(\"go\").expect(\"prefixed\").len() as i32 }", + "literal", + ); + + assert!(found.contains(&"literal.str_to_xyzzy"), "{found:?}"); +} + +#[test] +fn a_call_that_merely_shares_the_name_keeps_its_arguments() { + // `expect` with any other arity is somebody else's method, not the standard library's. + let found = mutators( + "struct S; impl S { fn expect(&self, _a: &str, _b: &str) {} } fn f(s: S) { s.expect(\"one\", \"two\") }", + "literal", + ); + + assert!(found.contains(&"literal.str_to_xyzzy"), "{found:?}"); +} + +#[test] +fn a_condition_that_is_already_a_literal_is_not_replaced_by_that_literal() { + let found = mutators("fn f() -> i32 { if true { 1 } else { 2 } }", "cond"); + + assert!(!found.contains(&"cond.always_true"), "{found:?}"); + assert!(found.contains(&"cond.always_false"), "{found:?}"); + + let found = mutators("fn f() -> i32 { if false { 1 } else { 2 } }", "cond"); + + assert!(found.contains(&"cond.always_true"), "{found:?}"); + assert!(!found.contains(&"cond.always_false"), "{found:?}"); +} + +#[test] +fn negated_integer_zero_has_no_mutant_but_negative_float_zero_does() { + assert!(candidates("fn f() -> i32 { -0 }", "unary.remove_neg").is_empty()); + assert!(!candidates("fn f() -> f64 { -0.0 }", "unary.remove_neg").is_empty()); + assert!(!candidates("fn f() -> i32 { -1 }", "unary.remove_neg").is_empty()); +} + +#[test] +fn a_parenthesised_zero_is_still_recognised_under_negation() { + // `-(0)` is exactly as much its own negation as `-0` is; a reader would not expect the + // redundant parentheses to turn a no-op mutant into a real one. + assert!(candidates("fn f() -> i32 { -(0) }", "unary.remove_neg").is_empty()); +} + +#[test] +fn associated_const_initializers_are_not_mutated() { + // A guard cannot be called in a const-evaluation context, so the mutant would not compile. + let source = "struct S; impl S { const N: i32 = 1 + 2; }"; + + assert!(candidates(source, "").is_empty(), "{:?}", candidates(source, "")); +} + +#[test] +fn trait_const_defaults_are_not_mutated() { + let source = "trait T { const N: i32 = 1 + 2; }"; + + assert!(candidates(source, "").is_empty(), "{:?}", candidates(source, "")); +} + +#[test] +fn a_string_returning_function_gets_an_owned_marker() { + // `"xyzzy"` is a `&'static str`, so a `String`-returning function needs the owned form or + // every one of these mutants is withdrawn as unviable. + let found = candidates("fn f() -> String { String::new() }", "fn_value.xyzzy_string"); + + assert_eq!(found[0].replacement, "\"xyzzy\".to_owned()"); +} + +#[test] +fn booleans_flip_to_the_other_value() { + let found = candidates("fn f() -> bool { true }", "literal.bool_flip"); + + assert_eq!(found[0].replacement, "false"); +} + +#[test] +fn compound_assignment_is_mutated() { + let found = mutators("fn f(a: &mut i32) { *a += 1; }", "assign"); + + assert_eq!(found, vec!["assign.add_to_sub"]); +} + +#[test] +fn unary_operators_can_be_removed() { + let found = mutators("fn f(a: i32) -> i32 { -a }", "unary"); + + assert_eq!(found, vec!["unary.remove_neg"]); +} + +#[test] +fn logical_not_can_be_removed() { + let found = candidates("fn f(a: bool) -> bool { !a }", "unary.remove_not"); + + assert_eq!(found[0].replacement, "a"); +} + +#[test] +fn statement_deletion_covers_calls_assignments_and_ignored_statements() { + let source = "fn f(v: &mut Vec, mut a: i32) { + v.push(1); + a = 2; + a += 3; + a + 4; + }"; + let found = mutators(source, "stmt"); + + assert!(found.contains(&"stmt.delete_call")); + assert_eq!(found.iter().filter(|name| **name == "stmt.delete_assign").count(), 2); +} + +/// A `let` with no value makes the assignment that settles it load-bearing, and deleting it +/// produces a mutant that cannot compile. The error rustc raises, E0381, is reported at the +/// binding's first *use* rather than at the deleted statement, so withdrawal cannot match the +/// diagnostic to the mutant that caused it and abandons the entire run. The assignment must +/// therefore never become a candidate in the first place. +#[test] +fn the_assignment_settling_a_deferred_let_is_not_deletable() { + let source = "fn f(flag: bool) -> i32 { + let scanned; + if flag { + scanned = 1; + } else { + scanned = 2; + } + scanned + }"; + let found = mutators(source, "stmt.delete_assign"); + + assert!(found.is_empty(), "{found:?}"); +} + +/// The exemption is about the binding being empty, not about the name. A `let` that supplies a +/// value leaves its later assignments as ordinary overwrites, which a test that reads the +/// variable afterwards should notice going missing. +#[test] +fn assigning_to_an_initialised_binding_stays_deletable() { + let source = "fn f() -> i32 { + let mut scanned = 0; + scanned = 1; + scanned + }"; + let found = mutators(source, "stmt.delete_assign"); + + assert_eq!(found, vec!["stmt.delete_assign"]); +} + +/// Shadowing re-uses the name for a different binding, and the new one is initialised. Because +/// declarations are recorded in source order, the second `let` must lift the exemption the +/// first one installed rather than leaving the name exempt for the rest of the function. +#[test] +fn a_later_initialised_let_lifts_the_deferral_for_that_name() { + let source = "fn f() -> i32 { + let scanned; + scanned = 1; + let mut scanned = scanned; + scanned = 2; + scanned + }"; + let found = mutators(source, "stmt.delete_assign"); + + assert_eq!(found, vec!["stmt.delete_assign"]); +} + +/// Deferred names are per-function: a nested function cannot see the outer one's locals, so an +/// assignment to its own fully-initialised binding must not inherit the outer exemption. +#[test] +fn a_nested_function_does_not_inherit_deferred_names() { + let source = "fn outer() -> i32 { + let scanned; + scanned = 1; + fn inner() -> i32 { + let mut scanned = 0; + scanned = 5; + scanned + } + scanned + inner() + }"; + let found = mutators(source, "stmt.delete_assign"); + + assert_eq!(found, vec!["stmt.delete_assign"]); +} + +/// The exemption is keyed on a bare name because only a bare name can be what a deferred `let` +/// is waiting on. Assigning through a field presupposes the binding it belongs to already holds +/// a value, so that assignment carries no such load-bearing weight and stays an ordinary +/// candidate a test should notice the loss of. +#[test] +fn assigning_through_a_field_is_never_treated_as_settling_a_deferred_let() { + let source = "fn f() -> i32 { + let mut pair = (0, 0); + pair.0 = 1; + pair.0 + }"; + let found = mutators(source, "stmt.delete_assign"); + + assert_eq!(found, vec!["stmt.delete_assign"]); +} + +/// Leaving a block restores the type evidence a shadow inside it displaced. `total` is a +/// `String` inside the block and the numeric parameter after it, so the perturbation family is +/// offered on the post-block use and withheld on the in-block one. If the inner shadow outlived +/// the block, the later use would be judged a `String` and lose its perturbation — the reverse +/// of the failure the block scoping exists to prevent, and just as wrong. +#[test] +fn a_block_shadow_of_a_binding_does_not_outlive_the_block() { + let source = "fn f(total: i32) { { let total: String = String::new(); sink(total); } sink(total); }"; + let found = mutators(source, "expr"); + + assert_eq!( + found.iter().filter(|name| **name == "expr.increment").count(), + 1, + "only the post-block use of the numeric `total` should be perturbed: {found:?}" + ); + assert_eq!(found.iter().filter(|name| **name == "expr.decrement").count(), 1, "{found:?}"); +} + +/// Leaving a block also restores the deferral state it changed. The first block re-declares +/// `scanned` with a value, lifting the deferral inside itself; the second then settles the +/// still-deferred outer `scanned`, an assignment that must stay exempt from deletion. If the +/// first block's removal outlived it, the outer `scanned` would look initialised and the +/// settling assignment would wrongly become a `stmt.delete_assign` candidate. +#[test] +fn a_block_change_to_a_deferral_does_not_outlive_the_block() { + let source = "fn f() -> i32 { + let scanned; + { let scanned = 0; let _ = scanned; } + { scanned = 1; let _ = scanned; } + scanned + }"; + let found = mutators(source, "stmt.delete_assign"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn repeat_lengths_and_enum_discriminants_are_const_contexts() { + let source = "enum E { A = 1 + 2 } fn f(n: i32) { let _ = [n + 1; 2 + 3]; }"; + let found = mutators(source, "arith"); + + assert!(found.contains(&"arith.add_to_sub"), "{found:?}"); + assert_eq!(found.len(), 2, "{found:?}"); +} + +#[test] +fn trait_default_methods_are_mutated_but_excluded_ones_are_not() { + let source = "trait T { + fn f(&self) -> i32 { 1 } + #[cfg(test)] + fn helper(&self) -> i32 { 2 } + }"; + let found = candidates(source, "fn_value.zero,literal.int_to_zero"); + + assert!(found.iter().any(|candidate| &*candidate.item_path == "T::f")); + assert!(found.iter().all(|candidate| &*candidate.item_path != "T::helper")); +} + +#[test] +fn const_trait_defaults_are_left_inert() { + let source = "trait T { const fn f(&self) -> i32 { 1 + 2 } }"; + + assert!(candidates(source, "arith,fn_value").is_empty()); +} + +#[test] +fn impl_paths_handle_reference_and_non_path_self_types() { + let source = "trait T { fn f(&self) -> i32; } + struct S; + impl T for &S { fn f(&self) -> i32 { 1 + 1 } } + impl T for (S,) { fn f(&self) -> i32 { 2 + 2 } }"; + let found = candidates(source, "arith.add_to_sub"); + let paths: Vec<&str> = found.iter().map(|candidate| &*candidate.item_path).collect(); + + assert!(paths.contains(&"::f"), "{paths:?}"); + assert!(paths.contains(&"<_ as T>::f"), "{paths:?}"); +} + +#[test] +fn borrowed_promotable_shapes_are_classified_without_touching_unary_minus() { + let source = "fn f() -> &'static (i32, [i32; 2], &'static i32, i32) { + &((-1), [0; 2], &3, 4) + }"; + let found = mutators(source, "literal,unary"); + + assert!(!found.contains(&"unary.remove_neg"), "{found:?}"); +} + +#[test] +fn function_value_replacements_cover_return_type_shapes() { + let source = "fn unit() { work(); } + fn explicit_unit() -> () { work(); } + fn unsigned() -> usize { 3 } + fn float() -> f64 { 3.0 } + fn owned_string() -> String { make() } + fn vec_deque() -> std::collections::VecDeque { make() } + fn reference(x: &i32) -> &i32 { x } + fn array() -> [i32; 1] { [1] } + fn unknown() -> Custom { make() }"; + let found = mutators(source, "fn_value"); + + for expected in [ + "fn_value.unit", + "fn_value.zero", + "fn_value.one", + "fn_value.minus_one", + "fn_value.empty_string", + "fn_value.xyzzy_string", + "fn_value.empty_collection", + "fn_value.one_element", + "fn_value.default", + ] { + assert!(found.contains(&expected), "{expected} not in {found:?}"); + } +} + +#[test] +fn a_map_return_offers_an_empty_map_and_a_single_pairing() { + // A map's element is a pair, so unlike an ordinary collection the one-element form has to + // supply both a key and a value; without a test naming a `HashMap` return type directly, + // nothing would ever exercise the branch that builds that pair rather than a bare element. + let source = "fn f() -> std::collections::HashMap { make() }"; + let found = candidates(source, "fn_value"); + let texts: Vec<_> = found.iter().map(|candidate| candidate.replacement.as_str()).collect(); + + assert!(texts.iter().any(|text| text.contains("HashMap::new()")), "{texts:?}"); + assert!( + texts + .iter() + .any(|text| text.contains("core::iter::once((") && text.contains(").collect()")), + "{texts:?}" + ); +} + +#[test] +fn a_map_whose_key_is_a_type_the_caller_chooses_offers_only_the_empty_map() { + // When nothing is known about the key, there is no value to pair with one that is, and the + // one-element form has to be withheld entirely rather than invented with half a pair + // missing; a `Default::default()` key would compile only by accident of the type the + // caller happened to choose. + let source = "fn f() -> std::collections::HashMap { make() }"; + let found = candidates(source, "fn_value"); + let texts: Vec<_> = found.iter().map(|candidate| candidate.replacement.as_str()).collect(); + + assert!(texts.iter().any(|text| text.contains("HashMap::new()")), "{texts:?}"); + assert!(!texts.iter().any(|text| text.contains("core::iter::once")), "{texts:?}"); +} + +#[test] +fn unselected_function_values_are_filtered_at_emit_time() { + let found = candidates("fn f() -> i32 { 2 }", "fn_value.one"); + + assert_eq!(found.len(), 1); + assert_eq!(found[0].mutator, "fn_value.one"); +} + +#[test] +fn literals_without_literal_mutators_are_ignored() { + let found = candidates("fn f() -> char { 'x' }", "literal"); + + assert!(found.is_empty()); +} + +#[test] +fn huge_integer_literals_skip_neighbour_replacements() { + let found = mutators("fn f() -> u128 { 340282366920938463463374607431768211455 }", "literal"); + + assert!(found.contains(&"literal.int_to_zero")); + assert!(!found.contains(&"literal.int_increment")); + assert!(!found.contains(&"literal.int_decrement")); +} + +#[test] +fn ids_are_stable_across_reformatting() { + let compact = "fn f(a: i32, b: i32) -> bool { a < b }"; + let spaced = "fn f(a: i32, b: i32) -> bool {\n\n a < b\n\n}\n"; + + let left = SourceFile::parse("test.rs", compact.to_owned()).unwrap(); + let right = SourceFile::parse("test.rs", spaced.to_owned()).unwrap(); + let selection = Selection::parse("relational").unwrap(); + + let left_ids: Vec = into_definitions(&left, collect(&left, &selection)) + .into_iter() + .map(|m| m.id.to_string()) + .collect(); + + let right_ids: Vec = into_definitions(&right, collect(&right, &selection)) + .into_iter() + .map(|m| m.id.to_string()) + .collect(); + + assert_eq!(left_ids, right_ids); +} + +#[test] +fn ids_survive_a_line_inserted_above() { + let before = "fn f(a: i32, b: i32) -> bool { a < b }"; + let after = "// a new comment\n\nfn f(a: i32, b: i32) -> bool { a < b }"; + + let left = SourceFile::parse("test.rs", before.to_owned()).unwrap(); + let right = SourceFile::parse("test.rs", after.to_owned()).unwrap(); + let selection = Selection::parse("relational").unwrap(); + + let left_ids: Vec = into_definitions(&left, collect(&left, &selection)) + .into_iter() + .map(|m| m.id.to_string()) + .collect(); + + let right_ids: Vec = into_definitions(&right, collect(&right, &selection)) + .into_iter() + .map(|m| m.id.to_string()) + .collect(); + + assert_eq!(left_ids, right_ids); +} + +#[test] +fn identical_sites_in_one_function_get_distinct_ids() { + let source = "fn f(a: i32, b: i32) -> bool { (a < b) && (a < b) }"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("relational.lt_to_le").unwrap(); + let mutants = into_definitions(&file, collect(&file, &selection)); + + assert_eq!(mutants.len(), 2); + assert_ne!(mutants[0].id, mutants[1].id); + assert_eq!(mutants[0].occurrence, 0); + assert_eq!(mutants[1].occurrence, 1); +} + +#[test] +fn identical_sites_in_different_functions_get_distinct_ids() { + let source = "fn f(a: i32, b: i32) -> bool { a < b }\nfn g(a: i32, b: i32) -> bool { a < b }"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("relational.lt_to_le").unwrap(); + let mutants = into_definitions(&file, collect(&file, &selection)); + + assert_eq!(mutants.len(), 2); + assert_ne!(mutants[0].id, mutants[1].id); + assert_eq!(mutants[0].occurrence, 0); + assert_eq!(mutants[1].occurrence, 0); +} + +#[test] +fn different_replacements_at_one_site_get_distinct_ids() { + let source = "fn f(a: i32, b: i32) -> bool { a < b }"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("relational").unwrap(); + let mutants = into_definitions(&file, collect(&file, &selection)); + + assert_eq!(mutants.len(), 2); + assert_ne!(mutants[0].id, mutants[1].id); +} + +#[test] +fn mutants_carry_line_and_column() { + let source = "fn f(a: i32, b: i32) -> bool {\n a < b\n}"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("relational.lt_to_le").unwrap(); + let mutants = into_definitions(&file, collect(&file, &selection)); + + assert_eq!(mutants[0].site.line, 2); + assert_eq!(mutants[0].site.column, 5); +} + +#[test] +fn doc_comments_are_not_string_literals() { + // A doc comment is desugared into `#[doc = "..."]`, so a visitor that walks attributes + // reports every line of documentation in the tree as a mutable string. + let source = "/// documentation\nfn f() {}"; + + assert!(candidates(source, "literal").is_empty()); +} + +#[test] +fn attribute_arguments_are_not_mutated() { + let source = "#[deprecated(note = \"use g instead\", since = \"1.0\")]\nfn f() {}"; + + assert!(candidates(source, "all").is_empty()); +} + +#[test] +fn string_literals_in_real_code_are_still_mutated() { + let source = "/// documentation\nfn f() -> &'static str { \"hello\" }"; + let found = mutators(source, "literal.str_to_empty"); + + assert_eq!(found, vec!["literal.str_to_empty"]); +} + +#[test] +fn an_empty_file_yields_nothing() { + assert!(candidates("", "all").is_empty()); +} + +#[test] +fn a_file_of_only_types_yields_nothing() { + assert!(candidates("struct S { a: i32 } enum E { A, B }", "all").is_empty()); +} + +// ---- Match guards. ----------------------------------------------------------------------- + +#[test] +fn a_match_guard_is_mutated_the_way_a_branch_condition_is() { + let source = "fn f(v: i32) -> i32 { match v { n if n > 0 => n, _ => 0 } }"; + let found = mutators(source, "match_guard"); + + // Without this family a guard is the one condition in the language nothing asks about, so a + // suite that never exercises the guarded case scores as though it had. + assert!(found.contains(&"match_guard.negate"), "{found:?}"); + assert!(found.contains(&"match_guard.always_true"), "{found:?}"); + assert!(found.contains(&"match_guard.always_false"), "{found:?}"); +} + +#[test] +fn an_unguarded_arm_offers_no_guard_mutants() { + let source = "fn f(v: i32) -> i32 { match v { 1 => 1, _ => 0 } }"; + + assert!(candidates(source, "match_guard").is_empty()); +} + +#[test] +fn a_guard_that_is_already_a_literal_is_not_replaced_by_that_literal() { + let source = "fn f(v: i32) -> i32 { match v { n if true => n, _ => 0 } }"; + let found = mutators(source, "match_guard"); + + // Replacing `true` with `true` is the original program, which can never be caught and + // would sit in the report as a permanent survivor. + assert!(!found.contains(&"match_guard.always_true"), "{found:?}"); + assert!(found.contains(&"match_guard.always_false"), "{found:?}"); +} + +// ---- Match arms. ------------------------------------------------------------------------- + +#[test] +fn an_arm_before_a_wildcard_can_be_stopped_from_matching() { + let source = "fn f(v: i32) -> i32 { match v { 1 => 10, 2 => 20, _ => 0 } }"; + let found = candidates(source, "match_arm"); + + assert_eq!(found.len(), 2, "{found:?}"); + assert!(found.iter().all(|c| c.shape == Shape::Arm), "{found:?}"); +} + +#[test] +fn the_wildcard_itself_is_never_stopped_from_matching() { + let source = "fn f(v: i32) -> i32 { match v { 1 => 10, _ => 0 } }"; + let found = candidates(source, "match_arm"); + + // Guarding the wildcard leaves the match non-exhaustive, which is a compile error rather + // than a question about the tests: the compiler does not count a guarded arm. + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!( + found[0].span, + span_of(source, "1 => 10").start..span_of(source, "1 => 10").start + 1 + ); +} + +#[test] +fn a_match_without_a_wildcard_offers_no_arm_mutants() { + let source = "fn f(v: bool) -> i32 { match v { true => 1, false => 0 } }"; + + assert!( + candidates(source, "match_arm").is_empty(), + "an exhaustive match has nothing to fall through to" + ); +} + +#[test] +fn an_arm_after_the_wildcard_offers_nothing() { + let source = "fn f(v: i32) -> i32 { match v { _ => 0, 1 => 10 } }"; + + // Nothing falls through to an arm the wildcard already swallowed, so the mutant would be + // an equivalent one that survives forever. + assert!(candidates(source, "match_arm").is_empty()); +} + +#[test] +fn a_guarded_arm_is_disabled_by_its_guard_rather_than_by_a_second_mutant() { + let source = "fn f(v: i32) -> i32 { match v { n if n > 0 => n, _ => 0 } }"; + + // `match_guard.always_false` already stops the arm matching. A second mutant saying the + // same thing would double the cost of one question. + assert!(candidates(source, "match_arm").is_empty()); +} + +// ---- Struct literal fields. -------------------------------------------------------------- + +#[test] +fn a_struct_field_is_omitted_only_when_a_base_supplies_it() { + let source = "fn f() -> C { C { a: 1, b: 2, ..Default::default() } }"; + let found = candidates(source, "struct_field"); + + assert_eq!(found.len(), 2, "{found:?}"); + assert!( + found + .iter() + .any(|c| c.replacement.contains("b: 2") && !c.replacement.contains("a: 1")) + ); + assert!( + found + .iter() + .any(|c| c.replacement.contains("a: 1") && !c.replacement.contains("b: 2")) + ); +} + +#[test] +fn a_struct_literal_without_a_base_offers_nothing() { + let source = "fn f() -> C { C { a: 1, b: 2 } }"; + + // Removing a field from a literal that names every one of them does not compile. + assert!(candidates(source, "struct_field").is_empty()); +} + +#[test] +fn omitting_the_last_field_leaves_the_base_intact() { + let source = "fn f() -> C { C { a: 1, ..Default::default() } }"; + let found = candidates(source, "struct_field"); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].replacement, "C { ..Default::default() }"); +} + +// ---- Ranges. ----------------------------------------------------------------------------- + +#[test] +fn a_half_open_range_offers_its_inclusive_form() { + let source = "fn f(n: usize) -> usize { let mut t = 0; for i in 0..n { t += i; } t }"; + let found = candidates(source, "range"); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].mutator, "range.exclusive_to_inclusive"); + + // Spelled as arithmetic on the endpoint rather than as `..=`, because the mutant and the + // original share the arms of an `if` and so have to share a type. + assert_eq!(found[0].replacement, "(0)..((n) + 1)"); +} + +#[test] +fn an_inclusive_range_offers_its_half_open_form() { + let source = "fn f(n: usize) -> usize { let mut t = 0; for i in 0..=n { t += i; } t }"; + let found = candidates(source, "range"); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].mutator, "range.inclusive_to_exclusive"); + assert_eq!(found[0].replacement, "(0)..=((n) - 1)"); +} + +#[test] +fn a_range_with_no_end_has_no_inclusive_form_to_offer() { + let source = "fn f(v: &[u8]) -> &[u8] { &v[1..] }"; + + assert!(candidates(source, "range").is_empty()); +} + +#[test] +fn a_range_with_no_start_still_moves_its_boundary() { + let source = "fn f(v: &[u8], n: usize) -> &[u8] { &v[..n] }"; + let found = candidates(source, "range"); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].replacement, "..((n) + 1)"); +} + +// ---- Loop exits. ------------------------------------------------------------------------- + +#[test] +fn break_and_continue_are_swapped_for_each_other() { + let source = "fn f(v: &[i32]) { for x in v { if *x == 0 { continue; } if *x == 1 { break; } } }"; + let found = mutators(source, "loop.break_to_continue,loop.continue_to_break"); + + assert!(found.contains(&"loop.break_to_continue"), "{found:?}"); + assert!(found.contains(&"loop.continue_to_break"), "{found:?}"); +} + +#[test] +fn a_break_carrying_a_value_is_left_alone() { + let source = "fn f() -> i32 { loop { break 1; } }"; + + // `continue` produces no value, so the loop would no longer have the type its context + // requires and the mutant would be withdrawn as unviable rather than measured. + assert!(!mutators(source, "loop.break_to_continue").contains(&"loop.break_to_continue")); +} + +#[test] +fn a_labelled_break_is_left_alone_but_a_labelled_continue_is_not() { + let source = "fn f(v: &[i32]) { 'outer: for x in v { for y in v { if x == y { continue 'outer; } break 'outer; } } }"; + let found = candidates(source, "loop"); + + // A label on `continue` can only name a loop, so `break` accepts it. A label on `break` + // may name a labelled block, which `continue` cannot leave at all. + assert!( + found + .iter() + .any(|c| c.mutator == "loop.continue_to_break" && c.replacement == "break 'outer") + ); + assert!(!found.iter().any(|c| c.mutator == "loop.break_to_continue")); +} + +#[test] +fn a_continue_is_not_changed_to_a_valueless_break_in_a_value_producing_loop() { + let source = "fn f(flag: bool) -> i32 { loop { if flag { continue; } break 1; } }"; + + assert!(candidates(source, "loop.continue_to_break").is_empty()); +} + +#[test] +fn a_labelled_continue_is_not_changed_to_a_valueless_break_in_its_value_producing_loop() { + let source = "fn f(flag: bool) -> i32 { 'outer: loop { while flag { continue 'outer; } break 'outer 1; } }"; + + assert!(candidates(source, "loop.continue_to_break").is_empty()); +} + +#[test] +fn a_break_or_continue_statement_can_be_deleted() { + let source = "fn f(v: &[i32]) { for x in v { if *x == 0 { continue; } } }"; + let found = candidates(source, "loop.delete_continue"); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].shape, Shape::Stmt); +} + +// ---- Focused numeric perturbation. -------------------------------------------------------- + +#[test] +fn a_call_argument_is_perturbed_by_one_in_both_directions() { + let source = "fn f(n: usize) { g(n); }"; + let found = candidates(source, "expr"); + + assert_eq!(found.len(), 2, "{found:?}"); + assert!(found.iter().any(|c| c.replacement == "(n) + 1")); + assert!(found.iter().any(|c| c.replacement == "(n) - 1")); +} + +#[test] +fn a_literal_argument_is_left_to_the_literal_family() { + let source = "fn f() { g(3); }"; + + // `literal.int_increment` already offers `4` here. Offering `(3) + 1` beside it would buy + // a second run of the whole suite for an answer already in hand. + assert!(candidates(source, "expr").is_empty()); +} + +#[test] +fn a_capacity_argument_is_not_perturbed() { + let source = "fn f(n: usize) -> Vec { Vec::with_capacity(n) }"; + + // A test that noticed would be a test pinning an allocation strategy, so reporting the + // survivor would accuse the suite of a gap it should not be asked to fill. + assert!(candidates(source, "expr").is_empty()); +} + +#[test] +fn an_index_and_a_range_bound_are_perturbed() { + let indexed = candidates("fn f(v: &[u8], i: usize) -> u8 { v[i] }", "expr"); + let bounded = candidates("fn f(v: &[u8], n: usize) -> &[u8] { &v[..n] }", "expr"); + + // Two positions overlap here and both are wanted: the subscript, where being wrong by one + // reads the neighbouring element, and the returned `u8`, where being wrong by one is the + // classic off-by-one in the answer itself. + assert!(indexed.iter().any(|c| c.replacement == "(i) + 1"), "{indexed:?}"); + + // The element is offered too, but on the signature's word rather than on a guess about + // what subscripting yields: `-> u8` says the returned value is a number. + assert!(indexed.iter().any(|c| c.replacement == "(v[i]) + 1"), "{indexed:?}"); + + // Take that word away and it falls silent, because nothing else here says what `v[i]` is. + let unsaid = candidates("fn f(v: &[u8], i: usize) { g(v[i]); }", "expr"); + + assert!(!unsaid.iter().any(|c| c.replacement == "(v[i]) + 1"), "{unsaid:?}"); + + // The returned `&[u8]` is not a number, so only the range bound is offered. + assert_eq!(bounded.len(), 2, "{bounded:?}"); + assert!(bounded.iter().any(|c| c.replacement == "(n) - 1"), "{bounded:?}"); +} + +#[test] +fn a_returned_value_is_perturbed_however_it_is_returned() { + let trailing = candidates("fn f(n: usize) -> usize { n }", "expr"); + let explicit = candidates("fn f(n: usize) -> usize { return n; }", "expr"); + + assert_eq!(trailing.len(), 2, "{trailing:?}"); + assert_eq!(explicit.len(), 2, "{explicit:?}"); +} + +#[test] +fn integer_literal_tails_and_explicit_returns_have_identical_mutants() { + let describe = |source| { + let mut found: Vec<(&'static str, String)> = candidates(source, "literal,expr") + .into_iter() + .map(|candidate| (candidate.mutator, candidate.replacement.to_string())) + .collect(); + found.sort(); + found + }; + + assert_eq!(describe("fn f() -> i32 { 3 }"), describe("fn f() -> i32 { return 3 }")); +} + +#[test] +fn a_returned_number_in_tail_position_perturbs_nothing() { + // `return 5` as the body's tail has type `!`, so it passes the signature-only proof that a + // tail is a number, but `(return 5) + 1` returns before the `+ 1` is ever reached: the + // mutant is a twin of the original that no test could distinguish. The bare literal `5` is + // still the literal family's to perturb; this asks only about the increment/decrement pair. + let found = candidates("fn f() -> i32 { return 5 }", "expr.increment,expr.decrement"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn statically_divergent_tails_perturb_nothing() { + // Each of these leaves the function -- or spins forever -- before its supposed value is + // used, so wrapping it in `+ 1` or `- 1` changes nothing a test could observe. The proof + // that a numeric tail is worth perturbing has to exclude the tails that never yield one: + // `return`, the never-returning macros, a breakless `loop`, and branches that all diverge. + let divergent = [ + "fn f() -> i32 { panic!() }", + "fn f() -> i32 { unreachable!() }", + "fn f() -> i32 { todo!() }", + "fn f() -> i32 { unimplemented!() }", + "fn f() -> i32 { loop {} }", + "fn f(c: bool) -> i32 { if c { return 1 } else { return 2 } }", + ]; + + for source in divergent { + let found = candidates(source, "expr.increment,expr.decrement"); + + assert!(found.is_empty(), "{source}: {found:?}"); + } +} + +#[test] +fn an_ordinary_numeric_tail_still_perturbs_both_ways() { + // The guard rejects only the tails that diverge. A plain value-producing tail is the whole + // point of the family, and dropping its neighbours would blind the suite to an off-by-one + // in the one place the signature guarantees a number. + let found: Vec = candidates("fn f(value: i32) -> i32 { value }", "expr.increment,expr.decrement") + .into_iter() + .map(|candidate| candidate.replacement.to_string()) + .collect(); + + assert!(found.contains(&"(value) + 1".to_owned()), "{found:?}"); + assert!(found.contains(&"(value) - 1".to_owned()), "{found:?}"); +} + +#[test] +fn a_loop_that_can_break_with_a_value_is_still_perturbed() { + // A `loop` is only rejected when it truly cannot terminate. One that breaks with a value + // does hand a number back, so the family still owes it the increment/decrement pair; the + // breakless check must not swallow it. + let found = mutators("fn f() -> i32 { loop { break 5; } }", "expr.increment,expr.decrement"); + + assert!(found.contains(&"expr.increment"), "{found:?}"); + assert!(found.contains(&"expr.decrement"), "{found:?}"); +} + +#[test] +fn a_non_numeric_argument_is_not_perturbed() { + let source = "fn f(s: &str, c: bool, n: usize) { g(s, c, &s, |x| x, \"lit\", n); }"; + let found = candidates(source, "expr"); + + // Nothing here adds to an integer except `n`. Every mutant that cannot compile costs a + // share of a rebuild that finds nothing, so the filter is what keeps the family + // affordable — but the one argument that does add has to survive it, or the filter has + // bought its saving by hiding a gap in the suite. + assert!(found.iter().all(|c| c.replacement.starts_with("(n)")), "{found:?}"); + assert!(found.iter().any(|c| c.replacement == "(n) + 1"), "{found:?}"); +} + +#[test] +fn a_parameter_the_source_declares_a_number_is_still_perturbed_through_a_reference() { + // `&usize + 1` compiles, so treating every reference as unaddable would throw away a + // mutant that builds, runs and can genuinely be missed. + let found = candidates("fn f(n: &usize) { g(n); }", "expr"); + + assert!(found.iter().any(|c| c.replacement == "(n) + 1"), "{found:?}"); +} + +#[test] +fn an_annotated_local_is_judged_by_the_type_the_source_wrote_down() { + let source = "fn f() { let name: String = h(); let count: u32 = h(); g(name); g(count); }"; + let found = candidates(source, "expr"); + + assert!(found.iter().any(|c| c.replacement == "(count) + 1"), "{found:?}"); + assert!(!found.iter().any(|c| c.replacement.starts_with("(name)")), "{found:?}"); +} + +#[test] +fn a_local_whose_type_was_never_written_down_is_left_alone_rather_than_guessed_at() { + // Guessing that anything unaccounted for is a number was measured on this repository and + // was wrong three times in four, which made the two perturbation operators alone 78% of + // every mutant that failed to build. Silence is the answer when the source says nothing. + let guessed = candidates("fn f() { let total = h(); g(total); }", "expr"); + + assert!(guessed.is_empty(), "{guessed:?}"); + + // But the source says a great deal short of an annotation, and the point of refusing to + // guess is that reading what it does say has to make up the difference. An initialiser + // this collector can type answers exactly what the annotation would have. + let inferred = candidates("fn f(v: &[u8]) { let total = v.len(); g(total); }", "expr"); + + assert!(inferred.iter().any(|c| c.replacement == "(total) + 1"), "{inferred:?}"); +} + +#[test] +fn a_local_used_where_only_a_number_fits_is_perturbed_without_an_annotation() { + // Most locals carry no type and no initialiser worth reading, so the last evidence left is + // how the name is used. Each of these uses admits nothing but a number. + for source in [ + "fn f() { let i = h(); let _ = v[i]; g(i); }", + "fn f() { let i = h(); let _ = i - 1; g(i); }", + "fn f() { let i = h(); if i > 0 { g(i); } }", + "fn f() { let i = h(); let _ = i.saturating_sub(1); g(i); }", + "fn f() { for i in 0..8 { g(i); } }", + ] { + let found = candidates(source, "expr"); + + assert!(found.iter().any(|c| c.replacement == "(i) + 1"), "{source}: {found:?}"); + } +} + +#[test] +fn a_field_is_judged_by_the_struct_that_declares_it() { + // A field is very often read above its own `struct`, so the declarations are gathered in a + // pass of their own before anything is offered. + let source = "fn f(s: S) { g(s.count, s.name); } struct S { name: String, count: usize }"; + let found = candidates(source, "expr"); + + assert!(found.iter().any(|c| c.replacement == "(s.count) + 1"), "{found:?}"); + assert!(!found.iter().any(|c| c.replacement.starts_with("(s.name)")), "{found:?}"); +} + +#[test] +fn a_type_named_where_a_value_is_expected_is_not_perturbed() { + let source = "fn f() { g(PhantomData, Vec::new(), items.iter(), MAX); }"; + let found = candidates(source, "expr"); + + // `MAX` is the point of the exception: constants are spelled in the screaming case and are + // among the most worthwhile things this family has to offer, so the camel-case rule that + // rejects `PhantomData` must not reject them too. + assert!(found.iter().any(|c| c.replacement == "(MAX) + 1"), "{found:?}"); + assert!(found.iter().all(|c| c.replacement.starts_with("(MAX)")), "{found:?}"); +} + +#[test] +fn a_local_binding_does_not_leak_into_a_nested_function() { + // A function defined inside another cannot see the outer one's locals, so reasoning from + // them would reach a confident conclusion about a completely unrelated name. + let source = "fn outer() { let value: String = h(); fn inner(value: u32) { g(value); } }"; + let found = candidates(source, "expr"); + + assert!(found.iter().any(|c| c.replacement == "(value) + 1"), "{found:?}"); +} + +#[test] +fn a_default_is_not_invented_for_a_type_the_caller_chooses() { + // `D::Error` is whatever the caller's deserializer says it is, and nothing promises it has + // a `Default`. On a serde-shaped API this was the single largest source of mutants that + // could not compile. + let source = "fn f(d: D) -> Result { g(d); Ok(1) }"; + let found = candidates(source, "fn_value"); + + assert!( + !found.iter().any(|c| c.replacement.contains("Err(Default::default())")), + "{found:?}" + ); + + // The other half of the return type is concrete, so it keeps everything it had. A rule + // that took the whole signature out would stop asking whether the value is tested at all. + assert!(found.iter().any(|c| c.replacement == "Ok(0)"), "{found:?}"); +} + +#[test] +fn a_default_is_still_invented_for_a_parameter_declared_to_have_one() { + // The promise this rule looks for was made explicitly, so the mutant it would otherwise + // withhold compiles and is worth offering. + let source = "fn f(t: T) -> Result { g(t); Ok(1) }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().any(|c| c.replacement == "Err(Default::default())"), "{found:?}"); +} + +#[test] +fn a_default_is_not_invented_for_a_trait_object() { + // `dyn Reader` names a capability rather than a type; there is no `default()` to call. + let found = candidates("fn f() -> Box { h() }", "fn_value"); + + assert!(!found.iter().any(|c| c.replacement.contains("Default::default()")), "{found:?}"); +} + +#[test] +fn an_associated_type_of_self_is_still_given_a_default() { + // `Self::Value` looks like `D::Error` but is not: inside an `impl` it resolves to a type + // that block chose, which often does have a `Default`. Treating it as abstract cost six + // mutants a real suite had caught. + let source = "impl Visitor for V { fn visit(self) -> Result { h() } }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().any(|c| c.replacement == "Ok(Default::default())"), "{found:?}"); +} + +#[test] +fn perturbation_is_on_by_default() { + let source = "fn f(n: usize) { g(n); }"; + let found = mutators(source, "@default"); + + assert!(found.contains(&"expr.increment"), "{found:?}"); +} + +#[test] +fn option_and_result_construction_is_mutated_both_ways() { + let found = mutators("fn f(flag: bool) { let _ = if flag { Some(1) } else { None }; }", "option"); + + assert!(found.contains(&"option.some_to_none"), "{found:?}"); + assert!(found.contains(&"option.none_to_some"), "{found:?}"); + + let found = mutators("fn f(flag: bool) { let _ = if flag { Ok(1) } else { Err(2) }; }", "result"); + + assert!(found.contains(&"result.ok_to_err"), "{found:?}"); + assert!(found.contains(&"result.err_to_ok"), "{found:?}"); +} + +#[test] +fn iterator_methods_swap_only_where_the_types_agree() { + let found = mutators("fn f(v: &[u32]) { let _ = v.iter().any(|n| *n > 0); }", "iter"); + + assert!(found.contains(&"iter.any_to_all"), "{found:?}"); + + // `take` and `skip` return different types, so no mutant may be offered for them. This + // would otherwise be generated on every chain in a codebase and withdrawn on every run. + let found = mutators("fn f(v: &[u32], n: usize) { let _ = v.iter().take(n).count(); }", "iter"); + + assert!(!found.contains(&"iter.take_to_skip"), "{found:?}"); +} + +#[test] +fn a_method_rename_needs_the_arity_that_identifies_it() { + // Without type resolution, the count is the only evidence that this `take` belongs to + // `Iterator` rather than to `Option` or `Cell`, where the rename would be nonsense. + let found = mutators( + "fn f(v: &[String], s: &str) { let _ = v.iter().any(|w| w.starts_with(s)); }", + "string", + ); + + assert!(found.contains(&"string.starts_with_to_ends_with"), "{found:?}"); + + let found = mutators("fn f(o: &mut Option) { let _ = o.take(); }", "iter,string"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_vec_literal_offers_each_element_for_omission() { + let found = candidates("fn f() { let _ = vec![1, 2, 3]; }", "collection.omit_element"); + + assert_eq!(found.len(), 3); + + // The replacement must still be an expression, because it becomes one arm of an `if`. + assert!(found.iter().all(|candidate| candidate.replacement.starts_with("vec!")), "{found:?}"); +} + +#[test] +fn an_assignment_offers_a_default_value() { + let found = mutators("fn f(mut n: u32) { n = n + 1; }", "assign_value"); + + assert!(found.contains(&"assign_value.default"), "{found:?}"); +} + +#[test] +fn nested_return_types_recurse_into_their_payloads() { + let source = "fn f() -> Result, String> { compute() }"; + let found = candidates(source, "fn_value"); + let texts: Vec<_> = found.iter().map(|candidate| candidate.replacement.as_str()).collect(); + + assert!(texts.contains(&"Ok(None)"), "{texts:?}"); + assert!(texts.contains(&"Ok(Some(true))"), "{texts:?}"); + assert!(texts.contains(&"Ok(Some(false))"), "{texts:?}"); +} + +#[test] +fn an_impl_iterator_return_is_mutated_through_the_either_wrapper() { + // An `impl Trait` return is one concrete type picked by the body, and a mutant shares an + // `if` with that body, so a bare replacement would be withdrawn after a wasted build. + // `Shape::IterBlock` wraps both arms so they agree on a type, which is what makes these + // mutants viable rather than a waste of a build. + let found = candidates("fn f() -> impl Iterator { core::iter::once(1) }", "fn_value"); + let texts: Vec<_> = found.iter().map(|candidate| candidate.replacement.as_str()).collect(); + + assert!(texts.contains(&"core::iter::empty()"), "{texts:?}"); + assert!(texts.contains(&"core::iter::once(0)"), "{texts:?}"); + + assert!( + found.iter().all(|candidate| candidate.shape == Shape::IterBlock), + "the wrapper is what makes them compile, so every one must ask for it: {found:?}" + ); +} + +#[test] +fn an_impl_iterator_return_without_an_item_type_still_offers_the_empty_case() { + // `empty()` needs no item type, because the wrapper infers it from the arm holding the + // original. `once(v)` needs a value, and there is no type here to name one of. + let found = candidates("fn f() -> impl Iterator { core::iter::once(1) }", "fn_value"); + let texts: Vec<_> = found.iter().map(|candidate| candidate.replacement.as_str()).collect(); + + assert_eq!(texts, vec!["core::iter::empty()"], "{texts:?}"); +} + +#[test] +fn an_impl_trait_return_that_is_not_an_iterator_still_offers_nothing() { + // `Either` only unifies iterators. A future, a closure or a writer has no expression this + // tool can name that is guaranteed to satisfy the signature. + let found = mutators("fn f() -> impl core::future::Future { async { 1 } }", "fn_value"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_vec_element_is_never_offered_when_the_mutator_is_not_selected() { + // `omit_elements` splices a candidate directly rather than going through the usual + // `emit_shaped` helper, so it carries its own `wants` check. Without one, disabling + // `collection.omit_element` would still leave the mutant in the population, and a user who + // turned the family off to shrink a run would get exactly the mutants they asked to skip. + let found = mutators("fn f() { let _ = vec![1, 2, 3]; }", "arith"); + + assert!(!found.contains(&"collection.omit_element"), "{found:?}"); +} + +#[test] +fn a_vec_that_uses_the_repeat_syntax_is_not_a_comma_separated_list() { + // `vec![value; count]` does not parse as `Punctuated`, so the body parse + // fails outright. Treating that as "nothing to offer" rather than propagating the error + // is what lets an ordinary array-style vec sit next to a repeat-style one in the same file + // without the whole file failing to collect. + let found = mutators("fn f() { let _ = vec![0; 5]; }", "collection.omit_element"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_single_element_vec_is_left_alone() { + // Omitting the one element a single-item `vec!` has would leave an empty collection, which + // is a different question — whether the collection is needed at all — and one that + // `Vec::new()` already asks on the function's behalf. Offering it here would be asking the + // same thing twice under two different names. + let found = mutators("fn f() { let _ = vec![1]; }", "collection.omit_element"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_cfg_test_match_arm_is_left_out_of_the_never_matches_family() { + // An arm that only exists under `#[cfg(test)]` is not compiled into the program a normal + // test run exercises, so a guard placed on it would sit on code that was never there and + // could never be activated by any test. + let source = "fn f(x: i32) -> i32 { match x { #[cfg(test)] 1 => 10, _ => 0 } }"; + let found = mutators(source, "match_arm.never_matches"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_struct_field_omission_is_withheld_when_its_mutator_is_not_selected() { + // Like the `vec!` element family, field omission splices its candidate directly and has + // to check the selection itself; skipping that check would ignore a selection that + // deliberately left the mutator out. + let found = mutators("fn f() -> C { C { a: 1, ..Default::default() } }", "arith"); + + assert!(!found.contains(&"struct_field.omit"), "{found:?}"); +} + +#[test] +fn a_struct_literal_with_no_base_offers_no_field_omissions() { + // Omitting a field only leaves a well-formed value when a `..base` supplies whatever was + // taken out. Without one, "omission" would just be a missing field, which is a compile + // error rather than a mutant. + let found = mutators("fn f() -> C { C { a: 1, b: 2 } }", "struct_field.omit"); + + assert!(found.is_empty(), "{found:?}"); +} + +#[test] +fn a_cfg_test_struct_field_is_never_offered_for_omission() { + // A field that only exists under `#[cfg(test)]` is not part of the struct a normal build + // sees, so a mutant that omits it there would be omitting something that was never there. + let source = "fn f() -> C { C { #[cfg(test)] a: 1, b: 2, ..Default::default() } }"; + let found = candidates(source, "struct_field.omit"); + + assert_eq!(found.len(), 1, "{found:?}"); + assert!(!found[0].replacement.contains('b'), "{found:?}"); +} + +#[test] +fn a_cfg_test_impl_block_is_not_mutated() { + // `#[cfg(test)]` on the `impl` itself takes every method in it out of the build, the same + // way the attribute does on a module or a function; the check has to be repeated at the + // `impl` because none of those other sites would have caught it. + let source = "#[cfg(test)] impl S { fn f(&self) -> i32 { 1 + 1 } }"; + + assert!(candidates(source, "arith").is_empty()); +} + +#[test] +fn parenthesised_bindings_still_answer_whether_a_value_is_a_number() { + // Redundant parentheses are common after a refactor or a macro expansion site, and a + // reader would not expect them to change what is offered. Without seeing through them + // here, a parenthesised reference to a `String` parameter would be perturbed as though it + // were a number, offering a mutant that could never compile. + let found = mutators("fn f(name: String) { g((name)); }", "expr"); + + assert!(!found.contains(&"expr.increment"), "{found:?}"); + assert!(!found.contains(&"expr.decrement"), "{found:?}"); +} + +#[test] +fn a_parenthesised_true_is_not_offered_true_again() { + // `if (true)` is the same condition as `if true`, so `cond.always_true` would replace it + // with a copy of itself — a mutant that compiles to the original program and can never be + // caught, forever occupying a line on the report as a survivor nothing could kill. + let found = mutators("fn f() { if (true) { g(); } }", "cond"); + + assert!(!found.contains(&"cond.always_true"), "{found:?}"); + assert!(found.contains(&"cond.always_false"), "{found:?}"); +} + +#[test] +fn a_parenthesised_arithmetic_expression_is_still_perturbable() { + // A caller sometimes parenthesises an argument for its own clarity, and that should not + // hide the arithmetic inside it from the family whose whole purpose is asking whether an + // off-by-one there would be caught. + let found = mutators("fn f(a: i32, b: i32) { g((a + b)); }", "expr"); + + assert!(found.contains(&"expr.increment"), "{found:?}"); +} + +/// A screaming-case name whose declaration is not a number is not perturbed. +/// +/// The screaming case is read as evidence of a constant, and a constant is one of the best +/// things this family has to offer — but the spelling says nothing about the type, and this +/// codebase alone is full of `const PREFIX: &str`. Adding one to those is `E0369` every time: a +/// mutant that cannot compile, and a share of a rollback round spent finding that out. +#[test] +fn a_constant_the_file_declares_as_text_is_not_perturbed() { + let found = mutators(r#"const PREFIX: &str = "x"; fn f() { g(PREFIX); }"#, "expr"); + + assert!(!found.contains(&"expr.increment"), "{found:?}"); +} + +/// The declaration is read in both directions, so a quiet name that is a number still perturbs. +#[test] +fn a_constant_the_file_declares_as_a_number_is_still_perturbed() { + let found = mutators("const LIMIT: usize = 4; fn f() { g(LIMIT); }", "expr"); + + assert!(found.contains(&"expr.increment"), "{found:?}"); +} + +/// An associated constant is declared just as plainly as a free one. +#[test] +fn an_associated_constant_declared_as_text_is_not_perturbed() { + let found = mutators( + r#"struct S; impl S { const NAME: &'static str = "s"; } fn f() { g(S::NAME); }"#, + "expr", + ); + + assert!(!found.contains(&"expr.increment"), "{found:?}"); +} + +/// Two declarations disagreeing about a name leave it unknown rather than letting one win. +/// +/// Without type resolution there is no way to say which `MAX` a bare `MAX` reached, so the +/// screaming-case guess would be as likely to be wrong as right — and being wrong here costs a +/// build. Neither answer is taken, which for this family means no perturbation. +#[test] +fn a_constant_two_declarations_disagree_about_is_not_perturbed() { + let source = r#"struct A; struct B; impl A { const CAP: usize = 1; } impl B { const CAP: &'static str = "b"; } fn f() { g(CAP); }"#; + let found = mutators(source, "expr"); + + assert!(!found.contains(&"expr.increment"), "{found:?}"); +} + +/// Text the source builds on the spot is never perturbed, whatever else looked numeric. +/// +/// `format!` has one result type, and the sum of a `String` and a `&str` is a `String` however +/// numeric either side looked. Both are decidable from the syntax alone, which is what makes +/// them worth refusing rather than paying a build to discover. +#[test] +fn text_the_source_builds_on_the_spot_is_not_perturbed() { + for source in [ + r#"fn f(n: usize) { g(format!("{n}")); }"#, + "fn f(n: usize) { g(n.to_string()); }", + r#"fn f() { g("a".to_owned()); }"#, + r#"const NAME: &str = "n"; fn f(s: String) { g(s + NAME); }"#, + ] { + let found = mutators(source, "expr"); + + assert!(!found.contains(&"expr.increment"), "{source}: {found:?}"); + assert!(!found.contains(&"expr.decrement"), "{source}: {found:?}"); + } +} + +#[test] +fn a_parenthesised_callee_is_still_recognised_by_its_type() { + // `(Vec::new)()` calls a parenthesised path, which is unusual but legal Rust, and the + // qualifying type still has to be read through the parentheses. Missing it here would + // offer `+ 1` on a `Vec`, a mutant that can never compile. + let found = mutators("fn f() { g((Vec::new)()); }", "expr"); + + assert!(!found.contains(&"expr.increment"), "{found:?}"); +} + +#[test] +fn an_assignment_already_holding_a_default_is_left_alone() { + // Replacing `Default::default()` with `Default::default()` reproduces the original + // program exactly, so the mutant could never be caught and would sit in every report as a + // permanent, uninformative survivor. + let found = mutators("fn f(mut n: i32) { n = Default::default(); }", "assign_value"); + + assert!(!found.contains(&"assign_value.default"), "{found:?}"); +} + +#[test] +fn a_parenthesised_default_call_is_still_recognised() { + let found = mutators("fn f(mut n: i32) { n = (Default::default()); }", "assign_value"); + + assert!(!found.contains(&"assign_value.default"), "{found:?}"); +} + +#[test] +fn a_trait_default_bound_suppresses_an_identical_assignment_mutant() { + let source = "trait Builder { fn build(mut value: T) { value = T::default(); } }"; + let found = mutators(source, "assign_value"); + + assert!(!found.contains(&"assign_value.default"), "{found:?}"); +} + +#[test] +fn a_bare_call_to_a_capacity_named_function_is_not_perturbed() { + // `with_capacity(5)` names an allocation strategy rather than a behavior even when it is + // not qualified by the type it belongs to, and perturbing its result would ask a question + // about a performance decision rather than about the program. + let found = mutators("fn f() { h(with_capacity(5)); }", "expr"); + + assert!(!found.contains(&"expr.increment"), "{found:?}"); +} + +#[test] +fn a_parenthesised_type_the_caller_chooses_is_still_treated_as_abstract() { + // `D::Error` wrapped in redundant parentheses is exactly as unconstrained as the bare + // form, and a rule that stopped recognising it once parenthesised would start inventing + // `Default::default()` values for a type nothing promises has one. + let source = "fn f(d: D) -> Result { g(d); Ok(1) }"; + let found = candidates(source, "fn_value"); + + assert!( + !found.iter().any(|c| c.replacement.contains("Err(Default::default())")), + "{found:?}" + ); +} + +#[test] +fn a_parenthesised_numeric_type_annotation_still_marks_a_binding_numeric() { + // `let x: (i32) = 1;` writes down a numeric type just as plainly as `let x: i32 = 1;` + // does, and a reader would not expect the redundant parentheses to hide that fact from the + // family that decides whether a bare identifier is worth perturbing. + let found = mutators("fn f() { let x: (i32) = 1; g(x); }", "expr"); + + assert!(found.contains(&"expr.increment"), "{found:?}"); +} + +#[test] +fn a_parenthesised_callee_path_still_names_the_variant_it_constructs() { + // `(Some)(1)` calls a parenthesised path, which is legal Rust and behaves exactly like + // `Some(1)`; missing the wrapped path here would quietly drop `option.some_to_none` for + // every call written this way. + let found = mutators("fn f() { let _ = (Some)(1); }", "option"); + + assert!(found.contains(&"option.some_to_none"), "{found:?}"); +} + +#[test] +fn a_lifetime_generic_parameter_never_needs_a_default_bound() { + // Only a type parameter can carry a `Default` bound in the first place, so a lifetime + // sitting beside one in the same parameter list has to be skipped rather than treated as + // an undefaulted type parameter, or it would be reported as a type this tool cannot + // assume a `Default` for. + let source = "fn f<'a, D: Reader>(d: &'a D) -> Result { g(d); Ok(1) }"; + let found = candidates(source, "fn_value"); + + assert!( + !found.iter().any(|c| c.replacement.contains("Err(Default::default())")), + "{found:?}" + ); +} + +/// Without type resolution the classification is by name, so a locally defined type spelled +/// `Vec` cannot be told apart by its name alone. It is told apart by its shape: the standard +/// `Vec` carries an element type and this one carries nothing, so it is not read as the +/// standard one and the family falls back to the guess that fits any type. +#[test] +fn a_local_type_that_shares_a_collection_name_but_has_no_generics_falls_back_to_default() { + let source = "struct Vec; fn f() -> Vec { Vec }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().all(|c| c.replacement == "Default::default()"), "{found:?}"); +} + +#[test] +fn a_bare_box_with_no_generic_argument_is_not_treated_as_unconstructable() { + // `Box` is only as unconstructable as whatever it wraps, and asking what it wraps when + // nothing was written down should be answered with "nothing", not with a guess. A `Box` + // spelled without its argument at all — a locally shadowed name, since the real type + // always requires one — must not be mistaken for `Box`, which is why it holds + // the `err_default` mutant rather than losing it. + let source = "struct Box; fn f() -> Result { g() }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().any(|c| c.replacement == "Err(Default::default())"), "{found:?}"); +} + +#[test] +fn a_return_type_nested_deeper_than_the_recursion_bound_falls_back_to_default() { + // The recursion is bounded so that a deeply nested return type costs a constant number of + // mutants rather than one for every level of nesting; past that bound, the family still + // has to offer something, and `Default::default()` is the one expression that type-checks + // regardless of how deep the type turned out to be. + let source = "fn f() -> Option>>> { None }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().any(|c| c.replacement.contains("Default::default()")), "{found:?}"); +} + +#[test] +fn an_option_whose_payload_is_a_type_the_caller_chooses_offers_only_none() { + // Just as with `Result`, an `Option` cannot be given a `Some(..)` mutant without + // guessing at a `Default` nothing promises exists, so the abstract payload has to leave + // the family with nothing to offer beyond the value it can always name: absence itself. + let source = "fn f(d: D) -> Option { g(d) }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().any(|c| c.replacement == "None"), "{found:?}"); + assert!(!found.iter().any(|c| c.replacement.contains("Some")), "{found:?}"); +} + +#[test] +fn a_result_whose_success_payload_is_abstract_offers_no_ok_mutant() { + // The mirror image of the caller-chosen-error case: when it is the success side that + // cannot be given a `Default`, the family still owes the reader the error mutant it can + // name, but must not fabricate a value for the side it cannot. + let source = "fn f(d: D) -> Result { g(d) }"; + let found = candidates(source, "fn_value"); + + assert!(!found.iter().any(|c| c.replacement.starts_with("Ok(")), "{found:?}"); + assert!(found.iter().any(|c| c.replacement == "Err(Default::default())"), "{found:?}"); +} + +#[test] +fn a_wide_tuple_return_is_capped_rather_than_multiplied_out_in_full() { + // Every element of a tuple return multiplies the count of mutants by its own number of + // values, so a tuple of several booleans would otherwise cost dozens of mutants for one + // function. The cap exists precisely to stop that multiplication, and this is the fixture + // that forces the running total past it mid-build. + let source = "fn f() -> (bool, bool, bool, bool, bool) { (true, true, true, true, true) }"; + let found = candidates(source, "fn_value.tuple"); + + assert!(found.len() <= 8, "{found:?}"); + assert!(!found.is_empty(), "{found:?}"); +} + +#[test] +fn a_destructured_parameter_contributes_no_binding_to_perturb() { + // A tuple-pattern parameter never binds a plain identifier the way `a: i32` does, so there + // is no name this family could look up later when deciding whether some later expression + // is the numeric parameter it appears to be. Recording nothing for it, rather than + // guessing at one of its parts, is what keeps that later lookup honest. + let source = "fn f((a, b): (i32, i32)) -> i32 { let a = a; a + b }"; + let found = mutators(source, "expr"); + + assert!(found.contains(&"expr.increment"), "{found:?}"); +} + +#[test] +fn a_parenthesised_return_type_is_still_classified_by_the_type_it_wraps() { + // Redundant parentheses around a return type are legal and occasionally left behind by a + // refactor; a reader would expect `(Vec)` to offer exactly what `Vec` offers; if + // the parentheses hid the type from classification, the whole `fn_value` family would fall + // silent for a function that has plenty to offer. + let found = mutators("fn f() -> (Vec) { make() }", "fn_value"); + + assert!(found.contains(&"fn_value.empty_collection"), "{found:?}"); +} + +#[test] +fn a_lifetime_argument_ahead_of_a_result_payload_is_skipped_rather_than_counted() { + // `Result`'s two payload types are found by position among a path's generic arguments, but + // a written-out lifetime shares that argument list. Counting it as though it were a type + // would shift every position after it by one and hand the wrong payload to the wrong side + // of the `Result`. + let found = candidates("fn f() -> Result<'static, u8> { Ok(1) }", "fn_value.ok"); + + assert!(found.iter().any(|c| c.replacement == "Ok(0)"), "{found:?}"); +} + +/// A body that already spells one of the values the family offers would be replaced by itself. +/// The compiled program is identical, so no test can tell the mutant from the original and it +/// survives every suite that will ever exist — a permanent accusation against tests that had +/// nothing to answer. The sibling values are still worth offering, so only the duplicate goes. +#[test] +fn a_replacement_identical_to_the_body_it_replaces_is_not_offered() { + let found = candidates("fn f() -> bool { true }", "fn_value"); + + assert!( + found.iter().all(|c| c.replacement != "true"), + "a body of `true` must not be replaced by `true`: {found:?}" + ); + assert!( + found.iter().any(|c| c.replacement == "false"), + "the other value must survive: {found:?}" + ); +} + +/// Layout must not decide the answer, since the question is whether the compiled program +/// changes. A body spread over several lines is the same program as one on a single line, and +/// comparing the text rather than the tokens would call the second a no-op and the first a +/// mutant. +#[test] +fn the_no_op_test_reads_tokens_rather_than_layout() { + let found = candidates("fn f() -> Option {\n // a note\n None\n}", "fn_value.none"); + + assert!(found.is_empty(), "{found:?}"); +} + +/// A body doing more than producing the value is not reproduced by that value alone, however +/// the two end. Dropping the statement is exactly what the mutant is asking about, so this one +/// must be kept. +#[test] +fn a_body_ending_in_the_offered_value_is_still_mutated() { + let found = candidates("fn f(c: &Cell) -> bool { c.set(1); true }", "fn_value.bool_true"); + + assert!(found.iter().any(|c| c.replacement == "true"), "{found:?}"); +} + +/// `Default::default()` inside `impl Default` names the function it is replacing, so the +/// mutant is unbounded recursion. It cannot be killed by a test seeing a wrong value, only by +/// the stack running out, which costs a full timeout to reach and is the slowest verdict there +/// is. Mutants elsewhere in the same body are ordinary and must be kept — which is why the +/// whole `impl` is not simply skipped. +#[test] +fn the_default_method_of_a_default_impl_is_not_replaced_by_a_call_to_itself() { + let source = "impl Default for Thing { fn default() -> Self { Thing { n: 7 } } }"; + let found = candidates(source, "all"); + + assert!( + found.iter().all(|c| c.replacement != "Default::default()"), + "`default` must not be replaced by a call to itself: {found:?}" + ); + assert!( + found.iter().any(|c| c.mutator == "literal.int_increment"), + "other mutants in the same body must be kept: {found:?}" + ); +} + +/// The rule is about the trait, not the name. An inherent `default` does not shadow +/// `Default::default`, so replacing its body with one is an ordinary mutant, and a differently +/// named method inside `impl Default` is not what `Default::default()` resolves to either. +#[test] +fn only_the_default_method_of_a_default_impl_loses_that_replacement() { + let inherent = candidates("impl Thing { fn default() -> Self { Thing { n: 7 } } }", "fn_value.default"); + + assert!(!inherent.is_empty(), "an inherent `default` is not the trait method: {inherent:?}"); + + let sibling = candidates( + "impl Default for Thing { fn helper() -> Other { Other { n: 7 } } }", + "fn_value.default", + ); + + assert!(!sibling.is_empty(), "a sibling method is not the trait method: {sibling:?}"); +} + +#[test] +fn a_concrete_self_without_default_is_not_replaced_with_default() { + let source = "struct Thing; impl Thing { fn make() -> Self { Self } }"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("fn_value.default").unwrap(); + let defaults = Defaults::of(&file.ast); + let found = collect_with(&file, &selection, &CfgSet::unconditional(), &defaults); + + assert!(found.is_empty(), "`Thing` does not implement `Default`: {found:?}"); +} + +#[test] +fn a_concrete_self_associated_type_without_default_is_not_defaulted() { + let source = " + struct Error; + struct Thing; + trait Make { type Err; fn make() -> Result; } + impl Make for Thing { + type Err = Error; + fn make() -> Result { unreachable!() } + } + "; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("fn_value.err_default").unwrap(); + let defaults = Defaults::of(&file.ast); + let found = collect_with(&file, &selection, &CfgSet::unconditional(), &defaults); + + assert!(found.is_empty(), "`Self::Err` resolves to non-defaultable `Error`: {found:?}"); +} + +#[test] +fn the_standard_fmt_result_alias_gets_a_compiling_result_value() { + let found = candidates( + "use std::fmt::{self, Display}; impl Display for Thing { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(\"x\") } }", + "fn_value", + ); + + assert!( + found.iter().any(|candidate| candidate.replacement == "Ok(Default::default())"), + "`fmt::Result` should be recognized as `Result<(), fmt::Error>`: {found:?}" + ); + assert!( + found.iter().all(|candidate| candidate.replacement != "Default::default()"), + "`Result` itself does not implement `Default`: {found:?}" + ); +} + +#[test] +fn standard_time_types_are_not_replaced_with_default() { + for source in [ + "use std::time::Instant; fn now() -> Instant { Instant::now() }", + "fn epoch() -> std::time::SystemTime { std::time::UNIX_EPOCH }", + ] { + let found = candidates(source, "fn_value.default"); + + assert!(found.is_empty(), "the standard time type has no `Default`: {found:?}"); + } +} + +/// A reference cannot point at a temporary, so `&Default::default()` would not compile and the +/// family would otherwise pass over every reference-returning function in silence. Leaking a box gives +/// a `&'static mut T`, which lives long enough for any signature, and the values are the +/// element type's own. +/// +/// A shared reference is reborrowed rather than left to coerce. Coercion is enough in a return +/// position, but not where the value is what a type is *inferred* from — an +/// `impl Iterator` would infer `Once<&mut T>` and be withdrawn as unviable. +#[test] +fn a_shared_reference_return_is_served_by_reborrowing_a_leaked_box() { + let found = candidates("fn f(v: &Vec) -> &String { &v[0] }", "fn_value"); + + assert!( + found.iter().any(|c| c.replacement == "&*Box::leak(Box::new(String::new()))"), + "{found:?}" + ); +} + +/// A mutable reference is not reborrowed, because `Box::leak` already yields exactly that. +#[test] +fn a_mutable_reference_return_is_served_by_the_leak_alone() { + let found = candidates("fn f(v: &mut Vec) -> &mut u8 { &mut v[0] }", "fn_value"); + + assert!(found.iter().any(|c| c.replacement == "Box::leak(Box::new(1))"), "{found:?}"); +} + +/// A reference to something abstract still yields nothing. `Box::new` needs a value of the +/// pointee type, and a trait object names a capability rather than a type, so there is none to +/// make — the leak does not rescue this case and must not be offered for it. +#[test] +fn a_reference_to_a_trait_object_is_still_left_alone() { + let found = candidates("fn f(&self) -> &dyn Debug { &self.inner }", "fn_value"); + + assert!(found.is_empty(), "{found:?}"); +} + +/// Only the last segment of a path is compared against the standard names, so a local type is +/// told apart by the type arguments it carries. `mine::Vec` takes none where the standard `Vec` +/// takes one, and calling `mine::Vec::new()` would not compile; the generic guess is offered +/// instead, which at least stands a chance. +#[test] +fn a_local_type_wearing_a_standard_name_is_not_treated_as_the_standard_one() { + let source = "mod mine { pub struct Vec { pub n: i32 } } fn f() -> mine::Vec { mine::Vec { n: 4 } }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().all(|c| c.replacement == "Default::default()"), "{found:?}"); +} + +/// The same rule applies to `Cow`, where the trap is sharper: a replacement that hard-codes +/// `std::borrow::Cow` names a different type from the one being returned. +#[test] +fn a_local_cow_is_not_given_the_standard_cow_constructor() { + let source = "mod mine { pub struct Cow { pub n: i32 } } fn f() -> mine::Cow { mine::Cow { n: 5 } }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().all(|c| !c.replacement.contains("std::borrow")), "{found:?}"); +} + +/// Qualification must keep working, since that is why only the last segment is compared. A +/// fully written `std::vec::Vec` is the standard one and keeps every value it had. +#[test] +fn a_fully_qualified_standard_type_is_still_recognised() { + let source = "fn f() -> std::vec::Vec { std::vec::Vec::from([1]) }"; + let found = candidates(source, "fn_value"); + + assert!(found.iter().any(|c| c.replacement == "std::vec::Vec::new()"), "{found:?}"); +} + +/// `type Result = core::result::Result` is everywhere in real crates. The error +/// is whatever the alias fixed it to, which is almost never something with a `Default`, so +/// guessing at it produced a mutant that did not compile. The `Ok` values are unaffected, +/// because those come from the argument the alias does spell. +#[test] +fn an_aliased_result_is_not_given_an_error_it_cannot_name() { + let found = candidates("fn f() -> Result { g() }", "fn_value"); + + assert!(found.iter().all(|c| c.mutator != "fn_value.err_default"), "{found:?}"); + assert!(found.iter().any(|c| c.replacement == "Ok(0)"), "{found:?}"); +} + +/// The error value is still offered when the path names the error, which is the case the rule +/// above must not disturb. +#[test] +fn a_result_that_names_its_error_still_offers_one() { + let found = candidates("fn f() -> Result { g() }", "fn_value"); + + assert!(found.iter().any(|c| c.mutator == "fn_value.err_default"), "{found:?}"); +} + +/// A lifetime is not a type argument, so `Cow<'a, str>` names one type and must still be +/// recognised despite carrying two arguments in the source. +#[test] +fn a_lifetime_does_not_count_towards_the_type_arguments() { + let found = candidates("fn f() -> Cow<'static, str> { borrow() }", "fn_value"); + + assert!(found.iter().any(|c| c.replacement.contains("Cow::Owned")), "{found:?}"); +} + +fn span_of(text: &str, needle: &str) -> core::ops::Range { + let start = text.find(needle).expect("the needle must be in the text"); + + start..start + needle.len() +} + +/// A selection that reads none of the prepass indexes must collect exactly what a full one does. +/// +/// The whole-file prepass is skipped when no selected mutator consults it, and skipping work is +/// only safe if it is invisible: the candidates a narrow selection yields have to be the ones a +/// selection that *does* build the indexes yields for those same mutators. When the gate was +/// first written it missed `result.ok_to_err`, which reads the import index to decide whether an +/// `Err` is constructible, and five mutants changed — a gate on an index is wrong exactly when +/// its output stops matching the ungated walk. +#[test] +fn a_selection_that_needs_no_prepass_collects_what_a_full_one_does() { + let source = "\ +use std::io::Error; +const CAP: usize = 8; +struct Held { count: usize } +fn f(held: &Held) -> Result { +for i in 0..CAP { + if held.count < i && i > 1 { + return Ok(i); + } +} +Ok(0) +} +"; + + let shape = |found: Vec| -> Vec { + found + .into_iter() + .filter(|c| c.mutator.starts_with("relational.")) + .map(|c| format!("{c:?}")) + .collect() + }; + + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let narrow = shape(collect(&file, &Selection::parse("relational").unwrap())); + let full = shape(collect(&file, &Selection::everything())); + + assert!( + !narrow.is_empty(), + "the fixture must offer relational sites for this to prove anything" + ); + assert_eq!(narrow, full); +} +/// A file that is not plain ASCII survives collection with its spans, text and columns intact. +/// +/// Every other fixture in this workspace is an ASCII string literal with Unix line endings, +/// and every stage downstream of the parser is byte-offset arithmetic. A byte-offset slip on a +/// multi-byte identifier does not fail loudly — it records the wrong `original`, reports the +/// wrong column, and splices the guard through the middle of a character, which is a build +/// failure blamed on the tree rather than on the tool. Real repositories contain all three of +/// these: a byte-order mark, `\r\n` terminators and non-ASCII identifiers and literals. +/// +/// The oracle is deliberately not `file.slice`, which is the same arithmetic the recording +/// used. It is `find` over the text, which knows nothing about spans. +#[test] +fn a_file_with_a_byte_order_mark_crlf_endings_and_multibyte_text_keeps_its_spans() { + let source = "\u{feff}fn tälle(gröÿe: usize) -> bool {\r\n let ändern = \"ünïcøde\";\r\n\r\n gröÿe < ändern.len()\r\n}\r\n"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let found = into_definitions(&file, collect(&file, &Selection::parse("relational").unwrap())); + + assert!(!found.is_empty(), "nothing was collected from a non-ASCII file"); + + assert!( + !file.text.starts_with('\u{feff}'), + "the mark has to be gone, or every span below is three bytes out" + ); + + for mutant in &found { + assert_eq!( + file.text.get(mutant.site.span.clone()), + Some(mutant.site.original.as_str()), + "the recorded original is not the text at the span" + ); + + let line = file.text.lines().nth(mutant.site.line - 1).expect("the line exists"); + + assert_eq!( + line.chars().nth(mutant.site.column - 1), + mutant.site.original.chars().next(), + "the column does not point at the start of `{}` on line {}", + mutant.site.original, + mutant.site.line + ); + } + + // The site itself, found without going anywhere near a span. + let comparison = found + .iter() + .find(|mutant| mutant.site.original == "gröÿe < ändern.len()") + .expect("the comparison over multi-byte operands was collected"); + + assert_eq!(comparison.site.line, 4, "the CRLF terminators shifted the line number"); + assert_eq!(comparison.site.column, 5, "the multi-byte indent shifted the column"); +} + +/// The same file instruments into something that still parses and still says what it said. +/// +/// Collection recording the right bytes is half of it; the splice has to put them back. A cut +/// on a byte that is not a character boundary produces invalid UTF-8 in the middle of a source +/// file, and a splice that is merely off by a byte silently changes an identifier. +#[test] +fn a_non_ascii_file_instruments_into_source_that_still_parses() { + let source = "\u{feff}fn tälle(gröÿe: usize) -> bool {\r\n let ändern = \"ünïcøde\";\r\n\r\n gröÿe < ändern.len()\r\n}\r\n"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let found = into_definitions(&file, collect(&file, &Selection::parse("relational").unwrap())); + let mutations: Vec<_> = found + .iter() + .enumerate() + .map(|(ordinal, mutant)| crate::schema::AssignedMutant::new(u32::try_from(ordinal).unwrap(), mutant)) + .collect(); + let instrumented = crate::schema::instrument(&file.text, &mutations).expect("instruments"); + + let _ = syn::parse_file(&instrumented).expect("the instrumented file no longer parses"); + + for identifier in ["tälle", "gröÿe", "ändern", "ünïcøde"] { + assert!( + instrumented.contains(identifier), + "`{identifier}` did not survive instrumentation:\n{instrumented}" + ); + } +} + +// ---- Values the site states for itself. -------------------------------------------------- + +/// Every replacement collected under the `fn_value` family, in emission order. +fn fn_values(source: &str) -> Vec<(&'static str, String)> { + candidates(source, "fn_value") + .into_iter() + .map(|c| (c.mutator, c.replacement.to_string())) + .collect() +} + +/// A stated value replaces what the return type made the tool guess. +/// +/// The point of the attribute: the author knows a value that is wrong in a way their suite should +/// notice, and `false`, `0` or `String::new()` was not it. Emitting the guess as well would ask +/// the same question twice at one site and charge the suite for failing to answer the version its +/// author had already rejected. +/// +/// The last two are the guess that most often turns out not to compile: behind an alias or an +/// associated type there is a concrete type the collector cannot see, and `Default::default()` is +/// a hope that it implements `Default`. A stated value is how that hope becomes knowledge. +#[test] +fn a_stated_value_displaces_the_guessed_ones() { + for (source, stated, guessed) in [ + ("fn f() -> bool { g() }", "#[gamma::value(h())] fn f() -> bool { g() }", "false"), + ("fn f() -> i32 { g() }", "#[gamma::value(h())] fn f() -> i32 { g() }", "0"), + ( + "fn f() -> String { g() }", + "#[gamma::value(h())] fn f() -> String { g() }", + "String::new()", + ), + ( + "fn f() -> MyAlias { g() }", + "#[gamma::value(h())] fn f() -> MyAlias { g() }", + "Default::default()", + ), + ( + "impl Trait for S { type Item = u8; fn f(&self) -> Self::Item { g() } }", + "impl Trait for S { type Item = u8; #[gamma::value(h())] fn f(&self) -> Self::Item { g() } }", + "Default::default()", + ), + ] { + assert!( + fn_values(source).iter().any(|(_, value)| value == guessed), + "the premise is that `{guessed}` is guessed" + ); + assert_eq!(fn_values(stated), vec![("fn_value.stated", "h()".to_owned())], "for `{source}`"); + } +} + +/// The shapes whose types the tool refuses to guess a value for, each with a value stated. +/// +/// A type parameter, an associated type reached through one, a trait object and an opaque return +/// are all types the collector cannot see a constructor behind — it parses, it does not resolve — +/// so each one silently costs a site its `fn_value` mutant. The attribute is the way back: the +/// author can see the constructor, and says it. +const WITHHELD: &[(&str, &str)] = &[ + ("fn f(t: T) -> T { g(t) }", "t"), + ("fn f(i: I) -> I::Item { g(i) }", "i.next().unwrap()"), + ("fn f() -> Box { g() }", "Box::new(Empty)"), + ("fn f() -> impl Reader { g() }", "Empty"), +]; + +/// A site the tool declines to guess for gets a mutant once the value is stated. +/// +/// Half of F4's reason to exist: these sites are not hard to mutate, they are hard to *guess* a +/// mutant for, and the difference showed up as a hole in the population rather than as a question. +#[test] +fn a_stated_value_adds_a_mutant_where_none_was_guessed() { + for (source, expression) in WITHHELD { + assert!( + fn_values(source).is_empty(), + "the premise is that `{source}` yields no guessed value" + ); + + let stated = format!("#[gamma::value({expression})]\n{source}"); + + assert_eq!( + fn_values(&stated), + vec![("fn_value.stated", (*expression).to_owned())], + "for `{source}`" + ); + } +} + +/// An `impl Iterator` return keeps the splice its type needs when the value is stated. +/// +/// Both arms of the guard have to be wrapped so they share one type, and that is decided by the +/// signature rather than by where the value came from. Emitting a stated value as a plain block +/// would produce a mutant that cannot compile for a reason the author had no way to see. +#[test] +fn a_stated_value_on_an_iterator_return_keeps_the_iterator_shape() { + let found = candidates( + "#[gamma::value(core::iter::empty())]\nfn f() -> impl Iterator { g() }", + "fn_value", + ); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].shape, Shape::IterBlock); +} + +/// The expression reaches the mutant exactly as written. +/// +/// It is shown in reports and spliced into the file, so a re-rendering that turns `Config { n: 1 }` +/// into `Config { n : 1 }` puts text in the author's diff that the author did not write. +#[test] +fn the_stated_expression_reaches_the_mutant_verbatim() { + let found = fn_values("#[gamma::value(Some(Config { n: 1 }))]\nfn f() -> Option { g() }"); + + assert_eq!(found, vec![("fn_value.stated", "Some(Config { n: 1 })".to_owned())]); +} + +/// A stated value that cannot possibly type-check is still collected. +/// +/// F4's rule about trust: the tool does not type-check, so it cannot tell a wrong value from a +/// right one, and guessing would mean quietly dropping the mutants of authors who were right. +/// A wrong one becomes a mutant that fails to build and is withdrawn by the same rollback that +/// withdraws every other unviable mutant, which is a reported outcome rather than a silence. +#[test] +fn a_stated_value_of_the_wrong_type_is_collected_rather_than_dropped() { + let found = fn_values("#[gamma::value(\"not a number\")]\nfn f() -> i32 { g() }"); + + assert_eq!(found, vec![("fn_value.stated", "\"not a number\"".to_owned())]); +} + +/// Stating a value adds a mutant; it never takes one away. +/// +/// The other half of F4's rule: the attribute is not a suppression channel. If stating a value +/// could remove mutants, an author could quietly delete the site's other questions by answering +/// one of them, and the mutation score would rise for a reason nobody reviewed. +#[test] +fn stating_a_value_leaves_every_other_family_alone() { + let body = "fn f(a: i32, b: i32) -> i32 { if a < b { a + 1 } else { b * 2 } }"; + let plain = mutators(body, "all"); + let stated = mutators(&format!("#[gamma::value(41)]\n{body}"), "all"); + + for mutator in &plain { + if mutator.starts_with("fn_value") { + continue; + } + + assert_eq!( + plain.iter().filter(|m| *m == mutator).count(), + stated.iter().filter(|m| *m == mutator).count(), + "`{mutator}` changed when a value was stated" + ); + } +} + +/// The named error values keep the replacement indices they had before. +/// +/// Their indices continue the guessed list's, so if a stated value shortened that list the +/// `--error` mutants at every annotated site would be renumbered, and a renumbered mutant is a +/// new id: suppressions by id stop matching and an incremental run re-runs work it had settled. +#[test] +fn a_stated_value_does_not_renumber_the_named_error_mutants() { + let plain = with_errors("fn f() -> Result { Ok(1) }", &["MyError::Io"]); + let stated = with_errors("#[gamma::value(Ok(7))]\nfn f() -> Result { Ok(1) }", &["MyError::Io"]); + + let indices = |found: &[Candidate]| -> Vec { + found + .iter() + .filter(|c| c.mutator == "fn_value.err_with") + .map(|c| c.replacement_index) + .collect() + }; + + assert_eq!(indices(&plain), vec![4], "the premise is that the errors follow the guessed values"); + assert_eq!(indices(&stated), indices(&plain)); +} + +/// A stated mutant does not inherit the identity of the guess it displaced. +/// +/// Identity is a hash of the mutator name and the site, not of the replacement text, so reusing +/// `fn_value.default` for a stated value would give the two the same id — and a cached +/// `CompileError` against the guess would then withhold the very mutant the author wrote the +/// attribute to obtain, without a word about why. +#[test] +fn a_stated_mutant_has_an_identity_of_its_own() { + let identify = |source: &str| -> String { + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let found = collect(&file, &Selection::parse("fn_value").unwrap()); + let mutants = into_definitions(&file, found); + + mutants + .into_iter() + .map(|m| m.id.to_string()) + .next() + .expect("the fixture yields a mutant") + }; + + let guessed = identify("fn f() -> bool { g() }"); + let stated = identify("#[gamma::value(true)]\nfn f() -> bool { g() }"); + + assert_ne!(guessed, stated); +} + +/// A file with no attribute remains pinned to the site-counted identity baseline. +/// +/// The baseline includes the corrected rule that replacements at one site share an occurrence. +/// Once established, these ids must remain stable across runs because stored verdicts and +/// suppressions name mutants by id. +#[test] +fn a_file_that_states_nothing_uses_the_site_counted_identity_baseline() { + let source = "fn a() -> bool { g() }\nfn b() -> Result { Ok(1) }\nfn c(x: i32) -> i32 { x + 1 }\n"; + let file = SourceFile::parse("test.rs", source.to_owned()).unwrap(); + let found = collect(&file, &Selection::parse("all").unwrap()); + let ids: Vec = into_definitions(&file, found).into_iter().map(|m| m.id.to_string()).collect(); + + assert_eq!( + ids, + vec![ + "2b643388f655", + "6ecb982becec", + "c0fc1efe0f11", + "3c5e85253765", + "0a42ad003f48", + "496b6928efc8", + "2c30ededa685", + "6b3d0f1f4c8a", + "7db3aae5c3f8", + "14e9e208d250", + "1a5c3cad1dc5", + "e7e1046f74d6", + "bee0a88bd144", + "3e4f9ce3258f", + "03ddbb807ce2", + "680f8f10ad99", + "1ad3b0366e51" + ], + "collected ids moved for a file that states no values" + ); +} + +/// Methods and trait methods are annotated the same way free functions are. +/// +/// The three positions are three different `syn` nodes and three different call sites in the +/// collector, so "it works on a function" says nothing about the two places most trait-shaped +/// code actually lives. +#[test] +fn a_value_can_be_stated_on_a_method_or_a_trait_method() { + let sources = [ + "impl S { #[gamma::value(9)] fn f(&self) -> u8 { g() } }", + "trait T { #[gamma::value(9)] fn f(&self) -> u8 { g() } }", + ]; + + for source in sources { + assert_eq!(fn_values(source), vec![("fn_value.stated", "9".to_owned())], "for `{source}`"); + } +} + +/// A stated value obeys the selection like every other mutator. +/// +/// The attribute states which expression to substitute, not that the substitution must happen: +/// a run narrowed to arithmetic asked for arithmetic, and a mutator that ignored the selection +/// because a source file asked nicely would make `--mutators` mean something different per file. +#[test] +fn a_stated_value_is_still_subject_to_the_selection() { + let source = "#[gamma::value(9)]\nfn f(a: u8) -> u8 { a + 1 }"; + + assert!( + mutators(source, "arith").iter().all(|m| *m != "fn_value.stated"), + "{:?}", + mutators(source, "arith") + ); + assert!(mutators(source, "fn_value.stated").contains(&"fn_value.stated")); + + // The other half of the rule, and the one the attribute must never break: a selection holding + // a sibling `fn_value` mutator but not the stated one still gets that sibling's mutant. The + // attribute says which expression a site substitutes; a narrowing flag must not turn it into a + // way of taking the site's only `fn_value` mutant away, because that shrinks the denominator + // and raises the score with nothing said about it anywhere. + let narrowed = mutators(source, "fn_value.zero"); + + assert_eq!(narrowed, vec!["fn_value.zero"], "{narrowed:?}"); +} + +/// A stated value on a `const fn` produces nothing, as every function value does there. +/// +/// The guard around the body cannot be called in a const context, so the mutant could not be +/// built whatever its value is. Honouring the attribute here would emit a mutant that fails to +/// compile for a reason that has nothing to do with what the author stated. +#[test] +fn a_stated_value_on_a_const_fn_produces_nothing() { + assert!(fn_values("#[gamma::value(9)]\nconst fn f() -> u8 { g() }").is_empty()); +} + +// ---- Every catalog entry must actually fire. --------------------------------------------- +/// A composite fixture holding at least one construct for every mutator family in the registry. +/// +/// One function per shape rather than one dense function, so that a mutator that fails to fire +/// can be diagnosed by reading the single function that was supposed to feed it. Undefined +/// callees and types are deliberate: collection only parses, it never type-checks, so the +/// fixture can name the exact return shapes the `fn_value` family keys on without carrying the +/// machinery to make them real. +const EVERY_FAMILY_FIXTURE: &str = r#" +fn returns_unit() { do_work(); } +fn returns_bool() -> bool { compute() } +fn returns_signed() -> i32 { compute() } +fn returns_unsigned() -> usize { compute() } +fn returns_float() -> f64 { compute() } +fn returns_nonzero() -> core::num::NonZeroU32 { compute() } +fn returns_string() -> String { compute() } +fn returns_static_str() -> &'static str { compute() } +fn returns_option() -> Option { compute() } +fn returns_option_ref() -> Option<&'static dyn core::fmt::Debug> { compute() } +fn returns_result() -> Result { compute() } +fn returns_result_ref() -> Result<&'static dyn core::fmt::Debug, MyError> { compute() } +fn returns_vec() -> Vec { compute() } +fn returns_tuple() -> (i32, bool) { compute() } +fn returns_custom() -> Custom { compute() } + +#[gamma::value(Custom::EMPTY)] +fn returns_a_stated_value() -> Custom { compute() } + +fn relational_ops(a: i32, b: i32) -> bool { + a < b && a <= b && a > b && a >= b && a == b && a != b +} + +fn arithmetic_ops(a: i32, b: i32) -> i32 { + a + b - a * b / a % b +} + +fn bitwise_ops(a: i32, b: i32) -> i32 { + a & b | a ^ b +} + +fn shift_ops(a: i32, b: i32) -> i32 { + (a << b) >> b +} + +fn logical_ops(x: bool, y: bool) -> bool { + x && y || x +} + +fn assign_ops(a: &mut i32, b: i32) { + *a += b; + *a -= b; + *a *= b; + *a /= b; + *a %= b; + *a &= b; + *a |= b; + *a ^= b; + *a <<= b; + *a >>= b; +} + +fn cond_op(flag: bool) -> i32 { + if flag { 1 } else { 2 } +} + +fn guard_op(v: i32) -> i32 { + match v { + n if n > 0 => n, + _ => 0, + } +} + +fn arm_op(v: i32) -> i32 { + match v { + 1 => 10, + 2 => 20, + _ => 0, + } +} + +fn struct_op() -> C { + C { a: 1, b: 2, ..Default::default() } +} + +fn range_op(n: usize) -> usize { + let mut t = 0; + for i in 0..n { + t += i; + } + for j in 0..=n { + t += j; + } + t +} + +fn loop_op(v: &[i32]) { + for x in v { + if *x == 0 { + continue; + } + if *x == 1 { + break; + } + } +} + +fn unary_op(a: i32, c: bool) -> i32 { + let _ = !c; + -a +} + +fn literal_op() -> i32 { + let _b = true; + let _s = "hello"; + 5 +} + +fn stmt_op(v: &mut Vec) { + let mut a = 0; + v.push(1); + a = 2; + let _ = a; +} + +fn expr_op(n: usize) { + sink(n); +} + +fn option_op(flag: bool) { + let _ = if flag { Some(1) } else { None }; +} + +fn result_op(flag: bool) { + let _ = if flag { Ok(1) } else { Err(2) }; +} + +fn iter_op(v: &[u32], mut w: Vec) { + let _ = v.iter().any(|n| *n > 0); + let _ = v.iter().all(|n| *n > 0); + let _ = v.iter().min(); + let _ = v.iter().max(); + let _ = v.first(); + let _ = v.last(); + w.sort(); + w.dedup(); +} + +fn string_op(s: &str) { + let _ = s.starts_with("a"); + let _ = s.ends_with("b"); + let _ = s.to_lowercase(); + let _ = s.to_uppercase(); + let _ = s.trim_start(); + let _ = s.trim_end(); +} + +fn collection_op() { + let _ = vec![1, 2, 3]; +} + +fn assign_value_op(mut n: u32) { + n = n + 1; + let _ = n; +} +"#; + +/// Every entry in `REGISTRY` must produce at least one candidate that is attributed to it. +/// +/// This is the one guard the aggregate tests structurally cannot be: a mutator that silently +/// fires zero times is invisible everywhere else. It compiles, it passes +/// `every_mutator_has_a_description`, it passes `registry_names_are_unique`, it shows up in +/// `cargo gamma list mutators` and in the generated documentation tables — and it never produces +/// a mutant. The consequence is the worst failure this tool has: a non-firing entry shrinks the +/// mutant population, which shrinks the denominator, which makes the mutation *score go up*, so a +/// tool whose entire purpose is to tell a user their tests are weaker than they think reports +/// that they are stronger. The same failure hides a mutator that regresses to producing nothing +/// after a refactor of the collectors. +/// +/// Because only the single entry under test is selected, every candidate collected is already +/// attributed to it, so `any(mutator == name)` asserts both that something fired and that it is +/// the right thing. There is deliberately no skip-list: an entry that will not fire is either a +/// gap in this fixture (extend the fixture) or a real defect in its collector (report it), and +/// the failure message names the entry, its family and its description so a maintainer can tell +/// the two apart at a glance rather than being told only that "some entry" is broken. +#[test] +fn every_registry_entry_produces_at_least_one_candidate_against_the_composite_fixture() { + let file = SourceFile::parse("fixture.rs", EVERY_FAMILY_FIXTURE.to_owned()).unwrap(); + + for mutator in REGISTRY { + let mut selection = Selection::parse(mutator.name).unwrap(); + + // `fn_value.err_with` consumes caller-supplied `Err(...)` payloads and produces nothing + // until some are named, exactly as it does on the command line. + if mutator.name == "fn_value.err_with" { + selection.set_errors(vec!["MyError::Boom".to_owned()]); + } + + let found = collect(&file, &selection); + let family = mutator.name.split('.').next().unwrap_or(mutator.name); + + assert!( + found.iter().any(|candidate| candidate.mutator == mutator.name), + "registry entry `{}` (family `{family}`: {}) fired against nothing in the composite fixture. \ + Either the fixture lacks the syntax this mutator needs — extend `EVERY_FAMILY_FIXTURE` — or the \ + collector has regressed to emitting nothing for it, which silently shrinks the mutant population \ + and inflates the mutation score. Candidates found under this selection: {:?}", + mutator.name, + mutator.description, + found.iter().map(|candidate| candidate.mutator).collect::>(), + ); + } +} + +/// A candidate reduced to the fields that identify it, so two independently produced `Vec`s can be +/// compared without `Candidate` needing `PartialEq` for production code that never asks two of them +/// whether they are equal. +fn candidate_key(candidate: &Candidate) -> (Range, &'static str, CompactString, u32, String, Shape) { + ( + candidate.span.clone(), + candidate.mutator, + candidate.replacement.clone(), + candidate.replacement_index, + candidate.item_path.to_string(), + candidate.shape, + ) +} + +/// The fused pass exists only to spend one walk of the syntax tree instead of two; it must never +/// spend it on a different answer. Every family in [`EVERY_FAMILY_FIXTURE`] — including the numeric +/// evidence and import indexes only [`check_stated_and_collect_with`]'s pre-pass builds — is +/// selected here, so this is the same walk [`collect_with`] would have driven from indexes +/// `check_stated`'s own pass never touches. +/// +/// Both sides are compared through [`candidate_key`] rather than by ordering the raw `Vec`s: both +/// already come out of [`finish`](super::traversal) sorted by the same span-then-mutator-then-index +/// rule, so the two lists line up position for position without this test inventing an ordering of +/// its own. +#[test] +fn the_fused_pass_produces_the_same_candidates_as_check_stated_then_collect_with() { + let file = SourceFile::parse("fixture.rs", EVERY_FAMILY_FIXTURE.to_owned()).unwrap(); + let selection = { + let names = REGISTRY.iter().map(|mutator| mutator.name).collect::>().join(","); + let mut selection = Selection::parse(&names).unwrap(); + selection.set_errors(vec!["MyError::Boom".to_owned()]); + selection + }; + let cfg = CfgSet::unconditional(); + let defaults = Defaults::of(&file.ast); + + check_stated(&file).expect("the composite fixture states nothing that cannot be honoured"); + let separately = collect_with(&file, &selection, &cfg, &defaults); + let fused = check_stated_and_collect_with(&file, &selection, &cfg, &defaults).expect("the composite fixture has no fault to report"); + + assert!( + !fused.is_empty(), + "the fixture is expected to produce candidates under the full registry selection" + ); + assert_eq!( + fused.len(), + separately.len(), + "the fused pass must offer as many candidates as the two separate passes would" + ); + + let separately: Vec<_> = separately.iter().map(candidate_key).collect(); + let fused: Vec<_> = fused.iter().map(candidate_key).collect(); + + assert_eq!( + fused, separately, + "the fused pass must offer exactly the candidates the two separate passes would, in the same order" + ); +} + +/// The fault the fused pass reports for a misplaced or malformed stated value must be the one +/// [`check_stated`] alone would have reported, in the same wording — a caller switching to the +/// fused entry point must never see its error messages, or its decision to stop before collecting +/// any candidates, change. +#[test] +fn the_fused_pass_reports_the_same_fault_as_check_stated_and_collects_nothing() { + let source = "#[gamma::value(0)]\n#[gamma::value(1)]\nfn f() -> u32 { 2 }"; + let file = SourceFile::parse("fixture.rs", source.to_owned()).unwrap(); + let selection = Selection::parse("arith.add_to_sub").unwrap(); + let cfg = CfgSet::unconditional(); + let defaults = Defaults::of(&file.ast); + + let expected = check_stated(&file) + .expect_err("the fixture states two values on one item") + .to_string(); + let actual = check_stated_and_collect_with(&file, &selection, &cfg, &defaults) + .expect_err("the fused pass must reject what check_stated alone rejects") + .to_string(); + + assert_eq!( + actual, expected, + "the fused pass must report the identical fault check_stated would have reported alone" + ); +} diff --git a/crates/cargo-gamma-engine/src/ops/collect/traversal.rs b/crates/cargo-gamma-engine/src/ops/collect/traversal.rs new file mode 100644 index 000000000..12c3c7e33 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/collect/traversal.rs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Driving the collector over a syntax tree to produce the candidates a file admits. + +use syn::visit::Visit; + +use super::collector::{Collector, phase_one}; +use super::{Candidate, Defaults}; +use crate::Result; +use crate::cfg::CfgSet; +use crate::ops::registry::Selection; +use crate::parse::SourceFile; + +/// Collects every candidate a file admits under the given selection, with nothing stripped for +/// configuration. +/// +/// Equivalent to [`collect_in`] with an unconditional set. Use it where the build's configuration +/// is not known, which is every caller that is examining a fragment of source rather than a real +/// workspace. +/// +/// The result is sorted by span start, then by mutator name, so that two runs over the same source +/// produce the same order regardless of how the traversal happened to visit siblings. +#[must_use] +pub fn collect(file: &SourceFile, selection: &Selection) -> Vec { + collect_in(file, selection, &CfgSet::unconditional()) +} + +/// Collects every candidate a file admits, under a selection and a build configuration. +/// +/// `cfg` decides which conditionally compiled code is actually in the build. Code behind a +/// predicate that does not hold produces no candidates at all: the compiler strips it, so a guard +/// there would never be compiled and its mutant could never be activated by any test. +#[must_use] +pub fn collect_in(file: &SourceFile, selection: &Selection, cfg: &CfgSet) -> Vec { + collect_with(file, selection, cfg, &Defaults::default()) +} + +/// Collects every candidate a file admits, told what the rest of the workspace implements. +/// +/// The extra argument is what lets a `Default::default()` be withheld for a type the workspace +/// defines and gives no `Default`. An empty index is not a claim that nothing has one; it says +/// nothing was looked at, and every type stays optimistic, which is what [`collect_in`] passes. +#[must_use] +pub fn collect_with(file: &SourceFile, selection: &Selection, cfg: &CfgSet, defaults: &Defaults) -> Vec { + let collector = Collector::new(file, selection, selection.errors(), cfg, defaults); + + finish(file, collector) +} + +/// Reports a file's stated-value errors and collects its candidates in one walk of the syntax tree, +/// rather than the two [`super::check_stated`] and [`collect_with`] would run one after the other. +/// +/// Equivalent to calling [`super::check_stated`] and then [`collect_with`]: the same fault, in the +/// same wording, stops candidate collection before it starts, and the candidates returned when there +/// is no fault are the same candidates `collect_with` would have produced from the same inputs. The +/// only difference is that both passes now read the file's syntax tree once between them, instead of +/// [`super::check_stated`]'s own pass, an index-building pass `collect_with` would otherwise run +/// internally, and `collect_with`'s own candidate-collecting pass. +pub fn check_stated_and_collect_with( + file: &SourceFile, + selection: &Selection, + cfg: &CfgSet, + defaults: &Defaults, +) -> Result> { + let indexes = phase_one::run(file, selection)?; + let collector = Collector::with_indexes(file, selection, selection.errors(), cfg, defaults, indexes); + + Ok(finish(file, collector)) +} + +/// Drives a collector's own traversal to completion and returns its candidates in the stable order +/// every caller depends on. +fn finish<'a>(file: &'a SourceFile, mut collector: Collector<'a>) -> Vec { + collector.visit_file(&file.ast); + + let mut candidates = collector.finish(); + + candidates.sort_by(|left, right| { + left.span + .start + .cmp(&right.span.start) + .then_with(|| left.span.end.cmp(&right.span.end)) + .then_with(|| left.mutator.cmp(right.mutator)) + .then_with(|| left.replacement_index.cmp(&right.replacement_index)) + }); + + candidates +} diff --git a/crates/cargo-gamma-engine/src/ops/mod.rs b/crates/cargo-gamma-engine/src/ops/mod.rs new file mode 100644 index 000000000..ef9a16a01 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The mutation operators and the registry that names them. + +pub mod collect; +pub mod registry; diff --git a/crates/cargo-gamma-engine/src/ops/registry/catalog.rs b/crates/cargo-gamma-engine/src/ops/registry/catalog.rs new file mode 100644 index 000000000..4c9e8f730 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/registry/catalog.rs @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The catalog itself: every mutator the tool knows, and the presets that name groups of them. + +use super::{Mutator, Preset}; + +macro_rules! mutators { + ($($name:literal, $default:literal, $aliases:expr, $description:literal;)+) => { + /// Every mutator known to the tool, in registry order. + pub const REGISTRY: &[Mutator] = &[ + $(Mutator { name: $name, description: $description, default_on: $default, aliases: $aliases },)+ + ]; + }; +} + +mutators! { + // ---- Function value replacement. --------------------------------------------------------- + "fn_value.default", true, &["RV"], "replace the function body with a default value"; + "fn_value.stated", true, &["RV"], "replace the body with the value the site states in #[gamma::value(...)]"; + "fn_value.unit", true, &[], "replace the body of a unit function with ()"; + "fn_value.bool_true", true, &[], "replace the body with true"; + "fn_value.bool_false", true, &[], "replace the body with false"; + "fn_value.zero", true, &[], "replace the body with 0"; + "fn_value.one", true, &[], "replace the body with 1"; + "fn_value.minus_one", true, &[], "replace the body with -1"; + "fn_value.empty_string", true, &[], "replace the body with an empty string"; + "fn_value.xyzzy_string", true, &[], "replace the body with a non-empty string"; + "fn_value.none", true, &[], "replace the body with None"; + "fn_value.some_default", true, &[], "replace the body with Some(Default::default())"; + "fn_value.ok_default", true, &[], "replace the body with Ok(Default::default())"; + "fn_value.err_default", true, &[], "replace the body with Err(Default::default())"; + "fn_value.err_with", true , &[], "replace the body with Err(v) for each --error value"; + "fn_value.two", true, &[], "replace the body with 2"; + "fn_value.some", false, &[], "replace the body with Some(value)"; + "fn_value.ok", true, &[], "replace the body with Ok(value)"; + "fn_value.empty_collection", true, &[], "replace the body with an empty collection or iterator"; + "fn_value.one_element", true, &[], "replace the body with a one-element collection or iterator"; + "fn_value.tuple", true, &[], "replace the body with a tuple of replacement values"; + + // ---- Relational and boundary. ------------------------------------------------------------ + "relational.lt_to_le", true, &["ROR"], "replace < with <="; + "relational.lt_to_gt", true, &["ROR"], "replace < with >"; + "relational.le_to_lt", true, &["ROR"], "replace <= with <"; + "relational.le_to_ge", true, &["ROR"], "replace <= with >="; + "relational.gt_to_ge", true, &["ROR"], "replace > with >="; + "relational.gt_to_lt", true, &["ROR"], "replace > with <"; + "relational.ge_to_gt", true, &["ROR"], "replace >= with >"; + "relational.ge_to_le", true, &["ROR"], "replace >= with <="; + "relational.eq_to_ne", true, &["ROR"], "replace == with !="; + "relational.ne_to_eq", true, &["ROR"], "replace != with =="; + + // ---- Arithmetic. ------------------------------------------------------------------------- + "arith.add_to_sub", true, &["AOR"], "replace + with -"; + "arith.add_to_mul", true , &["AOR"], "replace + with *"; + "arith.sub_to_add", true, &["AOR"], "replace - with +"; + "arith.sub_to_div", true , &["AOR"], "replace - with /"; + "arith.mul_to_div", true, &["AOR"], "replace * with /"; + "arith.mul_to_add", true , &["AOR"], "replace * with +"; + "arith.div_to_mul", true, &["AOR"], "replace / with *"; + "arith.div_to_rem", true , &["AOR"], "replace / with %"; + "arith.rem_to_div", true, &["AOR"], "replace % with /"; + "arith.rem_to_mul", true , &["AOR"], "replace % with *"; + + // ---- Bitwise and shift. ------------------------------------------------------------------ + "bitwise.and_to_or", true, &["AOR"], "replace & with |"; + "bitwise.or_to_and", true, &["AOR"], "replace | with &"; + "bitwise.xor_to_and", true, &["AOR"], "replace ^ with &"; + "bitwise.and_to_xor", true , &["AOR"], "replace & with ^"; + "shift.shl_to_shr", true, &["AOR"], "replace << with >>"; + "shift.shr_to_shl", true, &["AOR"], "replace >> with <<"; + + // ---- Compound assignment. ---------------------------------------------------------------- + "assign.add_to_sub", true, &["ASR"], "replace += with -="; + "assign.sub_to_add", true, &["ASR"], "replace -= with +="; + "assign.mul_to_div", true, &["ASR"], "replace *= with /="; + "assign.div_to_mul", true, &["ASR"], "replace /= with *="; + "assign.rem_to_div", true , &["ASR"], "replace %= with /="; + "assign.and_to_or", true, &["ASR"], "replace &= with |="; + "assign.or_to_and", true, &["ASR"], "replace |= with &="; + "assign.xor_to_and", true , &["ASR"], "replace ^= with &="; + "assign.shl_to_shr", true , &["ASR"], "replace <<= with >>="; + "assign.shr_to_shl", true , &["ASR"], "replace >>= with <<="; + + // ---- Logical and condition. -------------------------------------------------------------- + "logical.and_to_or", true, &["LCR"], "replace && with ||"; + "logical.or_to_and", true, &["LCR"], "replace || with &&"; + "cond.negate", true, &["COR"], "negate a branch condition"; + "cond.always_true", true , &["COR"], "force a branch condition to true"; + "cond.always_false", true , &["COR"], "force a branch condition to false"; + + // ---- Match guards. The condition family's blind spot. -------------------------------------- + "match_guard.negate", true, &["COR"], "negate a match arm's guard"; + "match_guard.always_true", true , &["COR"], "force a match arm's guard to true"; + "match_guard.always_false", true , &["COR"], "force a match arm's guard to false"; + + // ---- Match arms. ------------------------------------------------------------------------- + "match_arm.never_matches", true , &["SDL"], "stop a match arm from matching, falling through to the wildcard"; + + // ---- Struct literals. -------------------------------------------------------------------- + "struct_field.omit", true , &["SDL"], "omit a struct literal field, leaving the base expression to supply it"; + + // ---- Ranges. ----------------------------------------------------------------------------- + "range.exclusive_to_inclusive", true, &["ROR"], "extend a .. range to cover its endpoint"; + "range.inclusive_to_exclusive", true, &["ROR"], "shrink a ..= range to stop short of its endpoint"; + + // ---- Loop control flow. ------------------------------------------------------------------ + "loop.break_to_continue", true , &[], "replace break with continue"; + "loop.continue_to_break", true, &[], "replace continue with break"; + "loop.delete_break", true , &["SDL"], "delete a break statement"; + "loop.delete_continue", true , &["SDL"], "delete a continue statement"; + + // ---- Unary. ------------------------------------------------------------------------------ + "unary.remove_neg", true, &["UOI"], "remove a unary minus"; + "unary.remove_not", true, &["UOI"], "remove a unary not"; + + // ---- Literals. --------------------------------------------------------------------------- + "literal.int_to_zero", true, &["CRP"], "replace an integer literal with 0"; + "literal.int_to_one", true, &["CRP"], "replace an integer literal with 1"; + "literal.int_increment", true, &["CRP"], "add one to an integer literal"; + "literal.int_decrement", true, &["CRP"], "subtract one from an integer literal"; + "literal.bool_flip", true, &["CRP"], "invert a boolean literal"; + "literal.str_to_empty", true, &["CRP"], "replace a string literal with an empty string"; + "literal.str_to_xyzzy", true , &["CRP"], "replace a string literal with a different string"; + + // ---- Statement deletion and side-effect removal. ------------------------------------------ + "stmt.delete_call", true , &["SDL"], "delete a statement whose value is discarded"; + "stmt.delete_assign", true , &["SDL"], "delete an assignment statement, plain or compound"; + + // ---- Focused numeric perturbation, in boundary-sensitive positions only. ------------------- + "expr.increment", true, &["EVR"], "add one to a numeric expression in a boundary-sensitive position"; + "expr.decrement", true, &["EVR"], "subtract one from a numeric expression in a boundary-sensitive position"; + + // ---- Option and Result construction. ------------------------------------------------------- + // These ask about error handling at the point it is decided, which whole-function replacement + // can only ask about a function at a time. + "option.some_to_none", true, &["EVR"], "replace Some(value) with None"; + "option.none_to_some", true, &["EVR"], "replace None with Some(Default::default())"; + "result.ok_to_err", true, &["EVR"], "replace Ok(value) with Err(Default::default())"; + "result.err_to_ok", true, &["EVR"], "replace Err(value) with Ok(Default::default())"; + + // ---- Iterator quantifiers and selectors. --------------------------------------------------- + // Limited to a curated set of standard-library names. Without type resolution there is no way + // to know that a user's `take` means what the standard library's does, so the risk is applying + // a transformation that is not the one advertised. + "iter.any_to_all", true, &["EVR"], "replace any with all"; + "iter.all_to_any", true, &["EVR"], "replace all with any"; + "iter.min_to_max", true, &["EVR"], "replace min with max"; + "iter.max_to_min", true, &["EVR"], "replace max with min"; + "iter.first_to_last", true, &["EVR"], "replace first with last"; + "iter.last_to_first", true, &["EVR"], "replace last with first"; + "iter.remove_sort", true, &["SDL"], "remove a sort from a chain"; + "iter.remove_dedup", true, &["SDL"], "remove a deduplication from a chain"; + + // ---- String semantics. --------------------------------------------------------------------- + "string.starts_with_to_ends_with", true, &["EVR"], "replace starts_with with ends_with"; + "string.ends_with_to_starts_with", true, &["EVR"], "replace ends_with with starts_with"; + "string.lower_to_upper", true, &["EVR"], "replace to_lowercase with to_uppercase"; + "string.upper_to_lower", true, &["EVR"], "replace to_uppercase with to_lowercase"; + "string.trim_start_to_trim_end", true, &["EVR"], "replace trim_start with trim_end"; + "string.trim_end_to_trim_start", true, &["EVR"], "replace trim_end with trim_start"; + + // ---- Collection construction. -------------------------------------------------------------- + // Only `vec![]`, never an array: an array's length is part of its type, so removing an element + // changes the type rather than the behaviour. + "collection.omit_element", true, &["SDL"], "omit an element from a vec! literal"; + + // ---- Assignment values. -------------------------------------------------------------------- + "assign_value.default", true, &["EVR"], "replace an assigned value with its type's default"; +} + +/// Every mutator preset known to the tool. +pub const PRESETS: &[Preset] = &[ + Preset { + name: "all", + description: "every registered mutator", + members: &["*"], + }, + Preset { + name: "default", + description: "the mutators enabled when none are named", + members: &["@default"], + }, + Preset { + name: "pedantic", + description: "additional low-yield mutations excluded from the default selection", + members: &["fn_value.some"], + }, + Preset { + name: "boundary", + description: "relational and boundary conditions", + members: &["relational", "range"], + }, + Preset { + name: "arithmetic", + description: "arithmetic, bitwise, shift and compound assignment", + members: &["arith", "bitwise", "shift", "assign"], + }, + Preset { + name: "logical", + description: "logical operators and branch conditions", + members: &["logical", "cond", "match_guard"], + }, + Preset { + name: "control", + description: "the choices control flow makes: conditions, guards, arms and loop exits", + members: &["cond", "match_guard", "match_arm", "loop"], + }, + Preset { + name: "removal", + description: "statement and side-effect deletion", + members: &["stmt", "unary", "match_arm", "struct_field", "collection"], + }, + Preset { + name: "semantics", + description: "standard-library meaning: Option, Result, iterators, strings and collections", + members: &["option", "result", "iter", "string", "collection", "assign_value"], + }, + Preset { + name: "literals", + description: "literal and constant replacement", + members: &["literal"], + }, + Preset { + name: "numeric", + description: "literal replacement and focused numeric expression perturbation", + members: &["literal", "expr"], + }, + Preset { + name: "extreme", + description: "a synonym for `all`, kept because scripts name it", + members: &["*"], + }, +]; diff --git a/crates/cargo-gamma-engine/src/ops/registry/lookup.rs b/crates/cargo-gamma-engine/src/ops/registry/lookup.rs new file mode 100644 index 000000000..58716c5e7 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/registry/lookup.rs @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Resolving a name or a selector against the catalog. + +use super::catalog::{PRESETS, REGISTRY}; +use super::{Mutator, Preset}; +use crate::error::{Error, error}; +use crate::{HashSet, Result}; + +/// Looks up a mutator by its exact registry name. +#[must_use] +pub fn find(name: &str) -> Option<&'static Mutator> { + REGISTRY.iter().find(|mutator| mutator.name == name) +} + +/// Looks up a mutator preset by name, without the leading `@`. +#[must_use] +pub fn find_preset(name: &str) -> Option<&'static Preset> { + PRESETS.iter().find(|preset| preset.name == name) +} + +/// Returns the distinct family prefixes present in the registry, in registry order. +#[must_use] +pub fn families() -> Vec<&'static str> { + let mut seen = HashSet::default(); + let mut out = Vec::new(); + + for mutator in REGISTRY { + let family = mutator.name.split('.').next().unwrap_or(mutator.name); + + if seen.insert(family) { + out.push(family); + } + } + + out +} + +/// Expands one selector into the mutator names it matches. +/// +/// A selector is a full name, a family prefix, an `@preset`, an academic alias, or `all`. +pub fn resolve(selector: &str) -> Result> { + if selector == "all" { + return Ok(REGISTRY.iter().map(|m| m.name).collect()); + } + + if let Some(preset_name) = selector.strip_prefix('@') { + let preset = find_preset(preset_name).ok_or_else(|| unknown(selector))?; + let mut names = Vec::new(); + + for member in preset.members { + match *member { + "*" => names.extend(REGISTRY.iter().map(|m| m.name)), + "@default" => names.extend(REGISTRY.iter().filter(|m| m.default_on).map(|m| m.name)), + other => names.extend(resolve(other)?), + } + } + + return Ok(names); + } + + if let Some(mutator) = find(selector) { + return Ok(vec![mutator.name]); + } + + // Family or sub-family prefix: `arith`, or `combinator.iter`. + let prefix = format!("{selector}."); + let matched: Vec<&'static str> = REGISTRY + .iter() + .filter(|m| m.name.starts_with(prefix.as_str())) + .map(|m| m.name) + .collect(); + + if !matched.is_empty() { + return Ok(matched); + } + + // Academic or industry alias, matched case-insensitively. + let matched: Vec<&'static str> = REGISTRY + .iter() + .filter(|m| m.aliases.iter().any(|alias| alias.eq_ignore_ascii_case(selector))) + .map(|m| m.name) + .collect(); + + if matched.is_empty() { Err(unknown(selector)) } else { Ok(matched) } +} + +/// Builds the error for an unmatched selector, with a spelling suggestion. +fn unknown(selector: &str) -> Error { + let mut best: Option<(f64, String)> = None; + + let candidates = REGISTRY + .iter() + .map(|m| m.name.to_owned()) + .chain(families().into_iter().map(ToOwned::to_owned)) + .chain(REGISTRY.iter().flat_map(|m| m.aliases.iter().map(|a| (*a).to_owned()))) + .chain(PRESETS.iter().map(|preset| format!("@{}", preset.name))); + + for candidate in candidates { + let score = strsim::jaro_winkler(selector, &candidate); + + if score > best.as_ref().map_or(0.85, |(previous, _)| *previous) { + best = Some((score, candidate)); + } + } + + best.map_or_else( + || error!("unknown mutator selector `{selector}`; run `cargo gamma list mutators` to see the registry"), + |(_, suggestion)| error!("unknown mutator selector `{selector}`; did you mean `{suggestion}`?"), + ) + .usage() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_distant_unknown_selector_falls_back_to_the_registry_hint() { + let error = resolve("zzzzzzzzzz").expect_err("the selector is not in the registry"); + let message = error.to_string(); + + assert!(message.contains("unknown mutator selector `zzzzzzzzzz`"), "{message}"); + assert!(message.contains("run `cargo gamma list mutators`"), "{message}"); + assert!(!message.contains("did you mean"), "{message}"); + } + + #[test] + fn a_close_unknown_selector_offers_a_spelling_suggestion() { + let error = resolve("reltional").expect_err("the selector misspells a known family"); + let message = error.to_string(); + + assert!(message.contains("unknown mutator selector `reltional`"), "{message}"); + assert!(message.contains("did you mean `relational`?"), "{message}"); + } +} diff --git a/crates/cargo-gamma-engine/src/ops/registry/mod.rs b/crates/cargo-gamma-engine/src/ops/registry/mod.rs new file mode 100644 index 000000000..0eafc71b9 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/registry/mod.rs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The mutator registry: stable names, families, presets, and selector resolution. +//! +//! Every mutator has one stable, well-known name of the form `family.transform`. That name is the +//! single vocabulary used by all three suppression channels, by `--mutators`, by the report, and by +//! configuration. Nothing anywhere refers to a mutator by index, by description string, or by +//! position in a list, because all three of those change when the catalog grows. + +mod catalog; +mod lookup; +mod mutator; +mod preset; +mod selection; + +pub use catalog::{PRESETS, REGISTRY}; +pub use lookup::{families, find, find_preset, resolve}; +pub use mutator::Mutator; +pub use preset::Preset; +pub use selection::Selection; diff --git a/crates/cargo-gamma-engine/src/ops/registry/mutator.rs b/crates/cargo-gamma-engine/src/ops/registry/mutator.rs new file mode 100644 index 000000000..166438acf --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/registry/mutator.rs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/// A registered mutator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Mutator { + /// The stable name, `family.transform`. + pub name: &'static str, + + /// One-line description, used by `explain` and by the report. + pub description: &'static str, + + /// Whether the mutator is enabled by the default preset. + pub default_on: bool, + + /// Academic or industry aliases that resolve to this mutator. + pub aliases: &'static [&'static str], +} diff --git a/crates/cargo-gamma-engine/src/ops/registry/preset.rs b/crates/cargo-gamma-engine/src/ops/registry/preset.rs new file mode 100644 index 000000000..2b4218af3 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/registry/preset.rs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/// A named set of mutators. +#[derive(Debug, Clone, Copy)] +pub struct Preset { + /// The preset name, written `@name` in selectors. + pub name: &'static str, + + /// One-line description. + pub description: &'static str, + + /// Families and names that make up the preset. + pub members: &'static [&'static str], +} diff --git a/crates/cargo-gamma-engine/src/ops/registry/selection.rs b/crates/cargo-gamma-engine/src/ops/registry/selection.rs new file mode 100644 index 000000000..abf24d8d4 --- /dev/null +++ b/crates/cargo-gamma-engine/src/ops/registry/selection.rs @@ -0,0 +1,249 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::{REGISTRY, resolve}; +use crate::{HashSet, Result}; + +/// The mutator that consumes caller-supplied error values. +const ERR_WITH: &str = "fn_value.err_with"; + +/// A resolved set of mutator names. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Selection { + names: HashSet<&'static str>, + errors: Vec, +} + +impl Selection { + /// The set enabled when the user names nothing. + #[must_use] + pub fn default_preset() -> Self { + Self { + names: REGISTRY.iter().filter(|m| m.default_on).map(|m| m.name).collect(), + errors: Vec::new(), + } + } + + /// Every registered mutator. + #[must_use] + pub fn everything() -> Self { + Self { + names: REGISTRY.iter().map(|m| m.name).collect(), + errors: Vec::new(), + } + } + + /// An empty set. + #[must_use] + pub fn empty() -> Self { + Self { + names: HashSet::default(), + errors: Vec::new(), + } + } + + /// Sets the caller-supplied `Err(...)` payloads that `fn_value.err_with` will use. + /// + /// `Err(Default::default())` only reaches error types that implement `Default`, which most + /// hand-rolled error enums do not. Naming values here is how those functions get an error + /// mutant at all, so supplying any also turns the mutator on. + pub fn set_errors(&mut self, errors: Vec) { + if errors.is_empty() { + self.errors = errors; + return; + } + + let _ = self.names.insert(ERR_WITH); + self.errors = errors; + } + + /// Removes the error mutator, for a selection the user spelled out without it. + pub fn drop_errors(&mut self) { + let _ = self.names.remove(ERR_WITH); + self.errors.clear(); + } + + /// Returns the caller-supplied `Err(...)` payloads. + #[must_use] + pub fn errors(&self) -> &[String] { + &self.errors + } + + /// Returns whether a mutator is in the set. + #[must_use] + pub fn contains(&self, name: &str) -> bool { + self.names.contains(name) + } + + /// Returns whether any mutator of a family is selected. + /// + /// A family is the part of a name before the dot. Asked by the collector, which builds some of + /// its per-file indexes only for the family that consults them and would otherwise pay for an + /// answer nothing was going to read. + #[must_use] + pub fn any_in_family(&self, family: &str) -> bool { + self.names + .iter() + .any(|name| name.strip_prefix(family).is_some_and(|rest| rest.starts_with('.'))) + } + + /// Returns the names in sorted order. + #[must_use] + pub fn sorted(&self) -> Vec<&'static str> { + let mut names: Vec<&'static str> = self.names.iter().copied().collect(); + + names.sort_unstable(); + names + } + + /// Adds every mutator matched by one selector. + fn add(&mut self, selector: &str) -> Result<()> { + for name in resolve(selector)? { + let _ = self.names.insert(name); + } + + Ok(()) + } + + /// Removes every mutator matched by one selector. + fn remove(&mut self, selector: &str) -> Result<()> { + for name in resolve(selector)? { + let _ = self.names.remove(name); + } + + Ok(()) + } + + /// Applies a comma-separated selector list to this set. + /// + /// Selectors are applied left to right, so a later `!family` can carve out of an earlier + /// preset. A selector that matches nothing is an error, never a silent no-op: a suppression + /// that quietly does nothing is the single most damaging failure mode a mutation tool can + /// have, because the score stays high and nobody learns why. + pub fn apply(&mut self, selectors: &str) -> Result<()> { + for raw in selectors.split(',') { + let selector = raw.trim(); + + if selector.is_empty() { + continue; + } + + if let Some(rest) = selector.strip_prefix('!') { + self.remove(rest.trim())?; + } else { + self.add(selector)?; + } + } + + Ok(()) + } + + /// Builds a selection from a selector list, starting from nothing. + pub fn parse(selectors: &str) -> Result { + let mut selection = Self::empty(); + + selection.apply(selectors)?; + Ok(selection) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_selectors_are_errors_not_silent_no_ops() { + let error = resolve("nonsense_family").unwrap_err(); + + assert!(error.to_string().contains("unknown mutator selector")); + } + + #[test] + fn negation_carves_out_of_a_preset() { + let mut selection = Selection::parse("@arithmetic").unwrap(); + let before = selection.names.len(); + + selection.apply("!bitwise").unwrap(); + + assert!(selection.contains("arith.add_to_sub")); + assert!(!selection.contains("bitwise.and_to_or")); + assert!(selection.names.len() < before); + } + + #[test] + fn selectors_apply_left_to_right() { + let selection = Selection::parse("relational, !relational.eq_to_ne").unwrap(); + + assert!(selection.contains("relational.lt_to_le")); + assert!(!selection.contains("relational.eq_to_ne")); + } + + #[test] + fn default_and_pedantic_presets_partition_everything() { + let default = Selection::default_preset(); + let pedantic = Selection::parse("@pedantic").unwrap(); + let all = Selection::everything(); + + assert!(!default.names.is_empty()); + assert_eq!(pedantic.sorted(), ["fn_value.some"]); + assert!(!default.contains("fn_value.some")); + assert_eq!(default.names.len() + pedantic.names.len(), all.names.len()); + + let combined = Selection::parse("@default,@pedantic").unwrap(); + + for name in all.sorted() { + assert!(combined.contains(name), "{name} is not reachable through a shipped preset"); + } + } + + #[test] + fn the_noisier_families_are_still_on() { + let default = Selection::default_preset(); + + // Statement deletion has a high equivalent-mutant rate. It stays on anyway; this test + // exists so that turning it off again is a deliberate edit rather than a quiet drift. + assert!(default.contains("stmt.delete_call")); + } + + #[test] + fn empty_selectors_are_ignored() { + let selection = Selection::parse("relational, , ").unwrap(); + + assert_eq!(selection.names.len(), 10); + } + + /// `any_in_family` is the gate that builds the per-file imports index only when a `fn_value` + /// mutator is selected. It must answer on the family prefix: `true` when the family is present, + /// `false` when it is not, or the index is skipped and undefaultable-type mutants slip through. + #[test] + fn any_in_family_gates_on_the_selected_family() { + assert!(Selection::parse("fn_value.default").unwrap().any_in_family("fn_value")); + assert!(!Selection::parse("relational").unwrap().any_in_family("fn_value")); + } + + #[test] + fn caller_supplied_error_values_toggle_the_error_mutator() { + let mut selection = Selection::empty(); + + selection.set_errors(vec!["Error::Broken".to_owned()]); + + assert!(selection.contains(ERR_WITH)); + assert_eq!(selection.errors(), ["Error::Broken"]); + + selection.drop_errors(); + + assert!(!selection.contains(ERR_WITH)); + assert!(selection.errors().is_empty()); + } + + #[test] + fn an_empty_error_list_does_not_enable_the_error_mutator() { + let mut selection = Selection::everything(); + + selection.drop_errors(); + selection.set_errors(Vec::new()); + + assert!(!selection.contains(ERR_WITH)); + assert!(selection.errors().is_empty()); + } +} diff --git a/crates/cargo-gamma-engine/src/parse/comment.rs b/crates/cargo-gamma-engine/src/parse/comment.rs new file mode 100644 index 000000000..1de3319e6 --- /dev/null +++ b/crates/cargo-gamma-engine/src/parse/comment.rs @@ -0,0 +1,546 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Comment trivia: scanning raw source text for comments that `syn` never sees. + +use core::ops::Range; + +/// What kind of comment was found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CommentKind { + /// `// ...`, the only kind that can carry a directive. + Line, + + /// `/// ...` or `/** ... */`, documentation for the item that follows. + OuterDoc, + + /// `//! ...` or `/*! ... */`, documentation for the enclosing item. + InnerDoc, + + /// `/* ... */`. + Block, +} + +/// One comment, located in the file. +#[derive(Debug, Clone)] +pub struct Comment { + /// What kind of comment this is. + pub kind: CommentKind, + + /// Byte range of the comment including its delimiters. + pub span: Range, + + /// The comment text with its delimiters and one leading space removed. + pub body: String, + + /// 1-based line number of the comment's first line. + pub line: usize, + + /// Whether anything other than whitespace precedes the comment on its own line. + pub trailing: bool, +} + +/// Scans source text for comments, skipping over anything inside a string or character literal. +pub(super) fn scan_comments(text: &str, lines: &[usize]) -> Vec { + comment_spans(text) + .into_iter() + .map(|span| { + let raw = text.get(span.clone()).unwrap_or(""); + + build_comment( + if raw.starts_with("//") { + line_comment_kind(raw) + } else { + block_comment_kind(raw) + }, + raw, + span, + text, + lines, + ) + }) + .collect() +} + +/// Locates comments while skipping comment-shaped text inside literals. +pub(crate) fn comment_spans(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut comments = Vec::new(); + let mut i = 0; + + while i < bytes.len() { + if bytes[i] == b'/' && bytes.get(i + 1) == Some(&b'/') { + let end = line_comment_end(bytes, i); + + comments.push(i..end); + i = end; + } else if bytes[i] == b'/' && bytes.get(i + 1) == Some(&b'*') { + let end = block_comment_end(bytes, i); + + comments.push(i..end); + i = end; + } else if let Some(end) = literal_end(text, i) { + i = end; + } else { + i += 1; + } + } + + comments +} + +/// Returns the offset past a literal beginning at `start`, or `None` for ordinary syntax. +pub(crate) fn literal_end(text: &str, start: usize) -> Option { + let bytes = text.as_bytes(); + let end = match bytes.get(start)? { + b'"' => string_end(bytes, start), + b'r' if matches!(bytes.get(start + 1), Some(b'"' | b'#')) => raw_string_end(bytes, start), + b'\'' => quote_end(bytes, start), + _ => return None, + }; + + (end > start + 1).then_some(end) +} + +/// Classifies a `//` comment. Order matters: `//!` and `///` must be checked before `//`. +fn line_comment_kind(raw: &str) -> CommentKind { + if raw.starts_with("//!") { + CommentKind::InnerDoc + } else if raw.starts_with("///") && !raw.starts_with("////") { + CommentKind::OuterDoc + } else { + CommentKind::Line + } +} + +/// Classifies a `/* */` comment. +fn block_comment_kind(raw: &str) -> CommentKind { + if raw.starts_with("/*!") { + CommentKind::InnerDoc + } else if raw.starts_with("/**") && !raw.starts_with("/**/") && !raw.starts_with("/***") { + CommentKind::OuterDoc + } else { + CommentKind::Block + } +} + +/// Builds a comment record, stripping delimiters and working out whether it trails code. +fn build_comment(kind: CommentKind, raw: &str, span: Range, text: &str, lines: &[usize]) -> Comment { + let stripped = raw + .trim_start_matches('/') + .trim_start_matches('*') + .trim_start_matches('!') + .trim_end_matches('/') + .trim_end_matches('*'); + + let line_index = match lines.binary_search(&span.start) { + Ok(exact) => exact, + Err(insertion) => insertion.saturating_sub(1), + }; + + let line_start = lines.get(line_index).copied().unwrap_or(0); + let before = text.get(line_start..span.start).unwrap_or(""); + + Comment { + kind, + span, + body: stripped.trim().to_owned(), + line: line_index + 1, + trailing: !before.trim().is_empty(), + } +} + +/// Returns the offset just past the end of a `//` comment. +fn line_comment_end(bytes: &[u8], start: usize) -> usize { + let mut i = start + 2; + + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + + i +} + +/// Returns the offset just past the end of a `/* */` comment, which may nest. +fn block_comment_end(bytes: &[u8], start: usize) -> usize { + let mut i = start + 2; + let mut depth = 1_usize; + + while i < bytes.len() { + if bytes[i] == b'/' && bytes.get(i + 1) == Some(&b'*') { + depth += 1; + i += 2; + } else if bytes[i] == b'*' && bytes.get(i + 1) == Some(&b'/') { + depth -= 1; + i += 2; + + if depth == 0 { + return i; + } + } else { + i += 1; + } + } + + bytes.len() +} + +/// Returns the offset just past the end of a `"..."` literal. +fn string_end(bytes: &[u8], start: usize) -> usize { + let mut i = start + 1; + + while i < bytes.len() { + match bytes[i] { + b'\\' => i += 2, + b'"' => return i + 1, + _ => i += 1, + } + } + + bytes.len() +} + +/// Returns the offset just past the end of an `r#".."#` literal. +fn raw_string_end(bytes: &[u8], start: usize) -> usize { + let mut hashes = 0; + let mut i = start + 1; + + while bytes.get(i) == Some(&b'#') { + hashes += 1; + i += 1; + } + + if bytes.get(i) != Some(&b'"') { + // Not a raw string after all, just an identifier beginning with `r`. + return start + 1; + } + + i += 1; + + while i < bytes.len() { + if bytes[i] == b'"' { + let closing = i + 1; + let found = bytes[closing..].iter().take_while(|b| **b == b'#').count(); + + if found >= hashes { + return closing + hashes; + } + } + + i += 1; + } + + bytes.len() +} + +/// Returns the offset just past a `'`, which may open a character literal or a lifetime. +/// +/// This is the one genuinely ambiguous case in Rust's lexical grammar for a scanner this small. +/// Getting it wrong is not cosmetic: treating the `'a` in `fn f<'a>()` as an unterminated +/// character literal would swallow the rest of the file and lose every comment in it. +fn quote_end(bytes: &[u8], start: usize) -> usize { + // `'\n'`, `'\''`, `'\u{1f600}'` and friends. The escaped character is skipped rather than + // examined, or the quote in `'\''` would be mistaken for the closing one. + if bytes.get(start + 1) == Some(&b'\\') { + let mut i = start + 3; + + while i < bytes.len() && bytes[i] != b'\'' { + i += 1; + } + + // An unterminated literal at the end of the file has no closing quote to step past. + return (i + 1).min(bytes.len()); + } + + // A single character followed by a closing quote is a character literal. The character may be + // several bytes, so step by whole characters rather than by bytes. + let rest = &bytes[start + 1..]; + let width = utf8_width(rest.first().copied().unwrap_or(0)); + + if rest.get(width) == Some(&b'\'') { + return start + 1 + width + 1; + } + + // Otherwise it is a lifetime or a label; consume just the quote. + start + 1 +} + +/// Returns the number of bytes in a UTF-8 sequence given its leading byte. +const fn utf8_width(first: u8) -> usize { + match first { + 0x00..=0x7F => 1, + 0xC0..=0xDF => 2, + 0xE0..=0xEF => 3, + _ => 4, + } +} + +#[cfg(all(test, not(miri)))] +mod fuzz { + use super::scan_comments; + use crate::parse::source_file::line_starts; + + /// Arbitrary text is scanned without panicking, and every comment it finds is really there. + /// + /// This scanner indexes a byte slice directly and jumps over string, raw-string and character + /// literals, so it is the one parser here that can produce an out-of-bounds index or a span + /// that is not on a character boundary. Random text — especially multi-byte text next to a + /// quote — is the cheapest way to find that. + /// + /// Reaching the end at all is half the property: every arm has to advance the cursor, and one + /// that returns its own position would spin here rather than fail, which the harness's own + /// timeout reports. + #[test] + fn arbitrary_text_is_scanned_into_spans_that_exist() { + bolero::check!().with_type::().for_each(|text| { + super::tests::assert_comment_scan_invariants(text); + }); + } + + /// Text with no comment delimiter in it yields no comments. + /// + /// The scanner's job is to find `//` and `/*`, and its risk is finding them inside a literal + /// where they are not comments at all. This is the cheap half of that: with the delimiters + /// removed entirely, no amount of quoting or escaping should conjure one up. + #[test] + fn text_without_a_delimiter_yields_no_comments() { + bolero::check!().with_type::().for_each(|text| { + let text: String = text.replace('/', ""); + + assert!( + scan_comments(&text, &line_starts(&text)).is_empty(), + "a comment was found in text with no delimiter: {text:?}" + ); + }); + } +} + +#[cfg(test)] +mod tests { + use super::super::source_file::{SourceFile, line_starts}; + use super::*; + + pub(crate) fn assert_comment_scan_invariants(text: &str) { + let comments = scan_comments(text, &line_starts(text)); + let lines = text.lines().count(); + let mut previous = 0; + + for comment in &comments { + assert!(comment.span.start <= comment.span.end, "a span runs backwards: {comment:?}"); + assert!(comment.span.end <= text.len(), "a span runs past the text: {comment:?}"); + assert!( + text.get(comment.span.clone()).is_some(), + "a span is not on a character boundary: {comment:?}" + ); + + // Comments are found by one forward pass, so they come out in order and cannot + // overlap. A scanner that failed to advance past one would break this first. + assert!(comment.span.start >= previous, "spans are out of order: {comment:?}"); + previous = comment.span.end; + + assert!(comment.line >= 1, "a comment is before the first line: {comment:?}"); + assert!(comment.line <= lines.max(1), "a comment is past the last line: {comment:?}"); + } + } + + fn parse(text: &str) -> SourceFile { + SourceFile::parse("test.rs", text.to_owned()).unwrap() + } + + #[test] + fn known_comments_satisfy_the_fuzz_scan_invariants() { + assert_comment_scan_invariants("// one\nfn f() {} /* two */\n/// three\n"); + } + + #[test] + fn line_comments_are_found_and_stripped() { + let file = parse("// hello\nfn f() {}\n"); + + assert_eq!(file.comments.len(), 1); + assert_eq!(file.comments[0].kind, CommentKind::Line); + assert_eq!(file.comments[0].body, "hello"); + assert_eq!(file.comments[0].line, 1); + assert!(!file.comments[0].trailing); + } + + #[test] + fn doc_comments_are_distinguished_from_directive_comments() { + let file = parse("//! module\n/// item\n// plain\n//// also plain\nfn f() {}\n"); + let kinds: Vec = file.comments.iter().map(|c| c.kind).collect(); + + assert_eq!( + kinds, + vec![CommentKind::InnerDoc, CommentKind::OuterDoc, CommentKind::Line, CommentKind::Line] + ); + } + + #[test] + fn an_escaped_quote_literal_does_not_confuse_the_scanner() { + // Stopping at the escaped quote in `'\''` would leave the scanner one quote out of step + // for the rest of the line. + assert_eq!(quote_end(br"'\''", 0), 4); + assert_eq!(quote_end(br"'\n'", 0), 4); + assert_eq!(quote_end(br"'\u{27}'", 0), 8); + assert_eq!(quote_end(b"'a'", 0), 3); + + // A lifetime is not a literal, so only the quote is consumed. + assert_eq!(quote_end(b"'static", 0), 1); + } + + #[test] + fn an_unterminated_literal_stays_inside_the_text() { + // The result indexes the text, so running off the end would be a panic waiting to happen. + for text in [r"'\'", r"'\", "'"] { + let bytes = text.as_bytes(); + + assert!(quote_end(bytes, 0) <= bytes.len(), "{text} ran past the end"); + } + } + + #[test] + fn a_comment_after_an_escaped_quote_is_still_found() { + let file = parse("fn f() { let _q = '\\''; } // after\n"); + + assert_eq!(file.comments.len(), 1); + assert_eq!(file.comments[0].body, "after"); + } + + #[test] + fn trailing_comments_are_marked() { + let file = parse("fn f() { let _x = 1; } // after\n"); + + assert!(file.comments[0].trailing); + } + + #[test] + fn comment_markers_inside_strings_are_not_comments() { + let file = parse("fn f() -> &'static str { \"// not a comment\" }\n"); + + assert!(file.comments.is_empty(), "{:?}", file.comments); + } + + #[test] + fn comment_markers_inside_raw_strings_are_not_comments() { + let file = parse("fn f() -> &'static str { r#\"// nope /* nope */\"# }\n"); + + assert!(file.comments.is_empty(), "{:?}", file.comments); + } + + #[test] + fn an_escaped_quote_does_not_end_a_string() { + let file = parse("fn f() -> &'static str { \"a\\\"// no\" }\n// yes\n"); + + assert_eq!(file.comments.len(), 1); + assert_eq!(file.comments[0].body, "yes"); + } + + #[test] + fn lifetimes_do_not_swallow_the_rest_of_the_file() { + let file = parse("fn f<'a>(x: &'a str) -> &'a str { x }\n// found me\n"); + + assert_eq!(file.comments.len(), 1, "{:?}", file.comments); + assert_eq!(file.comments[0].body, "found me"); + } + + #[test] + fn character_literals_are_skipped() { + let file = parse("fn f() -> char { '\\'' }\n// found me\n"); + + assert_eq!(file.comments.len(), 1, "{:?}", file.comments); + assert_eq!(file.comments[0].body, "found me"); + } + + #[test] + fn multibyte_character_literals_are_skipped() { + let file = parse("fn f() -> char { 'é' }\n// found me\n"); + + assert_eq!(file.comments.len(), 1, "{:?}", file.comments); + } + + #[test] + fn a_quote_in_a_labelled_loop_is_not_a_literal() { + let file = parse("fn f() { 'outer: loop { break 'outer; } }\n// found me\n"); + + assert_eq!(file.comments.len(), 1, "{:?}", file.comments); + } + + #[test] + fn block_comments_nest() { + let file = parse("/* outer /* inner */ still outer */\nfn f() {}\n"); + + assert_eq!(file.comments.len(), 1); + assert!(file.comments[0].body.contains("still outer")); + } + + #[test] + fn block_doc_comments_are_classified() { + let text = "/*! module */\n/** item */\n/**/ /* plain */\nfn f() {}\n"; + let file = parse(text); + let kinds: Vec = file.comments.iter().map(|c| c.kind).collect(); + + assert_eq!( + kinds, + vec![CommentKind::InnerDoc, CommentKind::OuterDoc, CommentKind::Block, CommentKind::Block] + ); + } + + #[test] + fn an_unterminated_block_comment_stops_at_end_of_file() { + // syn rejects this, so scan the trivia directly. + let text = "fn f() {} /* never closed"; + let comments = scan_comments(text, &line_starts(text)); + + assert_eq!(comments.len(), 1); + assert_eq!(comments[0].span.end, text.len()); + } + + #[test] + fn spans_point_at_the_comment_itself() { + let file = parse("fn f() {}\n// directive\n"); + let span = file.comments[0].span.clone(); + + assert_eq!(file.slice(&span), "// directive"); + } + + #[test] + fn r_prefixed_identifiers_are_not_raw_strings() { + let file = parse("fn f() { let range = 1; let _ = range; }\n// found me\n"); + + assert_eq!(file.comments.len(), 1, "{:?}", file.comments); + } + + #[test] + fn raw_strings_with_many_hashes_are_handled() { + let file = parse("fn f() -> &'static str { r###\"a\"# b\"### }\n// found me\n"); + + assert_eq!(file.comments.len(), 1, "{:?}", file.comments); + } + + #[test] + fn unterminated_strings_and_raw_strings_stop_at_end_of_file() { + assert_eq!(string_end(b"\"unterminated", 0), b"\"unterminated".len()); + assert_eq!(raw_string_end(br#"r#"unterminated"#, 0), br#"r#"unterminated"#.len()); + } + + #[test] + fn invalid_raw_string_prefix_consumes_only_the_r() { + assert_eq!(raw_string_end(b"range", 0), 1); + } + + #[test] + fn multibyte_widths_cover_three_and_four_byte_characters() { + let euro = "€".as_bytes()[0]; + let crab = "🦀".as_bytes()[0]; + + assert_eq!(utf8_width(euro), 3); + assert_eq!(utf8_width(crab), 4); + } + + #[test] + fn comments_come_back_in_source_order() { + let file = parse("// one\nfn f() {}\n// two\nfn g() {}\n// three\n"); + let bodies: Vec<&str> = file.comments.iter().map(|c| c.body.as_str()).collect(); + + assert_eq!(bodies, vec!["one", "two", "three"]); + } +} diff --git a/crates/cargo-gamma-engine/src/parse/mod.rs b/crates/cargo-gamma-engine/src/parse/mod.rs new file mode 100644 index 000000000..2b2c23351 --- /dev/null +++ b/crates/cargo-gamma-engine/src/parse/mod.rs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Turning source text into a syntax tree with byte-accurate spans and comment trivia. +//! +//! Two things make this module more than a thin wrapper around `syn::parse_file`. +//! +//! The first is byte accuracy. Instrumentation is a text splice, not a token-tree rewrite, because +//! re-emitting a `syn` tree through `quote` would reformat every file it touches and destroy the +//! line numbers that every report, every suppression and every diff depends on. That requires +//! exact byte offsets for each node, which `proc-macro2` provides only with the `span-locations` +//! feature enabled. +//! +//! The second is comments. Comments are trivia: they are not in the syntax tree at all, so a +//! suppression written as a comment is invisible to `syn`. This module scans the raw text for +//! them, which means it needs a small but honest lexer that knows about raw strings, escapes and +//! the lifetime-versus-character-literal ambiguity. +//! +//! # The one file this module refuses +//! +//! Source under audit is untrusted input, and every stage that reads it — the parser here, the +//! collector's visitor, the scope walk, the render that splices guards back in — descends by +//! recursion. A file nested past [`nesting::NESTING_LIMIT`] levels is therefore refused with a +//! diagnostic naming it, rather than parsed into a stack overflow that names nothing. + +mod comment; +// `pub` (within this private module) so the crate's proc-macro agreement test can reach +// `nesting::NESTING_LIMIT` and `nesting::CHAIN_FACTOR` through the `internals` facade. +pub mod nesting; +mod source_file; + +pub use comment::{Comment, CommentKind}; +pub(crate) use comment::{comment_spans, literal_end}; +#[doc(hidden)] +pub use source_file::{BOM, strip_bom}; +pub use source_file::{SourceFile, exceeds_nesting_limit, without_bom}; diff --git a/crates/cargo-gamma-engine/src/parse/nesting.rs b/crates/cargo-gamma-engine/src/parse/nesting.rs new file mode 100644 index 000000000..7dfe00200 --- /dev/null +++ b/crates/cargo-gamma-engine/src/parse/nesting.rs @@ -0,0 +1,544 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! How deeply source text nests, measured before anything recursive is pointed at it. + +use super::Comment; +use super::comment::literal_end; + +/// The deepest delimiter nesting a file may have and still be analyzed. +/// +/// Source under audit is input, not code this project wrote: a user can point the tool at any file +/// that parses, and machine-generated files nest far deeper than hand-written ones. Everything +/// downstream descends by recursion — `syn`'s parser, the collector's visitor, the scope walk, and +/// the render that splices guards back in — so a file deep enough exhausts the stack, and a stack +/// overflow on Linux is a `SIGSEGV` that names neither the file nor the stage. Rejecting the file +/// by name costs one file's worth of coverage; the alternative costs the whole run and says +/// nothing about why. +/// +/// The number is set from measurement at both ends. Every file in this workspace nests at most 20 +/// levels, which is what hand-written Rust looks like, so the limit is three times what real code +/// needs. Against that, a debug build on a 2 MiB worker thread — the smallest stack the discovery +/// pass runs on — overflows between 140 and 200 levels on the parse-and-collect path. Sitting an +/// order of magnitude above real code and well under half the measured failure point leaves room +/// for both a deeper file than anyone here writes and a stage that grows its stack frames. +/// +/// `pub` within this private module so the crate's proc-macro agreement test can pin the +/// proc-macro's hand-copied `NESTING_LIMIT` against it through the `internals` facade. +pub const NESTING_LIMIT: usize = 64; + +/// How many links a chain may have per level of delimiter nesting allowed. +/// +/// One operator in a chain costs a shallower stack than one delimiter does — a delimiter opens a +/// block or a call, several frames apiece, where an operator adds one `Expr` node — so the two +/// bounds are not the same number. The multiple is what separates a chain long enough to matter +/// from ordinary code: a `|` pattern with dozens of alternatives, or a builder called forty times +/// over, is written by hand and parses flat or nearly so, and refusing it would cost a file's +/// coverage for nothing. +/// +/// `pub` within this private module for the same reason [`NESTING_LIMIT`] is: the proc-macro +/// agreement test pins the proc-macro's hand-copied `CHAIN_FACTOR` against this one. +pub const CHAIN_FACTOR: usize = 4; + +/// Whether the tokens just before the current byte can end an expression. +/// +/// The nesting precheck does not need to parse Rust to answer this narrowly: calls and indexes +/// are postfix only after an expression-shaped token, while an opening delimiter after an operator +/// or a separator starts an ordinary grouped, array, or block expression. +#[derive(Clone, Copy, Eq, PartialEq)] +enum Previous { + Other, + Expression, +} + +/// The offset of the token that first takes `text` past what `limit` allows. +/// +/// The scan is a byte walk with counters, which is the point: a recursive measurement of how deeply +/// something recurses would be the same defect wearing the fix's clothes. +/// +/// The cumulative path count covers more than delimiters. A run of prefix operators (`----1`), a +/// chain of same-precedence binary operators (`a || b || c || …`), an `else if` ladder, and a +/// postfix chain (`f()()…` or `x[0][0]…`) each build a tree one level deeper per token while opening +/// no bracket that remains open — and `syn`'s parser, the collector's visitor and the `Box` +/// drop all descend that tree once per level. Counting delimiters alone admits those files and +/// lets them overflow the stack, which on Linux is a `SIGSEGV` naming neither the file nor the +/// stage. +/// +/// The path inherits the cost accumulated by every enclosing expression and is reset to that +/// inherited base by `;` and `,`, since sibling expressions do not lie on one AST path. +/// +/// Comments and literals are stepped over rather than counted. A bracket inside a string or a +/// comment is text, and counting it would refuse a file for the shape of its documentation — a +/// module doc drawing a diagram out of brackets is not a nesting hazard. `comments` must be the +/// spans this module's own scanner found, in source order; it skips literals exactly as this walk +/// does, so the two agree on where a comment begins. +#[expect( + clippy::too_many_lines, + reason = "delimiter and expression-path state must advance together through one byte walk" +)] +pub(super) fn beyond(text: &str, comments: &[Comment], limit: usize) -> Option { + let bytes = text.as_bytes(); + let path_limit = limit.saturating_mul(CHAIN_FACTOR); + let mut depth: usize = 0; + let mut path: Vec = vec![0]; + let mut base: Vec = vec![0]; + let mut peak: Vec = vec![0]; + let mut ladders: Vec = vec![0]; + let mut awaiting_else: Vec = vec![false]; + let mut previous = Previous::Other; + let mut at = 0; + let mut next = 0; + + while at < bytes.len() { + if let Some(comment) = comments.get(next) + && comment.span.start <= at + { + at = at.max(comment.span.end); + next += 1; + continue; + } + + if let Some(end) = literal_end(text, at) { + if previous == Previous::Expression { + let inherited = base.get(depth).copied().unwrap_or_default(); + set(&mut path, depth, inherited); + } + at = end; + previous = Previous::Expression; + continue; + } + + if let Some(end) = identifier_end(text, at) { + let identifier = text.get(at..end).unwrap_or(""); + + if identifier == "else" && awaiting_else.get(depth).copied().unwrap_or(false) { + grow(&mut ladders, depth); + ladders[depth] += 1; + set_bool(&mut awaiting_else, depth, false); + + if ladders[depth] > path_limit || link(&mut path, &mut peak, depth, path_limit) { + return Some(at); + } + previous = Previous::Other; + } else if identifier == "as" { + set_bool(&mut awaiting_else, depth, false); + previous = Previous::Other; + + if link(&mut path, &mut peak, depth, path_limit) { + return Some(at); + } + } else { + if awaiting_else.get(depth).copied().unwrap_or(false) { + set(&mut ladders, depth, 0); + set_bool(&mut awaiting_else, depth, false); + } + if previous == Previous::Expression { + let inherited = base.get(depth).copied().unwrap_or_default(); + set(&mut path, depth, inherited); + } + previous = Previous::Expression; + } + + at = end; + continue; + } + + let byte = bytes[at]; + + // Counting every operator byte is conservative for multi-byte binary tokens and exact for + // prefix runs, where each punctuator contributes another unary expression node. + if is_operator(byte) { + if link(&mut path, &mut peak, depth, path_limit) { + return Some(at); + } + + previous = if matches!(byte, b'?' | b'>') { + Previous::Expression + } else { + Previous::Other + }; + at += 1; + continue; + } + + match byte { + b'(' | b'[' => { + let is_postfix = previous == Previous::Expression; + + if is_postfix && link(&mut path, &mut peak, depth, path_limit) { + return Some(at); + } + + let inherited = path.get(depth).copied().unwrap_or_default().saturating_add(CHAIN_FACTOR); + depth += 1; + + if inherited > path_limit { + return Some(at); + } + + set(&mut path, depth, inherited); + set(&mut base, depth, inherited); + set(&mut peak, depth, inherited); + set(&mut ladders, depth, 0); + set_bool(&mut awaiting_else, depth, false); + previous = Previous::Other; + } + + b'{' => { + let inherited = base.get(depth).copied().unwrap_or_default().saturating_add(CHAIN_FACTOR); + depth += 1; + + if inherited > path_limit { + return Some(at); + } + + set(&mut path, depth, inherited); + set(&mut base, depth, inherited); + set(&mut peak, depth, inherited); + set(&mut ladders, depth, 0); + set_bool(&mut awaiting_else, depth, false); + previous = Previous::Other; + } + + // An unbalanced closer means the file is not Rust, which the parser reports far better + // than a depth counter could. Saturating keeps the count from wrapping around into a + // depth that would reject every delimiter after it. + b')' | b']' => { + let nested = peak.get(depth).copied().unwrap_or_default(); + depth = depth.saturating_sub(1); + raise(&mut path, depth, nested); + raise(&mut peak, depth, nested); + previous = Previous::Expression; + } + + b'}' => { + let nested = peak.get(depth).copied().unwrap_or_default(); + depth = depth.saturating_sub(1); + raise(&mut path, depth, nested); + raise(&mut peak, depth, nested); + set_bool(&mut awaiting_else, depth, true); + previous = Previous::Expression; + } + + // A chain cannot span a separator, so the count starts again after one. + b';' | b',' => { + let inherited = base.get(depth).copied().unwrap_or_default(); + set(&mut path, depth, inherited); + set(&mut ladders, depth, 0); + set_bool(&mut awaiting_else, depth, false); + previous = Previous::Other; + } + + b':' | b'#' | b'@' | b'$' | b'\'' => { + previous = Previous::Other; + } + + byte if byte.is_ascii_whitespace() => {} + + byte if byte.is_ascii() => { + if previous == Previous::Expression { + let inherited = base.get(depth).copied().unwrap_or_default(); + set(&mut path, depth, inherited); + } + previous = Previous::Expression; + } + + _ => { + at += text.get(at..).and_then(|rest| rest.chars().next()).map_or(1, char::len_utf8); + continue; + } + } + + at += 1; + } + + None +} + +/// Counts one more link at `depth`, returning whether that took the chain past `limit`. +fn link(path: &mut Vec, peak: &mut Vec, depth: usize, limit: usize) -> bool { + grow(path, depth); + grow(peak, depth); + + let links = path.get_mut(depth).expect("grow(path, depth) made `depth` a valid index"); + + *links += 1; + peak[depth] = peak[depth].max(*links); + + *links > limit +} + +/// Sets the path cost at `depth`. +fn set(path: &mut Vec, depth: usize, value: usize) { + grow(path, depth); + path[depth] = value; +} + +/// Retains the deeper of the current and completed nested paths. +fn raise(path: &mut Vec, depth: usize, value: usize) { + grow(path, depth); + path[depth] = path[depth].max(value); +} + +/// Makes `depth` a valid index, which an unbalanced closer can otherwise leave it short of. +fn grow(chain: &mut Vec, depth: usize) { + while chain.len() <= depth { + chain.push(0); + } +} + +fn set_bool(values: &mut Vec, depth: usize, value: bool) { + while values.len() <= depth { + values.push(false); + } + values[depth] = value; +} + +/// Whether a byte can be part of an operator token. +/// +/// `.` is here because a field access and a method call nest their receiver exactly as a binary +/// operator nests its left operand, so a chain of forty calls is forty levels deep. `:` is not, +/// because a path is flat and `a::b::c::d` says nothing about depth. +const fn is_operator(byte: u8) -> bool { + matches!( + byte, + b'+' | b'-' | b'*' | b'/' | b'%' | b'&' | b'|' | b'^' | b'!' | b'<' | b'>' | b'=' | b'?' | b'.' + ) +} + +/// The end of the Rust identifier beginning at `at`. +fn identifier_end(text: &str, at: usize) -> Option { + let mut characters = text.get(at..)?.char_indices(); + let (_, first) = characters.next()?; + + if !rustc_lexer::is_id_start(first) { + return None; + } + + Some( + characters + .take_while(|(_, character)| rustc_lexer::is_id_continue(*character)) + .last() + .map_or_else(|| at + first.len_utf8(), |(offset, character)| at + offset + character.len_utf8()), + ) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use camino::Utf8Path; + use walkdir::WalkDir; + + use super::super::comment::scan_comments; + use super::super::source_file::line_starts; + use super::*; + + fn beyond_limit(text: &str, limit: usize) -> Option { + let comments = scan_comments(text, &line_starts(text)); + + beyond(text, &comments, limit) + } + + #[test] + fn a_file_within_the_limit_reports_nothing() { + assert_eq!(beyond_limit("fn f() -> i32 { (((1))) }\n", 4), None); + } + + #[test] + fn the_reported_offset_is_the_delimiter_that_crossed_the_limit() { + let text = "fn f() -> i32 { ((1)) }\n"; + + // `{`, `(`, `(` is three levels, so a limit of two is crossed by the second parenthesis. + assert_eq!(beyond_limit(text, 2), Some(text.find("((").expect("the fixture nests") + 1)); + } + + /// Brackets in prose and in strings are text, not structure. + /// + /// A doc comment drawing a tree out of brackets, or a parser test holding a string of them, is + /// ordinary and must not be refused: the depth that matters is the one the tree has, and + /// neither of those reaches the tree at all. + #[test] + fn brackets_inside_comments_and_literals_are_not_nesting() { + let commented = format!("// {}\nfn f() {{}}\n", "(".repeat(500)); + let documented = format!("/* {} */\nfn f() {{}}\n", "[".repeat(500)); + let quoted = format!("fn f() -> &'static str {{ \"{}\" }}\n", "{".repeat(500)); + let raw = format!("fn f() -> &'static str {{ r#\"{}\"# }}\n", "(".repeat(500)); + + for text in [commented, documented, quoted, raw] { + assert_eq!(beyond_limit(&text, 8), None, "for {text:.40}"); + } + } + + /// Nesting is depth, not count: a file of a thousand sibling constructs nests one level. + #[test] + fn siblings_do_not_accumulate_depth() { + let text = "fn f() {}\n".repeat(1_000); + + assert_eq!(beyond_limit(&text, 2), None); + } + + /// A closer with nothing open cannot lower the count below zero, and cannot make the + /// delimiters after it look deeper than they are. + #[test] + fn an_unbalanced_closer_does_not_wrap_the_count() { + assert_eq!(beyond_limit(")))))fn f() { (1) }\n", 4), None); + } + + #[test] + fn blocks_cross_the_limit_without_needing_parentheses() { + assert_eq!(beyond_limit("{{", 1), Some(1)); + } + + /// A run of prefix operators nests one level per operator while opening no delimiter. + /// + /// `syn` recurses once per operator to parse it, the visitor recurses once per operator to walk + /// it, and the `Box` chain recurses once per operator to drop it. Counted by delimiters + /// alone the file looks trivial, and a few hundred of them segfault a discovery worker with no + /// diagnostic at all. + #[test] + fn a_run_of_prefix_operators_is_nesting() { + let text = format!("fn f() -> i32 {{ {}1 }}\n", "-".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "a stack of unary operators must be refused"); + } + + /// The same run written with spaces between the operators is the same tree. + #[test] + fn a_spaced_run_of_prefix_operators_is_still_nesting() { + let text = format!("fn f() -> i32 {{ {}1 }}\n", "- ".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "whitespace does not flatten the tree"); + } + + /// A same-precedence binary chain is linearly nested, however flat it looks. + #[test] + fn a_long_binary_chain_is_nesting() { + let text = format!("fn f(a: bool) -> bool {{ a{} }}\n", " || a".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "a chain of operators must be refused"); + } + + #[test] + fn a_long_postfix_call_chain_is_nesting() { + let text = format!("fn f() {{ g{}; }}\n", "()".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "a chain of calls must be refused"); + } + + #[test] + fn a_long_postfix_index_chain_is_nesting() { + let text = format!("fn f() {{ value{}; }}\n", "[0]".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "a chain of indexes must be refused"); + } + + #[test] + fn a_long_postfix_method_chain_is_nesting() { + let text = format!("fn f() {{ value{}; }}\n", ".call()".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "a chain of method calls must be refused"); + } + + #[test] + fn mixed_method_and_index_links_accumulate_on_one_path() { + let text = format!("fn f() {{ value{}; }}\n", ".call()[0]".repeat(20)); + + assert!(beyond_limit(&text, 8).is_some(), "interleaved postfix links must be refused"); + } + + #[test] + fn operator_chains_inside_nested_delimiters_accumulate_on_one_path() { + let mut expression = "value".to_owned(); + + for _ in 0..8 { + expression = format!("({expression} + value + value + value)"); + } + + assert!( + beyond_limit(&format!("fn f() {{ {expression}; }}\n"), 8).is_some(), + "nested operator chains must be refused" + ); + } + + #[test] + fn a_long_cast_chain_is_nesting() { + let text = format!("fn f() {{ 1{}; }}\n", " as u64".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "a chain of casts must be refused"); + } + + /// An `else if` ladder de-recurses in the parser but not in the visitor or the drop. + #[test] + fn a_long_else_if_ladder_is_nesting() { + let text = format!("fn f(a: bool) {{ if a {{}} {} }}\n", "else if a {} ".repeat(64)); + + assert!(beyond_limit(&text, 8).is_some(), "an else-if ladder must be refused"); + } + + /// Ordinary code is not a chain: a block of separate statements starts the count again at each. + /// + /// Without the reset, any function long enough would be refused for being long rather than for + /// being deep, which costs a file's coverage and explains nothing. + #[test] + fn statements_in_sequence_are_not_a_chain() { + let body = "let x = 1 + 2 * 3; ".repeat(200); + let text = format!("fn f() {{ {body} }}\n"); + + assert_eq!(beyond_limit(&text, 8), None); + } + + /// Nor are the elements of a long literal, which are separated by commas. + #[test] + fn a_long_list_of_elements_is_not_a_chain() { + let elements = "1 + 2, ".repeat(200); + let text = format!("fn f() {{ let xs = [{elements}]; }}\n"); + + assert_eq!(beyond_limit(&text, 8), None); + } + + /// Operators inside comments and strings are text here exactly as brackets are. + #[test] + fn operators_inside_comments_and_literals_are_not_nesting() { + let commented = format!("// {}\nfn f() {{}}\n", "-".repeat(500)); + let quoted = format!("fn f() -> &'static str {{ \"{}\" }}\n", "|".repeat(500)); + + for text in [commented, quoted] { + assert_eq!(beyond_limit(&text, 8), None, "for {text:.40}"); + } + } + + #[test] + fn chain_keywords_inside_unicode_identifiers_are_not_counted() { + assert_eq!(beyond_limit("{ let caféelse; }\n", 1), None); + assert_eq!(beyond_limit("{ let µas; }\n", 1), None); + } + + #[test] + fn non_ascii_non_identifier_bytes_are_skipped_without_affecting_depth() { + assert_eq!(beyond_limit("{ 🦀 }\n", 1), None); + } + + /// Every source file in this workspace passes its own guard, at the limit the run uses. + /// + /// The counts added here are proxies, and a proxy that refuses ordinary Rust is worse than the + /// crash it prevents. This is the check that keeps them calibrated against real code. + #[test] + + fn this_workspace_is_within_the_limit() { + let root = Utf8Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); + + for entry in WalkDir::new(root.as_std_path()).into_iter().filter_map(Result::ok) { + let path = entry.path(); + + if path.extension() != Some("rs".as_ref()) { + continue; + } + + let actual = fs::read_to_string(path).ok().and_then(|text| beyond_limit(&text, NESTING_LIMIT)); + let message = format!("{} is refused by its own guard", path.display()); + + assert_eq!(actual, None, "{message}"); + } + } +} diff --git a/crates/cargo-gamma-engine/src/parse/source_file.rs b/crates/cargo-gamma-engine/src/parse/source_file.rs new file mode 100644 index 000000000..582bdc3d5 --- /dev/null +++ b/crates/cargo-gamma-engine/src/parse/source_file.rs @@ -0,0 +1,438 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Parsed source text and byte-offset navigation over it. + +use core::ops::Range; +use std::fs; + +use camino::{Utf8Path, Utf8PathBuf}; +use syn::File; + +use super::comment::{self, Comment}; +use super::nesting; +use crate::Result; +use crate::error::error; + +/// A parsed source file, with everything downstream stages need to work in byte offsets. +#[derive(Debug)] +pub struct SourceFile { + /// Path as it should appear in reports, relative to the workspace root where possible. + pub path: Utf8PathBuf, + + /// The exact bytes that were parsed. All spans index into this. + pub text: String, + + /// The syntax tree. + pub ast: File, + + /// Byte offset of the start of each line. + lines: Vec, + + /// Every comment in the file, in source order. + pub comments: Vec, +} + +/// Whether source text is too deeply nested to hand to a recursive parser. +#[doc(hidden)] +#[must_use] +pub fn exceeds_nesting_limit(text: &str) -> bool { + let lines = line_starts(text); + let comments = comment::scan_comments(text, &lines); + + nesting::beyond(text, &comments, nesting::NESTING_LIMIT).is_some() +} + +impl SourceFile { + /// Parses source text that has already been read. + /// + /// The path is used only for diagnostics and reporting; nothing is read from disk here, which + /// is what lets every test in this crate work on string literals. + /// + /// # Errors + /// + /// Returns an error if the text is not Rust, or if it nests deeper than + /// [`NESTING_LIMIT`](nesting::NESTING_LIMIT) — see that constant for why a file can be too + /// deep to look at. + pub fn parse(path: impl Into, text: String) -> Result { + let path = path.into(); + let text = without_bom(text); + let lines = line_starts(&text); + let comments = comment::scan_comments(&text, &lines); + + // Before `syn`, deliberately. The parser is the first of the recursive descents over this + // text and the guard is worth nothing after one of them has already run out of stack. + if let Some(at) = nesting::beyond(&text, &comments, nesting::NESTING_LIMIT) { + let line = lines.partition_point(|start| *start <= at); + + // Skippable, because this is the one parse failure that says nothing about whether the + // workspace is sound. A file that does not parse does not compile either, so refusing + // the run tells the user something they were about to find out anyway; a file nested + // past this limit is one `rustc` builds happily and only this tool cannot walk. Killing + // the whole run over it would make a valid workspace unmeasurable, which is a worse + // answer than measuring the rest of it and naming what was left out. + return Err(error!( + "{path}:{line}: nests deeper than {} levels of brackets, prefix operators, chained operators or postfix expressions", + nesting::NESTING_LIMIT + ) + .skippable()); + } + + let ast = syn::parse_file(&text).map_err(|cause| { + let start = cause.span().start(); + + error!("{path}:{}:{}: could not parse: {cause}", start.line, start.column) + })?; + + Ok(Self { + path, + text, + ast, + lines, + comments, + }) + } + + /// Reads and parses a file from disk. + pub fn read(path: impl AsRef) -> Result { + let path = path.as_ref(); + + let text = fs::read_to_string(path).map_err(|cause| error!("could not read `{path}`").caused_by(cause))?; + + Self::parse(path.to_owned(), text) + } + + /// Returns the 1-based line and column of a byte offset. + /// + /// The column is counted in characters rather than bytes, because it is shown to humans beside + /// a rendering of the line, and a byte column would point at the wrong place in any line + /// containing non-ASCII text. + #[must_use] + pub fn location(&self, offset: usize) -> (usize, usize) { + let line_index = match self.lines.binary_search(&offset) { + Ok(exact) => exact, + Err(insertion) => insertion.saturating_sub(1), + }; + + let line_start = self.lines.get(line_index).copied().unwrap_or(0); + let clamped = offset.min(self.text.len()); + let column = self.text.get(line_start..clamped).map_or(0, |s| s.chars().count()); + + (line_index + 1, column + 1) + } + + /// Returns the 1-based line number of a byte offset. + #[must_use] + pub fn line_of(&self, offset: usize) -> usize { + self.location(offset).0 + } + + /// Returns the source text covered by a byte range. + #[must_use] + pub fn slice(&self, span: &Range) -> &str { + self.text.get(span.start..span.end).unwrap_or("") + } +} + +/// Drops a leading byte-order mark. +/// +/// `syn` skips the mark and then counts byte offsets from zero, so a file that has one hands back +/// spans three bytes short of where the construct really is: the recorded original text is the +/// wrong slice, the reported column is wrong, and instrumenting the file on disk splices guards +/// three bytes off — through the middle of a character, if the file is not ASCII. +/// +/// Removing it here, at the one place source text enters, is what makes every offset downstream +/// agree. The mark carries no meaning in Rust source, so nothing is lost; callers that rewrite a +/// user's source retain it separately at the file boundary. +#[must_use] +pub fn without_bom(mut text: String) -> String { + if text.starts_with(BOM) { + let _removed = text.drain(..BOM.len_utf8()); + } + + text +} + +/// The byte-order mark, which is three bytes of UTF-8 and no characters of Rust. +pub const BOM: char = '\u{feff}'; + +/// Removes a leading byte-order mark without allocating. +/// +/// Source parsing owns normalized text, while source editing must retain the original bytes. This +/// lets generation comparisons use the parser's representation without making an edit move or +/// discard the mark. +#[must_use] +pub fn strip_bom(text: &str) -> &str { + text.strip_prefix(BOM).unwrap_or(text) +} + +/// Byte offset of the start of each line, indexing the text as `syn` sees it. +pub(super) fn line_starts(text: &str) -> Vec { + let mut starts = vec![0]; + + starts.extend(text.bytes().enumerate().filter(|(_, b)| *b == b'\n').map(|(i, _)| i + 1)); + + // A trailing newline does not open a line that anything can be on. + if starts.last() == Some(&text.len()) && !text.is_empty() { + let _ = starts.pop(); + } + + starts +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(text: &str) -> SourceFile { + SourceFile::parse("test.rs", text.to_owned()).unwrap() + } + + #[test] + fn a_parse_failure_names_the_file_and_position() { + let error = SourceFile::parse("bad.rs", "fn f( {".to_owned()).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains("bad.rs"), "{message}"); + assert!(message.contains("could not parse"), "{message}"); + } + + /// A file too deep to walk is refused by name, not by dying. + /// + /// Every stage that reads source descends by recursion, and a stack overflow on Linux is a + /// `SIGSEGV`: the process disappears with no diagnostic, no file named, and nothing to + /// distinguish it from a bug in the tool. This depth is past the measured overflow point of + /// the parse-and-collect path on the smallest stack a discovery worker runs on, so the file + /// has to be turned away before `syn` ever sees it. + #[test] + fn a_file_nested_deeper_than_the_limit_is_refused_rather_than_overflowing_the_stack() { + let depth = 4_096; + let text = format!("fn f() -> i32 {{ {}1{} }}\n", "(".repeat(depth), ")".repeat(depth)); + let message = SourceFile::parse("deep.rs", text).unwrap_err().to_string(); + + assert!(message.contains("deep.rs"), "{message}"); + assert!(message.contains("nests deeper"), "{message}"); + } + + #[test] + fn a_file_with_a_deep_postfix_chain_is_refused_before_parsing() { + let chains = [format!("call{}", "()".repeat(4_096)), format!("value{}", "[0]".repeat(4_096))]; + + for expression in chains { + let text = format!("fn f() {{ {expression}; }}\n"); + let message = SourceFile::parse("postfix.rs", text).unwrap_err().to_string(); + + assert!(message.contains("postfix.rs"), "{message}"); + assert!(message.contains("nests deeper"), "{message}"); + } + } + + /// Nesting a human would write is still analyzed, which is the other half of the bargain. + #[test] + fn a_file_within_the_nesting_limit_still_parses() { + let depth = 20; + let text = format!("fn f() -> i32 {{ {}1{} }}\n", "(".repeat(depth), ")".repeat(depth)); + + let parsed = SourceFile::parse("deep_enough.rs", text).expect("nesting a human writes is analyzed"); + + assert_eq!(parsed.path, "deep_enough.rs"); + } + + #[test] + + fn read_loads_and_parses_a_file_from_disk() { + let path = Utf8Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/src/parse/source_file.rs")); + let file = SourceFile::read(path).unwrap(); + + assert_eq!(file.path, path); + assert!(file.text.contains("pub struct SourceFile")); + } + + #[test] + + fn read_failures_name_the_missing_file() { + let error = SourceFile::read(Utf8Path::new("target/does-not-exist/source.rs")).unwrap_err(); + + assert!(error.to_string().contains("could not read")); + } + + #[test] + fn locations_are_one_based() { + let file = parse("fn a() {}\nfn b() {}\n"); + + assert_eq!(file.location(0), (1, 1)); + assert_eq!(file.location(3), (1, 4)); + assert_eq!(file.location(10), (2, 1)); + } + + #[test] + fn columns_count_characters_not_bytes() { + let file = parse("fn f() { let s = \"éé\"; let _ = s; }\n"); + let offset = file.text.find("let _").unwrap(); + let (line, column) = file.location(offset); + + assert_eq!(line, 1); + assert_eq!(file.text.get(..offset).unwrap().chars().count() + 1, column); + } + + /// Walks the text once, keeping line and column by hand, and reports what every boundary + /// offset should map to. + /// + /// Deliberately shares nothing with `location`: no binary search, no `lines` table, no + /// `chars().count()` over a slice. `columns_count_characters_not_bytes` computes its + /// expectation the way the implementation does, on line one where the line start is zero, so + /// an implementation that dropped the line start entirely would still satisfy it. This one + /// would not. + /// + /// The final offset is excluded because a text ending in a newline has no line there — + /// `line_starts` pops it — and there is nothing for an independent oracle to agree with. + fn walked(text: &str) -> Vec<(usize, (usize, usize))> { + let (mut line, mut column) = (1, 1); + let mut expected = Vec::new(); + + for (offset, character) in text.char_indices() { + expected.push((offset, (line, column))); + + if character == '\n' { + line += 1; + column = 1; + } else { + column += 1; + } + } + + expected + } + + #[test] + fn every_boundary_offset_lands_where_walking_the_text_says_it_should() { + for text in [ + "", + "\n", + "fn a() {}\n", + "fn a() {}\nfn b() {}\n", + "fn a() {}\r\nfn b() {}\r\n", + "fn é() { let ß = 1; }\nfn b() {}\n", + "\u{feff}fn a() {}\n", + "\u{feff}", + "\n\n\nfn a() {}", + "fn a() {}", + ] { + let file = parse(text); + + // The file's own text rather than the literal, because a byte-order mark is dropped on + // the way in — which is the point of the mark case being here. + for (offset, expected) in walked(&file.text) { + assert_eq!(file.location(offset), expected, "offset {offset} of {text:?} is misplaced"); + } + } + } + + /// A byte-order mark is dropped, so that spans mean what `syn` meant by them. + /// + /// Regression: `syn` skips a leading mark and then counts byte offsets from zero, so on a + /// file that had one every mutant's span was three bytes short of the construct. The recorded + /// original text was the wrong slice, the reported column was wrong, and instrumenting the + /// file on disk spliced guards three bytes off — through the middle of a character, on a file + /// that was not ASCII. + #[test] + fn a_byte_order_mark_is_dropped_so_that_offsets_mean_what_syn_meant() { + let file = parse("\u{feff}fn a() {}\n"); + + assert_eq!(file.text, "fn a() {}\n"); + assert_eq!(file.location(0), (1, 1)); + + // A mark anywhere but the front is ordinary text and is left alone. + let inner = parse("fn a() { let _ = \"\u{feff}\"; }\n"); + + assert!(inner.text.contains('\u{feff}')); + } + + #[test] + fn strip_bom_removes_only_a_leading_mark() { + assert_eq!(strip_bom("\u{feff}fn a() {}\n"), "fn a() {}\n"); + assert_eq!(strip_bom("fn a() {}\n"), "fn a() {}\n"); + assert_eq!( + strip_bom("fn a() { let _ = \"\u{feff}\"; }\n"), + "fn a() { let _ = \"\u{feff}\"; }\n" + ); + } + + #[test] + fn a_trailing_newline_does_not_open_a_line() { + assert_eq!(parse("fn a() {}\n").lines.len(), 1); + assert_eq!(parse("fn a() {}\nfn b() {}\n").lines.len(), 2); + } +} + +#[cfg(all(test, not(miri)))] +mod fuzz { + use super::{SourceFile, line_starts}; + + /// A file whose navigation tables describe `text`. + /// + /// The syntax tree comes from a fixed stub and is never looked at: `location` reads only the + /// text and the line table. Going through `SourceFile::parse` instead would mean discarding + /// every generated text that is not valid Rust, which is very nearly all of them, and the + /// property would then be checked against almost nothing. + fn navigable(text: &str) -> SourceFile { + let mut file = SourceFile::parse("fuzz.rs", "fn f() {}\n".to_owned()).expect("the stub parses"); + + file.lines = line_starts(text); + text.clone_into(&mut file.text); + + file + } + + /// Arbitrary text maps every boundary offset to a location that is really where it is. + /// + /// Everything downstream of the parser is byte-offset arithmetic over source nobody in this + /// project wrote, and every hand-written fixture here is text somebody chose. The two + /// properties are the ones the rest of the pipeline relies on and that a table of examples can + /// only sample: that the line and column of an offset are what counting from the start of the + /// text gives, and that slicing back to that line and column lands on the same offset. + #[test] + fn locations_over_arbitrary_text_agree_with_counting_from_the_start() { + // Generated as lines rather than as one string: a `String` generator emits a newline + // about as often as any other character, so a flat one would leave the line table barely + // exercised — an implementation that ignored the line start entirely survived that + // version of this test. Each line carries its own terminator so that `\r\n`, which is + // ordinary in real repositories and is *not* a line start of its own, is covered too. + bolero::check!().with_type::>().for_each(|lines| { + let mut text = String::new(); + + for (line, crlf) in lines { + text.push_str(line); + text.push_str(if *crlf { "\r\n" } else { "\n" }); + } + + let text = &text; + let file = navigable(text); + let (mut line, mut column) = (1, 1); + let mut line_start = 0; + + for (offset, character) in text.char_indices() { + let located = file.location(offset); + + assert_eq!(located, (line, column), "offset {offset} of {text:?} is misplaced"); + + // The other direction: the location has to name a real place in the text, so + // taking that many characters from the start of that line must arrive back here. + let back = text + .get(line_start..) + .and_then(|rest| rest.char_indices().nth(located.1 - 1)) + .map(|(index, _)| line_start + index); + + assert_eq!(back, Some(offset), "the column of {offset} in {text:?} does not lead back"); + + if character == '\n' { + line += 1; + column = 1; + line_start = offset + 1; + } else { + column += 1; + } + } + }); + } +} diff --git a/crates/cargo-gamma-engine/src/schema.rs b/crates/cargo-gamma-engine/src/schema.rs new file mode 100644 index 000000000..551944a71 --- /dev/null +++ b/crates/cargo-gamma-engine/src/schema.rs @@ -0,0 +1,835 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Encoding a whole population of mutants into a single compilable source tree. +//! +//! This is the idea the tool is built on. The conventional way to test a mutant is to edit the +//! source, compile, run the suite, and revert — which means one full build per mutant, and a build +//! is far and away the most expensive thing in the loop. Instead, every mutant is compiled into +//! the tree *at once*, each one behind a runtime guard, and a single environment variable picks +//! which one is live. The build happens once; testing a mutant costs a process launch. +//! +//! The construction that makes this possible is called a *mutant schema*, after Untch, Offutt and +//! Harrold, who introduced it in 1993 for Fortran. A guarded site looks like this: +//! +//! ```text +//! original: a < b +//! instrumented: (if ::gamma_rt::a(7u32) { (a) <= (b) } else { a < b }) +//! ``` +//! +//! # Why only the default branch carries nested guards +//! +//! A mutation site can contain others: in `a + b < c` the `<` site contains the `+` site. The +//! obvious encoding instruments both arms of every guard, which makes the output grow as +//! `2^depth` and duplicates whole subtrees. +//! +//! It is also unnecessary. Exactly one mutant is active in a process, so if the `<` mutant is +//! live then no `+` mutant can be, and the `<` arm can hold the plain original text of its +//! operands. Only the `else` arm — the one taken when this site is not the active mutant — needs +//! instrumented children. That makes the encoding linear in the size of the source. +//! +//! The alternative, binding operands to temporaries and sharing them between arms, was rejected: +//! it would defeat the short-circuit of `&&` and `||`, move values that were only borrowed, and +//! change when temporaries are dropped. Duplicating the operand text costs compile time; changing +//! any of those three would change what the tests prove. + +use core::fmt::Write as _; +use core::ops::Range; + +use crate::error::Error; +#[cfg(test)] +use crate::model::MutantDefinition; +use crate::ops::collect::Shape; +use crate::{HashMap, Result}; + +/// The crate path of the guard predicate, as it appears in instrumented source. +pub const GUARD_PATH: &str = "::gamma_rt::a"; + +/// The crate path of the two-variant iterator wrapper, as it appears in instrumented source. +/// +/// Used only by [`Shape::IterBlock`], which is the one shape whose two arms cannot be made to +/// agree on a type without it. +pub const EITHER_PATH: &str = "::gamma_rt::Either"; + +/// Maps each mutant ordinal to where its guard landed in the instrumented text. +/// +/// A guard emits both the mutated text and the original, so a multi-line site grows and every +/// later line shifts. Instrumented text therefore does not line up with its source, and anything +/// attributing a compiler diagnostic to a mutant has to use these positions rather than the +/// mutant's source line. +fn positions(text: &str, spans: &HashMap, Range)>) -> HashMap { + let mut starts: Vec = Vec::with_capacity(text.len() / 32); + + starts.push(0); + starts.extend(text.match_indices('\n').map(|(at, _matched)| at + 1)); + + let at = |offset: usize| -> Position { + let index = starts.partition_point(|start| *start <= offset).saturating_sub(1); + let start = starts.get(index).copied().unwrap_or(0); + let column = text.get(start..offset).map_or(0, |prefix| prefix.chars().count()); + + Position { + line: u32::try_from(index + 1).unwrap_or(u32::MAX), + column: u32::try_from(column + 1).unwrap_or(u32::MAX), + } + }; + + spans + .iter() + .map(|(ordinal, (site, mutated))| { + let guard = Guard { + site: at(site.start)..at(site.end), + mutated: (!mutated.is_empty()).then(|| at(mutated.start)..at(mutated.end)), + }; + + (*ordinal, guard) + }) + .collect() +} + +/// A one-based line and column in instrumented text, ordered as the text reads. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Position { + pub line: u32, + pub column: u32, +} + +/// Where one mutant's guard landed in instrumented text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Guard { + /// The whole guarded site: the `if`, the mutated branch, and the original text. + pub site: Range, + + /// Just the mutated branch — the only text in the tree that is not a copy of the original. + /// + /// Nested guards go exclusively in the `else` branch, so these ranges never overlap between + /// mutants, not even for two mutants of the same site. A compiler diagnostic landing inside + /// one therefore names its cause exactly. A deletion mutant has no replacement text at all, + /// and so has nothing here. + pub mutated: Option>, +} + +/// A node in the containment tree of mutation sites within one file. +#[derive(Debug)] +struct Node<'a> { + span: Range, + shape: Shape, + + /// Every mutant sharing exactly this span, in ordinal order. + mutants: Vec<(u32, &'a str)>, + + /// Sites strictly contained within this one. + children: Vec, +} + +/// A source-level mutant definition paired with the run-local ordinal that selects its guard. +#[derive(Clone, Copy, Debug)] +pub struct AssignedMutant<'a> { + ordinal: u32, + span: &'a Range, + replacement: &'a str, + shape: Shape, +} + +impl<'a> AssignedMutant<'a> { + #[cfg(test)] + #[must_use] + pub(crate) fn new(ordinal: u32, definition: &'a MutantDefinition) -> Self { + Self::from_parts(ordinal, definition.span(), &definition.replacement, definition.shape) + } + + #[doc(hidden)] + #[must_use] + pub const fn from_parts(ordinal: u32, span: &'a Range, replacement: &'a str, shape: Shape) -> Self { + Self { + ordinal, + span, + replacement, + shape, + } + } +} + +/// Rewrites one file so that it encodes every one of the given mutants. +/// +/// Any mutant whose span is not a region of `text` that can be replaced is skipped rather than +/// spliced at a wrong offset, because a bad offset would silently corrupt unrelated code and the +/// resulting failures would be blamed on the test suite. A valid span is nonempty, in bounds, and +/// starts and ends on UTF-8 character boundaries. +/// +/// # Errors +/// +/// Returns an error if two mutants have spans that overlap without nesting, which would make the +/// rewrite ambiguous. +pub fn instrument(text: &str, mutants: &[AssignedMutant<'_>]) -> Result { + instrument_with_guards(text, mutants).map(|(out, _guards)| out) +} + +/// Rewrites one file as [`instrument`] does, also reporting where each guard landed. +/// +/// The positions are one-based and refer to the returned text, not to `text`. +/// +/// # Errors +/// +/// Returns an error if two mutants have spans that overlap without nesting. +pub fn instrument_with_guards(text: &str, mutants: &[AssignedMutant<'_>]) -> Result<(String, HashMap)> { + let mut sites: Vec<&AssignedMutant<'_>> = mutants.iter().filter(|mutant| spliceable(text, mutant.span)).collect(); + + // Outermost first, and within one span the lowest ordinal first, so that the tree builds by a + // single stack walk and guard order is deterministic. + sites.sort_by(|left, right| { + left.span + .start + .cmp(&right.span.start) + .then_with(|| right.span.end.cmp(&left.span.end)) + .then_with(|| left.ordinal.cmp(&right.ordinal)) + }); + + let roots = build_tree(&sites)?; + let mut out = String::with_capacity(text.len() + text.len() / 4); + let mut spans = HashMap::default(); + let mut cursor = 0; + + for root in &roots { + out.push_str(copy(text, cursor..root.span.start)); + render(text, root, &mut out, &mut spans); + cursor = root.span.end; + } + + out.push_str(copy(text, cursor..text.len())); + + let guards = positions(&out, &spans); + + Ok((out, guards)) +} + +/// Whether a span names a region of `text` that can be replaced. +/// +/// Length is not the whole of it. A span whose endpoint falls inside a character is within the +/// text and still cannot be sliced, and the failure that follows is the worst of the three +/// available: not an error, not a splice at a wrong offset, but the silent disappearance of every +/// region the bad endpoint bounds. Nothing downstream can catch that — the guard the mutant asked +/// for is emitted correctly, so the invariant check over the guards passes, and only the code +/// around them is gone. +/// +/// Within one parse this cannot happen, because every span is a token span and token spans fall on +/// character boundaries. The spans and the text come from two independent reads of the file, so +/// "within one parse" is an assumption rather than a fact. +const fn spliceable(text: &str, span: &Range) -> bool { + span.start < span.end && span.end <= text.len() && text.is_char_boundary(span.start) && text.is_char_boundary(span.end) +} + +/// The text a range covers, for a range built from endpoints that have already been vetted. +/// +/// Every caller composes its range from the endpoints of spans that passed [`spliceable`], in an +/// order the containment tree fixes: a parent's start precedes its first child's, siblings do not +/// overlap, and each range ends no later than the enclosing node. So the slice exists, and saying +/// so out loud is the point — the alternative spelling substitutes an empty string for a range +/// that does not, which is how a region of the file goes missing without anyone being told. +fn copy(text: &str, range: Range) -> &str { + text.get(range) + .expect("every range here is built from vetted endpoints in containment order") +} + +/// Groups sites into a forest ordered by containment. +fn build_tree<'a>(sites: &[&'a AssignedMutant<'a>]) -> Result>> { + let mut roots: Vec> = Vec::new(); + let mut stack: Vec> = Vec::new(); + + for mutant in sites { + // Close every open node this site is not inside. + while let Some(top) = stack.last() { + if mutant.span.start < top.span.end { + break; + } + + let Some(finished) = stack.pop() else { break }; + + attach(finished, &mut stack, &mut roots); + } + + if let Some(top) = stack.last_mut() { + // Two mutants share a node only when they mutate the same text in the same way. Equal + // spans with different shapes would be spliced with the wrong wrapper — an expression + // guard around a statement, say — so they are kept apart and nested instead. + if top.span == *mutant.span && top.shape == mutant.shape { + top.mutants.push((mutant.ordinal, mutant.replacement)); + continue; + } + + // Partial overlap. A single parse cannot produce this, so it means spans from two + // different parses were mixed, and splicing either one would corrupt the other. + if mutant.span.end > top.span.end { + return Err(Error::new(format!( + "mutation sites {:?} and {:?} overlap without nesting", + top.span, mutant.span + ))); + } + } + + stack.push(Node { + span: (*mutant.span).clone(), + shape: mutant.shape, + mutants: vec![(mutant.ordinal, mutant.replacement)], + children: Vec::new(), + }); + } + + while let Some(finished) = stack.pop() { + attach(finished, &mut stack, &mut roots); + } + + roots.sort_by_key(|node| node.span.start); + + Ok(roots) +} + +/// Files a finished node under its parent, or under the roots when it has none. +fn attach<'a>(node: Node<'a>, stack: &mut [Node<'a>], roots: &mut Vec>) { + if let Some(parent) = stack.last_mut() { + parent.children.push(node); + } else { + roots.push(node); + } +} + +/// Writes the instrumented form of one node, recording the output each guard covers. +fn render(text: &str, node: &Node<'_>, out: &mut String, spans: &mut HashMap, Range)>) { + let start = out.len(); + let mut mutated = Vec::with_capacity(node.mutants.len()); + + for (ordinal, replacement) in &node.mutants { + match node.shape { + Shape::Expr | Shape::Block | Shape::IterBlock => { + let opening = if node.shape == Shape::Expr { "(" } else { "{ " }; + + let _ = write!(out, "{opening}if {GUARD_PATH}({ordinal}u32) {{ "); + + // The mutant is the left variant and the original the right, which is what makes + // the two arms one type. See `Shape::IterBlock`. + if node.shape == Shape::IterBlock { + let _ = write!(out, "{EITHER_PATH}::L("); + } + + let from = out.len(); + + out.push_str(replacement); + mutated.push((*ordinal, from..out.len())); + + if node.shape == Shape::IterBlock { + out.push(')'); + } + + out.push_str(" } else { "); + + // Every `else` arm is wrapped, not just the one holding the original text. An + // outer arm holds the next guard down, whose type is an `Either` of its own, and + // the two arms only agree once that is the right-hand side of this one. + if node.shape == Shape::IterBlock { + let _ = write!(out, "{EITHER_PATH}::R("); + } + } + Shape::Continue | Shape::Break => { + let _ = write!(out, "{{ if {GUARD_PATH}({ordinal}u32) {{ "); + let from = out.len(); + + out.push_str(replacement); + mutated.push((*ordinal, from..out.len())); + out.push_str("; } "); + } + Shape::Stmt => { + let _ = write!(out, "if !{GUARD_PATH}({ordinal}u32) {{ "); + mutated.push((*ordinal, out.len()..out.len())); + } + // Written after the pattern rather than before it, so nothing is emitted here. + Shape::Arm => {} + } + } + + // The innermost thing is the original text with this node's children instrumented in place. + let mut cursor = node.span.start; + + for child in &node.children { + out.push_str(copy(text, cursor..child.span.start)); + render(text, child, out, spans); + cursor = child.span.end; + } + + out.push_str(copy(text, cursor..node.span.end)); + + // An arm is disabled by a guard trailing its pattern. Several mutants on one arm chain with + // `&&`, which is correct however many there are, though only one can ever be active at once. + if node.shape == Shape::Arm { + for (index, (ordinal, _replacement)) in node.mutants.iter().enumerate() { + let joiner = if index == 0 { " if" } else { " &&" }; + + let _ = write!(out, "{joiner} !{GUARD_PATH}({ordinal}u32)"); + mutated.push((*ordinal, out.len()..out.len())); + } + } + + let close = match node.shape { + Shape::Expr => " })", + Shape::Block => " } }", + // The extra `)` closes the `Either::R` this arm was opened with. + Shape::IterBlock => ") } }", + Shape::Continue | Shape::Break | Shape::Stmt => " }", + // The guard is the whole of the change, and it is already written. + Shape::Arm => "", + }; + + for _ in 0..node.mutants.len() { + out.push_str(close); + } + + let end = out.len(); + + for (ordinal, region) in mutated { + let _ = spans.insert(ordinal, (start..end, region)); + } +} + +#[cfg(test)] +mod tests { + use core::ops::Range; + + use super::{AssignedMutant, Guard, Shape}; + use crate::{HashMap, Result}; + + #[derive(Debug)] + struct Mutant { + ordinal: u32, + span: Range, + replacement: String, + shape: Shape, + } + + fn instrument(text: &str, mutants: &[&Mutant]) -> Result { + let assigned = assigned_mutants(mutants); + + super::instrument(text, &assigned) + } + + fn instrument_with_guards(text: &str, mutants: &[&Mutant]) -> Result<(String, HashMap)> { + let assigned = assigned_mutants(mutants); + + super::instrument_with_guards(text, &assigned) + } + + fn assigned_mutants<'a>(mutants: &[&'a Mutant]) -> Vec> { + mutants + .iter() + .map(|mutant| AssignedMutant::from_parts(mutant.ordinal, &mutant.span, &mutant.replacement, mutant.shape)) + .collect() + } + + /// Every `else` arm of an `IterBlock` must be wrapped, not only the one holding the original. + /// + /// This is the invariant that makes the shape work, and it is not obvious. Wrapping just the + /// innermost arm looks right and compiles for a single mutant, but a second mutant on the same + /// site nests a whole `Either` into the outer `else`, and the outer `if` then has + /// `Either, _>` against `Either, _>` — two types, so the build fails and both + /// mutants are withdrawn as unviable. Nothing else in the suite would notice, because the + /// single-mutant case stays green. + #[test] + fn every_else_arm_of_an_iterator_body_is_wrapped_so_that_two_mutants_still_agree_on_a_type() { + let text = "fn f() -> impl Iterator { 0..10 }\n"; + let empty = mutant(36..45, 1, "core::iter::empty()", Shape::IterBlock); + let once = mutant(36..45, 2, "core::iter::once(0)", Shape::IterBlock); + + let out = instrument(text, &[&empty, &once]).expect("instrumented"); + + assert_eq!( + out.matches("::gamma_rt::Either::R(").count(), + 2, + "one wrapper per guard, not one for the whole site: {out}" + ); + assert_eq!(out.matches("::gamma_rt::Either::L(").count(), 2, "{out}"); + + // The original body is the innermost thing, still intact inside the wrappers. + assert!(out.contains("::gamma_rt::Either::R({ 0..10 })"), "{out}"); + } + + /// A body wrapped this way has to parse, and braces are easy to get wrong by one. + #[test] + fn an_instrumented_iterator_body_is_still_balanced_rust() { + let text = "fn f() -> impl Iterator { 0..10 }\n"; + let only = mutant(36..45, 1, "core::iter::empty()", Shape::IterBlock); + + let out = instrument(text, &[&only]).expect("instrumented"); + + let _parsed = parse_file(&out).expect("the instrumented form must parse"); + } + + #[test] + fn guards_report_the_lines_they_actually_landed_on() { + let text = "fn a() {}\nlet x = 1;\nv.push(1);\n"; + let first = mutant(18..19, 3, "0", Shape::Expr); + let second = mutant(21..31, 9, "", Shape::Stmt); + + let (_out, guards) = instrument_with_guards(text, &[&first, &second]).expect("instrumented"); + + assert_eq!(guards.get(&3).map(|guard| guard.site.start.line), Some(2)); + assert_eq!(guards.get(&9).map(|guard| guard.site.start.line), Some(3)); + assert_eq!(guards.len(), 2); + } + + #[test] + fn a_guard_reports_the_whole_range_it_spans() { + let text = "fn f(a: i32, b: i32) -> bool {\n a\n < b\n}\n"; + let only = mutant(35..48, 1, "(a)\n <= (b)", Shape::Expr); + + let (out, guards) = instrument_with_guards(text, &[&only]).expect("instrumented"); + let span = guards.get(&1).expect("recorded").clone(); + + assert!(out.lines().count() > text.lines().count(), "the site should have grown"); + assert!( + span.site.end.line > span.site.start.line, + "a multi-line site should span multiple lines" + ); + } + + #[test] + fn a_nested_guard_lies_outside_its_enclosing_guards_mutated_branch() { + // This is what lets a compile error be attributed exactly. An enclosing mutant that + // replaces a whole function body is far likelier to break the build than the literal + // nested inside it, so the two must be distinguishable: the enclosing mutant's own text + // has to be disjoint from where the innocent nested guard sits. + let text = "fn f() -> i32 {\n 0\n}\n"; + let outer = mutant(14..23, 1, "Default::default()", Shape::Block); + let inner = mutant(20..21, 2, "1", Shape::Expr); + + let (_out, guards) = instrument_with_guards(text, &[&outer, &inner]).expect("instrumented"); + let outer = guards.get(&1).expect("outer recorded").clone(); + let inner = guards.get(&2).expect("inner recorded").clone(); + + let replacement = outer.mutated.expect("the outer mutant has replacement text"); + + assert!( + outer.site.start <= inner.site.start && outer.site.end >= inner.site.end, + "the outer site should enclose" + ); + assert!( + replacement.end <= inner.site.start || replacement.start >= inner.site.end, + "the outer replacement must not overlap the nested guard" + ); + } + + #[test] + fn a_guarded_site_that_grows_shifts_the_guards_below_it() { + // The first site emits both the mutated text and the original, so the file gets longer and + // the second guard sits well below the line its mutant was written on. + let text = "fn f(a: i32, b: i32) -> bool {\n a\n < b\n}\nfn g() -> i32 { 1 }\n"; + let first = mutant(35..48, 1, "(a)\n <= (b)", Shape::Expr); + let second = mutant(67..68, 2, "0", Shape::Expr); + + let (out, guards) = instrument_with_guards(text, &[&first, &second]).expect("instrumented"); + let found = out + .lines() + .position(|line| line.contains("a(2u32)")) + .and_then(|at| u32::try_from(at + 1).ok()); + + assert_eq!(guards.get(&2).map(|guard| guard.site.start.line), found); + assert_ne!( + guards.get(&2).map(|guard| guard.site.start.line), + Some(5), + "the guard should not be on its source line" + ); + } + use syn::parse_file; + + fn mutant(span: Range, ordinal: u32, replacement: &str, shape: Shape) -> Mutant { + Mutant { + ordinal, + span, + replacement: replacement.to_owned(), + shape, + } + } + + #[test] + fn continue_to_break_keeps_continue_as_the_tail_expression() { + let text = "fn f() -> i32 { loop { let x = if true { continue } else { 1 }; return x } }\n"; + let site = span_of(text, "continue"); + let out = apply(text, &[mutant(site, 7, "break", Shape::Continue)]); + + assert!(out.contains("{ if ::gamma_rt::a(7u32) { break; } continue }"), "{out}"); + let _parsed = parse_file(&out).expect("the specialized guard must parse"); + } + + #[test] + fn break_to_continue_keeps_break_as_the_tail_expression() { + let text = "fn f() { loop { if true { break } } }\n"; + let site = span_of(text, "break"); + let out = apply(text, &[mutant(site, 8, "continue", Shape::Break)]); + + assert!(out.contains("{ if ::gamma_rt::a(8u32) { continue; } break }"), "{out}"); + let _parsed = parse_file(&out).expect("the specialized guard must parse"); + } + + fn apply(text: &str, mutants: &[Mutant]) -> String { + let refs: Vec<&Mutant> = mutants.iter().collect(); + + instrument(text, &refs).unwrap() + } + + fn span_of(text: &str, needle: &str) -> Range { + let start = text.find(needle).unwrap(); + + start..start + needle.len() + } + + #[test] + fn no_mutants_leaves_the_text_untouched() { + assert_eq!(apply("fn f() {}", &[]), "fn f() {}"); + } + + #[test] + fn an_arm_site_becomes_a_guard_trailing_the_pattern() { + let text = "fn f(v: Option) -> i32 { match v { Some(n) => n, _ => 0 } }"; + let out = apply(text, &[mutant(span_of(text, "Some(n)"), 4, "", Shape::Arm)]); + + // The arm has to keep its pattern: the body still binds `n`, so replacing the pattern + // rather than qualifying it would not compile. + assert!(out.contains("Some(n) if !::gamma_rt::a(4u32) => n"), "{out}"); + } + + #[test] + fn a_guarded_arm_still_parses_as_rust() { + let text = "fn f(v: Option) -> i32 { match v { Some(n) => n, _ => 0 } }"; + let out = apply(text, &[mutant(span_of(text, "Some(n)"), 4, "", Shape::Arm)]); + + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn two_arm_mutants_on_one_pattern_chain_with_and() { + let text = "fn f(v: Option) -> i32 { match v { Some(n) => n, _ => 0 } }"; + let span = span_of(text, "Some(n)"); + let out = apply(text, &[mutant(span.clone(), 4, "", Shape::Arm), mutant(span, 5, "", Shape::Arm)]); + + // Repeating `if` would not parse. Only one mutant is ever active, so the conjunction is + // never actually deciding between them, but it has to be syntactically well formed. + assert!(out.contains("Some(n) if !::gamma_rt::a(4u32) && !::gamma_rt::a(5u32) =>"), "{out}"); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn an_arm_guard_nests_inside_an_enclosing_expression_site() { + let text = "fn f(v: Option) -> i32 { match v { Some(n) => n, _ => 0 } }"; + let whole = span_of(text, "match v { Some(n) => n, _ => 0 }"); + let arm = span_of(text, "Some(n)"); + let out = apply(text, &[mutant(whole, 1, "0", Shape::Expr), mutant(arm, 2, "", Shape::Arm)]); + + _ = parse_file(&out).expect("the instrumented source does not parse"); + assert!(out.contains("if ::gamma_rt::a(1u32) { 0 }"), "{out}"); + assert!(out.contains("Some(n) if !::gamma_rt::a(2u32) =>"), "{out}"); + } + + #[test] + fn an_expression_site_becomes_a_parenthesized_guard() { + let text = "fn f(a: i32, b: i32) -> bool { a < b }"; + let out = apply(text, &[mutant(span_of(text, "a < b"), 7, "(a) <= (b)", Shape::Expr)]); + + assert_eq!( + out, + "fn f(a: i32, b: i32) -> bool { (if ::gamma_rt::a(7u32) { (a) <= (b) } else { a < b }) }" + ); + } + + #[test] + fn the_guarded_expression_still_parses_as_rust() { + let text = "fn f(a: i32, b: i32) -> bool { a < b }"; + let out = apply(text, &[mutant(span_of(text, "a < b"), 7, "(a) <= (b)", Shape::Expr)]); + + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn a_guard_in_condition_position_parses() { + // Without the parentheses this would be `if { .. } { .. }`, which Rust rejects. + let text = "fn f(a: i32, b: i32) { if a < b { g(); } }"; + let out = apply(text, &[mutant(span_of(text, "a < b"), 1, "(a) <= (b)", Shape::Expr)]); + + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn two_mutants_on_one_span_nest_rather_than_collide() { + let text = "fn f(a: i32, b: i32) -> bool { a < b }"; + let site = span_of(text, "a < b"); + let out = apply( + text, + &[ + mutant(site.clone(), 1, "(a) <= (b)", Shape::Expr), + mutant(site, 2, "(a) > (b)", Shape::Expr), + ], + ); + + assert!(out.contains("::gamma_rt::a(1u32)")); + assert!(out.contains("::gamma_rt::a(2u32)")); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn a_nested_site_is_instrumented_only_in_the_default_branch() { + // `a + b < c`: the `<` site contains the `+` site. Exactly one mutant is live per process, + // so the `<` replacement can use plain operands; only the fall-through arm needs children. + let text = "fn f(a: i32, b: i32, c: i32) -> bool { a + b < c }"; + let out = apply( + text, + &[ + mutant(span_of(text, "a + b < c"), 1, "(a + b) <= (c)", Shape::Expr), + mutant(span_of(text, "a + b"), 2, "(a) - (b)", Shape::Expr), + ], + ); + + assert_eq!(out.matches("::gamma_rt::a(2u32)").count(), 1); + + let then_arm = out.split("else").next().unwrap(); + + assert!(!then_arm.contains("a(2u32)")); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn nesting_does_not_grow_exponentially() { + // Three nested sites encode three guards, not eight. + let text = "fn f(a: i32, b: i32, c: i32, d: i32) -> bool { a + b * c < d }"; + let out = apply( + text, + &[ + mutant(span_of(text, "a + b * c < d"), 1, "(a + b * c) <= (d)", Shape::Expr), + mutant(span_of(text, "a + b * c"), 2, "(a) - (b * c)", Shape::Expr), + mutant(span_of(text, "b * c"), 3, "(b) / (c)", Shape::Expr), + ], + ); + + assert_eq!(out.matches("::gamma_rt::a(").count(), 3); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn a_block_site_stays_a_block() { + let text = "fn f() -> i32 { compute() }"; + let out = apply(text, &[mutant(span_of(text, "{ compute() }"), 4, "0", Shape::Block)]); + + assert!(out.contains("{ if ::gamma_rt::a(4u32) { 0 } else {")); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn a_block_site_containing_an_expression_site_nests() { + let text = "fn f(a: i32) -> i32 { a + 1 }"; + let out = apply( + text, + &[ + mutant(span_of(text, "{ a + 1 }"), 1, "0", Shape::Block), + mutant(span_of(text, "a + 1"), 2, "(a) - (1)", Shape::Expr), + ], + ); + + assert_eq!(out.matches("::gamma_rt::a(").count(), 2); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn a_statement_site_is_skipped_when_active() { + let text = "fn f(v: &mut Vec) { v.push(1); }"; + let out = apply(text, &[mutant(span_of(text, "v.push(1);"), 9, "", Shape::Stmt)]); + + assert!(out.contains("if !::gamma_rt::a(9u32) { v.push(1); }")); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn sites_are_spliced_at_the_right_offsets_when_there_are_several() { + let text = "fn f(a: i32, b: i32) -> i32 { let x = a + b; let y = a - b; x * y }"; + let out = apply( + text, + &[ + mutant(span_of(text, "a + b"), 1, "(a) - (b)", Shape::Expr), + mutant(span_of(text, "a - b"), 2, "(a) + (b)", Shape::Expr), + ], + ); + + assert!(out.contains("let x = (if ::gamma_rt::a(1u32)")); + assert!(out.contains("let y = (if ::gamma_rt::a(2u32)")); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn text_outside_every_site_is_preserved_exactly() { + let text = "// a comment\nfn f(a: i32) -> i32 { a + 1 }\n// trailing\n"; + let out = apply(text, &[mutant(span_of(text, "a + 1"), 1, "(a) - (1)", Shape::Expr)]); + + assert!(out.starts_with("// a comment\n")); + assert!(out.ends_with("// trailing\n")); + } + + #[test] + fn a_span_beyond_the_text_is_skipped_rather_than_spliced() { + let text = "fn f() {}"; + + assert_eq!(apply(text, &[mutant(100..200, 1, "0", Shape::Expr)]), text); + } + + #[test] + fn an_empty_span_is_skipped() { + let text = "fn f() {}"; + + assert_eq!(apply(text, &[mutant(3..3, 1, "0", Shape::Expr)]), text); + } + + /// A span landing inside a character is not a span this file can splice. + /// + /// Every offset is within the text, so a length check alone lets it through — and then every + /// slice taken from it is `None`, which as an empty string erases the region instead of + /// splicing it. That is worse than a wrong-offset splice: whole functions vanish from the + /// instrumented copy, the build fails somewhere unrelated, and nothing in the pipeline can + /// attribute it, because the guard the invariant check looks for was written correctly. + #[test] + fn a_span_bisecting_a_character_is_skipped_rather_than_deleting_the_region() { + let text = "fn f() -> usize { let s = \"ππ\"; s.len() }"; + let start = text.find('π').expect("the fixture holds a multi-byte character"); + + assert!(!text.is_char_boundary(start + 1), "the premise is an endpoint inside a character"); + + assert_eq!(apply(text, &[mutant(start..start + 1, 1, "0", Shape::Expr)]), text); + assert_eq!(apply(text, &[mutant(start + 1..text.len(), 2, "0", Shape::Expr)]), text); + } + + #[test] + fn overlapping_sites_are_rejected() { + let text = "fn f(a: i32, b: i32, c: i32) -> i32 { a + b + c }"; + let left = span_of(text, "a + b"); + let right = span_of(text, "b + c"); + let result = instrument(text, &[&mutant(left, 1, "x", Shape::Expr), &mutant(right, 2, "y", Shape::Expr)]); + + _ = result.expect_err("the instrumentation was expected to fail"); + } + + #[test] + fn multibyte_text_is_spliced_on_byte_boundaries() { + let text = "fn f(a: i32) -> i32 { /* π ≈ 3 */ a + 1 }"; + let out = apply(text, &[mutant(span_of(text, "a + 1"), 1, "(a) - (1)", Shape::Expr)]); + + assert!(out.contains("π ≈ 3")); + _ = parse_file(&out).expect("the instrumented source does not parse"); + } + + #[test] + fn ordinals_are_emitted_in_order_within_a_span() { + let text = "fn f(a: i32, b: i32) -> bool { a < b }"; + let site = span_of(text, "a < b"); + let out = apply( + text, + &[ + mutant(site.clone(), 5, "(a) > (b)", Shape::Expr), + mutant(site, 3, "(a) <= (b)", Shape::Expr), + ], + ); + + assert!(out.find("a(3u32)").unwrap() < out.find("a(5u32)").unwrap()); + } +} diff --git a/crates/cargo-gamma-lib/Cargo.toml b/crates/cargo-gamma-lib/Cargo.toml new file mode 100644 index 000000000..08b539dc2 --- /dev/null +++ b/crates/cargo-gamma-lib/Cargo.toml @@ -0,0 +1,120 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[package] +name = "cargo-gamma-lib" +description = "Internal library for cargo-gamma" +version = "0.1.0" +readme = "README.md" +keywords = ["cargo", "mutation-testing", "testing"] +categories = ["development-tools::testing"] + +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo-gamma-lib" + +[package.metadata.docs.rs] +all-features = true + +# Establish the cross-platform CI baseline for this new crate. Raise it as coverage improves. +[package.metadata.coverage-gate] +min-lines-percent = 96.8 + +[features] +# Exposes the crate's internal modules so that the integration tests in `tests/` can drive them +# directly. Enabled only by this crate's dev-dependency on itself; never enable it from outside. +internals = ["dep:tempfile"] +# Selects the dedicated Loom integration target. The scheduler itself is still activated by +# `RUSTFLAGS="--cfg loom"` in the repository check. +loom = [] + +[dependencies] +blake3 = { workspace = true, features = ["std"] } +camino = { workspace = true } +cargo-gamma-engine = { workspace = true } +cargo-gamma-process = { workspace = true } +cargo-gamma-rt = { workspace = true } +cargo-gamma-unsafe = { workspace = true } +cargo_metadata = { workspace = true } +clap = { workspace = true, features = ["std", "derive", "color", "help", "error-context", "usage", "suggestions", "env", "wrap_help"] } +clap_complete = { workspace = true } +compact_str = { workspace = true, features = ["serde", "std"] } +ignore = { workspace = true } +owo-colors = { workspace = true } +proc-macro2 = { workspace = true, features = ["span-locations", "proc-macro"] } +reflink-copy = { workspace = true } +rustc-hash = { workspace = true, features = ["std"] } +serde = { workspace = true, features = ["derive", "rc", "std"] } +serde_json = { workspace = true, features = ["std"] } +syn = { workspace = true, features = ["full", "parsing", "printing", "visit", "extra-traits", "clone-impls", "proc-macro"] } +tempfile = { workspace = true, optional = true } +toml = { workspace = true, features = ["parse", "display", "serde", "std"] } +toml_edit = { workspace = true, features = ["parse", "display"] } +walkdir = { workspace = true } + +# loom drives the deterministic-scheduler models in `exec::verdict::hubs`, which only compile under +# `--cfg loom`. Gating the dependency on that cfg keeps it out of every ordinary build and test: +# `cfg(loom)` is never set unless `RUSTFLAGS="--cfg loom"` is passed, so this resolves to nothing in +# a normal `cargo build`/`cargo test`. It is declared as a normal (not dev) dependency because cargo +# only honours a custom `--cfg` from `RUSTFLAGS` for `[target.'cfg(...)'.dependencies]`. +[target.'cfg(loom)'.dependencies] +loom = { workspace = true } + +[target.'cfg(unix)'.dependencies] +libc = { workspace = true, features = ["std"] } + +[dev-dependencies] +bolero = { workspace = true } +# Only the integration test that pins the proc-macro's hand-copied limit constants against this +# library's own needs it; it is never used outside `#[cfg(test)]`/`tests/`. +cargo-gamma-attrs-impl = { workspace = true } +cargo-gamma-process = { workspace = true, features = ["fault-injection"] } +tempfile = { workspace = true } + +[[test]] +name = "agreement" +required-features = ["internals"] + +[[test]] +name = "cli" +required-features = ["internals"] + +[[test]] +name = "docs" +required-features = ["internals"] + +[[test]] +name = "gate" +required-features = ["internals"] + +[[test]] +name = "instrumented_compiles" +required-features = ["internals"] + +[[test]] +name = "loom" +required-features = ["loom"] + +[[test]] +name = "regressions" +required-features = ["internals"] + +[[test]] +name = "schema_conformance" +required-features = ["internals"] + +[[test]] +name = "session" +required-features = ["internals"] + +[[test]] +name = "units" +required-features = ["internals"] + +# >>> anvil-managed: anvil-lints +[lints] +workspace = true +# <<< anvil-managed: anvil-lints diff --git a/crates/cargo-gamma-lib/README.md b/crates/cargo-gamma-lib/README.md new file mode 100644 index 000000000..ba093fbef --- /dev/null +++ b/crates/cargo-gamma-lib/README.md @@ -0,0 +1,27 @@ +
+ Cargo-Gamma-Lib Logo + +# Cargo-Gamma-Lib + +[![crates.io](https://img.shields.io/crates/v/cargo-gamma-lib.svg)](https://crates.io/crates/cargo-gamma-lib) +[![docs.rs](https://docs.rs/cargo-gamma-lib/badge.svg)](https://docs.rs/cargo-gamma-lib) +[![MSRV](https://img.shields.io/crates/msrv/cargo-gamma-lib)](https://crates.io/crates/cargo-gamma-lib) +[![CI](https://github.com/microsoft/ox-tools/actions/workflows/main.yml/badge.svg?event=push)](https://github.com/microsoft/ox-tools/actions/workflows/main.yml) +[![Coverage](https://codecov.io/gh/microsoft/ox-tools/graph/badge.svg?token=FCUG0EL5TI)](https://codecov.io/gh/microsoft/ox-tools) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](../../LICENSE) +This crate was developed as part of the Oxidizer project + +
+ +Internal implementation library for [`cargo-gamma`][__link0]. + +This crate is an implementation detail. Do not depend on it: it may change in incompatible +ways without warning, and it carries no semver commitment to anything it exposes. + + +
+ +This crate was developed as part of The Oxidizer Project. Browse this crate's source code. + + + [__link0]: https://crates.io/crates/cargo-gamma diff --git a/crates/cargo-gamma-lib/docs/DESIGN.md b/crates/cargo-gamma-lib/docs/DESIGN.md new file mode 100644 index 000000000..2d76f02dc --- /dev/null +++ b/crates/cargo-gamma-lib/docs/DESIGN.md @@ -0,0 +1,27 @@ +# cargo-gamma-lib — Design + +> Status: **Implemented**. +> Crate name: `cargo-gamma-lib`. + +## Purpose + +This crate coordinates cargo-gamma campaigns: configuration, discovery, +scratch workspaces, instrumented builds, test selection, process supervision, +verdicts, incremental reuse, reporting, and command dispatch. + +## Boundaries + +- Rust parsing and instrumentation are delegated to `cargo-gamma-engine`. +- Process-tree mechanics are delegated to `cargo-gamma-process` and + `cargo-gamma-unsafe`. +- The injected guard protocol is provided by dependency-free + `cargo-gamma-rt`. +- The `internals` feature exists only for this crate's integration tests and + is not a supported downstream API. +- The crate forbids unsafe code. + +## Public contract + +The primary public contract is the `cargo gamma` command surface and its +configuration, reports, diagnostics, and exit codes. The Rust API is an +implementation detail used by the thin executable crate. diff --git a/crates/cargo-gamma-lib/favicon.ico b/crates/cargo-gamma-lib/favicon.ico new file mode 100644 index 000000000..18acb282f --- /dev/null +++ b/crates/cargo-gamma-lib/favicon.ico @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82caca17fd4d08a23a9d76f5c895029ff6bb0d13b0e2693ef229a4ea691ffccd +size 46496 diff --git a/crates/cargo-gamma-lib/logo.png b/crates/cargo-gamma-lib/logo.png new file mode 100644 index 000000000..20aaf26cf --- /dev/null +++ b/crates/cargo-gamma-lib/logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc17b32e4b6b8c4f0c1d443b680a25b68e0cd8913954dfd047a7659a6bd0baff +size 131471 diff --git a/crates/cargo-gamma-lib/src/advise/analysis.rs b/crates/cargo-gamma-lib/src/advise/analysis.rs new file mode 100644 index 000000000..9d46f9f5a --- /dev/null +++ b/crates/cargo-gamma-lib/src/advise/analysis.rs @@ -0,0 +1,1152 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Turning a completed run into measured findings and per-family yields. + +use core::cmp::Ordering; +use core::time::Duration; + +use camino::Utf8Path; + +use super::text::{human, plural}; +use super::{Finding, Timing, Yield}; +use crate::exec::TestBinary; +use crate::model::{Mutant, Outcome, Summary}; +use crate::{HashMap, HashSet}; + +/// The fraction of the population one file must hold before it is worth naming. +const HOT_FILE_SHARE: f64 = 0.10; + +/// The population below which share-based findings are arithmetic rather than evidence. +/// +/// In a run of four mutants every file is a hot file and every family is a quarter of the budget. +/// Reporting that is not a smaller version of the real finding, it is a different and false one, +/// and a tool that cries wolf on a toy project is one nobody reads on a real one. +const MIN_POPULATION: usize = 50; + +/// The CPU time a family must consume before its yield is worth judging at all. +/// +/// A share threshold alone would flag a family that used 40% of six seconds. +const MIN_YIELD_CPU: Duration = Duration::from_mins(1); + +/// The fraction of mutant execution time a family must consume before its yield is worth judging. +/// +/// Below this, a family with no survivors is not a problem: disabling it would save nothing, and +/// the advice would be pure noise on a report someone has to read. +const YIELD_FLOOR_SHARE: f64 = 0.05; + +/// The fraction of wall time the fixed cost must exceed before it is the thing to fix. +const FIXED_COST_SHARE: f64 = 0.30; + +/// The fraction of wall time mutant execution must occupy before optimization is worth trying. +const EXECUTION_DOMINANT_SHARE: f64 = 0.60; + +/// The fraction of the population that must be unviable before rollback is worth reporting. +const UNVIABLE_SHARE: f64 = 0.05; + +/// The fraction of executed mutants that must exhaust memory before the ceiling is the suspect. +/// +/// A ceiling is at least twice its binary's measured baseline peak, so passing it means roughly +/// doubling the memory of the whole test binary. A handful of mutants genuinely can — an inverted +/// loop bound, a capacity computed by multiplication — but one mutant in twenty doing it says more +/// about the baseline the ceiling was derived from than about the mutants held to it. +const MEMORY_CEILING_SHARE: f64 = 0.05; + +/// The fraction of valid mutants that must be uncovered before it is the headline. +const UNCOVERED_SHARE: f64 = 0.10; + +/// The baseline duration above which every mutant is paying a noticeable fixed cost. +const SLOW_BASELINE: Duration = Duration::from_secs(10); + +/// The wall time above which a run will not fit in a routine CI job. +const LONG_RUN: Duration = Duration::from_mins(30); + +/// The wall time a shard should aim for, used to size the suggested rotation. +const TARGET_SHARD: Duration = Duration::from_mins(15); + +#[derive(Clone, Copy)] +enum CargoProfile<'a> { + Unknown, + Default, + Named(&'a str), +} + +/// Analyzes a completed run. +/// +/// Findings come back in a fixed diagnostic order: run-wide costs first, then costly verdicts, +/// population concentration, mutator yield, and uncovered code. +#[must_use] +pub fn analyze(mutants: &[Mutant], timing: &Timing) -> Vec { + analyze_context(mutants, timing, CargoProfile::Unknown, &[]) +} + +/// Analyzes a completed run with the user-controlled build and test context. +#[must_use] +pub fn analyze_run(mutants: &[Mutant], timing: &Timing, profile: Option<&str>, binaries: &[TestBinary]) -> Vec { + let profile = profile.map_or(CargoProfile::Default, CargoProfile::Named); + + analyze_context(mutants, timing, profile, binaries) +} + +fn analyze_context(mutants: &[Mutant], timing: &Timing, profile: CargoProfile<'_>, binaries: &[TestBinary]) -> Vec { + let mut findings = Vec::new(); + let summary = Summary::of(mutants); + let executed: Duration = mutants.iter().map(|mutant| Duration::from_millis(mutant.elapsed_ms)).sum(); + + if let Some(finding) = fixed_cost(timing) { + findings.push(finding); + } + + if let Some(finding) = slow_baseline(timing, mutants, binaries) { + findings.push(finding); + } + + if let Some(finding) = unoptimized_execution(timing, profile) { + findings.push(finding); + } + + if let Some(finding) = long_run(timing) { + findings.push(finding); + } + + if let Some(finding) = timeouts(summary, mutants) { + findings.push(finding); + } + + if let Some(finding) = out_of_memory(summary, mutants) { + findings.push(finding); + } + + if let Some(finding) = unviable(summary) { + findings.push(finding); + } + + findings.extend(hot_files(mutants)); + findings.extend(low_yield(mutants, executed)); + + if let Some(finding) = uncovered(summary) { + findings.push(finding); + } + + findings +} + +/// Reports the cost and value of each mutator family, worst ratio last. +#[must_use] +pub fn yields(mutants: &[Mutant]) -> Vec { + let mut buckets: HashMap = HashMap::default(); + + for mutant in mutants { + let family = family_of(&mutant.mutator).to_owned(); + let entry = buckets.entry(family.clone()).or_insert_with(|| Yield { + family, + mutants: 0, + cpu: Duration::ZERO, + survivors: 0, + }); + + entry.mutants += 1; + entry.cpu += Duration::from_millis(mutant.elapsed_ms); + + if mutant.outcome == Outcome::Survived { + entry.survivors += 1; + } + } + + let mut rows: Vec = buckets.into_values().collect(); + + rows.sort_by(|left, right| { + right + .per_cpu_hour() + .partial_cmp(&left.per_cpu_hour()) + .unwrap_or(Ordering::Equal) + .then_with(|| left.family.cmp(&right.family)) + }); + + rows +} + +/// The family part of a mutator name: everything before the first dot. +fn family_of(mutator: &str) -> &str { + mutator.split_once('.').map_or(mutator, |(family, _)| family) +} + +/// A build that costs more than the testing it enables. +fn fixed_cost(timing: &Timing) -> Option { + let wall = timing.wall.as_secs_f64(); + let fixed = timing.build.as_secs_f64() + timing.baseline.as_secs_f64(); + + if wall <= 0.0 || fixed / wall < FIXED_COST_SHARE { + return None; + } + + Some(Finding { + code: "fixed-cost", + headline: format!( + "{:.0}% of the run was the build and baseline, not mutation testing", + fixed / wall * 100.0 + ), + detail: vec![ + format!("build {}, baseline {}", human(timing.build), human(timing.baseline)), + format!( + "mutant execution {}", + human(timing.wall.saturating_sub(timing.build + timing.baseline)) + ), + ], + remedy: "test more mutants per build: widen `--mutators`, or drop `--shard-count` so each run \ + amortizes the build over more work. A build cache such as sccache helps the build \ + itself." + .to_owned(), + cost: "none — this is the one finding whose remedy costs no signal at all".to_owned(), + }) +} + +/// A suite whose fixed per-run cost is paid once per mutant. +#[expect(clippy::cast_precision_loss, reason = "a mutant count far exceeds any plausible workspace")] +fn slow_baseline(timing: &Timing, mutants: &[Mutant], binaries: &[TestBinary]) -> Option { + if timing.baseline < SLOW_BASELINE { + return None; + } + + let live = mutants.iter().filter(|mutant| ran_test_binary(mutant.outcome)).count(); + let projected = timing.baseline.mul_f64(live as f64).div_f64(timing.jobs.max(1) as f64); + + let mut detail = vec![ + format!("every one of the {live} tested mutants pays that cost"), + format!("floor for this run at {} jobs: {}", timing.jobs, human(projected)), + ]; + + let mut slowest: Vec<&TestBinary> = binaries.iter().filter(|binary| !binary.baseline.is_zero()).collect(); + + slowest.sort_by(|left, right| { + right + .baseline + .cmp(&left.baseline) + .then_with(|| left.package.cmp(&right.package)) + .then_with(|| left.target.cmp(&right.target)) + }); + + for binary in slowest.into_iter().take(3) { + let tests = binary.tests.map_or_else(String::new, |tests| format!(", {tests} tests")); + + detail.push(format!( + "test target `{}::{}`: {} baseline{tests}", + binary.package, + binary.target, + human(binary.baseline) + )); + } + + Some(Finding { + code: "slow-baseline", + headline: format!("the suite takes {} with no mutant active", human(timing.baseline)), + detail, + remedy: "profile the named test targets and shorten repeated fixture setup, sleeps, network \ + calls, and oversized inputs. Split performance or load checks from correctness \ + assertions so mutation runs can keep the fast oracle; if a target carries no \ + mutation-relevant assertions, omit it with `--exclude-test `." + .to_owned(), + cost: "optimizing or splitting tests preserves signal; `--exclude-test` removes every \ + assertion in that target from mutation verdicts" + .to_owned(), + }) +} + +/// A test-dominated run using Cargo's unoptimized test or development profile. +fn unoptimized_execution(timing: &Timing, profile: CargoProfile<'_>) -> Option { + let profile = match profile { + CargoProfile::Default => "Cargo's default test profile", + CargoProfile::Named(profile @ ("dev" | "test")) => profile, + CargoProfile::Unknown | CargoProfile::Named(_) => return None, + }; + + if timing.wall.is_zero() { + return None; + } + + let executed = timing.wall.saturating_sub(timing.build + timing.baseline); + let share = executed.as_secs_f64() / timing.wall.as_secs_f64(); + + if share < EXECUTION_DOMINANT_SHARE { + return None; + } + + Some(Finding { + code: "unoptimized-execution", + headline: format!("{:.0}% of the run was mutant execution under {profile}", share * 100.0), + detail: vec![ + format!("mutant execution {}, build {}", human(executed), human(timing.build)), + "the build is paid once, while the selected test code is executed for every mutant".to_owned(), + ], + remedy: "if the named test targets are CPU-bound, compare a representative shard with the \ + documented optimized profile: `cargo gamma run --profile gamma`. Keep that profile \ + fixed for every shard whose reports will be merged." + .to_owned(), + cost: "optimization makes the build slower, does not help I/O-bound tests, and can change \ + verdicts through code generation; scores from different profiles are not comparable" + .to_owned(), + }) +} + +/// Whether assigning this outcome ran a test binary with the mutant active. +const fn ran_test_binary(outcome: Outcome) -> bool { + matches!( + outcome, + Outcome::Killed | Outcome::Survived | Outcome::Timeout | Outcome::OutOfMemory | Outcome::Flaky + ) +} + +/// A run too long to sit in a routine CI job. +fn long_run(timing: &Timing) -> Option { + if timing.wall < LONG_RUN { + return None; + } + + let shards = (timing.wall.as_secs_f64() / TARGET_SHARD.as_secs_f64()).ceil().max(2.0); + + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "bounded by the ratio of two durations" + )] + let shards = shards as u32; + + Some(Finding { + code: "long-run", + headline: format!("the run took {}, which will not fit a per-commit job", human(timing.wall)), + detail: vec![format!("at {} per shard that is a rotation of {shards}", human(TARGET_SHARD))], + remedy: format!( + "run one shard a night — `--shard-count {shards} --shard-index ` — and combine the \ + reports with `cargo gamma merge`. Shards are assigned by content, so coverage \ + accumulates as the code changes instead of resetting." + ), + cost: "none in total coverage, but a verdict is up to one rotation old rather than current".to_owned(), + }) +} + +/// Mutants that hung, and the budget they burned proving it. +fn timeouts(summary: Summary, mutants: &[Mutant]) -> Option { + if summary.timeout == 0 { + return None; + } + + let spent: Duration = mutants + .iter() + .filter(|mutant| mutant.outcome == Outcome::Timeout) + .map(|mutant| Duration::from_millis(mutant.elapsed_ms)) + .sum(); + + Some(Finding { + code: "timeouts", + headline: format!( + "{} {} ran out their whole budget", + summary.timeout, + plural(summary.timeout, "mutant") + ), + detail: vec![format!("{} of CPU time spent waiting for them", human(spent))], + remedy: "a mutant that hangs is a mutant the suite detected, so this is signal, not \ + failure — it is just expensive signal. `cargo gamma suppress` writes suppressions \ + for them so the next run does not pay again." + .to_owned(), + cost: "a suppressed timeout leaves the score unchanged today, but stops being retested, so \ + a later edit that makes it terminate goes unnoticed" + .to_owned(), + }) +} + +/// Mutants the memory ceiling stopped, and whether the ceiling or the sites are the likelier fault. +#[expect(clippy::cast_precision_loss, reason = "a mutant count far exceeds any plausible workspace")] +fn out_of_memory(summary: Summary, mutants: &[Mutant]) -> Option { + let stopped: Vec<&Mutant> = mutants.iter().filter(|mutant| mutant.outcome == Outcome::OutOfMemory).collect(); + + if stopped.is_empty() { + return None; + } + + let spent: Duration = stopped.iter().map(|mutant| Duration::from_millis(mutant.elapsed_ms)).sum(); + let files: HashSet<&Utf8Path> = stopped.iter().map(|mutant| &*mutant.file).collect(); + + let located = match files.iter().copied().next() { + Some(only) if files.len() == 1 && stopped.len() == 1 => format!("the one is in {only}"), + Some(only) if files.len() == 1 => format!("all {} are in {only}", stopped.len()), + _ => format!("spread across {} files, so no one site explains them", files.len()), + }; + + let mut detail = vec![ + format!("{} of CPU time spent before the kernel stopped them", human(spent)), + located, + ]; + + // The share is only evidence once enough mutants ran to make it one, and only the mutants that + // ran a test binary could have reached a ceiling — an uncovered mutant never allocated anything, + // so counting it would dilute the very signal this line exists to raise. + let executed = mutants.iter().filter(|mutant| ran_test_binary(mutant.outcome)).count(); + + if executed >= MIN_POPULATION && stopped.len() as f64 / executed as f64 >= MEMORY_CEILING_SHARE { + detail.push(format!( + "that is {:.0}% of the {executed} mutants that ran, which points at the ceiling rather \ + than the sites", + stopped.len() as f64 / executed as f64 * 100.0 + )); + } + + Some(Finding { + code: "out-of-memory", + headline: format!( + "{} {} hit the memory ceiling", + summary.out_of_memory, + plural(summary.out_of_memory, "mutant") + ), + detail, + remedy: "this is the verdict most likely to be wrong, because a ceiling set too tight \ + convicts a healthy mutant — establish which it is before acting. `--memory \ + measure` reports each binary's peak without stopping anything, and \ + `--memory-multiplier` or `--memory-headroom` widen the ceiling if the baseline it \ + came from was unrepresentative. A site that is genuinely allowed to allocate this \ + much is eligible for `cargo gamma suppress` by default." + .to_owned(), + cost: "a widened ceiling stops bounding the runaway allocation it was there to catch, and a \ + suppressed site stops being retested, so a later edit that makes it allocate without \ + bound goes unnoticed" + .to_owned(), + }) +} + +/// Mutants that could not be compiled, and the rebuild rounds they forced. +fn unviable(summary: Summary) -> Option { + let total = summary.valid() + summary.unviable + summary.ignored; + + // #[gamma::skip(expr.decrement, reason = "on conversion failure both usize::MAX and usize::MAX - 1 are far above the 50-mutant threshold, so the branch is identical")] + if usize::try_from(total).unwrap_or(usize::MAX) < MIN_POPULATION || f64::from(summary.unviable) / f64::from(total) < UNVIABLE_SHARE { + return None; + } + + Some(Finding { + code: "unviable", + headline: format!( + "{} of {total} mutants could not compile ({:.0}%)", + summary.unviable, + f64::from(summary.unviable) / f64::from(total) * 100.0 + ), + detail: vec![ + "each withdrawal round is a full rebuild of the instrumented tree".to_owned(), + "they are excluded from the score, so the cost bought nothing".to_owned(), + ], + remedy: "`cargo gamma suppress --eligible unviable` records them in the source so later runs \ + skip them without discovering their unviability again. If they cluster in one \ + operator, narrow `--mutators` instead." + .to_owned(), + cost: "none — an unviable mutant never contributed to the score".to_owned(), + }) +} + +/// Files holding an outsized share of the population. +#[expect(clippy::cast_precision_loss, reason = "a mutant count far exceeds any plausible workspace")] +fn hot_files(mutants: &[Mutant]) -> Vec { + let total = mutants.len(); + + if total < MIN_POPULATION { + return Vec::new(); + } + + let mut counts: HashMap<&str, (u32, u32, Duration)> = HashMap::default(); + + for mutant in mutants { + let entry = counts.entry(mutant.file.as_str()).or_insert((0, 0, Duration::ZERO)); + + entry.0 += 1; + entry.2 += Duration::from_millis(mutant.elapsed_ms); + + if mutant.outcome == Outcome::Survived { + entry.1 += 1; + } + } + + let mut hot: Vec<(&str, u32, u32, Duration)> = counts + .into_iter() + .filter(|&(_, (count, _, _))| f64::from(count) / total as f64 >= HOT_FILE_SHARE) + .map(|(file, (count, survivors, cpu))| (file, count, survivors, cpu)) + .collect(); + + hot.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(right.0))); + + hot.into_iter() + .map(|(file, count, survivors, cpu)| Finding { + code: "hot-file", + headline: format!( + "{file} alone is {:.0}% of the population ({count} {})", + f64::from(count) / total as f64 * 100.0, + plural(count, "mutant") + ), + detail: vec![format!( + "{} of CPU time, {survivors} {} found there", + human(cpu), + plural(survivors, "survivor") + )], + remedy: "if it is generated, tabular or macro-expanded code, exclude it with \ + `--exclude-file` or the `exclude-files` config key. If it is hand-written, \ + this is not a problem — it is where the logic is." + .to_owned(), + cost: format!( + "exactly {count} {} stop being tested, {survivors} of which are currently finding \ + gaps in the suite", + plural(count, "mutant") + ), + }) + .collect() +} + +/// Families spending real time and finding nothing. +fn low_yield(mutants: &[Mutant], executed: Duration) -> Vec { + // #[gamma::skip(cond.always_false, reason = "when executed is zero every row's CPU is also zero, so the later 60-second filter returns the same empty result")] + if executed.is_zero() { + return Vec::new(); + } + + yields(mutants) + .into_iter() + .filter(|row| row.survivors == 0) + .filter(|row| row.cpu >= MIN_YIELD_CPU) + .filter(|row| row.cpu.as_secs_f64() / executed.as_secs_f64() >= YIELD_FLOOR_SHARE) + .map(|row| Finding { + code: "low-yield", + headline: format!("the `{}` family spent {} and found no survivors", row.family, human(row.cpu)), + detail: vec![format!( + "{} {}, {:.0}% of mutant execution time", + row.mutants, + plural(row.mutants, "mutant"), + row.cpu.as_secs_f64() / executed.as_secs_f64() * 100.0 + )], + remedy: format!("`--mutators 'all,!{}'` drops it", row.family), + cost: "real, and easy to underrate. A family that finds nothing today is a regression \ + detector for tomorrow: this says the suite currently covers it, not that it \ + always will" + .to_owned(), + }) + .collect() +} + +/// Code no test reaches at all. +fn uncovered(summary: Summary) -> Option { + let valid = summary.valid(); + + // #[gamma::skip(expr.decrement, reason = "on conversion failure both usize::MAX and usize::MAX - 1 are far above the 50-mutant threshold, so the branch is identical")] + if usize::try_from(valid).unwrap_or(usize::MAX) < MIN_POPULATION || f64::from(summary.uncovered) / f64::from(valid) < UNCOVERED_SHARE { + return None; + } + + Some(Finding { + code: "uncovered", + headline: format!( + "{} of {valid} mutants ({:.0}%) sit in code no test reaches", + summary.uncovered, + f64::from(summary.uncovered) / f64::from(valid) * 100.0 + ), + detail: vec!["they count against the score, because untested code is the finding".to_owned()], + remedy: "this is not a performance problem and there is nothing to tune. Write tests, or \ + delete the code." + .to_owned(), + cost: "—".to_owned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::advise_fixture::{binary, find, mutant, timing}; + + #[test] + fn a_healthy_run_produces_no_findings() { + let mutants = vec![ + mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 100), + mutant("b.rs", "arith.add_to_sub", Outcome::Survived, 100), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + + assert!(findings.is_empty(), "{findings:?}"); + } + + #[test] + fn findings_follow_the_documented_diagnostic_order() { + let mut mutants: Vec = (0..47) + .map(|index| mutant(&format!("killed-{index}.rs"), "relational.lt_to_le", Outcome::Killed, 1)) + .collect(); + mutants.push(mutant("timeout.rs", "relational.lt_to_le", Outcome::Timeout, 1)); + mutants.push(mutant("memory.rs", "relational.lt_to_le", Outcome::OutOfMemory, 1)); + mutants.extend((0..3).map(|index| mutant(&format!("unviable-{index}.rs"), "relational.lt_to_le", Outcome::CompileError, 1))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let codes: Vec<&str> = findings.iter().map(|finding| finding.code).collect(); + + assert_eq!(codes, ["timeouts", "out-of-memory", "unviable"]); + } + + #[test] + fn a_dominant_build_is_reported() { + let mutants = vec![mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 100)]; + let findings = analyze(&mutants, &timing(50, 5, 100)); + let finding = find(&findings, "fixed-cost").expect("expected a fixed-cost finding"); + + assert!(finding.headline.contains("55%"), "{}", finding.headline); + assert_eq!(finding.detail, ["build 50.0s, baseline 5.0s", "mutant execution 45.0s"]); + assert!(finding.remedy.contains("test more mutants per build"), "{}", finding.remedy); + assert!(finding.remedy.contains("sccache helps the build itself"), "{}", finding.remedy); + assert_eq!(finding.cost, "none — this is the one finding whose remedy costs no signal at all"); + } + + #[test] + fn a_build_that_is_a_small_part_of_the_run_is_not_reported() { + let mutants = vec![mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 100)]; + let findings = analyze(&mutants, &timing(5, 1, 100)); + + assert!(find(&findings, "fixed-cost").is_none(), "{findings:?}"); + } + + #[test] + fn fixed_cost_threshold_and_zero_wall_are_handled_exactly() { + assert!(fixed_cost(&timing(30, 0, 100)).is_some()); + assert!(fixed_cost(&timing(29, 0, 100)).is_none()); + assert!(fixed_cost(&timing(0, 0, 0)).is_none()); + } + + #[test] + fn a_slow_baseline_projects_the_floor_of_the_run() { + let mutants: Vec = (0..40) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + let findings = analyze(&mutants, &timing(1, 60, 4000)); + let finding = find(&findings, "slow-baseline").expect("expected a slow-baseline finding"); + + // 40 mutants x 60s / 4 jobs = 600s. + assert!(finding.detail.iter().any(|line| line.contains("10m")), "{finding:?}"); + assert_eq!(finding.detail[0], "every one of the 40 tested mutants pays that cost"); + assert!(finding.remedy.starts_with("profile the named test targets"), "{}", finding.remedy); + assert!(finding.cost.contains("optimizing or splitting tests preserves signal")); + } + + #[test] + fn slow_baseline_starts_at_ten_seconds_and_never_divides_by_zero_jobs() { + let mutants = vec![mutant("a.rs", "x", Outcome::Killed, 1)]; + assert!(slow_baseline(&timing(0, 9, 10), &mutants, &[]).is_none()); + let mut at_threshold = timing(0, 10, 10); + at_threshold.jobs = 0; + let finding = slow_baseline(&at_threshold, &mutants, &[]).expect("threshold is inclusive"); + assert_eq!(finding.detail[1], "floor for this run at 0 jobs: 10.0s"); + } + + #[test] + fn uncovered_mutants_do_not_inflate_the_baseline_projection() { + let mutants = vec![ + mutant("a.rs", "one", Outcome::Killed, 1), + mutant("a.rs", "two", Outcome::Survived, 1), + mutant("a.rs", "three", Outcome::Timeout, 1), + mutant("b.rs", "four", Outcome::NoCoverage, 1), + mutant("b.rs", "five", Outcome::NoCoverage, 1), + ]; + let mut run = timing(0, 60, 100); + run.jobs = 3; + + let finding = slow_baseline(&run, &mutants, &[]).expect("a 60-second baseline is slow"); + + assert_eq!( + finding.detail, + [ + "every one of the 3 tested mutants pays that cost", + "floor for this run at 3 jobs: 60.0s", + ] + ); + } + + #[test] + fn a_slow_baseline_names_the_slowest_test_targets() { + let mutants = vec![mutant("a.rs", "x", Outcome::Killed, 1)]; + let binaries = vec![ + binary("api", "unit", 3, Some(20)), + binary("api", "security", 18, Some(4)), + binary("cli", "robustness", 12, None), + binary("core", "fast", 1, Some(100)), + ]; + let finding = slow_baseline(&timing(0, 34, 100), &mutants, &binaries).expect("slow baseline"); + + assert_eq!( + &finding.detail[2..], + [ + "test target `api::security`: 18.0s baseline, 4 tests", + "test target `cli::robustness`: 12.0s baseline", + "test target `api::unit`: 3.0s baseline, 20 tests", + ] + ); + assert!(finding.remedy.contains("--exclude-test "), "{}", finding.remedy); + assert!(finding.cost.contains("removes every assertion"), "{}", finding.cost); + } + + #[test] + fn execution_dominance_under_the_default_profile_suggests_the_gamma_profile() { + let finding = unoptimized_execution(&timing(10, 5, 100), CargoProfile::Default).expect("execution dominates"); + + assert_eq!(finding.code, "unoptimized-execution"); + assert!(finding.remedy.contains("cargo gamma run --profile gamma"), "{}", finding.remedy); + assert!(finding.remedy.contains("CPU-bound"), "{}", finding.remedy); + assert!(finding.cost.contains("I/O-bound"), "{}", finding.cost); + assert!(finding.cost.contains("not comparable"), "{}", finding.cost); + } + + #[test] + fn profile_advice_requires_dominant_execution_and_an_unoptimized_profile() { + assert!(unoptimized_execution(&timing(0, 0, 0), CargoProfile::Default).is_none()); + assert!(unoptimized_execution(&timing(30, 20, 100), CargoProfile::Default).is_none()); + assert!(unoptimized_execution(&timing(10, 5, 100), CargoProfile::Named("gamma")).is_none()); + assert!(unoptimized_execution(&timing(10, 5, 100), CargoProfile::Unknown).is_none()); + } + + #[test] + fn a_long_run_suggests_a_rotation_sized_to_it() { + let mutants = vec![mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 100)]; + let findings = analyze(&mutants, &timing(1, 1, 3600)); + let finding = find(&findings, "long-run").expect("expected a long-run finding"); + + // An hour at fifteen minutes a shard is four shards. + assert!(finding.remedy.contains("--shard-count 4"), "{}", finding.remedy); + assert_eq!(finding.detail, ["at 15m per shard that is a rotation of 4"]); + assert_eq!( + finding.cost, + "none in total coverage, but a verdict is up to one rotation old rather than current" + ); + } + + #[test] + fn long_run_starts_at_thirty_minutes() { + assert!(long_run(&timing(0, 0, 1799)).is_none()); + assert!(long_run(&timing(0, 0, 1800)).is_some()); + } + + #[test] + fn timeouts_report_the_budget_they_burned() { + let mutants = vec![ + mutant("a.rs", "relational.lt_to_le", Outcome::Timeout, 30_000), + mutant("a.rs", "relational.le_to_lt", Outcome::Timeout, 30_000), + mutant("a.rs", "arith.add_to_sub", Outcome::Killed, 10), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "timeouts").expect("expected a timeouts finding"); + + assert!(finding.headline.contains('2'), "{}", finding.headline); + assert!(finding.detail[0].contains("60"), "{:?}", finding.detail); + assert!(finding.remedy.contains("does not pay again"), "{}", finding.remedy); + } + + #[test] + fn the_timeout_remedy_names_its_cost() { + let mutants = vec![mutant("a.rs", "relational.lt_to_le", Outcome::Timeout, 10)]; + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "timeouts").expect("expected a timeouts finding"); + + assert!(finding.cost.contains("goes unnoticed"), "{}", finding.cost); + } + + #[test] + fn out_of_memory_mutants_report_the_budget_they_burned_and_where_they_sit() { + let mutants = vec![ + mutant("a.rs", "iter.min_to_max", Outcome::OutOfMemory, 20_000), + mutant("a.rs", "arith.div_to_mul", Outcome::OutOfMemory, 10_000), + mutant("b.rs", "arith.add_to_sub", Outcome::Killed, 10), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "out-of-memory").expect("expected an out-of-memory finding"); + + assert_eq!(finding.headline, "2 mutants hit the memory ceiling"); + assert_eq!( + finding.detail, + ["30.0s of CPU time spent before the kernel stopped them", "all 2 are in a.rs",] + ); + } + + /// The finding leads with verifying the ceiling, because a ceiling set too tight convicts a + /// healthy mutant and suppressing it would then hide a working test rather than a hungry site. + #[test] + fn the_out_of_memory_remedy_names_both_routes_and_both_costs() { + let mutants = vec![mutant("a.rs", "iter.min_to_max", Outcome::OutOfMemory, 10)]; + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "out-of-memory").expect("expected an out-of-memory finding"); + + assert_eq!(finding.headline, "1 mutant hit the memory ceiling"); + assert_eq!(finding.detail[1], "the one is in a.rs"); + + let verify = finding.remedy.find("--memory measure").expect("the remedy must offer measurement"); + let suppress = finding + .remedy + .find("cargo gamma suppress") + .expect("the remedy must offer suppression"); + + assert!( + verify < suppress, + "measuring must be offered before suppressing: {}", + finding.remedy + ); + assert!(finding.remedy.contains("--memory-multiplier"), "{}", finding.remedy); + assert!(finding.cost.contains("stops bounding"), "{}", finding.cost); + assert!(finding.cost.contains("goes unnoticed"), "{}", finding.cost); + } + + #[test] + fn out_of_memory_mutants_in_many_files_are_reported_as_spread() { + let mutants = vec![ + mutant("a.rs", "iter.min_to_max", Outcome::OutOfMemory, 10), + mutant("b.rs", "iter.min_to_max", Outcome::OutOfMemory, 10), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "out-of-memory").expect("expected an out-of-memory finding"); + + assert_eq!(finding.detail[1], "spread across 2 files, so no one site explains them"); + } + + /// A share large enough to indict the ceiling is only evidence once enough mutants ran, and only + /// the mutants that ran count — an uncovered mutant never allocated anything. + #[test] + fn a_large_share_of_out_of_memory_mutants_indicts_the_ceiling() { + let mut mutants: Vec = (0..45) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.extend((0..5).map(|index| mutant("b.rs", "iter.min_to_max", Outcome::OutOfMemory, index))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "out-of-memory").expect("expected an out-of-memory finding"); + + assert_eq!( + finding.detail[2], + "that is 10% of the 50 mutants that ran, which points at the ceiling rather than the sites" + ); + } + + /// The share threshold is inclusive, and a run just under it says nothing about the ceiling. + #[test] + fn the_out_of_memory_share_threshold_is_inclusive() { + let indicts = |killed: u64, stopped: u64| { + let mut mutants: Vec = (0..killed) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.extend((0..stopped).map(|index| mutant("b.rs", "iter.min_to_max", Outcome::OutOfMemory, index))); + + let finding = out_of_memory(Summary::of(&mutants), &mutants).expect("expected a finding"); + + finding.detail.len() == 3 + }; + + assert!(indicts(95, 5), "5 of 100 is exactly the threshold"); + assert!(!indicts(96, 4), "4 of 100 is below it"); + } + + /// Uncovered mutants never ran a test binary, so counting them would dilute the share below the + /// threshold and silence a genuinely mis-calibrated ceiling. + #[test] + fn uncovered_mutants_do_not_dilute_the_out_of_memory_share() { + let mut mutants: Vec = (0..47) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.extend((0..3).map(|index| mutant("b.rs", "iter.min_to_max", Outcome::OutOfMemory, index))); + mutants.extend((0..500).map(|index| mutant("c.rs", "arith.add_to_sub", Outcome::NoCoverage, index))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "out-of-memory").expect("expected an out-of-memory finding"); + + assert_eq!( + finding.detail[2], + "that is 6% of the 50 mutants that ran, which points at the ceiling rather than the sites" + ); + } + + /// Below the population floor a share is arithmetic rather than evidence: five out of forty-five + /// is 11%, but a run that small cannot tell a mis-calibrated ceiling from three hungry sites. + #[test] + fn a_small_run_never_indicts_the_ceiling_however_large_the_share() { + let mut mutants: Vec = (0..44) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.extend((0..5).map(|index| mutant("b.rs", "iter.min_to_max", Outcome::OutOfMemory, index))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "out-of-memory").expect("expected an out-of-memory finding"); + + assert_eq!(finding.detail.len(), 2, "{:?}", finding.detail); + } + + #[test] + fn a_run_with_no_out_of_memory_mutants_has_no_such_finding() { + let mutants = vec![mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 10)]; + + assert!(out_of_memory(Summary::of(&mutants), &mutants).is_none()); + } + + #[test] + fn a_pile_of_unviable_mutants_is_reported() { + let mut mutants: Vec = (0..90) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.extend((0..10).map(|index| mutant("b.rs", "fn_value.default", Outcome::CompileError, index))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "unviable").expect("expected an unviable finding"); + + assert!(finding.headline.contains("10 of 100"), "{}", finding.headline); + assert_eq!( + finding.detail, + [ + "each withdrawal round is a full rebuild of the instrumented tree", + "they are excluded from the score, so the cost bought nothing", + ] + ); + assert!(finding.remedy.contains("cargo gamma suppress --eligible unviable")); + assert_eq!(finding.cost, "none — an unviable mutant never contributed to the score"); + } + + #[test] + fn unviable_requires_the_population_and_share_thresholds_inclusively() { + assert!( + unviable(Summary { + killed: 46, + unviable: 3, + ignored: 1, + ..Summary::default() + }) + .is_some() + ); + assert!( + unviable(Summary { + killed: 57, + unviable: 3, + ..Summary::default() + }) + .is_some() + ); + assert!( + unviable(Summary { + killed: 46, + unviable: 2, + ignored: 1, + ..Summary::default() + }) + .is_none() + ); + assert!( + unviable(Summary { + killed: 57, + unviable: 2, + ignored: 1, + ..Summary::default() + }) + .is_none() + ); + } + + #[test] + fn a_hot_file_names_the_survivors_that_would_be_lost() { + let mut mutants: Vec = (0..80) + .map(|index| mutant("generated.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.push(mutant("generated.rs", "arith.add_to_sub", Outcome::Survived, 1)); + mutants.extend((0..19).map(|index| mutant("real.rs", "relational.lt_to_le", Outcome::Killed, index))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "hot-file").expect("expected a hot-file finding"); + + assert!(finding.headline.starts_with("generated.rs"), "{}", finding.headline); + assert!(finding.cost.contains("81 mutants"), "{}", finding.cost); + assert!(finding.cost.contains("1 of which"), "{}", finding.cost); + assert_eq!(finding.detail, ["3.2s of CPU time, 1 survivor found there"]); + assert!(finding.remedy.contains("where the logic is"), "{}", finding.remedy); + } + + #[test] + fn a_file_below_the_share_is_not_named() { + let mut mutants: Vec = (0..95) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.extend((0..5).map(|index| mutant("b.rs", "arith.add_to_sub", Outcome::Killed, index))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let hot: Vec<&Finding> = findings.iter().filter(|finding| finding.code == "hot-file").collect(); + + assert_eq!(hot.len(), 1, "{hot:?}"); + assert!(hot[0].headline.starts_with("a.rs"), "{}", hot[0].headline); + } + + #[test] + fn a_file_at_exactly_ten_percent_is_hot_and_keeps_its_cpu_total() { + let mut mutants: Vec = (0..45).map(|_| mutant("a.rs", "x", Outcome::Killed, 1)).collect(); + mutants.extend((0..5).map(|_| mutant("b.rs", "x", Outcome::Survived, 1000))); + let hot = hot_files(&mutants); + let b = hot.iter().find(|finding| finding.headline.starts_with("b.rs")).expect("10% is hot"); + assert_eq!(b.detail, ["5.0s of CPU time, 5 survivors found there"]); + } + + #[test] + fn a_family_that_finds_nothing_expensively_is_reported() { + let mutants = vec![ + mutant("a.rs", "literal.int_bump", Outcome::Killed, 200_000), + mutant("a.rs", "relational.lt_to_le", Outcome::Survived, 1000), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "low-yield").expect("expected a low-yield finding"); + + assert!(finding.headline.contains("literal"), "{}", finding.headline); + assert!(finding.remedy.contains("!literal"), "{}", finding.remedy); + assert_eq!(finding.detail, ["1 mutant, 100% of mutant execution time"]); + assert!(finding.cost.contains("not that it always will"), "{}", finding.cost); + } + + #[test] + fn a_family_that_finds_nothing_cheaply_is_left_alone() { + let mutants = vec![ + mutant("a.rs", "literal.int_bump", Outcome::Killed, 10), + mutant("a.rs", "relational.lt_to_le", Outcome::Survived, 50_000), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + + assert!(find(&findings, "low-yield").is_none(), "{findings:?}"); + } + + #[test] + fn a_family_with_survivors_is_never_low_yield() { + let mutants = vec![mutant("a.rs", "literal.int_bump", Outcome::Survived, 50_000)]; + let findings = analyze(&mutants, &timing(1, 1, 100)); + + assert!(find(&findings, "low-yield").is_none(), "{findings:?}"); + } + + #[test] + fn low_yield_honours_both_inclusive_thresholds_and_zero_execution() { + assert!(low_yield(&[], Duration::ZERO).is_empty()); + let mutants = vec![ + mutant("a.rs", "literal.bump", Outcome::Killed, 60_000), + mutant("a.rs", "other.bump", Outcome::Survived, 1_140_000), + ]; + let findings = low_yield(&mutants, Duration::from_mins(20)); + assert_eq!(findings.len(), 1); + assert!(findings[0].headline.contains("literal")); + assert!(low_yield(&mutants, Duration::from_secs(1201)).is_empty()); + } + + #[test] + fn uncovered_code_is_reported_as_the_finding_it_is() { + let mut mutants: Vec = (0..80) + .map(|index| mutant("a.rs", "relational.lt_to_le", Outcome::Killed, index)) + .collect(); + + mutants.extend((0..20).map(|index| mutant("b.rs", "arith.add_to_sub", Outcome::NoCoverage, index))); + + let findings = analyze(&mutants, &timing(1, 1, 100)); + let finding = find(&findings, "uncovered").expect("expected an uncovered finding"); + + assert!(finding.headline.contains("20 of 100"), "{}", finding.headline); + assert!(finding.remedy.contains("Write tests"), "{}", finding.remedy); + assert_eq!( + finding.detail, + ["they count against the score, because untested code is the finding"] + ); + assert_eq!(finding.cost, "—"); + } + + #[test] + fn uncovered_requires_the_population_and_share_thresholds_inclusively() { + assert!( + uncovered(Summary { + killed: 45, + uncovered: 5, + ..Summary::default() + }) + .is_some() + ); + assert!( + uncovered(Summary { + killed: 44, + uncovered: 5, + ..Summary::default() + }) + .is_none() + ); + assert!( + uncovered(Summary { + killed: 46, + uncovered: 4, + ..Summary::default() + }) + .is_none() + ); + } + + #[test] + fn yields_rank_families_by_survivors_per_cpu_hour() { + let mutants = vec![ + mutant("a.rs", "stmt.delete", Outcome::Survived, 1000), + mutant("a.rs", "literal.int_bump", Outcome::Survived, 100_000), + ]; + + let rows = yields(&mutants); + + assert_eq!(rows[0].family, "stmt"); + assert_eq!(rows[1].family, "literal"); + assert!(rows[0].per_cpu_hour() > rows[1].per_cpu_hour()); + } + + #[test] + fn yields_aggregate_every_field_and_break_ties_by_family() { + let mutants = vec![ + mutant("a.rs", "z.one", Outcome::Killed, 1000), + mutant("a.rs", "z.two", Outcome::Survived, 2000), + mutant("a.rs", "a.one", Outcome::Survived, 3000), + ]; + let rows = yields(&mutants); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].family, "a"); + assert_eq!(rows[1].family, "z"); + assert_eq!((rows[1].mutants, rows[1].cpu, rows[1].survivors), (2, Duration::from_secs(3), 1)); + } + + #[test] + fn a_mutator_without_a_family_is_its_own_family() { + assert_eq!(family_of("relational.lt_to_le"), "relational"); + assert_eq!(family_of("odd"), "odd"); + } + + #[test] + fn a_tiny_run_is_never_diagnosed_by_share() { + // Every file in a two-mutant run is half the population, which is arithmetic, not evidence. + let mutants = vec![ + mutant("a.rs", "relational.lt_to_le", Outcome::NoCoverage, 100), + mutant("b.rs", "arith.add_to_sub", Outcome::CompileError, 100), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + + assert!(find(&findings, "hot-file").is_none(), "{findings:?}"); + assert!(find(&findings, "unviable").is_none(), "{findings:?}"); + assert!(find(&findings, "uncovered").is_none(), "{findings:?}"); + } + + #[test] + fn a_family_dominating_a_few_seconds_is_not_a_finding() { + // 90% of six seconds is not worth anybody's attention. + let mutants = vec![ + mutant("a.rs", "literal.int_bump", Outcome::Killed, 5000), + mutant("a.rs", "relational.lt_to_le", Outcome::Survived, 500), + ]; + + let findings = analyze(&mutants, &timing(1, 1, 100)); + + assert!(find(&findings, "low-yield").is_none(), "{findings:?}"); + } +} diff --git a/crates/cargo-gamma-lib/src/advise/finding.rs b/crates/cargo-gamma-lib/src/advise/finding.rs new file mode 100644 index 000000000..dfbad8693 --- /dev/null +++ b/crates/cargo-gamma-lib/src/advise/finding.rs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/// One diagnosis. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Finding { + /// A stable identifier, so a finding can be referred to, suppressed or searched for. + pub code: &'static str, + + /// The measured symptom, in one line. + pub headline: String, + + /// Supporting measurements, one per line. + pub detail: Vec, + + /// What to do about it. + pub remedy: String, + + /// What taking the remedy costs in signal. Never omitted, never softened. + pub cost: String, +} diff --git a/crates/cargo-gamma-lib/src/advise/mod.rs b/crates/cargo-gamma-lib/src/advise/mod.rs new file mode 100644 index 000000000..a3489d94d --- /dev/null +++ b/crates/cargo-gamma-lib/src/advise/mod.rs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Diagnosis of a mutation run: where the time went, and what can be done about it. +//! +//! Mutation testing is the kind of tool that gets adopted enthusiastically, runs for four hours, +//! and is then quietly deleted from the CI configuration. The run itself does not explain why it +//! was slow, so the only remedies available to a frustrated user are the blunt ones — fewer +//! operators, fewer files, or nothing at all — chosen without knowing what they cost in signal. +//! +//! This module turns a completed run into a list of findings. Each is a measured symptom, a named +//! cause, a remedy, and the signal cost of taking that remedy. The last part is the one that +//! matters: every mitigation here trades information for time, and a recommendation that hides the +//! trade is worse than no recommendation, because it will be taken. +mod analysis; +mod finding; +mod render; +mod text; +mod timing; +mod yield_; + +pub use analysis::{analyze, analyze_run, yields}; +pub use finding::Finding; +pub use render::{Layout, render_markdown}; +pub use text::human; +pub use timing::Timing; +pub use yield_::Yield; diff --git a/crates/cargo-gamma-lib/src/advise/render.rs b/crates/cargo-gamma-lib/src/advise/render.rs new file mode 100644 index 000000000..848a29172 --- /dev/null +++ b/crates/cargo-gamma-lib/src/advise/render.rs @@ -0,0 +1,629 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Rendering findings and yields as plain text or as Markdown. + +use core::fmt::Write as _; + +use super::text::{human, sentence, share, slug}; +use super::{Finding, Timing, Yield}; +use crate::model::{Outcome, Summary}; + +/// The heading of the section reporting what the run cost and decided. +const RUN_HEADING: &str = "This run"; + +/// The heading of the section holding the diagnoses. +const FINDINGS_HEADING: &str = "Findings"; + +/// The heading of the per-family cost and value table. +const YIELD_HEADING: &str = "Yield by mutator family"; + +/// The heading of the definitions. +const GLOSSARY_HEADING: &str = "What the verdicts mean"; + +/// Where the rendered Markdown is going to be read. +/// +/// The two destinations want genuinely different documents, not the same one at two sizes. A file +/// is opened on purpose by someone who wants the whole picture and needs to navigate it; a job +/// summary panel is scrolled past by someone who did not ask for it, sits under a heading the CI +/// renderer already owns, and has just been told the score and the verdict counts by the panel +/// above it. Repeating that there would be noise, and a level-one title nested under it would be +/// malformed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Layout { + /// A standalone file: title, table of contents, and what the run cost. + #[default] + Document, + + /// A fragment appended to something that already has a heading and a score. + Embedded, +} + +impl Layout { + /// The heading prefix for a top-level section. + const fn section(self) -> &'static str { + match self { + Self::Document => "##", + Self::Embedded => "###", + } + } + + /// The heading prefix for one finding. + const fn subsection(self) -> &'static str { + match self { + Self::Document => "###", + Self::Embedded => "####", + } + } +} + +/// Renders the diagnosis and the family table as a structured Markdown document. +/// +/// The same analysis as the console rendering, in the one format that travels: a job summary panel, +/// a pull request comment, an issue, a file checked into a repository. Prose is left unwrapped +/// because a Markdown renderer reflows to the reader's width, and hard-wrapping to ours would +/// fight it. +/// +/// The document is built to be navigated rather than read start to finish. A run that produced +/// eight findings is exactly the run whose reader wants to jump to one of them, so every section +/// is linkable, findings are numbered, and the table of contents names them. +#[must_use] +pub fn render_markdown(findings: &[Finding], rows: &[Yield], summary: Summary, timing: &Timing, layout: Layout) -> String { + let mut out = String::new(); + + if layout == Layout::Document { + out.push_str("# Mutation testing advice\n\n"); + + out.push_str( + "What this run cost, what it found, and what could be changed — with the signal cost \ + of every change stated alongside it.\n\n", + ); + + write_contents(&mut out, findings, rows); + write_outcome(&mut out, summary, timing); + } + + write_findings(&mut out, findings, layout); + write_yields(&mut out, rows, layout); + write_glossary(&mut out, layout); + + out +} + +/// Writes the table of contents. +fn write_contents(out: &mut String, findings: &[Finding], rows: &[Yield]) { + out.push_str("## Contents\n\n"); + let _ = writeln!(out, "- [{RUN_HEADING}](#{})", slug(RUN_HEADING)); + let _ = writeln!(out, "- [{FINDINGS_HEADING}](#{})", slug(FINDINGS_HEADING)); + + for (index, finding) in findings.iter().enumerate() { + let heading = finding_heading(index, finding); + + let _ = writeln!(out, " - [{heading}](#{})", slug(&heading)); + } + + if !rows.is_empty() { + let _ = writeln!(out, "- [{YIELD_HEADING}](#{})", slug(YIELD_HEADING)); + } + + let _ = writeln!(out, "- [{GLOSSARY_HEADING}](#{})\n", slug(GLOSSARY_HEADING)); +} + +/// Writes what the run cost and what it decided, as two tables. +/// +/// The verdicts and the cost are separate tables because they answer separate questions, and a +/// single table mixing counts with durations invites reading down a column that means two things. +/// +/// The verdict rows are enumerated from [`Outcome::ALL`] and scored through [`Outcome::scoring`] +/// rather than listed here, because this table is read against the score on its last row. A +/// hand-written list is a list that can fall behind the enum, and it did: with the three counters +/// it omitted, a run stopped entirely by its memory ceiling rendered a header, a separator, and a +/// perfect score over nothing at all. +fn write_outcome(out: &mut String, summary: Summary, timing: &Timing) { + let _ = writeln!(out, "## {RUN_HEADING}\n"); + let _ = writeln!(out, "| Verdict | Mutants | Share of score |\n|---|---:|---:|"); + + // #[gamma::skip(literal.int_decrement, reason = "a scored row implies valid is at least one, while valid zero emits no scored rows, so max(0) and max(1) are observationally identical")] + let valid = f64::from(summary.valid().max(1)); + + for outcome in Outcome::ALL { + let count = summary.count(outcome); + + if count == 0 { + continue; + } + + let share = if outcome.is_valid() { + format!("{:.1}%", f64::from(count) * 100.0 / valid) + } else { + "not scored".to_owned() + }; + + let _ = writeln!(out, "| {} | {count} | {share} |", verdict_label(outcome)); + } + + let _ = writeln!(out, "| **Score** | | **{:.1}%** |\n", summary.score()); + + let executed = timing.wall.saturating_sub(timing.build + timing.baseline); + + let _ = writeln!(out, "| Cost | Time | Share of run |\n|---|---:|---:|"); + + for (label, spent) in [ + ("Build", timing.build), + ("Baseline", timing.baseline), + ("Testing mutants", executed), + ] { + let _ = writeln!(out, "| {label} | {} | {} |", human(spent), share(spent, timing.wall)); + } + + let _ = writeln!(out, "| **Total** | **{}** | at {} jobs |\n", human(timing.wall), timing.jobs); +} + +/// How one verdict is named in the run table. +/// +/// A `match` so that a new outcome has to be given a name here before it can be rendered, and +/// spelled out in this document's own words rather than in the short forms the console prints: the +/// audience for this file is whoever it was forwarded to. +const fn verdict_label(outcome: Outcome) -> &'static str { + match outcome { + Outcome::Killed => "Killed", + Outcome::Timeout => "Timed out", + Outcome::OutOfMemory => "Stopped by the memory limit", + Outcome::Survived => "Survived", + Outcome::NoCoverage => "Uncovered", + Outcome::Flaky => "Flaky", + Outcome::CompileError => "Unviable", + Outcome::Ignored => "Ignored", + Outcome::NotBuilt => "Not built", + Outcome::Pending => "Not run", + } +} + +/// Writes the findings, numbered so they can be referred to by position. +fn write_findings(out: &mut String, findings: &[Finding], layout: Layout) { + let _ = writeln!(out, "{} {FINDINGS_HEADING}\n", layout.section()); + + if findings.is_empty() { + out.push_str( + "Nothing crossed its threshold. Every check this tool makes looks for a cost that is \ + large enough to be worth trading signal for, and none of them fired.\n\n", + ); + + return; + } + + out.push_str( + "Findings follow a fixed diagnostic order: run-wide costs first, then costly verdicts, \ + population concentration, mutator yield, and uncovered code.\n\n", + ); + + for (index, finding) in findings.iter().enumerate() { + let _ = writeln!(out, "{} {}\n", layout.subsection(), finding_heading(index, finding)); + let _ = writeln!(out, "Finding code: `{}`\n", finding.code); + + if !finding.detail.is_empty() { + out.push_str("What was measured:\n\n"); + + for line in &finding.detail { + let _ = writeln!(out, "- {line}"); + } + + out.push('\n'); + } + + let _ = writeln!(out, "> **Remedy.** {}\n>", sentence(&finding.remedy)); + + // The cost is never dropped, even here. A remedy quoted without what it gives up is how a + // team ends up raising a score by measuring less. + let _ = writeln!(out, "> **Costs.** {}\n", sentence(&finding.cost)); + } +} + +/// Writes the per-family cost and value table. +fn write_yields(out: &mut String, rows: &[Yield], layout: Layout) { + if rows.is_empty() { + return; + } + + let _ = writeln!(out, "{} {YIELD_HEADING}\n", layout.section()); + + out.push_str( + "Survivors per CPU-hour is what makes families comparable: it is the rate at which a \ + family bought the only thing a mutation run produces. A family near the bottom of this \ + table is the cheapest thing to turn off, and the last column says what turning it off \ + would have cost this run.\n\n", + ); + + out.push_str("| Family | Mutants | CPU | Survivors | Survivors/CPU-h |\n|---|---:|---:|---:|---:|\n"); + + for row in rows { + let _ = writeln!( + out, + "| `{}` | {} | {} | {} | {:.1} |", + row.family, + row.mutants, + human(row.cpu), + row.survivors, + row.per_cpu_hour() + ); + } + + out.push('\n'); +} + +/// Writes the definitions the rest of the document leans on. +/// +/// Included because this file is written to be shared, and the person it gets forwarded to is +/// usually not the person who ran the tool. +fn write_glossary(out: &mut String, layout: Layout) { + let _ = writeln!(out, "{} {GLOSSARY_HEADING}\n", layout.section()); + + for (term, meaning) in [ + ( + "Killed", + "a test failed while the mutant was active, which is the outcome you want.", + ), + ( + "Survived", + "every test still passed with the mutant active, so nothing asserted on the behaviour it changed.", + ), + ( + "Timed out", + "the suite never finished with the mutant active. Counted as undetected because no assertion rejected the change.", + ), + ( + "Out of memory", + "the suite crossed its memory ceiling with the mutant active. Counted as undetected because no assertion rejected the change.", + ), + ( + "Uncovered", + "no test reaches the code at all. Counted against the score exactly as a survivor is.", + ), + ( + "Unviable", + "the mutant did not compile, so it says nothing about the tests and is left out of the score.", + ), + ( + "Baseline", + "how long the suite takes with no mutant active. Every mutant pays this, so it multiplies by the population.", + ), + ] { + let _ = writeln!(out, "- **{term}** — {meaning}"); + } + + out.push('\n'); +} + +/// The heading for one finding, numbered by position. +fn finding_heading(index: usize, finding: &Finding) -> String { + format!("{}. {}", index + 1, sentence(&finding.headline)) +} + +#[cfg(test)] +mod tests { + use core::time::Duration; + + use super::*; + use crate::advise::analysis::{analyze, yields}; + use crate::testing::advise_fixture::{mutant, timing}; + + /// The verdict rows of the run table, as `(label, count, share)`. + /// + /// Taken from between the table's separator and the score line, so that neither the cost table + /// below — whose rows are the same shape — nor the score itself can stand in for a verdict row + /// a test is asserting the absence of. + fn run_table_rows(document: &str) -> Vec<(String, u32, String)> { + document + .lines() + .skip_while(|line| !line.starts_with("| Verdict |")) + .skip(2) + .take_while(|line| !line.starts_with("| **Score**")) + .map(|line| { + let cells: Vec<&str> = line.trim_matches('|').split('|').map(str::trim).collect(); + + assert_eq!(cells.len(), 3, "a verdict row has three cells: {line}"); + + ( + cells[0].to_owned(), + cells[1].parse().expect("a verdict row counts mutants"), + cells[2].to_owned(), + ) + }) + .collect() + } + + /// Every finding this crate constructs carries at least one measurement, but the field itself + /// is a plain `Vec` a caller could hand over empty; the section it introduces has to disappear + /// along with it; showing an empty "What was measured" header would read as though the + /// renderer forgot to fill it in rather than as though there was nothing to say. + #[test] + fn a_finding_with_no_detail_lines_omits_the_measured_section() { + let finding = Finding { + code: "bare", + headline: "a headline with nothing to back it up".to_owned(), + detail: Vec::new(), + remedy: "nothing to do".to_owned(), + cost: "none".to_owned(), + }; + + let document = render_markdown(&[finding], &[], Summary::default(), &timing(1, 1, 100), Layout::Document); + + assert!(!document.contains("What was measured"), "{document}"); + } + + #[test] + fn the_findings_introduction_describes_the_diagnostic_order() { + let finding = Finding { + code: "example", + headline: "example".to_owned(), + detail: Vec::new(), + remedy: "act".to_owned(), + cost: "none".to_owned(), + }; + let document = render_markdown(&[finding], &[], Summary::default(), &timing(1, 1, 100), Layout::Document); + + assert!(document.contains( + "Findings follow a fixed diagnostic order: run-wide costs first, then costly verdicts, \ + population concentration, mutator yield, and uncovered code.\n\n" + )); + assert!(!document.contains("remedy costs no signal"), "{document}"); + } + + #[test] + fn every_contents_entry_points_at_a_heading_that_exists() { + let mutants = vec![ + mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 100), + mutant("a.rs", "arith.add_to_sub", Outcome::Survived, 100), + ]; + + let timing = timing(50, 5, 100); + let findings = analyze(&mutants, &timing); + let summary = Summary::of(&mutants); + let document = render_markdown(&findings, &yields(&mutants), summary, &timing, Layout::Document); + + assert!(!findings.is_empty(), "the fixture must produce something to link to"); + + // A table of contents whose links do not resolve is worse than none, because it is only + // discovered to be broken by someone who already had to scroll. + let anchors: Vec = document + .lines() + .filter_map(|line| { + line.trim_start() + .strip_prefix("- [") + .or_else(|| line.trim_start().strip_prefix(" - [")) + }) + .filter_map(|entry| entry.split("](#").nth(1).map(|tail| tail.trim_end_matches(')').to_owned())) + .collect(); + + let headings: Vec = document + .lines() + .filter_map(|line| line.strip_prefix("## ").or_else(|| line.strip_prefix("### "))) + .map(slug) + .collect(); + + assert!(anchors.len() >= 4, "{document}"); + + for anchor in anchors { + assert!(headings.contains(&anchor), "`{anchor}` is not a heading in:\n{document}"); + } + } + + #[test] + fn a_run_with_nothing_to_report_still_says_what_it_cost() { + let mutants = vec![mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 100)]; + let timing = timing(1, 1, 100); + let document = render_markdown(&[], &yields(&mutants), Summary::of(&mutants), &timing, Layout::Document); + + assert!(document.contains("## This run"), "{document}"); + assert!(document.contains("Nothing crossed its threshold"), "{document}"); + assert!(document.contains("| Killed | 1 | 100.0% |"), "{document}"); + assert!(document.contains("| **Score** | | **100.0%** |"), "{document}"); + } + + /// Every mutant the run produced appears in the table, and only the scored ones take a share. + /// + /// The population is deliberately lopsided, and deliberately includes the three counters most + /// easily left without a row at all — memory exhaustion, a flake, and a mutant the build never + /// compiled. With those at zero the table cannot be caught omitting them: a run whose mutants + /// were all stopped by a memory ceiling would render a header and separator with nothing + /// between them, and an all-zero fixture reproduces none of that. Summing the rows + /// back up is what pins it, because a row that is missing, duplicated or reading the wrong + /// counter all fail the same assertion. + #[test] + fn every_verdict_in_the_population_is_accounted_for_in_the_run_table() { + let summary = Summary { + killed: 3, + survived: 1, + timeout: 2, + out_of_memory: 4, + flaky: 5, + unviable: 6, + ignored: 7, + uncovered: 8, + not_built: 9, + pending: 10, + }; + + let document = render_markdown(&[], &[], summary, &timing(1, 1, 10), Layout::Document); + let rows = run_table_rows(&document); + let counted: u32 = rows.iter().map(|(_, count, _)| count).sum(); + let scored: u32 = rows + .iter() + .filter(|(_, _, share)| share != "not scored") + .map(|(_, count, _)| count) + .sum(); + + assert_eq!(counted, 55, "every mutant in the population must have a row: {document}"); + assert_eq!(scored, summary.valid(), "the scoring rows must sum to the denominator: {document}"); + + // The score's numerator is on the page too: only assertion-driven kills. + let detected: u32 = rows + .iter() + .filter(|(label, _, _)| label == "Killed") + .map(|(_, count, _)| count) + .sum(); + + assert_eq!( + detected, + summary.detected(), + "the detected rows must sum to the numerator: {document}" + ); + } + + /// A run whose every mutant exhausted its memory ceiling scores 0%, and the table under that + /// score has to identify the undetected outcome rather than being empty. + #[test] + fn a_run_of_nothing_but_memory_exhaustion_still_has_a_breakdown() { + let summary = Summary { + out_of_memory: 3, + ..Summary::default() + }; + + let document = render_markdown(&[], &[], summary, &timing(1, 1, 10), Layout::Document); + + assert_eq!( + run_table_rows(&document), + vec![("Stopped by the memory limit".to_owned(), 3, "100.0%".to_owned())], + "{document}" + ); + assert!(document.contains("| **Score** | | **0.0%** |"), "{document}"); + } + + #[test] + fn unscored_outcomes_are_named_as_not_scored_in_the_run_table() { + let summary = Summary { + flaky: 1, + killed: 1, + survived: 0, + timeout: 0, + out_of_memory: 1, + unviable: 1, + ignored: 1, + uncovered: 0, + not_built: 1, + pending: 1, + }; + let timing = timing(1, 1, 10); + let document = render_markdown(&[], &[], summary, &timing, Layout::Document); + + // Flakes, unviable, ignored, unbuilt and pending mutants are not in the denominator, so the + // table must not present them as a share of the mutation score. Memory exhaustion is in the + // denominator and therefore still has a scored share despite earning no detection credit. + assert_eq!(document.matches("not scored").count(), 5, "{document}"); + assert!(document.contains("| Stopped by the memory limit | 1 | 50.0% |"), "{document}"); + } + + #[test] + fn the_document_renders_every_table_row_and_explanatory_section() { + let finding = Finding { + code: "example", + headline: "example finding".to_owned(), + detail: vec!["first measurement".to_owned(), "second measurement".to_owned()], + remedy: "fix it".to_owned(), + cost: "some signal".to_owned(), + }; + let row = Yield { + family: "relational".to_owned(), + mutants: 2, + cpu: Duration::from_mins(30), + survivors: 3, + }; + let summary = Summary { + killed: 1, + timeout: 1, + survived: 1, + uncovered: 1, + unviable: 1, + ignored: 1, + pending: 1, + ..Summary::default() + }; + let timing = Timing { + build: Duration::from_secs(10), + baseline: Duration::from_secs(20), + wall: Duration::from_secs(100), + jobs: 4, + }; + let document = render_markdown(&[finding], &[row], summary, &timing, Layout::Document); + + assert!(document.starts_with("# Mutation testing advice\n\nWhat this run cost, what it found, and what could be changed — with the signal cost of every change stated alongside it.\n\n"), "{document}"); + assert!(document.contains("## Contents\n\n- [This run](#this-run)\n- [Findings](#findings)\n - [1. Example finding](#1-example-finding)\n- [Yield by mutator family](#yield-by-mutator-family)\n- [What the verdicts mean](#what-the-verdicts-mean)\n"), "{document}"); + for row in [ + "| Killed | 1 | 25.0% |", + "| Timed out | 1 | 25.0% |", + "| Survived | 1 | 25.0% |", + "| Uncovered | 1 | 25.0% |", + "| Unviable | 1 | not scored |", + "| Ignored | 1 | not scored |", + "| Not run | 1 | not scored |", + "| Build | 10.0s | 10% |", + "| Baseline | 20.0s | 20% |", + "| Testing mutants | 70.0s | 70% |", + "| **Total** | **2m** | at 4 jobs |", + ] { + assert!(document.contains(row), "missing `{row}` in:\n{document}"); + } + assert!(document.contains("Findings follow a fixed diagnostic order: run-wide costs first, then costly verdicts, population concentration, mutator yield, and uncovered code.\n\n"), "{document}"); + assert!( + document.contains( + "What was measured:\n\n- first measurement\n- second measurement\n\n> **Remedy.** Fix it\n>\n> **Costs.** Some signal\n" + ), + "{document}" + ); + assert!(document.contains("Survivors per CPU-hour is what makes families comparable: it is the rate at which a family bought the only thing a mutation run produces."), "{document}"); + assert!(document.contains("| Family | Mutants | CPU | Survivors | Survivors/CPU-h |\n|---|---:|---:|---:|---:|\n| `relational` | 2 | 30m | 3 | 6.0 |\n\n"), "{document}"); + for definition in [ + "- **Killed** — a test failed while the mutant was active, which is the outcome you want.", + "- **Survived** — every test still passed with the mutant active, so nothing asserted on the behaviour it changed.", + "- **Timed out** — the suite never finished with the mutant active. Counted as undetected because no assertion rejected the change.", + "- **Out of memory** — the suite crossed its memory ceiling with the mutant active. Counted as undetected because no assertion rejected the change.", + "- **Uncovered** — no test reaches the code at all. Counted against the score exactly as a survivor is.", + "- **Unviable** — the mutant did not compile, so it says nothing about the tests and is left out of the score.", + "- **Baseline** — how long the suite takes with no mutant active. Every mutant pays this, so it multiplies by the population.", + ] { + assert!(document.contains(definition), "missing `{definition}` in:\n{document}"); + } + assert!(document.ends_with("\n\n"), "the glossary must end as a Markdown block:\n{document}"); + } + + #[test] + fn empty_rows_are_absent_from_both_contents_and_body() { + let document = render_markdown(&[], &[], Summary::default(), &timing(1, 1, 10), Layout::Document); + assert!(!document.contains("- [Yield by mutator family]"), "{document}"); + assert!(!document.contains("## Yield by mutator family"), "{document}"); + assert!(!document.contains("| Killed | 0 |"), "{document}"); + } + + #[test] + fn a_run_with_no_wall_time_reports_no_share_rather_than_a_division_by_zero() { + assert_eq!(share(Duration::from_secs(1), Duration::ZERO), "—"); + } + + #[test] + fn the_embedded_layout_nests_under_the_heading_its_host_already_wrote() { + let mutants = vec![mutant("a.rs", "relational.lt_to_le", Outcome::Killed, 100)]; + let timing = timing(50, 5, 100); + let findings = analyze(&mutants, &timing); + let summary = Summary::of(&mutants); + let panel = render_markdown(&findings, &yields(&mutants), summary, &timing, Layout::Embedded); + + assert!(!panel.contains("# Mutation testing advice"), "{panel}"); + assert!(!panel.contains("## Contents"), "{panel}"); + + // The job summary panel states the score and the verdict counts itself, directly above. + assert!(!panel.contains(RUN_HEADING), "{panel}"); + assert!(panel.starts_with("### Findings"), "{panel}"); + } + + #[test] + fn finding_headings_are_numbered_and_capitalized() { + let finding = Finding { + code: "x", + headline: "lowercase headline".to_owned(), + detail: Vec::new(), + remedy: String::new(), + cost: String::new(), + }; + assert_eq!(finding_heading(1, &finding), "2. Lowercase headline"); + } +} diff --git a/crates/cargo-gamma-lib/src/advise/text.rs b/crates/cargo-gamma-lib/src/advise/text.rs new file mode 100644 index 000000000..01919dc1f --- /dev/null +++ b/crates/cargo-gamma-lib/src/advise/text.rs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Shared phrasing helpers: durations, counts, shares, wrapping, and heading slugs. + +use core::time::Duration; + +/// Pluralizes a count's noun, because "1 mutants" reads as a bug in the tool. +pub(super) fn plural(count: u32, noun: &str) -> String { + if count == 1 { noun.to_owned() } else { format!("{noun}s") } +} + +/// Renders a duration the way a person would say it. +#[must_use] +pub fn human(duration: Duration) -> String { + let seconds = duration.as_secs_f64(); + + if seconds < 1.0 { + return format!("{}ms", duration.as_millis()); + } + + if seconds < 90.0 { + return format!("{seconds:.1}s"); + } + + let minutes = seconds / 60.0; + + if minutes < 90.0 { + return format!("{minutes:.0}m"); + } + + format!("{:.1}h", minutes / 60.0) +} + +/// Capitalizes the first letter, for prose written to follow a lowercase console label. +/// +/// The findings phrase themselves to sit after `remedy:` and `costs:`, which reads correctly there +/// and like a typo after a bold Markdown heading. +pub(super) fn sentence(text: &str) -> String { + let mut chars = text.chars(); + + chars + .next() + .map_or_else(String::new, |first| first.to_uppercase().collect::() + chars.as_str()) +} + +/// One duration as a percentage of another, for a table column. +pub(super) fn share(part: Duration, whole: Duration) -> String { + if whole.is_zero() { + return "—".to_owned(); + } + + format!("{:.0}%", part.as_secs_f64() * 100.0 / whole.as_secs_f64()) +} + +/// A GitHub-style anchor for a heading, so the table of contents actually resolves. +/// +/// Matches the rule GitHub, GitLab and most static site generators share: lowercase, drop anything +/// that is not a letter, a digit, a space or a hyphen, then turn spaces into hyphens. +pub(super) fn slug(heading: &str) -> String { + let mut anchor = String::with_capacity(heading.len()); + + for character in heading.chars() { + if character.is_alphanumeric() { + anchor.extend(character.to_lowercase()); + } else if character == ' ' || character == '-' { + anchor.push('-'); + } + } + + anchor +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_of_one_are_not_pluralized() { + assert_eq!(plural(1, "mutant"), "mutant"); + assert_eq!(plural(0, "mutant"), "mutants"); + assert_eq!(plural(2, "mutant"), "mutants"); + } + + #[test] + fn durations_read_the_way_a_person_says_them() { + assert_eq!(human(Duration::from_millis(250)), "250ms"); + assert_eq!(human(Duration::from_secs(9)), "9.0s"); + assert_eq!(human(Duration::from_mins(10)), "10m"); + assert_eq!(human(Duration::from_hours(2)), "2.0h"); + assert_eq!(human(Duration::from_secs(90)), "2m"); + assert_eq!(human(Duration::from_mins(90)), "1.5h"); + } + + #[test] + fn sentences_and_shares_cover_empty_and_nonzero_inputs() { + assert_eq!(sentence(""), ""); + assert_eq!(sentence("hello world"), "Hello world"); + assert_eq!(share(Duration::from_secs(1), Duration::from_secs(4)), "25%"); + assert_eq!(share(Duration::from_secs(1), Duration::ZERO), "—"); + } + + #[test] + fn a_slug_matches_the_anchor_a_markdown_renderer_would_generate() { + assert_eq!(slug("Yield by mutator family"), "yield-by-mutator-family"); + assert_eq!(slug("1. 97% of the run was the build"), "1-97-of-the-run-was-the-build"); + } +} diff --git a/crates/cargo-gamma-lib/src/advise/timing.rs b/crates/cargo-gamma-lib/src/advise/timing.rs new file mode 100644 index 000000000..6e4fe9033 --- /dev/null +++ b/crates/cargo-gamma-lib/src/advise/timing.rs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; + +/// What a run cost, beyond the verdicts. +#[derive(Debug, Clone, Copy)] +pub struct Timing { + /// How long the single instrumented build took. + pub build: Duration, + + /// Elapsed wall time for the suite with no mutant active. + pub baseline: Duration, + + /// Total wall time for the whole run. + pub wall: Duration, + + /// How many mutants were tested at once. + pub jobs: usize, +} diff --git a/crates/cargo-gamma-lib/src/advise/yield_.rs b/crates/cargo-gamma-lib/src/advise/yield_.rs new file mode 100644 index 000000000..3047e18d5 --- /dev/null +++ b/crates/cargo-gamma-lib/src/advise/yield_.rs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; + +/// Cost and value for one mutator family. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Yield { + /// The family name, such as `relational`. + pub family: String, + + /// How many mutants it produced. + pub mutants: u32, + + /// CPU time spent deciding them. + pub cpu: Duration, + + /// How many of them survived — the only output of a mutation run that teaches anything. + pub survivors: u32, +} + +impl Yield { + /// Survivors found per CPU-hour, the ratio that makes families comparable. + #[must_use] + pub fn per_cpu_hour(&self) -> f64 { + let hours = self.cpu.as_secs_f64() / 3600.0; + + if hours <= 0.0 { + return 0.0; + } + + f64::from(self.survivors) / hours + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_family_with_no_time_has_no_ratio_rather_than_an_infinite_one() { + let row = Yield { + family: "x".to_owned(), + mutants: 1, + cpu: Duration::ZERO, + survivors: 1, + }; + + assert!((row.per_cpu_hour() - 0.0).abs() < f64::EPSILON); + } + + #[test] + fn the_ratio_is_survivors_per_cpu_hour() { + let row = Yield { + family: "x".to_owned(), + mutants: 7, + cpu: Duration::from_mins(30), + survivors: 3, + }; + assert!((row.per_cpu_hour() - 6.0).abs() < f64::EPSILON); + } +} diff --git a/crates/cargo-gamma-lib/src/bounds.rs b/crates/cargo-gamma-lib/src/bounds.rs new file mode 100644 index 000000000..57f05ba48 --- /dev/null +++ b/crates/cargo-gamma-lib/src/bounds.rs @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Range checks for the numbers a user can supply. +//! +//! Every one of these values reaches a `Duration` eventually, and `Duration`'s float constructors +//! *panic* on a negative, `NaN` or over-large argument. A mistyped flag or a bad line in +//! `gamma.toml` should be a diagnostic, not a crash report, so the checks live here and are +//! applied at both entry points rather than at the point of use. + +use core::time::Duration; + +/// The largest multiplier worth accepting. +/// +/// The bound exists so that scaling a baseline by it cannot overflow a `Duration`; the exact value +/// is arbitrary, since nothing sane is anywhere near it. +/// +/// `pub` within this private module so the crate's proc-macro agreement test can reach it through +/// the `internals` facade and pin the proc-macro's hand-copied `MOST_FACTOR` against it. +pub const MOST_FACTOR: f64 = 1e6; + +/// The largest timeout accepted from configuration: one year. +pub(crate) const MOST_SECONDS: u64 = 365 * 24 * 60 * 60; +const MOST_SECONDS_F64: f64 = 31_536_000.0; + +/// Checks a number of seconds that a `Duration` has to be able to represent. +/// +/// # Errors +/// +/// Returns a message naming the offending text if it is not a positive, finite, representable +/// number of seconds. +pub fn seconds(text: &str, value: f64) -> Result { + let value = positive(text, value)?; + + if value > MOST_SECONDS_F64 { + return Err(format!("`{text}` is unreasonably large; the most is {MOST_SECONDS} seconds")); + } + + Duration::try_from_secs_f64(value) + .map(|_duration| value) + .map_err(|_cause| format!("`{text}` is too large to be a duration")) +} + +/// Checks a multiplier, bounded so that scaling a baseline by it cannot overflow a `Duration`. +/// +/// # Errors +/// +/// Returns a message naming the offending text if it is not a positive, finite, reasonable factor. +pub fn factor(text: &str, value: f64) -> Result { + let value = positive(text, value)?; + + if value > MOST_FACTOR { + return Err(format!("`{text}` is unreasonably large; the most is {MOST_FACTOR}")); + } + + Ok(value) +} + +/// Parses a memory size, with or without a binary unit suffix. +/// +/// Sizes are the one quantity here people habitually write with a unit, and a flag that silently +/// read `512M` as five hundred and twelve bytes would install a ceiling no test could ever fit +/// under and report every mutant as caught. A suffix is therefore understood rather than rejected, +/// and an unrecognized one is an error rather than a number quietly taken from the digits in front +/// of it. +/// +/// The units are binary — `K` is 1024 — because that is what every other memory figure in this tool +/// and in the kernel interfaces behind it means. +/// +/// # Errors +/// +/// Returns a message naming the offending text if it is not a positive, finite, representable +/// number of at least one byte, or if its suffix is not a unit. +pub fn size(text: &str) -> Result { + let trimmed = text.trim(); + let digits = trimmed.trim_end_matches(|character: char| character.is_ascii_alphabetic()); + let suffix = trimmed.strip_prefix(digits).unwrap_or_default(); + + let scale: u64 = match suffix.to_ascii_lowercase().as_str() { + "" | "b" => 1, + "k" | "kb" | "kib" => 1024, + "m" | "mb" | "mib" => 1024 * 1024, + "g" | "gb" | "gib" => 1024 * 1024 * 1024, + "t" | "tb" | "tib" => 1024 * 1024 * 1024 * 1024, + _other => return Err(format!("`{text}` does not end in a size unit such as `KiB`, `MiB` or `GiB`")), + }; + + let count: f64 = digits + .trim() + .parse() + .map_err(|_cause| format!("`{text}` is not a number of bytes"))?; + + let _positive = positive(text, count)?; + + #[expect(clippy::cast_precision_loss, reason = "the comparison only needs to be right near the boundary")] + let bytes = count * scale as f64; + + #[expect(clippy::cast_precision_loss, reason = "as above")] + let most = u64::MAX as f64; + + if bytes >= most { + return Err(format!("`{text}` is too large to be a number of bytes")); + } + + // Positivity was checked on the unscaled count, so a positive size that scales to less than one + // byte reaches this point and would truncate to a zero ceiling. A zero ceiling is not a small + // ceiling: every mutant exceeds it, so the whole population reports as detected and the run + // claims a perfect score off a typo. + if bytes < 1.0 { + return Err(format!("`{text}` is less than one byte")); + } + + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "the value is finite, positive and below `u64::MAX` by the checks above" + )] + let bytes = bytes as u64; + + Ok(bytes) +} + +/// Checks a percentage between zero and a hundred. +/// +/// # Errors +/// +/// Returns a message naming the offending text if it is outside that range. `NaN` is rejected +/// rather than passed through: it compares false against every score, so a gate set to it would +/// silently never fire. +pub fn percentage(text: &str, value: f64) -> Result { + if !value.is_finite() || !(0.0..=100.0).contains(&value) { + return Err(format!("`{text}` is not a percentage between 0 and 100")); + } + + Ok(value) +} + +/// Checks that a number is finite and greater than zero. +fn positive(text: &str, value: f64) -> Result { + if !value.is_finite() || value <= 0.0 { + return Err(format!("`{text}` must be a number greater than zero")); + } + + Ok(value) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_duration_that_would_panic_is_rejected_instead() { + // Each of these reaches `Duration::from_secs_f64`, which panics rather than returning. + _ = seconds("-5", -5.0).expect_err("out of range"); + _ = seconds("nan", f64::NAN).expect_err("out of range"); + _ = seconds("1e300", 1e300).expect_err("out of range"); + _ = seconds("0", 0.0).expect_err("out of range"); + _ = seconds("1.5", 1.5).expect("in range"); + assert_eq!(seconds(&MOST_SECONDS.to_string(), MOST_SECONDS_F64), Ok(MOST_SECONDS_F64)); + _ = seconds(&(MOST_SECONDS + 1).to_string(), MOST_SECONDS_F64 + 1.0).expect_err("past the usable deadline range"); + } + + #[test] + fn a_factor_is_positive_and_not_absurd() { + _ = factor("-1", -1.0).expect_err("out of range"); + _ = factor("nan", f64::NAN).expect_err("out of range"); + _ = factor("1e9", 1e9).expect_err("out of range"); + _ = factor("1.2", 1.2).expect("in range"); + assert_eq!(factor("1000000", MOST_FACTOR), Ok(MOST_FACTOR)); + } + + #[test] + fn a_size_understands_the_units_people_write() { + assert_eq!(size("1024"), Ok(1024)); + assert_eq!(size("1k"), Ok(1024)); + assert_eq!(size("512MiB"), Ok(512 * 1024 * 1024)); + assert_eq!(size(" 2GB "), Ok(2 * 1024 * 1024 * 1024)); + assert_eq!(size("1.5G"), Ok(1024 * 1024 * 1024 + 512 * 1024 * 1024)); + assert_eq!(size("1TiB"), Ok(1024_u64.pow(4))); + } + + #[test] + fn a_size_that_is_not_one_is_refused_rather_than_guessed_at() { + // Reading `512Mb/s` as five hundred and twelve would install a ceiling no suite could fit + // under, and every mutant would then be reported as caught by tests that never ran. + _ = size("512Mb/s").expect_err("not a size"); + _ = size("many").expect_err("not a size"); + _ = size("0").expect_err("not a size"); + _ = size("-1M").expect_err("not a size"); + _ = size("1e30G").expect_err("not a size"); + _ = size("18446744073709551616").expect_err("exactly one past u64::MAX"); + } + + #[test] + fn a_size_below_one_byte_is_refused_rather_than_truncated_to_a_zero_ceiling() { + // Positivity is checked on the unscaled count, so these pass it and then truncate. A zero + // ceiling is the worst possible failure: every mutant exceeds it, so the run reports a + // perfect score without a single test having judged anything. + assert_eq!(size("0.5"), Err("`0.5` is less than one byte".to_owned())); + assert_eq!(size("0.5B"), Err("`0.5B` is less than one byte".to_owned())); + assert_eq!(size("0.0001k"), Err("`0.0001k` is less than one byte".to_owned())); + + // The boundary itself is a legitimate size and must survive the new check. + assert_eq!(size("1"), Ok(1)); + assert_eq!(size("1B"), Ok(1)); + } + + #[test] + fn a_percentage_stays_within_its_range() { + // `NaN` is the interesting one: it compares false against every score, so a gate set to it + // would pass silently forever. + _ = percentage("nan", f64::NAN).expect_err("out of range"); + _ = percentage("150", 150.0).expect_err("out of range"); + _ = percentage("-1", -1.0).expect_err("out of range"); + _ = percentage("0", 0.0).expect("in range"); + _ = percentage("100", 100.0).expect("in range"); + } +} diff --git a/crates/cargo-gamma-lib/src/cfg/build.rs b/crates/cargo-gamma-lib/src/cfg/build.rs new file mode 100644 index 000000000..4dfd76aa9 --- /dev/null +++ b/crates/cargo-gamma-lib/src/cfg/build.rs @@ -0,0 +1,1773 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! What the build cargo will actually run compiles for. +//! +//! `rustc --print cfg` answers about the compiler's own host under its own defaults, and that is +//! not the build a run measures. Cargo is asked for a target, for a profile and with whatever +//! `RUSTFLAGS` the environment or the cargo configuration hold, and every one of those changes +//! which `#[cfg(...)]` predicates hold. Discovery that ignored them would classify target-only, +//! release-only or `--cfg`-gated code as absent and drop its mutants from the population, which +//! raises the score by removing the code nobody tested. +//! +//! So the settings are resolved here, from the same places cargo reads them, and handed to the +//! probe as command-line flags — because `rustc` itself does not read `RUSTFLAGS`, which is a +//! Cargo-facing variable, and inheriting it changes nothing at all. + +use std::process::Command; +use std::{env, fs}; + +use camino::{Utf8Path, Utf8PathBuf}; +use cargo_gamma_engine::cfg::{CfgSet, Verdict}; +use toml::{Table, Value}; + +use crate::HashMap; + +/// How many `inherits` hops a profile chain may take before it is called malformed. +const PROFILE_DEPTH: usize = 16; + +/// The parts of a cargo build that decide which configuration predicates hold. +/// +/// Built by [`Build::resolve`] from the settings cargo will read, or by +/// [`CargoOptions::cfg_build`](crate::exec::CargoOptions::cfg_build) from the options the run will +/// build with, so that discovery and the build cannot describe different compilations. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Build { + /// The target triple cargo will compile for, or `None` for the compiler's host. + pub target: Option, + + /// The custom predicates `--cfg` puts in force, as `loom` or `flavor="x"`. + pub cfgs: Vec, + + /// The codegen options that change which predicates hold, as `target-feature=+avx2`. + /// + /// Only the ones that decide a `cfg` are carried. A probe handed the whole of `RUSTFLAGS` + /// would fail on the first nightly-only or lint-shaped flag in it and take the entire + /// evaluation down with it, which costs far more than the predicates it would have refined. + pub codegen: Vec, + + /// Whether `debug_assertions` holds, or `None` when nothing consulted here can say. + /// + /// `None` is not "off". A profile whose chain cannot be followed leaves both halves of + /// `#[cfg(debug_assertions)]` in the population rather than deleting one of them on a guess. + pub debug_assertions: Option, + + /// Names some configuration puts in force under conditions this cannot evaluate. + /// + /// A `--cfg` in a `[target.'cfg(…)'.rustflags]` table applies only when its own predicate + /// holds, and that predicate is about the very target being resolved. Rather than decide it, + /// the name is left unanswerable, which keeps the code it gates mutable. + pub undecided: Vec, + + /// Whether the build compiles for more than one target at once. + /// + /// One set of predicates cannot describe two targets, so this suppresses evaluation entirely + /// rather than describing whichever target happened to be written first. + pub several_targets: bool, +} + +impl Build { + /// Works out what cargo will compile, for a workspace at `root`. + /// + /// `profile` and `extra` are the run's own `--profile` and passthrough cargo arguments, which + /// outrank the configuration files. Everything else comes from where cargo would read it: the + /// `CARGO_BUILD_TARGET`, `RUSTFLAGS` and `CARGO_ENCODED_RUSTFLAGS` variables, the + /// `.cargo/config.toml` files above the workspace, and the profile tables in the workspace + /// manifest. + /// + /// Nothing here fails. A file that cannot be read or parsed simply says nothing, and what it + /// would have said is either supplied by a file further out or left unanswered. + #[must_use] + pub fn resolve(root: &Utf8Path, profile: Option<&str>, extra: &[String]) -> Self { + Self::resolve_in(root, profile, extra, &Environment::ambient()) + } + + /// Resolves against a given environment, so the tests do not depend on the ambient one. + /// + /// The variables cargo reads are taken as a value so tests need not write the process + /// environment. + fn resolve_in(root: &Utf8Path, profile: Option<&str>, extra: &[String], environment: &Environment) -> Self { + let config = CargoConfig::load(root, environment); + let (target, several_targets) = target(extra, environment, &config); + let (flags, undecided) = rustflags(environment, &config, target.as_deref()); + let profile = named_profile(extra).or_else(|| profile.map(ToOwned::to_owned)); + + Self { + debug_assertions: assertions(&flags).or_else(|| profile_assertions(root, &config, profile.as_deref())), + cfgs: valued(&flags, "--cfg"), + codegen: codegen(&flags), + target, + undecided, + several_targets, + } + } + + /// Everything [`Self::resolve`] reads other than the run's own `--profile` and passthrough + /// arguments. + /// + /// Named here, beside the code that reads them, because another module has to know this set: + /// the run record decides whether a cached "this mutant does not compile" was reached under the + /// same build, and a second list of build inputs kept over there would fall behind this one + /// without anything failing. When it did, unviability would be carried across a change that + /// decides what compiles, and a mutant withheld from the denominator on that basis turns a gap + /// in the suite into a better score. Anything the resolution learns to read belongs here. + #[cfg_attr( + not(any(test, feature = "internals")), + expect( + dead_code, + reason = "the list exists to be compared against, not to be read: its only consumer is \ + the invariant test that fails when the resolution learns to read an input \ + nobody added here, so it is live only where that test can see it" + ) + )] + pub const INPUTS: &'static [&'static str] = &[ + "CARGO_BUILD_TARGET", + "CARGO_ENCODED_RUSTFLAGS", + "RUSTFLAGS", + "CARGO_BUILD_RUSTFLAGS", + "CARGO_TARGET__RUSTFLAGS", + "CARGO_HOME", + "cargo config files", + "build.target", + "build.rustflags", + "target.*.rustflags", + "profile.*", + ]; + + /// The triples the command line and the environment ask for, in cargo's precedence order. + /// + /// Split out of the resolution for the run record, which has to key what it cached on the + /// target it was built for and cannot call [`Self::resolve`]: that needs a workspace root, and + /// the record's context is settled before the workspace has been located. The configured + /// `build.target` is not consulted here for exactly that reason — it lives in a file, so the + /// record covers it through [`Self::settings`] instead. + /// + /// Several triples come back as several rather than as one answer, because a build for two + /// targets is one no single set of predicates describes and a caller has to be able to tell + /// that apart from a build for one. + #[must_use] + pub fn requested_targets(extra: &[String], configured: Option<&str>) -> Vec { + let mut named = valued(extra, "--target"); + + if named.is_empty() { + named = configured.map(ToOwned::to_owned).into_iter().collect(); + } + + named.sort(); + named.dedup(); + + named + } + + /// The values of the [`Self::INPUTS`] that live in files, rendered so a change moves the bytes. + /// + /// The variables are left out because a caller that digests this has the environment in hand + /// already; what it cannot see without a root is the workspace. Rendered as named parts rather + /// than as a resolved [`Build`] because the resolution collapses its inputs — a `--cfg` in a + /// table this build does not read comes back as one undecided name whatever the flag was, and a + /// profile body reached through `inherits` is folded into a single boolean — so two genuinely + /// different configurations can resolve to the same value. What is wanted here is noticing the + /// change, and the settings themselves are cheaper to be right about than their consequences. + /// + /// Deliberately over-inclusive: every profile table is rendered, not the one profile this run + /// builds, and every target table, not the ones whose predicate holds. Being over-inclusive + /// costs a cache that could have been kept, which costs time; being under-inclusive costs a + /// mutant that is silently dropped from the denominator. + #[must_use] + pub fn settings(root: &Utf8Path) -> Vec { + Self::settings_in(root, &Environment::ambient()) + } + + /// Reads the settings against a given environment, so the tests do not depend on the ambient + /// one; see [`Self::resolve_in`]. + fn settings_in(root: &Utf8Path, environment: &Environment) -> Vec { + let config = CargoConfig::load(root, environment); + let mut parts = Vec::new(); + + // Cargo configurations can set far more than the predicates this resolver projects. The + // run record must still notice every one: `[env]`, wrappers and future Cargo keys can + // change the build or test process without changing a selected `build.*` value. + parts.extend(config.sources.iter().cloned()); + + if let Some(target) = config.string(&["build", "target"]) { + parts.push(format!("build.target={target}")); + } + + for flag in config.strings(&["build", "rustflags"]) { + parts.push(format!("build.rustflags={flag}")); + } + + for table in config.keys(&["target"]) { + for flag in config.strings(&["target", &table, "rustflags"]) { + parts.push(format!("target.{table}.rustflags={flag}")); + } + } + + // Whole tables rather than the keys the profile chain happens to follow, and one part per + // file rather than a merged view, because the merge is the resolution's business: a key + // that moves from one file to another changes which of them wins, and a reader that only + // kept the winner could not see that it had. + let manifest = read_table(&root.join("Cargo.toml")); + + for table in config.tables.iter().chain(manifest.as_ref()) { + if let Some(profiles) = table.get("profile") { + parts.push(format!("profile={profiles}")); + } + } + + parts + } + + /// The `rustc` command line that answers which predicates hold for this build. + pub(super) fn probe_args(&self) -> Vec { + let mut args = vec!["--print".to_owned(), "cfg".to_owned()]; + + if let Some(target) = self.target.as_ref() { + args.push("--target".to_owned()); + args.push(target.clone()); + } + + if let Some(on) = self.debug_assertions { + args.push("-C".to_owned()); + args.push(format!("debug-assertions={}", if on { "on" } else { "off" })); + } + + for option in &self.codegen { + args.push("-C".to_owned()); + args.push(option.clone()); + } + + for predicate in &self.cfgs { + args.push("--cfg".to_owned()); + args.push(predicate.clone()); + } + + args + } +} + +/// The environment variables cargo reads that decide what a build compiles. +#[derive(Clone, Debug, Default)] +struct Environment { + /// `CARGO_BUILD_TARGET`, the variable spelling of `build.target`. + target: Option, + + /// `CARGO_ENCODED_RUSTFLAGS`, whose entries are separated by unit separators. + encoded_rustflags: Option, + + /// `RUSTFLAGS`, whose entries are separated by spaces. + rustflags: Option, + + /// `CARGO_BUILD_RUSTFLAGS`, the variable spelling of `build.rustflags`. + build_rustflags: Option, + + /// `CARGO_TARGET__RUSTFLAGS`, the variable spelling of `target..rustflags`. + /// + /// Keyed by the variable's own triple component rather than by the triple, because the mapping + /// from a config key to a variable name loses information — `-` and `.` both become `_` — and + /// cannot be run backwards. A lookup normalizes the triple it holds the same way instead. + target_rustflags: HashMap, + + /// Where the user-wide cargo configuration lives. + cargo_home: Option, +} + +impl Environment { + /// The variable-spelled rustflags for a triple, if the environment sets them. + fn target_flags(&self, triple: &str) -> Option<&str> { + self.target_rustflags.get(&variable_component(triple)).map(String::as_str) + } + + // #[gamma::skip(fn_value.default, reason = "this adapter reads process-wide variables that parallel tests cannot safely replace; Environment::read is tested with an injected lookup")] + /// Reads the variables this process was launched with. + fn ambient() -> Self { + Self::read( + |name| env::var(name), + env::vars_os().filter_map(|(name, _value)| name.into_string().ok()), + ) + } + + /// Reads the variables, given a lookup and the names the environment holds. + /// + /// The names are needed as well as the lookup because one of the variables cargo reads has a + /// name this cannot know in advance: `CARGO_TARGET__RUSTFLAGS` names its own triple, + /// and the triple in force is not settled until the configuration has been read. + fn read(mut get: impl FnMut(&str) -> Result, names: impl IntoIterator) -> Self { + let home = get("CARGO_HOME").ok().map(Utf8PathBuf::from).or_else(|| { + get("HOME") + .or_else(|_absent| get("USERPROFILE")) + .ok() + .map(|home| Utf8PathBuf::from(home).join(".cargo")) + }); + + let target = get("CARGO_BUILD_TARGET").ok(); + let encoded_rustflags = get("CARGO_ENCODED_RUSTFLAGS").ok(); + let rustflags = get("RUSTFLAGS").ok(); + let build_rustflags = get("CARGO_BUILD_RUSTFLAGS").ok(); + + let target_rustflags = names + .into_iter() + .filter_map(|name| { + let component = name.strip_prefix("CARGO_TARGET_")?.strip_suffix("_RUSTFLAGS")?.to_owned(); + + Some((component, get(&name).ok()?)) + }) + .collect(); + + Self { + target, + encoded_rustflags, + rustflags, + build_rustflags, + target_rustflags, + cargo_home: home, + } + } +} + +/// The cargo configuration files that apply to a build under `dir`, nearest first. +/// +/// Cargo reads `.cargo/config.toml` from the directory it runs in and every directory above it, +/// then the user-wide file. A setting from a nearer file wins, and array-valued settings are +/// joined, which is what this preserves by keeping the files in order rather than merging them. +#[derive(Debug, Default)] +struct CargoConfig { + tables: Vec, + sources: Vec, +} + +impl CargoConfig { + fn load(dir: &Utf8Path, environment: &Environment) -> Self { + let mut config = Self::default(); + let mut at = Some(dir); + + while let Some(directory) = at { + for name in ["config.toml", "config"] { + let path = directory.join(".cargo").join(name); + if config.include(&path) { + break; + } + } + + // #[gamma::skip(stmt.delete_assign, reason = "without advancing to the parent this loop is intrinsically nonterminating; the mutation runner observes that only as its timeout")] + at = directory.parent(); + } + + if let Some(home) = environment.cargo_home.as_ref() + && !cargo_home_was_walked(dir, home) + { + for name in ["config.toml", "config"] { + let path = home.join(name); + if config.include(&path) { + break; + } + } + } + + config + } + + /// Reads one Cargo configuration as both a parsed resolver input and opaque record input. + /// + /// A syntactically invalid file still changes Cargo's behavior by making the build fail, so + /// its bytes must invalidate a prior successful record even though no table can be resolved. + fn include(&mut self, path: &Utf8Path) -> bool { + let bytes = match fs::read(path.as_std_path()) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return false, + Err(error) => { + self.sources.push(format!("cargo-config:{path}:unreadable:{:?}", error.kind())); + return true; + } + }; + + self.sources.push(format!("cargo-config:{path}:{}", blake3::hash(&bytes).to_hex())); + + if let Ok(text) = core::str::from_utf8(&bytes) + && let Ok(table) = toml::from_str(text) + { + self.tables.push(table); + } + + true + } + + /// The nearest file's value for a dotted key, when it is a string. + fn string(&self, path: &[&str]) -> Option<&str> { + self.tables.iter().find_map(|table| lookup(table, path)?.as_str()) + } + + /// Every file's value for a dotted key, as the list of strings cargo would join them into. + /// + /// A cargo configuration key of this shape accepts either one string or a list of them, and + /// the lists in several files are concatenated rather than shadowing one another. + fn strings(&self, path: &[&str]) -> Vec { + self.tables + .iter() + .filter_map(|table| lookup(table, path)) + .flat_map(|value| match value { + Value::String(one) => vec![one.clone()], + Value::Array(many) => many.iter().filter_map(|entry| entry.as_str().map(ToOwned::to_owned)).collect(), + _other => Vec::new(), + }) + .collect() + } + + /// The keys of a table, across every file that declares one, nearest first. + fn keys(&self, path: &[&str]) -> Vec { + let mut names: Vec = self + .tables + .iter() + .filter_map(|table| lookup(table, path)?.as_table()) + .flat_map(|table| table.keys().cloned()) + .collect(); + + names.sort(); + names.dedup(); + names + } +} + +/// Whether the upward `.cargo` search already visited this Cargo home. +fn cargo_home_was_walked(dir: &Utf8Path, home: &Utf8Path) -> bool { + home.file_name() == Some(".cargo") && home.parent().is_some_and(|parent| dir.starts_with(parent)) +} + +/// Reads a TOML file, or nothing at all when it is absent or malformed. +fn read_table(path: &Utf8Path) -> Option
{ + let text = fs::read_to_string(path.as_std_path()).ok()?; + + toml::from_str(&text).ok() +} + +/// Follows a dotted key through nested tables. +fn lookup<'table>(table: &'table Table, path: &[&str]) -> Option<&'table Value> { + let (last, leading) = path.split_last()?; + let mut at = table; + + for name in leading { + at = at.get(*name)?.as_table()?; + } + + at.get(*last) +} + +/// The target triple the build compiles for, and whether there is more than one of them. +/// +/// `--target` on the command line outranks `CARGO_BUILD_TARGET`, which outranks `build.target`, +/// exactly as cargo orders them. +fn target(extra: &[String], environment: &Environment, config: &CargoConfig) -> (Option, bool) { + let mut named = Build::requested_targets(extra, environment.target.as_deref()); + + if named.is_empty() { + named = config.strings(&["build", "target"]); + } + + // #[gamma::skip(iter.remove_sort, reason = "sorting only makes equal targets adjacent before `dedup`; zero, one, or several distinct targets is unchanged because an all-equal input is already adjacent")] + named.sort(); + named.dedup(); + + match named.len() { + 0 => (None, false), + 1 => (named.pop(), false), + _several => (None, true), + } +} + +/// The rustc flags cargo will pass, and the names it may pass under a predicate of its own. +/// +/// Cargo takes these from the first source that has anything to say, rather than merging them: +/// `CARGO_ENCODED_RUSTFLAGS`, then `RUSTFLAGS`, then the target tables — the triple's own joined +/// with every `cfg(…)` table whose predicate holds for that target, as cargo joins them — then +/// `build.rustflags`, whose environment spelling `CARGO_BUILD_RUSTFLAGS` shares that last slot +/// because it is the same setting written another way. +/// +/// A target table this build does not read decides nothing, and what it *would* have decided is +/// reported as unanswerable rather than answered from a lower slot: a `--cfg` name, and equally a +/// `-C debug-assertions`, `-C panic` or `-C target-feature`, each settles a predicate, and letting +/// the profile chain or the compiler's own default answer instead describes a different +/// compilation from the one cargo will run. +fn rustflags(environment: &Environment, config: &CargoConfig, target: Option<&str>) -> (Vec, Vec) { + let tables = target_tables(config, target, environment); + let joined: Vec = tables + .iter() + .filter(|(applies, _flags)| *applies == Verdict::Yes) + .flat_map(|(_applies, flags)| flags.iter().cloned()) + .collect(); + + let ambient = environment + .encoded_rustflags + .as_ref() + .map(|encoded| { + encoded + .split('\u{1f}') + .filter(|flag| !flag.is_empty()) + .map(ToOwned::to_owned) + .collect() + }) + .or_else(|| environment.rustflags.as_ref().map(|flags| split(flags))); + + // Whether the tables are the source in force decides both the flags and what is left hanging: + // a table whose flags this build really passes has answered its own names, and every other + // table has not. + let reading_tables = ambient.is_none() && !joined.is_empty(); + + let chosen = ambient + .or_else(|| reading_tables.then(|| joined.clone())) + .or_else(|| environment.build_rustflags.as_ref().map(|flags| split(flags))) + .or_else(|| { + let flags = config.strings(&["build", "rustflags"]); + + (!flags.is_empty()).then_some(flags) + }) + .unwrap_or_default(); + + let mut undecided: Vec = Vec::new(); + + for (applies, flags) in &tables { + if reading_tables && *applies == Verdict::Yes { + continue; + } + + undecided.extend(decided_names(flags)); + } + + // #[gamma::skip(iter.remove_sort, reason = "the only consumer collects these names into a HashSet, so their order cannot affect cfg evaluation")] + undecided.sort(); + undecided.dedup(); + + (chosen, undecided) +} + +/// Every `[target.…]` table's rustflags, paired with whether they are in force for this build. +/// +/// A table named by a triple is in force when that triple is the one being built; a table named by +/// a `cfg(…)` predicate is in force when the predicate holds for it, which is a question about the +/// target and so is put to `rustc`. Asking costs one process, and only for a workspace that +/// actually writes such a table — but not asking means either applying flags that are not in force +/// or ignoring flags that are, and both describe a compilation cargo will not run. +/// +/// The probe answers about the bare target rather than about the target as these very flags will +/// leave it, which is the circularity cargo has too. A predicate that turns on a name some +/// rustflag sets is therefore answered `Unknown`, and the table it names is left unread and +/// unanswerable rather than resolved on a guess. +fn target_tables(config: &CargoConfig, target: Option<&str>, environment: &Environment) -> Vec<(Verdict, Vec)> { + let names = config.keys(&["target"]); + let asked_about = !names.is_empty() || !environment.target_rustflags.is_empty(); + let triple = || target.or_else(|| host_triple(config, asked_about)); + let mut cfgs: Option> = None; + + let mut tables: Vec<(Verdict, Vec)> = names + .iter() + .map(|name| { + let applies = predicate_of(name).map_or_else( + || triple().map_or(Verdict::Unknown, |triple| Verdict::from(triple == name)), + |predicate| { + cfgs.get_or_insert_with(|| probe_cfgs(target)) + .as_ref() + .map_or(Verdict::Unknown, |cfgs| cfgs.decide_str(predicate)) + }, + ); + + // The variable spelling of a key wins over every file that states it, as it does for + // any cargo configuration value. Only a table named by a triple has one: the variable + // form of a `cfg(…)` key is not a name cargo will construct. + let flags = predicate_of(name) + .is_none() + .then(|| environment.target_flags(name).map(split)) + .flatten() + .unwrap_or_else(|| config.strings(&["target", name, "rustflags"])); + + (applies, flags) + }) + .collect(); + + // A variable can also name a triple no file mentions at all. Only the triple in force is + // reachable, because the variable's name is an uppercased, underscored spelling of the triple + // and that mapping cannot be run backwards to recover the triples the environment names. + if let Some(triple) = triple() + && !names.iter().any(|name| name == triple) + && let Some(flags) = environment.target_flags(triple) + { + tables.push((Verdict::Yes, split(flags))); + } + + tables +} + +/// The component a triple contributes to a `CARGO_TARGET__RUSTFLAGS` variable name. +/// +/// Cargo's rule for spelling any configuration key as a variable: uppercase it, and replace `-` and +/// `.` with `_`. +fn variable_component(triple: &str) -> String { + triple.to_uppercase().replace(['-', '.'], "_") +} + +/// The predicate inside a `cfg(…)` table name, or nothing for a table named by a triple. +fn predicate_of(table: &str) -> Option<&str> { + table.strip_prefix("cfg(")?.strip_suffix(')') +} + +/// The predicates that hold for a target, as the compiler's own answer about it. +/// +/// Deliberately asked without the flags being resolved: they are what this is being asked in order +/// to choose. Failing to run says nothing, which leaves every `cfg(…)` table unread and its names +/// unanswerable — the same direction every other uncertainty in this module resolves in. +fn probe_cfgs(target: Option<&str>) -> Option { + let program = env::var("RUSTC").unwrap_or_else(|_absent| "rustc".to_owned()); + let mut command = Command::new(program); + + let _builder = command.arg("--print").arg("cfg"); + + if let Some(triple) = target { + let _builder = command.arg("--target").arg(triple); + } + + let output = command.output().ok().filter(|output| output.status.success())?; + + Some(CfgSet::parse(&String::from_utf8_lossy(&output.stdout))) +} + +/// The predicate names a flag vector settles, whichever spelling it settles them in. +/// +/// Read from a table this build does not use, so that each name can be marked unanswerable. The +/// `-C` options are here for the same reason the `--cfg` names are: [`assertions`] and [`codegen`] +/// read them out of the chosen vector, so a table that sets one and is not chosen leaves that +/// predicate to be answered by something describing a different compilation. +fn decided_names(flags: &[String]) -> Vec { + let mut names: Vec = valued(flags, "--cfg").into_iter().map(|predicate| name_of(&predicate)).collect(); + + for option in options(flags) { + if option + .strip_prefix("debug-assertions") + .is_some_and(|rest| rest.is_empty() || rest.starts_with('=')) + { + names.push("debug_assertions".to_owned()); + } else if option.starts_with("panic=") { + names.push("panic".to_owned()); + } else if option.starts_with("target-feature=") { + names.push("target_feature".to_owned()); + } + } + + names +} + +/// The compiler's own triple, asked for only when a target table might apply to it. +/// +/// A `target..rustflags` table is in force for a build with no `--target` when the triple +/// is the host's, so answering that needs the host's name — and nothing else here does, which is +/// why it is not asked for up front. `asked_about` is the caller's statement that something — +/// a configuration table or a `CARGO_TARGET__RUSTFLAGS` variable — names a triple at all. +fn host_triple(config: &CargoConfig, asked_about: bool) -> Option<&'static str> { + use std::sync::OnceLock; + + /// One `rustc -vV` per process, since the answer cannot change under a running run. + static HOST: OnceLock> = OnceLock::new(); + + if !asked_about && config.keys(&["target"]).is_empty() { + return None; + } + + HOST.get_or_init(|| { + // #[gamma::skip(literal.str_to_empty, literal.str_to_xyzzy, reason = "the process-wide compiler override cannot be mutated safely by parallel tests; host parsing and command behavior are exercised independently")] + let program = env::var("RUSTC").unwrap_or_else(|_absent| "rustc".to_owned()); + let output = Command::new(program).arg("-vV").output().ok()?; + let printed = String::from_utf8_lossy(&output.stdout).into_owned(); + + parse_host(&printed) + }) + .as_deref() +} + +fn parse_host(printed: &str) -> Option { + printed + .lines() + .find_map(|line| line.strip_prefix("host: ")) + .map(|triple| triple.trim().to_owned()) +} + +/// Splits a space-separated flag string, as cargo does for `RUSTFLAGS`. +fn split(flags: &str) -> Vec { + flags.split_whitespace().map(ToOwned::to_owned).collect() +} + +/// Collects the values of a flag, written either as two arguments or joined by `=`. +/// +/// The prefix has to be matched exactly before the `=`, or `--target-dir` would be read as a +/// target triple and every predicate would then describe a build that does not exist. +fn valued(args: &[String], flag: &str) -> Vec { + let mut found = Vec::new(); + let mut expecting = false; + + for argument in args { + if expecting { + found.push(argument.clone()); + // #[gamma::skip(assign_value.default, reason = "`expecting` is bool, whose `Default::default()` is exactly false")] + expecting = false; + + continue; + } + + if argument == flag { + expecting = true; + } else if let Some(value) = argument.strip_prefix(flag).and_then(|rest| rest.strip_prefix('=')) { + found.push(value.to_owned()); + } + } + + found +} + +/// The name a `--cfg` value puts in force, without whatever it is set to. +fn name_of(predicate: &str) -> String { + predicate.split_once('=').map_or(predicate, |(name, _value)| name).trim().to_owned() +} + +/// The codegen options that decide a predicate, in the spelling `-C` takes. +/// +/// `target-feature` decides the `target_feature` values and `panic` decides the `panic` one. The +/// profile's own `panic` setting is not consulted, because cargo ignores it for the `test` profile +/// — which is the profile gamma's `cargo test --no-run` build uses — and a flag is the only way it +/// can reach the compilation this describes. +fn codegen(flags: &[String]) -> Vec { + options(flags) + .filter(|option| option.starts_with("target-feature=") || option.starts_with("panic=")) + .map(ToOwned::to_owned) + .collect() +} + +/// Whether the flags settle `debug_assertions` themselves. +/// +/// `RUSTFLAGS` reaches rustc after the flags cargo derives from the profile, so a +/// `-C debug-assertions` there is the last word whatever the profile says. +fn assertions(flags: &[String]) -> Option { + options(flags) + .filter_map(|option| option.strip_prefix("debug-assertions")) + .filter_map(|rest| match rest { + // A bare `-C debug-assertions` turns them on. + "" => Some(Some(true)), + + // Anything else that merely starts with the name — `-C debug-assertions-foo` — is a + // different option, and must not be read as this one. + _other => rest.strip_prefix('=').map(|value| match value.trim() { + "y" | "yes" | "on" | "true" => Some(true), + "n" | "no" | "off" | "false" => Some(false), + _unrecognised => None, + }), + }) + .last() + .flatten() +} + +/// The `-C` options among a flag list, however they were spelled. +fn options(flags: &[String]) -> impl Iterator { + let mut expecting = false; + + flags.iter().filter_map(move |flag| { + if expecting { + // #[gamma::skip(assign_value.default, reason = "`expecting` is bool, whose `Default::default()` is exactly false")] + expecting = false; + + return Some(flag.as_str()); + } + + if flag == "-C" || flag == "--codegen" { + expecting = true; + + return None; + } + + flag.strip_prefix("-C").or_else(|| flag.strip_prefix("--codegen=")) + }) +} + +/// The profile named on the command line, which outranks anything configured. +fn named_profile(extra: &[String]) -> Option { + if extra.iter().any(|argument| argument == "--release" || argument == "-r") { + return Some("release".to_owned()); + } + + valued(extra, "--profile").pop() +} + +/// Whether the profile the build will use has debug assertions on. +/// +/// A run that names no profile builds with `cargo test`, which uses the `test` profile, so that is +/// what an unnamed profile resolves to rather than `dev`. +/// +/// Returns `None` for a profile whose chain cannot be followed — a custom profile that inherits +/// from nothing, or from itself — because a guess either way deletes half of the conditionally +/// compiled code from the population. +fn profile_assertions(root: &Utf8Path, config: &CargoConfig, profile: Option<&str>) -> Option { + let manifest = read_table(&root.join("Cargo.toml")); + let declared = |name: &str, key: &str| { + config.string(&["profile", name, key]).map(ToOwned::to_owned).or_else(|| { + manifest + .as_ref() + .and_then(|table| lookup(table, &["profile", name, key]))? + .as_str() + .map(ToOwned::to_owned) + }) + }; + let switched = |name: &str| { + ["debug-assertions", "debug_assertions"] + .into_iter() + .find_map(|key| lookup_bool(config, manifest.as_ref(), name, key)) + }; + + let mut current = profile.unwrap_or("test").to_owned(); + + for _hop in 0..PROFILE_DEPTH { + if let Some(on) = switched(¤t) { + return Some(on); + } + + // The two built-in profiles that settle the question outright. Both can be overridden, + // which is why the declared value is consulted first. + match current.as_str() { + "dev" => return Some(true), + "release" => return Some(false), + _custom => {} + } + + let parent = declared(¤t, "inherits").or_else(|| match current.as_str() { + "test" => Some("dev".to_owned()), + "bench" => Some("release".to_owned()), + _custom => None, + })?; + + current = parent; + } + + None +} + +/// A profile's boolean setting, from the cargo configuration first and the manifest second. +fn lookup_bool(config: &CargoConfig, manifest: Option<&Table>, profile: &str, key: &str) -> Option { + let path = ["profile", profile, key]; + + config + .tables + .iter() + .find_map(|table| lookup(table, &path)?.as_bool()) + .or_else(|| lookup(manifest?, &path)?.as_bool()) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use tempfile::TempDir; + + use super::*; + + /// A workspace root with the given files written into it, and no environment at all. + fn tree(files: &[(&str, &str)]) -> (TempDir, Utf8PathBuf) { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + for (relative, contents) in files { + let path = root.join(relative); + + fs::create_dir_all(path.parent().expect("every fixture path has a parent").as_std_path()).expect("directories"); + fs::write(path.as_std_path(), contents).expect("the fixture file is written"); + } + + (directory, root) + } + + /// An environment holding nothing, so a fixture answers for itself. + /// + /// `cargo_home` points into the fixture, which has no user-wide file, so whatever the machine + /// running the tests has configured cannot reach them. + fn empty(root: &Utf8Path) -> Environment { + Environment { + cargo_home: Some(root.join("nowhere")), + ..Environment::default() + } + } + + #[test] + fn environment_names_and_home_fallbacks_are_read_exactly() { + let values = std::collections::HashMap::from([ + ("HOME", "home"), + ("USERPROFILE", "profile"), + ("CARGO_BUILD_TARGET", "target"), + ("CARGO_ENCODED_RUSTFLAGS", "encoded"), + ("RUSTFLAGS", "plain"), + ("CARGO_BUILD_RUSTFLAGS", "build"), + ]); + let mut asked = Vec::new(); + let environment = Environment::read( + |name| { + asked.push(name.to_owned()); + values.get(name).map(|value| (*value).to_owned()).ok_or(env::VarError::NotPresent) + }, + [], + ); + + assert_eq!( + asked, + [ + "CARGO_HOME", + "HOME", + "CARGO_BUILD_TARGET", + "CARGO_ENCODED_RUSTFLAGS", + "RUSTFLAGS", + "CARGO_BUILD_RUSTFLAGS", + ] + ); + assert_eq!(environment.cargo_home.as_deref(), Some(Utf8Path::new("home/.cargo"))); + assert_eq!(environment.target.as_deref(), Some("target")); + assert_eq!(environment.encoded_rustflags.as_deref(), Some("encoded")); + assert_eq!(environment.rustflags.as_deref(), Some("plain")); + assert_eq!(environment.build_rustflags.as_deref(), Some("build")); + + let profile = Environment::read( + |name| { + (name == "USERPROFILE") + .then(|| "profile".to_owned()) + .ok_or(env::VarError::NotPresent) + }, + [], + ); + assert_eq!(profile.cargo_home.as_deref(), Some(Utf8Path::new("profile/.cargo"))); + } + + fn resolve(root: &Utf8Path, profile: Option<&str>, extra: &[&str]) -> Build { + let extra: Vec = extra.iter().map(|argument| (*argument).to_owned()).collect(); + + Build::resolve_in(root, profile, &extra, &empty(root)) + } + + /// The settings a record has to notice are read whole, not through the resolution that + /// collapses them: a target table this build does not read still decides what a build that does + /// read it compiles. + #[test] + fn the_settings_name_every_file_borne_input_the_resolution_reads() { + let (_directory, root) = tree(&[ + ( + ".cargo/config.toml", + "[build]\ntarget = \"wasm32-unknown-unknown\"\nrustflags = [\"--cfg\", \"loom\"]\n\n[target.'cfg(unix)']\nrustflags = [\"-Cdebug-assertions=on\"]\n\n[profile.dev]\ndebug-assertions = false\n", + ), + ("Cargo.toml", "[profile.mutants]\ninherits = \"release\"\ndebug-assertions = true\n"), + ]); + + let settings = Build::settings_in(&root, &empty(&root)); + + assert!(settings.contains(&"build.target=wasm32-unknown-unknown".to_owned()), "{settings:?}"); + assert!(settings.contains(&"build.rustflags=--cfg".to_owned()), "{settings:?}"); + assert!(settings.contains(&"build.rustflags=loom".to_owned()), "{settings:?}"); + assert!( + settings.contains(&"target.cfg(unix).rustflags=-Cdebug-assertions=on".to_owned()), + "{settings:?}" + ); + assert_eq!( + settings.iter().filter(|part| part.starts_with("profile=")).count(), + 2, + "both the configuration's profiles and the manifest's are read: {settings:?}" + ); + } + + /// A workspace nobody has configured says nothing, rather than saying something that varies + /// with the machine the tests run on. + #[test] + fn an_unconfigured_workspace_holds_no_settings() { + let (_directory, root) = tree(&[]); + + assert!(Build::settings_in(&root, &empty(&root)).is_empty()); + } + + #[test] + fn an_unprojected_cargo_setting_still_moves_the_record_input() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[env]\nSUBJECT_MODE = \"one\"\n")]); + let before = Build::settings_in(&root, &empty(&root)); + + fs::write(root.join(".cargo/config.toml").as_std_path(), "[env]\nSUBJECT_MODE = \"two\"\n").expect("configuration is writable"); + + assert_ne!(before, Build::settings_in(&root, &empty(&root))); + } + + #[test] + fn a_user_cargo_setting_still_moves_the_record_input() { + let (_workspace, root) = tree(&[]); + let (_home_directory, home) = tree(&[("config.toml", "[env]\nSUBJECT_MODE = \"one\"\n")]); + let environment = Environment { + cargo_home: Some(home.clone()), + ..Environment::default() + }; + let before = Build::settings_in(&root, &environment); + + fs::write(home.join("config.toml").as_std_path(), "[env]\nSUBJECT_MODE = \"two\"\n").expect("configuration is writable"); + + assert_ne!(before, Build::settings_in(&root, &environment)); + } + + /// The record keys on the target before it has a workspace, so this answers from the two + /// sources that do not need one — and from neither of them when they disagree with each other, + /// which is a build no single set of predicates describes. + #[test] + fn the_requested_targets_are_the_ones_the_command_line_and_the_environment_ask_for() { + let extra = ["--target=wasm32-unknown-unknown".to_owned()]; + let both = [ + "--target=wasm32-unknown-unknown".to_owned(), + "--target".to_owned(), + "aarch64-apple-darwin".to_owned(), + ]; + + assert_eq!(Build::requested_targets(&[], None), Vec::::new()); + assert_eq!( + Build::requested_targets(&[], Some("x86_64-unknown-linux-musl")), + ["x86_64-unknown-linux-musl"] + ); + assert_eq!( + Build::requested_targets(&extra, Some("x86_64-unknown-linux-musl")), + ["wasm32-unknown-unknown"], + "a passthrough target outranks the variable, as it does for cargo" + ); + assert_eq!( + Build::requested_targets(&both, None), + ["aarch64-apple-darwin", "wasm32-unknown-unknown"] + ); + } + + #[test] + fn a_passthrough_target_is_read_however_it_is_written() { + let (_directory, root) = tree(&[]); + + assert_eq!( + resolve(&root, None, &["--target", "wasm32-unknown-unknown"]).target.as_deref(), + Some("wasm32-unknown-unknown") + ); + assert_eq!( + resolve(&root, None, &["--target=wasm32-unknown-unknown"]).target.as_deref(), + Some("wasm32-unknown-unknown") + ); + } + + /// `--target-dir` shares the first eight characters with `--target` and says nothing about the + /// triple. Reading it as one would describe a build that does not exist. + #[test] + fn a_target_directory_is_not_a_target() { + let (_directory, root) = tree(&[]); + let build = resolve(&root, None, &["--target-dir=elsewhere"]); + + assert_eq!(build.target, None); + assert!(!build.several_targets); + } + + #[test] + fn public_resolution_reads_the_workspace_configuration() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\ntarget=\"configured-publicly\"\n")]); + + assert_eq!(Build::resolve(&root, None, &[]).target.as_deref(), Some("configured-publicly")); + } + + #[test] + fn the_cargo_configuration_supplies_the_target() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\ntarget = \"aarch64-apple-darwin\"\n")]); + + assert_eq!(resolve(&root, None, &[]).target.as_deref(), Some("aarch64-apple-darwin")); + } + + /// A configured target is what the build uses only until something more specific says + /// otherwise, which is the same order cargo itself applies. + #[test] + fn a_passthrough_target_outranks_the_configured_one() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\ntarget = \"aarch64-apple-darwin\"\n")]); + + assert_eq!( + resolve(&root, None, &["--target", "wasm32-unknown-unknown"]).target.as_deref(), + Some("wasm32-unknown-unknown") + ); + } + + /// A configuration file above the workspace still applies, because cargo reads every one of + /// them on the way up. + #[test] + fn a_configuration_file_above_the_workspace_is_read() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\ntarget = \"aarch64-apple-darwin\"\n")]); + let inner = root.join("member"); + + fs::create_dir_all(inner.as_std_path()).expect("a member directory"); + + assert_eq!( + Build::resolve_in(&inner, None, &[], &empty(&root)).target.as_deref(), + Some("aarch64-apple-darwin") + ); + } + + #[test] + fn configuration_search_obeys_cargo_precedence_and_reads_every_level() { + let (_directory, root) = tree(&[ + ( + ".cargo/config.toml", + "[build]\ntarget = \"near\"\nrustflags = [\"--cfg\", \"near\"]\n", + ), + ( + ".cargo/config", + "[build]\ntarget = \"legacy-near\"\nrustflags = [\"--cfg\", \"legacy-near\"]\n", + ), + ("member/.cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"member\"]\n"), + ("member/.cargo/config", "[build]\nrustflags = [\"--cfg\", \"legacy-member\"]\n"), + ]); + let config = CargoConfig::load(&root.join("member"), &empty(&root)); + + assert_eq!(config.string(&["build", "target"]), Some("near")); + assert_eq!(config.strings(&["build", "rustflags"]), vec!["--cfg", "member", "--cfg", "near"]); + } + + #[test] + fn legacy_configuration_names_are_used_when_toml_is_absent() { + let (_directory, root) = tree(&[ + ("workspace/.cargo/config", "[build]\ntarget=\"legacy-workspace\"\n"), + ("home/config", "[build]\nrustflags=[\"legacy-home\"]\n"), + ]); + let workspace = root.join("workspace"); + let environment = Environment { + cargo_home: Some(root.join("home")), + ..Environment::default() + }; + let config = CargoConfig::load(&workspace, &environment); + + assert_eq!(config.string(&["build", "target"]), Some("legacy-workspace")); + assert_eq!(config.strings(&["build", "rustflags"]), vec!["legacy-home"]); + } + + #[test] + fn user_configuration_is_read_only_when_it_is_outside_the_workspace() { + let (_directory, root) = tree(&[ + ("workspace/.cargo/config.toml", "[build]\nrustflags = [\"workspace\"]\n"), + ("home/config.toml", "[build]\nrustflags = [\"home\"]\n"), + ("home/config", "[build]\nrustflags = [\"legacy-home\"]\n"), + ]); + let workspace = root.join("workspace"); + let outside = Environment { + cargo_home: Some(root.join("home")), + ..Environment::default() + }; + let inside = Environment { + cargo_home: Some(workspace.join(".cargo")), + ..Environment::default() + }; + + assert_eq!( + CargoConfig::load(&workspace, &outside).strings(&["build", "rustflags"]), + vec!["workspace", "home"] + ); + assert_eq!( + CargoConfig::load(&workspace, &inside).strings(&["build", "rustflags"]), + vec!["workspace"] + ); + } + + #[test] + fn descendant_non_dot_cargo_home_reaches_resolution_and_record_settings_once() { + let (_directory, root) = tree(&[("cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"from_descendant_home\"]\n")]); + let environment = Environment { + cargo_home: Some(root.join("cargo")), + ..Environment::default() + }; + + let build = Build::resolve_in(&root, None, &[], &environment); + let settings = Build::settings_in(&root, &environment); + + assert_eq!(build.cfgs, ["from_descendant_home"]); + assert_eq!( + settings.iter().filter(|part| part.starts_with("cargo-config:")).count(), + 1, + "the configuration digest must contain the descendant home exactly once: {settings:?}" + ); + assert!( + settings.contains(&"build.rustflags=from_descendant_home".to_owned()), + "{settings:?}" + ); + } + + #[test] + fn a_deeply_nested_cargo_cfg_key_is_left_undecided_before_parsing() { + let predicate = format!("{}unix{}", "all(".repeat(100), ")".repeat(100)); + let config = format!("[target.'cfg({predicate})']\nrustflags = [\"--cfg\", \"deep_guard\"]\n"); + let (_directory, root) = tree(&[(".cargo/config.toml", &config)]); + let build = resolve(&root, None, &[]); + + assert_eq!(build.cfgs, Vec::::new()); + assert_eq!(build.undecided, ["deep_guard"]); + } + + #[test] + fn table_keys_are_sorted_and_deduplicated() { + let (_directory, root) = tree(&[ + (".cargo/config.toml", "[target.z]\nrustflags=[]\n[target.a]\nrustflags=[]\n"), + ("member/.cargo/config.toml", "[target.z]\nrustflags=[]\n[target.m]\nrustflags=[]\n"), + ]); + let config = CargoConfig::load(&root.join("member"), &empty(&root)); + + assert_eq!(config.keys(&["target"]), vec!["a", "m", "z"]); + assert!(config.keys(&["missing"]).is_empty()); + } + + /// No single set of predicates describes two targets, so a build of both is not evaluated at + /// all rather than evaluated as whichever was written first. + #[test] + fn several_targets_are_reported_rather_than_picked_between() { + let (_directory, root) = tree(&[]); + let build = resolve( + &root, + None, + &["--target", "wasm32-unknown-unknown", "--target", "aarch64-apple-darwin"], + ); + + assert!(build.several_targets); + assert_eq!(build.target, None); + + // The same triple twice is still one target. + let repeated = resolve( + &root, + None, + &["--target", "wasm32-unknown-unknown", "--target", "wasm32-unknown-unknown"], + ); + + assert!(!repeated.several_targets); + assert_eq!(repeated.target.as_deref(), Some("wasm32-unknown-unknown")); + } + + #[test] + fn target_sources_fall_back_in_order_and_are_normalised() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\ntarget = [\"z\", \"a\", \"z\"]\n")]); + let config = CargoConfig::load(&root, &empty(&root)); + let environment = Environment { + target: Some("environment".to_owned()), + ..empty(&root) + }; + + assert_eq!(target(&[], &environment, &config), (Some("environment".to_owned()), false)); + assert_eq!(target(&[], &empty(&root), &config), (None, true)); + } + + #[test] + fn configured_rustflags_carry_their_custom_predicates() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[build]\nrustflags = [\"--cfg\", \"loom\", \"-C\", \"target-feature=+avx2\", \"-C\", \"panic=abort\", \"-C\", \"opt-level=3\"]\n", + )]); + let build = resolve(&root, None, &[]); + + assert_eq!(build.cfgs, vec!["loom".to_owned()]); + + // `opt-level` decides no predicate, and a probe handed every flag would fail on the first + // one the compiler refuses in this position. + assert_eq!(build.codegen, vec!["target-feature=+avx2".to_owned(), "panic=abort".to_owned()]); + } + + /// A target table's flags are what cargo uses when that target is the one being built, and + /// they replace `build.rustflags` rather than adding to them. + #[test] + fn a_target_table_supplies_the_flags_for_its_own_triple() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[build]\nrustflags = [\"--cfg\", \"everywhere\"]\n\n[target.wasm32-unknown-unknown]\nrustflags = [\"--cfg\", \"woven\"]\n", + )]); + let build = resolve(&root, None, &["--target", "wasm32-unknown-unknown"]); + + assert_eq!(build.cfgs, vec!["woven".to_owned()]); + } + + /// The variable spelling of a target table is read, and wins over the file that states it. + /// + /// `CARGO_TARGET__RUSTFLAGS` is a level of cargo's rustflags precedence of its own: it + /// applies where none of the global variables do, so a resolver that only modeled those would + /// resolve `#[cfg(live)]` code as absent and drop every mutant in it from the population. + #[test] + fn the_variable_spelling_of_a_target_table_is_read_and_beats_the_file() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[build]\nrustflags = [\"--cfg\", \"everywhere\"]\n\n[target.wasm32-unknown-unknown]\nrustflags = [\"--cfg\", \"filed\"]\n", + )]); + let environment = Environment { + target_rustflags: core::iter::once(("WASM32_UNKNOWN_UNKNOWN".to_owned(), "--cfg live".to_owned())).collect(), + ..empty(&root) + }; + let extra = ["--target".to_owned(), "wasm32-unknown-unknown".to_owned()]; + let build = Build::resolve_in(&root, None, &extra, &environment); + + assert_eq!(build.cfgs, vec!["live".to_owned()]); + } + + /// The variable is read even when no configuration file names that triple at all, which is the + /// case a reader that only ever walked the `[target.…]` tables could not reach. + #[test] + fn a_target_variable_applies_with_no_table_to_hang_it_on() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"everywhere\"]\n")]); + let environment = Environment { + target_rustflags: core::iter::once(("WASM32_UNKNOWN_UNKNOWN".to_owned(), "--cfg live".to_owned())).collect(), + ..empty(&root) + }; + let extra = ["--target".to_owned(), "wasm32-unknown-unknown".to_owned()]; + + assert_eq!(Build::resolve_in(&root, None, &extra, &environment).cfgs, vec!["live".to_owned()]); + + // A variable naming some other triple is not in force and decides nothing. + let elsewhere = Environment { + target_rustflags: core::iter::once(("AARCH64_APPLE_DARWIN".to_owned(), "--cfg live".to_owned())).collect(), + ..empty(&root) + }; + + assert_eq!( + Build::resolve_in(&root, None, &extra, &elsewhere).cfgs, + vec!["everywhere".to_owned()] + ); + } + + /// The variable names its triple the way cargo spells a configuration key as a variable. + #[test] + fn a_triple_is_spelled_into_a_variable_name_the_way_cargo_spells_it() { + assert_eq!(variable_component("x86_64-unknown-linux-gnu"), "X86_64_UNKNOWN_LINUX_GNU"); + assert_eq!(variable_component("thumbv8m.main-none-eabi"), "THUMBV8M_MAIN_NONE_EABI"); + } + + /// Every `CARGO_TARGET_…_RUSTFLAGS` in the environment is read, since the triple in force is + /// not known until the configuration has been. + #[test] + fn the_target_specific_variables_are_read_by_enumeration() { + let environment = Environment::read( + |name| match name { + "CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS" => Ok("--cfg live".to_owned()), + _other => Err(env::VarError::NotPresent), + }, + [ + "CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS".to_owned(), + "CARGO_TARGET_DIR".to_owned(), + "RUSTFLAGS".to_owned(), + ], + ); + + assert_eq!(environment.target_flags("wasm32-unknown-unknown"), Some("--cfg live")); + assert_eq!(environment.target_flags("x86_64-unknown-linux-gnu"), None); + assert_eq!(environment.target_rustflags.len(), 1, "a variable that is not one was read"); + } + + /// A `cfg(…)` table is in force only when its own predicate holds, which is a question about + /// the target being resolved. The names it would set are left unanswerable, so the code they + /// gate stays in the population instead of being deleted on a guess. + #[test] + fn a_predicate_gated_table_leaves_its_names_unanswerable() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[build]\nrustflags = [\"--cfg\", \"everywhere\"]\n\n[target.'cfg(target_os = \"redox\")']\nrustflags = [\"--cfg\", \"sometimes\"]\n", + )]); + let build = resolve(&root, None, &[]); + + assert_eq!(build.cfgs, vec!["everywhere".to_owned()]); + assert_eq!(build.undecided, vec!["sometimes".to_owned()]); + } + + #[test] + fn a_valued_predicate_in_a_gated_table_leaves_its_bare_name_unanswerable() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[target.'cfg(target_os = \"redox\")']\nrustflags = [\"--cfg\", \"flavor=\\\"strawberry\\\"\"]\n", + )]); + let build = resolve(&root, None, &[]); + + assert_eq!(build.undecided, vec!["flavor".to_owned()]); + } + + /// Cargo joins the triple's table with every `cfg(…)` table whose predicate holds for the + /// target it is building. Looking the triple up alone means the flags of a workspace that + /// configures itself through `cfg(…)` — the common spelling for "every unix" — never reach the + /// probe, so the predicates they decide are answered from the compiler's own defaults instead. + #[test] + fn a_matching_predicate_table_is_joined_with_the_triples_own() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[target.wasm32-unknown-unknown]\nrustflags = [\"--cfg\", \"triple\"]\n\n\ + [target.'cfg(target_family = \"wasm\")']\nrustflags = [\"--cfg\", \"family\"]\n\n\ + [target.'cfg(target_os = \"linux\")']\nrustflags = [\"--cfg\", \"elsewhere\"]\n", + )]); + let build = resolve(&root, None, &["--target", "wasm32-unknown-unknown"]); + + assert!(build.cfgs.contains(&"triple".to_owned()), "{:?}", build.cfgs); + assert!(build.cfgs.contains(&"family".to_owned()), "{:?}", build.cfgs); + + // The table whose predicate is false for this target decides nothing, and is not in force + // either, so it stays unanswerable rather than becoming a name the build passes. + assert!(!build.cfgs.contains(&"elsewhere".to_owned()), "{:?}", build.cfgs); + assert_eq!(build.undecided, vec!["elsewhere".to_owned()]); + + // A table this run does read is decided, so nothing about it is left hanging. + assert!(!build.undecided.contains(&"family".to_owned())); + } + + /// `-C debug-assertions`, `-C panic` and `-C target-feature` each decide a predicate exactly as + /// a `--cfg` does. A table that sets one and is not the table this build reads leaves that + /// predicate answered by the profile chain or by the compiler's own default, which is an answer + /// about a different compilation — so the name is unanswerable instead. + #[test] + fn a_table_this_build_does_not_read_leaves_its_codegen_predicates_unanswerable() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[target.wasm32-unknown-unknown]\nrustflags = [\"-C\", \"debug-assertions=off\", \"-C\", \"panic=abort\", \"-C\", \"target-feature=+simd128\"]\n", + )]); + let build = resolve(&root, None, &[]); + + assert_eq!( + build.undecided, + vec!["debug_assertions".to_owned(), "panic".to_owned(), "target_feature".to_owned()] + ); + } + + /// `CARGO_BUILD_RUSTFLAGS` is the environment spelling of `build.rustflags`, so it belongs in + /// that slot rather than above the target tables. Ranking it higher means probing with one set + /// of flags while cargo builds with another, and whichever set carries a `--cfg` then decides + /// predicates the other contradicts. + #[test] + fn the_build_rustflags_variable_ranks_below_the_target_tables() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[target.wasm32-unknown-unknown]\nrustflags = [\"--cfg\", \"tabled\"]\n", + )]); + let environment = Environment { + target: Some("wasm32-unknown-unknown".to_owned()), + build_rustflags: Some("--cfg from_the_variable".to_owned()), + ..empty(&root) + }; + let build = Build::resolve_in(&root, None, &[], &environment); + + assert_eq!(build.cfgs, vec!["tabled".to_owned()]); + } + + /// The slot it does hold is the one `build.rustflags` holds, and it outranks the file there: + /// an environment variable is the more specific statement of the same setting. + #[test] + fn the_build_rustflags_variable_outranks_the_configured_build_rustflags() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"configured\"]\n")]); + let environment = Environment { + build_rustflags: Some("--cfg from_the_variable".to_owned()), + ..empty(&root) + }; + let build = Build::resolve_in(&root, None, &[], &environment); + + assert_eq!(build.cfgs, vec!["from_the_variable".to_owned()]); + + // And `RUSTFLAGS` still outranks it, since that is a different setting and a higher slot. + let outranked = Build::resolve_in( + &root, + None, + &[], + &Environment { + rustflags: Some("--cfg plain".to_owned()), + build_rustflags: Some("--cfg from_the_variable".to_owned()), + ..empty(&root) + }, + ); + + assert_eq!(outranked.cfgs, vec!["plain".to_owned()]); + } + + #[test] + fn the_environment_outranks_the_configuration_files() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"configured\"]\n")]); + let environment = Environment { + rustflags: Some("--cfg loom -C debug-assertions=off".to_owned()), + ..empty(&root) + }; + let build = Build::resolve_in(&root, None, &[], &environment); + + assert_eq!(build.cfgs, vec!["loom".to_owned()]); + assert_eq!(build.debug_assertions, Some(false), "the flags say so outright"); + } + + #[test] + fn cargo_build_rustflags_are_used_when_the_other_variables_are_absent() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"configured\"]\n")]); + let environment = Environment { + build_rustflags: Some("--cfg environment".to_owned()), + ..empty(&root) + }; + + assert_eq!(Build::resolve_in(&root, None, &[], &environment).cfgs, vec!["environment"]); + } + + #[test] + fn target_table_uncertainty_is_sorted_deduplicated_and_excludes_the_chosen_flags() { + let (_directory, root) = tree(&[( + ".cargo/config.toml", + "[target.chosen]\nrustflags=[\"--cfg\", \"same\"]\n\ + [target.z]\nrustflags=[\"--cfg\", \"maybe\", \"--cfg\", \"maybe\"]\n\ + [target.a]\nrustflags=[\"--cfg\", \"alpha\"]\n", + )]); + let build = resolve(&root, None, &["--target", "chosen"]); + + assert_eq!(build.cfgs, vec!["same"]); + assert_eq!(build.undecided, vec!["alpha", "maybe"]); + } + + /// The encoded variable is what cargo writes when a flag contains a space, and it outranks the + /// space-separated one; splitting it on spaces would tear one flag into two. + #[test] + fn encoded_flags_are_split_on_the_unit_separator() { + let (_directory, root) = tree(&[]); + let environment = Environment { + encoded_rustflags: Some("--cfg\u{1f}flavor=\"two words\"\u{1f}".to_owned()), + rustflags: Some("--cfg ignored".to_owned()), + ..empty(&root) + }; + let build = Build::resolve_in(&root, None, &[], &environment); + + assert_eq!(build.cfgs, vec!["flavor=\"two words\"".to_owned()]); + } + + #[test] + fn flag_value_parsing_requires_exact_spellings_and_keeps_order() { + let args = [ + "--cfg", + "one", + "--cfg=two", + "--cfg-dir=wrong", + "--cfg", + "three", + "--cfg", + "--cfg=four", + ] + .map(ToOwned::to_owned); + + assert_eq!(valued(&args, "--cfg"), vec!["one", "two", "three", "--cfg=four"]); + } + + #[test] + fn codegen_options_accept_every_rustc_spelling_but_only_cfg_relevant_values() { + let flags = [ + "-C", + "target-feature=+avx2", + "--codegen", + "panic=abort", + "-Cdebug-assertions", + "--codegen=target-feature=-sse", + "-C", + "opt-level=3", + ] + .map(ToOwned::to_owned); + + assert_eq!(codegen(&flags), vec!["target-feature=+avx2", "panic=abort", "target-feature=-sse"]); + assert_eq!(assertions(&flags), Some(true)); + } + + #[test] + fn debug_assertion_flags_use_the_last_recognised_exact_value() { + for enabled in ["y", "yes", "on", "true"] { + assert_eq!(assertions(&["-C".to_owned(), format!("debug-assertions={enabled}")]), Some(true)); + } + for disabled in ["n", "no", "off", "false"] { + assert_eq!( + assertions(&["--codegen".to_owned(), format!("debug-assertions={disabled}")]), + Some(false) + ); + } + assert_eq!( + assertions(&["-Cdebug-assertions=true", "-Cdebug-assertions=perhaps", "-Cdebug-assertions=false"].map(ToOwned::to_owned)), + Some(false) + ); + assert_eq!(assertions(&["-Cdebug-assertions=perhaps".to_owned()]), None); + assert_eq!(assertions(&["-Cdebug-assertions-extra=true".to_owned()]), None); + assert_eq!(assertions(&["-C".to_owned()]), None); + assert!(options(&["-C".to_owned()]).next().is_none()); + } + + #[test] + fn release_shorthand_must_match_exactly() { + assert_eq!(named_profile(&["-r".to_owned()]).as_deref(), Some("release")); + assert_eq!(named_profile(&["--release".to_owned()]).as_deref(), Some("release")); + assert_eq!(named_profile(&["-really-not-release".to_owned()]), None); + } + + #[test] + fn host_output_parsing_requires_the_host_field() { + assert_eq!( + parse_host("rustc 1.0\nhost: x86_64-example\nrelease: 1.0\n").as_deref(), + Some("x86_64-example") + ); + assert_eq!(parse_host("rustc 1.0\ntarget: x86_64-example\n"), None); + } + + #[test] + fn host_lookup_is_skipped_without_target_tables_and_answers_with_them() { + let with_target = CargoConfig { + tables: vec![toml::from_str("[target.host]\nrustflags=[]\n").expect("the fixture parses")], + sources: Vec::new(), + }; + let host = host_triple(&with_target, false).expect("the compiler that built the tests reports its host"); + + assert!(!host.is_empty()); + assert_ne!(host, "xyzzy"); + assert_eq!(host_triple(&CargoConfig::default(), false), None); + } + + /// A run that names no profile builds with `cargo test`, whose profile inherits `dev`. + #[test] + fn the_default_profile_has_debug_assertions_on() { + let (_directory, root) = tree(&[]); + + assert_eq!(resolve(&root, None, &[]).debug_assertions, Some(true)); + } + + #[test] + fn the_release_profile_has_them_off() { + let (_directory, root) = tree(&[]); + + assert_eq!(resolve(&root, Some("release"), &[]).debug_assertions, Some(false)); + assert_eq!(resolve(&root, None, &["--release"]).debug_assertions, Some(false)); + assert_eq!(resolve(&root, None, &["--profile", "bench"]).debug_assertions, Some(false)); + } + + /// A passthrough `--profile` is the run's own last word, above whatever was configured. + #[test] + fn a_passthrough_profile_outranks_the_configured_one() { + let (_directory, root) = tree(&[]); + + assert_eq!(resolve(&root, Some("dev"), &["--profile=release"]).debug_assertions, Some(false)); + } + + #[test] + fn a_custom_profile_follows_what_it_inherits() { + let (_directory, root) = tree(&[( + "Cargo.toml", + "[workspace]\n\n[profile.mutants]\ninherits = \"release\"\n\n[profile.loud]\ninherits = \"dev\"\n", + )]); + + assert_eq!(resolve(&root, Some("mutants"), &[]).debug_assertions, Some(false)); + assert_eq!(resolve(&root, Some("loud"), &[]).debug_assertions, Some(true)); + } + + #[test] + fn a_profile_that_switches_them_on_is_believed_over_what_it_inherits() { + let (_directory, root) = tree(&[( + "Cargo.toml", + "[workspace]\n\n[profile.mutants]\ninherits = \"release\"\ndebug-assertions = true\n", + )]); + + assert_eq!(resolve(&root, Some("mutants"), &[]).debug_assertions, Some(true)); + } + + #[test] + fn the_underscore_spelling_of_debug_assertions_is_accepted() { + let (_directory, root) = tree(&[( + "Cargo.toml", + "[workspace]\n\n[profile.mutants]\ninherits = \"release\"\ndebug_assertions = true\n", + )]); + + assert_eq!(resolve(&root, Some("mutants"), &[]).debug_assertions, Some(true)); + } + + /// The cargo configuration can override the manifest's profile tables, so it is asked first. + #[test] + fn a_configured_profile_overrides_the_manifest() { + let (_directory, root) = tree(&[ + ("Cargo.toml", "[workspace]\n\n[profile.release]\ndebug-assertions = true\n"), + (".cargo/config.toml", "[profile.release]\ndebug-assertions = false\n"), + ]); + + assert_eq!(resolve(&root, Some("release"), &[]).debug_assertions, Some(false)); + } + + /// A profile nothing describes is unanswerable rather than assumed, because assuming either + /// answer deletes one half of every `#[cfg(debug_assertions)]` from the population. + #[test] + fn a_profile_that_cannot_be_followed_is_left_unanswered() { + let (_directory, root) = tree(&[("Cargo.toml", "[workspace]\n")]); + + assert_eq!(resolve(&root, Some("nowhere"), &[]).debug_assertions, None); + } + + #[test] + fn a_missing_parent_is_not_replaced_with_the_empty_profile() { + let (_directory, root) = tree(&[( + "Cargo.toml", + "[workspace]\n\n[profile.custom]\n\n[profile.\"\"]\ndebug-assertions = true\n", + )]); + + assert_eq!(resolve(&root, Some("custom"), &[]).debug_assertions, None); + } + + /// A profile that inherits from itself is malformed, and must not spin. + #[test] + fn a_cyclic_profile_chain_terminates() { + let (_directory, root) = tree(&[( + "Cargo.toml", + "[workspace]\n\n[profile.a]\ninherits = \"b\"\n\n[profile.b]\ninherits = \"a\"\n", + )]); + + assert_eq!(resolve(&root, Some("a"), &[]).debug_assertions, None); + } + + fn profile_chain(length: usize, switched_at: usize) -> String { + use core::fmt::Write as _; + let mut manifest = String::from("[workspace]\n"); + for index in 0..=length { + let _ = writeln!(manifest, "\n[profile.p{index}]"); + if index < length { + let _ = writeln!(manifest, "inherits = \"p{}\"", index + 1); + } + if index == switched_at { + manifest.push_str("debug-assertions = true\n"); + } + } + manifest + } + + #[test] + fn profile_chain_iteration_starts_at_zero_and_ends_before_the_limit() { + for (switched_at, expected) in [(0, Some(true)), (PROFILE_DEPTH - 1, Some(true)), (PROFILE_DEPTH, None)] { + let manifest = profile_chain(PROFILE_DEPTH, switched_at); + let (_directory, root) = tree(&[("Cargo.toml", &manifest)]); + + assert_eq!(resolve(&root, Some("p0"), &[]).debug_assertions, expected); + } + } + + /// A malformed configuration file says nothing rather than stopping discovery, which is the + /// same direction every other uncertainty here resolves in. + #[test] + fn an_unparsable_configuration_file_is_ignored() { + let (_directory, root) = tree(&[(".cargo/config.toml", "[build\ntarget =\n")]); + + assert_eq!(resolve(&root, None, &[]).target, None); + } + + #[test] + fn the_probe_command_line_carries_every_setting() { + let build = Build { + target: Some("wasm32-unknown-unknown".to_owned()), + cfgs: vec!["loom".to_owned()], + codegen: vec!["target-feature=+atomics".to_owned()], + debug_assertions: Some(false), + undecided: Vec::new(), + several_targets: false, + }; + + assert_eq!( + build.probe_args(), + vec![ + "--print", + "cfg", + "--target", + "wasm32-unknown-unknown", + "-C", + "debug-assertions=off", + "-C", + "target-feature=+atomics", + "--cfg", + "loom", + ] + ); + + // An unanswered profile leaves the flag off entirely, so the compiler answers with its own + // default and the name is marked unanswerable by the set that is built from it. + assert!( + !Build::default() + .probe_args() + .iter() + .any(|argument| argument.starts_with("debug-assertions")) + ); + } +} diff --git a/crates/cargo-gamma-lib/src/cfg/cfgs.rs b/crates/cargo-gamma-lib/src/cfg/cfgs.rs new file mode 100644 index 000000000..fad0063d7 --- /dev/null +++ b/crates/cargo-gamma-lib/src/cfg/cfgs.rs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! One resolved configuration per package. + +use super::CfgSet; +use crate::HashMap; + +/// One [`CfgSet`] per package, since features differ between packages but the target does not. +#[derive(Clone, Debug, Default)] +pub struct Cfgs { + per_package: HashMap, + fallback: CfgSet, +} + +impl Cfgs { + /// Builds a set for every package named in `features`, sharing one target answer. + /// + /// Every set describes the instrumented build, which is `cargo test --no-run`, so `test` is + /// among the predicates that hold; see [`CfgSet::with_test`]. + #[must_use] + pub fn new(target: &CfgSet, features: &HashMap>) -> Self { + let per_package = features + .iter() + .map(|(package, enabled)| (package.clone(), target.clone().with_features(enabled.iter().cloned()).with_test())) + .collect(); + + Self { + per_package, + fallback: CfgSet::unconditional(), + } + } + + /// Returns a map under which nothing is stripped, for callers with no cfg information. + #[must_use] + pub fn unconditional() -> Self { + Self::default() + } + + /// Returns the set for a package. + /// + /// A package that was never resolved gets the unconditional set, so an unexpected name leaves + /// its code mutable rather than silently emptying it. + #[must_use] + pub fn for_package(&self, package: &str) -> &CfgSet { + self.per_package.get(package).unwrap_or(&self.fallback) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_package_gets_its_own_features() { + let mut features: HashMap> = HashMap::default(); + + let _old = features.insert("alpha".to_owned(), vec!["std".to_owned()]); + let _old = features.insert("beta".to_owned(), Vec::new()); + + let cfgs = Cfgs::new(&CfgSet::parse("unix\n"), &features); + + assert!(cfgs.for_package("alpha").holds_str("feature = \"std\"")); + assert!(!cfgs.for_package("beta").holds_str("feature = \"std\"")); + assert!(cfgs.for_package("alpha").holds_str("unix")); + } + + #[test] + fn an_unknown_package_is_left_alone() { + let cfgs = Cfgs::new(&CfgSet::parse("unix\n"), &HashMap::default()); + + assert!(cfgs.for_package("nobody").holds_str("windows")); + } + + /// Every set the collector is handed comes from here, so this is where `--cfg test` has to be + /// attached; a set built without it silently loses the unit-test target's code. + #[test] + fn every_package_set_describes_a_test_build() { + let mut features: HashMap> = HashMap::default(); + + let _old = features.insert("alpha".to_owned(), Vec::new()); + + let cfgs = Cfgs::new(&CfgSet::parse("unix\n"), &features); + + assert!(cfgs.for_package("alpha").holds_str("any(feature = \"absent\", test)")); + assert!(cfgs.for_package("alpha").holds_str("not(test)")); + } + + #[test] + fn the_unconditional_map_strips_nothing() { + let cfgs = Cfgs::unconditional(); + + assert!(cfgs.for_package("anything").holds_str("windows")); + } +} diff --git a/crates/cargo-gamma-lib/src/cfg/features.rs b/crates/cargo-gamma-lib/src/cfg/features.rs new file mode 100644 index 000000000..b8ceec371 --- /dev/null +++ b/crates/cargo-gamma-lib/src/cfg/features.rs @@ -0,0 +1,855 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Working out which Cargo features are on, without asking Cargo to resolve the whole graph. +//! +//! `#[cfg(feature = "...")]` is the most common gate in a real workspace, and `rustc` cannot answer +//! it: features are Cargo's concept, not the compiler's. The obvious source is `cargo metadata` +//! with its dependency resolution, but that requires a resolvable graph and can reach the network, +//! which would make discovery fail on a tree that builds perfectly well offline. +//! +//! So the answer is computed here from the metadata this tool already loads, which lists every +//! workspace member's own feature table and its declared dependencies. That is enough to be exact +//! about workspace members, which are the only packages whose code gets mutated. +//! +//! # What is resolved +//! +//! Starting from what the command line selected, three things propagate to a fixed point: +//! +//! 1. A feature's own entries, so `default = ["std"]` turns `default` on into `std` on. +//! 2. A `member/feature` entry, so one member can turn on another member's feature. +//! 3. A dependency declaration on another member, which contributes its `features` list and, unless +//! it opted out, that member's `default`. +//! +//! Development and build dependencies count too, because the schema is built with `cargo test`, +//! which compiles all of them. +//! +//! # Erring toward keeping a mutant +//! +//! Anything ambiguous resolves toward the feature being *on*, which keeps the code mutable and the +//! mutants in the population. An optional dependency is treated as enabled if anything could enable +//! it, because being wrong in the other direction would silently drop mutants from live code. + +use cargo_metadata::{DependencyKind, Metadata, Package}; + +use crate::commands::FeatureArgs; +use crate::{HashMap, HashSet}; + +/// Returns the features enabled for each workspace member. +/// +/// Only workspace members appear: a registry dependency is never mutated, so its features are of no +/// interest, and guessing at them would cost a full resolve. +/// +/// ```rust,no_run +/// # use cargo_gamma_lib::internals::cfg::features::enabled; +/// # use cargo_gamma_lib::internals::commands::FeatureArgs; +/// # fn example(metadata: &cargo_metadata::Metadata) { +/// let features = enabled(metadata, &FeatureArgs::default()); +/// +/// // Every member is present, even one with no features at all. +/// assert!(features.contains_key("my-crate")); +/// # } +/// ``` +#[must_use] +pub fn enabled(metadata: &Metadata, args: &FeatureArgs) -> HashMap> { + let members: Vec<&Package> = metadata.workspace_packages(); + let named = requested(args); + let renames = renames(&members); + let mut on: HashMap> = HashMap::default(); + + for package in &members { + let _old = on.insert(package.name.as_str().to_owned(), seed(package, args, &named)); + } + + propagate_worklist(&members, &renames, &mut on); + + on.into_iter() + .map(|(package, features)| { + let mut sorted: Vec = features.into_iter().collect(); + + sorted.sort(); + + (package, sorted) + }) + .collect() +} + +/// Splits the `--features` values into the flat list of names they denote. +/// +/// Cargo accepts both `--features a,b` and `--features a --features b`, and a `package/feature` +/// entry names a feature of another package. +fn requested(args: &FeatureArgs) -> Vec<(Option, String)> { + args.features + .iter() + .flat_map(|entry| entry.split([',', ' '])) + .filter(|entry| !entry.is_empty()) + .map(|entry| match entry.split_once('/') { + Some((package, feature)) => (Some(package.to_owned()), feature.to_owned()), + None => (None, entry.to_owned()), + }) + .collect() +} + +/// Returns the features a package starts with, before anything propagates. +fn seed(package: &Package, args: &FeatureArgs, named: &[(Option, String)]) -> HashSet { + let mut on: HashSet = HashSet::default(); + + if args.all_features { + on.extend(package.features.keys().cloned()); + + return on; + } + + if !args.no_default_features && package.features.contains_key("default") { + let _added = on.insert("default".to_owned()); + } + + for (owner, feature) in named { + // An unqualified name applies to whichever selected packages declare it, which is what + // Cargo does for a workspace build. A qualified one names its package outright. + let mine = owner.as_ref().is_none_or(|owner| owner == package.name.as_str()); + + if mine && package.features.contains_key(feature) { + let _added = on.insert(feature.clone()); + } + } + + on +} + +/// Runs worklist-driven propagation to a fixed point. +/// +/// Instead of full rescans, each newly enabled feature is pushed onto a queue. When a feature is +/// popped, its entries (other features it activates) are processed. If any of those turn on a new +/// feature, that new feature is pushed. Dependency edges are processed once per member. This is +/// O(E) in the number of feature edges rather than O(V * passes). +fn propagate_worklist(members: &[&Package], renames: &Renames, on: &mut HashMap>) { + use std::collections::VecDeque; + + let packages: HashMap<&str, &Package> = members.iter().map(|package| (package.name.as_str(), *package)).collect(); + + // Seed the worklist with every feature that is already on. + let mut queue: VecDeque<(String, String)> = VecDeque::new(); + + for (package, features) in on.iter() { + for feature in features { + queue.push_back((package.clone(), feature.clone())); + } + } + + // Also enqueue dependency-contributed features from the initial state. + for package in members { + for dependency in &package.dependencies { + let target = dependency.name.as_str(); + + if !on.contains_key(target) { + continue; + } + + if matches!(dependency.kind, DependencyKind::Unknown) { + continue; + } + + for feature in &dependency.features { + if turn_on(target, feature, on) { + queue.push_back((target.to_owned(), feature.clone())); + } + } + + if dependency.uses_default_features && turn_on(target, "default", on) { + queue.push_back((target.to_owned(), "default".to_owned())); + } + } + } + + while let Some((package_name, feature)) = queue.pop_front() { + let Some(package) = packages.get(package_name.as_str()).copied() else { + continue; + }; + + // A feature's own entries: `foo = ["bar", "dep/baz"]`. + if let Some(entries) = package.features.get(&feature) { + for entry in entries { + if let Some((pkg, feat)) = apply_worklist(entry, &package_name, renames, on) { + queue.push_back((pkg, feat)); + } + } + } + } +} + +/// Applies one feature-table entry, returning the (package, feature) pair if something new was +/// turned on (for enqueueing). +fn apply_worklist(entry: &str, owner: &str, renames: &Renames, on: &mut HashMap>) -> Option<(String, String)> { + if entry.starts_with("dep:") { + return None; + } + + match entry.split_once('/') { + Some((token, feature)) => { + let token = token.trim_end_matches('?'); + let package = renames + .get(owner) + .and_then(|by_alias| by_alias.get(token)) + .map_or(token, String::as_str); + + turn_on(package, feature, on).then(|| (package.to_owned(), feature.to_owned())) + } + + None => turn_on(owner, entry, on).then(|| (owner.to_owned(), entry.to_owned())), + } +} + +/// Runs one propagation pass, returning whether anything changed. +/// +/// Retained for the test that exercises cycle termination; the worklist above is what `enabled` +/// actually calls. +#[cfg(test)] +fn propagate(members: &[&Package], renames: &Renames, on: &mut HashMap>) -> bool { + let mut changed = false; + + for package in members { + let name = package.name.as_str(); + let mine = on.get(name).cloned().unwrap_or_default(); + + for feature in &mine { + let Some(entries) = package.features.get(feature) else { + continue; + }; + + for entry in entries { + changed |= apply(entry, name, renames, on); + } + } + + for dependency in &package.dependencies { + let target = dependency.name.as_str(); + + if !on.contains_key(target) { + continue; + } + + if matches!(dependency.kind, DependencyKind::Unknown) { + continue; + } + + for feature in &dependency.features { + changed |= turn_on(target, feature, on); + } + + if dependency.uses_default_features { + changed |= turn_on(target, "default", on); + } + } + } + + changed +} + +/// For each member, the manifest-local dependency names that stand for another package. +type Renames = HashMap>; + +/// Collects the dependency aliases each member's manifest declares. +/// +/// `bruce = { package = "beta" }` lets the rest of that manifest — including its feature table — +/// call `beta` by the name `bruce`, and nothing outside the manifest knows the alias. The enabled +/// map is keyed by real package names, so a `bruce/y` entry has to be translated before it is +/// looked up or the feature it forwards lands nowhere and `beta`'s `#[cfg(feature = "y")]` code is +/// read as absent. +/// +/// Only renamed dependencies are recorded, since every other token already is the package name. +fn renames(members: &[&Package]) -> Renames { + members + .iter() + .filter_map(|package| { + let mapped: HashMap = package + .dependencies + .iter() + .filter_map(|dependency| { + dependency + .rename + .as_ref() + .map(|alias| (alias.clone(), dependency.name.as_str().to_owned())) + }) + .collect(); + + (!mapped.is_empty()).then(|| (package.name.as_str().to_owned(), mapped)) + }) + .collect() +} + +/// Applies one entry from a feature table, which may name this package's feature or another's. +/// +/// The spellings are `plain`, `dep:some-crate`, `some-crate/feature` and `some-crate?/feature`. +/// Only the ones that name a feature matter here; `dep:` merely activates an optional dependency, +/// which this module already assumes. +/// +/// The token before the slash is whatever `owner`'s manifest calls the dependency, so it is +/// resolved through that manifest's renames before anything is looked up. +#[cfg(test)] +fn apply(entry: &str, owner: &str, renames: &Renames, on: &mut HashMap>) -> bool { + if entry.starts_with("dep:") { + return false; + } + + match entry.split_once('/') { + Some((token, feature)) => { + let token = token.trim_end_matches('?'); + let package = renames + .get(owner) + .and_then(|by_alias| by_alias.get(token)) + .map_or(token, String::as_str); + + turn_on(package, feature, on) + } + + None => turn_on(owner, entry, on), + } +} + +/// Turns a feature on for a package, returning whether that was news. +/// +/// A package that is not a workspace member is ignored: its code is never mutated, so what it +/// compiles is not this tool's business. +fn turn_on(package: &str, feature: &str, on: &mut HashMap>) -> bool { + on.get_mut(package).is_some_and(|features| features.insert(feature.to_owned())) +} + +/// The feature selection `args` asks for, updated by selectors in the passthrough arguments. +/// +/// `-C --all-features`, and a `cargo_args` entry in `gamma.toml` naming `--features`, reach the +/// cargo the run really invokes, so the build genuinely enables what they name. A closure computed +/// from the typed arguments alone has never heard of them: every `#[cfg(feature = "…")]` item they +/// turn on resolves absent and yields no mutants, and a `[[bin]]` whose `required-features` they +/// satisfy is judged unbuildable and leaves the population whole — a score raised by exactly the +/// code nobody measured. +/// +/// The mining mirrors [`Build::resolve`](crate::cfg::Build::resolve)'s reading of `--target` and +/// `--profile` from the same vector, and covers the spellings cargo accepts: `--features x`, +/// `--features=x`, `-F x` and `-Fx`, plus the two flags that override the selection outright. +/// The result mirrors both enabling selectors and `--no-default-features`, which narrows the +/// selection. +#[must_use] +pub fn from_extra(args: &FeatureArgs, extra: &[String]) -> FeatureArgs { + let mut widened = args.clone(); + let mut expecting = false; + + for argument in extra { + if expecting { + widened.features.push(argument.clone()); + expecting = false; + + continue; + } + + match argument.as_str() { + "--all-features" => widened.all_features = true, + "--no-default-features" => widened.no_default_features = true, + "--features" | "-F" => expecting = true, + + // The prefix has to be matched exactly before the `=`, or `--features-of-interest` + // would be read as a selection; `-F` is the one spelling cargo also accepts attached. + other => { + if let Some(value) = other + .strip_prefix("--features=") + .or_else(|| other.strip_prefix("-F=")) + .or_else(|| other.strip_prefix("-F")) + { + widened.features.push(value.to_owned()); + } + } + } + } + + widened +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::fs; + + use camino::Utf8PathBuf; + use tempfile::TempDir; + + use super::*; + use crate::discover::load_metadata; + + /// Writes a workspace of manifests and gives back the metadata cargo reads from it. + /// + /// A `src/lib.rs` is written beside every manifest that declares a package, because a package + /// with no target at all is not something cargo will describe. + fn metadata_for(files: &[(&str, &str)]) -> (TempDir, Metadata) { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + for (relative, contents) in files { + let path = root.join(relative); + + fs::create_dir_all(path.parent().expect("a manifest has a directory").as_std_path()).expect("directories"); + fs::write(path.as_std_path(), contents).expect("the manifest is written"); + + if contents.contains("[package]") { + let source = path.parent().expect("a manifest has a directory").join("src"); + + fs::create_dir_all(source.as_std_path()).expect("a source directory"); + fs::write(source.join("lib.rs").as_std_path(), "pub fn f() {}\n").expect("a library root"); + } + } + + let metadata = load_metadata(&root, &FeatureArgs::default()).expect("the fixture workspace has metadata"); + + (directory, metadata) + } + + /// The features enabled for `package` under `args`, as a sorted list. + fn features_of(files: &[(&str, &str)], args: &FeatureArgs, package: &str) -> Vec { + let (_directory, metadata) = metadata_for(files); + + enabled(&metadata, args).remove(package).unwrap_or_default() + } + + fn enabled_by_rescan(metadata: &Metadata, args: &FeatureArgs) -> HashMap> { + let members: Vec<&Package> = metadata.workspace_packages(); + let named = requested(args); + let renames = renames(&members); + let mut on: HashMap> = members + .iter() + .map(|package| (package.name.as_str().to_owned(), seed(package, args, &named))) + .collect(); + + while propagate(&members, &renames, &mut on) {} + + on.into_iter() + .map(|(package, features)| { + let mut features: Vec<_> = features.into_iter().collect(); + features.sort(); + (package, features) + }) + .collect() + } + + /// A single-package workspace whose library declares the given feature table. + fn alone(table: &str) -> String { + format!("[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[features]\n{table}\n\n[workspace]\n") + } + + #[test] + fn default_is_on_and_expands() { + let manifest = alone("default = [\"std\"]\nstd = []\nstats = []\n"); + let found = features_of(&[("Cargo.toml", &manifest)], &FeatureArgs::default(), "alpha"); + + assert_eq!(found, vec!["default".to_owned(), "std".to_owned()]); + } + + #[test] + fn worklist_matches_fixed_point_rescans() { + let files = [ + ("Cargo.toml", "[workspace]\nmembers = [\"alpha\", \"beta\"]\nresolver = \"3\"\n"), + ( + "alpha/Cargo.toml", + "[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [features]\ndefault = [\"cycle\", \"bruce/y\"]\ncycle = [\"default\"]\nextra = []\n\n\ + [dependencies]\nbruce = { package = \"beta\", path = \"../beta\", features = [\"z\"] }\n", + ), + ( + "beta/Cargo.toml", + "[package]\nname = \"beta\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [features]\ndefault = [\"z\"]\ny = [\"z\"]\nz = []\n", + ), + ]; + let (_directory, metadata) = metadata_for(&files); + + for args in [ + FeatureArgs::default(), + FeatureArgs { + all_features: true, + ..FeatureArgs::default() + }, + ] { + assert_eq!(enabled(&metadata, &args), enabled_by_rescan(&metadata, &args)); + } + } + + #[test] + fn no_default_features_leaves_nothing_on() { + let manifest = alone("default = [\"std\"]\nstd = []\n"); + let args = FeatureArgs { + no_default_features: true, + ..FeatureArgs::default() + }; + + assert!(features_of(&[("Cargo.toml", &manifest)], &args, "alpha").is_empty()); + } + + #[test] + fn all_features_turns_on_everything_declared() { + let manifest = alone("default = [\"std\"]\nstd = []\nstats = []\n"); + let args = FeatureArgs { + all_features: true, + ..FeatureArgs::default() + }; + let found = features_of(&[("Cargo.toml", &manifest)], &args, "alpha"); + + assert_eq!(found, vec!["default".to_owned(), "stats".to_owned(), "std".to_owned()]); + } + + #[test] + fn a_named_feature_is_turned_on() { + let manifest = alone("default = [\"std\"]\nstd = []\nstats = []\n"); + let args = FeatureArgs { + features: vec!["stats".to_owned()], + ..FeatureArgs::default() + }; + + assert!(features_of(&[("Cargo.toml", &manifest)], &args, "alpha").contains(&"stats".to_owned())); + } + + #[test] + fn several_named_features_may_share_one_argument() { + // Cargo accepts `--features a,b`, so a value that was never split would name no feature at + // all and quietly leave both off. + let manifest = alone("a = []\nb = []\nc = []\n"); + let args = FeatureArgs { + features: vec!["a,b".to_owned()], + ..FeatureArgs::default() + }; + let found = features_of(&[("Cargo.toml", &manifest)], &args, "alpha"); + + assert_eq!(found, vec!["a".to_owned(), "b".to_owned()]); + } + + #[test] + fn a_feature_that_is_not_declared_is_not_invented() { + let manifest = alone("a = []\n"); + let args = FeatureArgs { + features: vec!["nope".to_owned()], + ..FeatureArgs::default() + }; + + assert!(features_of(&[("Cargo.toml", &manifest)], &args, "alpha").is_empty()); + } + + #[test] + fn a_qualified_name_only_reaches_its_own_package() { + let manifest = alone("a = []\n"); + let args = FeatureArgs { + features: vec!["beta/a".to_owned()], + ..FeatureArgs::default() + }; + + assert!(features_of(&[("Cargo.toml", &manifest)], &args, "alpha").is_empty()); + } + + #[test] + fn features_chain_through_their_own_entries() { + let manifest = alone("default = [\"a\"]\na = [\"b\"]\nb = [\"c\"]\nc = []\n"); + let found = features_of(&[("Cargo.toml", &manifest)], &FeatureArgs::default(), "alpha"); + + assert_eq!(found, vec!["a".to_owned(), "b".to_owned(), "c".to_owned(), "default".to_owned()]); + } + + #[test] + fn a_cycle_between_features_terminates() { + // A malformed manifest must not hang discovery. + let manifest = alone("default = [\"a\"]\na = [\"b\"]\nb = [\"a\"]\n"); + let found = features_of(&[("Cargo.toml", &manifest)], &FeatureArgs::default(), "alpha"); + + assert!(found.contains(&"a".to_owned()), "{found:?}"); + assert!(found.contains(&"b".to_owned()), "{found:?}"); + } + + #[test] + fn a_dep_entry_names_no_feature() { + // `dep:beta` switches on an optional dependency. Reading it as a feature named `dep:beta` + // would put a name in the set that no `#[cfg]` can ever spell. + let files = pair( + "[features]\ndefault = [\"dep:beta\"]\n\n[dependencies]\nbeta = { path = \"../beta\", optional = true, default-features = false }\n", + "loud = []\n", + ); + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "alpha"); + + assert_eq!(found, vec!["default".to_owned()], "`dep:` activates a crate, not a feature"); + assert!(features_of(&borrowed(&files), &FeatureArgs::default(), "beta").is_empty()); + } + + /// A two-member workspace, where `alpha` relates to `beta` however the caller says. + fn pair(alpha_extra: &str, beta_features: &str) -> Vec<(String, String)> { + vec![ + ( + "Cargo.toml".to_owned(), + "[workspace]\nmembers = [\"alpha\", \"beta\"]\nresolver = \"3\"\n".to_owned(), + ), + ( + "alpha/Cargo.toml".to_owned(), + format!("[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n{alpha_extra}"), + ), + ( + "beta/Cargo.toml".to_owned(), + format!("[package]\nname = \"beta\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[features]\n{beta_features}"), + ), + ] + } + + /// Borrows an owned fixture into the shape the loader wants. + fn borrowed(files: &[(String, String)]) -> Vec<(&str, &str)> { + files.iter().map(|(path, text)| (path.as_str(), text.as_str())).collect() + } + + #[test] + fn one_member_can_turn_on_anothers_feature() { + let files = pair( + "[features]\ndefault = [\"beta/loud\"]\n\n[dependencies]\nbeta = { path = \"../beta\", default-features = false }\n", + "loud = []\nquiet = []\n", + ); + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "beta"); + + assert_eq!(found, vec!["loud".to_owned()]); + } + + #[test] + fn a_dependency_declaration_contributes_its_features() { + let files = pair( + "[dependencies]\nbeta = { path = \"../beta\", features = [\"loud\"] }\n", + "default = [\"quiet\"]\nloud = []\nquiet = []\n", + ); + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "beta"); + + assert!(found.contains(&"loud".to_owned()), "{found:?}"); + assert!(found.contains(&"quiet".to_owned()), "the default came through too: {found:?}"); + } + + /// A manifest may call a dependency whatever it likes — `bruce = { package = "beta" }` — and + /// everything in that manifest, its feature table included, then speaks of `bruce`. The map + /// being resolved is keyed by real package names, so a `bruce/y` entry that was not translated + /// landed nowhere and `beta`'s `#[cfg(feature = "y")]` code was read as absent, taking its live + /// mutants out of the population. + #[test] + fn a_renamed_dependency_forwards_its_features_to_the_package_it_names() { + let files = pair( + "[features]\ndefault = [\"bruce/y\"]\n\n\ + [dependencies]\nbruce = { package = \"beta\", path = \"../beta\", default-features = false }\n", + "y = []\nz = []\n", + ); + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "beta"); + + assert_eq!(found, vec!["y".to_owned()], "{found:?}"); + } + + /// The weak spelling forwards through an alias exactly as the plain one does; only the + /// question of whether the optional dependency is switched on differs, and this module already + /// assumes it is. + #[test] + fn a_weak_reference_through_an_alias_resolves_too() { + let files = pair( + "[features]\ndefault = [\"bruce?/y\"]\n\n\ + [dependencies]\nbruce = { package = \"beta\", path = \"../beta\", optional = true, default-features = false }\n", + "y = []\nz = []\n", + ); + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "beta"); + + assert_eq!(found, vec!["y".to_owned()], "{found:?}"); + } + + /// An alias is local to the manifest that declares it, so a name another member happens to use + /// for something else must not be translated by it — that would turn a feature on for a + /// package the entry never named. + #[test] + fn an_alias_does_not_reach_beyond_the_manifest_that_declares_it() { + let files = [ + ( + "Cargo.toml".to_owned(), + "[workspace]\nmembers = [\"alpha\", \"beta\", \"bruce\"]\nresolver = \"3\"\n".to_owned(), + ), + ( + "alpha/Cargo.toml".to_owned(), + "[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [features]\ndefault = [\"bruce/y\"]\n\n\ + [dependencies]\nbruce = { path = \"../bruce\", default-features = false }\n" + .to_owned(), + ), + ( + "beta/Cargo.toml".to_owned(), + "[package]\nname = \"beta\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[features]\ny = []\n".to_owned(), + ), + ( + "bruce/Cargo.toml".to_owned(), + "[package]\nname = \"bruce\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[features]\ny = []\n".to_owned(), + ), + ]; + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "beta"); + + assert!(found.is_empty(), "the entry named the crate called bruce, not beta: {found:?}"); + assert_eq!( + features_of(&borrowed(&files), &FeatureArgs::default(), "bruce"), + vec!["y".to_owned()] + ); + } + + #[test] + fn a_dev_dependency_counts_because_the_schema_is_built_with_cargo_test() { + let files = pair( + "[dev-dependencies]\nbeta = { path = \"../beta\", features = [\"loud\"], default-features = false }\n", + "loud = []\nquiet = []\n", + ); + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "beta"); + + assert_eq!(found, vec!["loud".to_owned()]); + } + + #[test] + fn a_feature_of_a_package_outside_the_workspace_is_ignored() { + // Only members are mutated, so what a dependency outside the workspace compiles is not + // this tool's business, and naming one must not invent an entry for it. + let manifest = alone("default = [\"a\"]\na = []\n"); + let (_directory, metadata) = metadata_for(&[("Cargo.toml", &manifest)]); + let found = enabled(&metadata, &FeatureArgs::default()); + + assert_eq!(found.len(), 1, "only the one member is described: {found:?}"); + } + + /// A `some-crate/feature` reference is turned on for the named crate the moment it is written, + /// before anything checks whether that crate actually declares the feature. Propagation must + /// still tolerate the mismatch afterwards rather than panicking, or crediting the package with + /// a feature nothing in its own manifest ever named. + #[test] + fn a_member_feature_reference_to_an_undeclared_feature_does_not_invent_downstream_entries() { + let files = pair( + "[features]\ndefault = [\"beta/ghost\"]\n\n[dependencies]\nbeta = { path = \"../beta\", default-features = false }\n", + "loud = []\n", + ); + + let found = features_of(&borrowed(&files), &FeatureArgs::default(), "beta"); + + // The reference switched the name on for `beta`, but nothing there declares `ghost`, so no + // further features chain off it and the set stays exactly what was named. + assert_eq!(found, vec!["ghost".to_owned()], "{found:?}"); + } + + /// A dependency on a crate outside the workspace has no entry in the feature map at all, since + /// only members are tracked; the propagation pass has to recognise that and move on rather + /// than fabricate an entry or panic reaching for one that was never created. + #[test] + fn a_dependency_outside_the_workspace_contributes_nothing_to_propagate() { + let files = [ + ( + "Cargo.toml", + "[workspace]\nmembers = [\"alpha\"]\nexclude = [\"external\"]\nresolver = \"3\"\n", + ), + ( + "alpha/Cargo.toml", + "[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [features]\ndefault = []\n\n\ + [dependencies]\nexternal = { path = \"../external\", features = [\"derive\"] }\n", + ), + ( + "external/Cargo.toml", + // Excluded from the workspace above and carrying its own `[workspace]` table, this + // is outside the tool's business entirely, exactly like a registry dependency + // would be — and outside the feature map, which only ever tracks members. + "[package]\nname = \"external\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [features]\nderive = []\n\n[workspace]\n", + ), + ]; + + let found = features_of(&files, &FeatureArgs::default(), "alpha"); + + assert_eq!(found, vec!["default".to_owned()], "{found:?}"); + } + + /// Cargo never actually emits `DependencyKind::Unknown` — it is a `#[serde(other)]` catch-all + /// for a variant this build of `cargo_metadata` does not recognise, kept forward-compatible + /// against a future cargo. A dependency of that kind must still be inert here rather than + /// having its features silently propagate, which is the whole reason the check exists at all. + #[test] + fn a_dependency_of_an_unrecognised_kind_does_not_propagate_its_features() { + let files = pair( + "[dependencies]\nbeta = { path = \"../beta\", features = [\"loud\"] }\n", + "loud = []\n", + ); + + let (_directory, metadata) = metadata_for(&borrowed(&files)); + + // Round-trip the real metadata through JSON, forging the one field that real cargo never + // produces, so the dependency in `alpha` is now of a kind this module cannot recognise. + let mut value = serde_json::to_value(&metadata).expect("metadata serialises"); + let packages = value + .get_mut("packages") + .and_then(serde_json::Value::as_array_mut) + .expect("packages array"); + let alpha = packages + .iter_mut() + .find(|package| package["name"] == "alpha") + .expect("the fixture always declares an alpha package"); + let dependency = alpha["dependencies"] + .as_array_mut() + .expect("alpha has dependencies") + .first_mut() + .expect("alpha declares exactly one dependency"); + dependency["kind"] = serde_json::Value::String("build-tool".to_owned()); + + let forged: Metadata = serde_json::from_value(value).expect("the forged metadata still deserialises"); + let found = enabled(&forged, &FeatureArgs::default()).remove("beta").unwrap_or_default(); + + assert!(!found.contains(&"loud".to_owned()), "{found:?}"); + } + + #[test] + fn every_member_appears_even_with_no_features() { + let manifest = alone(""); + let (_directory, metadata) = metadata_for(&[("Cargo.toml", &manifest)]); + let found = enabled(&metadata, &FeatureArgs::default()); + + assert!(found.contains_key("alpha"), "a missing package would be left unconditional"); + assert!(found["alpha"].is_empty()); + } + + /// Every spelling cargo itself accepts, because the vector is passed to cargo verbatim: a + /// spelling this misses is a feature the build has and the closure does not. + #[test] + fn a_passthrough_selector_is_read_however_it_is_written() { + let extra = |arguments: &[&str]| { + let owned: Vec = arguments.iter().map(|argument| (*argument).to_owned()).collect(); + + from_extra(&FeatureArgs::default(), &owned) + }; + + assert_eq!(extra(&["--features", "cli"]).features, vec!["cli".to_owned()]); + assert_eq!(extra(&["--features=cli,extra"]).features, vec!["cli,extra".to_owned()]); + assert_eq!(extra(&["-F", "cli"]).features, vec!["cli".to_owned()]); + assert_eq!(extra(&["-Fcli"]).features, vec!["cli".to_owned()]); + assert!(extra(&["--all-features"]).all_features); + assert!(extra(&["--no-default-features"]).no_default_features); + } + + /// The prefix is matched exactly before the `=`, or an unrelated flag that merely starts the + /// same way would name a feature nobody asked for and turn on code the build does not compile. + #[test] + fn an_argument_that_merely_begins_like_a_selector_names_no_feature() { + let extra = vec!["--features-of-interest=cli".to_owned(), "--target".to_owned(), "wasm32".to_owned()]; + let widened = from_extra(&FeatureArgs::default(), &extra); + + assert!(widened.features.is_empty(), "{:?}", widened.features); + assert!(!widened.all_features); + } + + /// The typed arguments are widened, never replaced: both routes reach the same cargo, so what + /// either of them names is on. + #[test] + fn a_passthrough_selector_adds_to_the_typed_one() { + let args = FeatureArgs { + features: vec!["typed".to_owned()], + ..FeatureArgs::default() + }; + let widened = from_extra(&args, &["--features".to_owned(), "passed".to_owned()]); + + assert_eq!(widened.features, vec!["typed".to_owned(), "passed".to_owned()]); + } +} diff --git a/crates/cargo-gamma-lib/src/cfg/mod.rs b/crates/cargo-gamma-lib/src/cfg/mod.rs new file mode 100644 index 000000000..47076ff78 --- /dev/null +++ b/crates/cargo-gamma-lib/src/cfg/mod.rs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Deciding which conditionally compiled code is actually in the build. +//! +//! Rust strips `#[cfg(...)]` items before anything else runs, so code behind a predicate that does +//! not hold is not in the compiled artifact at all. A mutant placed there is therefore unkillable +//! by construction: activating it changes nothing, every test passes, and the tool reports a +//! survivor no test could ever have caught. +//! +//! That is not merely a wrong number. Each such mutant costs a full run of every test binary that +//! links its package, so a workspace with a lot of platform-specific or feature-gated code spends +//! most of its time proving things about code it did not build. On one real workspace, 378 of +//! 2,290 survivors — 16.5% — sat behind a gate that did not hold. +//! +//! # What this module decides +//! +//! [`CfgSet`] holds the configuration predicates that are true for the build, expands active +//! `cfg_attr` attributes, and answers whether the resulting `#[cfg(...)]` attributes hold: +//! +//! ```rust +//! # use cargo_gamma_lib::internals::cfg::CfgSet; +//! let set = CfgSet::parse("unix\ntarget_arch=\"x86_64\"\n").with_features(["std".to_owned()]); +//! +//! assert!(set.holds_str("unix")); +//! assert!(set.holds_str("feature = \"std\"")); +//! assert!(!set.holds_str("windows")); +//! assert!(!set.holds_str("feature = \"stats\"")); +//! assert!(set.holds_str("any(unix, windows)")); +//! assert!(set.holds_str("not(windows)")); +//! assert!(!set.holds_str("all(unix, feature = \"stats\")")); +//! ``` +//! +//! The names and values come from `rustc --print cfg`, asked about the build cargo will actually +//! run rather than about the compiler's own defaults: [`Build`] resolves the target, the profile's +//! `debug_assertions` and any custom `--cfg` from the same places cargo reads them — the +//! command line, `CARGO_BUILD_TARGET`, `RUSTFLAGS`, `CARGO_ENCODED_RUSTFLAGS`, the +//! `.cargo/config.toml` files and the manifest's profile tables — and hands them to the probe as +//! flags. They have to be passed rather than inherited: `RUSTFLAGS` is a Cargo-facing variable +//! that `rustc` does not interpret, so a probe that merely inherited it would answer about a +//! different compilation from the one the run builds. +//! +//! Features are the one thing `rustc` cannot answer, because they are Cargo's concept. They are +//! resolved separately, per package, by [`features`]. +//! +//! # Erring toward keeping a mutant +//! +//! Every uncertainty resolves toward the predicate holding, which keeps the mutant. A mutant that +//! should not exist is visible and annoying; a mutant silently missing from the population is a +//! hole in the measurement that nobody can see. So an unparsable attribute, an unrecognised +//! predicate function, a build whose profile could not be followed, a `--cfg` a configuration file +//! sets under a predicate of its own, and a package whose features could not be resolved all leave +//! the code mutable: +//! +//! ```rust +//! # use cargo_gamma_lib::internals::cfg::CfgSet; +//! let set = CfgSet::parse("unix\n"); +//! +//! // `version` is a predicate this module does not model, so it is assumed to hold. +//! assert!(set.holds_str("version(\"1.80\")")); +//! +//! // And a set that was never resolved holds everything. +//! assert!(CfgSet::unconditional().holds_str("windows")); +//! ``` + +mod build; +mod cfgs; +mod probe; + +pub mod features; + +pub use build::Build; +pub(crate) use cargo_gamma_engine::cfg::test_gated_for; +pub use cargo_gamma_engine::cfg::{CfgSet, test_gated}; +pub use cfgs::Cfgs; +pub(crate) use probe::for_build; diff --git a/crates/cargo-gamma-lib/src/cfg/probe.rs b/crates/cargo-gamma-lib/src/cfg/probe.rs new file mode 100644 index 000000000..15bb62288 --- /dev/null +++ b/crates/cargo-gamma-lib/src/cfg/probe.rs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Resolving engine cfg evidence from the compiler selected by Cargo policy. + +use std::process::Command; + +use super::{Build, CfgSet}; +use crate::Result; +use crate::error::error; + +pub(crate) fn for_build(build: &Build) -> Result { + if build.several_targets { + return Ok(CfgSet::unconditional()); + } + + let program = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_owned()); + for_build_with(build, &program) +} + +fn for_build_with(build: &Build, program: impl AsRef) -> Result { + let mut command = Command::new(program); + let _builder = command.args(build.probe_args()); + let output = command + .output() + .map_err(|cause| error!("could not run `rustc --print cfg`").caused_by(cause))?; + + if !output.status.success() { + return Err(error!( + "`rustc --print cfg` failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + + let mut set = CfgSet::parse(&String::from_utf8_lossy(&output.stdout)); + + if build.debug_assertions.is_none() { + set = set.with_undecided(["debug_assertions".to_owned()]); + } + + Ok(set.with_undecided(build.undecided.iter().cloned())) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + + #[test] + fn a_rustc_probe_failure_is_reported() { + let executable = std::env::current_exe().expect("the test executable has a path"); + let error = for_build_with(&Build::default(), executable).expect_err("libtest rejects rustc's arguments"); + + assert!(error.to_string().contains("`rustc --print cfg` failed"), "{error}"); + } + + #[test] + fn the_host_answers_about_itself() { + let set = for_build(&Build::default()).expect("the compiler that built the tests is available"); + + assert!(set.holds_str("target_pointer_width = \"64\"") || set.holds_str("target_pointer_width = \"32\"")); + assert!(!set.holds_str("target_arch = \"there-is-no-such-architecture\"")); + } + + #[test] + fn a_build_for_another_target_answers_about_that_target() { + let host = for_build(&Build::default()).expect("the host answers"); + let (elsewhere, family) = if host.holds_str("windows") { + ("x86_64-unknown-linux-gnu", "unix") + } else { + ("x86_64-pc-windows-msvc", "windows") + }; + let build = Build { + target: Some(elsewhere.to_owned()), + debug_assertions: Some(true), + ..Build::default() + }; + let set = for_build(&build).expect("a built-in triple answers"); + + assert!(set.holds_str(family), "{elsewhere} is a {family} target"); + assert!(!set.holds_str(if family == "unix" { "windows" } else { "unix" })); + assert!(set.holds_str("target_arch = \"x86_64\"")); + assert_ne!(host.holds_str("windows"), set.holds_str("windows")); + } + + #[test] + fn a_profile_without_debug_assertions_strips_the_other_half() { + let release = for_build(&Build { + debug_assertions: Some(false), + ..Build::default() + }) + .expect("the host answers"); + let debug = for_build(&Build { + debug_assertions: Some(true), + ..Build::default() + }) + .expect("the host answers"); + + assert!(!release.holds_str("debug_assertions")); + assert!(release.holds_str("not(debug_assertions)")); + assert!(debug.holds_str("debug_assertions")); + assert!(!debug.holds_str("not(debug_assertions)")); + } + + #[test] + fn an_unanswered_profile_leaves_both_halves_mutable() { + let set = for_build(&Build::default()).expect("the host answers"); + + assert!(set.holds_str("debug_assertions")); + assert!(set.holds_str("not(debug_assertions)")); + assert!(!set.holds_str("all(debug_assertions, there_is_no_such_predicate)")); + } + + #[test] + fn a_custom_predicate_the_build_passes_holds() { + let set = for_build(&Build { + cfgs: vec!["loom".to_owned(), "flavor=\"strawberry\"".to_owned()], + debug_assertions: Some(true), + ..Build::default() + }) + .expect("the host answers"); + + assert!(set.holds_str("loom")); + assert!(set.holds_str("flavor = \"strawberry\"")); + assert!(!set.holds_str("flavor = \"vanilla\"")); + assert!(!set.holds_str("kani")); + } + + #[test] + fn a_name_the_build_cannot_settle_is_unanswerable() { + let set = for_build(&Build { + undecided: vec!["sometimes".to_owned()], + debug_assertions: Some(true), + ..Build::default() + }) + .expect("the host answers"); + + assert!(set.holds_str("sometimes")); + assert!(set.holds_str("not(sometimes)")); + assert!(!set.holds_str("all(sometimes, there_is_no_such_predicate)")); + } + + #[test] + fn a_valued_predicate_the_build_cannot_settle_is_unanswerable() { + let set = for_build(&Build { + undecided: vec!["flavor".to_owned()], + debug_assertions: Some(true), + ..Build::default() + }) + .expect("the host answers"); + + assert!(set.holds_str("flavor = \"strawberry\"")); + assert!(set.holds_str("not(flavor = \"strawberry\")")); + assert!(!set.holds_str("all(flavor = \"strawberry\", there_is_no_such_predicate)")); + } + + #[test] + fn a_build_of_several_targets_strips_nothing() { + let build = Build { + several_targets: true, + ..Build::default() + }; + let set = for_build(&build).expect("no probe is needed"); + + assert!(set.holds_str("windows")); + assert!(set.holds_str("unix")); + } +} diff --git a/crates/cargo-gamma-lib/src/ci/annotations.rs b/crates/cargo-gamma-lib/src/ci/annotations.rs new file mode 100644 index 000000000..7ef8830bd --- /dev/null +++ b/crates/cargo-gamma-lib/src/ci/annotations.rs @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! How much CI surfacing to emit. + +use camino::Utf8Path; +use clap::ValueEnum; + +use super::finding::{describe, findings, relative}; +use crate::model::Mutant; + +/// How much of the CI surfacing to emit. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum Annotations { + /// Emit nothing. + None, + + /// Emit the GitHub renderings when running inside GitHub Actions, and nothing otherwise. + #[default] + Auto, + + /// Emit the GitHub renderings regardless of where we are running. + Github, +} + +/// The most findings any one annotation run will print. +/// +/// GitHub keeps only the first ten annotations of a level per step and silently discards the rest, +/// so printing more produces a log full of commands that had no effect and a reviewer who believes +/// they have seen everything. The report and the SARIF log carry the full population. +const ANNOTATION_LIMIT: usize = 10; + +/// Whether the GitHub renderings should be emitted. +/// +/// `Auto` keys off `GITHUB_ACTIONS`, which the runner sets on every step. That means a workflow +/// gets annotations by adding nothing to its command line, which is the only adoption path that +/// reliably happens. +#[must_use] +pub const fn wanted(annotations: Annotations, github_actions: bool) -> bool { + match annotations { + Annotations::None => false, + Annotations::Auto => github_actions, + Annotations::Github => true, + } +} + +/// Renders the GitHub Actions workflow commands that place survivors on the diff. +/// +/// The message is the mutation itself rather than a summary of it. A reviewer looking at the line +/// needs to know what was changed and that nothing complained, and any wording that does not +/// contain the replacement makes them go and look it up. +#[must_use] +pub fn annotations(mutants: &[Mutant], root: &Utf8Path) -> Vec { + let survivors = findings(mutants); + let mut lines: Vec = survivors + .iter() + .take(ANNOTATION_LIMIT) + .map(|mutant| { + let file = relative(&mutant.file, root); + let title = format!("Surviving mutant ({})", mutant.mutator); + let message = describe(mutant); + + format!( + "::warning file={},line={},col={},title={}::{}", + escape_property(&file), + mutant.line, + mutant.column, + escape_property(&title), + escape_data(&message) + ) + }) + .collect(); + + if survivors.len() > ANNOTATION_LIMIT { + lines.push(format!( + "::notice title=Surviving mutants::{} of {} findings annotated, which is all GitHub keeps per step; \ + the rest are in the report", + ANNOTATION_LIMIT, + survivors.len() + )); + } + + lines +} + +/// Escapes a workflow command property value. +fn escape_property(text: &str) -> String { + escape_data(text).replace(':', "%3A").replace(',', "%2C") +} + +/// Escapes a workflow command message. +/// +/// A newline inside a message would end the command and turn the remainder into log noise, so the +/// escaping is not cosmetic. +fn escape_data(text: &str) -> String { + text.replace('%', "%25").replace('\r', "%0D").replace('\n', "%0A") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Outcome; + use crate::testing::ci_fixture::{mutant, root}; + + #[test] + fn an_annotation_points_at_a_relative_path() { + let mutants = vec![mutant("/w/src/a.rs", 12, "relational.gt_to_ge", Outcome::Survived)]; + let lines = annotations(&mutants, &root()); + + assert_eq!(lines.len(), 1); + assert!(lines[0].starts_with("::warning file=src/a.rs,line=12,col=5,"), "{}", lines[0]); + } + + #[test] + fn an_annotation_says_what_the_mutation_was() { + let mutants = vec![mutant("/w/src/a.rs", 12, "relational.gt_to_ge", Outcome::Survived)]; + let lines = annotations(&mutants, &root()); + + // A reviewer standing on the line has to be told the replacement, or they go and look it up. + assert!(lines[0].contains("a >= b"), "{}", lines[0]); + } + + #[test] + fn an_uncovered_mutant_says_so() { + let mutants = vec![mutant("/w/src/a.rs", 12, "relational.gt_to_ge", Outcome::NoCoverage)]; + let lines = annotations(&mutants, &root()); + + assert!(lines[0].contains("No test reaches this code"), "{}", lines[0]); + } + + #[test] + fn too_many_annotations_are_capped_and_the_cap_is_announced() { + let mutants: Vec = (0..ANNOTATION_LIMIT + 5) + .map(|line| mutant("/w/src/a.rs", line + 1, "relational.gt_to_ge", Outcome::Survived)) + .collect(); + + let lines = annotations(&mutants, &root()); + + assert_eq!(lines.len(), ANNOTATION_LIMIT + 1); + assert!(lines.last().expect("a notice").contains("of 15 findings annotated")); + } + + #[test] + fn nothing_survived_means_nothing_to_annotate() { + let mutants = vec![mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Killed)]; + + assert!(annotations(&mutants, &root()).is_empty()); + } + + #[test] + fn a_newline_cannot_escape_a_message() { + // A raw newline would end the workflow command and turn the rest into log noise. Mutant + // text is already flattened before it gets here, so this is the belt to that suspenders. + assert_eq!(escape_data("a\r\nb"), "a%0D%0Ab"); + } + + #[test] + fn an_annotation_does_not_repeat_the_location_it_already_carries() { + let mutants = vec![mutant("/w/src/a.rs", 12, "relational.gt_to_ge", Outcome::Survived)]; + let lines = annotations(&mutants, &root()); + + assert!(!lines[0].contains("src/a.rs:12"), "{}", lines[0]); + } + + #[test] + fn a_comma_cannot_escape_a_property() { + assert_eq!(escape_property("a,b:c"), "a%2Cb%3Ac"); + } + + #[test] + fn a_path_cannot_escape_its_file_property() { + let mutants = vec![mutant("/w/src/a,b:c\r\n.rs", 12, "relational.gt_to_ge", Outcome::Survived)]; + let lines = annotations(&mutants, &root()); + + assert!( + lines[0].starts_with("::warning file=src/a%2Cb%3Ac%0D%0A.rs,line=12,col=5,"), + "{}", + lines[0] + ); + } + + #[test] + fn a_percent_is_escaped_before_anything_else() { + // Escaping it last would double-escape the escapes. + assert_eq!(escape_data("%0A\n"), "%250A%0A"); + } + + #[test] + fn auto_follows_the_runner() { + assert!(wanted(Annotations::Auto, true)); + assert!(!wanted(Annotations::Auto, false)); + assert!(wanted(Annotations::Github, false)); + assert!(!wanted(Annotations::None, true)); + } +} diff --git a/crates/cargo-gamma-lib/src/ci/finding.rs b/crates/cargo-gamma-lib/src/ci/finding.rs new file mode 100644 index 000000000..794d68a99 --- /dev/null +++ b/crates/cargo-gamma-lib/src/ci/finding.rs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! What counts as a finding, and where it points. + +use camino::Utf8Path; + +use crate::model::{Mutant, Outcome}; + +/// The undetected mutants, in report order. +/// +/// Each asks for different work, but all lower the score because no assertion killed them. +pub(super) fn findings(mutants: &[Mutant]) -> Vec<&Mutant> { + mutants + .iter() + .filter(|mutant| { + matches!( + mutant.outcome, + Outcome::Survived | Outcome::Timeout | Outcome::OutOfMemory | Outcome::NoCoverage + ) + }) + .collect() +} + +/// A path relative to the workspace root, with forward slashes. +/// +/// Every consumer here resolves against the repository checkout, so an absolute path from the +/// machine that ran the job points at nothing. Forward slashes because SARIF and the workflow +/// commands both specify them regardless of the host. +/// +/// The separator rewrite is conditional on the host, because on Unix a backslash is an ordinary +/// character in a file name: rewriting it there maps `src/a\b.rs` and `src/a/b.rs` — two distinct +/// files — onto one string, and this result is what the summary keys its rows by and what SARIF and +/// the annotations emit as a location. A finding attributed to a file that did not earn it is +/// exactly what the escaping downstream of here was written to prevent, and no escaping can undo a +/// collision introduced on the key. Nothing is lost by the condition: [`Mutant::file`] already +/// carries forward slashes, so on Unix the rewrite never had anything to do. +pub(super) fn relative(path: &Utf8Path, root: &Utf8Path) -> String { + let path = path.strip_prefix(root).unwrap_or(path).as_str(); + + #[cfg(windows)] + let relative = path.replace('\\', "/"); + + #[cfg(not(windows))] + let relative = path.to_owned(); + + relative +} + +/// What a survivor is, in one sentence. +/// +/// The location is carried in fields of its own by both consumers, so repeating it here would only +/// take space from the part a reader cannot get anywhere else. +pub(super) fn describe(mutant: &Mutant) -> String { + match mutant.outcome { + Outcome::NoCoverage => format!("No test reaches this code: {}.", mutant.summary()), + Outcome::Timeout => format!("{} and the test run timed out before an assertion rejected it.", mutant.summary()), + Outcome::OutOfMemory => { + format!( + "{} and the test run exceeded its memory limit before an assertion rejected it.", + mutant.summary() + ) + } + _other => format!("{} and no test failed.", mutant.summary()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::ci_fixture::{mutant, root}; + + #[test] + fn every_undetected_outcome_is_a_finding() { + let mutants = vec![ + mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Killed), + mutant("/w/src/a.rs", 2, "relational.gt_to_ge", Outcome::Survived), + mutant("/w/src/a.rs", 3, "relational.gt_to_ge", Outcome::Timeout), + mutant("/w/src/a.rs", 4, "relational.gt_to_ge", Outcome::NoCoverage), + mutant("/w/src/a.rs", 5, "relational.gt_to_ge", Outcome::CompileError), + mutant("/w/src/a.rs", 6, "relational.gt_to_ge", Outcome::OutOfMemory), + ]; + + let found = findings(&mutants); + + assert_eq!(found.len(), 4); + assert_eq!(found[0].line, 2); + assert_eq!(found[1].line, 3); + assert_eq!(found[2].line, 4); + assert_eq!(found[3].line, 6); + } + + #[test] + fn a_path_outside_the_root_is_left_alone() { + // Better an absolute path a consumer cannot resolve than a relative one pointing at the + // wrong file inside the checkout. + assert_eq!(relative(Utf8Path::new("/elsewhere/a.rs"), &root()), "/elsewhere/a.rs"); + } + + /// Two files whose names differ only by a backslash stay two files. + /// + /// On Unix a backslash is an ordinary character in a file name, so rewriting it into a slash + /// merges `src/a\b.rs` into `src/a/b.rs` — and the result is what the summary keys its rows by + /// and what SARIF emits as a location, so one file's survivors are printed under the other's + /// name and an alert lands on whichever of the two exists. The escaping downstream is + /// one-to-one and cannot see this, because the collision happens on the key. + #[cfg(not(windows))] + #[test] + fn two_paths_differing_only_by_a_backslash_do_not_collide() { + use crate::ci::{Level, sarif}; + + let mutants = vec![ + mutant(r"/w/src/a\b.rs", 1, "relational.gt_to_ge", Outcome::Survived), + mutant("/w/src/a/b.rs", 1, "relational.gt_to_ge", Outcome::Survived), + ]; + + assert_ne!( + relative(&mutants[0].file, &root()), + relative(&mutants[1].file, &root()), + "two files must not share one name" + ); + + let (log, _truncation) = sarif(&mutants, &root(), Level::Note).expect("a sarif log"); + let document: serde_json::Value = serde_json::from_str(&log).expect("valid json"); + let uris: Vec = document["runs"][0]["results"] + .as_array() + .expect("results") + .iter() + .map(|result| result["locations"][0]["physicalLocation"]["artifactLocation"]["uri"].to_string()) + .collect(); + + assert_eq!(uris.len(), 2, "{document}"); + assert_ne!(uris[0], uris[1], "{document}"); + + // The job summary keys its rows on the same string, so a collision there sums one file's + // survivors into the other's row rather than merely renaming it. + let table = crate::ci::summary(&mutants, &root()); + let rows = table.lines().filter(|line| line.contains("b.rs")).count(); + + assert_eq!(rows, 2, "{table}"); + } +} diff --git a/crates/cargo-gamma-lib/src/ci/level.rs b/crates/cargo-gamma-lib/src/ci/level.rs new file mode 100644 index 000000000..5cbc9bba6 --- /dev/null +++ b/crates/cargo-gamma-lib/src/ci/level.rs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! How loudly a survivor is reported to a SARIF consumer. + +use clap::ValueEnum; + +/// How loudly a survivor is reported to a SARIF consumer. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum Level { + /// An observation about the test suite. The default. + #[default] + Note, + + /// A problem the team wants raised. + Warning, +} + +impl Level { + /// The SARIF spelling. + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Note => "note", + Self::Warning => "warning", + } + } +} diff --git a/crates/cargo-gamma-lib/src/ci/mod.rs b/crates/cargo-gamma-lib/src/ci/mod.rs new file mode 100644 index 000000000..e8d407ff2 --- /dev/null +++ b/crates/cargo-gamma-lib/src/ci/mod.rs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Surfacing a run inside a continuous integration system. +//! +//! A mutation report that lives in an artifact zip is a report nobody reads. The findings have to +//! arrive where the reviewer already is — on the diff, in the job summary, in the security tab — +//! or the tool gets adopted, run nightly, and ignored. +//! +//! Three renderings share one rule: **every undetected mutant is a finding.** A mutant killed by a +//! failing assertion is the tool working; survivors, uncovered sites, timeouts, and memory +//! exhaustion all need attention. + +mod annotations; +mod finding; +mod level; +pub(crate) mod sarif; +mod summary; +mod truncation; + +pub use annotations::{Annotations, annotations, wanted}; +pub use level::Level; +pub use sarif::sarif; +pub(crate) use summary::append; +pub use summary::summary; +pub use truncation::Truncation; diff --git a/crates/cargo-gamma-lib/src/ci/sarif.rs b/crates/cargo-gamma-lib/src/ci/sarif.rs new file mode 100644 index 000000000..d862475c9 --- /dev/null +++ b/crates/cargo-gamma-lib/src/ci/sarif.rs @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The twelve pieces of a SARIF 2.1.0 log document, all serialized together as one wire format. + +use camino::Utf8Path; +use serde::Serialize; + +use super::finding::{describe, findings, relative}; +use super::level::Level; +use super::truncation::Truncation; +use crate::model::Mutant; +use crate::{HashMap, HashSet, Result}; + +/// A SARIF 2.1.0 log. +#[derive(Debug, Serialize)] +pub(super) struct Log { + pub(super) version: &'static str, + #[serde(rename = "$schema")] + pub(super) schema: &'static str, + pub(super) runs: Vec, +} + +#[derive(Debug, Serialize)] +pub(super) struct Run { + pub(super) tool: Tool, + pub(super) results: Vec, +} + +#[derive(Debug, Serialize)] +pub(super) struct Tool { + pub(super) driver: Driver, +} + +#[derive(Debug, Serialize)] +pub(super) struct Driver { + pub(super) name: &'static str, + #[serde(rename = "informationUri")] + pub(super) information_uri: &'static str, + pub(super) version: &'static str, + pub(super) rules: Vec, +} + +#[derive(Debug, Serialize)] +pub(super) struct Rule { + pub(super) id: String, + pub(super) name: String, + #[serde(rename = "shortDescription")] + pub(super) short_description: Text, + #[serde(rename = "fullDescription")] + pub(super) full_description: Text, + #[serde(rename = "defaultConfiguration")] + pub(super) default_configuration: Configuration, +} + +#[derive(Debug, Serialize)] +pub(super) struct Configuration { + pub(super) level: &'static str, +} + +#[derive(Debug, Serialize)] +pub(super) struct Text { + pub(super) text: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct Finding { + #[serde(rename = "ruleId")] + pub(super) rule_id: String, + pub(super) level: &'static str, + pub(super) message: Text, + pub(super) locations: Vec, + #[serde(rename = "partialFingerprints")] + pub(super) partial_fingerprints: HashMap, +} + +#[derive(Debug, Serialize)] +pub(super) struct Location { + #[serde(rename = "physicalLocation")] + pub(super) physical_location: Physical, +} + +#[derive(Debug, Serialize)] +pub(super) struct Physical { + #[serde(rename = "artifactLocation")] + pub(super) artifact_location: Artifact, + pub(super) region: Region, +} + +#[derive(Debug, Serialize)] +pub(super) struct Artifact { + pub(super) uri: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct Region { + #[serde(rename = "startLine")] + pub(super) start_line: usize, + #[serde(rename = "startColumn")] + pub(super) start_column: usize, +} + +/// GitHub rejects a SARIF upload with more results than this. +/// +/// It is a hard limit on their side, not a preference on ours: exceeding it fails the upload +/// outright, which is a worse outcome than a report that says what it left out. +pub(crate) const SARIF_LIMIT: usize = 5_000; + +/// GitHub rejects a SARIF upload larger than this, whatever it contains. +/// +/// The result count is not a reliable proxy for the size, because a finding carries a message, a +/// path and a fingerprint whose lengths are the code's business rather than ours. A log under the +/// count limit and over the byte limit is rejected just as completely, so both are enforced. +const SARIF_BYTES: usize = 10 * 1024 * 1024; + +/// Renders survivors as a SARIF 2.1.0 log, and says what it had to leave out. +/// +/// Rule identifiers are our stable mutator names, which is what makes GitHub's alert grouping and +/// dismissal work per operator: a team can permanently dismiss every `literal.int_zero` alert +/// without touching anything else, and that decision keeps applying to code written next year. +pub fn sarif(mutants: &[Mutant], root: &Utf8Path, level: Level) -> Result<(String, Option)> { + let survivors = findings(mutants); + let found = survivors.len(); + let mut kept: Vec<&Mutant> = survivors.into_iter().take(SARIF_LIMIT).collect(); + + // Shrunk until it fits rather than estimated, because the size of a finding is decided by the + // length of a path, a message and an identifier, none of which this can predict. Halving + // converges in a handful of serializations even from the count limit, and the alternative to + // any of it is an upload GitHub refuses whole. + loop { + let text = render(&kept, root, level)?; + + if text.len() <= SARIF_BYTES || kept.is_empty() { + let truncation = (found > kept.len()).then_some(Truncation { + found, + written: kept.len(), + }); + + return Ok((text, truncation)); + } + + kept.truncate(kept.len() / 2); + } +} + +/// Serializes one SARIF log over exactly the findings it is given. +fn render(kept: &[&Mutant], root: &Utf8Path, level: Level) -> Result { + let mut seen = HashSet::default(); + let mut rules = Vec::new(); + + for mutant in kept { + if !seen.insert(mutant.mutator.to_string()) { + continue; + } + + rules.push(Rule { + id: mutant.mutator.to_string(), + name: mutant.mutator.to_string(), + short_description: Text { + text: format!("Surviving mutant: {}", mutant.mutator), + }, + full_description: Text { + text: format!( + "The {} mutation was applied and the test suite still passed, so nothing asserts on the \ + behavior it changed.", + mutant.mutator + ), + }, + default_configuration: Configuration { level: level.as_str() }, + }); + } + + rules.sort_by(|left, right| left.id.cmp(&right.id)); + + let results = kept + .iter() + .map(|mutant| { + let mut fingerprints = HashMap::default(); + + // The mutant id is content-addressed, so an alert follows its code through reformatting + // and through edits elsewhere in the file instead of being dismissed and resurrected. + let _previous = fingerprints.insert(format!("gammaMutantId/v{}", crate::model::MUTANT_ID_VERSION), mutant.id.to_string()); + + Finding { + rule_id: mutant.mutator.to_string(), + level: level.as_str(), + message: Text { text: describe(mutant) }, + locations: vec![Location { + physical_location: Physical { + artifact_location: Artifact { + uri: relative(&mutant.file, root), + }, + region: Region { + start_line: mutant.line, + start_column: mutant.column, + }, + }, + }], + partial_fingerprints: fingerprints, + } + }) + .collect(); + + let log = Log { + version: "2.1.0", + schema: "https://json.schemastore.org/sarif-2.1.0.json", + runs: vec![Run { + tool: Tool { + driver: Driver { + name: "cargo-gamma", + information_uri: "https://github.com/microsoft/ox-tools/tree/main/crates/cargo-gamma", + version: env!("CARGO_PKG_VERSION"), + rules, + }, + }, + results, + }], + }; + + serde_json::to_string_pretty(&log).map_err(|cause| crate::error::error!("could not serialize the SARIF log").caused_by(cause)) +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + + use super::*; + use crate::model::Outcome; + use crate::testing::ci_fixture::{mutant, root}; + + #[test] + fn sarif_carries_one_rule_per_mutator() { + let mutants = vec![ + mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Survived), + mutant("/w/src/a.rs", 2, "relational.gt_to_ge", Outcome::Survived), + mutant("/w/src/a.rs", 3, "literal.int_zero", Outcome::Survived), + ]; + + let (text, truncation) = sarif(&mutants, &root(), Level::Note).expect("sarif"); + let log: Value = serde_json::from_str(&text).expect("valid json"); + + assert_eq!(truncation, None); + + let rules = log["runs"][0]["tool"]["driver"]["rules"].as_array().expect("rules"); + + assert_eq!(rules.len(), 2); + assert_eq!(rules[0]["id"], "literal.int_zero"); + assert_eq!(log["runs"][0]["results"].as_array().expect("results").len(), 3); + } + + #[test] + fn a_sarif_result_is_fingerprinted_by_mutant_id() { + let mutants = vec![mutant("/w/src/a.rs", 7, "relational.gt_to_ge", Outcome::Survived)]; + let (text, _) = sarif(&mutants, &root(), Level::Warning).expect("sarif"); + let log: Value = serde_json::from_str(&text).expect("valid json"); + let result = &log["runs"][0]["results"][0]; + + assert_eq!(result["level"], "warning"); + assert_eq!( + result["partialFingerprints"]["gammaMutantId/v4"], + "/w/src/a.rs:7:relational.gt_to_ge" + ); + + let region = &result["locations"][0]["physicalLocation"]; + + assert_eq!(region["artifactLocation"]["uri"], "src/a.rs"); + assert_eq!(region["region"]["startLine"], 7); + } + + /// The count cap only matters at its boundary, and the boundary is the one place nothing + /// exercised it: the shrink-to-fit test stops at exactly `SARIF_LIMIT`, so neither the + /// constant nor the `take` that enforces it was ever asked to drop a result. Exceeding + /// GitHub's cap fails the upload whole, losing every finding rather than the excess. + #[test] + fn one_finding_past_the_cap_is_dropped_and_reported() { + let mutants: Vec = (0..=SARIF_LIMIT) + .map(|line| mutant("/w/src/a.rs", line, "relational.gt_to_ge", Outcome::Survived)) + .collect(); + + assert_eq!(mutants.len(), SARIF_LIMIT + 1); + + let (text, truncation) = sarif(&mutants, &root(), Level::Warning).expect("sarif"); + let log: Value = serde_json::from_str(&text).expect("valid json"); + let truncation = truncation.expect("one finding past the cap must be reported as truncated"); + + // The rendered log is what GitHub sees, so the cap is asserted there and not only on the + // bookkeeping that describes it. + assert_eq!(log["runs"][0]["results"].as_array().expect("results").len(), SARIF_LIMIT); + assert_eq!(truncation.found, SARIF_LIMIT + 1); + assert_eq!(truncation.written, SARIF_LIMIT); + assert_eq!(truncation.found - truncation.written, 1); + } + + #[test] + #[cfg(not(miri))] + fn a_log_too_large_to_upload_is_shrunk_until_it_fits() { + // The count limit is not a size limit: a finding's size is decided by the length of a path + // and a message, and a log GitHub refuses is worth nothing however many results it holds. + let deep = format!("/w/src/{}/a.rs", "nested/".repeat(500)); + let mutants: Vec = (0..SARIF_LIMIT) + .map(|line| mutant(&deep, line, "relational.gt_to_ge", Outcome::Survived)) + .collect(); + + let (text, truncation) = sarif(&mutants, &root(), Level::Warning).expect("sarif"); + let truncation = truncation.expect("a log this large cannot have been written whole"); + + assert!(text.len() <= SARIF_BYTES, "{} bytes", text.len()); + assert_eq!(truncation.found, SARIF_LIMIT); + assert!(truncation.written < SARIF_LIMIT, "{}", truncation.written); + } +} diff --git a/crates/cargo-gamma-lib/src/ci/summary.rs b/crates/cargo-gamma-lib/src/ci/summary.rs new file mode 100644 index 000000000..7f3a5fd43 --- /dev/null +++ b/crates/cargo-gamma-lib/src/ci/summary.rs @@ -0,0 +1,629 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The Markdown written to `$GITHUB_STEP_SUMMARY`. + +use core::fmt::Write as _; +use std::fs::{File, OpenOptions}; +use std::io; + +use camino::Utf8Path; + +use super::finding::{findings, relative}; +use crate::HashMap; +use crate::model::{Mutant, Outcome, Summary}; + +/// How many under-tested files the job summary lists. +const SUMMARY_FILES: usize = 10; + +/// Appends one complete panel to a GitHub step summary. +/// +/// `O_APPEND` only makes each individual write append-safe. `write_all` can retry after a short +/// write, so the file lock spans the whole panel and prevents another cargo-gamma process from +/// placing its panel between those retries. +pub(crate) fn append(path: &Utf8Path, panel: &str) -> io::Result<()> { + let lock = OpenOptions::new().append(true).create(true).read(true).open(path)?; + let mut writer = lock.try_clone()?; + + append_locked(&lock, &mut writer, panel) +} + +fn append_locked(lock: &File, writer: &mut impl io::Write, panel: &str) -> io::Result<()> { + lock.lock()?; + + match writer.write_all(panel.as_bytes()) { + Ok(()) => lock.unlock(), + Err(cause) => { + let _ = lock.unlock(); + + Err(cause) + } + } +} + +/// Renders the Markdown written to `$GITHUB_STEP_SUMMARY`. +/// +/// This is the artifact a team actually reads every morning, so it leads with the number that +/// decides whether anyone reads further, and then spends its space on where the gaps are rather +/// than on restating the run's configuration. +#[must_use] +pub fn summary(mutants: &[Mutant], root: &Utf8Path) -> String { + let totals = Summary::of(mutants); + let mut text = String::from("## Mutation testing\n\n"); + + // The headline is the score's own arithmetic, not a second opinion about it: the mutants + // counted as detected are exactly `Summary::detected`: mutants rejected by a failing + // assertion. The total is `Summary::valid`, its denominator. Timeouts and memory exhaustion + // remain in that denominator because no assertion rejected them. + // + // The complement is labelled for what it counts, which is not survivors: a mutant no test + // reaches is undetected too, and the table below lists the two separately because they ask the + // reader to do different things — "no test links this code" and "the tests that run it did not + // notice" are different problems. Calling the sum "survived" put a figure above a table that + // contradicted it, with nothing on the page to explain the difference. + let detected = totals.detected(); + let valid = totals.valid(); + + let _ = writeln!( + text, + "**Score {:.1}%** — {detected} detected, {} not detected of {valid} mutants.\n", + totals.score(), + valid - detected + ); + + text.push_str("| Outcome | Count |\n|---|---:|\n"); + + // Every outcome has a row, enumerated from `Outcome::ALL` rather than listed here, so the table + // sums to the population. The ones that score nothing are here because they explain a + // population smaller than the run: a mutant that would not compile, one that was suppressed, + // one the build never produced and one that never ran are all absent from the number above, and + // a reader who cannot see them reads the difference as mutants going missing — which is exactly + // what a hand-written list did to the three outcomes it had fallen behind the enum on. + for outcome in Outcome::ALL { + let count = totals.count(outcome); + + if count > 0 { + let _ = writeln!(text, "| {} | {count} |", label(outcome)); + } + } + + let hot = under_tested(mutants, root); + + if !hot.is_empty() { + text.push_str("\n### Where the gaps are\n\n| File | Not detected |\n|---|---:|\n"); + + for (file, count) in hot { + let _ = writeln!(text, "| {} | {count} |", cell(&file)); + } + } + + text +} + +/// How one outcome is named in the summary table. +/// +/// A `match`, so a new outcome cannot reach a report without a name. +const fn label(outcome: Outcome) -> &'static str { + match outcome { + Outcome::Killed => "Killed", + Outcome::Timeout => "Timed out", + Outcome::OutOfMemory => "Out of memory", + Outcome::Survived => "Survived", + Outcome::NoCoverage => "Uncovered", + Outcome::Flaky => "Flaky", + Outcome::CompileError => "Unviable", + Outcome::Ignored => "Suppressed", + Outcome::NotBuilt => "Not built", + Outcome::Pending => "Not run", + } +} + +/// Renders one path as a single Markdown table cell. +/// +/// A file name is the one thing in this document that neither this tool nor the reader chooses, and +/// on Unix it may hold anything but a null byte and a slash — a pipe, which ends a table cell; a +/// backtick, which ends a code span; a newline, which ends the row and moves the count onto a line +/// of its own, where the table silently attributes it to some other file. A count printed beside +/// the wrong path is worse than an ugly one, so the path is escaped rather than trusted. +/// +/// The escape is one-to-one, which is what "unambiguous" means here: two different paths cannot +/// render as the same cell, so a reader is never shown one file's name over another's count. The +/// backslash goes first for exactly that reason, since it is what every other escape is built out +/// of, and the delimiter that closes the code span is chosen longer than any run of backticks +/// inside it — the rule Markdown itself provides for this, and the only one that works for every +/// input. +fn cell(path: &str) -> String { + let escaped = escape(path); + let fence = "`".repeat(longest_run(&escaped) + 1); + + // A code span drops one space from each end when it has one at both, and cannot be empty at + // all, so anything that starts or ends with a space or a backtick is given a pair of its own to + // lose. Without it `` ` a ` `` and `` `a` `` are the same cell for two different files. + let padding = if escaped.is_empty() || escaped.starts_with(['`', ' ']) || escaped.ends_with(['`', ' ']) { + " " + } else { + "" + }; + + format!("{fence}{padding}{escaped}{padding}{fence}") +} + +/// Rewrites the characters that cannot survive a table row, so that no two paths collide. +/// +/// A pipe is escaped the way GitHub's tables specify, which is understood before the row is split +/// into cells and so works inside a code span as well as outside one. The rest are control +/// characters, which have no Markdown spelling at all: they are written as the escapes a Rust or C +/// programmer already reads, and the backslash that introduces them is escaped first so that the +/// mapping can be undone — a path holding the two characters `\` and `n` has to stay distinct from +/// one holding a newline. +fn escape(path: &str) -> String { + let mut out = String::with_capacity(path.len()); + + for character in path.chars() { + match character { + '\\' => out.push_str(r"\\"), + '|' => out.push_str(r"\|"), + '\n' => out.push_str(r"\n"), + '\r' => out.push_str(r"\r"), + '\t' => out.push_str(r"\t"), + other if other.is_control() => { + let _ = write!(out, r"\u{{{:04x}}}", u32::from(other)); + } + other => out.push(other), + } + } + + out +} + +/// The longest run of backticks in a string, which is what a code span around it has to beat. +fn longest_run(text: &str) -> usize { + let mut longest = 0; + let mut run = 0; + + for character in text.chars() { + if character == '`' { + run += 1; + longest = longest.max(run); + } else { + run = 0; + } + } + + longest +} + +/// The files with the most survivors, worst first. +fn under_tested(mutants: &[Mutant], root: &Utf8Path) -> Vec<(String, usize)> { + let mut counts: HashMap = HashMap::default(); + + for mutant in findings(mutants) { + *counts.entry(relative(&mutant.file, root)).or_default() += 1; + } + + let mut ranked: Vec<(String, usize)> = counts.into_iter().collect(); + + // Ties broken by path so a summary is reproducible run to run; an unstable order turns every + // morning's table into a diff nobody can read. + ranked.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0))); + ranked.truncate(SUMMARY_FILES); + ranked +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use core::time::Duration; + use std::process::Command; + use std::{env, fs, thread}; + + use camino::Utf8PathBuf; + + use super::*; + use crate::testing::ci_fixture::{mutant, root}; + + const SHORT_WRITER_PATH: &str = "CARGO_GAMMA_SHORT_WRITER_PATH"; + const SHORT_WRITER_GATE: &str = "CARGO_GAMMA_SHORT_WRITER_GATE"; + const SHORT_WRITER_PANEL: &str = "CARGO_GAMMA_SHORT_WRITER_PANEL"; + + struct ShortWriter { + file: File, + } + + impl io::Write for ShortWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.file.write(&buf[..buf.len().min(1)])?; + + thread::sleep(Duration::from_millis(1)); + + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush() + } + } + + #[test] + fn concurrent_short_writes_keep_each_summary_panel_contiguous() { + if let (Some(path), Some(gate), Some(panel)) = ( + env::var_os(SHORT_WRITER_PATH), + env::var_os(SHORT_WRITER_GATE), + env::var_os(SHORT_WRITER_PANEL), + ) { + let path = Utf8PathBuf::from(path.into_string().expect("the test path is UTF-8")); + let gate = Utf8PathBuf::from(gate.into_string().expect("the test gate is UTF-8")); + let panel = panel.into_string().expect("the test panel is UTF-8"); + + while !gate.exists() { + thread::sleep(Duration::from_millis(1)); + } + + let lock = OpenOptions::new() + .append(true) + .create(true) + .read(true) + .open(&path) + .expect("open step summary"); + let writer = lock.try_clone().expect("clone summary handle"); + let mut writer = ShortWriter { file: writer }; + + append_locked(&lock, &mut writer, &panel).expect("append one panel"); + + return; + } + + let directory = crate::testing::workdir("ci-summary-lock-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the test directory is UTF-8"); + let path = root.join("summary.md"); + let gate = root.join("go"); + let first = format!("\n{}\n\n", "a".repeat(64)); + let second = format!("\n{}\n\n", "b".repeat(64)); + let test = "ci::summary::tests::concurrent_short_writes_keep_each_summary_panel_contiguous"; + let executable = env::current_exe().expect("the test executable is known"); + + let mut children = Vec::new(); + + for panel in [&first, &second] { + children.push( + Command::new(&executable) + .args(["--exact", test, "--nocapture"]) + .env(SHORT_WRITER_PATH, path.as_str()) + .env(SHORT_WRITER_GATE, gate.as_str()) + .env(SHORT_WRITER_PANEL, panel) + .spawn() + .expect("start concurrent writer"), + ); + } + + thread::sleep(Duration::from_millis(20)); + fs::write(&gate, "").expect("release writers"); + + for mut child in children { + assert!(child.wait().expect("wait for writer").success(), "the writer process failed"); + } + + let written = fs::read_to_string(&path).expect("read step summary"); + + assert!( + written == format!("{first}{second}") || written == format!("{second}{first}"), + "short writes interleaved panels: {written:?}" + ); + } + + /// How many cells a Markdown reader finds in a table row. + /// + /// Cells are split on pipes before anything else is parsed, so a code span does not protect one + /// and only a backslash does — and a backslash escapes whatever follows it, including another + /// backslash, which is why an escape that left those alone could still hide an active pipe. + fn cells(row: &str) -> usize { + let mut count = 1; + let mut characters = row.chars(); + + while let Some(character) = characters.next() { + match character { + '\\' => { + let _ = characters.next(); + } + '|' => count += 1, + _other => {} + } + } + + count + } + + /// The rows of the file table, which is the only place a path is rendered. + /// + /// Taken from below the table's separator so that the outcome table above, whose rows are the + /// same shape, cannot stand in for the row under test. + fn file_rows(text: &str) -> Vec { + let (_, gaps) = text.split_once("### Where the gaps are").expect("a file table"); + + gaps.lines() + .skip_while(|line| !line.starts_with("|---")) + .skip(1) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect() + } + + #[test] + fn the_summary_leads_with_the_score() { + let mutants = vec![ + mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Killed), + mutant("/w/src/a.rs", 2, "relational.gt_to_ge", Outcome::Survived), + ]; + + let text = summary(&mutants, &root()); + + assert!(text.contains("**Score 50.0%**"), "{text}"); + assert!(text.contains("| Killed | 1 |"), "{text}"); + assert!(text.contains("| Survived | 1 |"), "{text}"); + } + + #[test] + fn an_empty_outcome_is_left_out_of_the_summary_table() { + let mutants = vec![mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Killed)]; + let text = summary(&mutants, &root()); + + assert!(!text.contains("Unviable"), "{text}"); + assert!(!text.contains("Where the gaps are"), "{text}"); + } + + /// Resource exhaustion remains in the denominator and out of the numerator. + /// + /// The population is deliberately lopsided — every scoring outcome a different size — so that + /// dropping any one from the denominator changes a number this test names. Detected is the 3 + /// assertion kills; valid also includes 2 timeouts, 4 out-of-memory verdicts, 1 survivor and 5 + /// uncovered mutants, for a 20.0% score. + #[test] + fn the_headline_and_the_table_account_for_a_memory_kill() { + let mut mutants = Vec::new(); + + for (outcome, many) in [ + (Outcome::Killed, 3), + (Outcome::Timeout, 2), + (Outcome::OutOfMemory, 4), + (Outcome::Survived, 1), + (Outcome::NoCoverage, 5), + (Outcome::CompileError, 6), + (Outcome::Ignored, 7), + ] { + for line in 0..many { + mutants.push(mutant("/w/src/a.rs", line + 1, "relational.gt_to_ge", outcome)); + } + } + + let totals = Summary::of(&mutants); + let text = summary(&mutants, &root()); + + // The headline is the score's own arithmetic, so it reconciles with the score printed in + // front of it and with what the console reports for the same run. Its second figure is the + // score's complement, which includes resource exhaustion, the survivor and uncovered + // mutants, so it is labelled for that union rather than any one outcome. + assert!( + text.contains("**Score 20.0%** — 3 detected, 12 not detected of 15 mutants."), + "{text}" + ); + assert_eq!(totals.detected(), 3); + assert_eq!(totals.valid(), 15); + assert_eq!(totals.survived, 1, "the headline's 6 must not be readable as survivors"); + + assert!(text.contains("| Killed | 3 |"), "{text}"); + assert!(text.contains("| Timed out | 2 |"), "{text}"); + assert!(text.contains("| Out of memory | 4 |"), "{text}"); + assert!(text.contains("| Survived | 1 |"), "{text}"); + assert!(text.contains("| Uncovered | 5 |"), "{text}"); + + // Every row that scores adds up to the headline, and so to the score above it. + let scoring = [("Killed", 3)]; + let detected: u32 = scoring + .iter() + .filter(|(label, _)| text.contains(&format!("| {label} | "))) + .map(|(_, count)| count) + .sum(); + + assert_eq!(detected, totals.detected(), "the killed rows must sum to the headline: {text}"); + } + + /// A run with nothing but memory exhaustion scores zero because no assertion killed it. + #[test] + fn a_memory_kill_is_the_whole_breakdown_when_it_is_the_whole_run() { + let mutants = vec![mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::OutOfMemory)]; + let text = summary(&mutants, &root()); + + assert!(text.contains("**Score 0.0%** — 0 detected, 1 not detected of 1 mutants."), "{text}"); + assert!(text.contains("| Out of memory | 1 |"), "{text}"); + } + + /// The headline's complement counts uncovered mutants as well as survivors, so nothing on the + /// page may call it survivors. + /// + /// The two are separate rows in the table directly beneath, and separate on purpose: "no test + /// links this code" and "the tests that run it did not notice" send a reader to do different + /// things. A figure labelled `survived` that disagrees with the `Survived` row two lines below + /// leaves the reader with no way to tell which number is wrong. + #[test] + fn no_figure_in_the_summary_labels_uncovered_mutants_as_survivors() { + let mut mutants = vec![mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Survived)]; + + for line in 0..5 { + mutants.push(mutant("/w/src/b.rs", line + 2, "relational.gt_to_ge", Outcome::NoCoverage)); + } + + let totals = Summary::of(&mutants); + let text = summary(&mutants, &root()); + + assert_eq!(totals.survived, 1); + assert_eq!(totals.valid() - totals.detected(), 6); + + // Every count the document states against the word "survived", wherever it appears, has to + // be the number of survivors. + let figures: Vec<&str> = text.split(" survived").collect(); + + for figure in figures.iter().take(figures.len().saturating_sub(1)) { + let count: u32 = figure + .rsplit(|character: char| !character.is_ascii_digit()) + .next() + .expect("a split always yields one part") + .parse() + .expect("every figure printed beside `survived` is a count"); + + assert_eq!(count, totals.survived, "`{count} survived` is not the survivor count: {text}"); + } + + assert!(text.contains("6 not detected"), "{text}"); + assert!(text.contains("| Survived | 1 |"), "{text}"); + assert!(text.contains("| Uncovered | 5 |"), "{text}"); + } + + /// Every mutant in the run is in the table, whatever became of it. + /// + /// The table carries no score, so an omission does not move a number — it makes the rows stop + /// adding up to the run, and a reader cannot tell a category nobody listed from mutants that + /// went missing. One of each outcome, with distinct multiplicities so a row reading the wrong + /// counter fails as loudly as a row that is absent. + #[test] + fn the_summary_table_accounts_for_every_mutant_in_the_run() { + let mut mutants = Vec::new(); + + for (index, outcome) in Outcome::ALL.into_iter().enumerate() { + for line in 0..=index { + mutants.push(mutant("/w/src/a.rs", line + 1, "relational.gt_to_ge", outcome)); + } + } + + let totals = Summary::of(&mutants); + let text = summary(&mutants, &root()); + + let counted: usize = text + .lines() + .skip_while(|line| !line.starts_with("| Outcome |")) + .skip(2) + .take_while(|line| line.starts_with('|')) + .map(|line| { + line.trim_matches('|') + .split('|') + .nth(1) + .expect("an outcome row states a count") + .trim() + .parse::() + .expect("an outcome row's count is a number") + }) + .sum(); + + assert_eq!(counted, mutants.len(), "the table must account for every mutant: {text}"); + + for outcome in Outcome::ALL { + let count = totals.count(outcome); + + assert!( + text.contains(&format!("| {} | {count} |", label(outcome))), + "{outcome} is missing from the table: {text}" + ); + } + } + + #[test] + fn the_summary_ranks_files_by_survivors() { + let mut mutants = vec![mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Survived)]; + + for line in 0..3 { + mutants.push(mutant("/w/src/b.rs", line + 1, "relational.gt_to_ge", Outcome::Survived)); + } + + let ranked = under_tested(&mutants, &root()); + + assert_eq!(ranked, vec![("src/b.rs".to_owned(), 3), ("src/a.rs".to_owned(), 1)]); + } + + #[test] + fn files_with_the_same_count_are_ordered_by_path() { + let mutants = vec![ + mutant("/w/src/z.rs", 1, "relational.gt_to_ge", Outcome::Survived), + mutant("/w/src/a.rs", 1, "relational.gt_to_ge", Outcome::Survived), + ]; + + let ranked = under_tested(&mutants, &root()); + + assert_eq!(ranked[0].0, "src/a.rs"); + } + + /// An ordinary path is rendered as it always was: a plain code span, with nothing escaped. + #[test] + fn an_ordinary_path_is_left_alone() { + assert_eq!(cell("src/lib.rs"), "`src/lib.rs`"); + } + + /// Each of these characters is legal in a Unix file name and each ends something in Markdown. + /// The row stays two cells wide and one line long whichever one turns up. + #[test] + fn a_path_that_ends_a_cell_a_span_or_a_line_stays_in_one_cell() { + for (what, name) in [ + ("a pipe", "a|b.rs"), + ("a backtick", "a`b.rs"), + ("a run of backticks", "a```b.rs"), + ("a leading backtick", "`ab.rs"), + ("a trailing backtick", "ab.rs`"), + ("a newline", "a\nb.rs"), + ("a carriage return", "a\rb.rs"), + ("a tab", "a\tb.rs"), + ("a backslash", "a\\b.rs"), + ("a bell", "a\u{7}b.rs"), + ("everything at once", "a|`\n`|b.rs"), + ] { + let mutants = vec![mutant(&format!("/w/src/{name}"), 1, "relational.gt_to_ge", Outcome::Survived)]; + let text = summary(&mutants, &root()); + let rows = file_rows(&text); + + assert_eq!(rows.len(), 1, "{what} must leave the file on one row: {text}"); + assert_eq!(cells(&rows[0]), 4, "{what} must leave one row of two cells: {}", rows[0]); + assert!(rows[0].ends_with("| 1 |"), "{what} must keep the count in the row: {}", rows[0]); + } + } + + /// Two files whose names differ have to render as cells that differ, or the survivor count in + /// the next column is attributed to a file that did not earn it. + #[test] + fn no_two_paths_render_as_the_same_cell() { + let names = [ + "a|b.rs", + "a\\|b.rs", + "a\\b.rs", + "ab.rs", + "a\nb.rs", + "a\\nb.rs", + "a`b.rs", + "a``b.rs", + " a.rs", + "a.rs ", + "a.rs", + "`a.rs`", + "a\u{7}b.rs", + "a\\u{0007}b.rs", + "", + ]; + + for (index, left) in names.iter().enumerate() { + for right in &names[index + 1..] { + assert_ne!(cell(left), cell(right), "{left:?} and {right:?} render alike"); + } + } + } + + /// The delimiter has to be longer than any run of backticks inside the cell, and the content + /// must not begin or end with one, or the span closes early and the rest of the path leaks into + /// the table as Markdown. + #[test] + fn a_code_span_is_fenced_longer_than_the_backticks_it_holds() { + assert_eq!(cell("a`b"), "``a`b``"); + assert_eq!(cell("a``b"), "```a``b```"); + assert_eq!(cell("`a`"), "`` `a` ``"); + + // A code span cannot be empty, so the empty path is a pair of spaces rather than nothing: + // "consists entirely of spaces" is the one case the code-span rule leaves alone, and a cell + // that renders as nothing at all would read as a table this tool failed to fill in. + assert_eq!(cell(""), "` `"); + } +} diff --git a/crates/cargo-gamma-lib/src/ci/truncation.rs b/crates/cargo-gamma-lib/src/ci/truncation.rs new file mode 100644 index 000000000..33be1f289 --- /dev/null +++ b/crates/cargo-gamma-lib/src/ci/truncation.rs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! What a SARIF rendering left out, if anything. + +/// What a SARIF rendering left out, if anything. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Truncation { + /// How many findings there were. + /// + /// Uncovered mutants are among them, not only survivors. + pub found: usize, + + /// How many the log contains. + pub written: usize, +} diff --git a/crates/cargo-gamma-lib/src/commands/clean.rs b/crates/cargo-gamma-lib/src/commands/clean.rs new file mode 100644 index 000000000..ec84ed2b2 --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/clean.rs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::io::Write; + +use super::cli::{CleanArgs, FeatureArgs}; +use super::dispatch::EXIT_OK; +use super::host::Host; +use crate::discover::load_metadata; +use crate::exec::{clean_cache, gamma_base}; +use crate::report::Styler; + +/// Deletes the external cache belonging to the resolved workspace. +pub(super) fn clean(host: &mut H, args: &CleanArgs, styler: Styler) -> crate::Result { + let metadata = load_metadata(&args.dir, &FeatureArgs::default())?; + let root = camino::Utf8Path::new(metadata.workspace_root.as_str()); + let base = gamma_base(root, None); + + if clean_cache(root)? { + writeln!(host.error(), "{} `{base}`", styler.verb("Cleaned"))?; + } else { + writeln!(host.error(), "{} no cached data under `{base}`", styler.verb("Finished"))?; + } + + Ok(EXIT_OK) +} diff --git a/crates/cargo-gamma-lib/src/commands/cli.rs b/crates/cargo-gamma-lib/src/commands/cli.rs new file mode 100644 index 000000000..ca37b5a4b --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/cli.rs @@ -0,0 +1,1263 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use camino::Utf8PathBuf; +use clap::{Args, Parser, Subcommand, ValueEnum}; +use clap_complete::Shell; + +use super::When; +use crate::ci::{Annotations, Level}; +use crate::error::error; +use crate::ops::registry::Selection; + +/// Adapts a [`crate::bounds`] check to clap's parser signature. +macro_rules! bounded { + ($name:ident) => { + fn $name(text: &str) -> Result { + let value: f64 = text.parse().map_err(|_cause| format!("`{text}` is not a number"))?; + + crate::bounds::$name(text, value) + } + }; +} + +bounded!(seconds); +bounded!(factor); +bounded!(percentage); + +/// Adapts the memory-size check to clap's parser signature. +fn size(text: &str) -> Result { + crate::bounds::size(text) +} + +/// Fast mutation testing for Rust. +/// +/// With no subcommand, `run` is implied by argument normalization rather than by flattening +/// `RunArgs` here, so each help page lists only its own options. +#[derive(Debug, Parser)] +#[command( + name = "cargo-gamma", + bin_name = "cargo gamma", + version, + propagate_version = true, + about = "Fast mutation testing for Rust.", + long_about = "Fast mutation testing for Rust.\n\nEvery selected mutant is compiled into one \ + set of test binaries and chosen at run time, so a whole workspace is mutated \ + without rebuilding it once per mutant.\n\nWith no subcommand, `run` is implied.", + max_term_width = 100 +)] +pub struct Cli { + /// The subcommand to run. Defaults to `run`. + #[command(subcommand)] + pub command: Command, + + /// When to use color in output. + #[arg(long, global = true, value_name = "WHEN", default_value = "auto", help_heading = "Global options")] + pub color: When, + + /// When to show the progress display. + #[arg(long, global = true, value_name = "WHEN", default_value = "auto", help_heading = "Global options")] + pub progress: When, +} + +/// The subcommands. +#[derive(Debug, Subcommand)] +pub enum Command { + /// Run mutation testing. + Run(RunArgs), + + /// List what would be done, without doing it. + List(ListArgs), + + /// Explain a mutator, a mutant, or a suppression. + Explain(ExplainArgs), + + /// Write suppressions into the source for mutants that cannot usefully be tested. + Suppress(SuppressArgs), + + /// Remove skip directives that no longer suppress anything. + Unsuppress(UnsuppressArgs), + + /// Combine per-shard reports into one answer. + Merge(MergeArgs), + + /// Promote what a run learned about speed into a file the workspace can check in. + Hints(HintsArgs), + + /// Delete cargo-gamma's cached data for a workspace. + Clean(CleanArgs), + + /// Print a shell completion script. + Completions(CompletionsArgs), +} + +/// Arguments for `clean`. +#[derive(Debug, Args, Default)] +pub struct CleanArgs { + /// Path to the workspace or package whose cache should be deleted. + #[arg(short = 'd', long, value_name = "PATH", default_value = ".")] + pub dir: Utf8PathBuf, +} + +/// Arguments for `merge`. +#[derive(Debug, Args)] +pub struct MergeArgs { + /// The reports to merge. A directory is read for its `*.json` files. + #[arg(value_name = "REPORTS", required = true)] + pub inputs: Vec, + + /// Write the merged `mutation-testing-elements` document here. + #[arg(long, value_name = "PATH", help_heading = "Reporting")] + pub json_report: Option, + + /// Write a self-contained merged HTML report here. + #[arg(long, value_name = "PATH", help_heading = "Reporting")] + pub html_report: Option, + + /// Days after which a verdict is reported as stale. Zero disables the freshness window. + /// + /// Stale verdicts are still counted. Dropping them would shrink the denominator, which raises + /// the score by forgetting rather than by testing. + #[arg(long, value_name = "DAYS", default_value = "30", help_heading = "Merging")] + pub window: u64, + + /// Fail if the merged assertion-killed score is below this percentage. + /// + /// Score gates belong here rather than on a shard run: a shard's own score moves by a third of + /// a point per undetected mutant, so a threshold set on one fires on noise. Timeouts count as + /// undetected because no test assertion rejected them. + #[arg(long, value_name = "PERCENT", value_parser = percentage, help_heading = "Run control")] + pub min_score: Option, +} + +/// Arguments for `suppress`. +#[derive(Debug, Args)] +pub struct SuppressArgs { + /// The run to perform before writing anything. + #[command(flatten)] + pub run: RunArgs, + + /// Print the diff without changing anything. + /// + /// Spelled apart from the run's own `--dry-run`, which stops before building at all: this one + /// runs everything and holds back only the source edit. + #[arg(long, help_heading = "Suppressing")] + pub dry_run_suppress: bool, + + /// Which verdicts may be suppressed. + /// + /// A surviving mutant is never eligible and cannot be made eligible: it is a real gap in the + /// test suite, and suppressing it would remove the gap from the score rather than from the code. + /// + /// Timeouts and out-of-memory verdicts are both eligible by default because they are one + /// situation seen through two ceilings: whichever the runaway mutant reaches first is a property + /// of the machine, so suppressing only one produces directives that hold on one host and not on + /// another. + #[arg(long, value_name = "LIST", default_value = "timeout,outofmem", help_heading = "Suppressing")] + pub eligible: String, + + /// Edit source files that have uncommitted changes. + /// + /// The edit is undone in this process if anything goes wrong, but nothing survives the process + /// being killed part-way through it. A committed file needs no journal of its own, because + /// version control already is one; a file with uncommitted changes has nothing to be put back + /// from, so it is refused rather than edited. + #[arg(long, help_heading = "Suppressing")] + pub allow_dirty: bool, +} + +/// Arguments for `unsuppress`. +#[derive(Debug, Args)] +pub struct UnsuppressArgs { + /// What to look at. + #[command(flatten)] + pub select: SelectArgs, + + /// Remove the directives instead of printing what would be removed. + /// + /// The preview is the default, which is the reverse of `suppress`. Writing a directive can be + /// read back and reverted at leisure; deleting one that was in fact load-bearing turns a + /// considered decision into a survivor nobody chose to accept, and by then the reason it + /// carried is gone too. + #[arg(long, help_heading = "Suppressing")] + pub apply: bool, + + /// Remove directives from source files that have uncommitted changes. + /// + /// The removal is undone in this process if anything goes wrong, but nothing survives the + /// process being killed part-way through it — and what a removal takes out is a hand-written + /// reason nobody can reconstruct. A committed file needs no journal of its own, because version + /// control already is one; a file with uncommitted changes has nothing to be put back from, so + /// it is refused rather than edited. + #[arg(long, help_heading = "Suppressing")] + pub allow_dirty: bool, +} + +/// Arguments for `hints`. +#[derive(Debug, Args, Default)] +pub struct HintsArgs { + /// What to look at. + #[command(flatten)] + pub select: SelectArgs, + + /// Read the run record from this cache directory instead of cargo-gamma's default. + /// + /// Spelled the same as the run's own flag and for the same reason: a run told to scratch + /// elsewhere left its record there, and a promotion looking under `target` would find nothing + /// and say so, which reads exactly like "that run learned nothing". + #[arg(long, value_name = "PATH", help_heading = "Cache")] + pub cache_dir: Option, + + /// Report what would be promoted without writing anything. + #[arg(long, help_heading = "Run control")] + pub dry_run: bool, +} + +/// Arguments shared by commands that select mutants. +#[derive(Debug, Args, Clone)] +#[command(next_help_heading = "Selecting what to mutate")] +pub struct SelectArgs { + /// Path to the workspace or package to analyze. + #[arg(short = 'd', long, value_name = "PATH", default_value = ".")] + pub dir: Utf8PathBuf, + + /// Mutators to apply, as a comma-separated selector list. + /// + /// A selector is a mutator name (`arith.add_to_sub`), a family (`relational`), a preset + /// (`@arithmetic`), or `all`. Prefix a selector with `!` to remove it from the set. Selectors + /// are applied left to right, so `@arithmetic,!bitwise` means what it reads as. + #[arg(long, value_name = "SELECTORS", allow_hyphen_values = true)] + pub mutators: Option, + + /// Only mutate files matching these glob patterns. + #[arg(long = "file", value_name = "GLOB")] + pub files: Vec, + + /// Skip files matching these glob patterns. + #[arg(long = "exclude-file", value_name = "GLOB")] + pub exclude_files: Vec, + + /// Number of shards to divide the mutants into. + #[arg(long, value_name = "COUNT")] + pub shard_count: Option, + + /// Which shard to run, from 0. + #[arg(long, value_name = "INDEX")] + pub shard_index: Option, + + /// Only mutate lines added or changed by this unified diff, or `-` for standard input. + /// + /// This is what makes a run affordable on a pull request: the population is restricted to the + /// code under review rather than sampled from the whole tree, so the result speaks about the + /// change. Sharding is not a substitute, since a shard is a slice of everything. + #[arg(short = 'D', long, value_name = "PATH")] + pub in_diff: Option, + + /// Only mutate these packages. Defaults to Cargo's package selection for the current directory. + #[arg(short = 'p', long = "package", value_name = "NAME")] + pub packages: Vec, + + /// Mutate every package in the workspace. + /// + /// Selection follows cargo: without this flag and without `--package`, a run mutates the + /// package owning the directory it was invoked from, and the workspace's default members when + /// that directory is the workspace root. + #[arg(long, conflicts_with = "packages")] + pub workspace: bool, + + /// Additional values for `fn_value.err_with`, which replaces a function body with `Err(...)`. + /// + /// `fn_value.err_default` only reaches error types that implement `Default`. Naming a value + /// here — `--error 'std::io::Error::from(std::io::ErrorKind::Other)'` — reaches the rest. + #[arg(long = "error", value_name = "EXPR")] + pub errors: Vec, + + /// Which cargo features to build with. + #[command(flatten)] + pub features: FeatureArgs, + + /// Where the configuration comes from. + #[command(flatten)] + pub config: ConfigArgs, +} + +/// Cargo feature selection, shared by discovery and the build. +/// +/// Discovery and the build must agree: finding files under one feature set and compiling under +/// another produces guards that are not in the compiled tree. +#[derive(Debug, Args, Clone, Default)] +#[command(next_help_heading = "Cargo features")] +pub struct FeatureArgs { + /// Cargo features to activate, comma-separated or repeated. + #[arg(long = "features", value_name = "FEATURES")] + pub features: Vec, + + /// Activate every feature of every selected package. + #[arg(long, conflicts_with = "no_default_features")] + pub all_features: bool, + + /// Do not activate the `default` feature. + #[arg(long)] + pub no_default_features: bool, +} + +/// Where the configuration file comes from. +#[derive(Debug, Args, Clone, Default)] +#[command(next_help_heading = "Configuration")] +pub struct ConfigArgs { + /// Read configuration from this file instead of `gamma.toml`. + #[arg(long = "config", value_name = "PATH", conflicts_with = "no_config")] + pub path: Option, + + /// Ignore the configuration file entirely. + /// + /// Without this there is no way to script a run that is independent of whatever the project + /// happens to have committed. + #[arg(long)] + pub no_config: bool, +} + +/// How long a build may take before it is abandoned. +/// +/// Not offered to `estimate`. These change nothing an estimate reports — they can only turn a +/// working estimate into an error — and capping the build is at odds with a subcommand whose job is +/// to tell you what the build costs. +#[derive(Debug, Args, Default)] +#[command(next_help_heading = "Building")] +pub struct BuildLimitArgs { + /// Seconds the build may take before the run is abandoned. + /// + /// A run builds once, so a build that never finishes costs everything rather than one mutant. + #[arg(long, value_name = "SECONDS", value_parser = seconds, conflicts_with = "build_timeout_multiplier")] + pub build_timeout: Option, + + /// Multiple of the first successful build's duration that a later build round is allowed. + /// + /// Rollback rounds rebuild the same tree with fewer mutants, so a round that runs far longer + /// than the first one is not making progress. + #[arg(long, value_name = "FACTOR", value_parser = factor)] + pub build_timeout_multiplier: Option, + + /// How many times the tree may be rebuilt while withdrawing mutants that do not compile. + /// + /// A mutant like `Some(Default::default())` only compiles when the type happens to implement + /// `Default`, and rustc reports only the errors it reaches before it stops, so a large tree can + /// need many rounds to converge. Raise this when a run stops with a rollback-limit error and the + /// withdrawal counts it prints are still falling. + #[arg(long, value_name = "ROUNDS", default_value_t = crate::exec::DEFAULT_ROLLBACK_ROUNDS)] + pub rollback_rounds: u32, +} + +/// The options common to every command that builds, measures a baseline and runs tests. +/// +/// Shared by `run`, `estimate` and `advise`, because all three build the tree and measure the +/// baseline the same way — an estimate that measured differently from the run it predicts would be +/// predicting a different run. +#[derive(Debug, Args, Default)] +#[command(next_help_heading = "Running tests")] +#[expect( + clippy::struct_excessive_bools, + reason = "each is an independent command-line flag, and grouping them would only obscure that" +)] +pub struct MeasureArgs { + /// Let cargo's own build output through, instead of only its progress bar. + /// + /// A run reports how far along the build is and any errors it hits, and swallows the rest so + /// that compiling several thousand instrumented files does not bury the run. This shows all of + /// it, which is what you want when the build itself is what is going wrong. + #[arg(long)] + pub show_build: bool, + + /// How many mutants to test at once. Defaults to one more than the available parallelism. + #[arg(short = 'j', long, value_name = "N")] + pub jobs: Option, + + /// Multiple of each test binary's baseline duration that a mutant is allowed. + #[arg(long, value_name = "FACTOR", value_parser = factor)] + pub test_timeout_multiplier: Option, + + /// Lower bound on a test binary's timeout, however fast the baseline was. + /// + /// A test binary that finishes in a fraction of a second gets a budget of just over that + /// duration, which a loaded machine can miss for reasons that have nothing to do with the mutant. + #[arg(long, value_name = "SECONDS", value_parser = seconds)] + pub minimum_test_timeout: Option, + + /// How much memory control to place around each test binary. `enforce` by default. + /// + /// A mutation can turn bounded allocation into unbounded allocation, which the timeout only + /// stops after the machine has already been driven into swap. `measure` records what each test + /// binary's whole process tree uses during the baseline and reports it, without ever stopping a + /// mutant. `enforce` also holds each mutant to a ceiling derived from that measurement, and + /// reports a mutant that breaches it as killed, shown as `OUTOFMEM`. + /// + /// Needs a delegated cgroup v2 on Linux, or a job object on Windows. A run that asked for this + /// explicitly says so and stops where the host cannot provide it, rather than pretend to be + /// protected; a run that merely inherited the default drops to `off` and says so. + #[arg(long, value_name = "MODE", help_heading = "Memory")] + pub memory: Option, + + /// Multiple of a test binary's baseline peak memory a mutant of it may reach. + #[arg(long, value_name = "FACTOR", value_parser = factor, help_heading = "Memory")] + pub memory_multiplier: Option, + + /// Absolute headroom added to a test binary's baseline peak memory. + /// + /// The ceiling is the larger of this and the multiplier, so this is what governs a binary whose + /// baseline peak is small enough that doubling it would still leave no room for a lazily + /// initialized table or a randomized test that picked a larger input. + #[arg(long, value_name = "SIZE", value_parser = size, help_heading = "Memory")] + pub memory_headroom: Option, + + /// An explicit memory ceiling for every test binary, instead of one derived from the baseline. + /// + /// Implies `--memory enforce`, and is the only way to bound a run that skips the baseline, + /// since there is then nothing to calibrate a ceiling from. + #[arg(long, value_name = "SIZE", value_parser = size, help_heading = "Memory")] + pub memory_limit: Option, + + /// A memory ceiling for the baseline runs themselves. + /// + /// A ceiling derived from the baseline cannot protect the machine from a baseline that is + /// itself runaway, which is the risk the first time an unfamiliar suite is measured. Implies + /// `--memory measure`. + #[arg(long, value_name = "SIZE", value_parser = size, help_heading = "Memory")] + pub baseline_memory_limit: Option, + + /// Do not re-run inside a systemd scope to obtain the cgroup memory control needs. + /// + /// Bounding a test subtree needs a cgroup this process may create children under, and a host + /// that started cargo-gamma outside a systemd user session never handed it one. Rather than + /// give up the ceiling, cargo-gamma asks the systemd user manager for a delegated scope and + /// re-runs itself inside it, reporting that it did so. + /// + /// Pass this to keep the original process, accepting that memory control is then unavailable: + /// a run that asked for it explicitly stops, and one that inherited the default continues + /// unbounded. Worth doing where a new process is itself the problem — a supervisor tracking + /// this pid, or a session that must not gain a scope. + #[arg(long, help_heading = "Memory")] + pub no_relaunch: bool, + + /// Which Cargo profile to build with. + /// + /// Worth more here than in a per-mutant tool: the build is paid once and then thousands of + /// mutants run against it, so an optimized profile usually pays for itself many times over. + /// + /// `release` also turns `debug_assertions` off, so a mutant that only a `debug_assert!` would + /// have caught survives instead. Scores from two profiles are not comparable for that reason. + #[arg(long, value_name = "NAME", help_heading = "Building")] + pub profile: Option, + + /// Pass an argument through to every cargo invocation. + #[arg( + short = 'C', + long = "cargo-arg", + value_name = "ARG", + allow_hyphen_values = true, + help_heading = "Building" + )] + pub cargo_args: Vec, + + /// Pass an argument through to every test binary. + #[arg(long = "cargo-test-arg", value_name = "ARG", allow_hyphen_values = true)] + pub cargo_test_args: Vec, + + /// Run the tests of these packages when deciding a verdict. + /// + /// Separate from `--package`, which chooses what to mutate. By default each mutant is judged + /// only by tests from its own package, so this is how it gets judged by a suite living + /// somewhere else — a workspace that keeps its integration tests in a package of their own + /// wants naming that package here. + #[arg(long = "test-package", value_name = "NAME")] + pub test_packages: Vec, + + /// Only let these test targets decide a verdict. + /// + /// Matches cargo target names — a package's unit tests take the name of the lib or bin they + /// live in, and each file under `tests/` is a target named after the file. Globs use `*` and + /// `?`. Finer than `--test-package`, which cannot separate a package's real tests from the + /// conformance corpus sitting beside them. + #[arg(long = "include-test", value_name = "GLOB")] + pub include_tests: Vec, + + /// Do not let these test targets decide a verdict. + /// + /// Applied after `--include-test`, so an exclusion always wins. The usual reason is a target + /// that is slow, flaky, or not an oracle at all — a conformance or fuzz corpus whose failures + /// say nothing about whether a mutant was noticed. A pattern matching no target is an error. + #[arg(long = "exclude-test", value_name = "GLOB")] + pub exclude_tests: Vec, + + /// Run test binaries through `cargo nextest` for per-test process isolation. + /// + /// The default launches each compiled test binary directly, which is faster. Passing `--nextest` + /// hands those same binaries to `cargo nextest`, which gives every test its own process. Use it + /// for a suite that depends on that isolation: such a suite is not merely slower without it but + /// red, and a red baseline stops the run before it judges anything. + /// + /// Nextest is given the binaries this run already built rather than asked to build its own, so + /// it never invokes cargo and a mutant costs one extra process rather than one extra build. + #[arg(long)] + pub nextest: bool, + + /// Let every workspace package's tests decide a verdict. + /// + /// By default each mutant is judged by its package's own suite and not by its dependents'. + /// This lifts that restriction and asks every workspace package that can reach the mutant, + /// which is more thorough and costs a workspace-wide build and test run per mutant. + #[arg(long, conflicts_with = "test_packages")] + pub test_workspace: bool, + + /// Run every selected test in each reachable test binary. + /// + /// By default every mutation site doubles as a reachability probe: gamma measures which + /// individual tests execute each site, then runs only those tests against its mutant. This + /// disables that measurement and runs each reachable binary whole instead. + /// + /// Use it when test reachability is nondeterministic because control flow depends on threads, + /// the clock, randomness or hash iteration order. It is usually much slower, especially for + /// survivors, because every test in a linked binary is repeated for every mutant it can reach. + /// + /// This does not override `--include-test`, `--exclude-test`, `--test-package` or package + /// reachability; it changes only whether selected binaries are filtered to specific test cases. + #[arg(long)] + pub whole_test_binaries: bool, + + /// Put cargo-gamma's reusable workspace and Cargo artifacts in this directory. + /// + /// Lets a read-only checkout be mutated and moves the copy off a slow or network filesystem. It + /// is refused when relocating would hide VCS metadata a build script can see from the source + /// checkout. The path is the cache itself, must be empty on first use, and becomes owned by this + /// workspace; another workspace cannot share it. Build artifacts live here too, so reusing the + /// directory across this workspace's runs keeps them incremental while a fresh one starts cold. + #[arg(long, value_name = "PATH", help_heading = "Cache")] + pub cache_dir: Option, + + /// Copy files version control ignores into the cached workspace as well. + /// + /// Files git tracks are always copied, whatever an ignore rule says about them; this is for the + /// untracked ones. Reach for it when the build reads something a shared `.gitignore` excludes — + /// a generated module, a downloaded fixture — which would otherwise have to be fixed by editing + /// an ignore file that is not this tool's to edit. Everything else the tree has ever built is + /// copied too, so a run with it on can be markedly slower to start. + #[arg(long, help_heading = "Cache")] + pub copy_ignored: bool, + + /// Arguments passed to every test binary, after `--`. + /// + /// The natural place to name the tests a run should consider, as in `-- --skip slow_`. + #[arg(last = true, value_name = "TEST_ARGS")] + pub test_args: Vec, +} + +/// Arguments for `run`. +#[derive(Debug, Args, Default)] +#[expect( + clippy::struct_excessive_bools, + reason = "each is an independent command-line flag, and grouping them would only obscure that" +)] +pub struct RunArgs { + /// Which mutants to consider. + #[command(flatten)] + pub select: SelectArgs, + + /// How the build and the baseline are measured. + #[command(flatten)] + pub measure: MeasureArgs, + + /// How long the build may take. + #[command(flatten)] + pub limits: BuildLimitArgs, + + /// Write all user-facing artifacts to this directory. + /// + /// The directory is created when it does not exist. It contains the JSON, HTML and SARIF + /// reports, performance advice and diagnostics bundle. + #[arg(long, value_name = "PATH", help_heading = "Reporting")] + pub artifact_dir: Option, + + /// Fail the run if the assertion-killed mutation score is below this percentage. + /// + /// Timeouts and out-of-memory mutants count against the score because no test assertion + /// rejected them. + #[arg(long, value_name = "PERCENT", value_parser = percentage, help_heading = "Run control")] + pub min_score: Option, + + /// How an incremental run reuses the last run: `no` starts cold; `build` reuses compiler + /// unviability and checked execution hints. + #[arg(long, value_enum, help_heading = "Run control")] + pub incremental: Option, + + /// List the mutants the suite killed, not just the ones that survived. + /// + /// A run reports what escaped, because that is what needs acting on. This shows the other side: + /// what the suite actually killed, which is how you confirm it is testing what you think it is. + #[arg(long, help_heading = "Reporting")] + pub show_killed: bool, + + /// List every mutant that could not be compiled, not just how many there were. + /// + /// A mutant that does not compile says nothing about the test suite, and a large workspace + /// produces thousands of them, so the summary counts them instead. Ask for the list when the + /// question is which constructs the encoding could not express. + #[arg(long, help_heading = "Reporting")] + pub show_unviable: bool, + + /// Keep an incomplete scratch workspace after errors so it can be inspected. + #[arg(long, help_heading = "Run control")] + pub leak_dirs: bool, + + /// Skip the baseline run. + /// + /// Faster, and strictly less trustworthy: without it there is no evidence that a failure was + /// caused by the mutant rather than by the suite already being red. + #[arg(long, help_heading = "Run control")] + pub no_baseline: bool, + + /// Believe a failing test without re-running it with no mutant active. + /// + /// Saves one run per kill, and gives up the ability to tell a detection from a flaky test: a + /// test that fails either way is counted as having caught every mutant it was run against. + #[arg(long, help_heading = "Run control")] + pub no_confirm: bool, + + /// Find and report mutants without building or running anything. + #[arg(long, help_heading = "Run control")] + pub dry_run: bool, + + /// Load the report viewer from a CDN instead of embedding it. + /// + /// Produces a much smaller file, at the cost of needing network access to read it. + #[arg(long, help_heading = "Reporting")] + pub html_external: bool, + + /// How loudly a survivor is reported to a SARIF consumer. + /// + /// A surviving mutant is an observation about the test suite rather than a defect in the code, + /// and drowning the security tab is how a good signal gets turned off. + #[arg(long, value_name = "LEVEL", default_value = "note", help_heading = "Reporting")] + pub sarif_level: Level, + + /// Annotate the diff and write a job summary when running inside a CI system. + #[arg(long, value_name = "WHEN", default_value = "auto", help_heading = "Reporting")] + pub annotations: Annotations, + + /// What to do with package and binary names in the diagnostics bundle. + /// + /// Hashed by default. A timing profile needs to tell one row from another and to group rows + /// that belong together, and a stable hash gives both without naming an unreleased codebase. + #[arg(long, value_name = "POLICY", default_value = "hashed", help_heading = "Reporting")] + pub diag_names: crate::diag::Redaction, + + /// Project what the rest of the run will cost, once the build and baseline have been measured. + /// + /// Printed at the only moment it is both possible and useful: everything before it was + /// measured, and everything after it is the wait you are deciding whether to sit through. The + /// range assumes a killed mutant gets through 60% of the tests that can reach it before one + /// of them fails. + #[arg(long, help_heading = "Run control")] + pub estimate: bool, + + /// Dump what the run measured about itself, for people working on this tool. + /// + /// Hidden, unstable and undocumented on purpose: it exists so that a change to the scheduler, + /// the build sequencing or the mutator catalog can be judged against numbers instead of + /// against how the run felt. Nothing here is a promise, and none of it is meant to be parsed. + #[arg(long, hide = true, help_heading = "Run control")] + pub diag: bool, + + /// Wait out the whole budget for every mutant instead of cutting off one that has stopped + /// making progress. + /// + /// A hung mutant is normally detected as soon as its test binary goes quiet for longer than + /// the baseline ever did, which is usually far sooner than its timeout. Turn this off if a + /// test legitimately goes silent for much longer under mutation than it ever does healthy. + #[arg(long, help_heading = "Run control")] + pub no_stall_detection: bool, +} + +/// Arguments for `completions`. +#[derive(Debug, Args)] +pub struct CompletionsArgs { + /// The shell to generate a completion script for. + #[arg(value_name = "SHELL")] + pub shell: Shell, +} + +/// What `list` can enumerate. +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum ListKind { + /// The mutants that would be generated. + Mutants, + + /// The mutator registry. + Mutators, + + /// The source files that would be analyzed. + Files, + + /// The named mutator presets. + #[value(alias = "profiles")] + Presets, +} + +/// Arguments for `list`. +#[derive(Debug, Args)] +pub struct ListArgs { + /// What to list. + #[arg(value_enum, default_value = "mutants")] + pub what: ListKind, + + /// Which mutants to consider. + #[command(flatten)] + pub select: SelectArgs, + + /// Emit machine-readable JSON instead of text. + #[arg(long, help_heading = "Reporting")] + pub json: bool, + + /// Write the population as a report document, for `merge` to withdraw retired mutants against. + #[arg(long, value_name = "PATH", help_heading = "Reporting")] + pub json_report: Option, +} + +/// Arguments for `explain`. +#[derive(Debug, Args)] +pub struct ExplainArgs { + /// A mutator name, family, preset, or mutant id. + #[arg(value_name = "SUBJECT")] + pub subject: String, +} + +impl Default for SelectArgs { + fn default() -> Self { + Self { + dir: Utf8PathBuf::from("."), + mutators: None, + files: Vec::new(), + exclude_files: Vec::new(), + shard_count: None, + shard_index: None, + in_diff: None, + packages: Vec::new(), + workspace: false, + errors: Vec::new(), + features: FeatureArgs::default(), + config: ConfigArgs::default(), + } + } +} + +impl SelectArgs { + /// Resolves the `--mutators` selector list into a concrete set of mutators. + pub fn selection(&self) -> crate::Result { + let mut selection = self + .mutators + .as_deref() + .map_or_else(|| Ok(Selection::default_preset()), Selection::parse)?; + + // An explicit `--mutators` list is the whole set, so `--error` must not smuggle a mutator into + // it. Dropping the values silently would be worse still, so say so. + if self.mutators.is_some() && !self.errors.is_empty() && !selection.contains("fn_value.err_with") { + selection.drop_errors(); + return Ok(selection); + } + + selection.set_errors(self.errors.clone()); + Ok(selection) + } + + /// Validates the effective sharding settings and names the shard to run. + /// + /// This is the only place the count and the index are checked against each other, and it runs + /// on the values the configuration file and the command line have already been folded into. + /// The pair is whole nowhere else: the count belongs in the committed file, because every job + /// in the matrix has to agree on it, while the index differs per job and arrives on the command + /// line — so a check made while the command line is parsed would reject the split the file + /// exists to support, and a check made on the file alone would not see the index at all. + /// + /// A half pair is refused rather than rounded down to "no sharding". Silently running the whole + /// population when a CI job asked for a slice of it is the failure that costs a night: the job + /// passes, the report looks complete, and nothing says the run was eight times the size it was + /// budgeted for. + pub fn shard(&self) -> crate::Result> { + match (self.shard_count, self.shard_index) { + (None, None) => Ok(None), + + (Some(count), None) => Err(error!( + "a shard count of {count} was set without a shard index; a shard is both, so pass `--shard-index` or set `index` in the `[shard]` table" + ) + .usage()), + + (None, Some(index)) => Err(error!( + "a shard index of {index} was set without a shard count; a shard is both, so pass `--shard-count` or set `count` in the `[shard]` table" + ) + .usage()), + + (Some(count), Some(index)) => { + if count == 0 { + return Err(error!("a shard count of 0 divides the population into nothing; it must be at least 1").usage()); + } + + if count > crate::merge::MAX_SHARDS { + return Err(error!( + "a shard count of {count} is more than the {} a merge will account for; a rotation that large cannot be merged back together", + crate::merge::MAX_SHARDS + ) + .usage()); + } + + if index >= count { + return Err(error!( + "shard index {index} is out of range for a shard count of {count}; valid indices are 0..{}", + count - 1 + ) + .usage()); + } + + Ok(Some((count, index))) + } + } + } +} + +impl FeatureArgs { + /// Renders the selection as the cargo arguments that express it. + #[must_use] + pub fn to_cargo_args(&self) -> Vec { + let mut args = Vec::new(); + + if self.all_features { + args.push("--all-features".to_owned()); + } + + if self.no_default_features { + args.push("--no-default-features".to_owned()); + } + + if !self.features.is_empty() { + args.push("--features".to_owned()); + args.push(self.features.join(",")); + } + + args + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shard_index_must_be_in_range() { + let args = SelectArgs { + shard_count: Some(4), + shard_index: Some(4), + ..SelectArgs::default() + }; + + let error = args.shard().unwrap_err(); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("out of range"), "{error}"); + assert!(error.to_string().contains("0..3"), "{error}"); + } + + /// The last index is inside the range, and an off-by-one here would reject a whole shard of a + /// matrix — the last job, which is the one nobody notices is missing. + #[test] + fn the_last_shard_index_is_in_range() { + let args = SelectArgs { + shard_count: Some(4), + shard_index: Some(3), + ..SelectArgs::default() + }; + + assert_eq!(args.shard().expect("the last index is valid"), Some((4, 3))); + } + + #[test] + fn a_zero_shard_count_is_rejected() { + let args = SelectArgs { + shard_count: Some(0), + shard_index: Some(0), + ..SelectArgs::default() + }; + + let error = args.shard().unwrap_err(); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("at least 1"), "{error}"); + } + + /// A rotation wider than a merge can account for is refused where it is asked for, not where it + /// is discovered. + /// + /// The merge side carries a ceiling on how many shards it will track, and a count above it + /// leaves a rotation whose slices each run fine and which can never be put back together. The + /// producing run is the only place that failure is cheap: it costs one refused command instead + /// of a whole matrix of green jobs and a merge that cannot report on them. + #[test] + fn a_shard_count_beyond_what_a_merge_can_account_for_is_rejected() { + let args = SelectArgs { + shard_count: Some(crate::merge::MAX_SHARDS + 1), + shard_index: Some(0), + ..SelectArgs::default() + }; + + let error = args.shard().unwrap_err(); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("cannot be merged back together"), "{error}"); + + let at_ceiling = SelectArgs { + shard_count: Some(crate::merge::MAX_SHARDS), + shard_index: Some(0), + ..SelectArgs::default() + }; + + assert_eq!( + at_ceiling.shard().expect("the ceiling itself is a mergeable rotation"), + Some((crate::merge::MAX_SHARDS, 0)) + ); + } + + /// A count with no index is the shape of a configuration file that names the matrix width and a + /// job that forgot to say which slice it is. Running the whole population instead would pass, + /// look complete, and cost the entire budget. + #[test] + fn a_count_without_an_index_is_rejected() { + let args = SelectArgs { + shard_count: Some(8), + shard_index: None, + ..SelectArgs::default() + }; + + let error = args.shard().unwrap_err(); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("--shard-index"), "{error}"); + assert!(error.to_string().contains("shard count of 8"), "{error}"); + } + + #[test] + fn an_index_without_a_count_is_rejected() { + let args = SelectArgs { + shard_count: None, + shard_index: Some(3), + ..SelectArgs::default() + }; + + let error = args.shard().unwrap_err(); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("--shard-count"), "{error}"); + assert!(error.to_string().contains("shard index of 3"), "{error}"); + } + + /// Zero is a value like any other, so a lone zero is a half pair rather than an absence. The + /// distinction matters because `Option::unwrap_or_default` on either field would turn one of + /// these into a silent full-population run. + #[test] + fn a_lone_zero_is_still_a_half_pair() { + let count_only = SelectArgs { + shard_count: Some(0), + shard_index: None, + ..SelectArgs::default() + }; + let index_only = SelectArgs { + shard_index: Some(0), + ..SelectArgs::default() + }; + + assert!(count_only.shard().unwrap_err().to_string().contains("--shard-index")); + assert!(index_only.shard().unwrap_err().to_string().contains("--shard-count")); + } + + /// One shard containing everything is a degenerate but legitimate matrix of one job. + #[test] + fn a_single_shard_is_accepted() { + let args = SelectArgs { + shard_count: Some(1), + shard_index: Some(0), + ..SelectArgs::default() + }; + + assert_eq!(args.shard().expect("one shard of one"), Some((1, 0))); + } + + #[test] + fn a_valid_shard_is_accepted() { + let args = SelectArgs { + shard_count: Some(4), + shard_index: Some(3), + ..SelectArgs::default() + }; + + assert_eq!(args.shard().unwrap(), Some((4, 3))); + } + + #[test] + fn no_sharding_arguments_means_no_shard() { + assert_eq!(SelectArgs::default().shard().unwrap(), None); + } + + /// The command line no longer requires the two together, because the file supplies one of them + /// and is read after parsing. What must not follow is that a half pair typed on the command + /// line becomes acceptable — it is refused later instead, on the effective values. + #[test] + fn half_a_pair_parses_and_is_refused_afterwards() { + let cli = Cli::try_parse_from(["cargo-gamma", "list", "--shard-count", "4"]).expect("parsing must not reject the split"); + + let Command::List(args) = cli.command else { + panic!("the list subcommand was parsed as something else"); + }; + + assert_eq!(args.select.shard_count, Some(4)); + assert!(args.select.shard().unwrap_err().is_usage()); + } + + #[test] + fn no_ops_argument_selects_the_default_preset() { + let selection = SelectArgs::default().selection().unwrap(); + + assert!(selection.contains("fn_value.default")); + assert!(selection.contains("stmt.delete_call")); + } + + #[test] + fn mutator_presets_are_listed_by_the_new_name_and_the_old_name_remains_an_alias() { + for name in ["presets", "profiles"] { + let cli = Cli::try_parse_from(["cargo-gamma", "list", name]).expect("list kind parses"); + let Command::List(args) = cli.command else { + panic!("expected list"); + }; + + assert_eq!(args.what, ListKind::Presets); + } + } + + #[test] + fn naming_error_values_turns_the_error_mutator_on() { + // The mutator is registered on like everything else, but it is inert until the user names + // something for it to substitute, so supplying a value has to keep it on rather than being + // the thing that enables it. + let args = SelectArgs { + errors: vec!["MyError::Io".to_owned()], + ..SelectArgs::default() + }; + + let selection = args.selection().unwrap(); + + assert!(selection.contains("fn_value.err_with")); + assert_eq!(selection.errors(), ["MyError::Io".to_owned()]); + } + + #[test] + fn error_values_are_ignored_when_the_mutator_is_deselected() { + let args = SelectArgs { + mutators: Some("relational".to_owned()), + errors: vec!["MyError::Io".to_owned()], + ..SelectArgs::default() + }; + + assert!(args.selection().unwrap().errors().is_empty()); + } + + #[test] + fn feature_arguments_render_as_cargo_spells_them() { + let features = FeatureArgs { + features: vec!["a,b".to_owned()], + all_features: false, + no_default_features: true, + }; + + assert_eq!( + features.to_cargo_args(), + vec!["--no-default-features".to_owned(), "--features".to_owned(), "a,b".to_owned()] + ); + } + + #[test] + fn no_feature_arguments_render_as_nothing() { + assert!(FeatureArgs::default().to_cargo_args().is_empty()); + } + + #[test] + fn all_features_render_before_named_features() { + let features = FeatureArgs { + features: vec!["serde".to_owned(), "cli".to_owned()], + all_features: true, + no_default_features: false, + }; + + assert_eq!( + features.to_cargo_args(), + vec!["--all-features".to_owned(), "--features".to_owned(), "serde,cli".to_owned()] + ); + } + + #[test] + fn the_cli_definition_is_valid() { + use clap::CommandFactory as _; + + Cli::command().debug_assert(); + } + + /// The default eligibility is both ceilings, and it is a real parse rather than a string check. + /// + /// Asserting on the `default_value` literal alone would pass even if the token were misspelled, + /// because nothing in clap requires the default to be a value `Eligible::parse` accepts — the + /// failure would surface only when a user ran `suppress` with no `--eligible`. + #[test] + fn suppress_defaults_to_suppressing_both_timeouts_and_out_of_memory() { + let cli = Cli::try_parse_from(["cargo-gamma", "suppress"]).expect("suppress parses with no arguments"); + + let Command::Suppress(args) = cli.command else { + panic!("expected the suppress subcommand"); + }; + + let eligible = crate::fix::Eligible::parse(&args.eligible).expect("the default must be a value the parser accepts"); + + assert_eq!(eligible, vec![crate::fix::Eligible::Timeout, crate::fix::Eligible::OutOfMemory]); + } + + #[test] + fn a_zero_merge_window_is_the_documented_disabled_value() { + use clap::CommandFactory as _; + + let cli = Cli::try_parse_from(["cargo-gamma", "merge", "report.json", "--window", "0"]).expect("zero window"); + let Command::Merge(args) = cli.command else { + panic!("expected merge"); + }; + + assert_eq!(args.window, 0); + let help = Cli::command() + .find_subcommand_mut("merge") + .expect("merge") + .render_long_help() + .to_string(); + assert!(help.contains("Zero disables the freshness window"), "{help}"); + } + + /// The `bounded!`-generated wrappers and the memory-size wrapper are what clap actually calls + /// as `value_parser`s; `crate::bounds` has its own tests for the underlying range checks, but + /// those never exercise the adapters here, and a wrapper that stopped forwarding its argument + /// correctly would only show up once a user typed a flag, not in any test of `bounds` itself. + #[test] + fn the_bound_wrappers_forward_to_the_underlying_checks() { + _ = seconds("1.5").expect("in range"); + _ = seconds("-5").expect_err("out of range"); + assert!(seconds("not-a-number").expect_err("not a number").contains("not a number")); + + _ = factor("1.2").expect("in range"); + _ = factor("-1").expect_err("out of range"); + + _ = percentage("50").expect("in range"); + _ = percentage("150").expect_err("out of range"); + + assert_eq!(size("1024"), Ok(1024)); + _ = size("not-a-size").expect_err("not a size"); + } + + /// `-V` is `--version` everywhere, and `-v` is not bound at all. + /// + /// Both short flags were once bound to `run` reporting options — `-V` to `--unviable` and `-v` + /// to `--killed` — so `cargo gamma run -V` silently started a full run instead of printing a + /// version. Nothing failed, which is why it survived: clap only rejects a collision when the + /// flag it collides with exists on that subcommand, and `version` was not propagated then. + #[test] + fn short_v_is_version_on_a_subcommand_and_never_a_reporting_flag() { + use clap::CommandFactory as _; + + for short in ['v', 'V'] { + let error = Cli::try_parse_from(["cargo gamma", "run", &format!("-{short}")]).expect_err("neither short flag may start a run"); + + let kind = error.kind(); + assert_ne!(kind, clap::error::ErrorKind::DisplayHelp, "-{short} must not be help"); + + if short == 'V' { + assert_eq!( + kind, + clap::error::ErrorKind::DisplayVersion, + "-V must print the version, not run: {error}" + ); + } else { + assert_eq!( + kind, + clap::error::ErrorKind::UnknownArgument, + "-v must stay free for a future --verbose: {error}" + ); + } + } + + let run = Cli::command() + .get_subcommands() + .find(|command| command.get_name() == "run") + .expect("run exists") + .clone(); + + for argument in run.get_arguments() { + assert_ne!(argument.get_short(), Some('v'), "{} claimed -v", argument.get_id()); + } + } + + /// Artifact routing is directory-wide; individual report path flags are not accepted. + #[test] + fn artifact_dir_replaces_individual_report_paths() { + let cli = Cli::try_parse_from(["cargo gamma", "run", "--artifact-dir", "out"]).expect("the directory parses"); + + match cli.command { + Command::Run(args) => assert_eq!(args.artifact_dir.unwrap(), "out"), + _ => panic!("expected run"), + } + + for removed in ["--html-report", "--json-report", "--sarif-report", "--advice", "--diag-bundle"] { + _ = Cli::try_parse_from(["cargo gamma", "run", removed, "out"]).expect_err("individual report paths are gone"); + } + + let cli = Cli::try_parse_from(["cargo gamma", "run", "--cache-dir", "cache"]).expect("cache directory parses"); + + match cli.command { + Command::Run(args) => assert_eq!(args.measure.cache_dir.unwrap(), "cache"), + _ => panic!("expected run"), + } + + _ = Cli::try_parse_from(["cargo gamma", "run", "--scratch-dir", "cache"]).expect_err("the old cache option is gone"); + } + + /// Every flag taking a filesystem path presents the same `` placeholder. + /// + /// `--config` once said `` and `--cache-dir` said ``, which reads as though they + /// accept different things from the eleven other path-valued flags. They do not. + #[test] + fn path_valued_flags_all_use_the_same_placeholder() { + use clap::CommandFactory as _; + + let command = Cli::command(); + let mut checked = 0; + + for subcommand in command.get_subcommands() { + for argument in subcommand.get_arguments() { + let id = argument.get_id().as_str(); + + if !matches!(id, "config" | "cache_dir" | "artifact_dir" | "dir") { + continue; + } + + let names = argument.get_value_names().unwrap_or_default(); + + assert_eq!( + names.first().map(clap::builder::Str::as_str), + Some("PATH"), + "{}'s {id} does not say PATH", + subcommand.get_name() + ); + + checked += 1; + } + } + + assert!(checked >= 4, "the filter matched almost nothing: {checked}"); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/completions.rs b/crates/cargo-gamma-lib/src/commands/completions.rs new file mode 100644 index 000000000..c9a99f92d --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/completions.rs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use clap::CommandFactory as _; + +use super::cli::{Cli, CompletionsArgs}; +use super::dispatch::EXIT_OK; +use super::host::Host; + +/// Writes a shell completion script to standard output. +pub(super) fn completions(host: &mut H, args: &CompletionsArgs) -> i32 { + let mut command = Cli::command(); + + // The generator treats spaces in the name as a subcommand path, so the script is generated for + // the executable rather than for the `cargo gamma` spelling that goes through cargo's plugin + // dispatch. Completion still works, because cargo hands the word after `gamma` straight on. + let name = command.get_name().to_owned(); + + clap_complete::generate(args.shell, &mut command, name, &mut host.results()); + + EXIT_OK +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use clap_complete::Shell; + + use super::*; + use crate::testing::Sink; + + #[test] + fn every_supported_shell_produces_a_script() { + for shell in [Shell::Bash, Shell::Zsh, Shell::Fish, Shell::PowerShell, Shell::Elvish] { + let mut host = Sink::default(); + let code = completions(&mut host, &CompletionsArgs { shell }); + + assert_eq!(code, EXIT_OK); + assert!(!host.out.is_empty(), "{shell} produced nothing"); + } + } + + #[test] + fn the_script_names_the_subcommands() { + let mut host = Sink::default(); + let _code = completions(&mut host, &CompletionsArgs { shell: Shell::Bash }); + let script = String::from_utf8(host.out).expect("the script is not UTF-8"); + + assert!(script.contains("merge"), "{script}"); + assert!(script.contains("estimate"), "{script}"); + } + + #[test] + fn sink_reports_the_shape_the_generator_ignores() { + let mut host = Sink::default(); + + assert_eq!(host.error().write(b"ignored").expect("write"), 7); + assert!(!host.is_terminal()); + assert_eq!(host.terminal_width(), None); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/console_events.rs b/crates/cargo-gamma-lib/src/commands/console_events.rs new file mode 100644 index 000000000..1070a5853 --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/console_events.rs @@ -0,0 +1,492 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::fmt::Write as _; +use std::io::Write as _; + +use camino::Utf8Path; + +use super::host::Host; +use super::verdict_log::VerdictLog; +use crate::model::Outcome; +use crate::report::{Progress, Styler}; + +/// Drives the console progress display from execution events. +pub(super) struct ConsoleEvents<'host, H: Host> { + pub(super) host: &'host mut H, + pub(super) progress: Progress, + pub(super) styler: Styler, + + /// Whether `--estimate` asked for a projection of the wait still to come. + pub(super) estimate: bool, + + /// Whether `--show-build` asked for cargo's own narration of the build. + pub(super) show_build: bool, + + pub(super) verdict_log: VerdictLog, +} + +impl ConsoleEvents<'_, H> { + /// Closes a phase line whose phase failed, so the error that follows starts on its own line. + pub(super) fn abandon(&mut self) { + self.progress.abandon(self.host); + } + + pub(super) fn finish_verdict_log(&mut self) -> crate::Result<()> { + self.verdict_log.finish() + } +} + +impl crate::exec::Events for ConsoleEvents<'_, H> { + fn testing_log(&mut self, scratch: &Utf8Path) -> crate::Result<()> { + self.verdict_log.start(scratch) + } + + fn phase(&mut self, verb: &str, detail: &str) { + self.progress.status(self.host, verb, detail); + } + + fn begin(&mut self, active: &str, completed: &str, detail: &str) { + self.progress.begin(self.host, active, completed, detail); + } + + fn end(&mut self, detail: &str) { + self.progress.end(self.host, detail); + } + + fn complete(&mut self, detail: &str) { + self.progress.complete(self.host, detail); + } + + fn phase_progress(&mut self, completed: usize, total: usize, unit: &str) { + self.progress.phase_progress(self.host, completed, total, unit); + } + + fn outcome(&mut self, detail: &str) { + self.progress.labelled(self.host, &crate::report::continuation(), detail); + } + + fn build_progress(&mut self, bar: &str) { + // Suppressed while cargo's own output is coming through, because both redraw the same line. + if self.show_build { + return; + } + + self.progress.borrowed(self.host, bar); + } + + fn build_output(&mut self, line: &str) { + if !self.show_build { + return; + } + + // Written whether or not the display is on. The display goes quiet when output is piped, + // and that is precisely where a user who asked to see the build needs to see it. + self.progress.insist(self.host, &crate::report::continuation(), line); + } + + fn wants_build_output(&self) -> bool { + self.show_build + } + + fn build_finished(&mut self) { + self.progress.restore(self.host); + } + + fn warn(&mut self, message: &str) { + let label = self.styler.warning(); + + for line in message.lines() { + self.progress.insist(self.host, &label, line); + } + } + + fn mutant(&mut self, mutant: &crate::model::Mutant) { + self.verdict_log.record(mutant); + + // A survivor is the entire point of the exercise; a timeout is the most expensive thing a + // run can find; and a mutant stopped by its memory ceiling is usually a sign the ceiling is + // wrong rather than a finding about the code. All three are printed as they happen rather + // than held back for the summary. Everything else only moves the bar. The label is the one + // the summary would use, so the same mutant is never named two different things. + match mutant.outcome { + Outcome::Survived => { + let label = self.styler.outcome(Outcome::Survived); + + self.progress.labelled(self.host, &label, &mutant.describe()); + } + + // Both carry a note that says something the label cannot: which test a timeout stalled + // in, and what a memory kill peaked at against what ceiling. Neither is worth repeating + // the label for, so only the note is appended. + outcome @ (Outcome::Timeout | Outcome::OutOfMemory) => { + let label = self.styler.outcome(outcome); + let detail = mutant_detail(mutant); + + self.progress.labelled(self.host, &label, &detail); + } + + _ => {} + } + + self.progress.record(mutant.outcome); + self.progress.tick(self.host); + } + + fn measured(&mut self, plan: &crate::discover::Plan, _session: &crate::exec::Session, estimate: &crate::estimate::Estimate) { + for mutant in &plan.mutants { + if mutant.outcome != Outcome::Pending { + self.verdict_log.record(mutant); + } + } + + // The bar's scale is the population that is about to be tested, which is not known until + // every package has been scanned — and this is the moment that becomes true. + let live = plan.mutants.iter().filter(|mutant| mutant.ordinal > 0).count(); + + self.progress.set_total(live); + + if !self.estimate { + return; + } + + // Written straight to the stream rather than through the progress display, because the + // display goes quiet when output is piped and an explicitly requested estimate must not. + self.progress.clear(self.host); + + let projection = format!("{} {}", self.styler.verb("Estimate"), crate::estimate::render(estimate)); + let mut stream = self.host.error(); + + let _ = writeln!(stream, "{projection}"); + let _ = stream.flush(); + } +} + +pub(super) fn mutant_detail(mutant: &crate::model::Mutant) -> String { + let mut detail = match mutant.outcome { + Outcome::Timeout | Outcome::OutOfMemory | Outcome::Flaky => mutant + .note + .as_deref() + .map_or_else(|| mutant.describe(), |note| format!("{}: {note}", mutant.describe())), + _ => mutant.describe(), + }; + + // A timeout that does not say how long it was given cannot be acted on: a genuine hang and a + // budget calibrated too tightly print identically, and they call for opposite responses. + if mutant.outcome == Outcome::Timeout { + let _ = write!(detail, " (after {})", crate::advise::human(mutant.elapsed())); + } + + detail +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use camino::Utf8PathBuf; + + use super::super::verdict_log::TESTING_PROGRESS_LOG; + use super::*; + use crate::exec::Events as _; + use crate::fixtures; + use crate::model::Mutant; + use crate::testing::Sink; + + fn mutant(outcome: Outcome, note: Option<&str>) -> Mutant { + timed(outcome, note, 0) + } + + fn timed(outcome: Outcome, note: Option<&str>, elapsed_ms: u64) -> Mutant { + Mutant { + line: 3, + column: 4, + item_path: ("subject::less".to_owned()).into(), + original: "a < b".to_owned().into(), + replacement: "a <= b".to_owned().into(), + outcome, + elapsed_ms, + note: note.map(str::to_owned), + ..fixtures::mutant() + } + } + + #[test] + fn survivors_and_timeouts_are_announced_as_they_happen() { + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(true, Styler::new(false), Some(80)), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.mutant(&mutant(Outcome::Survived, None)); + events.mutant(&mutant(Outcome::Timeout, Some("stalled, last test named was `slow`"))); + + let err = String::from_utf8(host.err).expect("utf-8"); + + assert!(err.contains("SURVIVED"), "{err}"); + assert!(err.contains("TIMEOUT"), "{err}"); + assert!(err.contains("stalled, last test named was `slow`"), "{err}"); + } + + /// A timeout says how long the mutant was given, alongside whatever the note already said. + /// + /// Without the figure a genuine hang and a budget calibrated too tightly print the same line, + /// and they call for opposite responses — lengthen the budget, or go and look at the code. + #[test] + fn a_timeout_says_how_long_the_mutant_ran_before_it_was_stopped() { + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(true, Styler::new(false), Some(200)), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.mutant(&timed(Outcome::Timeout, Some("stalled, last test named was `slow`"), 22_400)); + + let err = String::from_utf8(host.err).expect("utf-8"); + + assert!(err.contains("(after 22.4s)"), "{err}"); + + // The note names the likely culprit where the figure only sizes the problem, so the figure + // has to compose with it rather than displace it. + assert!(err.contains("stalled, last test named was `slow`"), "{err}"); + } + + /// Only a timeout gains the figure. Every other outcome's line is unchanged. + /// + /// A memory kill's note already carries the two numbers that matter — the peak and the ceiling + /// — and how long it took to get there says nothing about either. + #[test] + fn no_other_outcome_gains_an_elapsed_figure() { + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(true, Styler::new(false), Some(200)), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.mutant(&timed(Outcome::Survived, None, 9_000)); + events.mutant(&timed( + Outcome::OutOfMemory, + Some("peaked at 2.1 GiB against a 512 MiB ceiling"), + 9_000, + )); + + let err = String::from_utf8(host.err).expect("utf-8"); + + assert!(err.contains("SURVIVED"), "{err}"); + assert!(err.contains("peaked at 2.1 GiB"), "{err}"); + assert!(!err.contains("after"), "{err}"); + } + + /// The phase verbs all reach the display, and a caught mutant is left for the summary. + #[test] + fn phase_events_are_displayed_and_ordinary_verdicts_are_not_announced() { + let mut host = Sink::default().terminal(80); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(true, Styler::new(false), Some(80)), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.phase("Baseline", "measuring the suite"); + events.begin("Building", "Built", "the test binaries"); + events.end(", done"); + events.outcome("withdrew 2 mutants"); + events.mutant(&mutant(Outcome::Killed, None)); + + let err = host.err(); + + assert!(err.contains("Baseline"), "{err}"); + assert!(err.contains("Building"), "{err}"); + assert!(err.contains("withdrew 2 mutants"), "{err}"); + assert!(!err.contains("SURVIVED"), "{err}"); + } + + /// The display is off wherever output is piped, which is where a CI log is written — so a build + /// the user asked to see by name has to be printed anyway, or the flag does nothing in the one + /// place it was asked for. + #[test] + fn requested_build_output_survives_a_display_that_is_off() { + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(false, Styler::new(false), None), + styler: Styler::new(false), + estimate: false, + show_build: true, + verdict_log: VerdictLog::default(), + }; + + events.build_output("warning: unused variable: `x`"); + + assert!(host.err().contains("warning: unused variable: `x`"), "{}", host.err()); + } + + /// The question asked before the work is answered by the same flag that gates the work's result. + /// + /// `wants_build_output` exists so that the megabytes of cargo's JSON stream are not decoded to + /// produce a line that `build_output` is about to drop. That is only sound while the two agree: + /// a `wants` that said yes when `build_output` discards would cost the parse it is there to + /// avoid, and one that said no when `build_output` would have shown the line would silently + /// empty `--show-build`. + #[test] + fn asking_whether_build_output_is_wanted_agrees_with_what_is_done_with_it() { + for show_build in [false, true] { + let mut host = Sink::default(); + + { + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(false, Styler::new(false), None), + styler: Styler::new(false), + estimate: false, + show_build, + verdict_log: VerdictLog::default(), + }; + + assert_eq!(events.wants_build_output(), show_build); + events.build_output("warning: unused variable: `x`"); + } + + assert_eq!(!host.err().is_empty(), show_build, "{}", host.err()); + } + } + + /// And an unrequested build stays unrequested: the flag is what turns cargo's narration on, not + /// the state of the display. + #[test] + fn build_output_stays_hidden_when_it_was_not_asked_for() { + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(false, Styler::new(false), None), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.build_output("warning: unused variable: `x`"); + + assert!(host.err().is_empty(), "{}", host.err()); + } + + /// A warning about what a run is about to cost is worth nothing if it is only shown when + /// someone is watching. The display turns itself off wherever output is piped, and a CI job is + /// exactly where a run that quietly takes six hours is least affordable. + #[test] + fn a_warning_survives_a_display_that_is_off() { + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(false, Styler::new(false), None), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.warn("router_compile_fail runs the compiler\nexclude it with --exclude-test router_compile_fail"); + + assert!(host.err().contains("router_compile_fail runs the compiler"), "{}", host.err()); + assert!(host.err().contains("--exclude-test router_compile_fail"), "{}", host.err()); + } + + /// A multi-line warning is written line by line, because the status column is per line and a + /// message written as one string would put the second line under the label rather than beside + /// it. + #[test] + fn a_multi_line_warning_is_labelled_on_every_line() { + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(false, Styler::new(false), None), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.warn("first\nsecond"); + + assert_eq!( + host.err().lines().filter(|line| line.contains("warning")).count(), + 2, + "{}", + host.err() + ); + } + + #[test] + fn the_testing_progress_log_is_truncated_and_flushed_after_every_verdict() { + let directory = crate::testing::workdir("testing-progress-log-"); + let scratch = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the test path is UTF-8"); + let path = scratch.join(TESTING_PROGRESS_LOG); + + std::fs::write(path.as_std_path(), "stale verdict\n").expect("seed the old log"); + + let mut host = Sink::default(); + let mut events = ConsoleEvents { + host: &mut host, + progress: Progress::new(false, Styler::new(false), None), + styler: Styler::new(false), + estimate: false, + show_build: false, + verdict_log: VerdictLog::default(), + }; + + events.testing_log(&scratch).expect("create the verdict log"); + events.mutant(&mutant(Outcome::Killed, None)); + + let first = std::fs::read_to_string(path.as_std_path()).expect("read the still-open log"); + + assert!(first.contains("killed"), "{first}"); + assert!(!first.contains("stale verdict"), "{first}"); + + for outcome in [ + Outcome::Survived, + Outcome::Timeout, + Outcome::OutOfMemory, + Outcome::Flaky, + Outcome::NoCoverage, + Outcome::CompileError, + Outcome::Ignored, + Outcome::NotBuilt, + ] { + events.mutant(&mutant(outcome, Some("verdict detail"))); + } + + events.finish_verdict_log().expect("finish the verdict log"); + + let text = std::fs::read_to_string(path.as_std_path()).expect("read the verdict log"); + + for label in [ + "killed", + "SURVIVED", + "TIMEOUT", + "OUTOFMEM", + "FLAKY", + "uncovered", + "unviable", + "skipped", + "notbuilt", + ] { + assert!(text.contains(label), "{label} was absent from:\n{text}"); + } + assert!(!text.contains("\u{1b}["), "the durable log contains terminal styling: {text:?}"); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/dispatch.rs b/crates/cargo-gamma-lib/src/commands/dispatch.rs new file mode 100644 index 000000000..c5138edab --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/dispatch.rs @@ -0,0 +1,829 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::panic::AssertUnwindSafe; +use std::ffi::OsString; +use std::io::Write; +use std::panic; + +use clap::Parser; +use clap::error::ErrorKind; + +use super::clean::clean; +use super::cli::{Cli, Command, SelectArgs}; +use super::completions::completions; +use super::explain::explain; +use super::hints::hints_with_cargo; +use super::host::Host; +use super::list::list_with_cargo; +use super::merge::merge; +use super::run::{configure, run_session}; +use super::suppress::suppress; +use super::unsuppress::unsuppress_with_cargo; +use crate::config::Config; +use crate::error::error; +use crate::report::Styler; + +/// The multiple of each test binary's baseline duration a mutant is allowed when nothing says otherwise. +/// +/// Tight on purpose: a mutant that hangs costs its whole budget, and that budget is paid once per +/// hang across the population, so a generous multiplier is one of the few ways a run can still take +/// far longer than the cost model predicts. The floor keeps a fast suite from reading scheduler +/// noise as a hang, and stall detection catches the common hangs long before the budget expires. +pub(super) const DEFAULT_TEST_TIMEOUT_MULTIPLIER: f64 = 1.5; + +/// Exit code for a run in which every gate passed. +pub const EXIT_OK: i32 = 0; + +/// Exit code for a usage error: bad arguments or bad configuration. +pub const EXIT_USAGE: i32 = 1; + +/// Exit code for a run that completed but in which some gate failed. +pub const EXIT_GATE_FAILED: i32 = 2; + +/// Exit code for a run that could not proceed. +pub const EXIT_CANNOT_PROCEED: i32 = 3; + +/// Exit code for a bug in this tool: a panic that reached the top of the CLI. +/// +/// Distinct from the three above because it is the one code that says nothing about the code under +/// test. A job that reads a `3` retries the build or reports a broken workspace; a job that reads +/// this should file a bug, and leaving a panic to Rust's default `101` puts it in the same bucket +/// as a test binary that aborted, which is a thing the tool is supposed to be able to observe. +/// +/// The value follows `sysexits.h`'s `EX_SOFTWARE`. +pub const EXIT_INTERNAL: i32 = 70; + +/// Runs the tool and returns the process exit code. +/// +/// This returns rather than exits so that every path through the CLI, including the failure paths +/// and the exit codes themselves, is reachable from an ordinary integration test. +pub fn run(host: &mut H, args: impl IntoIterator + Clone>) -> i32 { + #[cfg(windows)] + cargo_gamma_unsafe::job::suppress_error_dialogs(); + + let notes = crate::notes::Run::new(); + let _notes = crate::notes::enter(Some(¬es)); + + // A panic anywhere below here is a bug in this tool rather than a finding about the code under + // test, and it is the only outcome the three ordinary codes cannot express. Caught here rather + // than left to unwind out of `main` so that the distinction the documentation promises actually + // reaches a caller: without this the process exits `101`, which a CI job cannot tell from a + // test binary that aborted. + // + // The payload is dropped rather than reported. The default hook has already written the + // message and the location to stderr by the time this runs, and `host` is not usable from + // inside the unwind — its streams are exactly what a panic here may have left half-written. + // + // `AssertUnwindSafe` because `host` is a `&mut` and the compiler cannot know what state a panic + // leaves it in. Nothing reads it afterwards: this branch returns the code and nothing else. + match panic::catch_unwind(AssertUnwindSafe(|| dispatched(host, args))) { + Ok(code) => code, + Err(_payload) => EXIT_INTERNAL, + } +} + +/// Runs the tool, letting a panic escape. +/// +/// Split out of [`run`] only so that the catch has a single call to wrap, which keeps the `?`-less +/// early returns below readable. +fn dispatched(host: &mut H, args: impl IntoIterator + Clone>) -> i32 { + let normalized = normalize(args); + + let cli = match Cli::try_parse_from(normalized) { + Ok(cli) => cli, + + Err(cause) => { + // clap renders help and version to stdout and errors to stderr, matching cargo. + let is_help = matches!(cause.kind(), ErrorKind::DisplayHelp | ErrorKind::DisplayVersion); + + let text = cause.render().ansi().to_string(); + + if is_help { + let _ = write!(host.output(), "{text}"); + return EXIT_OK; + } + + let _ = write!(host.error(), "{text}"); + return EXIT_USAGE; + } + }; + + let styler = Styler::new(cli.color.resolve(host.is_terminal())); + + let code = match dispatch(host, cli, styler) { + Ok(code) => code, + + Err(cause) => { + let code = if cause.is_usage() { EXIT_USAGE } else { EXIT_CANNOT_PROCEED }; + let label = styler.error("error:"); + let mut stream = host.error(); + + let _ = writeln!(stream, "{label} {cause}"); + + code + } + }; + + say_notes(host, styler); + + code +} + +/// Says whatever was raised from below the output seam, through the `Host` like everything else. +/// +/// Said after the command rather than as it happens: workers and low-level file publishers can +/// discover a warning while the progress display owns the terminal. Said on the way out of a +/// failed command too, because a warning about incomplete output is often the context that makes +/// the failure legible. +fn say_notes(host: &mut H, styler: Styler) { + let notes = crate::notes::drain(); + + if notes.is_empty() { + return; + } + + let label = styler.warning(); + let mut stream = host.error(); + + for note in notes { + let _ = writeln!(stream, "{label} {note}"); + } +} + +/// Strips the argument cargo inserts when invoking a subcommand. +/// +/// Invoked as `cargo gamma ...`, the process sees `["cargo-gamma", "gamma", ...]`. Invoked +/// directly as `cargo-gamma ...` it does not. Both must work, so drop a second argument that is +/// exactly `gamma` and nothing else. +fn normalize(args: impl IntoIterator + Clone>) -> Vec { + let mut normalized: Vec = args.into_iter().map(Into::into).collect(); + + if normalized.get(1).is_some_and(|entry| entry == "gamma") { + let _ = normalized.remove(1); + } + + if !normalized.is_empty() && implies_run(normalized.get(1..).unwrap_or_default()) { + normalized.insert(1, "run".into()); + } + + normalized +} + +/// The top-level options that may legitimately appear before a subcommand. +/// +/// Each takes one value, which has to be stepped over when looking for the subcommand. +const GLOBAL_OPTIONS: [&str; 2] = ["--color", "--progress"]; + +/// Whether `args` is a bare `run` invocation with the word `run` left off. +/// +/// The rule is deliberately shallow: after stepping over the global options, an argument that +/// begins with a dash cannot be a subcommand, so `run` is what was meant. Anything else is left +/// exactly as written, including a misspelled subcommand — clap's "did you mean" is far more useful +/// there than an unexpected-argument error from a `run` the user never asked for. +fn implies_run(args: &[OsString]) -> bool { + let mut rest = args; + + while let Some(first) = rest.first().and_then(|entry| entry.to_str()) { + if GLOBAL_OPTIONS + .iter() + .any(|option| first.strip_prefix(option).is_some_and(|rest| rest.starts_with('='))) + { + rest = &rest[1..]; + } else if GLOBAL_OPTIONS.contains(&first) { + // The value is skipped along with the option, or a `--color never merge` would look + // like it begins with the word `never`. + rest = rest.get(2..).unwrap_or_default(); + } else { + break; + } + } + + let Some(first) = rest.first().and_then(|entry| entry.to_str()) else { + // Nothing at all means a default run, which is the shortest path into the tool. + return true; + }; + + // Help and version are answered by the top-level parser; routing them through `run` would print + // that subcommand's page instead of the overview the user asked for. + first.starts_with('-') && !matches!(first, "-h" | "--help" | "-V" | "--version") +} + +/// Re-runs this process inside a delegated cgroup when that is what memory control is missing. +/// +/// Placed here, ahead of every command that executes tests, because a relaunch has to happen before +/// any work is done: the point is for the new process to do the run, and a process that has already +/// built a scratch tree would either duplicate it or hand over half-finished state. +/// +/// Returns the exit code of the relaunched run when there was one, and `None` when the run should +/// continue in this process — which covers both "no relaunch was needed" and "no relaunch was +/// possible". The distinction between those two is deliberately not made here. When it was not +/// needed there is nothing to say, and when it was not possible the run continues to +/// `admit_memory_control`, which already explains the absence of a ceiling and already decides +/// whether that is an error or a degradation based on whether the user asked for one. +#[cfg(target_os = "linux")] +fn relaunch_for_memory_control(host: &H, args: &super::cli::RunArgs) -> Option { + use crate::exec::relaunch; + + if !host.may_replace_process() || args.measure.no_relaunch || relaunch::relaunched() { + return None; + } + + // Asking the host first means a machine that already delegates a cgroup — the ordinary case on + // a desktop Linux session — never spawns a second process to discover it did not need one. + if !super::run::memory_policy(args).measuring() || crate::exec::memory_support().is_ok() { + return None; + } + + relaunch::relaunch().ok().flatten() +} + +/// Runs the parsed command. +/// +/// The configuration file is folded into the arguments here rather than inside each command, so +/// there is exactly one place where precedence between the file and the command line is decided — +/// and exactly one place where the settings that can arrive half from each side are checked. +pub(super) fn dispatch(host: &mut H, cli: Cli, styler: Styler) -> crate::Result { + match cli.command { + Command::Run(mut args) => { + configure(host, &mut args, styler)?; + check_shard(&args.select)?; + + #[cfg(target_os = "linux")] + if let Some(code) = relaunch_for_memory_control(host, &args) { + return Ok(code); + } + + run_session(host, &args, cli.progress, styler) + } + + Command::List(mut args) => { + let config = Config::resolve(&args.select)?; + let cargo = config.cargo_options(); + config.apply_selection(&mut args.select)?; + check_shard(&args.select)?; + list_with_cargo(host, &args, styler, &cargo) + } + + Command::Explain(args) => explain(host, &args), + Command::Suppress(mut args) => { + configure(host, &mut args.run, styler)?; + check_shard(&args.run.select)?; + + #[cfg(target_os = "linux")] + if let Some(code) = relaunch_for_memory_control(host, &args.run) { + return Ok(code); + } + + suppress(host, &args, cli.progress, styler) + } + + Command::Unsuppress(mut args) => { + let config = Config::resolve(&args.select)?; + let cargo = config.cargo_options(); + config.apply_selection(&mut args.select)?; + check_shard(&args.select)?; + unsuppress_with_cargo(host, &args, styler, &cargo) + } + + Command::Merge(args) => merge(host, &args, styler), + + Command::Hints(mut args) => { + // Before the merge, so that a shard count sitting in the committed configuration for + // the benefit of the matrix does not stop the one job that promotes the artifact. + refuse_shard(&args.select)?; + + let config = Config::resolve(&args.select)?; + let cargo = config.cargo_options(); + config.apply_selection(&mut args.select)?; + hints_with_cargo(host, &args, styler, &cargo) + } + + Command::Clean(args) => clean(host, &args, styler), + + Command::Completions(args) => Ok(completions(host, &args)), + } +} + +/// Checks the settings that can arrive half from the command line and half from the file. +/// +/// The shard count and index are one such setting, and the only one: the count belongs in the +/// committed configuration, because every job in a matrix has to agree on it, while the index +/// differs per job and arrives on the command line. Neither source holds the whole pair, so the +/// check cannot be made while the command line is parsed — which is why the pair is validated by +/// [`SelectArgs::shard`] rather than by clap. +/// +/// Made here, immediately after the merge, so that no command that *uses* a shard can reach its work +/// without it. On `run` in particular that is ahead of the relaunch into a cgroup: a shard nobody +/// can honour should cost a second, not a second process and a build. `hints` is the one command +/// that uses no shard at all, and it refuses one outright rather than checking it — see +/// [`refuse_shard`]. +fn check_shard(select: &SelectArgs) -> crate::Result<()> { + let _shard = select.shard()?; + + Ok(()) +} + +/// Refuses the shard flags on the one command that deliberately ignores them. +/// +/// `hints` promotes an artifact from the whole population, because a shard sees a fraction of it and +/// promoting from one would publish an almost-empty artifact while every other job in the matrix +/// overwrote it. That is the right behaviour, but accepting the flags and then ignoring them is not +/// how to have it: a user who adds `hints` to an existing sharded matrix step, where +/// `--shard-index ${{ matrix.i }}` is already on the line, would get exactly the race the design +/// avoids and no diagnostic at all. Refusing says which command they want instead. +fn refuse_shard(select: &SelectArgs) -> crate::Result<()> { + if select.shard_count.is_none() && select.shard_index.is_none() { + return Ok(()); + } + + Err(error!( + "`hints` is deliberately unsharded: it promotes from the whole population, because a shard sees a fraction of it and every job in the matrix would race to overwrite the artifact. Drop `--shard-count` and `--shard-index`, and promote from one job rather than all of them" + ) + .usage()) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::fs; + use std::sync::Barrier; + + use super::*; + use crate::testing::Sink as TestSink; + + struct ClosedOutput { + err: Vec, + } + + impl Host for ClosedOutput { + fn output(&mut self) -> impl Write { + crate::testing::Broken + } + + fn error(&mut self) -> impl Write { + &mut self.err + } + + fn is_terminal(&self) -> bool { + false + } + + fn terminal_width(&self) -> Option { + None + } + } + + #[test] + fn a_closed_results_pipe_ends_a_listing_successfully_without_a_diagnostic() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-broken-pipe-", None); + let root = dir.path().to_string_lossy().into_owned(); + let mut host = ClosedOutput { err: Vec::new() }; + + let code = run(&mut host, ["cargo-gamma", "gamma", "list", "mutants", "--dir", &root]); + + assert_eq!(code, EXIT_OK); + assert!(host.err.is_empty(), "{}", String::from_utf8_lossy(&host.err)); + }); + } + + /// A panic below the top of the CLI is reported as a bug in this tool, not as a finding. + /// + /// Regression: an escaping panic exited `101`, which is also what a test binary that aborted + /// under the tool exits with, so a CI job could not tell "cargo-gamma is broken" from + /// "the code under test is broken" — the one distinction the exit codes exist to draw. + #[test] + fn a_panic_inside_the_tool_is_reported_as_an_internal_error_rather_than_escaping() { + let previous = panic::take_hook(); + + // The default hook would write this deliberate panic's message and backtrace to the + // suite's stderr, where it reads as a failure. Silenced only for the call below, and put + // back immediately, because the hook is process-wide. + panic::set_hook(Box::new(|_| {})); + + let code = run(&mut Exploding, ["cargo-gamma", "gamma", "list", "files"]); + + panic::set_hook(previous); + + assert_eq!(code, EXIT_INTERNAL, "a panic reached the caller as something other than a tool bug"); + } + + /// A host that panics the moment the CLI asks it anything. + /// + /// Stands in for any bug below [`run`]: what is being pinned is that the boundary catches, not + /// that this particular question is dangerous. + struct Exploding; + + impl Host for Exploding { + fn output(&mut self) -> impl Write { + Vec::new() + } + + fn error(&mut self) -> impl Write { + Vec::new() + } + + fn is_terminal(&self) -> bool { + panic!("a bug in the tool") + } + + fn terminal_width(&self) -> Option { + None + } + } + + /// A crate small enough to enumerate mutants in, with whatever configuration the test needs. + fn crate_dir(name: &str, config: Option<&str>) -> tempfile::TempDir { + let (dir, root) = crate::fixtures::crate_dir(name, "pub fn less(a: i32, b: i32) -> bool { a < b }\n"); + + if let Some(text) = config { + fs::write(root.join("gamma.toml"), text).expect("config"); + } + + dir + } + + /// The split a CI matrix wants: the width is committed and the slice is per job. Clap cannot + /// judge this before the file has been read, so the check runs on the merged values. + #[test] + fn a_count_from_the_file_and_an_index_from_the_command_line_run() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-shard-split-", Some("[shard]\ncount = 3\n")); + let root = dir.path().to_string_lossy().into_owned(); + let mut host = crate::testing::Sink::default(); + + let code = run( + &mut host, + ["cargo-gamma", "gamma", "list", "mutants", "--dir", &root, "--shard-index", "1"], + ); + + assert_eq!(code, EXIT_OK, "{}", host.err()); + }); + } + + /// And the half pair that nothing completes is refused, rather than quietly running the whole + /// population under a name that says it ran a third of it. + #[test] + fn a_shard_count_with_nothing_to_complete_it_is_a_usage_error() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-shard-half-", Some("[shard]\ncount = 3\n")); + let root = dir.path().to_string_lossy().into_owned(); + let mut host = crate::testing::Sink::default(); + + let code = run(&mut host, ["cargo-gamma", "gamma", "list", "mutants", "--dir", &root]); + + assert_eq!(code, EXIT_USAGE, "{}", host.out()); + assert!(host.err().contains("--shard-index"), "{}", host.err()); + assert!(host.out().is_empty(), "a rejected shard listed mutants anyway: {}", host.out()); + }); + } + + /// The same, typed on the command line: parsing accepts it now, and the effective-value check + /// is what refuses it. + #[test] + fn half_a_shard_on_the_command_line_is_refused_by_the_command_rather_than_the_parser() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-shard-cli-half-", None); + let root = dir.path().to_string_lossy().into_owned(); + let mut host = crate::testing::Sink::default(); + + let code = run( + &mut host, + ["cargo-gamma", "gamma", "list", "mutants", "--dir", &root, "--shard-index", "1"], + ); + + assert_eq!(code, EXIT_USAGE, "{}", host.out()); + assert!(host.err().contains("--shard-count"), "{}", host.err()); + }); + } + + /// The command that promotes an artifact from the whole population has no use for a shard, and + /// accepting the flags anyway would let someone add it to a sharded matrix step and get every + /// job overwriting the same file — the race the design already refuses to run. + #[test] + fn a_sharded_hints_is_refused_rather_than_quietly_promoting_from_everything() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-shard-hints-", None); + let root = dir.path().to_string_lossy().into_owned(); + let mut host = crate::testing::Sink::default(); + + let code = run( + &mut host, + [ + "cargo-gamma", + "gamma", + "hints", + "--dir", + &root, + "--shard-index", + "1", + "--shard-count", + "4", + ], + ); + + assert_eq!(code, EXIT_USAGE, "{}", host.out()); + assert!(host.err().contains("deliberately unsharded"), "{}", host.err()); + }); + } + + /// A `gamma.toml` selection key reaches a discovery command. + /// + /// `list mutators` narrows to what the file selects, which is only possible if the `List` arm + /// folds the file's `mutators` in with `apply_selection` before listing. Drop that call and the + /// default family leaks through, marking `relational.lt_to_gt` enabled again — so this is a + /// direct, behavioural pin on the arm rather than an inspection of its source. + #[test] + fn a_config_selection_key_narrows_a_listing() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-config-list-", Some("mutators = [\"relational.lt_to_le\"]\n")); + let root = dir.path().to_string_lossy().into_owned(); + let mut host = crate::testing::Sink::default(); + + let code = run(&mut host, ["cargo-gamma", "gamma", "list", "mutators", "--json", "--dir", &root]); + assert_eq!(code, EXIT_OK, "{}", host.err()); + + let entries: Vec = serde_json::from_str(&host.out()).expect("the listing is JSON"); + let enabled = |name: &str| { + entries + .iter() + .find(|entry| entry["name"] == name) + .and_then(|entry| entry["enabled"].as_bool()) + }; + + assert_eq!(enabled("relational.lt_to_le"), Some(true), "{}", host.out()); + assert_eq!(enabled("relational.lt_to_gt"), Some(false), "{}", host.out()); + }); + } + + /// Exactly the discovery commands fold `gamma.toml` selection keys in before discovery; + /// `explain` deliberately does not. + /// + /// A `packages` key in the file, together with `--workspace` on the command line, is a + /// contradiction that only `apply_selection` (through `validate_effective`) catches — clap sees + /// only the flag. So a command that reaches that diagnostic is one that folded the file's + /// selection in, and one that runs clean is one that did not. Removing `apply_selection` from + /// the `list`, `unsuppress`, or `hints` arm, or adding it to `explain`, flips exactly one of + /// these assertions. It stops at the merge, before any build, so it stays a cheap unit test. + #[test] + fn only_the_discovery_commands_apply_config_selection() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-config-selection-", Some("packages = [\"subject\"]\n")); + let root = dir.path().to_string_lossy().into_owned(); + + for command in ["list", "unsuppress", "hints"] { + let mut host = crate::testing::Sink::default(); + let code = run(&mut host, ["cargo-gamma", "gamma", command, "--workspace", "--dir", &root]); + + assert_eq!(code, EXIT_USAGE, "`{command}` did not fold in the file's selection: {}", host.err()); + assert!(host.err().contains("packages"), "`{command}`: {}", host.err()); + assert!(host.err().contains("workspace"), "`{command}`: {}", host.err()); + } + + // `explain` resolves a named subject, not a selection, so the file never reaches it and + // the whole `relational` family is still explained. + let mut host = crate::testing::Sink::default(); + let code = run(&mut host, ["cargo-gamma", "gamma", "explain", "relational"]); + + assert_eq!(code, EXIT_OK, "explain applied config selection: {}", host.err()); + assert!(host.out().contains("relational.lt_to_le"), "{}", host.out()); + assert!(host.out().contains("relational.lt_to_gt"), "{}", host.out()); + }); + } + + /// An unsatisfiable shard has to be caught before the run does anything, which for `run` means + /// before the build and before this process would relaunch itself into a cgroup. + #[test] + fn a_run_with_an_impossible_shard_stops_before_it_builds() { + crate::notes::alone(|| { + let dir = crate_dir("dispatch-shard-run-", Some("[shard]\ncount = 2\n")); + let root = dir.path().to_string_lossy().into_owned(); + let mut host = crate::testing::Sink::default(); + + let code = run(&mut host, ["cargo-gamma", "gamma", "run", "--dir", &root, "--shard-index", "9"]); + + assert_eq!(code, EXIT_USAGE, "{}", host.out()); + assert!(host.err().contains("out of range"), "{}", host.err()); + }); + } + + /// The whole point of the shell scripts is that they reach the results stream. + #[test] + fn completions_are_dispatched_to_the_results_stream() { + crate::notes::alone(|| { + let mut host = crate::testing::Sink::default(); + + let code = run(&mut host, ["cargo-gamma", "gamma", "completions", "bash"]); + + assert_eq!(code, EXIT_OK); + assert!(host.out().contains("cargo-gamma"), "{}", host.out()); + }); + } + + /// A diagnostic raised from below the output seam still comes out through the `Host`. + /// + /// This is what the seam buys: the two places that have something to say and no `Host` in reach + /// — the cargo command line being assembled, and a verdict being reached on a worker thread — + /// — would otherwise write to `stderr` themselves, bypassing the colour, the width and the + /// progress display that everything else respects, and staying invisible to a test like this one. + #[test] + fn a_note_raised_below_the_seam_is_said_through_the_host() { + crate::notes::alone(|| { + let mut host = crate::testing::Sink::default(); + + crate::notes::note("something worth saying"); + + say_notes(&mut host, Styler::new(false)); + + assert!(host.err().contains("something worth saying"), "{}", host.err()); + assert!(host.err().contains("warning"), "{}", host.err()); + assert!(host.out().is_empty(), "a diagnostic reached the results stream: {}", host.out()); + }); + } + + /// Every note is said, not only the first: they describe different events. + #[test] + fn every_pending_note_is_said() { + crate::notes::alone(|| { + let mut host = crate::testing::Sink::default(); + + crate::notes::note("the first"); + crate::notes::note("the second"); + + say_notes(&mut host, Styler::new(false)); + + assert_eq!(host.err().lines().count(), 2, "{}", host.err()); + }); + } + + /// Saying them is what hands them over, so a second command does not repeat the first's. + #[test] + fn a_note_is_said_once_and_not_again_by_the_next_command() { + crate::notes::alone(|| { + let mut first = crate::testing::Sink::default(); + let mut second = crate::testing::Sink::default(); + + crate::notes::note("said once"); + + say_notes(&mut first, Styler::new(false)); + say_notes(&mut second, Styler::new(false)); + + assert!(first.err().contains("said once"), "{}", first.err()); + assert!(second.err().is_empty(), "{}", second.err()); + }); + } + + #[test] + fn concurrent_hosts_receive_only_the_notes_from_their_own_runs() { + let first = crate::notes::Run::new(); + let second = crate::notes::Run::new(); + let ready = Barrier::new(2); + + std::thread::scope(|scope| { + let first = first.clone(); + let first_ready = &ready; + let left = scope.spawn(move || { + let _notes = crate::notes::enter(Some(&first)); + let mut host = TestSink::default(); + + crate::notes::note("first run"); + let _ready = first_ready.wait(); + say_notes(&mut host, Styler::new(false)); + + String::from_utf8(host.err).expect("note output is UTF-8") + }); + + let second = second.clone(); + let second_ready = &ready; + let right = scope.spawn(move || { + let _notes = crate::notes::enter(Some(&second)); + let mut host = TestSink::default(); + + crate::notes::note("second run"); + let _ready = second_ready.wait(); + say_notes(&mut host, Styler::new(false)); + + String::from_utf8(host.err).expect("note output is UTF-8") + }); + + let left = left.join().expect("first run"); + let right = right.join().expect("second run"); + + assert!(left.contains("first run"), "{left}"); + assert!(!left.contains("second run"), "{left}"); + assert!(right.contains("second run"), "{right}"); + assert!(!right.contains("first run"), "{right}"); + }); + } + + /// A run with nothing to add says nothing, rather than an empty label on an empty line. + #[test] + fn a_command_that_raised_no_note_writes_nothing() { + crate::notes::alone(|| { + let mut host = crate::testing::Sink::default(); + + say_notes(&mut host, Styler::new(false)); + + assert!(host.err().is_empty(), "{}", host.err()); + }); + } + + #[test] + fn cargos_inserted_argument_is_stripped() { + let normalized = normalize(["cargo-gamma", "gamma", "list"]); + + assert_eq!(normalized, vec!["cargo-gamma", "list"]); + } + + #[test] + fn direct_invocation_is_left_alone() { + let normalized = normalize(["cargo-gamma", "list"]); + + assert_eq!(normalized, vec!["cargo-gamma", "list"]); + } + + #[test] + fn only_the_second_argument_named_gamma_is_stripped() { + let normalized = normalize(["cargo-gamma", "list", "gamma"]); + + assert_eq!(normalized, vec!["cargo-gamma", "list", "gamma"]); + } + + #[test] + fn an_empty_argument_list_does_not_panic() { + assert!(normalize(Vec::::new()).is_empty()); + } + + #[test] + fn a_bare_invocation_implies_run() { + assert_eq!(normalize(["cargo-gamma", "gamma"]), vec!["cargo-gamma", "run"]); + } + + #[test] + fn a_leading_option_implies_run() { + // The top level accepts no options of its own beyond the two globals, so an option here can + // only have been meant for `run`. + assert_eq!( + normalize(["cargo-gamma", "gamma", "--mutators", "relational"]), + vec!["cargo-gamma", "run", "--mutators", "relational"] + ); + } + + #[test] + fn a_named_subcommand_is_not_second_guessed() { + for command in ["run", "list", "explain", "suppress", "merge", "help"] { + assert_eq!(normalize(["cargo-gamma", "gamma", command]), vec!["cargo-gamma", command]); + } + } + + #[test] + fn a_misspelled_subcommand_is_left_for_clap_to_diagnose() { + // Wrapping it in `run` would turn "did you mean `merge`?" into an unexpected-value error + // about a subcommand the user did name. + assert_eq!(normalize(["cargo-gamma", "gamma", "mrege"]), vec!["cargo-gamma", "mrege"]); + } + + #[test] + fn help_and_version_stay_at_the_top_level() { + for flag in ["-h", "--help", "-V", "--version"] { + assert_eq!(normalize(["cargo-gamma", "gamma", flag]), vec!["cargo-gamma", flag]); + } + } + + #[test] + fn a_global_option_before_a_subcommand_is_stepped_over() { + assert_eq!( + normalize(["cargo-gamma", "gamma", "--color", "never", "merge", "a.json"]), + vec!["cargo-gamma", "--color", "never", "merge", "a.json"] + ); + assert_eq!( + normalize(["cargo-gamma", "gamma", "--progress=never", "merge", "a.json"]), + vec!["cargo-gamma", "--progress=never", "merge", "a.json"] + ); + } + + #[test] + fn a_global_option_before_no_subcommand_still_implies_run() { + assert_eq!( + normalize(["cargo-gamma", "gamma", "--color", "never", "--mutators", "stmt"]), + vec!["cargo-gamma", "run", "--color", "never", "--mutators", "stmt"] + ); + } + + #[test] + fn a_dangling_global_option_does_not_panic() { + assert_eq!( + normalize(["cargo-gamma", "gamma", "--color"]), + vec!["cargo-gamma", "run", "--color"] + ); + } + + #[test] + fn a_global_option_with_value_before_run_options_still_implies_run() { + assert_eq!( + normalize(["cargo-gamma", "gamma", "--progress=never", "--mutators", "relational"]), + vec!["cargo-gamma", "run", "--progress=never", "--mutators", "relational"] + ); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/explain.rs b/crates/cargo-gamma-lib/src/commands/explain.rs new file mode 100644 index 000000000..8629fbafb --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/explain.rs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::io::Write; + +use super::cli::ExplainArgs; +use super::dispatch::EXIT_OK; +use super::host::Host; +use crate::ops::registry; + +/// Implements `explain`. +pub(super) fn explain(host: &mut H, args: &ExplainArgs) -> crate::Result { + let names = registry::resolve(&args.subject)?; + let mut stream = host.results(); + + for name in names { + let Some(mutator) = registry::find(name) else { + continue; + }; + + writeln!(stream, "{}", mutator.name)?; + writeln!(stream, " {}", mutator.description)?; + writeln!(stream, " enabled by default: {}", if mutator.default_on { "yes" } else { "no" })?; + + if !mutator.aliases.is_empty() { + writeln!(stream, " also known as: {}", mutator.aliases.join(", "))?; + } + + writeln!(stream, " suppress with: // #[gamma::skip({})]", mutator.name)?; + writeln!(stream)?; + } + + Ok(EXIT_OK) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::{BrokenHost, Sink}; + + #[test] + fn explanation_names_aliases_and_suppressions() { + let mut host = Sink::default(); + + let code = explain( + &mut host, + &ExplainArgs { + subject: "fn_value.default".to_owned(), + }, + ) + .expect("explain"); + let text = String::from_utf8(host.out).expect("utf-8"); + + assert_eq!(code, EXIT_OK); + assert!(text.contains("also known as: RV"), "{text}"); + assert!(text.contains("suppress with: // #[gamma::skip(fn_value.default)]"), "{text}"); + } + + /// Piping into a consumer that exits early is successful consumption. + #[test] + fn a_closed_output_stream_ends_explanation_successfully() { + let code = explain( + &mut BrokenHost, + &ExplainArgs { + subject: "relational".to_owned(), + }, + ) + .expect("closed pipe"); + + assert_eq!(code, EXIT_OK); + } + + /// A mutator with no academic alias simply omits the line rather than printing an empty one. + #[test] + fn a_mutator_without_aliases_omits_the_alias_line() { + let named: Vec<&str> = registry::REGISTRY + .iter() + .filter(|mutator| mutator.aliases.is_empty()) + .map(|mutator| mutator.name) + .collect(); + + assert!(!named.is_empty(), "the registry should have at least one unaliased mutator"); + + for name in named { + let mut host = Sink::default(); + + let _code = explain(&mut host, &ExplainArgs { subject: name.to_owned() }).expect("explain"); + + assert!(!host.out().contains("also known as"), "{}", host.out()); + } + } + + /// A selector naming a whole family explains every mutator in it. + #[test] + fn a_family_selector_explains_each_of_its_mutators() { + let mut host = Sink::default(); + + let code = explain( + &mut host, + &ExplainArgs { + subject: "relational".to_owned(), + }, + ) + .expect("explain"); + + assert_eq!(code, EXIT_OK); + assert!( + host.out().lines().filter(|line| line.starts_with("relational.")).count() > 1, + "{}", + host.out() + ); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/hints.rs b/crates/cargo-gamma-lib/src/commands/hints.rs new file mode 100644 index 000000000..3a8831805 --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/hints.rs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::io::Write; + +use super::cli::HintsArgs; +use super::dispatch::EXIT_OK; +use super::host::Host; +use crate::discover::{Hints, RunRecord, hints_path}; +use crate::exec::{CargoOptions, gamma_base}; +use crate::report::{Styler, quantity}; + +/// Implements `hints`. +/// +/// Promoting rather than copying is the whole point of the command existing. A documented `cp` from +/// the scratch record would put a file in version control with a format nobody owns, no version, no +/// provenance, entries for mutants that were edited away three months ago, and — worst — every +/// verdict the record holds, which the next run would then be reading from somebody else's machine. +/// Everything this does that a copy does not is one of those problems: +/// +/// - it admits only the tiers that cannot move a score, and drops the rest on the floor; +/// - it joins against the population as it stands now, so the file does not grow forever; +/// - it stamps the format version and the tool that wrote it; +/// - it writes atomically and reads back what it wrote. +/// +/// Errors are surfaced rather than absorbed, which is the reverse of how the artifact is read. A +/// run consults the file automatically and must never fail over it, so reading is best-effort to +/// the point of treating a corrupt file as an empty one. Writing it is something somebody asked +/// for, and a promotion that quietly did nothing would be discovered as a slow CI run weeks later. +/// Implements `hints` with the configuration generation dispatch already resolved. +pub(super) fn hints_with_cargo(host: &mut H, args: &HintsArgs, styler: Styler, cargo: &CargoOptions) -> crate::Result { + let selection = args.select.selection()?; + + // Deliberately unsharded. A shard sees a fraction of the population, so promoting from one + // would drop every hint outside it — the file would be correct and almost empty, and each shard + // in a matrix would fight the others over it. + let plan = crate::discover::plan_for_build(&args.select, &selection, None, cargo, &mut |_| {})?; + + let base = gamma_base(&plan.root, args.cache_dir.as_deref()); + let record = RunRecord::load(&base); + let promoted = Hints::promoted(&record, &plan.mutants); + + if promoted.is_empty() { + writeln!( + host.error(), + "{} nothing to promote: no run under `{base}` has recorded a killing test or an unviable mutant for the current population", + styler.verb("Finished") + )?; + + return Ok(EXIT_OK); + } + + let path = hints_path(&plan.root); + + if args.dry_run { + let counts = promoted.counts(); + + writeln!( + host.error(), + "{} `{path}` would carry {} and {}, for {}", + styler.verb("Preview"), + quantity(counts.probes, "killing test"), + quantity(counts.ordering, "build-order hint"), + quantity(counts.mutants, "mutant") + )?; + + return Ok(EXIT_OK); + } + + let promotion = promoted.write(&path)?; + + let verb = if promotion.changed { "Wrote" } else { "Unchanged" }; + + writeln!( + host.error(), + "{} `{path}`: {} and {}, for {}", + styler.verb(verb), + quantity(promotion.probes, "killing test"), + quantity(promotion.ordering, "build-order hint"), + quantity(promotion.mutants, "mutant") + )?; + + Ok(EXIT_OK) +} diff --git a/crates/cargo-gamma-lib/src/commands/host.rs b/crates/cargo-gamma-lib/src/commands/host.rs new file mode 100644 index 000000000..bcd0a7fac --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/host.rs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::env; +use std::io::{self, Write}; + +/// Everything the library needs from the outside world. +/// +/// Routing all output and all terminal interrogation through one trait is what makes the console +/// UI testable. A fake host captures both streams and reports a fixed width, so the progress +/// rendering, the color decisions and the exit codes are all ordinary assertions in an +/// integration test rather than things verified by eye. +pub trait Host { + /// The stream for results the user might pipe into another program. + fn output(&mut self) -> impl Write; + + /// The result stream with normal early pipe closure treated as successful consumption. + fn results(&mut self) -> impl Write { + Results(self.output()) + } + + /// The stream for progress and diagnostics. + fn error(&mut self) -> impl Write; + + /// Whether the diagnostic stream is a terminal. + fn is_terminal(&self) -> bool; + + /// The width of the terminal in columns, if there is one. + fn terminal_width(&self) -> Option; + + /// The value of an environment variable. + /// + /// Reading the real environment is right for every caller but a test, and a test that wants to + /// pretend it is running inside a CI runner should not have to mutate the process it shares + /// with every other test to do it. + fn env(&self, name: &str) -> Option { + env::var(name).ok() + } + + /// Whether this host stands for the real process, and so may replace it with another. + /// + /// Relaunching re-runs `current_exe`, which is cargo-gamma only when cargo-gamma is what the + /// operating system actually started. Under a test harness `current_exe` is the harness, so + /// relaunching there would spawn a second copy of the test suite rather than a second copy of + /// the tool — which is why this is false unless a host says otherwise. + fn may_replace_process(&self) -> bool { + false + } +} + +struct Results(W); + +impl Write for Results { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self.0.write(buf) { + Err(cause) if cause.kind() == io::ErrorKind::BrokenPipe => Ok(buf.len()), + outcome => outcome, + } + } + + fn flush(&mut self) -> io::Result<()> { + match self.0.flush() { + Err(cause) if cause.kind() == io::ErrorKind::BrokenPipe => Ok(()), + outcome => outcome, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A host that only says it is a terminal is not thereby the real process. + /// + /// Regression: relaunching under a test harness re-runs the harness, so the suite spawns a + /// second copy of itself, and every test that inspects output sees two runs interleaved. The + /// default has to be the safe answer, because a host that forgets to say is a host that + /// cannot survive being replaced. + #[test] + fn a_host_does_not_permit_being_replaced_unless_it_says_so() { + assert!(!PlainHost.may_replace_process()); + } + + struct PlainHost; + + impl Host for PlainHost { + fn output(&mut self) -> impl Write { + Vec::new() + } + + fn error(&mut self) -> impl Write { + Vec::new() + } + + fn is_terminal(&self) -> bool { + false + } + + fn terminal_width(&self) -> Option { + None + } + } + + /// A host that overrides nothing gets the real environment, which is right for the real binary. + #[test] + #[cfg(not(miri))] + fn default_env_reads_the_process_environment() { + let host = PlainHost; + let path = host.env("PATH").expect("PATH should be set for cargo test"); + + assert!(!path.is_empty()); + assert_eq!(host.env("GAMMA_DEFINITELY_NOT_SET_IN_THE_ENVIRONMENT"), None); + } + + /// The rest of the contract is answered too, so the default double stays honest. + #[test] + fn a_minimal_host_still_answers_the_whole_contract() { + let mut host = PlainHost; + + host.output().write_all(b"out").expect("output is a sink"); + host.error().write_all(b"err").expect("error is a sink"); + + assert!(!host.is_terminal()); + assert_eq!(host.terminal_width(), None); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/list.rs b/crates/cargo-gamma-lib/src/commands/list.rs new file mode 100644 index 000000000..5305e30d7 --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/list.rs @@ -0,0 +1,570 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::io::Write; +use std::time::{SystemTime, UNIX_EPOCH}; + +use camino::Utf8PathBuf; +use serde_json::Value; + +use super::cli::{ListArgs, ListKind}; +use super::dispatch::EXIT_OK; +use super::host::Host; +use crate::error::error; +use crate::exec::CargoOptions; +use crate::ops::registry; +use crate::report::Styler; + +/// Implements `list`. +#[cfg(test)] +pub(super) fn list(host: &mut H, args: &ListArgs, styler: Styler) -> crate::Result { + let config = crate::config::Config::resolve(&args.select)?; + let cargo = config.cargo_options(); + + list_with_cargo(host, args, styler, &cargo) +} + +/// Implements `list` with the configuration generation dispatch already resolved. +pub(super) fn list_with_cargo(host: &mut H, args: &ListArgs, styler: Styler, cargo: &CargoOptions) -> crate::Result { + match args.what { + ListKind::Mutators => list_mutators(host, args), + ListKind::Files => list_files(host, args, styler, cargo), + ListKind::Mutants => list_mutants(host, args, styler, cargo), + ListKind::Presets => list_presets(host, args), + } +} + +/// Lists the named mutator presets. +/// +/// The selection is resolved against each preset so the listing says which one you are actually +/// running, rather than making you match `--mutators` against the table by eye. +fn list_presets(host: &mut H, args: &ListArgs) -> crate::Result { + let selection = args.select.selection()?; + let mut stream = host.results(); + + if args.json { + let entries: Vec = registry::PRESETS + .iter() + .map(|preset| { + serde_json::json!({ + "name": preset.name, + "description": preset.description, + "members": preset.members, + "enabled": registry::resolve(&format!("@{}", preset.name)) + .is_ok_and(|resolved| resolved.iter().all(|mutator| selection.contains(mutator))), + }) + }) + .collect(); + + writeln!( + stream, + "{}", + serde_json::to_string_pretty(&entries).map_err(|cause| { error!("could not serialize the presets").caused_by(cause) })? + )?; + + return Ok(EXIT_OK); + } + + let width = registry::PRESETS.iter().map(|preset| preset.name.len() + 1).max().unwrap_or(0); + + for preset in registry::PRESETS { + let enabled = registry::resolve(&format!("@{}", preset.name)) + .is_ok_and(|resolved| resolved.iter().all(|mutator| selection.contains(mutator))); + let mark = if enabled { "*" } else { " " }; + + writeln!(stream, "{mark} @{:width$} {}", preset.name, preset.description)?; + } + + writeln!(stream)?; + writeln!(stream, "* = enabled by the current selection")?; + + Ok(EXIT_OK) +} + +/// Lists the mutator registry. +fn list_mutators(host: &mut H, args: &ListArgs) -> crate::Result { + let selection = args.select.selection()?; + let mut stream = host.results(); + + if args.json { + let entries: Vec = registry::REGISTRY + .iter() + .map(|mutator| { + serde_json::json!({ + "name": mutator.name, + "description": mutator.description, + "default": mutator.default_on, + "enabled": selection.contains(mutator.name), + "aliases": mutator.aliases, + }) + }) + .collect(); + + writeln!( + stream, + "{}", + serde_json::to_string_pretty(&entries) + .map_err(|cause| { error!("could not serialize the mutator registry").caused_by(cause) })? + )?; + + return Ok(EXIT_OK); + } + + let width = registry::REGISTRY.iter().map(|m| m.name.len()).max().unwrap_or(0); + + for mutator in registry::REGISTRY { + let mark = if selection.contains(mutator.name) { "*" } else { " " }; + + writeln!(stream, "{mark} {:width$} {}", mutator.name, mutator.description)?; + } + + writeln!(stream)?; + writeln!(stream, "* = enabled by the current selection")?; + + Ok(EXIT_OK) +} + +/// Lists the files that would be analyzed. +fn list_files(host: &mut H, args: &ListArgs, styler: Styler, cargo: &CargoOptions) -> crate::Result { + let selection = args.select.selection()?; + let plan = crate::discover::plan_for_build(&args.select, &selection, args.select.shard()?, cargo, &mut |_| {})?; + + crate::report::skipped(host, &plan, styler)?; + + let mut stream = host.results(); + + if args.json { + let paths: Vec<&Utf8PathBuf> = plan.files.iter().map(|file| &file.path).collect(); + + writeln!( + stream, + "{}", + serde_json::to_string_pretty(&paths).map_err(|cause| error!("could not serialize the file list").caused_by(cause))? + )?; + + return Ok(EXIT_OK); + } + + for file in &plan.files { + writeln!(stream, "{}", file.path)?; + } + + Ok(EXIT_OK) +} + +/// Lists the mutants that would be generated. +fn list_mutants(host: &mut H, args: &ListArgs, styler: Styler, cargo: &CargoOptions) -> crate::Result { + let selection = args.select.selection()?; + let shard = args.select.shard()?; + let plan = crate::discover::plan_for_build(&args.select, &selection, shard, cargo, &mut |_| {})?; + + crate::report::skipped(host, &plan, styler)?; + + if let Some(path) = args.json_report.as_ref() { + write_population(host, &plan, shard, path)?; + } + + let mut stream = host.results(); + + if args.json { + writeln!( + stream, + "{}", + serde_json::to_string_pretty(&plan.mutants).map_err(|cause| error!("could not serialize the mutant list").caused_by(cause))? + )?; + + return Ok(EXIT_OK); + } + + for mutant in &plan.mutants { + writeln!(stream, "{}", describe_for_listing(mutant))?; + } + + let suppressed = plan + .mutants + .iter() + .filter(|mutant| mutant.outcome == crate::model::Outcome::Ignored) + .count(); + + if suppressed > 0 { + writeln!(stream)?; + writeln!(stream, "{suppressed} suppressed")?; + } + + Ok(EXIT_OK) +} + +/// Describes one mutant for the plain listing, marking the ones a run will not test. +/// +/// A suppressed mutant stays in the population so reports can show what was skipped and why, so +/// without the mark the listing would read as a promise to test every line it prints. +fn describe_for_listing(mutant: &crate::model::Mutant) -> String { + let Some(channel) = mutant + .suppression + .as_ref() + .filter(|_| mutant.outcome == crate::model::Outcome::Ignored) + .map(|suppression| suppression.channel.as_str()) + else { + return mutant.describe(); + }; + + format!("{} [suppressed: {channel}]", mutant.describe()) +} + +/// Writes the listing as a report document. +/// +/// `merge` withdraws a mutant only when a newer unsharded input states the whole population of its +/// file, and producing that from a run means paying for a run. Listing is the cheap way to say what +/// exists now, so it is the one a nightly rotation can afford beside its shard. +fn write_population( + host: &mut H, + plan: &crate::discover::Plan, + shard: Option<(u32, u32)>, + path: &Utf8PathBuf, +) -> crate::Result<()> { + let info = crate::elements::RunInfo { + started_at: SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |since| since.as_secs()), + merged: false, + shard: shard.map(|(count, index)| crate::elements::ShardInfo { index, count }), + tests: None, + // Filled in by `build` from the plan it is given, so that it cannot disagree with the + // mutants in the same report. + not_built: None, + dropped_test_packages: Vec::new(), + merge_provenance: None, + }; + + let report = crate::elements::build(plan, crate::elements::Thresholds::default(), Some(info))?; + + crate::elements::write_json(&report, path)?; + writeln!(host.error(), "Wrote {path}")?; + + Ok(()) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::fs; + + use super::*; + use crate::testing::{BrokenHost, Sink}; + + fn crate_dir(name: &str) -> tempfile::TempDir { + crate::fixtures::crate_dir(name, "pub fn less(a: i32, b: i32) -> bool { a < b }\n").0 + } + + fn args(dir: Utf8PathBuf, what: ListKind, json: bool) -> ListArgs { + ListArgs { + what, + select: crate::commands::SelectArgs { + dir, + ..crate::commands::SelectArgs::default() + }, + json, + json_report: None, + } + } + + #[test] + fn ops_can_be_listed_as_json() { + let mut host = Sink::default(); + + let code = list( + &mut host, + &args(Utf8PathBuf::from("."), ListKind::Mutators, true), + Styler::new(false), + ) + .expect("list"); + let text = String::from_utf8(host.out).expect("utf-8"); + let value: Value = serde_json::from_str(&text).expect("json"); + + assert_eq!(code, EXIT_OK); + assert!(value.as_array().is_some_and(|entries| !entries.is_empty()), "{text}"); + assert!(text.contains("\"enabled\""), "{text}"); + } + + /// `--json` is a contract for scripts, so the oracle has to be that the output *parses* and has + /// the promised shape. Asserting a substring the plain-text listing also contains would leave + /// the JSON branch free to disappear entirely. + #[test] + fn files_can_be_listed_as_json() { + let dir = crate_dir("list-files-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + + let code = list(&mut host, &args(root, ListKind::Files, true), Styler::new(false)).expect("list files"); + let text = String::from_utf8(host.out).expect("utf-8"); + let value: Value = serde_json::from_str(&text).expect("the --json file listing must be JSON"); + + assert_eq!(code, EXIT_OK); + + let paths = value.as_array().expect("the file listing is a JSON array"); + + assert!(paths.iter().all(Value::is_string), "every entry is a path string: {text}"); + assert!(paths.iter().any(|entry| entry.as_str() == Some("src/lib.rs")), "{text}"); + } + + #[test] + fn mutants_can_be_listed_as_json() { + let dir = crate_dir("list-mutants-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + + let code = list(&mut host, &args(root, ListKind::Mutants, true), Styler::new(false)).expect("list mutants"); + let text = String::from_utf8(host.out).expect("utf-8"); + let value: Value = serde_json::from_str(&text).expect("the --json mutant listing must be JSON"); + + assert_eq!(code, EXIT_OK); + + let mutants = value.as_array().expect("the mutant listing is a JSON array"); + let first = mutants.first().expect("the fixture produces mutants").as_object(); + let first = first.expect("every entry is a JSON object"); + + for field in ["id", "file", "line", "mutator"] { + assert!(first.contains_key(field), "missing `{field}`: {text}"); + } + + assert!( + mutants.iter().any(|entry| entry["mutator"].as_str() == Some("relational.lt_to_le")), + "{text}" + ); + } + + /// The plain listing marks the mutators the current selection turns on. + #[test] + fn ops_can_be_listed_as_text_with_the_selection_marked() { + let mut host = Sink::default(); + + let code = list( + &mut host, + &args(Utf8PathBuf::from("."), ListKind::Mutators, false), + Styler::new(false), + ) + .expect("list"); + + assert_eq!(code, EXIT_OK); + assert!(host.out().contains("* = enabled by the current selection"), "{}", host.out()); + assert!(host.out().lines().any(|line| line.starts_with("* ")), "{}", host.out()); + } + + /// The plain file listing is one path per line, so it can be piped into `xargs`. + #[test] + fn files_can_be_listed_as_text() { + let dir = crate_dir("list-files-text-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + + let code = list(&mut host, &args(root, ListKind::Files, false), Styler::new(false)).expect("list files"); + + assert_eq!(code, EXIT_OK); + assert_eq!(host.out().trim(), "src/lib.rs"); + } + + /// The plain mutant listing describes each mutant on its own line. + #[test] + fn mutants_can_be_listed_as_text() { + let dir = crate_dir("list-mutants-text-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + + let code = list(&mut host, &args(root, ListKind::Mutants, false), Styler::new(false)).expect("list mutants"); + + assert_eq!(code, EXIT_OK); + assert!(host.out().contains("relational.lt_to_le"), "{}", host.out()); + } + + /// A suppressed mutant is marked and counted, so the listing does not overstate the run. + #[test] + fn a_suppressed_mutant_is_marked_and_counted_in_the_text_listing() { + let dir = crate_dir("list-mutants-suppressed-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + + fs::write( + root.join("src/lib.rs"), + "#[gamma::skip]\npub fn less(a: i32, b: i32) -> bool { a < b }\n\npub fn more(a: i32, b: i32) -> bool { a > b }\n", + ) + .expect("lib"); + + let mut host = Sink::default(); + let code = list(&mut host, &args(root, ListKind::Mutants, false), Styler::new(false)).expect("list mutants"); + let text = host.out(); + let marked: Vec<&str> = text.lines().filter(|line| line.contains("[suppressed:")).collect(); + + assert_eq!(code, EXIT_OK); + assert!(!marked.is_empty(), "nothing was marked: {text}"); + assert!(marked.iter().all(|line| line.contains("[suppressed: attribute]")), "{text}"); + assert!( + text.lines() + .any(|line| line.contains("relational.gt_to_ge") && !line.contains("[suppressed:")), + "an ordinary mutant was marked: {text}" + ); + assert!( + text.lines().any(|line| line == format!("{} suppressed", marked.len())), + "the tail line is missing: {text}" + ); + } + + /// Nothing suppressed means no tail line at all, rather than a `0 suppressed` line. + #[test] + fn the_suppressed_tail_is_omitted_when_nothing_is_suppressed() { + let dir = crate_dir("list-mutants-unsuppressed-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + + let code = list(&mut host, &args(root, ListKind::Mutants, false), Styler::new(false)).expect("list mutants"); + + assert_eq!(code, EXIT_OK); + assert!(!host.out().contains("suppressed"), "{}", host.out()); + } + + /// The preset listing says which presets the current selection actually turns on, which is + /// the whole reason it exists rather than being a table in the README. + #[test] + fn presets_can_be_listed_as_json_with_the_selection_resolved_against_each() { + let mut listing = args(Utf8PathBuf::from("."), ListKind::Presets, true); + let mut host = Sink::default(); + + listing.select.mutators = Some("relational,range".to_owned()); + + let code = list(&mut host, &listing, Styler::new(false)).expect("list presets"); + let text = String::from_utf8(host.out).expect("utf-8"); + let value: Value = serde_json::from_str(&text).expect("json"); + let entries = value.as_array().expect("an array"); + let enabled = |name: &str| { + entries + .iter() + .find(|entry| entry["name"] == name) + .unwrap_or_else(|| panic!("no {name} preset: {text}"))["enabled"] + .as_bool() + .expect("a boolean") + }; + + assert_eq!(code, EXIT_OK); + assert_eq!(entries.len(), registry::PRESETS.len(), "{text}"); + assert!(enabled("boundary"), "a preset whose every member was selected reads as off: {text}"); + assert!(!enabled("semantics"), "a preset no selected mutator belongs to reads as on: {text}"); + assert!(entries.iter().all(|entry| entry["description"].is_string()), "{text}"); + assert!(entries.iter().all(|entry| entry["members"].is_array()), "{text}"); + } + + /// The plain preset listing names every preset with the `@` its argument needs. + #[test] + fn presets_can_be_listed_as_text() { + let mut host = Sink::default(); + + let code = list( + &mut host, + &args(Utf8PathBuf::from("."), ListKind::Presets, false), + Styler::new(false), + ) + .expect("list presets"); + let text = host.out(); + + assert_eq!(code, EXIT_OK); + assert_eq!(text.lines().count(), registry::PRESETS.len() + 2, "{text}"); + + for preset in registry::PRESETS { + assert!( + text.lines() + .any(|line| line.contains(&format!("@{}", preset.name)) && line.contains(preset.description)), + "{} is missing: {text}", + preset.name + ); + } + } + + #[test] + fn the_text_preset_listing_marks_a_preset_selected_by_its_exact_members() { + let preset = registry::PRESETS + .iter() + .find(|preset| preset.name == "boundary") + .expect("the boundary preset exists"); + let mut listing = args(Utf8PathBuf::from("."), ListKind::Presets, false); + listing.select.mutators = Some(preset.members.join(",")); + let mut host = Sink::default(); + + let code = list(&mut host, &listing, Styler::new(false)).expect("list presets"); + + assert_eq!(code, EXIT_OK); + assert!( + host.out().lines().any(|line| line.starts_with(&format!("* @{}", preset.name))), + "{}", + host.out() + ); + assert!(host.out().contains("* = enabled by the current selection"), "{}", host.out()); + } + + /// Every listing shape treats a closed consumer as successful completion. + #[test] + fn a_closed_output_stream_ends_every_listing_successfully() { + let dir = crate_dir("list-broken-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + + for (what, json) in [ + (ListKind::Mutators, true), + (ListKind::Mutators, false), + (ListKind::Files, true), + (ListKind::Files, false), + (ListKind::Mutants, true), + (ListKind::Mutants, false), + (ListKind::Presets, true), + (ListKind::Presets, false), + ] { + let code = list(&mut BrokenHost, &args(root.clone(), what, json), Styler::new(false)).expect("closed pipe"); + + assert_eq!(code, EXIT_OK, "{what:?} json={json}"); + } + } + + #[test] + fn the_population_can_be_written_as_a_report() { + // `merge` withdraws a retired mutant only against an unsharded population, and a rotation + // that could afford a full run would not be sharding in the first place. + let dir = crate_dir("list-population-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("population.json"); + let mut host = Sink::default(); + let mut listing = args(root, ListKind::Mutants, false); + + listing.json_report = Some(path.clone()); + + let code = list(&mut host, &listing, Styler::new(false)).expect("list"); + let text = fs::read_to_string(&path).expect("report"); + let report: crate::elements::Report = serde_json::from_str(&text).expect("json"); + + assert_eq!(code, EXIT_OK); + assert!(report.config.as_ref().is_some_and(|run| run.shard.is_none()), "{text}"); + assert!( + report.files.values().any(|file| !file.mutants.is_empty()), + "the population is empty: {text}" + ); + assert!( + String::from_utf8(host.err).expect("utf-8").contains("Wrote"), + "the path was not echoed" + ); + } + + #[test] + fn a_sharded_population_says_which_shard_it_is() { + // A shard's silence about a mutant is not evidence that the mutant is gone, so the merge + // has to be able to tell the two kinds of listing apart. + let dir = crate_dir("list-population-shard-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("population.json"); + let mut host = Sink::default(); + let mut listing = args(root, ListKind::Mutants, false); + + listing.json_report = Some(path.clone()); + listing.select.shard_count = Some(4); + listing.select.shard_index = Some(2); + + let code = list(&mut host, &listing, Styler::new(false)).expect("list"); + let text = fs::read_to_string(&path).expect("report"); + let report: crate::elements::Report = serde_json::from_str(&text).expect("json"); + let shard = report.config.as_ref().and_then(|run| run.shard.as_ref()).expect("shard"); + + assert_eq!(code, EXIT_OK); + assert_eq!((shard.index, shard.count), (2, 4)); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/merge.rs b/crates/cargo-gamma-lib/src/commands/merge.rs new file mode 100644 index 000000000..2cfa9d9df --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/merge.rs @@ -0,0 +1,818 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::fs; +use std::io::Write; +use std::time::{SystemTime, UNIX_EPOCH}; + +use camino::{Utf8Path, Utf8PathBuf}; + +use super::cli::MergeArgs; +use super::dispatch::{EXIT_GATE_FAILED, EXIT_OK}; +use super::host::Host; +use crate::elements::Report; +use crate::error::error; +use crate::report::{Styler, quantity}; + +/// The most independently produced reports one merge retains. +/// +/// A rotation is normally in the tens or hundreds. This leaves room for a large history while +/// making a directory full of individually valid reports an actionable error rather than an OOM. +const MAX_REPORTS: usize = 4_096; + +/// The input bytes whose decoded reports one merge retains. +/// +/// String data in a decoded report cannot exceed its JSON representation, so this bounds the +/// untrusted text retained by the report vector in addition to the per-report read cap. +const MAX_TOTAL_BYTES: u64 = 512 * 1024 * 1024; + +/// Implements `merge`. +pub(super) fn merge(host: &mut H, args: &MergeArgs, styler: Styler) -> crate::Result { + let now = SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|since| since.as_secs()); + + merge_at(host, args, styler, now) +} + +fn merge_at(host: &mut H, args: &MergeArgs, styler: Styler, now: Option) -> crate::Result { + validate_output_paths(args)?; + let inputs = collect_reports(&args.inputs)?; + + if inputs.is_empty() { + return Err(error!("no reports were found in the given paths").usage()); + } + + let window = (args.window > 0).then(|| args.window.saturating_mul(86_400)); + let mut merged = crate::merge::merge(&inputs, now.unwrap_or(0), window); + + if now.is_none() { + merged.fresh = 0; + merged.stale = 0; + merged.freshness_unavailable = true; + } + + report_merge(host, args, &merged, styler)?; + + if let Some(report) = merged.report.as_ref() { + if let Some(path) = args.json_report.as_ref() { + crate::elements::write_json(report, path)?; + writeln!(host.error(), "{} {path}", styler.verb("Wrote"))?; + } + + if let Some(path) = args.html_report.as_ref() { + crate::html::write_page(report, crate::html::Source::Inline, path)?; + writeln!(host.error(), "{} {path}", styler.verb("Wrote"))?; + } + } + + if let Some(minimum) = args.min_score { + let Some(score) = merged.scored() else { + // Every mutant was withdrawn, never tested, or otherwise ungradeable, so the merged + // score is a ratio with nothing in its denominator. That prints as 100%, which is the + // right answer to "how much of what ran was caught" and the wrong one to hand a + // threshold, so the gate refuses rather than passing a merge that scored nothing. + writeln!( + host.error(), + "{} no mutant counted toward the merged score, so the `--min-score` gate was never evaluated; \ + check that the inputs cover a population that was actually run", + styler.error("error:") + )?; + + return Ok(EXIT_GATE_FAILED); + }; + + if score < minimum { + let (shown_score, shown_minimum) = super::run::distinguish(score, minimum); + + writeln!( + host.error(), + "{} merged mutation score {shown_score}% is below the required {shown_minimum}%", + styler.error("error:") + )?; + + return Ok(EXIT_GATE_FAILED); + } + } + + Ok(EXIT_OK) +} + +/// Refuses a merged HTML and JSON report that would publish over one another. +fn validate_output_paths(args: &MergeArgs) -> crate::Result<()> { + let mut outputs = Vec::new(); + + if let Some(path) = args.json_report.as_deref() { + outputs.push(("JSON report", path)); + } + + if let Some(path) = args.html_report.as_deref() { + outputs.push(("HTML report", path)); + } + + crate::paths::reject_collisions(&outputs) +} + +/// Reads every report named, expanding directories to the JSON files they contain. +/// +/// Directories are accepted because the natural place to keep a rotation's history is a directory, +/// and requiring a glob would mean the command behaves differently under shells that do not expand +/// one. +fn collect_reports(inputs: &[Utf8PathBuf]) -> crate::Result> { + collect_reports_limited(inputs, MAX_REPORTS, MAX_TOTAL_BYTES) +} + +/// Collects inputs under explicit budgets, so the production limits have cheap boundary tests. +fn collect_reports_limited(inputs: &[Utf8PathBuf], max_reports: usize, max_bytes: u64) -> crate::Result> { + let mut out = Vec::new(); + let mut retained_bytes = 0; + + for input in inputs { + if input.is_dir() { + let entries = fs::read_dir(input).map_err(|cause| error!("could not read `{input}`").caused_by(cause))?; + let mut paths: Vec = Vec::new(); + + for entry in entries { + let entry = entry.map_err(|cause| error!("could not read `{input}`").caused_by(cause))?; + let path = Utf8PathBuf::from_path_buf(entry.path()).map_err(|path| error!("{} is not a UTF-8 path", path.display()))?; + + if path.extension() == Some("json") { + if paths.len() >= max_reports.saturating_sub(out.len()) { + return Err(error!( + "`{input}` contains more than the {max_reports} reports a merge will retain; split the directory into smaller merges" + ) + .usage()); + } + + paths.push(path); + } + } + + // Directory order is not defined, and the merge must not depend on it. + paths.sort(); + + for path in paths { + read_report(&path, max_reports, max_bytes, &mut retained_bytes, &mut out)?; + } + } else { + read_report(input, max_reports, max_bytes, &mut retained_bytes, &mut out)?; + } + } + + Ok(out) +} + +/// Reads and retains one report after applying the aggregate bounds. +fn read_report( + path: &Utf8Path, + max_reports: usize, + max_bytes: u64, + retained_bytes: &mut u64, + out: &mut Vec<(String, Report)>, +) -> crate::Result<()> { + if out.len() >= max_reports { + return Err(error!("merge has more than the {max_reports} reports it will retain; split the inputs into smaller merges").usage()); + } + + let remaining = max_bytes.saturating_sub(*retained_bytes); + let input = crate::merge::read_limited(path, remaining)?; + + *retained_bytes = retained_bytes + .checked_add(input.bytes) + .expect("a report read under the remaining budget cannot overflow the aggregate"); + out.push((path.to_string(), input.report)); + + Ok(()) +} + +/// Prints what the merge concluded. +fn report_merge(host: &mut H, args: &MergeArgs, merged: &crate::merge::Merged, styler: Styler) -> crate::Result<()> { + let mut stream = host.error(); + + // "detected" and "not detected" rather than "killed" and "survived", because these are the two + // halves of the score's fraction and neither is the outcome it would otherwise be named after. + // Only a failing assertion detects a mutant. The remainder also holds timeouts, memory + // exhaustion and `NoCoverage`, so naming it "survived" would hide several distinct remedies. + writeln!( + stream, + "{} {} detected, {} not detected, score {}%", + styler.verb("Merged"), + merged.detected, + merged.valid.saturating_sub(merged.detected), + crate::report::score(merged.score(), merged.detected, merged.valid) + )?; + + if merged.freshness_unavailable { + writeln!( + stream, + "{} unavailable because the system clock is before the Unix epoch; {} never tested", + styler.verb("Freshness"), + merged.never_tested + )?; + } else if args.window == 0 { + writeln!( + stream, + "{} window disabled, {} verdicts current, {} never tested", + styler.verb("Freshness"), + merged.fresh, + merged.never_tested + )?; + } else { + writeln!( + stream, + "{} {} fresh, {} older than {} days, {} never tested", + styler.verb("Freshness"), + merged.fresh, + merged.stale, + args.window, + merged.never_tested + )?; + } + + if merged.withdrawn > 0 { + writeln!( + stream, + "{} {} dropped, tested against code that has since changed", + styler.verb("Withdrawn"), + merged.withdrawn + )?; + } + + if merged.incompatible > 0 { + writeln!( + stream, + "{} {} excluded because their source presentation is incompatible with the selected source", + styler.verb("Incompatible"), + quantity(merged.incompatible, "verdict") + )?; + } + + if merged.unchecked > 0 { + writeln!( + stream, + "{} withdrawals unchecked for {}: no input supplied a complete population", + styler.verb("Note"), + quantity(merged.unchecked, "file") + )?; + } + + if let Some(count) = merged.shard_count { + writeln!( + stream, + "{} {} of {count} shards seen, {:.0}% of the rotation", + styler.verb("Rotation"), + merged.shards_seen.len(), + merged.coverage() + )?; + + let missing = merged.missing_shards(); + + if !missing.is_empty() { + let names: Vec = missing.iter().map(u32::to_string).collect(); + + writeln!(stream, "{} shards never run: {}", styler.verb("Note"), names.join(", "))?; + } + } + + // Two runs at different shard counts partitioned the population differently, so the coverage + // number above is not the claim it appears to be. + for input in &merged.inconsistent { + writeln!( + stream, + "{} {input} used a different shard count; rotation coverage is not meaningful across it", + styler.verb("Warning") + )?; + } + + Ok(()) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + use crate::elements::{FileResult, RunInfo, ShardInfo}; + use crate::fixtures::mutant_result_at as mutant; + use crate::testing::{Sink, fails_at_every_line, workdir}; + use crate::{HashMap, fixtures}; + + fn report(index: u32, count: u32, status: &str) -> Report { + let mut files = HashMap::default(); + let _ = files.insert( + "src/lib.rs".to_owned(), + FileResult { + source: "pub fn f() {}\n".to_owned(), + language: "rust".to_owned(), + mutants: vec![fixtures::mutant_result_at( + &format!("m{index}"), + usize::try_from(index + 1).unwrap(), + status, + )], + }, + ); + + Report { + files, + config: Some(RunInfo { + started_at: 100 + u64::from(index), + merged: false, + shard: Some(ShardInfo { index, count }), + tests: None, + not_built: None, + dropped_test_packages: Vec::new(), + merge_provenance: None, + }), + ..fixtures::report() + } + } + + /// Builds a report with no shard identity at all, the shape a full (non-sharded) run writes. + /// `populations` only trusts an unsharded report to say what currently exists at a path, so a + /// withdrawn mutant can only be produced from a pair of these. + fn unsharded_report(started_at: u64, id: &str, line: usize, status: &str) -> Report { + let mut files = HashMap::default(); + let _ = files.insert( + "src/lib.rs".to_owned(), + FileResult { + source: "pub fn f() {}\n".to_owned(), + language: "rust".to_owned(), + mutants: vec![mutant(id, line, status)], + }, + ); + + Report { + files, + config: Some(RunInfo { + started_at, + merged: false, + shard: None, + tests: None, + not_built: None, + dropped_test_packages: Vec::new(), + merge_provenance: None, + }), + ..fixtures::report() + } + } + + fn write_report(path: &Utf8Path, report: &Report) { + crate::elements::write(&path.to_path_buf(), &crate::elements::to_json(report).expect("json")).expect("write"); + } + + #[test] + fn directories_are_scanned_and_outputs_are_written() { + let dir = workdir("merge-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input_dir = root.join("reports"); + fs::create_dir_all(&input_dir).expect("reports"); + write_report(&input_dir.join("a.json"), &report(0, 3, "Killed")); + write_report(&input_dir.join("b.json"), &report(1, 4, "Survived")); + fs::write(input_dir.join("ignored.txt"), "not a report").expect("ignore"); + + let args = MergeArgs { + inputs: vec![input_dir], + json_report: Some(root.join("out/report.json")), + html_report: Some(root.join("out/report.html")), + window: 30, + min_score: Some(75.0), + }; + let mut host = Sink::default(); + + let code = merge(&mut host, &args, Styler::new(false)).expect("merge"); + let err = String::from_utf8(host.err).expect("utf-8"); + + assert_eq!(code, EXIT_GATE_FAILED); + assert!(root.join("out/report.json").exists()); + assert!(root.join("out/report.html").exists()); + assert!(err.contains("shards never run"), "{err}"); + assert!(err.contains("different shard count"), "{err}"); + assert!(err.contains("below the required"), "{err}"); + assert!(host.out.is_empty()); + } + + #[test] + fn a_directory_with_too_many_reports_is_refused_before_they_are_retained() { + let dir = workdir("merge-report-count-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input_dir = root.join("reports"); + fs::create_dir_all(&input_dir).expect("reports"); + write_report(&input_dir.join("a.json"), &report(0, 2, "Killed")); + write_report(&input_dir.join("b.json"), &report(1, 2, "Survived")); + + let error = collect_reports_limited(&[input_dir], 1, u64::MAX) + .expect_err("the second report exceeds the count budget") + .to_string(); + + assert!(error.contains("1 reports"), "{error}"); + assert!(error.contains("split the directory"), "{error}"); + } + + #[test] + fn aggregate_report_bytes_are_bounded_before_a_report_is_retained() { + let dir = workdir("merge-total-bytes-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input = root.join("report.json"); + let report = report(0, 1, "Killed"); + let text = crate::elements::to_json(&report).expect("json"); + let bytes = u64::try_from(text.len()).expect("report text length fits u64"); + fs::write(&input, text).expect("report"); + + let error = collect_reports_limited(&[input], 2, bytes - 1) + .expect_err("the aggregate byte budget is too small") + .to_string(); + + assert!(error.contains(&bytes.to_string()), "{error}"); + assert!(error.contains(&(bytes - 1).to_string()), "{error}"); + assert!(error.contains("merge will retain"), "{error}"); + } + + #[test] + fn empty_inputs_are_a_usage_error() { + let dir = workdir("merge-empty-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let args = MergeArgs { + inputs: vec![root], + json_report: None, + html_report: None, + window: 30, + min_score: None, + }; + let mut host = Sink::default(); + + let err = merge(&mut host, &args, Styler::new(false)).unwrap_err(); + + assert!(err.is_usage()); + } + + /// A closed diagnostic stream has to surface from every line the merge prints. + #[test] + fn a_closed_diagnostic_stream_is_reported_from_any_line() { + let dir = workdir("merge-broken-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input_dir = root.join("reports"); + fs::create_dir_all(&input_dir).expect("reports"); + write_report(&input_dir.join("a.json"), &report(0, 3, "Killed")); + write_report(&input_dir.join("b.json"), &report(1, 4, "Survived")); + + let args = MergeArgs { + inputs: vec![input_dir], + json_report: Some(root.join("out/report.json")), + html_report: Some(root.join("out/report.html")), + window: 30, + min_score: Some(75.0), + }; + + fails_at_every_line(8, |host| merge(host, &args, Styler::new(false)).map(|_| ())); + } + + /// Merging without a gate or any report to write still succeeds and says what it saw. + #[test] + fn a_merge_with_no_gate_and_no_outputs_succeeds() { + let dir = workdir("merge-plain-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input = root.join("a.json"); + write_report(&input, &report(0, 1, "Killed")); + + let args = MergeArgs { + inputs: vec![input], + json_report: None, + html_report: None, + window: 30, + min_score: Some(10.0), + }; + let mut host = Sink::default(); + + let code = merge(&mut host, &args, Styler::new(false)).expect("merge"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("Merged"), "{}", host.err()); + assert!(!host.err().contains("shards never run"), "{}", host.err()); + } + + #[test] + fn zero_window_and_clock_failure_are_reported_truthfully() { + let dir = workdir("merge-clock-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input = root.join("a.json"); + write_report(&input, &report(0, 1, "Killed")); + let mut args = MergeArgs { + inputs: vec![input], + json_report: None, + html_report: None, + window: 0, + min_score: None, + }; + let mut host = Sink::default(); + + let _code = merge_at(&mut host, &args, Styler::new(false), Some(200)).expect("merge"); + assert!(host.err().contains("window disabled"), "{}", host.err()); + assert!(!host.err().contains("older than 0 days"), "{}", host.err()); + + args.window = 30; + let mut host = Sink::default(); + let _code = merge_at(&mut host, &args, Styler::new(false), None).expect("merge"); + assert!( + host.err().contains("unavailable because the system clock is before the Unix epoch"), + "{}", + host.err() + ); + assert!(!host.err().contains("1 fresh"), "{}", host.err()); + } + + #[test] + fn inexact_merged_boundary_scores_do_not_render_as_exact_boundaries() { + let args = MergeArgs { + inputs: vec![Utf8PathBuf::from("unused")], + json_report: None, + html_report: None, + window: 30, + min_score: None, + }; + for (detected, expected, forbidden) in [(9_999, "score 99.99%", "score 100.0%"), (1, "score 0.01%", "score 0.0%")] { + let merged = crate::merge::Merged { + detected, + valid: 10_000, + incompatible: 1, + ..crate::merge::Merged::default() + }; + let mut host = Sink::default(); + + report_merge(&mut host, &args, &merged, Styler::new(false)).expect("summary"); + assert!(host.err().contains(expected), "{}", host.err()); + assert!(!host.err().contains(forbidden), "{}", host.err()); + assert!(host.err().contains("1 verdict excluded"), "{}", host.err()); + } + } + + /// The merge headline names the two halves of the fraction, not two of the ten outcomes. + /// + /// The remainder of the denominator is every mutant the run judged and nothing noticed, which + /// includes the uncovered ones — mutants no test ever reached. Calling that figure "survived" + /// sends the reader looking for a test with a weak assertion when the truth is that no test ran + /// the line at all, and it does so on the summary line that is read most often and checked + /// least. + #[test] + fn the_merge_headline_does_not_call_uncovered_mutants_survivors() { + let dir = workdir("merge-headline-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input = root.join("a.json"); + write_report(&input, &report(0, 1, "NoCoverage")); + + let args = MergeArgs { + inputs: vec![input], + json_report: None, + html_report: None, + window: 30, + min_score: None, + }; + let mut host = Sink::default(); + + let _code = merge(&mut host, &args, Styler::new(false)).expect("merge"); + + assert!(host.err().contains("0 detected, 1 not detected"), "{}", host.err()); + assert!( + !host.err().contains("survived"), + "an uncovered mutant was called a survivor: {}", + host.err() + ); + } + + /// A directory that cannot be read names itself rather than failing anonymously. + #[test] + fn an_unreadable_input_directory_names_itself() { + let dir = workdir("merge-unreadable-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let missing = root.join("gone"); + + let args = MergeArgs { + inputs: vec![missing.clone()], + json_report: None, + html_report: None, + window: 30, + min_score: None, + }; + let mut host = Sink::default(); + + let error = merge(&mut host, &args, Styler::new(false)).expect_err("missing input"); + + assert!(error.to_string().contains(missing.as_str()), "{error}"); + } + + /// A mutant whose site was edited between two full runs is dropped from the merge, and the + /// summary calls out how many were withdrawn so the count is never confused with a mutant that + /// simply failed to run; without the note, a shrinking denominator would look identical to a + /// suite that stopped testing something. + #[test] + fn a_withdrawn_mutant_is_called_out_by_name() { + let dir = workdir("merge-withdrawn-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input_dir = root.join("reports"); + fs::create_dir_all(&input_dir).expect("reports"); + + // The old survivor's site no longer exists in the newer run, so it is withdrawn rather than + // counted as a gap in the current code. + write_report(&input_dir.join("old.json"), &unsharded_report(100, "aaa", 1, "Survived")); + write_report(&input_dir.join("new.json"), &unsharded_report(200, "bbb", 1, "Killed")); + + let args = MergeArgs { + inputs: vec![input_dir], + json_report: None, + html_report: None, + window: 30, + min_score: None, + }; + let mut host = Sink::default(); + + let code = merge(&mut host, &args, Styler::new(false)).expect("merge"); + + assert_eq!(code, EXIT_OK); + assert!( + host.err().contains("1 dropped, tested against code that has since changed"), + "{}", + host.err() + ); + } + + #[test] + fn a_merged_only_input_says_no_complete_population_was_supplied() { + let dir = workdir("merge-incomplete-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input = root.join("merged.json"); + let mut prior = unsharded_report(100, "aaa", 1, "Killed"); + + prior.config.as_mut().expect("config").merged = true; + write_report(&input, &prior); + let args = MergeArgs { + inputs: vec![input], + json_report: None, + html_report: None, + window: 30, + min_score: None, + }; + let mut host = Sink::default(); + + let _code = merge_at(&mut host, &args, Styler::new(false), Some(200)).expect("merge"); + assert!(host.err().contains("no input supplied a complete population"), "{}", host.err()); + } + + /// A closed stream has to surface from the withdrawn note too, just like every other line the + /// merge summary writes. + #[test] + fn a_closed_stream_is_reported_by_the_withdrawn_note() { + let dir = workdir("merge-withdrawn-broken-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input_dir = root.join("reports"); + fs::create_dir_all(&input_dir).expect("reports"); + + write_report(&input_dir.join("old.json"), &unsharded_report(100, "aaa", 1, "Survived")); + write_report(&input_dir.join("new.json"), &unsharded_report(200, "bbb", 1, "Killed")); + + let args = MergeArgs { + inputs: vec![input_dir], + json_report: None, + html_report: None, + window: 30, + min_score: None, + }; + + // "Merged", "Freshness" and then "Withdrawn" is the third line the summary writes, so + // closing the stream there is what exercises it. + fails_at_every_line(3, |host| merge(host, &args, Styler::new(false)).map(|_| ())); + } + + /// Builds an unsharded report whose single file holds one mutant per status given. + /// + /// The near-miss and empty-population gate tests need a population with a chosen detected/valid + /// split, which the per-shard `report` helper cannot express; an unsharded report states a whole + /// file's population, so the merge takes the statuses exactly as written. + fn population(statuses: &[&str]) -> Report { + let mutants = statuses + .iter() + .enumerate() + .map(|(index, status)| mutant(&format!("m{index}"), index + 1, status)) + .collect(); + + let mut files = HashMap::default(); + let _ = files.insert( + "src/lib.rs".to_owned(), + FileResult { + source: "pub fn f() {}\n".to_owned(), + language: "rust".to_owned(), + mutants, + }, + ); + + Report { + files, + config: Some(RunInfo { + started_at: 100, + merged: false, + shard: None, + tests: None, + not_built: None, + dropped_test_packages: Vec::new(), + merge_provenance: None, + }), + ..fixtures::report() + } + } + + /// A merge whose population never scored must fail `--min-score`, not pass it. + /// + /// Every mutant here was ignored, so the denominator is empty and the printed score is 100% — + /// the right thing to display for "how much of what ran was caught" and a catastrophe to hand a + /// threshold, because `--min-score 100` against a merge that scored nothing is a gate that never + /// ran, not one that passed. The gate routes through `scored`, so it refuses structurally rather + /// than relying on the placeholder being unflattering, and says why it could not grade. + #[test] + fn a_merge_that_scored_nothing_fails_the_min_score_gate() { + let dir = workdir("merge-ungraded-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input = root.join("a.json"); + write_report(&input, &population(&["Ignored", "Ignored"])); + + let args = MergeArgs { + inputs: vec![input], + json_report: None, + html_report: None, + window: 30, + min_score: Some(100.0), + }; + let mut host = Sink::default(); + + let code = merge(&mut host, &args, Styler::new(false)).expect("merge"); + + assert_eq!(code, EXIT_GATE_FAILED, "{}", host.err()); + assert!(host.err().contains("no mutant counted toward the merged score"), "{}", host.err()); + } + + /// The gate message must never print the required score as already met. + /// + /// Two killed against one survived is 66.666…%, which fails a 66.7% bar on the full-precision + /// comparison — the correct verdict — but at one decimal both the score and the threshold read + /// "66.7%", so the old message said a score of 66.7% was below the required 66.7%. The score now + /// keeps a second decimal until it is visibly under the bar. + #[test] + fn the_merge_gate_message_does_not_print_the_required_score_as_met() { + let dir = workdir("merge-near-miss-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let input = root.join("a.json"); + write_report(&input, &population(&["Killed", "Killed", "Survived"])); + + let args = MergeArgs { + inputs: vec![input], + json_report: None, + html_report: None, + window: 30, + min_score: Some(66.7), + }; + let mut host = Sink::default(); + + let code = merge(&mut host, &args, Styler::new(false)).expect("merge"); + let err = host.err(); + + assert_eq!(code, EXIT_GATE_FAILED, "{err}"); + assert!(err.contains("66.67% is below the required 66.70%"), "{err}"); + assert!( + !err.contains("66.7% is below the required 66.7%"), + "the message denies itself: {err}" + ); + } + + #[test] + fn merge_outputs_with_identical_names_are_refused_before_any_report_is_written() { + let directory = workdir("merge-output-collision-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("UTF-8 path"); + let output = root.join("merged"); + let args = MergeArgs { + inputs: vec![root.join("missing.json")], + json_report: Some(output.clone()), + html_report: Some(output), + window: 30, + min_score: None, + }; + + let error = merge(&mut Sink::default(), &args, Styler::new(false)).expect_err("colliding outputs are rejected first"); + + assert!(error.is_usage(), "{error}"); + assert!(!root.join("merged").exists()); + } + + #[cfg(unix)] + #[test] + fn merge_outputs_with_symlink_aliased_names_are_refused_before_any_report_is_written() { + let directory = workdir("merge-output-symlink-collision-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("UTF-8 path"); + let target = root.join("merged"); + let alias = root.join("alias"); + + std::os::unix::fs::symlink(target.as_std_path(), alias.as_std_path()).expect("symlink"); + let args = MergeArgs { + inputs: vec![root.join("missing.json")], + json_report: Some(target), + html_report: Some(alias), + window: 30, + min_score: None, + }; + + let _error = merge(&mut Sink::default(), &args, Styler::new(false)).expect_err("symlink-aliased outputs must be refused"); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/mod.rs b/crates/cargo-gamma-lib/src/commands/mod.rs new file mode 100644 index 000000000..73af9f7a9 --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/mod.rs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The command-line surface and the orchestration behind it. + +mod clean; +mod cli; +mod completions; +mod console_events; +mod dispatch; +mod explain; +mod hints; +mod host; +mod list; +mod merge; +mod run; +mod suppress; +mod unsuppress; +mod verdict_log; +mod when; + +pub use cli::{ + CleanArgs, Cli, Command, CompletionsArgs, ConfigArgs, ExplainArgs, FeatureArgs, HintsArgs, ListArgs, ListKind, MergeArgs, RunArgs, + SelectArgs, SuppressArgs, UnsuppressArgs, +}; +pub use dispatch::{EXIT_CANNOT_PROCEED, EXIT_GATE_FAILED, EXIT_INTERNAL, EXIT_OK, EXIT_USAGE, run}; +pub use host::Host; +pub use when::When; diff --git a/crates/cargo-gamma-lib/src/commands/run.rs b/crates/cargo-gamma-lib/src/commands/run.rs new file mode 100644 index 000000000..abc930f88 --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/run.rs @@ -0,0 +1,2439 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; +use std::fs; +use std::io::Write; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use camino::{Utf8Path, Utf8PathBuf}; + +use super::cli::RunArgs; +use super::console_events::ConsoleEvents; +use super::dispatch::{DEFAULT_TEST_TIMEOUT_MULTIPLIER, EXIT_CANNOT_PROCEED, EXIT_GATE_FAILED, EXIT_OK}; +use super::host::Host; +use super::verdict_log::VerdictLog; +use super::when::When; +use crate::config::Config; +use crate::discover::Plan; +use crate::error::{Error, error}; +use crate::exec; +use crate::model::{Mutant, Outcome}; +use crate::report::{Listings, Progress, Styler, quantity}; + +/// Which of the bulk outcome listings the caller asked for. +const fn listings(args: &RunArgs, announced: bool) -> Listings { + Listings { + killed: args.show_killed, + unviable: args.show_unviable, + announced, + } +} + +fn missing_hints(mode: exec::IncrementalMode, root: &Utf8Path) -> bool { + mode.is_enabled() && crate::discover::Hints::is_missing(root) +} + +/// Loads `gamma.toml` and folds it into `args`. +pub(super) fn configure(host: &mut H, args: &mut RunArgs, styler: Styler) -> crate::Result<()> { + // Said before anything is loaded, because the settings in that file are about to not happen and + // the run would otherwise look like it honoured them. + if Config::foreign_present(&args.select.dir) && !args.select.config.no_config && args.select.config.path.is_none() { + let hint = styler.note("Hint"); + + writeln!( + host.error(), + "{hint} .cargo/mutants.toml is not supported or read; configure gamma.toml explicitly" + )?; + } + + Config::resolve(&args.select)?.apply(args)?; + + Ok(()) +} + +/// Works out how much memory control the run should place around each test binary. +/// +/// The two size flags imply a mode, because asking for a specific ceiling and then being told +/// nothing was enforced would be a surprising way to learn that a separate switch existed. Naming +/// `--memory` explicitly still wins, so a configuration file that turns metering on can be turned +/// back off for one run. +/// +/// Whether any of that was said out loud is recorded rather than discarded. Enforcement is the +/// default, and a host that cannot deliver it gets a note and an unbounded run — but a user who +/// named a memory setting gets an error instead, because they asked for a guarantee and silently +/// not having it is the one outcome that could cost them the machine. +pub(super) fn memory_policy(args: &RunArgs) -> exec::MemoryPolicy { + let implied = exec::implied_memory_control(args.measure.memory_limit, args.measure.baseline_memory_limit); + let stated = args.measure.memory.or(implied); + let demand = if stated.is_some() { + exec::Demand::Stated + } else { + exec::Demand::Inherited + }; + + exec::MemoryPolicy { + control: stated.unwrap_or_default(), + demand, + multiplier: args.measure.memory_multiplier.unwrap_or(exec::DEFAULT_MULTIPLIER), + headroom: args.measure.memory_headroom.unwrap_or(exec::DEFAULT_HEADROOM), + limit: args.measure.memory_limit, + baseline_limit: args.measure.baseline_memory_limit, + } +} + +/// Records what `merge` needs to know about this run. +/// +/// The shard identity travels in the report rather than in the filename, because a filename is a +/// convention and this has to survive being copied into an artifact bucket by someone who does not +/// know the convention. +fn run_info(args: &RunArgs, tests: Option, dropped: &[String]) -> crate::elements::RunInfo { + crate::elements::RunInfo { + tests, + started_at: SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |since| since.as_secs()), + merged: false, + shard: args + .select + .shard_count + .zip(args.select.shard_index) + .map(|(count, index)| crate::elements::ShardInfo { index, count }), + // Filled in by `build` from the plan it is given, so that it cannot disagree with the + // mutants in the same report. + not_built: None, + dropped_test_packages: dropped.to_vec(), + merge_provenance: None, + } +} + +/// Where a run's documents go, once the defaults and any overrides have been settled. +/// +/// Every run writes all five — HTML, JSON and SARIF reports, Markdown advice and a diagnostics bundle. +/// The reason they are produced by default rather than on request is +/// that the flags were a trap: a run that takes an hour and answers a question about your test +/// suite is exactly the run you do not want to repeat because you forgot to ask for the artifact +/// that answers it. The cost of writing them is a few hundred milliseconds against that hour, and +/// they land under `target` where nothing is precious. +/// +/// Normal runs publish under the original workspace's `target/cargo-gamma`; `--artifact-dir` +/// redirects the complete set without changing where reusable cache state lives. +struct Documents { + /// Where the self-contained HTML report goes. + html: Utf8PathBuf, + + /// Where the `mutation-testing-elements` JSON report goes. + json: Utf8PathBuf, + + /// Where the Markdown diagnosis goes. + advice: Utf8PathBuf, + + /// Where the diagnostics bundle goes. + diag: Utf8PathBuf, + + /// Where the SARIF report goes. + sarif: Utf8PathBuf, +} + +impl Documents { + fn directory(args: &RunArgs, root: &Utf8Path) -> Utf8PathBuf { + args.artifact_dir.clone().unwrap_or_else(|| root.join("target/cargo-gamma")) + } + + /// The fixed artifact names under the configured directory. + fn resolve(args: &RunArgs, root: &Utf8Path) -> Self { + let base = Self::directory(args, root); + + Self { + html: base.join("gamma-report.html"), + json: base.join("gamma-report.json"), + advice: base.join("gamma-perf-advice.md"), + diag: base.join("gamma-diagnostics.json"), + sarif: base.join("gamma-report.sarif"), + } + } +} + +/// Writes the file reports, and says where they went. +/// +/// The path is echoed because a report written to a path nobody looks at is the same as no report, +/// and in CI the message is often the only trace that the artifact exists. +fn emit_reports( + host: &mut H, + args: &RunArgs, + plan: &Plan, + advice: Option<&str>, + tests: Option, + dropped: &[String], + styler: Styler, +) -> crate::Result<()> { + let documents = Documents::resolve(args, &plan.root); + let report = crate::elements::build(plan, crate::elements::Thresholds::default(), Some(run_info(args, tests, dropped)))?; + let mut stream = host.error(); + + crate::elements::write_json(&report, &documents.json)?; + writeln!(stream, "{} {}", styler.verb("Wrote"), documents.json)?; + + let source = if args.html_external { + crate::html::Source::External + } else { + crate::html::Source::Inline + }; + + crate::html::write_page(&report, source, &documents.html)?; + writeln!(stream, "{} {}", styler.verb("Wrote"), documents.html)?; + drop(stream); + + emit_ci(host, args, plan, advice, styler)?; + + Ok(()) +} + +/// Writes the SARIF log and, when requested, the diff annotations and job summary. +fn emit_ci(host: &mut H, args: &RunArgs, plan: &Plan, advice: Option<&str>, styler: Styler) -> crate::Result<()> { + let path = Documents::resolve(args, &plan.root).sarif; + let (log, truncation) = crate::ci::sarif(&plan.mutants, &plan.root, args.sarif_level)?; + + crate::elements::write(&path, &log)?; + + let mut stream = host.error(); + + writeln!(stream, "{} {path}", styler.verb("Wrote"))?; + + if let Some(truncation) = truncation { + // Saying so is the whole difference between a report that is smaller than the truth and + // a report that is quietly wrong. + writeln!( + stream, + "{} {} of {} findings written; SARIF consumers reject a larger log outright", + styler.warning(), + truncation.written, + truncation.found + )?; + } + + drop(stream); + + if !crate::ci::wanted(args.annotations, host.env("GITHUB_ACTIONS").is_some()) { + return Ok(()); + } + + for line in crate::ci::annotations(&plan.mutants, &plan.root) { + writeln!(host.results(), "{line}")?; + } + + if let Some(path) = host.env("GITHUB_STEP_SUMMARY") { + // The diagnosis rides along with the score rather than in a file of its own: the summary + // panel is the artifact a team reads every morning, and a score nobody knows what to do + // about is the reason mutation testing gets run nightly and then ignored. + let mut summary = crate::ci::summary(&plan.mutants, &plan.root); + + if let Some(advice) = advice { + summary.push('\n'); + summary.push_str(advice); + } + + let written = crate::ci::append(Utf8Path::new(&path), &summary); + + if let Err(cause) = written { + let _ = writeln!( + host.error(), + "{} could not write the job summary to `{path}`: {cause}", + styler.warning() + ); + } + } + + Ok(()) +} + +/// Collects the mutants whose `expect_survived` or `expect_killed` directive did not hold. +/// +/// An expectation is a claim about the suite that the author asked to be held to. Parsing it and +/// then ignoring it is worse than not supporting it at all: the directive reads as a guarantee, and +/// the thing it guards can rot indefinitely without anyone hearing about it. +/// +/// Only mutants that actually ran are judged. One that failed to compile or was never reached is +/// not evidence either way, and failing a run over it would make the check depend on whether the +/// build happened to produce that mutant at all. +fn broken_expectations(mutants: &[Mutant]) -> Vec<(&Mutant, &'static str)> { + let mut broken = Vec::new(); + + for mutant in mutants { + let Some(expectation) = &mutant.expectation else { + continue; + }; + + if !mutant.outcome.is_valid() { + continue; + } + + let detected = mutant.outcome.is_detected(); + + if detected != expectation.killed { + broken.push((mutant, if expectation.killed { "killed" } else { "survived" })); + } + } + + broken +} + +/// Fails a run whose gate never got a population to judge. +/// +/// A gate that cannot fail is worse than no gate. Every route to an empty score — an exclude +/// pattern that matched everything, a shard that held nothing but suppressions, a diff that named +/// no code, an incremental run that had already settled the lot — would otherwise end in a +/// summary that said nothing was tested and an exit code that said everything was fine. A job that +/// asked for `--min-score 100` would then pass on the strength of having tested nothing at all, +/// indefinitely, which is the exact failure the flag exists to prevent. +/// +/// A run that asked for no gate is left alone: an empty population is a perfectly ordinary answer +/// to a narrow selection, and turning it into a failure would break every run that never made a +/// claim about its score. +fn ungraded(host: &mut H, args: &RunArgs, styler: Styler, expectations: bool) -> i32 { + let gate = if args.min_score.is_some() { + "the `--min-score` gate" + } else if expectations { + "the expectations this run carried" + } else { + return EXIT_OK; + }; + + let _ = writeln!( + host.error(), + "{} no mutant counted toward the score, so {gate} was never evaluated; \ + check that the selection — `--in-diff`, `--exclude-file`, `--shard-count`/`--shard-index`, `--incremental` — leaves something to test", + styler.error("error:") + ); + + EXIT_GATE_FAILED +} + +/// Renders a failing score and its threshold at just enough precision to tell them apart. +/// +/// The gate compares the full-precision `f64`s — gating on the printed value instead would be a real +/// defect, since a score of 79.96% has not met an 80% bar — but at one decimal place that same near +/// miss renders as "80.0% is below the required 80.0%", a sentence that denies itself. The two are +/// shown at the coarsest precision that still distinguishes them, so the common case stays at one +/// decimal and only a genuine near miss grows extra digits. The caller guarantees `score < minimum`, +/// so the two are distinct reals and the search always terminates at a precision that separates them. +pub(super) fn distinguish(score: f64, minimum: f64) -> (String, String) { + for precision in 1..=12 { + let shown_score = format!("{score:.precision$}"); + let shown_minimum = format!("{minimum:.precision$}"); + + if shown_score != shown_minimum { + return (shown_score, shown_minimum); + } + } + + (format!("{score}"), format!("{minimum}")) +} + +pub(super) fn run_session(host: &mut H, args: &RunArgs, progress_when: When, styler: Styler) -> crate::Result { + let Executed { plan, stuck } = measured(host, args, progress_when, styler)?; + + // A build the tool could not make compile is not a run that passed, whatever the mutants it did + // get to say. It is reported first and it wins: the population the gate would judge is missing + // a part nobody measured, so a score computed over what is left is not the score the gate was + // written for, and letting it decide the exit code would turn a run that got stuck into a green + // tick. The report has already been written by this point, so nothing is lost by refusing. + if !stuck.is_empty() { + return Ok(EXIT_CANNOT_PROCEED); + } + + let Some(plan) = plan else { + // Nothing was generated at all, so no mutant carries an expectation either: whether a gate + // exists is entirely what the command line asked for. + return Ok(ungraded(host, args, styler, false)); + }; + + let summary = crate::model::Summary::of(&plan.mutants); + let broken = broken_expectations(&plan.mutants); + + if !broken.is_empty() { + let mut stream = host.error(); + + for (mutant, wanted) in &broken { + let line = mutant.expectation.as_ref().map_or(mutant.line, |expectation| expectation.line); + let was = mutant.outcome; + let _ = writeln!( + stream, + "{} {}:{line}: `{}` expected {wanted}, but this mutant was {was}", + styler.error("error:"), + mutant.file, + mutant.mutator + ); + } + + let count = broken.len(); + let _ = writeln!( + stream, + "{} {count} {} not hold", + styler.error("error:"), + if count == 1 { "expectation did" } else { "expectations did" } + ); + + return Ok(EXIT_GATE_FAILED); + } + + // Every mutant was suppressed, unviable, or never built, so the score is a ratio with nothing + // in its denominator. That prints as 100%, which is the right answer to "how much of what ran + // was caught" and the wrong one to hand a threshold. + let Some(score) = summary.scored() else { + return Ok(ungraded( + host, + args, + styler, + plan.mutants.iter().any(|mutant| mutant.expectation.is_some()), + )); + }; + + if let Some(minimum) = args.min_score + && score < minimum + { + let (shown_score, shown_minimum) = distinguish(score, minimum); + let mut stream = host.error(); + let _ = writeln!( + stream, + "{} mutation score {shown_score}% is below the required {shown_minimum}%", + styler.error("error:") + ); + + return Ok(EXIT_GATE_FAILED); + } + + Ok(EXIT_OK) +} + +/// Collects the arguments every test binary should receive. +/// +/// `--cargo-test-arg` and everything after `--` mean the same thing to the harness, so they are +/// concatenated in the order they were written rather than kept apart. +fn test_arguments(args: &RunArgs) -> Vec { + let mut collected = args.measure.cargo_test_args.clone(); + + collected.extend(args.measure.test_args.iter().cloned()); + collected +} + +/// Reads the verdicts an earlier report already settled. +/// +/// How many of this run's mutants the record settled for free. +/// +/// Counted against the population rather than taken from the size of the record, because an entry +/// only spares a run if the mutant is still there to spare: a file that was deleted, a `--package` +/// that no longer selects it, or a shard that never held it all leave an entry that matched +/// nothing. +fn adopted_from_cache(plan: &Plan, cached: &crate::HashMap) -> usize { + plan.mutants + .iter() + .filter(|mutant| mutant.outcome == Outcome::CompileError && cached.contains_key(&mutant.id)) + .count() +} + +/// Says how many mutants did not have to be rebuilt to be found unviable again. +/// +/// Said out loud rather than left implicit, because this is on by default and it changes how long a +/// run takes. A user comparing two runs' timings deserves to know which of them started warm. +/// Names the skip directives that suppressed nothing this run, so they can be reconsidered. +/// +/// A skip directive is a standing claim that something there cannot be tested. Once the code under +/// it changes, the claim keeps applying to nothing, and nothing in the report says so — which makes +/// a stale directive indistinguishable from a live one, and leaves the next reader believing a +/// decision that no longer holds. Reporting it is what lets that claim be audited. +/// +/// Each is named with its file, line, selectors and stated reason. It is a note rather than a +/// failure, because a directive can legitimately be idle for a run — a mutant whose site the +/// diff or the shard excluded is not there to be suppressed — and the run's verdict must not turn +/// on how the population happened to be narrowed. +/// +/// See [`crate::suppress::idle`] for exactly which directives reach here. +fn report_idle(host: &mut H, plan: &Plan, styler: Styler) -> crate::Result<()> { + if plan.idle.is_empty() { + return Ok(()); + } + + writeln!( + host.error(), + "{} {} suppressed nothing and may no longer be needed", + styler.verb("Unused"), + quantity(plan.idle.len(), "skip directive") + )?; + + for idle in &plan.idle { + let reason = idle.reason.as_ref().map_or_else(String::new, |reason| format!(" — {reason}")); + + writeln!(host.error(), " {}:{}: skip({}){reason}", idle.file, idle.line, idle.selectors)?; + } + + Ok(()) +} + +fn report_cache(host: &mut H, adopted: usize, styler: Styler) -> crate::Result<()> { + if adopted == 0 { + return Ok(()); + } + + writeln!( + host.error(), + "{} {} known not to compile, carried forward rather than rebuilt", + styler.verb("Cached"), + quantity(adopted, "mutant") + )?; + + Ok(()) +} + +/// Says which part of the build context cost this run the record's unviability. +/// +/// Only the tier that was refused is reported. Unviability is a claim about what compiles, so it +/// requires every term of the context; the probes and the build order the same record holds require +/// none and are used regardless. Saying "the cache did not apply" would send the reader through +/// their whole configuration, and would also be wrong — most of the record still applied. +fn report_context(host: &mut H, moved: &[crate::discover::Term], styler: Styler) -> crate::Result<()> { + let Some(first) = moved.first() else { + return Ok(()); + }; + + let named: Vec<&str> = moved.iter().map(|term| term.name()).collect(); + let axes = if moved.len() == 1 { + first.name().to_owned() + } else { + named.join(", ") + }; + + writeln!( + host.error(), + "{} the record's unviability: {} differs from the run that wrote it. Its build order is still used, so the mutants that failed last time are compiled first", + styler.note("Rebuilding"), + axes + )?; + + Ok(()) +} + +/// Digests the things other than the sources that decide whether a mutant compiles. +/// +/// `None` means this run has no trustworthy key — the compiler could not be asked what it is — and +/// the cache is then neither read nor written. That costs the run the time it would have saved, +/// which is the right side to fail on for a cache whose entries are believed rather than re-checked. +fn cache_context(args: &RunArgs) -> Option { + let features = &args.select.features; + let toolchain = crate::discover::toolchain(); + let rustflags = crate::discover::rustflags(); + + crate::discover::record_context(&crate::discover::RecordContext { + features: &features.features, + all_features: features.all_features, + no_default_features: features.no_default_features, + profile: args.measure.profile.as_deref(), + extra: &args.measure.cargo_args, + rustflags: rustflags.as_deref(), + toolchain: toolchain.as_deref(), + test_packages: &args.measure.test_packages, + include_tests: &args.measure.include_tests, + exclude_tests: &args.measure.exclude_tests, + test_workspace: args.measure.test_workspace, + whole_test_binaries: args.measure.whole_test_binaries, + nextest: args.measure.nextest, + cargo_test_args: &args.measure.cargo_test_args, + test_args: &args.measure.test_args, + baseline: !args.no_baseline, + confirm: !args.no_confirm, + stall: !args.no_stall_detection, + test_timeout_multiplier: args.measure.test_timeout_multiplier, + minimum_test_timeout: args.measure.minimum_test_timeout, + memory: args.measure.memory, + memory_multiplier: args.measure.memory_multiplier, + memory_headroom: args.measure.memory_headroom, + memory_limit: args.measure.memory_limit, + baseline_memory_limit: args.measure.baseline_memory_limit, + no_relaunch: args.measure.no_relaunch, + copy_ignored: args.measure.copy_ignored, + jobs: args.measure.jobs, + build_timeout: args.limits.build_timeout, + build_timeout_multiplier: args.limits.build_timeout_multiplier, + rollback_rounds: args.limits.rollback_rounds, + }) +} + +struct IncrementalPreparation { + base: Utf8PathBuf, + context: crate::discover::ContextDigest, + inputs: crate::discover::WorkspaceSnapshot, +} + +impl IncrementalPreparation { + fn for_run(args: &RunArgs, survey: &crate::discover::Survey, context: crate::discover::ContextDigest) -> Self { + let base = exec::gamma_base(&survey.root, args.measure.cache_dir.as_deref()); + let inputs = crate::discover::RunRecord::snapshot_with_external( + &survey.root, + &base, + survey.external_inputs(), + survey.has_untracked_build_script_inputs(), + ); + + Self { base, context, inputs } + } +} + +fn incremental_context(args: &RunArgs) -> Option { + let mode = args.incremental.unwrap_or(exec::IncrementalMode::Build); + + (!args.dry_run && mode.is_enabled()).then(|| cache_context(args)).flatten() +} + +/// Settles everything the measured run needs from the command line. +pub(super) fn run_config(args: &RunArgs, styler: Styler) -> exec::Config { + exec::Config { + jobs: exec::resolve_jobs(args.measure.jobs), + test_timeout_multiplier: args.measure.test_timeout_multiplier.unwrap_or(DEFAULT_TEST_TIMEOUT_MULTIPLIER), + baseline: !args.no_baseline, + confirm: !args.no_confirm, + stall: !args.no_stall_detection, + cargo: exec::CargoOptions { + features: args.select.features.to_cargo_args(), + profile: args.measure.profile.clone(), + extra: args.measure.cargo_args.clone(), + test_args: test_arguments(args), + color: styler.enabled(), + }, + memory: memory_policy(args), + build: exec::BuildLimits { + timeout: args.limits.build_timeout.map(Duration::from_secs_f64), + multiplier: args.limits.build_timeout_multiplier, + rollback_rounds: args.limits.rollback_rounds, + }, + leak_dirs: args.leak_dirs, + cache_dir: args.measure.cache_dir.clone(), + copy_ignored: args.measure.copy_ignored, + test_packages: args.measure.test_packages.clone(), + include_tests: args.measure.include_tests.clone(), + exclude_tests: args.measure.exclude_tests.clone(), + test_workspace: args.measure.test_workspace, + whole_test_binaries: args.measure.whole_test_binaries, + nextest: args.measure.nextest, + incremental: args.incremental.unwrap_or(exec::IncrementalMode::Build), + timeout_floor: args + .measure + .minimum_test_timeout + .map_or_else(|| exec::Config::default().timeout_floor, Duration::from_secs_f64), + ..exec::Config::default() + } +} + +/// Discovers, runs and reports, returning everything the caller needs to judge the run. +/// +/// Returns the whole [`Executed`], not just the plan it produced. `suppress` derives source edits +/// from these verdicts, and a run that could not build part of its population has verdicts for the +/// rest and none at all for that part — which is a decision only the caller can make, so the +/// stuck builds travel with the plan rather than being dropped on the way out. Split out of +/// [`run_session`] so `suppress` can act on the verdicts rather than re-deriving them from a +/// second run. +pub(super) fn execute(host: &mut H, args: &RunArgs, progress_when: When, styler: Styler) -> crate::Result { + measured(host, args, progress_when, styler) +} + +/// What a run produced: the plan, and whatever the build could not be made to compile. +pub(super) struct Executed { + /// The completed plan, or `None` when nothing was generated at all. + pub(super) plan: Option, + + /// One entry per build the run gave up on, already rendered for a reader. + /// + /// Carried out of the run rather than turned into an error at the point it happens, because + /// the whole point is that the report is still written: a run that got stuck has verdicts, + /// diagnostics and a population worth publishing, and the failure belongs in the exit code + /// rather than in place of all of that. + pub(super) stuck: Vec, +} + +/// Discovers, runs and reports, returning everything [`run_session`] needs to decide an exit code. +/// Everything an earlier run already answered, folded into the survey before anything is built. +/// +/// Incremental mode governs whether compiler unviability is reused from `last-gamma-run.json`. +fn adopt( + args: &RunArgs, + survey: &mut crate::discover::Survey, + base: &Utf8Path, + context: Option<&crate::discover::ContextDigest>, + inputs: &crate::discover::WorkspaceSnapshot, +) -> (crate::HashMap, usize, Vec) { + let mode = args.incremental.unwrap_or(exec::IncrementalMode::Build); + + if !mode.is_enabled() { + return (crate::HashMap::default(), 0, Vec::new()); + } + + // Read before the tree is copied. A dry run is deliberately left out of it: nothing is built, + // so there is nothing to save, and marking a mutant unviable in a listing of what *would* run + // would answer a question nobody asked. + let (recorded, declined, moved) = match context { + Some(context) if !args.dry_run => { + let record = crate::discover::RunRecord::load(base); + + // Asked before the record is consumed, because what it costs the run is decided here + // and reported nowhere else. A record whose unviability is refused still holds a build + // order, so the reader is told which axis moved rather than that the cache "did not + // apply" — the first is something to act on or accept, the second sends them through + // their whole configuration. + // + // Resolved against the workspace first, because the record's own digest was. A term + // one side does not state is not reported as a difference, so comparing against the + // command line's unresolved digest would answer "nothing moved" for precisely the + // configuration terms — the target and the `.cargo/config.toml` body — that a reader + // has no other way to find. + let moved = if record.holds_unviability() { + record.context().differences(&context.resolved_at(&survey.root)) + } else { + Vec::new() + }; + + let (settled, declined) = record.settled_against( + &survey.root, + crate::discover::Trust::Free, + &crate::discover::Killers::default(), + context, + inputs, + ); + + (settled, declined, moved) + } + _absent_or_disabled => (crate::HashMap::default(), 0, Vec::new()), + }; + + // What the record contributed for free, which is what `report_cache` announces. A run that + // asked for more must not have the rest credited to the free tier. + let free: crate::HashMap = recorded + .iter() + .filter(|(_id, outcome)| **outcome == Outcome::CompileError) + .map(|(id, outcome)| (id.clone(), *outcome)) + .collect(); + + if !recorded.is_empty() { + survey.settle(recorded); + } + + (free, declined, moved) +} + +#[expect( + clippy::too_many_lines, + reason = "the command orchestrator keeps its ordered reporting and resource-cleanup paths together" +)] +fn measured(host: &mut H, args: &RunArgs, progress_when: When, styler: Styler) -> crate::Result { + let started = Instant::now(); + let selection = args.select.selection()?; + let shard = args.select.shard()?; + let visible = progress_when.resolve(host.is_terminal()); + let mut progress = Progress::new(visible, styler, host.terminal_width()); + + progress.status(host, "Analyzing", "the workspace"); + + // Settled before discovery rather than just before the run, because discovery evaluates + // `#[cfg(...)]` against the build these options describe. Resolving it once and handing the + // same value to both is what keeps the tree that is surveyed and the tree that is compiled from + // describing different builds — a dry run included, since its listing is a claim about what a + // real run would do. + let config = run_config(args, styler); + let context = incremental_context(args); + let mut survey = crate::discover::Survey::for_build_with_cache_inputs(&args.select, shard, &config.cargo, context.is_some())?; + let artifact_dir = Documents::directory(args, &survey.root); + fs::create_dir_all(&artifact_dir).map_err(|cause| error!("could not create artifact directory `{artifact_dir}`").caused_by(cause))?; + + let incremental = context.map(|context| IncrementalPreparation::for_run(args, &survey, context)); + let cache_locks = if incremental.is_some() && !args.dry_run && args.measure.cache_dir.is_some() { + Some(exec::claim_cache(&survey.root, args.measure.cache_dir.as_deref())?) + } else { + None + }; + let (cached, _declined, moved) = incremental.as_ref().map_or_else( + || (crate::HashMap::default(), 0, Vec::new()), + |prepared| adopt(args, &mut survey, &prepared.base, Some(&prepared.context), &prepared.inputs), + ); + drop(cache_locks); + + // A dry run reports on the whole population and builds nothing, so there is no package-by- + // package sequence to interleave the scan with; it is simply scanned. + if args.dry_run { + let mut ordinals = 0; + let scanned = survey.scan(None, &selection, &mut ordinals)?; + let plan = survey.into_plan(scanned); + + progress.finish(host); + report_idle(host, &plan, styler)?; + + if plan.mutants.is_empty() { + let _ = writeln!(host.error(), "no mutants were generated"); + + return Ok(Executed { + plan: None, + stuck: Vec::new(), + }); + } + + crate::report::summarize(host, &plan, styler, listings(args, false))?; + emit_reports(host, args, &plan, None, None, &[], styler)?; + emit_diag(host, args, &plan, None, started, styler)?; + + return Ok(Executed { + plan: Some(plan), + stuck: Vec::new(), + }); + } + + let mut events = ConsoleEvents { + host, + progress, + styler, + estimate: args.estimate, + show_build: args.measure.show_build, + verdict_log: VerdictLog::default(), + }; + + let outcome = exec::run(&survey, &selection, &config, &mut events); + + // A phase that failed never got to say what it found, so the line it opened is still waiting + // for an ending. Close it before the error is printed, or the error arrives as the rest of + // that sentence. + if outcome.is_err() { + events.abandon(); + } + + let log_result = events.finish_verdict_log(); + + let exec::Measured { + plan, + built, + stuck, + dropped, + } = outcome?; + + let log_failure = log_result.err(); + + let mut progress = events.progress; + + // The live display named every survivor and timeout as it happened, so the summary must not + // name them again. + let announced = progress.is_enabled(); + + progress.finish(host); + + // Written from the whole population so an adopted cache is preserved. Failing here is not failing the run: + // every verdict has already been reached, and a scratch file that could not be written must + // only ever cost the next run some time. + let mode = args.incremental.unwrap_or(exec::IncrementalMode::Build); + if let Some(IncrementalPreparation { base, context, inputs }) = incremental + && let Some(record) = crate::discover::RunRecord::from_plan_snapshot(&plan, &context, inputs, &crate::discover::Killers::default()) + { + let record_locks = if built.is_none() { + match exec::claim_cache(&plan.root, args.measure.cache_dir.as_deref()) { + Ok(locks) => Some(locks), + Err(failure) => { + crate::notes::note(format!("could not lock the run-record cache: {failure}")); + None + } + } + } else { + None + }; + + if built.is_some() || record_locks.is_some() { + record.store(&base, &plan.root); + } + } + + let adopted = adopted_from_cache(&plan, &cached); + + report_cache(host, adopted, styler)?; + report_context(host, &moved, styler)?; + report_idle(host, &plan, styler)?; + + if plan.mutants.is_empty() { + let _ = writeln!(host.error(), "no mutants were generated"); + warn_auxiliary(host, log_failure.as_ref(), styler); + + return Ok(Executed { plan: None, stuck }); + } + + // With nothing live there was no build to pay for, or the build that would have decided the run + // could not be made to compile. Either way the summary already accounts for every mutant — + // suppressed, sharded away, already settled, or never built — and it is written before the + // failure is reported, because a report that exists is the whole point of getting this far. + let Some(mut built) = built else { + crate::report::summarize(host, &plan, styler, listings(args, announced))?; + + emit_reports(host, args, &plan, stuck_panel(&stuck).as_deref(), None, &dropped, styler)?; + emit_diag(host, args, &plan, None, started, styler)?; + report_dropped(host, &dropped, styler)?; + report_stuck(host, &stuck, styler)?; + warn_auxiliary(host, log_failure.as_ref(), styler); + + return Ok(Executed { plan: Some(plan), stuck }); + }; + + if args.leak_dirs { + let tree = exec::scratch_tree(&plan.root, args.measure.cache_dir.as_deref()); + + writeln!(host.error(), "{} {tree}", styler.verb("Kept"))?; + } + + crate::report::summarize(host, &plan, styler, listings(args, announced))?; + let has_suppressible_mutants = plan + .mutants + .iter() + .any(|mutant| matches!(mutant.outcome, Outcome::Timeout | Outcome::OutOfMemory)); + crate::report::session_notes( + host, + &built.session, + missing_hints(mode, &plan.root), + has_suppressible_mutants, + styler, + )?; + + let wall = started.elapsed(); + let panel = summary_panel(args, &plan, &built.session, &stuck, &dropped, wall); + + emit_reports(host, args, &plan, Some(&panel), built.session.tests, &dropped, styler)?; + emit_advice(host, args, &plan, &built.session, wall, styler)?; + emit_diag(host, args, &plan, Some(&built.session), started, styler)?; + report_dropped(host, &dropped, styler)?; + report_stuck(host, &stuck, styler)?; + warn_auxiliary(host, log_failure.as_ref(), styler); + + // Driven here rather than left to the destructor. The scratch tree is a copy of the workspace + // plus its build artifacts, so removing it walks every file in both, and doing that on the way + // out of the function looks from outside like a tool that has finished and will not exit. Its + // failure is said out loud rather than swallowed, and does not fail a run whose every verdict + // is already reached and reported: what is left behind costs the disk, not the answer. + if let Err(failure) = built.work.teardown() { + writeln!(host.error(), "{} {failure}", styler.warning())?; + } + + Ok(Executed { plan: Some(plan), stuck }) +} + +fn warn_auxiliary(host: &mut H, failure: Option<&Error>, styler: Styler) { + if let Some(failure) = failure { + let _ = writeln!(host.error(), "{} {failure}", styler.warning()); + } +} + +/// Renders the builds that could not be made to compile for the job summary. +/// +/// The console says this too, but a console is not an artifact: the summary panel is what a team +/// reads the next morning, and a score with a silently missing population is exactly the thing that +/// should not be readable without the caveat beside it. +fn stuck_panel(stuck: &[String]) -> Option { + if stuck.is_empty() { + return None; + } + + let mut panel = "### Builds that could not be made to compile\n\n".to_owned(); + + for reason in stuck { + panel.push_str("```\n"); + panel.push_str(reason); + panel.push_str("\n```\n"); + } + + Some(panel) +} + +/// Assembles the job summary: the advice, and every caveat that belongs beside a score. +/// +/// The caveats go under the advice rather than in place of it. The run did produce verdicts, and +/// the panel is where a team reads them, so the part of the population nobody could build — and +/// the part of the suite nobody could run — belongs on the same page as the score they are missing +/// from. +fn summary_panel(args: &RunArgs, plan: &Plan, session: &exec::Session, stuck: &[String], dropped: &[String], wall: Duration) -> String { + // The job summary wants a fragment under the heading it already owns; the artifact wants a whole + // document. Same analysis, two shapes. + let mut panel = advice_markdown(args, plan, session, wall, crate::advise::Layout::Embedded); + + for section in [stuck_panel(stuck), dropped_panel(dropped)].into_iter().flatten() { + panel.push('\n'); + panel.push_str(§ion); + } + + panel +} + +/// Renders the dropped test packages for the job summary. +fn dropped_panel(dropped: &[String]) -> Option { + if dropped.is_empty() { + return None; + } + + Some(format!( + "### Test packages dropped from this run\n\n These packages do not compile in this workspace, and neither the selected packages nor the whole workspace would check while they were in it. The run went ahead over the packages being mutated instead of refusing to start.\n\n Their test targets were not built and did not run, so any mutant they would have killed is reported here as a survivor. Fix them, or narrow the run yourself with `--test-package`, before reading this score against one taken over the whole workspace.\n\n```\n{}\n```\n", + dropped.join("\n") + )) +} + +/// Says which test packages the run had to drop, and what that costs the score. +/// +/// A warning rather than an error: every verdict this run reached is real, and the run is not a +/// failure. What it is not is comparable with a run over the whole workspace, and nothing else on +/// screen says so. +fn report_dropped(host: &mut H, dropped: &[String], styler: Styler) -> crate::Result<()> { + if dropped.is_empty() { + return Ok(()); + } + + writeln!( + host.error(), + "{} these packages do not compile, so their tests were neither built nor run and could \ + convict nothing: {}. Mutants they would have killed are reported as survivors.", + styler.warning(), + dropped.join(", ") + )?; + + Ok(()) +} + +/// Says, loudly and last, which builds could not be made to compile. +/// +/// Last because it has to be the thing still on screen when the run ends, and on the error stream +/// because it is a failure: the report beside it is real, but it describes a population the tool +/// could not finish judging, and a reader who scrolls past this line would take a partial answer +/// for a complete one. +fn report_stuck(host: &mut H, stuck: &[String], styler: Styler) -> crate::Result<()> { + for reason in stuck { + writeln!(host.error(), "{} {reason}", styler.error("error:"))?; + } + + Ok(()) +} + +/// Dumps the run's own numbers, when the hidden `--diag` asked for them. +/// +/// Last, and to the diagnostic stream, because it is neither a result nor something a person +/// reading the summary asked to see. +fn emit_diag( + host: &mut H, + args: &RunArgs, + plan: &Plan, + session: Option<&exec::Session>, + started: Instant, + styler: Styler, +) -> crate::Result<()> { + let jobs = exec::resolve_jobs(args.measure.jobs); + + if args.diag { + write!(host.error(), "\n{}", crate::diag::render(plan, session, jobs, started.elapsed()))?; + } + + emit_diag_bundle(host, args, plan, session, jobs, started, styler) +} + +/// Writes the diagnostics bundle. +/// +/// Written on every run, like the reports beside it: the file is only useful if it is already there +/// when someone decides the run was slow, and a flag they have to know about beforehand is a flag +/// nobody has set by the time it matters. +fn emit_diag_bundle( + host: &mut H, + args: &RunArgs, + plan: &Plan, + session: Option<&exec::Session>, + jobs: usize, + started: Instant, + styler: Styler, +) -> crate::Result<()> { + let path = Documents::resolve(args, &plan.root).diag; + let context = crate::diag::Context { + cores: exec::available_parallelism(), + jobs, + wall: started.elapsed(), + mutators: args.select.selection().map(|selection| selection.sorted()).unwrap_or_default(), + shard: args.select.shard().ok().flatten(), + + // Only when `--diag` asked for it. Measuring it is a walk of every build artifact the run + // produced, which on a large workspace costs more than the figure is worth to a run that + // did not ask a question about disk. + scratch_bytes: session + .filter(|_measured| args.diag) + .map(|session| exec::footprint(&session.scratch)), + redaction: args.diag_names, + version: env!("CARGO_PKG_VERSION"), + }; + + let bundle = crate::diag::bundle(plan, session, &context); + + crate::elements::write(&path, &crate::diag::to_json(&bundle)?) + .map_err(|cause| crate::error::error!("could not write the diagnostics bundle to `{path}`").caused_by(cause))?; + + writeln!(host.error(), "{} {}", styler.verb("Wrote"), path)?; + + Ok(()) +} + +/// Writes the Markdown diagnosis. +fn emit_advice( + host: &mut H, + args: &RunArgs, + plan: &Plan, + session: &exec::Session, + wall: Duration, + styler: Styler, +) -> crate::Result<()> { + let path = Documents::resolve(args, &plan.root).advice; + let advice = advice_markdown(args, plan, session, wall, crate::advise::Layout::Document); + + // Whole or not at all, like the reports beside it: a half-written diagnosis is one a reader + // takes for the whole story. + crate::elements::write(&path, &advice) + .map_err(|cause| crate::error::error!("could not write the advice to `{path}`").caused_by(cause))?; + + writeln!(host.error(), "{} {path}", styler.verb("Wrote"))?; + + Ok(()) +} + +/// Renders the diagnosis and the family table as Markdown. +/// +/// The family table is part of the diagnosis rather than a separate feature: knowing that a run's +/// time went somewhere is only actionable alongside what that somewhere caught. +fn advice_markdown(args: &RunArgs, plan: &Plan, session: &exec::Session, wall: Duration, layout: crate::advise::Layout) -> String { + let timing = crate::advise::Timing { + build: session.build, + baseline: session.baseline_wall, + wall, + jobs: exec::resolve_jobs(args.measure.jobs), + }; + + let findings = crate::advise::analyze_run(&plan.mutants, &timing, args.measure.profile.as_deref(), &session.binaries); + + let summary = crate::model::Summary::of(&plan.mutants); + + crate::advise::render_markdown(&findings, &crate::advise::yields(&plan.mutants), summary, &timing, layout) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::super::cli::MeasureArgs; + use super::*; + use crate::discover::TargetFile; + use crate::exec::Session; + use crate::fixtures; + + /// Lays out an empty single-package `subject` workspace in `dir` and returns its root. + /// + /// Only the manifest and the `src` directory: what the source under test is varies per test, + /// so each caller writes its own `src/lib.rs`. + fn subject_root(dir: &tempfile::TempDir) -> Utf8PathBuf { + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + fs::create_dir(root.join("src")).expect("src"); + fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"subject\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n", + ) + .expect("manifest"); + root + } + + use crate::testing::{Broken, Sink, fails_at_every_line, workdir}; + + #[derive(Debug, Default)] + struct ClosedResults { + diagnostics: Vec, + } + + impl Host for ClosedResults { + fn output(&mut self) -> impl Write { + Broken + } + + fn error(&mut self) -> impl Write { + &mut self.diagnostics + } + + fn is_terminal(&self) -> bool { + false + } + + fn terminal_width(&self) -> Option { + None + } + } + + #[test] + fn disabled_incremental_mode_suppresses_missing_hints_advice() { + let dir = workdir("run-missing-hints-"); + let root = Utf8Path::from_path(dir.path()).expect("UTF-8 work directory"); + + assert!(missing_hints(exec::IncrementalMode::Build, root)); + assert!(!missing_hints(exec::IncrementalMode::No, root)); + } + + #[test] + fn disabled_incremental_and_dry_runs_prepare_no_cache_context() { + let disabled = RunArgs { + incremental: Some(exec::IncrementalMode::No), + ..RunArgs::default() + }; + let dry = RunArgs { + dry_run: true, + ..RunArgs::default() + }; + + assert!(incremental_context(&disabled).is_none()); + assert!(incremental_context(&dry).is_none()); + } + + #[test] + fn discovery_resolves_external_inputs_only_for_cache_provenance() { + let dir = workdir("run-cache-inputs-"); + let container = Utf8Path::from_path(dir.path()).expect("UTF-8 work directory"); + let root = container.join("workspace"); + let dependency = container.join("dependency"); + fs::create_dir_all(root.join("src")).expect("workspace source"); + fs::create_dir_all(dependency.join("src")).expect("dependency source"); + fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"subject\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\ + [dependencies]\ndependency = { path = \"../dependency\" }\n\n[workspace]\n", + ) + .expect("workspace manifest"); + fs::write(root.join("src/lib.rs"), "pub fn subject() {}\n").expect("workspace source"); + fs::write( + dependency.join("Cargo.toml"), + "[package]\nname = \"dependency\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ) + .expect("dependency manifest"); + fs::write(dependency.join("src/lib.rs"), "pub fn dependency() {}\n").expect("dependency source"); + let select = crate::commands::SelectArgs { + dir: root, + ..crate::commands::SelectArgs::default() + }; + let cargo = exec::CargoOptions::default(); + + let uncached = crate::discover::Survey::for_build_with_cache_inputs(&select, None, &cargo, false).expect("uncached survey"); + let cached = crate::discover::Survey::for_build_with_cache_inputs(&select, None, &cargo, true).expect("cached survey"); + + assert!(uncached.external_inputs().is_empty()); + assert_eq!( + cached.external_inputs(), + &[crate::paths::physical(&dependency).expect("dependency path")] + ); + } + + fn plan() -> Plan { + let dir = workdir("run-plan-"); + let root = Utf8PathBuf::from_path_buf(dir.keep()).expect("utf8"); + let src = root.join("src"); + fs::create_dir(&src).expect("src"); + let source = "pub fn less(a: i32, b: i32) -> bool { a < b }\n"; + let absolute = src.join("lib.rs"); + fs::write(&absolute, source).expect("source"); + let start = source.find("a < b").expect("span"); + + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root, + files: vec![TargetFile { + path: Utf8PathBuf::from("src/lib.rs"), + absolute, + package: "subject".to_owned(), + }], + mutants: vec![mutant(start, Outcome::Survived, 120), mutant(start, Outcome::NoCoverage, 80)], + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + } + } + + fn mutant(start: usize, outcome: Outcome, elapsed_ms: u64) -> Mutant { + Mutant { + id: format!("m{elapsed_ms}").into(), + span: start..start + 5, + column: start + 1, + item_path: ("subject::less".to_owned()).into(), + original: "a < b".to_owned().into(), + replacement: "a <= b".to_owned().into(), + outcome, + elapsed_ms, + ..fixtures::mutant() + } + } + + fn expecting(outcome: Outcome, killed: bool) -> Mutant { + Mutant { + expectation: Some(crate::model::Expectation { + killed, + line: 3, + reason: None, + }), + ..mutant(0, outcome, 1) + } + } + + /// Naming only `--memory-limit` implies enforcement without the caller having to also spell + /// out `--memory enforce`; failing to imply it would mean a ceiling that was asked for + /// silently does nothing, which is the exact surprise the memory guard exists to prevent. + #[test] + fn a_memory_limit_alone_implies_enforcement() { + let args = RunArgs { + measure: MeasureArgs { + memory_limit: Some(1024), + ..MeasureArgs::default() + }, + ..Default::default() + }; + + let policy = memory_policy(&args); + + assert_eq!(policy.control, exec::MemoryControl::Enforce); + assert_eq!(policy.demand, exec::Demand::Stated); + } + + /// Naming only `--baseline-memory-limit` implies measurement rather than enforcement, because + /// a ceiling on the baseline alone is a request to observe, not to kill the run over. + #[test] + fn a_baseline_memory_limit_alone_implies_measurement() { + let args = RunArgs { + measure: MeasureArgs { + baseline_memory_limit: Some(1024), + ..MeasureArgs::default() + }, + ..Default::default() + }; + + let policy = memory_policy(&args); + + assert_eq!(policy.control, exec::MemoryControl::Measure); + assert_eq!(policy.demand, exec::Demand::Stated); + } + + /// Resource exhaustion satisfies `expect_survived`, not `expect_killed`, because no assertion + /// rejected the mutant. + #[test] + fn every_scoring_outcome_is_judged_by_assertion_detection() { + let mutants = vec![ + expecting(Outcome::Killed, true), + expecting(Outcome::Survived, false), + expecting(Outcome::Timeout, false), + expecting(Outcome::OutOfMemory, false), + ]; + + assert!(broken_expectations(&mutants).is_empty()); + } + + #[test] + fn an_expectation_that_does_not_hold_is_reported_with_what_was_wanted() { + let mutants = vec![expecting(Outcome::Survived, true), expecting(Outcome::Killed, false)]; + let broken = broken_expectations(&mutants); + + assert_eq!(broken.len(), 2); + assert_eq!(broken[0].1, "killed"); + assert_eq!(broken[1].1, "survived"); + } + + #[test] + fn an_expectation_on_a_mutant_that_never_ran_is_not_judged() { + // A mutant that failed to compile or that nothing reaches is not evidence about the suite + // either way, so holding the author to a claim about it would fail runs for no reason. + let mutants = vec![ + expecting(Outcome::CompileError, true), + expecting(Outcome::Ignored, true), + expecting(Outcome::Pending, true), + ]; + + assert!(broken_expectations(&mutants).is_empty()); + } + + #[test] + fn an_uncovered_mutant_is_judged_against_its_expectation() { + // Nothing reaching a site is exactly what `expect_survived` claims, and the opposite of what + // `expect_killed` claims, so both are real answers. + assert!(broken_expectations(&[expecting(Outcome::NoCoverage, false)]).is_empty()); + assert_eq!(broken_expectations(&[expecting(Outcome::NoCoverage, true)]).len(), 1); + } + + #[test] + fn a_mutant_with_no_expectation_is_never_reported() { + assert!(broken_expectations(&[mutant(0, Outcome::Survived, 1)]).is_empty()); + } + + fn session() -> Session { + Session { + census: Vec::new(), + baseline: Duration::from_millis(20), + baseline_wall: Duration::from_millis(20), + tests: None, + quiet: Duration::from_millis(10), + stall: Some(Duration::from_millis(30)), + build: Duration::from_millis(40), + metered: false, + unbounded: None, + withdrawn: 0, + rounds: 1, + rounds_taken: Vec::new(), + binaries: Vec::new(), + peak: None, + scratch: Utf8PathBuf::new(), + filtered: 0, + widened: false, + ordering: exec::OrderingHints::default(), + phases: exec::Phases::default(), + } + } + + #[test] + fn advice_uses_elapsed_wall_time_for_concurrent_baselines_and_sparse_workers() { + let mut measured = session(); + measured.build = Duration::from_secs(7); + measured.baseline = Duration::from_secs(5); + measured.baseline_wall = Duration::from_secs(2); + let mut args = RunArgs::default(); + + args.measure.jobs = Some(8); + + let text = advice_markdown(&args, &plan(), &measured, Duration::from_secs(20), crate::advise::Layout::Document); + + assert!(text.contains("| Build | 7.0s | 35% |"), "{text}"); + assert!(text.contains("| Baseline | 2.0s | 10% |"), "{text}"); + assert!(text.contains("| Testing mutants | 11.0s | 55% |"), "{text}"); + assert!(text.contains("45% of the run was the build and baseline"), "{text}"); + } + + /// Every line the report writer emits is checked to propagate a closed stream. + #[test] + fn a_closed_stream_stops_the_report_writer_at_whichever_line_it_reached() { + let dir = workdir("run-reports-closed-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let plan = plan(); + + // A run that has written a SARIF log, a JSON report and an HTML report emits several + // lines; a pipe that closes partway through must fail the command rather than be ignored, + // or CI would record a success for reports nobody received. + fails_at_every_line(3, |host| { + let args = RunArgs { + artifact_dir: Some(root.clone()), + html_external: true, + annotations: crate::ci::Annotations::None, + ..Default::default() + }; + + emit_reports(host, &args, &plan, None, None, &[], Styler::new(false)) + }); + } + + /// Annotations go to the results stream, where a closed consumer is successful completion. + #[test] + fn a_closed_results_stream_ends_annotation_writing_successfully() { + let plan = plan(); + let args = RunArgs { + annotations: crate::ci::Annotations::Github, + ..Default::default() + }; + + emit_ci(&mut ClosedResults::default(), &args, &plan, None, Styler::new(false)).expect("closed results pipe"); + } + + #[test] + fn reports_write_json_html_sarif_annotations_and_summary() { + let dir = workdir("run-reports-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let summary = root.join("summary.md"); + let args = RunArgs { + artifact_dir: Some(root.join("reports")), + html_external: true, + annotations: crate::ci::Annotations::Github, + ..Default::default() + }; + + let mut host = Sink::default().with_env("GITHUB_STEP_SUMMARY", summary.as_str()); + let plan = plan(); + + emit_reports(&mut host, &args, &plan, Some("embedded advice"), None, &[], Styler::new(false)).expect("reports"); + + let out = String::from_utf8(host.out).expect("utf-8"); + let err = String::from_utf8(host.err).expect("utf-8"); + let summary_text = fs::read_to_string(summary).expect("summary"); + + let documents = Documents::resolve(&args, &plan.root); + + assert!(documents.json.exists()); + assert!(documents.html.exists()); + assert!(documents.sarif.exists()); + assert!(out.contains("::warning"), "{out}"); + assert!(err.contains("Wrote"), "{err}"); + assert!(summary_text.contains("embedded advice"), "{summary_text}"); + } + + /// A run that names no report path still gets every report under the gamma directory. + /// + /// This is the whole point of the change that made them default: the run that answers a + /// question about your suite is the one you cannot afford to repeat because the flag was + /// forgotten. + #[test] + fn a_run_that_asks_for_nothing_still_writes_all_reports_under_the_gamma_directory() { + let mut host = Sink::default(); + let plan = plan(); + let args = RunArgs::default(); + let base = plan.root.join("target/cargo-gamma"); + + emit_reports(&mut host, &args, &plan, None, None, &[], Styler::new(false)).expect("reports"); + + assert!(base.join("gamma-report.json").exists()); + assert!(base.join("gamma-report.html").exists()); + assert!(base.join("gamma-report.sarif").exists()); + } + + #[test] + fn default_document_names_identify_their_gamma_artifacts() { + let plan = plan(); + let base = plan.root.join("target/cargo-gamma"); + let documents = Documents::resolve(&RunArgs::default(), &plan.root); + + assert_eq!(documents.json, base.join("gamma-report.json")); + assert_eq!(documents.html, base.join("gamma-report.html")); + assert_eq!(documents.advice, base.join("gamma-perf-advice.md")); + assert_eq!(documents.diag, base.join("gamma-diagnostics.json")); + assert_eq!(documents.sarif, base.join("gamma-report.sarif")); + } + + /// Naming an artifact directory moves every artifact and leaves the default directory alone. + #[test] + fn naming_an_artifact_directory_moves_every_report() { + let dir = workdir("run-report-override-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + let plan = plan(); + let base = plan.root.join("target/cargo-gamma"); + let artifact_dir = root.join("elsewhere"); + let args = RunArgs { + artifact_dir: Some(artifact_dir.clone()), + ..Default::default() + }; + + emit_reports(&mut host, &args, &plan, None, None, &[], Styler::new(false)).expect("reports"); + + assert!(artifact_dir.join("gamma-report.json").exists()); + assert!(artifact_dir.join("gamma-report.html").exists()); + assert!(artifact_dir.join("gamma-report.sarif").exists()); + assert!(!base.join("gamma-report.json").exists()); + assert!(!base.exists()); + } + + /// Cache relocation is internal state and must not move user-facing reports. + #[test] + fn a_cache_directory_does_not_move_the_default_reports() { + let dir = workdir("run-report-scratch-"); + let cache = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + let plan = plan(); + let args = RunArgs { + measure: MeasureArgs { + cache_dir: Some(cache.clone()), + ..Default::default() + }, + ..Default::default() + }; + + emit_reports(&mut host, &args, &plan, None, None, &[], Styler::new(false)).expect("reports"); + + assert!(plan.root.join("target/cargo-gamma/gamma-report.json").exists()); + assert!(!cache.join("gamma-report.json").exists()); + } + + /// A default report location that cannot be written fails the run rather than being skipped. + /// + /// Defaulting the reports made this reachable without anyone asking for it: before, a report + /// only existed because a flag named it, and a caller who named a path was there to see it + /// fail. Now every run writes three files nobody requested, and the temptation to make those + /// writes best-effort is real. It must be resisted — a run that spent an hour and produced no + /// report has failed, and exiting zero would hide that behind a score printed to a terminal + /// that CI throws away. + /// + /// The block is a *directory* standing where each default report file belongs, which the + /// writer cannot rename a file onto. Blocking one artifact at a time is what makes the test + /// say which write was dropped, rather than merely that some write was. + #[test] + fn a_default_report_path_that_cannot_be_written_fails_the_run() { + for blocked in [ + "gamma-report.json", + "gamma-report.html", + "gamma-report.sarif", + "gamma-perf-advice.md", + ] { + let dir = workdir("run-report-blocked-"); + let artifact_dir = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let plan = plan(); + let base = artifact_dir.clone(); + + fs::create_dir_all(base.join(blocked).as_std_path()).expect("blocker"); + + let mut host = Sink::default(); + let args = RunArgs { + artifact_dir: Some(artifact_dir), + measure: MeasureArgs { + jobs: Some(0), + ..Default::default() + }, + ..Default::default() + }; + + let error = if blocked == "gamma-perf-advice.md" { + emit_advice(&mut host, &args, &plan, &session(), Duration::from_secs(20), Styler::new(false)) + } else { + emit_reports(&mut host, &args, &plan, None, None, &[], Styler::new(false)) + } + .expect_err("a blocked default artifact path is an error"); + + assert!(error.to_string().contains(blocked), "{blocked}: {error}"); + } + } + + /// When no advice was generated, the job summary still gets written with the score alone; a + /// summary that only appeared once advice existed would make the panel disappear on exactly + /// the runs where nothing needed following up. + #[test] + fn a_step_summary_omits_advice_when_none_was_given() { + let dir = workdir("run-summary-no-advice-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let summary = root.join("summary.md"); + let args = RunArgs { + annotations: crate::ci::Annotations::Github, + ..Default::default() + }; + + let mut host = Sink::default().with_env("GITHUB_STEP_SUMMARY", summary.as_str()); + let plan = plan(); + + emit_ci(&mut host, &args, &plan, None, Styler::new(false)).expect("ci"); + + let summary_text = fs::read_to_string(summary).expect("summary"); + + assert!(!summary_text.is_empty(), "{summary_text}"); + } + + /// A broken job-summary path is auxiliary: it is warned about only after the durable reports + /// have been written, and cannot turn a completed run into one that "could not proceed". + #[test] + fn an_empty_step_summary_path_warns_after_writing_the_reports() { + let plan = plan(); + let base = plan.root.join("target/cargo-gamma"); + let args = RunArgs { + annotations: crate::ci::Annotations::Github, + ..Default::default() + }; + + let mut host = Sink::default().with_env("GITHUB_STEP_SUMMARY", ""); + + emit_reports(&mut host, &args, &plan, None, None, &[], Styler::new(false)).expect("primary reports"); + + assert!(base.join("gamma-report.json").exists()); + assert!(base.join("gamma-report.html").exists()); + assert!(host.err().contains("could not write the job summary"), "{}", host.err()); + } + + /// A closed results stream that stops the SARIF truncation note must fail the command, just + /// like every other line the run writes; letting the note alone go unenforced would mean the + /// one line that says the log is incomplete is exactly the one line allowed to vanish. + #[test] + fn a_closed_stream_is_reported_by_the_sarif_truncation_note() { + let dir = workdir("run-sarif-truncation-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut plan = plan(); + let survivor = plan.mutants[0].clone(); + + plan.mutants = (0..=crate::ci::sarif::SARIF_LIMIT) + .map(|index| { + let mut mutant = survivor.clone(); + + mutant.id = format!("m{index}").into(); + mutant + }) + .collect(); + + // The SARIF log is written, then the "Wrote" line, then the truncation note; both of the + // latter go to the error stream, so closing the stream at either point must fail the run. + fails_at_every_line(2, |host| { + let args = RunArgs { + artifact_dir: Some(root.clone()), + annotations: crate::ci::Annotations::None, + ..Default::default() + }; + + emit_ci(host, &args, &plan, None, Styler::new(false)) + }); + } + + /// The advice document goes in the artifact directory, while prose diagnostics remain opt-in. + #[test] + fn advice_goes_where_it_was_asked_for_and_diag_is_emitted_only_when_requested() { + let dir = workdir("run-advice-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut args = RunArgs::default(); + + args.measure.jobs = Some(0); + args.artifact_dir = Some(root.clone()); + args.diag = true; + + let plan = plan(); + let session = session(); + let mut host = Sink::default(); + + emit_advice(&mut host, &args, &plan, &session, Duration::from_secs(20), Styler::new(false)).expect("advice"); + emit_diag(&mut host, &args, &plan, Some(&session), Instant::now(), Styler::new(false)).expect("diag"); + + let advice = fs::read_to_string(root.join("gamma-perf-advice.md")).expect("advice file"); + let err = String::from_utf8(host.err).expect("utf-8"); + + assert!(advice.contains("Mutation testing"), "{advice}"); + assert!(err.contains("Wrote"), "{err}"); + assert!(err.contains("diag"), "{err}"); + } + + /// The bundle exists to be attached to an issue, which only works if it is already on disk by + /// the time anyone decides the run was worth reporting. + #[test] + fn every_run_writes_the_diagnostics_bundle_without_being_asked() { + let mut args = RunArgs::default(); + + args.measure.jobs = Some(0); + + let plan = plan(); + let session = session(); + let mut host = Sink::default(); + + emit_diag(&mut host, &args, &plan, Some(&session), Instant::now(), Styler::new(false)).expect("diag"); + + let path = Documents::resolve(&args, &plan.root).diag; + let text = fs::read_to_string(&path).expect("bundle file"); + let parsed: serde_json::Value = serde_json::from_str(&text).expect("valid json"); + + assert!(!args.diag, "the prose dump was not asked for and must not be the trigger"); + assert_eq!(parsed["schemaVersion"], "3"); + assert!(parsed["run"]["wallMs"].is_number(), "{text}"); + + // The phase profile is what makes the census's cost and its sweep dividend readable from one + // run. The copy, preflight and baseline always ran, so they are always present and named in + // camelCase; this session had no census, so that phase is omitted rather than a lying zero. + assert!(parsed["phases"]["copy"]["elapsedMs"].is_number(), "{text}"); + assert!(parsed["phases"]["preflight"]["elapsedMs"].is_number(), "{text}"); + assert!(parsed["phases"]["baseline"]["elapsedMs"].is_number(), "{text}"); + assert!(parsed["phases"]["census"].is_null(), "no census ran, so it is omitted: {text}"); + + assert!(String::from_utf8(host.err).expect("utf-8").contains(path.as_str())); + } + + /// A run that censused says so in its bundle: the phase carries its own elapsed time, the tests + /// it walked, and the binaries it examined, so the census's cost is legible on its own rather + /// than folded inside the build's total. + #[test] + fn a_censused_run_records_the_census_phase_in_its_bundle() { + let mut args = RunArgs::default(); + + args.measure.jobs = Some(0); + + let plan = plan(); + let mut session = session(); + + session.phases.census = Some(exec::CensusCost { + elapsed: Duration::from_secs(8), + walked: 1_681, + binaries: 30, + }); + + let mut host = Sink::default(); + + emit_diag(&mut host, &args, &plan, Some(&session), Instant::now(), Styler::new(false)).expect("diag"); + + let path = Documents::resolve(&args, &plan.root).diag; + let text = fs::read_to_string(&path).expect("bundle file"); + let parsed: serde_json::Value = serde_json::from_str(&text).expect("valid json"); + + assert_eq!(parsed["phases"]["census"]["elapsedMs"], 8_000); + assert_eq!(parsed["phases"]["census"]["walked"], 1_681); + assert_eq!(parsed["phases"]["census"]["binaries"], 30); + } + + /// The promise the file is worth attaching on: nothing in it names the tree it came from. + #[test] + fn the_diagnostics_bundle_carries_neither_source_text_nor_a_path() { + let dir = workdir("run-bundle-safety-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut args = RunArgs::default(); + + args.measure.jobs = Some(0); + args.artifact_dir = Some(root.clone()); + + let plan = plan(); + let mut host = Sink::default(); + + emit_diag(&mut host, &args, &plan, None, Instant::now(), Styler::new(false)).expect("diag"); + + let text = fs::read_to_string(root.join("gamma-diagnostics.json")).expect("bundle file"); + + assert!(!text.contains("a < b"), "the mutated source reached the bundle: {text}"); + assert!(!text.contains("src/lib.rs"), "a file path reached the bundle: {text}"); + assert!(!text.contains(plan.root.as_str()), "the workspace root reached the bundle: {text}"); + assert!(!text.contains("subject"), "a package name reached the bundle: {text}"); + } + + /// `--diag-names names` is for a tree whose names are already public, and it must actually + /// change what is written or the flag is decoration. + #[test] + fn naming_is_opt_in_and_changes_what_is_written() { + let dir = workdir("run-bundle-names-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut args = RunArgs::default(); + + args.measure.jobs = Some(0); + args.artifact_dir = Some(root.clone()); + args.diag_names = crate::diag::Redaction::Names; + + let plan = plan(); + let mut host = Sink::default(); + + emit_diag(&mut host, &args, &plan, None, Instant::now(), Styler::new(false)).expect("diag"); + + let text = fs::read_to_string(root.join("gamma-diagnostics.json")).expect("bundle file"); + + assert!(text.contains("subject"), "{text}"); + assert!(!text.contains("a < b"), "even named, no source text: {text}"); + } + + /// A run that names no advice path still gets the document, beside the reports. + #[test] + fn a_run_that_asks_for_no_advice_path_still_writes_it_under_the_gamma_directory() { + let mut args = RunArgs::default(); + + args.measure.jobs = Some(0); + + let plan = plan(); + let session = session(); + let mut host = Sink::default(); + let base = plan.root.join("target/cargo-gamma"); + + fs::create_dir_all(&base).expect("gamma base"); + emit_advice(&mut host, &args, &plan, &session, Duration::from_secs(20), Styler::new(false)).expect("advice"); + + assert!(base.join("gamma-perf-advice.md").exists()); + } + + /// A leftover foreign config is called out, because the run does not honour it. + #[test] + fn a_foreign_config_is_called_out_before_anything_is_loaded() { + let dir = workdir("run-foreign-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + fs::create_dir_all(root.join(".cargo")).expect("cargo dir"); + fs::write(root.join(".cargo/mutants.toml"), "examine_globs = [\"src/**\"]\n").expect("foreign"); + + let mut args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + ..crate::commands::SelectArgs::default() + }, + ..Default::default() + }; + let mut host = Sink::default(); + + configure(&mut host, &mut args, Styler::new(false)).expect("configure"); + + assert!(host.err().contains("Hint"), "{}", host.err()); + assert!(host.err().contains("is not supported or read"), "{}", host.err()); + assert!(host.err().contains("configure gamma.toml explicitly"), "{}", host.err()); + } + + /// Asking not to read config suppresses the note along with the loading. + #[test] + fn a_foreign_config_is_not_mentioned_when_config_is_disabled() { + let dir = workdir("run-foreign-off-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + fs::create_dir_all(root.join(".cargo")).expect("cargo dir"); + fs::write(root.join(".cargo/mutants.toml"), "examine_globs = [\"src/**\"]\n").expect("foreign"); + + let mut select = crate::commands::SelectArgs { + dir: root, + ..crate::commands::SelectArgs::default() + }; + + select.config.no_config = true; + + let mut args = RunArgs { + select, + ..Default::default() + }; + let mut host = Sink::default(); + + configure(&mut host, &mut args, Styler::new(false)).expect("configure"); + + assert!(host.err().is_empty(), "{}", host.err()); + } + + /// A directive that suppressed nothing must be named with enough to go and find it: the file, + /// the line, what it claimed, and why it said it was needed. + #[test] + fn an_unused_skip_directive_is_named_with_its_place_and_its_reason() { + let mut plan = plan(); + + plan.idle = vec![crate::suppress::Idle { + file: Utf8PathBuf::from("src/lib.rs"), + line: 42, + selectors: "arith".to_owned(), + reason: Some("the compiler folds it".to_owned()), + }]; + + let mut host = Sink::default(); + + report_idle(&mut host, &plan, Styler::new(false)).expect("note"); + + let text = host.err(); + + assert!(text.contains("1 skip directive"), "{text}"); + assert!(text.contains("src/lib.rs:42"), "{text}"); + assert!(text.contains("skip(arith)"), "{text}"); + assert!(text.contains("the compiler folds it"), "{text}"); + } + + /// The quiet case, and the one that makes the loud case worth trusting. + #[test] + fn a_run_whose_skips_all_still_apply_says_nothing_about_them() { + let plan = plan(); + let mut host = Sink::default(); + + report_idle(&mut host, &plan, Styler::new(false)).expect("note"); + + assert!(host.err().is_empty(), "{}", host.err()); + } + + /// Every idle directive is named, not just the first: a tree that has drifted has drifted in + /// several places, and a report that stops after one leaves the rest looking deliberate. + #[test] + fn every_unused_skip_directive_is_named() { + let mut plan = plan(); + + plan.idle = (1..=3) + .map(|line| crate::suppress::Idle { + file: Utf8PathBuf::from(format!("src/m{line}.rs")), + line, + selectors: "arith".to_owned(), + reason: None, + }) + .collect(); + + let mut host = Sink::default(); + + report_idle(&mut host, &plan, Styler::new(false)).expect("note"); + + let text = host.err(); + + assert!(text.contains("3 skip directives"), "{text}"); + + for line in 1..=3 { + assert!(text.contains(&format!("src/m{line}.rs:{line}")), "{text}"); + } + } + + /// The build discovery is given is the build the run will actually perform. + /// + /// `measured` resolves the run configuration once and hands `config.cargo` to both + /// `Survey::for_build` and `exec::run`, because discovery evaluates `#[cfg(...)]` against it. If + /// the command line stopped reaching these options, a run under `--profile release --cargo-arg + /// --target=…` would be surveyed as a different build and every item behind a gate those + /// settings decide would be mutated wrongly or not at all — silently, and only in the + /// configurations nobody tests locally. + #[test] + fn the_build_discovery_is_given_carries_the_command_lines_profile_and_arguments() { + let args = RunArgs { + measure: MeasureArgs { + profile: Some("release".to_owned()), + cargo_args: vec!["--target=x86_64-pc-solaris".to_owned()], + ..MeasureArgs::default() + }, + ..Default::default() + }; + + let cargo = run_config(&args, Styler::new(false)).cargo; + let build = cargo.cfg_build(Utf8Path::new(".")); + + assert_eq!(cargo.profile.as_deref(), Some("release")); + assert_eq!(build.target.as_deref(), Some("x86_64-pc-solaris")); + } + + /// With nothing on the command line, a run inherits the 50% default margin. + /// + /// The default is pinned to both the shared constant and its literal value, so raising or + /// lowering it is a deliberate edit here rather than a silent change to every default run. + #[test] + fn a_run_with_no_multiplier_uses_the_default_margin() { + let config = run_config(&RunArgs::default(), Styler::new(false)); + + // Bit-exact equality on values that are exactly representable, which also keeps the pedantic + // `float_cmp` lint quiet on a deliberate exact comparison. + assert_eq!(config.test_timeout_multiplier.to_bits(), DEFAULT_TEST_TIMEOUT_MULTIPLIER.to_bits()); + assert_eq!(config.test_timeout_multiplier.to_bits(), 1.5_f64.to_bits()); + } + + /// An explicit multiplier wins outright over the default, whichever way the default moves. + #[test] + fn an_explicit_multiplier_overrides_the_default_margin() { + let args = RunArgs { + measure: MeasureArgs { + test_timeout_multiplier: Some(2.0), + ..MeasureArgs::default() + }, + ..Default::default() + }; + + assert_eq!( + run_config(&args, Styler::new(false)).test_timeout_multiplier.to_bits(), + 2.0_f64.to_bits() + ); + } + + /// A survivor count past the SARIF ceiling is said out loud rather than silently trimmed. + #[test] + fn a_truncated_sarif_log_says_how_much_it_left_out() { + let dir = workdir("run-sarif-truncated-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let args = RunArgs { + artifact_dir: Some(root), + ..Default::default() + }; + + let mut plan = plan(); + let survivor = plan.mutants[0].clone(); + + plan.mutants = (0..=crate::ci::sarif::SARIF_LIMIT) + .map(|index| { + let mut mutant = survivor.clone(); + + mutant.id = format!("m{index}").into(); + mutant + }) + .collect(); + + let mut host = Sink::default(); + + emit_ci(&mut host, &args, &plan, None, Styler::new(false)).expect("ci"); + + assert!(host.err().contains("warning"), "{}", host.err()); + assert!(host.err().contains("findings written"), "{}", host.err()); + assert!(host.err().contains("reject a larger log outright"), "{}", host.err()); + } + + /// A mutant whose `expect_survived` directive did not hold — because the suite actually caught + /// it — fails the whole run and names the mutant that broke the promise; a gate that only + /// checked the score would let a contradicted claim about the suite go unnoticed forever. + #[test] + fn a_contradicted_expectation_fails_the_run_and_names_the_mutant() { + let dir = workdir("run-contradicted-"); + let root = subject_root(&dir); + fs::write( + root.join("src/lib.rs"), + "// #[gamma::expect_survived(relational)]\n\ + pub fn less(a: i32, b: i32) -> bool { a < b }\n\n\ + #[test]\n\ + fn catches_it() {\n\ + \x20\x20\x20\x20assert!(less(1, 2));\n\ + \x20\x20\x20\x20assert!(!less(1, 1));\n\ + }\n", + ) + .expect("lib"); + + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_GATE_FAILED); + assert!(host.err().contains("expected survived, but this mutant was"), "{}", host.err()); + assert!(host.err().contains("1 expectation did not hold"), "{}", host.err()); + } + + /// A run whose score falls short of `--min-score` fails the gate even when every expectation + /// held; the score threshold and the expectation directives are separate promises, and one + /// holding does not excuse the other from being checked. + #[test] + fn a_run_below_the_minimum_score_fails_the_gate() { + let dir = workdir("run-min-score-"); + let root = subject_root(&dir); + // No test asserts anything about `less`, so the survivor is left uncontested and the + // score comes in at 0%. + fs::write( + root.join("src/lib.rs"), + "pub fn less(a: i32, b: i32) -> bool { a < b }\n\n\ + #[test]\n\ + fn calls_it() {\n\ + \x20\x20\x20\x20let _ = less(1, 2);\n\ + }\n", + ) + .expect("lib"); + + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + min_score: Some(50.0), + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_GATE_FAILED); + assert!(host.err().contains("is below the required 50.0%"), "{}", host.err()); + } + + /// A diff written with `diff.mnemonicPrefix` names its paths `i/` and `w/` rather than `a/` and + /// `b/`. Every mutant in the change is still a mutant, so the gate still has a population to + /// judge and still fails on a survivor — where before the paths resolved to nothing, the run + /// tested nothing, and `--min-score 100` passed. + #[test] + fn a_diff_with_mnemonic_prefixes_is_still_gated() { + let dir = workdir("run-in-diff-mnemonic-"); + let root = subject_root(&dir); + // Nothing asserts anything about `less`, so the mutant on the changed line survives. + fs::write( + root.join("src/lib.rs"), + "pub fn less(a: i32, b: i32) -> bool { a < b }\n\n\ + #[test]\n\ + fn calls_it() {\n\ + \x20\x20\x20\x20let _ = less(1, 2);\n\ + }\n", + ) + .expect("lib"); + fs::write( + root.join("change.patch"), + "diff --git i/src/lib.rs w/src/lib.rs\n\ + --- i/src/lib.rs\n\ + +++ w/src/lib.rs\n\ + @@ -0,0 +1 @@\n\ + +pub fn less(a: i32, b: i32) -> bool { a < b }\n", + ) + .expect("patch"); + + let args = RunArgs { + select: crate::commands::SelectArgs { + in_diff: Some(root.join("change.patch")), + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + min_score: Some(100.0), + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_GATE_FAILED); + assert!(host.err().contains("is below the required 100.0%"), "{}", host.err()); + } + + /// A run whose every mutant is suppressed never builds anything, and the summary must still + /// account for them all; if the summary were skipped along with the build, a fully suppressed + /// package would look like a run that simply found nothing, hiding the fact that everything in + /// it was deliberately excluded. + #[test] + fn a_run_with_every_mutant_suppressed_never_builds_and_still_summarizes() { + let dir = workdir("run-all-suppressed-"); + let root = subject_root(&dir); + fs::write( + root.join("src/lib.rs"), + "// #[gamma::skip(relational)]\n\ + pub fn less(a: i32, b: i32) -> bool { a < b }\n", + ) + .expect("lib"); + + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_OK); + assert!( + host.out().contains("none tested") || host.err().contains("none tested"), + "{}{}", + host.out(), + host.err() + ); + } + + /// `--leak-dirs` says where the scratch tree that would otherwise be deleted was left, so a + /// caller who asked to inspect a build after the fact can find it; forgetting to print the path + /// would make the flag indistinguishable from doing nothing. + #[test] + fn leak_dirs_reports_where_the_scratch_tree_was_kept() { + let dir = workdir("run-leak-dirs-"); + let root = subject_root(&dir); + fs::write( + root.join("src/lib.rs"), + "pub fn less(a: i32, b: i32) -> bool { a < b }\n\n\ + #[test]\n\ + fn catches_it() {\n\ + \x20\x20\x20\x20assert!(less(1, 2));\n\ + \x20\x20\x20\x20assert!(!less(1, 1));\n\ + }\n", + ) + .expect("lib"); + + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + leak_dirs: true, + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("Kept"), "{}", host.err()); + } + + /// An artifact directory that cannot be created is reported as an error rather than silently + /// dropped, for the same reason a broken job-summary path is: the caller asked for a specific + /// artifact, and a swallowed failure there is indistinguishable from a run that never analyzed + /// anything. + /// + /// A missing parent directory is not that path — the writer creates one, as it does for the + /// reports beside it — so the broken path here is one whose parent is a file, which no amount + /// of creating directories can make writable. + #[test] + fn a_broken_artifact_directory_is_reported_as_an_error() { + let dir = workdir("run-advice-broken-"); + let root = subject_root(&dir); + fs::write( + root.join("src/lib.rs"), + "pub fn less(a: i32, b: i32) -> bool { a < b }\n\n\ + #[test]\n\ + fn catches_it() {\n\ + \x20\x20\x20\x20assert!(less(1, 2));\n\ + \x20\x20\x20\x20assert!(!less(1, 1));\n\ + }\n", + ) + .expect("lib"); + + fs::write(root.join("blocked"), "not a directory").expect("blocker"); + + let args = RunArgs { + artifact_dir: Some(root.join("blocked/artifacts")), + select: crate::commands::SelectArgs { + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + ..Default::default() + }; + let mut host = Sink::default(); + + let error = run_session(&mut host, &args, When::Never, Styler::new(false)).expect_err("broken artifact directory"); + + assert!(error.to_string().contains("could not create artifact directory"), "{error}"); + } + + /// A gate that cannot fail is worse than no gate. A selection that leaves nothing to test used + /// to exit zero, so `--min-score 100` passed on the strength of having tested nothing. + #[test] + fn a_gated_run_with_nothing_to_test_fails_rather_than_passing_silently() { + let dir = workdir("run-empty-gated-"); + let root = subject_root(&dir); + fs::write(root.join("src/lib.rs"), "pub fn less(a: i32, b: i32) -> bool { a < b }\n").expect("lib"); + + // Every file is excluded, so discovery finds nothing at all: the shape an over-eager + // `--exclude-file`, an empty shard and a diff that named no code all arrive in. + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + exclude_files: vec!["**/*.rs".to_owned()], + ..crate::commands::SelectArgs::default() + }, + dry_run: true, + min_score: Some(100.0), + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_GATE_FAILED); + assert!(host.err().contains("never evaluated"), "{}", host.err()); + assert!(host.err().contains("--exclude-file"), "{}", host.err()); + assert!(host.err().contains("--shard-count"), "{}", host.err()); + } + + /// The same run without a gate made no claim about its score, so an empty population is an + /// ordinary answer and must go on succeeding. + #[test] + fn an_ungated_run_with_nothing_to_test_still_succeeds() { + let dir = workdir("run-empty-ungated-"); + let root = subject_root(&dir); + fs::write(root.join("src/lib.rs"), "pub fn less(a: i32, b: i32) -> bool { a < b }\n").expect("lib"); + + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + exclude_files: vec!["**/*.rs".to_owned()], + ..crate::commands::SelectArgs::default() + }, + dry_run: true, + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_OK); + } + + /// A population that is not empty but holds nothing scorable — the shard that contained only + /// suppressed mutants — is the same failure wearing a different hat: the score is a ratio with + /// nothing in its denominator, and a threshold judged against it never ran. + #[test] + fn a_gated_run_whose_population_is_entirely_suppressed_fails() { + let dir = workdir("run-suppressed-gated-"); + let root = subject_root(&dir); + fs::write( + root.join("src/lib.rs"), + "// #[gamma::skip(relational)]\n\ + pub fn less(a: i32, b: i32) -> bool { a < b }\n", + ) + .expect("lib"); + + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + min_score: Some(100.0), + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + assert_eq!(code, EXIT_GATE_FAILED); + assert!(host.err().contains("never evaluated"), "{}", host.err()); + } + + /// And from the foreign-config note. + #[test] + fn a_closed_stream_is_reported_by_the_foreign_config_note() { + let dir = workdir("run-foreign-broken-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + fs::create_dir_all(root.join(".cargo")).expect("cargo dir"); + fs::write(root.join(".cargo/mutants.toml"), "examine_globs = [\"src/**\"]\n").expect("foreign"); + + // `configure` folds the file into `args`, so each attempt needs its own copy of them. + fails_at_every_line(1, |host| { + let mut args = RunArgs { + select: crate::commands::SelectArgs { + dir: root.clone(), + ..crate::commands::SelectArgs::default() + }, + ..Default::default() + }; + + configure(host, &mut args, Styler::new(false)) + }); + } + + /// A run whose build cannot be made to compile still reports, and still fails. + /// + /// This is the whole of the bargain: a workspace that gets stuck after an hour would otherwise + /// end with an error and nothing else — no JSON, no HTML, no annotations, no score — so the + /// time would be spent and nothing learned. The population that never ran is recorded as + /// `notbuilt`, the report is written, the diagnostic that would have been the error is printed, + /// and the exit code still says the run did not succeed. + #[test] + fn a_run_whose_build_cannot_converge_still_writes_a_report_and_still_fails() { + let dir = workdir("run-unconverged-"); + let root = subject_root(&dir); + + // A call to a symbol that does not exist passes `cargo check --tests`, so the preflight + // clears the tree, and then fails to link when the test targets are actually built. No + // mutant can be blamed for a linker error, which is exactly the failure that would + // otherwise end the run with nothing to show for it. + fs::write(root.join("src/lib.rs"), fixtures::UNRESOLVED_LINK_SOURCE).expect("lib"); + + let artifact_dir = root.join("artifacts"); + let report = artifact_dir.join("gamma-report.json"); + let args = RunArgs { + select: crate::commands::SelectArgs { + dir: root, + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + artifact_dir: Some(artifact_dir), + ..Default::default() + }; + let mut host = Sink::default(); + + let code = run_session(&mut host, &args, When::Never, Styler::new(false)).expect("run session"); + + // A build the tool could not converge is not a successful run, whatever was written. + assert_ne!(code, EXIT_OK, "{}", host.err()); + assert_eq!(code, EXIT_CANNOT_PROCEED, "{}", host.err()); + + let written = fs::read_to_string(&report).expect("the report has to exist even so"); + + // The distinction the whole issue is about: these mutants never ran, so calling them + // survivors would report a test-suite gap that does not exist. The schema has no status of + // its own for a mutant nobody built, and `Ignored` is what `NotBuilt` exports as — what + // matters is that it is not in the denominator and not called a survivor. + assert!(written.contains("\"status\": \"Ignored\""), "{written}"); + assert!(!written.contains("Survived"), "{written}"); + + // The diagnostic that would otherwise be the error, kept whole, and grouped so the reader can see + // where the tool got stuck rather than merely that it did. + let said = host.err(); + + assert!(said.contains("could not be made to compile"), "{said}"); + assert!(said.contains("never ran"), "{said}"); + assert!(said.contains("Not run, by mutator: relational.lt_to_le"), "{said}"); + assert!(said.contains("Not run, by scope"), "{said}"); + } + + /// A near miss must never render as if it met the gate. + /// + /// The gate compares the full-precision score, so 66.666…% correctly fails a 66.7% bar — but + /// printed at one decimal both read "66.7%", and the failure message would deny itself. The run + /// gate builds its message through `distinguish`, which grows the precision until the two are + /// visibly different: at two decimals the score reads "66.67" and the threshold "66.70", so the + /// sentence no longer prints the same number twice. + #[test] + fn the_run_gate_shows_a_near_miss_as_below_the_threshold_not_equal_to_it() { + let (score, minimum) = distinguish(200.0 / 3.0, 66.7); + + assert_ne!(score, minimum); + assert_eq!(score, "66.67"); + assert_eq!(minimum, "66.70"); + } + + /// A clear miss stays at the one decimal place the report has always used. + #[test] + fn a_clear_miss_is_shown_at_a_single_decimal_place() { + assert_eq!(distinguish(50.0, 80.0), ("50.0".to_owned(), "80.0".to_owned())); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/suppress.rs b/crates/cargo-gamma-lib/src/commands/suppress.rs new file mode 100644 index 000000000..d8308ae05 --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/suppress.rs @@ -0,0 +1,1498 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::BTreeSet; +use std::fs; +use std::io::Write; +use std::process::{Command, Stdio}; + +use camino::{Utf8Path, Utf8PathBuf}; + +use super::cli::SuppressArgs; +use super::dispatch::{EXIT_CANNOT_PROCEED, EXIT_OK}; +use super::host::Host; +use super::run::execute; +use super::when::When; +use crate::discover::Plan; +use crate::elements::Publication; +use crate::error::error; +use crate::fix::Edit; +use crate::report::{Styler, quantity}; + +/// Implements `suppress`. +/// +/// The order is deliberate: run first, then write. Suppressions are derived from observed verdicts, +/// never from static guesses, because the whole justification for editing someone's source is that +/// the tool watched the mutant misbehave. +pub(super) fn suppress(host: &mut H, args: &SuppressArgs, progress_when: When, styler: Styler) -> crate::Result { + let eligible = crate::fix::Eligible::parse(&args.eligible)?; + + if eligible.is_empty() { + return Err(error!("--eligible named no verdicts; nothing could be suppressed").usage()); + } + + let executed = execute(host, &args.run, progress_when, styler)?; + + // Refused before anything is planned, for the reason `run` refuses to score such a run: the + // population is missing the part nobody could build, so an edit derived from it comes out of an + // incomplete run and the zero this would otherwise exit with is a completeness claim the run + // did not earn. `cargo gamma suppress --apply && git commit -am …` is a real pipeline, and it + // must not commit source edits on a tree where `cargo gamma run` would have exited 3. + if !executed.stuck.is_empty() { + writeln!( + host.error(), + "{} nothing was suppressed: {} could not be made to compile, so this run measured only part of the population", + styler.verb("Refusing"), + quantity(executed.stuck.len(), "build") + )?; + + return Ok(EXIT_CANNOT_PROCEED); + } + + let Some(plan) = executed.plan else { + return Ok(EXIT_OK); + }; + + let edits = crate::fix::plan(&plan.mutants, &eligible); + + if edits.is_empty() { + writeln!( + host.error(), + "{} nothing to suppress: no mutant had an eligible verdict", + styler.verb("Finished") + )?; + + return Ok(EXIT_OK); + } + + let intended = intended(&plan.mutants, &edits, &eligible); + let date = crate::fix::today(); + let written = apply_all(host, args, &plan, &edits, &date)?; + + if args.dry_run_suppress { + return Ok(EXIT_OK); + } + + verify_or_revert(host, args, &plan, &intended, edits.len(), written, styler) +} + +/// The mutants the edits were written for, which the verification is then allowed to find suppressed. +/// +/// The same three conditions `fix::plan` inserted on, restated rather than widened: the line is one +/// an edit was written for, the directive on that line names this mutant's mutator, and this +/// mutant's own verdict was eligible. +/// +/// Restating them is the whole point. Deriving the set from the edit *sites* alone — every mutant on +/// a line an edit touched — admits the other mutants sharing that line, and two mutants at one site +/// with the same mutator is ordinary rather than exotic: `x + y + w` yields two `arith.add_to_sub` +/// mutants. Let one time out and the other survive, and the directive written for the timeout +/// suppresses both, because directives match by mutator name and not by occurrence. Told that the +/// survivor was intended, the verification exempts it from the collateral check and reports clean — +/// a survivor out of the denominator with nothing said about it, which is precisely what this +/// command exists to make impossible. The narrower set lets it land in `collateral` instead, where +/// it takes the whole edit back and says so. +/// +/// A map rather than a scan of the edits per mutant: both grow with the workspace, and the pairing +/// has no business being quadratic in it. +fn intended(mutants: &[crate::model::Mutant], edits: &[Edit], eligible: &[crate::fix::Eligible]) -> BTreeSet { + let touched: crate::HashMap<(&Utf8Path, usize), &BTreeSet> = edits + .iter() + .map(|edit| ((edit.file.as_path(), edit.line), &edit.mutators)) + .collect(); + + mutants + .iter() + .filter(|mutant| { + touched + .get(&(&*mutant.file, mutant.line)) + .is_some_and(|mutators| mutators.contains(&*mutant.mutator)) + && eligible.iter().any(|entry| entry.outcome() == mutant.outcome) + }) + .map(|mutant| mutant.id.to_string()) + .collect() +} + +/// One file a command has rewritten, with both generations the command observed. +/// +/// Rollback may restore `before` only when the path still holds `after`. Recording that published +/// generation turns compensation into a compare-and-replace, so a save made after this command's +/// edit remains the editor's save rather than becoming stale rollback input. +#[derive(Debug)] +pub(super) struct WrittenFile { + path: Utf8PathBuf, + before: String, + after: String, +} + +impl WrittenFile { + #[must_use] + pub(super) const fn new(path: Utf8PathBuf, before: String, after: String) -> Self { + Self { path, before, after } + } +} + +/// Ordered as they were written, so that putting the tree back is the exact reverse of what was +/// done to it. +pub(super) type Written = Vec; + +/// Writes every file the plan has an edit for, taking back whatever it wrote if any of them fails. +/// +/// The compensation is the point. A read, a parse or a write can fail on file N with files 1 to N-1 +/// already rewritten, and returning that error on its own would leave a tree nobody asked for and +/// nothing recorded — for `suppress`, directives standing over mutants that the next run will +/// therefore skip, which is the one thing this command must never do quietly. +fn apply_all(host: &mut H, args: &SuppressArgs, plan: &Plan, edits: &[Edit], date: &str) -> crate::Result { + if !args.dry_run_suppress { + let paths: Vec<&Utf8Path> = edits.iter().map(|edit| edit.file.as_path()).collect(); + + reject_external_sources(&plan.root, &paths)?; + recoverable(&plan.root, &paths, args.allow_dirty)?; + } + + let mut written = Written::new(); + + match edit_files(host, args, plan, edits, date, &mut written) { + Ok(()) => Ok(written), + + // `written` is handed over rather than borrowed so that the compensation owns what it puts + // back: whatever this returns, those files are no longer this command's to revert. + Err(cause) => Err(reverted(&plan.root, written, cause)), + } +} + +/// Refuses every source edit that would leave the workspace before anything is published. +/// +/// Checking the whole batch first keeps a later external link from turning a refusal into an +/// edit-and-rollback transaction. `recoverable` also inspects these paths, so it must not reach an +/// external referent first either. +pub(super) fn reject_external_sources(root: &Utf8Path, paths: &[&Utf8Path]) -> crate::Result<()> { + for path in paths { + let absolute = root.join(path); + let _destination = crate::paths::require_within(&absolute, root, "a source edit")?; + } + + Ok(()) +} + +/// Refuses to edit a file that nothing on disk could put back. +/// +/// The compensation set below lives in process memory and nowhere else. Every failure the loop can +/// see is answered by it, but the interrupt handler kills the process where it stands, and an +/// interrupt after file three of seven leaves four untouched files, three rewritten ones, and no +/// record anywhere of which were which. For `unsuppress` what is left in those three is *deleted* +/// directives, hand-written reasons included, which nothing can reconstruct. +/// +/// Rather than keep a journal of its own, this command requires the one every user of it already +/// has: a committed file is recoverable by `git checkout`, and the failure mode above costs a +/// command rather than an afternoon. A file with uncommitted changes has nothing behind it, so it +/// is refused — and the refusal names the flag, because someone working outside version control +/// deliberately is entitled to say so once rather than be stopped forever. +/// +/// A tree that is not a repository at all, or a host with no git, gets no opinion: demanding +/// version control from someone who is not using it would be a different command. +/// +/// Shared with `unsuppress`, whose edit loop has the same shape and the same hazard. +pub(super) fn recoverable(root: &Utf8Path, paths: &[&Utf8Path], allow_dirty: bool) -> crate::Result<()> { + if allow_dirty { + return Ok(()); + } + + // A status with no pathspec is a status of the whole repository, which is a different and much + // ruder question than the one being asked here. + if paths.is_empty() { + return Ok(()); + } + + let Some(dirty) = uncommitted(root, paths) else { + return Ok(()); + }; + + if dirty.is_empty() { + return Ok(()); + } + + Err(error!( + "{} about to be edited {} uncommitted changes: {}. This command's rollback lives in this process only, so an interrupt part-way through the edit would leave a tree nothing on disk records — version control is the journal. Commit or stash first, or pass `--allow-dirty` to edit anyway", + quantity(dirty.len(), "file"), + if dirty.len() == 1 { "has" } else { "have" }, + dirty.join(", ") + ) + .usage()) +} + +/// The paths among `paths` that git reports as changed, or `None` when there is no repository. +/// +/// Asked of git rather than derived from the index directly, for the reason the copy path already +/// asks it: a worktree, a submodule and an index format newer than any library understands all +/// answer correctly, and a directory that is not a repository answers by failing. +fn uncommitted(root: &Utf8Path, paths: &[&Utf8Path]) -> Option> { + let output = Command::new("git") + .arg("-C") + .arg(root.as_std_path()) + .args(["status", "--porcelain", "-z", "--"]) + .args(paths.iter().map(|path| path.as_std_path())) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + Some( + output + .stdout + .split(|byte| *byte == 0) + .filter(|entry| !entry.is_empty()) + .filter_map(|entry| core::str::from_utf8(entry).ok()) + // Each entry is two status characters, a space, and the path they are about. + .map(|entry| entry.get(3..).unwrap_or(entry).to_owned()) + .collect(), + ) +} + +/// Rewrites each file the plan has an edit for, recording what it held before in `written`. +/// +/// Every step is fallible and every step uses `?`, which is what makes the caller's compensation +/// necessary: this stops where it fails and says nothing about the files it already changed beyond +/// what it has put in `written`. +fn edit_files( + host: &mut H, + args: &SuppressArgs, + plan: &Plan, + edits: &[Edit], + date: &str, + written: &mut Written, +) -> crate::Result<()> { + for file in &plan.files { + let for_file: Vec<&Edit> = edits.iter().filter(|edit| edit.file == file.path).collect(); + + if for_file.is_empty() { + continue; + } + + let path = plan.root.join(&file.path); + let before = fs::read_to_string(&path).map_err(|cause| error!("could not read `{path}`").caused_by(cause))?; + let source = crate::parse::strip_bom(&before); + + // The line numbers in `for_file` were decided by a discovery that ran before the measured + // run, which on a real workspace is hours ago, and nothing has held the tree still since. + // A line number means nothing against text it was not computed from: applied to a file the + // author has edited, the directive lands on whichever line moved into that position and + // suppresses a mutant nobody chose, or annotates a statement. Refusing names the one thing + // the user can act on — re-run — where the alternative is a source edit that looks + // deliberate. + // + // Checked even in a dry run: a diff computed against text that is no longer there is a + // diff of an edit that will not be made, which is worse than no diff at all. + if let Some(recorded) = plan.digests.get(&file.path) + && crate::discover::digest(source.as_bytes()) != *recorded + { + return Err(error!( + "`{path}` changed since the run that planned this edit; nothing was written to it. Re-run to plan against the file as it is now" + )); + } + + let mut after = crate::fix::apply(source, &for_file, date); + + if before.len() != source.len() { + after.insert(0, crate::parse::BOM); + } + + // Parsing before writing, not after: a patch that does not parse must never reach the disk, + // because the revert path is only as good as the copy it holds. + let _ = syn::parse_file(&after).map_err(|cause| error!("the generated directive would not parse in {path}").caused_by(cause))?; + + if args.dry_run_suppress { + write!(host.results(), "{}", crate::fix::diff(&file.path, &before, &after))?; + } else { + // The final comparison happens after the replacement has been staged. The earlier + // digest protects the long run; this check catches changes through that comparison. + // The publication API deliberately makes no claim about a non-cooperating replacement + // in the syscall interval between comparison and rename. + match crate::elements::write_if_unchanged(&plan.root, &path, Some(&before), &after)? { + Publication::Conflict => { + return Err(error!( + "`{path}` changed while this command was preparing to publish its edit; the editor's bytes were left alone. Re-run to plan against the file as it is now" + )); + } + Publication::Published => written.push(WrittenFile::new(path, before, after)), + Publication::PublishedUndurable(cause) => { + // Rename made the source edit visible even though syncing its directory did + // not finish. Record it before returning so `apply_all` can compensate it. + written.push(WrittenFile::new(path, before, after)); + return Err(cause); + } + } + } + } + + Ok(()) +} + +/// Puts every file in `written` back the way it was, and folds what happened into `cause`. +/// +/// Both failures are reported, rather than the second replacing the first. What went wrong first is +/// why the command stopped; whether the tree went back is what the user has to act on, and a +/// message carrying only one of the two sends them to the wrong place — either hunting for an edit +/// that was taken back, or trusting a rollback that did not happen. +/// +/// Shared with `unsuppress`, whose edit loop has the same shape and the same hazard. +pub(super) fn reverted(root: &Utf8Path, written: Written, cause: crate::error::Error) -> crate::error::Error { + if written.is_empty() { + return cause; + } + + // Backwards, because the compensation set is ordered as it was written and two entries can name + // one file: a symlinked source directory, or one file reached under two of the plan's paths. + // Replaying forwards would restore the pristine text and then restore the text that already + // held the first edit, leaving the file at the intermediate state under a message saying every + // edit had been taken back. + let mut stranded = Vec::new(); + let mut undurable = Vec::new(); + + for entry in written.into_iter().rev() { + match crate::elements::write_if_unchanged(root, &entry.path, Some(&entry.after), &entry.before) { + Ok(Publication::Conflict) => { + stranded.push(format!("{} (changed after this command wrote it and was left alone)", entry.path)); + } + Ok(Publication::Published) => {} + Ok(Publication::PublishedUndurable(failure)) => { + undurable.push(format!("{} ({failure})", entry.path)); + } + Err(failure) => stranded.push(format!("{} ({failure})", entry.path)), + } + } + + if stranded.is_empty() { + if undurable.is_empty() { + return error!("{cause}; every edit has been reverted"); + } + + return error!( + "{cause}; every edit has been reverted, but {} could not be made durable because their directories could not be synced: {}", + quantity(undurable.len(), "file"), + undurable.join(", ") + ); + } + + let failure = error!( + "{cause}; every edit has been reverted except {}, which changed after this command wrote them and were left alone: {}", + quantity(stranded.len(), "file"), + stranded.join(", ") + ); + + if undurable.is_empty() { + return failure; + } + + error!( + "{failure}; {} reverted {} could not be made durable because their directories could not be synced: {}", + quantity(undurable.len(), "file"), + if undurable.len() == 1 { "edit" } else { "edits" }, + undurable.join(", ") + ) +} + +/// Re-runs discovery over the edited tree and reverts unless the suppressed set is exactly right. +/// +/// Over-suppression is the hazard: a directive attached to a multi-line construct silently takes out +/// everything inside it, which can include survivors. Checking both directions is what makes an +/// automated source edit something a reviewer can trust without reading every line of it. +fn verify_or_revert( + host: &mut H, + args: &SuppressArgs, + before: &Plan, + intended: &BTreeSet, + directives: usize, + written: Written, + styler: Styler, +) -> crate::Result { + let verified = (|| { + let selection = args.run.select.selection()?; + let cargo = super::run::run_config(&args.run, styler).cargo; + let after = crate::discover::plan_for_build(&args.run.select, &selection, args.run.select.shard()?, &cargo, &mut |_| {})?; + + Ok(crate::fix::verify(&before.mutants, &after.mutants, intended)) + })(); + let result = match verified { + Ok(result) => result, + Err(cause) => return Err(reverted(&before.root, written, cause)), + }; + + if result.is_clean() { + let mut stream = host.error(); + + // The directives written, not the mutants they cover: one comment naming three mutators + // over a line carrying three mutants is one directive, and reporting three would tell the + // reader to look for two comments that are not there. + writeln!( + stream, + "{} {} in {}", + styler.verb("Suppressed"), + quantity(directives, "directive"), + quantity(written.len(), "file") + )?; + + writeln!( + stream, + "{} every generated directive is tagged; grep for `cargo gamma suppress` to audit them", + styler.verb("Note") + )?; + + return Ok(EXIT_OK); + } + + Err(reverted(&before.root, written, unclean(&result))) +} + +/// Says which half of the verification failed, and names what it was about. +/// +/// All three vectors, because [`crate::fix::Verification::is_clean`] requires all three to be +/// empty: a rollback caused by `released` alone would otherwise print two zeroes and a message about +/// directives missing their target, which is a failure the reader can neither explain nor act on. +/// The ids are named as well as counted, because "3 unintended" leaves them re-running discovery by +/// hand to learn which three. +fn unclean(result: &crate::fix::Verification) -> crate::error::Error { + /// How many ids are worth putting in a one-line message before it stops being one. + const NAMED: usize = 3; + + let listed = |ids: &[String]| -> String { + let shown = ids.iter().take(NAMED).map(String::as_str).collect::>().join(", "); + + ids.len() + .checked_sub(NAMED) + .filter(|rest| *rest > 0) + .map_or_else(|| shown.clone(), |rest| format!("{shown} and {rest} more")) + }; + + let mut parts = Vec::new(); + + for (count, label, ids) in [ + (result.missing.len(), "not suppressed", &result.missing), + (result.collateral.len(), "unintended", &result.collateral), + (result.released.len(), "no longer suppressed", &result.released), + ] { + if count > 0 { + parts.push(format!("{count} {label} ({})", listed(ids))); + } + } + + error!( + "the generated directives did not suppress what they were meant to: {}", + parts.join(", ") + ) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + use crate::commands::RunArgs; + use crate::discover::TargetFile; + #[cfg(unix)] + use crate::testing::workdir; + use crate::testing::{Sink, fails_at_every_line}; + + fn crate_dir(name: &str) -> tempfile::TempDir { + crate::fixtures::crate_dir(name, "pub fn answer() -> i32 { 42 }\n").0 + } + + /// Builds a `suppress` invocation that only discovers, so no cargo build is involved. + fn dry_args(root: &Utf8PathBuf, eligible: &str) -> SuppressArgs { + SuppressArgs { + run: RunArgs { + select: crate::commands::SelectArgs { + dir: root.clone(), + ..crate::commands::SelectArgs::default() + }, + dry_run: true, + ..RunArgs::default() + }, + dry_run_suppress: false, + allow_dirty: false, + eligible: eligible.to_owned(), + } + } + + /// A plan over the named files, which is all the edit loop reads out of one. + fn plan_over(root: &Utf8PathBuf, files: &[&str]) -> Plan { + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: root.clone(), + files: files + .iter() + .map(|name| TargetFile { + path: Utf8PathBuf::from(name), + absolute: root.join(name), + package: "subject".to_owned(), + }) + .collect(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + } + } + + /// One directive over the first line of `file`. + fn edit_for(file: &str) -> Edit { + Edit { + file: Utf8PathBuf::from(file), + line: 1, + mutators: core::iter::once("arith.add_to_sub".to_owned()).collect(), + tag: "timeout", + } + } + + #[test] + fn empty_eligibility_is_a_usage_error() { + let mut host = Sink::default(); + let args = SuppressArgs { + run: RunArgs::default(), + dry_run_suppress: false, + allow_dirty: false, + eligible: String::new(), + }; + + let err = suppress(&mut host, &args, When::Never, Styler::new(false)).unwrap_err(); + + assert!(err.is_usage()); + } + + #[test] + fn clean_verification_reports_what_was_suppressed() { + let dir = crate_dir("suppress-verify-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let args = SuppressArgs { + run: RunArgs { + select: crate::commands::SelectArgs { + dir: root.clone(), + ..crate::commands::SelectArgs::default() + }, + ..RunArgs::default() + }, + dry_run_suppress: false, + allow_dirty: false, + eligible: "timeout".to_owned(), + }; + let before = Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: root.clone(), + files: vec![TargetFile { + path: Utf8PathBuf::from("src/lib.rs"), + absolute: root.join("src/lib.rs"), + package: "subject".to_owned(), + }], + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + }; + let mut host = Sink::default(); + + let code = verify_or_revert(&mut host, &args, &before, &BTreeSet::new(), 0, Vec::new(), Styler::new(false)).expect("verify"); + let err = String::from_utf8(host.err).expect("utf-8"); + + assert_eq!(code, EXIT_OK); + assert!(err.contains("Suppressed 0 directives"), "{err}"); + assert!(err.contains("grep for `cargo gamma suppress`"), "{err}"); + assert!(host.out.is_empty()); + } + + #[test] + fn verification_keeps_the_configuration_generation_merged_before_the_edit() { + let dir = crate_dir("suppress-config-generation-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let config = root.join("gamma.toml"); + + fs::create_dir_all(config.parent().expect("configuration parent")).expect("configuration parent"); + fs::write(&config, "cargo-args = [\"--cfg\", \"recorded_configuration\"]\n").expect("configuration"); + let mut args = dry_args(&root, "timeout"); + crate::config::Config::resolve(&args.run.select) + .expect("initial configuration") + .apply(&mut args.run) + .expect("configuration merges"); + + // This is the interval between argument merging and verification after a suppression + // edit. Discovery must use the already merged cargo options rather than re-read this + // unrelated generation, which is malformed and would previously be swallowed as defaults. + fs::write(&config, "cargo-args = [\n").expect("broken later configuration"); + + let mut host = Sink::default(); + + let code = verify_or_revert( + &mut host, + &args, + &plan_over(&root, &["src/lib.rs"]), + &BTreeSet::new(), + 0, + Vec::new(), + Styler::new(false), + ) + .expect("verification uses its resolved configuration generation"); + + assert_eq!(code, EXIT_OK); + } + + #[test] + fn a_verification_error_reverts_every_written_file() { + let dir = crate_dir("suppress-verify-error-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let original = fs::read_to_string(&path).expect("original"); + let generated = "// #[gamma::skip]\npub fn answer() -> i32 { 42 }\n"; + fs::write(&path, generated).expect("edited"); + let mut args = dry_args(&root, "timeout"); + args.run.select.mutators = Some("not.a.mutator".to_owned()); + let mut host = Sink::default(); + + let failure = verify_or_revert( + &mut host, + &args, + &plan_over(&root, &["src/lib.rs"]), + &BTreeSet::new(), + 1, + vec![WrittenFile::new(path.clone(), original.clone(), generated.to_owned())], + Styler::new(false), + ) + .expect_err("selection must fail"); + + assert!(failure.to_string().contains("every edit has been reverted"), "{failure}"); + assert_eq!(fs::read_to_string(path).expect("restored"), original); + } + + /// A run that produced no mutants leaves nothing to suppress and is not a failure. + #[test] + fn a_run_with_no_mutants_at_all_succeeds_quietly() { + let dir = crate_dir("suppress-none-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + fs::write(root.join("src/lib.rs"), "pub struct Empty;\n").expect("lib"); + + let mut host = Sink::default(); + + let code = suppress(&mut host, &dry_args(&root, "timeout"), When::Never, Styler::new(false)).expect("suppress"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("no mutants were generated"), "{}", host.err()); + } + + /// Mutants that were never run have no eligible verdict, so nothing gets edited. + #[test] + fn a_population_with_no_eligible_verdict_edits_nothing() { + // Determining an `unviable` or `timeout` verdict means building and running the + let dir = crate_dir("suppress-ineligible-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let source = fs::read_to_string(root.join("src/lib.rs")).expect("read"); + + let mut host = Sink::default(); + + let code = suppress(&mut host, &dry_args(&root, "timeout"), When::Never, Styler::new(false)).expect("suppress"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("nothing to suppress"), "{}", host.err()); + assert_eq!(fs::read_to_string(root.join("src/lib.rs")).expect("read"), source); + + fails_at_every_line(1, |host| { + suppress(host, &dry_args(&root, "timeout"), When::Never, Styler::new(false)).map(|_| ()) + }); + } + + /// Directives that missed their target take the whole edit back rather than leaving it half done. + #[test] + fn a_verification_that_fails_reverts_every_file_it_wrote() { + let dir = crate_dir("suppress-revert-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let original = fs::read_to_string(&path).expect("read"); + + fs::write(&path, "pub fn answer() -> i32 { 0 }\n").expect("edited"); + + let before = Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: root.clone(), + files: vec![TargetFile { + path: Utf8PathBuf::from("src/lib.rs"), + absolute: path.clone(), + package: "subject".to_owned(), + }], + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + }; + + // Naming an id that no mutant carries makes the intended set impossible to satisfy, which + // is exactly the shape of an over- or under-reaching directive. + let intended: BTreeSet = core::iter::once("never-generated".to_owned()).collect(); + let mut host = Sink::default(); + + let error = verify_or_revert( + &mut host, + &dry_args(&root, "timeout"), + &before, + &intended, + 1, + vec![WrittenFile::new( + path.clone(), + original.clone(), + "pub fn answer() -> i32 { 0 }\n".to_owned(), + )], + Styler::new(false), + ) + .expect_err("verification should fail"); + + assert!(error.to_string().contains("every edit has been reverted"), "{error}"); + assert_eq!(fs::read_to_string(&path).expect("read"), original); + } + + /// A closed stream has to surface from the success report. + #[test] + fn a_closed_stream_is_reported_by_the_success_report() { + let dir = crate_dir("suppress-broken-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let before = Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: root.clone(), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + }; + + fails_at_every_line(2, |host| { + verify_or_revert( + host, + &dry_args(&root, "timeout"), + &before, + &BTreeSet::new(), + 0, + Vec::new(), + Styler::new(false), + ) + .map(|_| ()) + }); + } + + /// A real, non-dry-run suppress writes the directive to the file that earned one and leaves + /// every other file in the workspace untouched; touching a file that had nothing eligible in + /// it would be an edit nobody asked for and nobody could explain from the run's own report. + #[test] + fn a_multi_file_run_edits_only_the_file_with_an_eligible_mutant() { + // Determining an `unviable` or `timeout` verdict means building and running the + let dir = crate_dir("suppress-multi-file-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + + // `find`'s only mutable site is `fn_value.some_default`: a trait object names a capability + // rather than a type, so there is no value to put inside the `Some`, and the family falls + // back on `Default::default()`, which cannot compile here. That unviable verdict is what + // the eligibility filter under test is keyed on. + fs::write( + root.join("src/lib.rs"), + "pub fn find() -> Option<&'static dyn core::fmt::Debug> { None }\n", + ) + .expect("lib"); + // A file with nothing for `fn_value.some_default` to touch, so it is scanned and then + // skipped over rather than edited. + fs::write(root.join("src/other.rs"), "pub struct Marker;\n").expect("other"); + + let args = SuppressArgs { + run: RunArgs { + select: crate::commands::SelectArgs { + dir: root.clone(), + mutators: Some("fn_value.some_default".to_owned()), + ..crate::commands::SelectArgs::default() + }, + ..RunArgs::default() + }, + dry_run_suppress: false, + allow_dirty: false, + eligible: "unviable".to_owned(), + }; + let mut host = Sink::default(); + + let code = suppress(&mut host, &args, When::Never, Styler::new(false)).expect("suppress"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("Suppressed"), "{}", host.err()); + + let lib = fs::read_to_string(root.join("src/lib.rs")).expect("lib after"); + let other = fs::read_to_string(root.join("src/other.rs")).expect("other after"); + + assert!(lib.contains("gamma::skip"), "{lib}"); + assert!(!other.contains("gamma::skip"), "{other}"); + } + + /// `--dry-run-suppress` writes the diff to stdout instead of touching the source; a caller + /// previewing a suppression run needs the file to still hold the mutant afterwards, or the + /// preview would be lying about what it is a preview of. + #[test] + fn a_dry_run_suppress_prints_the_diff_and_leaves_the_file_alone() { + // Determining an `unviable` or `timeout` verdict means building and running the + let dir = crate_dir("suppress-dry-run-write-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let source = "pub fn find() -> Option<&'static dyn core::fmt::Debug> { None }\n"; + + fs::write(root.join("src/lib.rs"), source).expect("lib"); + + let args = SuppressArgs { + run: RunArgs { + select: crate::commands::SelectArgs { + dir: root.clone(), + mutators: Some("fn_value.some_default".to_owned()), + ..crate::commands::SelectArgs::default() + }, + ..RunArgs::default() + }, + dry_run_suppress: true, + allow_dirty: false, + eligible: "unviable".to_owned(), + }; + let mut host = Sink::default(); + + let code = suppress(&mut host, &args, When::Never, Styler::new(false)).expect("suppress"); + + assert_eq!(code, EXIT_OK); + assert!(host.out().contains("gamma::skip"), "{}", host.out()); + assert_eq!(fs::read_to_string(root.join("src/lib.rs")).expect("lib after"), source); + } + + /// The two-file case the compensation exists for: the first file is rewritten, the second one + /// cannot be read, and the command must not leave the first one holding a directive nobody + /// asked for. A directive left behind that way is worse than a failed command — it silently + /// removes mutants from the next run, so the score goes up because the tool broke. + #[test] + fn a_failure_on_the_second_file_puts_the_first_one_back_byte_for_byte() { + let dir = crate_dir("suppress-rollback-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let first = root.join("src/lib.rs"); + let original = fs::read(first.as_std_path()).expect("the original bytes"); + + // Named so it sorts after the first, and absent so that reading it fails on every platform + // and for every user, root included. + let plan = plan_over(&root, &["src/lib.rs", "src/zzz_gone.rs"]); + let edits = vec![edit_for("src/lib.rs"), edit_for("src/zzz_gone.rs")]; + let mut host = Sink::default(); + + let error = apply_all(&mut host, &dry_args(&root, "timeout"), &plan, &edits, "2026-01-01").expect_err("the second file"); + + assert!(error.to_string().contains("zzz_gone.rs"), "{error}"); + assert!(error.to_string().contains("every edit has been reverted"), "{error}"); + assert_eq!( + fs::read(first.as_std_path()).expect("the bytes afterwards"), + original, + "the first file was left edited" + ); + } + + /// The same loop, with nothing in its way, does rewrite both files — so the test above is + /// asserting that a rollback happened rather than that the loop never got started. + #[test] + fn both_files_are_rewritten_when_nothing_fails() { + let dir = crate_dir("suppress-both-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + + fs::write(root.join("src/other.rs"), "pub fn other() -> i32 { 7 }\n").expect("other"); + + let plan = plan_over(&root, &["src/lib.rs", "src/other.rs"]); + let edits = vec![edit_for("src/lib.rs"), edit_for("src/other.rs")]; + let mut host = Sink::default(); + + let written = apply_all(&mut host, &dry_args(&root, "timeout"), &plan, &edits, "2026-01-01").expect("both files"); + + assert_eq!(written.len(), 2); + assert!(fs::read_to_string(root.join("src/lib.rs")).expect("lib").contains("gamma::skip")); + assert!( + fs::read_to_string(root.join("src/other.rs")) + .expect("other") + .contains("gamma::skip") + ); + } + + /// A file edited since the plan was made is refused, not written to by its old line numbers. + /// + /// The window here is the whole measured run — hours on a real workspace — and nothing holds + /// the tree still across it. The line numbers were decided against the text discovery read; + /// applied to text somebody has since edited they land on whichever line moved into that + /// position, so a directive appears over a function nobody chose and the mutants it suppresses + /// go unmeasured with a comment that says the author meant it. Content-addressed mutant ids + /// catch the *consequence* afterwards and revert, but the message they produce says the edit + /// "did not suppress what it was meant to", which sends the reader looking for a bug in their + /// eligibility selection rather than at the file they saved. + #[test] + fn a_file_edited_since_the_plan_was_made_is_refused_rather_than_edited_by_stale_line_numbers() { + let dir = crate_dir("suppress-moved-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let planned = fs::read_to_string(path.as_std_path()).expect("the text the plan saw"); + + let mut plan = plan_over(&root, &["src/lib.rs"]); + let _recorded = plan + .digests + .insert(Utf8PathBuf::from("src/lib.rs"), crate::discover::digest(planned.as_bytes())); + + // The user saves the file while the run is measuring, which is all it takes: every line + // number the plan holds is now one line out. + fs::write(path.as_std_path(), format!("// a note added while the run was going\n{planned}")).expect("the edit"); + + let after_edit = fs::read(path.as_std_path()).expect("the bytes as the user left them"); + let edits = vec![edit_for("src/lib.rs")]; + let mut host = Sink::default(); + + let error = apply_all(&mut host, &dry_args(&root, "timeout"), &plan, &edits, "2026-01-01").expect_err("the file moved"); + + assert!( + error.to_string().contains("changed since the run that planned this edit"), + "{error}" + ); + assert_eq!( + fs::read(path.as_std_path()).expect("the bytes afterwards"), + after_edit, + "the refusal did not leave the user's own edit alone" + ); + } + + /// The same loop writes the file when it is the one the plan was made against, so the test + /// above is asserting that the digest refused it rather than that nothing was ever attempted. + #[test] + fn a_file_that_still_matches_the_plan_is_edited_as_planned() { + let dir = crate_dir("suppress-unmoved-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let planned = fs::read_to_string(path.as_std_path()).expect("the text the plan saw"); + + let mut plan = plan_over(&root, &["src/lib.rs"]); + let _recorded = plan + .digests + .insert(Utf8PathBuf::from("src/lib.rs"), crate::discover::digest(planned.as_bytes())); + + let edits = vec![edit_for("src/lib.rs")]; + let mut host = Sink::default(); + + let written = apply_all(&mut host, &dry_args(&root, "timeout"), &plan, &edits, "2026-01-01").expect("the file is unchanged"); + + assert_eq!(written.len(), 1); + assert!(fs::read_to_string(path.as_std_path()).expect("lib").contains("gamma::skip")); + } + + /// Discovery strips a leading BOM before hashing and locating lines. Suppression must compare + /// that same normalized source, then put the mark back before the first generated directive so + /// it remains a file marker rather than becoming an in-source character. + #[test] + fn bom_prefixed_files_match_the_plan_and_keep_the_bom_for_first_and_later_edits() { + for (name, source, line, expected) in [ + ( + "first", + "\u{feff}pub fn first() -> i32 { 1 }\n", + 1, + "\u{feff}// #[gamma::skip(arith.add_to_sub,", + ), + ( + "later", + "\u{feff}pub fn first() -> i32 { 1 }\npub fn later() -> i32 { 2 }\n", + 2, + "\u{feff}pub fn first() -> i32 { 1 }\n// #[gamma::skip(arith.add_to_sub,", + ), + ] { + let dir = crate_dir(&format!("suppress-bom-{name}-")); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + fs::write(&path, source).expect("bom source"); + + let mut plan = plan_over(&root, &["src/lib.rs"]); + let _recorded = plan.digests.insert( + Utf8PathBuf::from("src/lib.rs"), + crate::discover::digest(crate::parse::strip_bom(source).as_bytes()), + ); + let edit = Edit { + line, + ..edit_for("src/lib.rs") + }; + + let written = + apply_all(&mut Sink::default(), &dry_args(&root, "timeout"), &plan, &[edit], "2026-01-01").expect("apply bom edit"); + let after = fs::read_to_string(path).expect("edited source"); + + assert_eq!(written.len(), 1, "{name}"); + assert!(after.starts_with(expected), "{name}: {after:?}"); + assert_eq!(after.chars().next(), Some(crate::parse::BOM), "{name}: {after:?}"); + } + } + + /// The final publication check, unlike the discovery digest, guards the short interval after + /// the replacement has been prepared. An editor save there is a conflict, not a generated + /// directive over text the command did not inspect. + #[test] + fn a_save_after_validation_and_before_suppress_publication_is_left_alone() { + let dir = crate_dir("suppress-publication-conflict-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let editor = "// saved by the editor\npub fn answer() -> i32 { 43 }\n".to_owned(); + let editor_path = path.clone(); + + crate::elements::before_next_publication(move |_| { + fs::write(editor_path, &editor).expect("the editor save"); + }); + + let error = apply_all( + &mut Sink::default(), + &dry_args(&root, "timeout"), + &plan_over(&root, &["src/lib.rs"]), + &[edit_for("src/lib.rs")], + "2026-01-01", + ) + .expect_err("the generation changed after validation"); + + assert!( + error.to_string().contains("changed while this command was preparing to publish"), + "{error}" + ); + assert_eq!( + fs::read_to_string(path).expect("the editor's bytes"), + "// saved by the editor\npub fn answer() -> i32 { 43 }\n" + ); + } + + /// A directory sync comes after the rename, so this is the failure where the source was + /// changed even though the writer returned an error. The entry must reach `written` before + /// that error escapes, or compensation has no way to put the directive back. + #[test] + fn a_post_rename_suppress_sync_failure_is_reverted() { + let dir = crate_dir("suppress-sync-failure-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let original = fs::read_to_string(&path).expect("original"); + + crate::elements::fail_next_directory_sync(); + + let error = apply_all( + &mut Sink::default(), + &dry_args(&root, "timeout"), + &plan_over(&root, &["src/lib.rs"]), + &[edit_for("src/lib.rs")], + "2026-01-01", + ) + .expect_err("the post-rename sync fails"); + + assert!(error.to_string().contains("injected directory sync failure"), "{error}"); + assert!(error.to_string().contains("every edit has been reverted"), "{error}"); + assert_eq!(fs::read_to_string(path).expect("source restored"), original); + } + + /// A write that cannot be staged leaves the source exactly as it was, rather than truncated. + /// + /// This is the injected partial write: blocking the staging file stands in for the disk filling + /// up or the process being killed between the truncate and the replacement bytes, which is what + /// writing straight into the file left no answer for. + #[test] + fn a_write_that_cannot_be_staged_leaves_the_source_untouched() { + let dir = crate_dir("suppress-partial-write-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let original = fs::read(path.as_std_path()).expect("the original bytes"); + + let scratch = root.join(".blocked-stage"); + fs::create_dir(scratch.as_std_path()).expect("block the staging file"); + crate::elements::next_scratch_path(scratch); + + let plan = plan_over(&root, &["src/lib.rs"]); + let edits = vec![edit_for("src/lib.rs")]; + let mut host = Sink::default(); + + let error = apply_all(&mut host, &dry_args(&root, "timeout"), &plan, &edits, "2026-01-01").expect_err("the write"); + + assert!(error.to_string().contains("lib.rs"), "{error}"); + assert_eq!( + fs::read(path.as_std_path()).expect("the bytes afterwards"), + original, + "a failed write did not leave the source alone" + ); + } + + /// When the rollback itself cannot be done, both failures are reported: the one that stopped + /// the command, and the fact that the tree is not as it was found. Reporting only the first + /// would tell the user their source is back when it is not. + #[test] + fn a_rollback_that_fails_is_reported_with_what_went_wrong_first() { + let dir = crate_dir("suppress-rollback-failure-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src"); + + // A directory cannot be replaced by a file on any platform, so restoring this one fails + // for a reason no permission model can talk the test out of. + let written = vec![WrittenFile::new( + path, + "pub fn answer() -> i32 { 42 }\n".to_owned(), + "// #[gamma::skip]\npub fn answer() -> i32 { 42 }\n".to_owned(), + )]; + let error = reverted(&root, written, error!("could not read `src/zzz_gone.rs`")); + let text = error.to_string(); + + assert!(text.contains("could not read `src/zzz_gone.rs`"), "{text}"); + assert!(text.contains("changed after this command wrote them"), "{text}"); + assert!(text.contains("src"), "{text}"); + } + + /// Nothing written means nothing to revert, and no claim that anything was. + #[test] + fn a_failure_before_the_first_write_makes_no_claim_about_reverting() { + let error = reverted(Utf8Path::new("."), Written::new(), error!("could not read `src/lib.rs`")); + + assert_eq!(error.to_string(), "could not read `src/lib.rs`"); + } + + /// A failed verification must not put stale source back over a save made after this command + /// published its directive. The rollback has the same generation guard as publication. + #[test] + fn a_save_before_suppress_rollback_is_left_alone() { + let dir = crate_dir("suppress-rollback-conflict-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let original = "pub fn answer() -> i32 { 42 }\n"; + let generated = "// #[gamma::skip(arith.add_to_sub)]\npub fn answer() -> i32 { 42 }\n"; + let editor = "// saved by the editor\npub fn answer() -> i32 { 43 }\n".to_owned(); + let editor_path = path.clone(); + + fs::write(&path, generated).expect("the generated edit"); + crate::elements::before_next_publication(move |_| { + fs::write(editor_path, &editor).expect("the editor save"); + }); + + let error = reverted( + &root, + vec![WrittenFile::new(path.clone(), original.to_owned(), generated.to_owned())], + error!("verification failed"), + ); + + assert!(error.to_string().contains("changed after this command wrote it"), "{error}"); + assert_eq!( + fs::read_to_string(path).expect("the editor's bytes"), + "// saved by the editor\npub fn answer() -> i32 { 43 }\n" + ); + } + + /// The compensating rename can reach the original bytes before its directory sync fails. The + /// error must retain that durability failure without claiming the source was left edited. + #[test] + fn a_post_rename_rollback_sync_failure_is_reported_as_restored() { + let dir = crate_dir("suppress-rollback-sync-failure-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let original = "pub fn answer() -> i32 { 42 }\n"; + let generated = "// #[gamma::skip(arith.add_to_sub)]\npub fn answer() -> i32 { 42 }\n"; + + fs::write(&path, generated).expect("the generated edit"); + crate::elements::fail_next_directory_sync(); + + let error = reverted( + &root, + vec![WrittenFile::new(path.clone(), original.to_owned(), generated.to_owned())], + error!("verification failed"), + ); + + assert!(error.to_string().contains("injected directory sync failure"), "{error}"); + assert!(error.to_string().contains("every edit has been reverted, but 1 file"), "{error}"); + assert_eq!(fs::read_to_string(path).expect("source restored"), original); + } + + /// One file can reach the compensation set twice, and only the bytes found first are the + /// user's. Replaying forwards restores those and then overwrites them with the intermediate + /// text, which is worse than not reverting at all: the message says the tree is as it was. + #[test] + fn a_file_written_twice_is_restored_to_the_bytes_that_were_there_first() { + let dir = crate_dir("suppress-rollback-order-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + + let original = "pub fn answer() -> i32 { 42 }\n"; + let first = "// #[gamma::skip(arith.add_to_sub)]\npub fn answer() -> i32 { 42 }\n"; + let second = "// #[gamma::skip(arith.add_to_sub)]\n// #[gamma::skip(arith.add_to_sub)]\npub fn answer() -> i32 { 42 }\n"; + fs::write(&path, second).expect("the second generated version"); + let written = vec![ + WrittenFile::new(path.clone(), original.to_owned(), first.to_owned()), + WrittenFile::new(path.clone(), first.to_owned(), second.to_owned()), + ]; + let error = reverted(&root, written, error!("could not read `src/other.rs`")); + + assert!(error.to_string().contains("every edit has been reverted"), "{error}"); + assert_eq!( + fs::read_to_string(&path).expect("the file afterwards"), + "pub fn answer() -> i32 { 42 }\n", + "the rollback left the file holding an edit this command made" + ); + } + + /// Builds a mutant at a named site with a named verdict. + fn mutant(id: &str, line: usize, mutator: &str, outcome: crate::model::Outcome) -> crate::model::Mutant { + crate::model::Mutant { + id: id.to_owned().into(), + line, + mutator: (mutator.to_owned()).into(), + outcome, + ..crate::fixtures::mutant() + } + } + + /// Marks a mutant as suppressed, the way the apply pass does once a directive names it. + fn suppressed(mutant: &crate::model::Mutant) -> crate::model::Mutant { + crate::model::Mutant { + suppression: Some(crate::model::Suppression { + channel: crate::model::Channel::Comment, + reason: None, + tag: Some("timeout".to_owned()), + line: Some(mutant.line), + }), + ..mutant.clone() + } + } + + /// The critical one. Two mutants of the same mutator at one site is ordinary — `x + y + w` has + /// two `arith.add_to_sub` occurrences — and if one times out and the other survives, the + /// directive written for the timeout suppresses both, because matching is by mutator name. + /// Exempting the survivor from the collateral check would take a survivor out of the + /// denominator and report success, which is the one thing this command may never do. + #[test] + fn a_survivor_sharing_a_site_with_a_timeout_is_caught_as_collateral() { + let eligible = crate::fix::Eligible::parse("timeout").expect("eligibility"); + let before = vec![ + mutant("timed-out", 2, "arith.add_to_sub", crate::model::Outcome::Timeout), + mutant("survivor", 2, "arith.add_to_sub", crate::model::Outcome::Survived), + ]; + let edits = crate::fix::plan(&before, &eligible); + + let intended = intended(&before, &edits, &eligible); + + assert_eq!(intended, core::iter::once("timed-out".to_owned()).collect::>()); + + // What the directive actually does to the file: both occurrences carry that mutator name. + let after: Vec = before.iter().map(suppressed).collect(); + let result = crate::fix::verify(&before, &after, &intended); + + assert!(!result.is_clean(), "a suppressed survivor was reported as a clean edit"); + assert_eq!(result.collateral, vec!["survivor".to_owned()]); + } + + /// The other half of the same mistake, in the harmless-looking direction: a killed mutant + /// sharing a line with a timeout is not named by the directive and is not meant to be + /// suppressed, so requiring it to be suppressed reverts a perfectly good edit and blames a + /// directive placement problem that does not exist. + #[test] + fn a_killed_mutant_sharing_a_line_is_not_required_to_be_suppressed() { + let eligible = crate::fix::Eligible::parse("timeout").expect("eligibility"); + let before = vec![ + mutant("timed-out", 2, "arith.add_to_sub", crate::model::Outcome::Timeout), + mutant("killed", 2, "stmt.delete", crate::model::Outcome::Killed), + ]; + let edits = crate::fix::plan(&before, &eligible); + let intended = intended(&before, &edits, &eligible); + + // Only the timeout's mutator is named, so only the timeout is suppressed. + let after = vec![suppressed(&before[0]), before[1].clone()]; + let result = crate::fix::verify(&before, &after, &intended); + + assert!(result.is_clean(), "{result:?}"); + assert!(result.missing.is_empty(), "{:?}", result.missing); + } + + /// The count belongs to the comments written, not to the mutants they cover: three mutants at + /// one site are one directive, and reporting three sends the reader looking for two comments + /// that were never written. + #[test] + fn the_success_line_counts_directives_rather_than_mutants() { + let dir = crate_dir("suppress-count-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let mut host = Sink::default(); + let before = plan_over(&root, &["src/lib.rs"]); + + let code = verify_or_revert( + &mut host, + &dry_args(&root, "timeout"), + &before, + &["a".to_owned(), "b".to_owned(), "c".to_owned()].into_iter().collect(), + 1, + Vec::new(), + Styler::new(false), + ); + + // The intended set is unsatisfiable here, so the message is the failure's; what matters is + // that the clean path below counts what it was told rather than what it verified. + let _ = code.expect_err("an unsatisfiable intended set"); + + let mut host = Sink::default(); + let code = verify_or_revert( + &mut host, + &dry_args(&root, "timeout"), + &before, + &BTreeSet::new(), + 1, + Vec::new(), + Styler::new(false), + ) + .expect("verify"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("Suppressed 1 directive in"), "{}", host.err()); + } + + /// A rollback caused by a directive that stopped suppressing something must not print two + /// zeroes and a sentence about directives missing their target, which is a failure the reader + /// can neither explain nor act on. + #[test] + fn a_verification_failure_names_which_half_failed_and_what_it_was_about() { + let released = crate::fix::Verification { + missing: Vec::new(), + collateral: Vec::new(), + released: vec!["one".to_owned(), "two".to_owned(), "three".to_owned(), "four".to_owned()], + }; + let text = unclean(&released).to_string(); + + assert!(text.contains("4 no longer suppressed"), "{text}"); + assert!(text.contains("one, two, three and 1 more"), "{text}"); + assert!(!text.contains('0'), "a count that is zero has nothing to report: {text}"); + } + + /// Version control is the only journal this command has, and an interrupt part-way through the + /// edit loop leaves a tree nothing on disk records. A file with uncommitted changes has nothing + /// behind it, so it is refused before the first write rather than after the third. + #[test] + fn an_edit_over_a_file_with_uncommitted_changes_is_refused_before_anything_is_written() { + let dir = crate_dir("suppress-dirty-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let path = root.join("src/lib.rs"); + let original = fs::read_to_string(&path).expect("the original"); + + if !git(&root, &["init", "--quiet"]) { + return; + } + + let plan = plan_over(&root, &["src/lib.rs"]); + let edits = vec![edit_for("src/lib.rs")]; + let mut args = dry_args(&root, "timeout"); + let mut host = Sink::default(); + + let error = apply_all(&mut host, &args, &plan, &edits, "2026-01-01").expect_err("a dirty tree"); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("src/lib.rs"), "{error}"); + assert!(error.to_string().contains("--allow-dirty"), "{error}"); + assert_eq!( + fs::read_to_string(&path).expect("afterwards"), + original, + "the refusal still wrote the file" + ); + + // Someone working outside version control deliberately is entitled to say so. + args.allow_dirty = true; + + let _ = apply_all(&mut host, &args, &plan, &edits, "2026-01-01").expect("the override"); + + assert_ne!(fs::read_to_string(&path).expect("afterwards"), original); + } + + #[cfg(unix)] + #[test] + fn applying_refuses_the_whole_batch_when_a_source_symlink_leaves_the_workspace() { + let dir = crate_dir("suppress-external-link-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + let first = root.join("src/lib.rs"); + let first_before = fs::read_to_string(&first).expect("first source"); + let second = root.join("src/other.rs"); + let external = workdir("suppress-external-referent-"); + let outside = Utf8PathBuf::from_path_buf(external.path().join("outside.rs")).expect("utf8"); + let outside_before = "pub fn other() -> i32 { 7 }\n"; + + fs::write(&outside, outside_before).expect("external source"); + std::os::unix::fs::symlink(outside.as_std_path(), second.as_std_path()).expect("source link"); + + let plan = plan_over(&root, &["src/lib.rs", "src/other.rs"]); + let edits = vec![edit_for("src/lib.rs"), edit_for("src/other.rs")]; + let mut args = dry_args(&root, "timeout"); + + args.allow_dirty = true; + + let error = apply_all(&mut Sink::default(), &args, &plan, &edits, "2026-01-01") + .expect_err("an external source referent must refuse the batch"); + + assert!(error.to_string().contains("outside"), "{error}"); + assert_eq!(fs::read_to_string(first).expect("first source"), first_before); + assert_eq!(fs::read_to_string(outside).expect("external source"), outside_before); + } + + /// Demanding version control from someone who is not using it would be a different command, + /// so a tree that is not a repository gets no opinion. + #[test] + fn a_tree_that_is_not_a_repository_is_edited_without_complaint() { + let dir = crate_dir("suppress-no-repo-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + + recoverable(&root, &[Utf8Path::new("src/lib.rs")], false).expect("no repository, no opinion"); + } + + /// A run that could not build part of its population measured only part of it, and an exit + /// code of zero from this command is a completeness claim that run did not earn — + /// `cargo gamma suppress --apply && git commit -am ...` is a real pipeline, and it must not + /// commit source edits on a tree where `cargo gamma run` would have exited 3. + #[test] + fn a_run_that_could_not_build_part_of_its_population_refuses_rather_than_exiting_zero() { + let dir = crate_dir("suppress-stuck-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + + // A call to a symbol that does not exist passes `cargo check --tests`, so the preflight + // clears the tree, and then fails to link when the test targets are actually built. No + // mutant can be blamed for a linker error, so the run gets stuck rather than scoring it. + fs::write(root.join("src/lib.rs"), crate::fixtures::UNRESOLVED_LINK_SOURCE).expect("lib"); + + let source = fs::read_to_string(root.join("src/lib.rs")).expect("the fixture"); + let args = SuppressArgs { + run: RunArgs { + select: crate::commands::SelectArgs { + dir: root.clone(), + mutators: Some("relational.lt_to_le".to_owned()), + ..crate::commands::SelectArgs::default() + }, + ..RunArgs::default() + }, + dry_run_suppress: false, + allow_dirty: false, + eligible: "timeout".to_owned(), + }; + let mut host = Sink::default(); + + let code = suppress(&mut host, &args, When::Never, Styler::new(false)).expect("suppress"); + + assert_eq!(code, EXIT_CANNOT_PROCEED, "{}", host.err()); + assert!(host.err().contains("could not be made to compile"), "{}", host.err()); + assert_eq!(fs::read_to_string(root.join("src/lib.rs")).expect("afterwards"), source); + } + + /// Runs git in `root`, reporting whether it could be run at all. + fn git(root: &Utf8Path, args: &[&str]) -> bool { + Command::new("git") + .arg("-C") + .arg(root.as_std_path()) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} diff --git a/crates/cargo-gamma-lib/src/commands/unsuppress.rs b/crates/cargo-gamma-lib/src/commands/unsuppress.rs new file mode 100644 index 000000000..175f117ac --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/unsuppress.rs @@ -0,0 +1,880 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::Write; + +use camino::{Utf8Path, Utf8PathBuf}; + +use super::cli::UnsuppressArgs; +use super::dispatch::EXIT_OK; +use super::host::Host; +use super::suppress::{Written, recoverable, reject_external_sources, reverted}; +use crate::discover::Plan; +use crate::error::error; +use crate::exec::CargoOptions; +use crate::report::{Styler, quantity}; + +/// Implements `unsuppress`. +/// +/// Discovery on its own decides everything here, so nothing is built and no test is run: a directive +/// is idle when the mutants it would govern are not there, which is a fact about the source. That +/// makes this the cheap counterpart to `suppress`, which cannot know what it wants to write until a +/// run has watched every mutant misbehave. +/// +/// The safety argument is the mirror of that module's. Writing a directive is dangerous because it +/// might suppress more than intended; removing one is dangerous because it might have been holding +/// something down that the report failed to notice. Both are answered the same way — do it, discover +/// again, compare the suppressed sets, and put the tree back if anything moved. +#[cfg(test)] +pub(super) fn unsuppress(host: &mut H, args: &UnsuppressArgs, styler: Styler) -> crate::Result { + let config = crate::config::Config::resolve(&args.select)?; + let cargo = config.cargo_options(); + + unsuppress_with_cargo(host, args, styler, &cargo) +} + +/// Implements `unsuppress` with the configuration generation dispatch already resolved. +pub(super) fn unsuppress_with_cargo( + host: &mut H, + args: &UnsuppressArgs, + styler: Styler, + cargo: &CargoOptions, +) -> crate::Result { + let selection = args.select.selection()?; + let before = crate::discover::plan_for_build(&args.select, &selection, args.select.shard()?, cargo, &mut |_| {})?; + + if before.idle.is_empty() { + writeln!( + host.error(), + "{} nothing to remove: every skip directive in scope suppressed something", + styler.verb("Finished") + )?; + + return Ok(EXIT_OK); + } + + let (removable, declined) = sort_out(&before)?; + + report_declined(host, &declined, styler)?; + + if removable.is_empty() { + return Ok(EXIT_OK); + } + + let written = remove_all(host, args, &before, &removable)?; + + let removed: usize = removable.values().map(|removal| removal.lines.len()).sum(); + + if !args.apply { + writeln!( + host.error(), + "{} {} in {} would be removed; pass `--apply` to do it", + styler.verb("Preview"), + quantity(removed, "skip directive"), + quantity(removable.len(), "file") + )?; + + return Ok(EXIT_OK); + } + + verify_or_revert_with_cargo(host, args, &before, removed, written, styler, cargo) +} + +/// Rewrites every file with an idle directive in it, putting back whatever it wrote if any of them +/// fails. +/// +/// The mirror of `suppress`'s compensation, for the same reason. A read, a parse or a write can +/// fail on file N with files 1 to N-1 already rewritten, and stopping there would leave directives +/// deleted out of somebody's source by a command that reported failure — the outcome `--apply` is a +/// deliberate opt-in for, arrived at by accident. +fn remove_all(host: &mut H, args: &UnsuppressArgs, before: &Plan, removable: &Removals) -> crate::Result { + if args.apply { + let paths: Vec<&Utf8Path> = removable.keys().map(Utf8PathBuf::as_path).collect(); + + reject_external_sources(&before.root, &paths)?; + recoverable(&before.root, &paths, args.allow_dirty)?; + } + + let mut written = Written::new(); + + match remove_directives(host, args, before, removable, &mut written) { + Ok(()) => Ok(written), + + Err(cause) => Err(reverted(&before.root, written, cause)), + } +} + +/// Rewrites each file without its idle directives, recording what it held before in `written`. +/// +/// Every step is fallible and every step uses `?`, so this stops where it fails: what it has +/// already changed is in `written`, and putting that back is the caller's business. Leaving those +/// files rewritten after the command has failed would be a directive silently deleted from +/// somebody's source — the same hazard `--apply` is a deliberate opt-in for. +fn remove_directives( + host: &mut H, + args: &UnsuppressArgs, + before: &Plan, + removable: &Removals, + written: &mut Written, +) -> crate::Result<()> { + for (path, removal) in removable { + let absolute = before.root.join(path); + let source = crate::parse::strip_bom(&removal.text); + let mut after = crate::fix::remove(source, &removal.lines); + + if removal.text.len() != source.len() { + after.insert(0, crate::parse::BOM); + } + + // Parsing before writing, not after: a file that does not parse must never reach the disk, + // because the revert path is only as good as the copy it holds. + let _ = syn::parse_file(&after) + .map_err(|cause| error!("removing the directives would leave {absolute} unparseable").caused_by(cause))?; + + if args.apply { + // The path discovery accepted is lexical, because it has to preserve how a source is + // named in reports. Publishing is physical: a source symlink that leaves the + // workspace must never turn an edit requested here into a write elsewhere. + let destination = crate::paths::require_within(&absolute, &before.root, "a source edit")?; + + // Read again, not to work from, but to check that the text the line numbers were + // validated against is still the text on disk. The generation check in `sort_out` + // protects the planned directive; this one covers a save after that check but before + // publication. A line delete must never take out whatever moved into its position. + let current = fs::read_to_string(&destination).map_err(|cause| error!("could not read `{absolute}`").caused_by(cause))?; + + if current != removal.text { + return Err(error!( + "`{absolute}` changed since the run that planned this edit; nothing was removed from it" + )); + } + + // `current` is the generation the directive lines were validated against. Checking it + // again after staging catches changes through that comparison. A non-cooperating save + // in the syscall interval after comparison is outside the publication API's guarantee. + match crate::elements::write_if_unchanged(&before.root, &destination, Some(¤t), &after)? { + crate::elements::Publication::Conflict => { + return Err(error!( + "`{absolute}` changed while this command was preparing to publish its edit; the editor's bytes were left alone" + )); + } + crate::elements::Publication::Published => { + written.push(super::suppress::WrittenFile::new(destination, current, after)); + } + crate::elements::Publication::PublishedUndurable(cause) => { + // The directive is already gone from the visible generation. Keep its + // rollback state before reporting that the parent directory was not synced. + written.push(super::suppress::WrittenFile::new(destination, current, after)); + return Err(cause); + } + } + } else { + write!(host.results(), "{}", crate::fix::diff(path, &removal.text, &after))?; + } + } + + Ok(()) +} + +/// What is to be deleted from each file, gathered so that one file is read and rewritten once. +type Removals = BTreeMap; + +/// The lines to delete from one file, and the text they were decided against. +/// +/// The text travels with the line numbers because a line number means nothing without it: the +/// validation that decided line 42 holds a directive and the delete that takes line 42 out are two +/// operations over a file anybody else may write between them, and re-reading it for the second one +/// is what lets them disagree. +#[derive(Debug)] +struct Removal { + /// The one-based lines holding a directive that a plain line delete removes cleanly. + lines: BTreeSet, + + /// The file's contents at the moment those lines were validated. + text: String, +} + +/// Splits the idle directives into the ones a line delete removes cleanly and the ones it does not. +/// +/// Grouped by file rather than kept flat, because the removal reads and rewrites each file once and +/// the line numbers within it have to be applied together. +fn sort_out(plan: &Plan) -> crate::Result<(Removals, Vec<&crate::suppress::Idle>)> { + let mut removable = Removals::new(); + let mut declined = Vec::new(); + let mut sources: BTreeMap<&Utf8PathBuf, (String, Vec)> = BTreeMap::new(); + + for idle in &plan.idle { + let (text, lines) = match sources.entry(&idle.file) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let path = plan.root.join(&idle.file); + let text = fs::read_to_string(&path).map_err(|cause| error!("could not read `{path}`").caused_by(cause))?; + let source = crate::parse::strip_bom(&text); + let Some(recorded) = plan.digests.get(&idle.file) else { + return Err(error!( + "`{path}` has no recorded generation for this planned edit; the planned directive was left alone" + )); + }; + + // A removable-looking current line is not enough: an editor could have replaced + // the idle directive with a live one at the same line. The digest binds the line + // deletion to the exact source discovery classified as idle. + if crate::discover::digest(source.as_bytes()) != *recorded { + return Err(error!( + "`{path}` changed since the run that planned this edit; the planned directive was left alone. Re-run to plan against the file as it is now" + )); + } + + let split = source.lines().map(str::to_owned).collect(); + + entry.insert((text, split)) + } + }; + + if idle + .line + .checked_sub(1) + .and_then(|index| lines.get(index)) + .is_some_and(|line| crate::fix::removable(line)) + { + // The text is carried out with the line numbers rather than read again by the removal, + // so that what was validated and what is edited are one and the same bytes. + let entry = removable.entry(idle.file.clone()).or_insert_with(|| Removal { + lines: BTreeSet::new(), + text: text.clone(), + }); + + let _ = entry.lines.insert(idle.line); + } else { + declined.push(idle); + } + } + + Ok((removable, declined)) +} + +/// Names the idle directives that were left alone, and says why. +/// +/// Silence here would be the worst outcome: the run has just reported these as suppressing nothing, +/// and a removal that quietly skips them leaves the user believing they are gone. +fn report_declined(host: &mut H, declined: &[&crate::suppress::Idle], styler: Styler) -> crate::Result<()> { + if declined.is_empty() { + return Ok(()); + } + + writeln!( + host.error(), + "{} {} share a line with something else and must be removed by hand", + styler.verb("Skipping"), + quantity(declined.len(), "skip directive") + )?; + + for idle in declined { + writeln!(host.error(), " {}:{}: skip({})", idle.file, idle.line, idle.selectors)?; + } + + Ok(()) +} + +/// Re-runs discovery over the edited tree and reverts unless nothing about the population moved. +/// +/// The check is exact, and that is what makes this safe to run unattended. A directive that +/// suppressed nothing cannot, by removing it, change anything at all: the same mutants must be +/// found, and the same ones must still be suppressed. Anything else means the report that named the +/// directive was wrong, and the tree goes back. +#[cfg(test)] +fn verify_or_revert( + host: &mut H, + args: &UnsuppressArgs, + before: &Plan, + removed: usize, + written: Written, + styler: Styler, +) -> crate::Result { + let config = crate::config::Config::resolve(&args.select)?; + let cargo = config.cargo_options(); + + verify_or_revert_with_cargo(host, args, before, removed, written, styler, &cargo) +} + +/// Verifies against the Cargo options that planned the edit. +fn verify_or_revert_with_cargo( + host: &mut H, + args: &UnsuppressArgs, + before: &Plan, + removed: usize, + written: Written, + styler: Styler, + cargo: &CargoOptions, +) -> crate::Result { + let verified = (|| { + let selection = args.select.selection()?; + let after = crate::discover::plan_for_build(&args.select, &selection, args.select.shard()?, cargo, &mut |_| {})?; + + Ok(crate::fix::verify(&before.mutants, &after.mutants, &BTreeSet::new())) + })(); + let result = match verified { + Ok(result) => result, + Err(cause) => return Err(reverted(&before.root, written, cause)), + }; + + if result.is_clean() { + writeln!( + host.error(), + "{} {} from {}", + styler.verb("Removed"), + quantity(removed, "skip directive"), + quantity(written.len(), "file") + )?; + + return Ok(EXIT_OK); + } + + Err(reverted( + &before.root, + written, + error!( + "removing the directives changed what the run found ({} mutants stopped being suppressed, {} started)", + result.released.len(), + result.collateral.len() + ), + )) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + use crate::fixtures::crate_dir; + use crate::suppress::Idle; + use crate::testing::Sink; + #[cfg(unix)] + use crate::testing::workdir; + + /// The first line of `source` marked for removal, as `sort_out` would have decided it. + fn removal(source: &str) -> Removal { + Removal { + lines: core::iter::once(1).collect(), + text: source.to_owned(), + } + } + + fn args(root: &Utf8PathBuf, apply: bool) -> UnsuppressArgs { + UnsuppressArgs { + select: crate::commands::SelectArgs { + dir: root.clone(), + ..crate::commands::SelectArgs::default() + }, + apply, + allow_dirty: false, + } + } + + /// The default has to be the preview, because deleting a directive that was in fact + /// load-bearing turns a considered decision into a survivor nobody chose to accept. + #[test] + fn a_preview_shows_the_removal_and_leaves_the_file_alone() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-preview-", source); + let mut host = Sink::default(); + + let code = unsuppress(&mut host, &args(&root, false), Styler::new(false)).expect("preview"); + + assert_eq!(code, EXIT_OK); + assert!(host.out().contains("-// #[gamma::skip(arith)]"), "{}", host.out()); + assert!(host.err().contains("--apply"), "{}", host.err()); + assert_eq!(fs::read_to_string(root.join("src/lib.rs")).expect("source"), source); + } + + #[test] + fn applying_removes_the_directive_and_nothing_else() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-apply-", source); + let mut host = Sink::default(); + + let code = unsuppress(&mut host, &args(&root, true), Styler::new(false)).expect("apply"); + + assert_eq!(code, EXIT_OK); + assert_eq!( + fs::read_to_string(root.join("src/lib.rs")).expect("source"), + "pub fn f(a: i32) -> bool { a > 1 }\n" + ); + assert!(host.err().contains("Removed 1 skip directive"), "{}", host.err()); + } + + #[cfg(unix)] + #[test] + fn applying_refuses_a_source_symlink_whose_referent_is_outside_the_workspace() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_workspace, root) = crate_dir("unsuppress-external-link-", source); + let external = workdir("unsuppress-external-referent-"); + let outside = Utf8PathBuf::from_path_buf(external.path().join("outside.rs")).expect("UTF-8 path"); + let link = root.join("src/lib.rs"); + + fs::write(&outside, source).expect("external source"); + fs::remove_file(&link).expect("replace source with link"); + std::os::unix::fs::symlink(outside.as_std_path(), link.as_std_path()).expect("source link"); + + let failure = unsuppress(&mut Sink::default(), &args(&root, true), Styler::new(false)) + .expect_err("an external source referent must not be edited"); + + assert!(failure.to_string().contains("outside"), "{failure}"); + assert_eq!(fs::read_to_string(&outside).expect("external source"), source); + } + + /// A BOM is a marker for the whole file, not for the first directive. Removing that directive + /// must retain the marker before the code that follows, whether the directive was on the first + /// line or later in the file. + #[test] + fn bom_prefixed_files_keep_the_bom_when_removing_first_and_later_directives() { + for (name, source, expected) in [ + ( + "first", + "\u{feff}// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n", + "\u{feff}pub fn f(a: i32) -> bool { a > 1 }\n", + ), + ( + "later", + "\u{feff}//! A crate.\n// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n", + "\u{feff}//! A crate.\npub fn f(a: i32) -> bool { a > 1 }\n", + ), + ] { + let (_dir, root) = crate_dir(&format!("unsuppress-bom-{name}-"), source); + let path = root.join("src/lib.rs"); + let mut host = Sink::default(); + + let code = unsuppress(&mut host, &args(&root, true), Styler::new(false)).expect("apply"); + let after = fs::read_to_string(path).expect("source"); + + assert_eq!(code, EXIT_OK, "{name}"); + assert_eq!(after, expected, "{name}"); + assert_eq!(after.chars().next(), Some(crate::parse::BOM), "{name}: {after:?}"); + } + } + + #[test] + fn a_verification_error_restores_every_removed_directive() { + let original = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-verify-error-", original); + let path = root.join("src/lib.rs"); + let removed = "pub fn f(a: i32) -> bool { a > 1 }\n"; + fs::write(&path, removed).expect("edited"); + let mut args = args(&root, true); + args.select.mutators = Some("not.a.mutator".to_owned()); + let mut host = Sink::default(); + + let failure = verify_or_revert( + &mut host, + &args, + &Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root, + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + }, + 1, + vec![super::super::suppress::WrittenFile::new( + path.clone(), + original.to_owned(), + removed.to_owned(), + )], + Styler::new(false), + ) + .expect_err("selection must fail"); + + assert!(failure.to_string().contains("every edit has been reverted"), "{failure}"); + assert_eq!(fs::read_to_string(path).expect("restored"), original); + } + + /// The quiet case. A directive that is still doing its job must not be touched, and the + /// command must say so rather than printing an empty diff. + #[test] + fn a_directive_that_still_suppresses_something_is_left_alone() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> i32 { a + 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-live-", source); + let mut host = Sink::default(); + + let code = unsuppress(&mut host, &args(&root, true), Styler::new(false)).expect("nothing"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("nothing to remove"), "{}", host.err()); + assert_eq!(fs::read_to_string(root.join("src/lib.rs")).expect("source"), source); + } + + /// A directive sharing its line with code cannot be deleted by deleting the line, and saying + /// nothing about it would leave the user believing it was gone. + #[test] + fn a_directive_that_shares_its_line_is_named_rather_than_removed() { + let source = "pub fn f(a: i32) -> bool { a > 1 } // #[gamma::skip(arith)]\n"; + let (_dir, root) = crate_dir("unsuppress-declined-", source); + let mut host = Sink::default(); + + let code = unsuppress(&mut host, &args(&root, true), Styler::new(false)).expect("declined"); + + assert_eq!(code, EXIT_OK); + assert!(host.err().contains("by hand"), "{}", host.err()); + assert!(host.err().contains("src/lib.rs:1"), "{}", host.err()); + assert_eq!(fs::read_to_string(root.join("src/lib.rs")).expect("source"), source); + } + + /// The line numbers have to be applied together, or removing the first shifts the second. + #[test] + fn several_directives_in_one_file_are_all_removed() { + let source = + "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n// #[gamma::skip(arith)]\npub fn g(a: i32) -> bool { a < 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-several-", source); + let mut host = Sink::default(); + + let _ = unsuppress(&mut host, &args(&root, true), Styler::new(false)).expect("apply"); + + let text = fs::read_to_string(root.join("src/lib.rs")).expect("source"); + + assert!(!text.contains("gamma::skip"), "{text}"); + assert!(text.contains("fn f") && text.contains("fn g"), "{text}"); + } + + #[test] + fn a_declined_directive_is_reported_with_its_place_and_selectors() { + let idle = Idle { + file: Utf8PathBuf::from("src/lib.rs"), + line: 7, + selectors: "arith".to_owned(), + reason: None, + }; + let mut host = Sink::default(); + + report_declined(&mut host, &[&idle], Styler::new(false)).expect("note"); + + let text = host.err(); + + assert!(text.contains("1 skip directive"), "{text}"); + assert!(text.contains("src/lib.rs:7"), "{text}"); + assert!(text.contains("skip(arith)"), "{text}"); + } + + #[test] + fn nothing_declined_says_nothing() { + let mut host = Sink::default(); + + report_declined(&mut host, &[], Styler::new(false)).expect("note"); + + assert!(host.err().is_empty(), "{}", host.err()); + } + + /// The defence in depth, and the one no ordinary tree can reach: a directive reported as idle + /// cannot change anything by being removed, so the only way to exercise the revert is to hand + /// the verification a "before" that claims otherwise. If it ever fires for real, the report that + /// named the directive was wrong, and the tree has to go back exactly as it was. + #[test] + fn a_removal_that_changes_what_is_suppressed_puts_every_file_back() { + let source = "pub fn f(a: i32) -> i32 { a + 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-revert-", source); + let path = root.join("src/lib.rs"); + + fs::write(&path, "pub fn f(a: i32) -> i32 { a - 1 }\n").expect("edited"); + + let mut before = crate::discover::plan( + &crate::commands::SelectArgs { + dir: root.clone(), + ..crate::commands::SelectArgs::default() + }, + &crate::ops::registry::Selection::default_preset(), + None, + &mut |_| {}, + ) + .expect("plan"); + + // A claim that something was suppressed, which the fresh discovery will not agree with. + before.mutants[0].suppression = Some(crate::model::Suppression { + channel: crate::model::Channel::Comment, + reason: None, + tag: None, + line: Some(1), + }); + + let mut host = Sink::default(); + let written = vec![super::super::suppress::WrittenFile::new( + path.clone(), + source.to_owned(), + "pub fn f(a: i32) -> i32 { a - 1 }\n".to_owned(), + )]; + let error = verify_or_revert(&mut host, &args(&root, true), &before, 1, written, Styler::new(false)).unwrap_err(); + + assert!(error.to_string().contains("reverted"), "{error}"); + assert_eq!(fs::read_to_string(&path).expect("source"), source, "the file was not put back"); + } + + /// A plan holding only the root, which is all the removal loop reads out of one. + fn plan_at(root: &Utf8PathBuf) -> Plan { + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: root.clone(), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + } + } + + /// The two-file case the compensation exists for. A directive deleted out of the first file by + /// a command that then failed is the worst of both worlds: the user is told nothing happened, + /// and the reason somebody wrote that directive is gone from the tree. + #[test] + fn a_failure_on_the_second_file_puts_the_first_one_back_byte_for_byte() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-rollback-", source); + let first = root.join("src/lib.rs"); + let original = fs::read(first.as_std_path()).expect("the original bytes"); + + // Absent, and named so that it sorts after the first: reading it fails on every platform + // and for every user, root included. + let removable: Removals = [ + (Utf8PathBuf::from("src/lib.rs"), removal(source)), + (Utf8PathBuf::from("src/zzz_gone.rs"), removal(source)), + ] + .into_iter() + .collect(); + let mut host = Sink::default(); + + let error = remove_all(&mut host, &args(&root, true), &plan_at(&root), &removable).expect_err("the second file"); + + assert!(error.to_string().contains("zzz_gone.rs"), "{error}"); + assert!(error.to_string().contains("every edit has been reverted"), "{error}"); + assert_eq!( + fs::read(first.as_std_path()).expect("the bytes afterwards"), + original, + "the first file was left with its directive removed" + ); + } + + /// The same loop, with nothing in its way, does rewrite both files — so the test above is + /// asserting that a rollback happened rather than that the loop never got started. + #[test] + fn both_files_are_rewritten_when_nothing_fails() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-both-", source); + + fs::write(root.join("src/other.rs"), source).expect("other"); + + let removable: Removals = [ + (Utf8PathBuf::from("src/lib.rs"), removal(source)), + (Utf8PathBuf::from("src/other.rs"), removal(source)), + ] + .into_iter() + .collect(); + let mut host = Sink::default(); + + let written = remove_all(&mut host, &args(&root, true), &plan_at(&root), &removable).expect("both files"); + + assert_eq!(written.len(), 2); + assert!(!fs::read_to_string(root.join("src/lib.rs")).expect("lib").contains("gamma::skip")); + assert!( + !fs::read_to_string(root.join("src/other.rs")) + .expect("other") + .contains("gamma::skip") + ); + } + + /// A write that cannot be staged leaves the source exactly as it was, rather than truncated: + /// the injected partial write, standing in for a full disk or a kill between the truncate and + /// the replacement bytes. + #[test] + fn a_write_that_cannot_be_staged_leaves_the_source_untouched() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-partial-write-", source); + let path = root.join("src/lib.rs"); + let original = fs::read(path.as_std_path()).expect("the original bytes"); + + let scratch = root.join(".blocked-stage"); + fs::create_dir(scratch.as_std_path()).expect("block the staging file"); + crate::elements::next_scratch_path(scratch); + + let removable: Removals = core::iter::once((Utf8PathBuf::from("src/lib.rs"), removal(source))).collect(); + let mut host = Sink::default(); + + let error = remove_all(&mut host, &args(&root, true), &plan_at(&root), &removable).expect_err("the write"); + + assert!(error.to_string().contains("lib.rs"), "{error}"); + assert_eq!( + fs::read(path.as_std_path()).expect("the bytes afterwards"), + original, + "a failed write did not leave the source alone" + ); + } + + /// Validating a line and deleting it are two operations over a file the user may write between + /// them. A save that shifts the contents by a line turns "delete the directive on line 1" into + /// "delete line 1", which is now somebody's code. The source generation from discovery is the + /// binding between the planned directive and that line, so it is checked before removals are + /// even selected. + #[test] + fn a_file_that_changed_since_planning_is_left_alone_rather_than_edited_by_line_number() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-shifted-", source); + let path = root.join("src/lib.rs"); + let mut plan = Plan { + idle: vec![Idle { + file: Utf8PathBuf::from("src/lib.rs"), + line: 1, + selectors: "arith".to_owned(), + reason: None, + }], + ..plan_at(&root) + }; + let _recorded = plan.digests.insert( + Utf8PathBuf::from("src/lib.rs"), + crate::discover::digest(crate::parse::strip_bom(source).as_bytes()), + ); + + // The user saves the file between discovery and selecting a directive to remove. + let edited = "//! A crate.\n// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + fs::write(&path, edited).expect("the concurrent save"); + + let error = sort_out(&plan).expect_err("the changed file"); + + assert!( + error.to_string().contains("changed since the run that planned this edit"), + "{error}" + ); + assert_eq!( + fs::read_to_string(&path).expect("the file afterwards"), + edited, + "the removal edited a file it had not examined" + ); + } + + /// An editor can replace an idle directive with a live one without moving its line. Checking + /// only that the current line is removable would then delete a new suppression and report a + /// successful no-op verification. The planned source generation rejects the replacement. + #[test] + fn a_planned_idle_directive_replaced_with_a_live_one_is_retained() { + let idle = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let live = "// #[gamma::skip(relational)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-replaced-directive-", idle); + let path = root.join("src/lib.rs"); + let mut plan = Plan { + idle: vec![Idle { + file: Utf8PathBuf::from("src/lib.rs"), + line: 1, + selectors: "arith".to_owned(), + reason: None, + }], + ..plan_at(&root) + }; + let _recorded = plan.digests.insert( + Utf8PathBuf::from("src/lib.rs"), + crate::discover::digest(crate::parse::strip_bom(idle).as_bytes()), + ); + + fs::write(&path, live).expect("replacement directive"); + + let error = sort_out(&plan).expect_err("the planned directive no longer exists"); + + assert!(error.to_string().contains("planned directive was left alone"), "{error}"); + assert_eq!(fs::read_to_string(path).expect("source"), live); + } + + /// Rechecking after staging closes the remaining interval after line validation: the editor's + /// new generation wins, and the deletion is reported as a conflict rather than published. + #[test] + fn a_save_after_validation_and_before_unsuppress_publication_is_left_alone() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-publication-conflict-", source); + let path = root.join("src/lib.rs"); + let editor = "//! saved by the editor\n// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n".to_owned(); + let editor_path = path.clone(); + let removable: Removals = core::iter::once((Utf8PathBuf::from("src/lib.rs"), removal(source))).collect(); + + crate::elements::before_next_publication(move |_| { + fs::write(editor_path, &editor).expect("the editor save"); + }); + + let error = remove_all(&mut Sink::default(), &args(&root, true), &plan_at(&root), &removable) + .expect_err("the generation changed after validation"); + + assert!( + error.to_string().contains("changed while this command was preparing to publish"), + "{error}" + ); + assert_eq!( + fs::read_to_string(path).expect("the editor's bytes"), + "//! saved by the editor\n// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n" + ); + } + + /// The deletion is visible before syncing its directory can fail. It must be added to the + /// compensation set first, so an error after the rename restores the directive. + #[test] + fn a_post_rename_unsuppress_sync_failure_is_reverted() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-sync-failure-", source); + let path = root.join("src/lib.rs"); + let removable: Removals = core::iter::once((Utf8PathBuf::from("src/lib.rs"), removal(source))).collect(); + + crate::elements::fail_next_directory_sync(); + + let error = + remove_all(&mut Sink::default(), &args(&root, true), &plan_at(&root), &removable).expect_err("the post-rename sync fails"); + + assert!(error.to_string().contains("injected directory sync failure"), "{error}"); + assert!(error.to_string().contains("every edit has been reverted"), "{error}"); + assert_eq!(fs::read_to_string(path).expect("directive restored"), source); + } + + /// Deleting a directive deletes the reason somebody wrote it, and an interrupt part-way through + /// the loop leaves no artifact behind to reconstruct it from. Version control is the journal, + /// so a file with nothing committed behind it is refused before the first delete. + #[test] + fn removing_from_a_file_with_uncommitted_changes_is_refused_before_anything_is_deleted() { + let source = "// #[gamma::skip(arith)]\npub fn f(a: i32) -> bool { a > 1 }\n"; + let (_dir, root) = crate_dir("unsuppress-dirty-", source); + let path = root.join("src/lib.rs"); + + let started = std::process::Command::new("git") + .arg("-C") + .arg(root.as_std_path()) + .args(["init", "--quiet"]) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + + if !started.is_ok_and(|status| status.success()) { + return; + } + + let removable: Removals = core::iter::once((Utf8PathBuf::from("src/lib.rs"), removal(source))).collect(); + let mut arguments = args(&root, true); + + let error = remove_all(&mut Sink::default(), &arguments, &plan_at(&root), &removable).expect_err("a dirty tree"); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("--allow-dirty"), "{error}"); + assert_eq!( + fs::read_to_string(&path).expect("afterwards"), + source, + "the refusal still edited the file" + ); + + arguments.allow_dirty = true; + + let _ = remove_all(&mut Sink::default(), &arguments, &plan_at(&root), &removable).expect("the override"); + + assert!(!fs::read_to_string(&path).expect("afterwards").contains("gamma::skip")); + } +} diff --git a/crates/cargo-gamma-lib/src/commands/verdict_log.rs b/crates/cargo-gamma-lib/src/commands/verdict_log.rs new file mode 100644 index 000000000..8bf6a165b --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/verdict_log.rs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::fs::File; +use std::io::Write as _; + +use camino::{Utf8Path, Utf8PathBuf}; + +use super::console_events::mutant_detail; +use crate::error::error; +use crate::model::Mutant; +use crate::report::Styler; + +pub(super) const TESTING_PROGRESS_LOG: &str = "gamma-progress.log"; + +#[derive(Debug, Default)] +pub(super) enum VerdictLog { + #[default] + Disabled, + Writing { + file: File, + path: Utf8PathBuf, + }, + Failed { + path: Utf8PathBuf, + cause: std::io::Error, + }, +} + +impl VerdictLog { + pub(super) fn start(&mut self, scratch: &Utf8Path) -> crate::Result<()> { + let path = scratch.join(TESTING_PROGRESS_LOG); + let file = File::create(path.as_std_path()) + .map_err(|cause| error!("could not create the testing progress log at `{path}`").caused_by(cause))?; + + *self = Self::Writing { file, path }; + + Ok(()) + } + + /// Writes and flushes one verdict, retaining the first failure for [`Self::finish`]. + pub(super) fn record(&mut self, mutant: &Mutant) { + let failed = match self { + Self::Writing { file, .. } => { + let label = Styler::new(false).outcome(mutant.outcome); + let detail = mutant_detail(mutant); + + writeln!(file, "{label} {detail}").and_then(|()| file.flush()).err() + } + Self::Disabled | Self::Failed { .. } => None, + }; + + let Some(cause) = failed else { return }; + let path = match self { + Self::Writing { path, .. } => path.clone(), + Self::Disabled | Self::Failed { .. } => return, + }; + + *self = Self::Failed { path, cause }; + } + + pub(super) fn finish(&mut self) -> crate::Result<()> { + let previous = core::mem::take(self); + + match previous { + Self::Failed { path, cause } => Err(error!("could not write the testing progress log at `{path}`").caused_by(cause)), + Self::Disabled | Self::Writing { .. } => Ok(()), + } + } +} diff --git a/crates/cargo-gamma-lib/src/commands/when.rs b/crates/cargo-gamma-lib/src/commands/when.rs new file mode 100644 index 000000000..55bcde89f --- /dev/null +++ b/crates/cargo-gamma-lib/src/commands/when.rs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use clap::ValueEnum; + +/// When to colorize output. +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Default)] +pub enum When { + /// Colorize when the stream is a terminal. + #[default] + Auto, + + /// Never colorize. + Never, + + /// Always colorize. + Always, +} + +impl When { + /// Resolves the setting against whether the stream is actually a terminal. + #[must_use] + pub const fn resolve(self, is_terminal: bool) -> bool { + match self { + Self::Auto => is_terminal, + Self::Never => false, + Self::Always => true, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn color_auto_follows_the_terminal() { + assert!(When::Auto.resolve(true)); + assert!(!When::Auto.resolve(false)); + assert!(!When::Never.resolve(true)); + assert!(When::Always.resolve(false)); + } +} diff --git a/crates/cargo-gamma-lib/src/config.rs b/crates/cargo-gamma-lib/src/config.rs new file mode 100644 index 000000000..c4fa60c08 --- /dev/null +++ b/crates/cargo-gamma-lib/src/config.rs @@ -0,0 +1,997 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The project configuration file, `gamma.toml`. +//! +//! A mutation run has a lot of knobs, and a project that has settled on its selection and run shape +//! should not have to repeat them in every CI job and every developer's shell history. One-off +//! output and control options remain command-line concerns. +//! +//! Two decisions shape the whole module. +//! +//! **Unknown keys are errors.** A configuration file whose settings are silently ignored is worse +//! than no configuration file, because the project believes it is configured. A misspelled key, or a +//! key for a feature this build does not have, stops the run and names the offender. +//! +//! **`.cargo/mutants.toml` is never read.** It is a different schema for a different tool, and +//! honoring it silently would mean that file's `exclude_re` entries quietly changing which +//! mutants this one suppresses. + +use std::fs; +use std::io::ErrorKind; + +use camino::{Utf8Path, Utf8PathBuf}; +use serde::Deserialize; + +use crate::commands::{RunArgs, SelectArgs}; +use crate::error::{Error, error}; +use crate::{Result, bounds}; + +/// Where the file lives, relative to the directory being analyzed. +const RELATIVE_PATH: &str = "gamma.toml"; + +/// A foreign configuration file that is noticed but deliberately not read. +const FOREIGN_PATH: &str = ".cargo/mutants.toml"; + +/// A parsed `gamma.toml`. +/// +/// Every field is optional: a file that sets one key is a valid file, and the rest keep whatever +/// the command line or the built-in default says. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(default, deny_unknown_fields, rename_all = "kebab-case")] +pub struct Config { + /// The mutator selector list, as it would be written after `--mutators`. + /// + /// A list rather than a string, because a configuration file has room to put one selector per + /// line with a comment explaining why. The entries are joined with commas and parsed by exactly + /// the same code that parses the flag, so the two cannot drift. + pub mutators: Option>, + + /// Globs limiting which files are mutated. + pub files: Vec, + + /// Globs excluding files from mutation. + pub exclude_files: Vec, + + /// Fail the run below this mutation score. + pub min_score: Option, + + /// How many mutants to test at once. + pub jobs: Option, + + /// The multiple of each test binary's baseline duration a mutant is allowed. + pub test_timeout_multiplier: Option, + + /// How an incremental run reuses the last run: `no` or `build`. + pub incremental: Option, + + /// Skip the baseline measurement. + pub no_baseline: Option, + + /// Believe a failing test without re-running it with no mutant active. + pub no_confirm: Option, + + /// Packages to mutate. Empty means every package in the workspace. + pub packages: Vec, + + /// Packages whose tests decide a verdict. Empty means each mutant's own package. + pub test_packages: Vec, + + /// Let tests from every workspace package judge mutants they can reach. + pub test_workspace: Option, + + /// Run every selected test in each reachable test binary instead of selecting cases by reachability. + pub whole_test_binaries: Option, + + /// Test target name globs whose tests may decide a verdict. Empty means all of them. + pub include_tests: Vec, + + /// Test target name globs whose tests must not decide a verdict. + pub exclude_tests: Vec, + + /// Cargo features to activate. + pub features: Vec, + + /// Activate every feature of every selected package. + pub all_features: Option, + + /// Do not activate the `default` feature. + pub no_default_features: Option, + + /// The cargo profile to build with. + pub profile: Option, + + /// Extra arguments for every cargo invocation. + pub cargo_args: Vec, + + /// Extra arguments for every test binary. + pub cargo_test_args: Vec, + + /// Additional `Err(...)` values for `fn_value.err_with`. + pub errors: Vec, + + /// A lower bound on the per-mutant timeout, in seconds. + pub minimum_test_timeout: Option, + + /// Run test binaries through `cargo nextest` for per-test process isolation. + pub nextest: Option, + + /// How much memory control to place around each test binary. + pub memory: Option, + + /// The multiple of a test binary's baseline peak memory a mutant of it may reach. + pub memory_multiplier: Option, + + /// Absolute headroom added to a test binary's baseline peak memory, as a size such as `128MiB`. + pub memory_headroom: Option, + + /// An explicit memory ceiling for every test binary, as a size such as `2GiB`. + pub memory_limit: Option, + + /// A memory ceiling for the baseline runs themselves, as a size such as `4GiB`. + pub baseline_memory_limit: Option, + + /// A fixed build timeout, in seconds. + pub build_timeout: Option, + + /// The multiple of the first build's duration a later build round is allowed. + pub build_timeout_multiplier: Option, + + /// Directory for all user-facing artifacts. + pub artifact_dir: Option, + + /// Sharding. + #[serde(default)] + pub shard: Shard, + + /// File reports. + #[serde(default)] + pub reporters: Reporters, +} + +/// Reads a size key that [`Config::validate`] has already accepted. +/// +/// A key that did not parse stopped the run before this point, so there is nothing left here to +/// report and nothing to fall back to but leaving the setting unset. +fn size(text: Option<&str>) -> Option { + let text = text?; + + bounds::size(text).ok() +} + +/// The `[shard]` table. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(default, deny_unknown_fields, rename_all = "kebab-case")] +pub struct Shard { + /// How many shards to divide the mutants into. + pub count: Option, + + /// Which shard to run, from zero. + pub index: Option, +} + +/// The `[reporters]` table. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(default, deny_unknown_fields, rename_all = "kebab-case")] +pub struct Reporters { + /// Load the viewer from a CDN instead of embedding it. + pub html_external: Option, +} + +impl Config { + /// The Cargo-only settings discovery must share with the eventual run. + #[must_use] + pub(crate) fn cargo_options(&self) -> crate::exec::CargoOptions { + crate::exec::CargoOptions { + profile: self.profile.clone(), + extra: self.cargo_args.clone(), + ..crate::exec::CargoOptions::default() + } + } + + /// Loads the configuration named by the command line, honoring `--config` and `--no-config`. + /// + /// An explicit path must exist: asking for a file and silently getting the defaults because it + /// was misspelled is the failure this guards against, whereas a missing conventional file is + /// the ordinary case. + pub fn resolve(select: &SelectArgs) -> Result { + if select.config.no_config { + return Ok(Self::default()); + } + + let Some(path) = select.config.path.as_ref() else { + return Self::load(&select.dir); + }; + + let text = fs::read_to_string(path).map_err(|cause| error!("could not read `{path}`").caused_by(cause))?; + + Self::parse(&text).map_err(|cause| error!("{path}: {cause}").usage()) + } + + /// Loads the configuration for a directory, if there is one. + /// + /// Returns the default configuration when the file is absent, which is the overwhelmingly + /// common case and is not worth distinguishing from an empty file. + pub fn load(dir: &Utf8Path) -> Result { + let path = dir.join(RELATIVE_PATH); + + let text = match fs::read_to_string(&path) { + Ok(text) => text, + + Err(cause) if cause.kind() == ErrorKind::NotFound => return Ok(Self::default()), + + Err(cause) => return Err(error!("could not read `{path}`").caused_by(cause)), + }; + + Self::parse(&text).map_err(|cause| error!("{path}: {cause}").usage()) + } + + /// Parses configuration text. + /// + /// Separated from [`Self::load`] so the schema can be tested without touching a file system. + pub fn parse(text: &str) -> Result { + let config: Self = toml::from_str(text).map_err(|cause| { + // toml's own message carries the line, the column and a caret, so it is a better + // diagnostic than anything reconstructed here would be. + cause.message().to_owned() + })?; + + config.validate()?; + + Ok(config) + } + + /// Range-checks the numeric keys. + /// + /// The command line checks the same values through its own parsers, but a setting can arrive + /// from either place and only one of the two would otherwise be guarded. + fn validate(&self) -> Result<(), String> { + /// A key, its value if set, and the range check that applies to it. + type Check = (&'static str, Option, fn(&str, f64) -> Result); + + let checks: [Check; 6] = [ + ("test-timeout-multiplier", self.test_timeout_multiplier, bounds::factor), + ("minimum-test-timeout", self.minimum_test_timeout, bounds::seconds), + ("build-timeout", self.build_timeout, bounds::seconds), + ("build-timeout-multiplier", self.build_timeout_multiplier, bounds::factor), + ("min-score", self.min_score, bounds::percentage), + ("memory-multiplier", self.memory_multiplier, bounds::factor), + ]; + + for (key, value, check) in checks { + if let Some(value) = value { + let _checked = check(&value.to_string(), value).map_err(|cause| format!("{key}: {cause}"))?; + } + } + + let sizes = [ + ("memory-headroom", self.memory_headroom.as_deref()), + ("memory-limit", self.memory_limit.as_deref()), + ("baseline-memory-limit", self.baseline_memory_limit.as_deref()), + ]; + + for (key, value) in sizes { + if let Some(value) = value { + let _checked = bounds::size(value).map_err(|cause| format!("{key}: {cause}"))?; + } + } + + Ok(()) + } + + /// Reports whether a foreign configuration file exists but is not being read. + /// + /// A project with only this foreign file would otherwise see its settings silently do nothing, + /// so the run says so out loud. + #[must_use] + pub fn foreign_present(dir: &Utf8Path) -> bool { + dir.join(FOREIGN_PATH).is_file() && !dir.join(RELATIVE_PATH).is_file() + } + + /// Applies the configuration underneath the command line. + /// + /// Scalars set on the command line win outright: a flag typed for this one run is the most + /// specific statement of intent available. Lists concatenate, with the command line first, so a + /// configured exclusion cannot be lost by adding one more on the command line. + /// + /// # Errors + /// + /// Returns a usage error if the merged settings contradict one another; see + /// [`validate_effective`](Self::validate_effective). + pub fn apply(&self, args: &mut RunArgs) -> Result<()> { + self.apply_selection(&mut args.select)?; + + let implied_by_cli = crate::exec::implied_memory_control(args.measure.memory_limit, args.measure.baseline_memory_limit); + + args.min_score = args.min_score.or(self.min_score); + args.measure.jobs = args.measure.jobs.or(self.jobs); + args.measure.test_timeout_multiplier = args.measure.test_timeout_multiplier.or(self.test_timeout_multiplier); + args.measure.minimum_test_timeout = args.measure.minimum_test_timeout.or(self.minimum_test_timeout); + args.measure.nextest = args.measure.nextest || self.nextest.unwrap_or(false); + args.measure.memory = args.measure.memory.or(implied_by_cli).or(self.memory); + args.measure.memory_multiplier = args.measure.memory_multiplier.or(self.memory_multiplier); + args.measure.memory_headroom = args.measure.memory_headroom.or_else(|| size(self.memory_headroom.as_deref())); + args.measure.memory_limit = args.measure.memory_limit.or_else(|| size(self.memory_limit.as_deref())); + args.measure.baseline_memory_limit = args + .measure + .baseline_memory_limit + .or_else(|| size(self.baseline_memory_limit.as_deref())); + args.limits.build_timeout = args.limits.build_timeout.or(self.build_timeout); + args.limits.build_timeout_multiplier = args.limits.build_timeout_multiplier.or(self.build_timeout_multiplier); + args.incremental = args.incremental.or(self.incremental); + args.measure.profile = args.measure.profile.take().or_else(|| self.profile.clone()); + args.measure.cargo_args.extend(self.cargo_args.iter().cloned()); + args.measure.cargo_test_args.extend(self.cargo_test_args.iter().cloned()); + args.measure.test_packages.extend(self.test_packages.iter().cloned()); + args.measure.test_workspace = args.measure.test_workspace || self.test_workspace.unwrap_or(false); + args.measure.whole_test_binaries = args.measure.whole_test_binaries || self.whole_test_binaries.unwrap_or(false); + args.measure.include_tests.extend(self.include_tests.iter().cloned()); + args.measure.exclude_tests.extend(self.exclude_tests.iter().cloned()); + args.no_baseline = args.no_baseline || self.no_baseline.unwrap_or(false); + args.no_confirm = args.no_confirm || self.no_confirm.unwrap_or(false); + args.artifact_dir = args.artifact_dir.take().or_else(|| self.artifact_dir.clone()); + args.html_external = args.html_external || self.reporters.html_external.unwrap_or(false); + + if !args.measure.test_packages.is_empty() && args.measure.test_workspace { + return Err(contradiction( + "test-packages", + !self.test_packages.is_empty(), + "test-workspace", + self.test_workspace == Some(true), + )); + } + + Ok(()) + } + + /// Folds the file's selection keys into `select` — the step `list`, `unsuppress`, and `hints` + /// each take before discovery, and that `run` and `suppress` reach through [`apply`](Self::apply). + /// + /// `explain` is deliberately not in that set: it resolves a named subject rather than a + /// selection, so it never calls this and the file's selection keys do not reach it. + /// + /// # Errors + /// + /// Returns a usage error if the merged settings contradict one another; see + /// [`validate_effective`](Self::validate_effective). + pub fn apply_selection(&self, select: &mut SelectArgs) -> Result<()> { + if select.mutators.is_none() + && let Some(selectors) = self.mutators.as_ref() + { + select.mutators = Some(selectors.join(",")); + } + + select.files.extend(self.files.iter().cloned()); + select.exclude_files.extend(self.exclude_files.iter().cloned()); + select.packages.extend(self.packages.iter().cloned()); + select.errors.extend(self.errors.iter().cloned()); + select.features.features.extend(self.features.iter().cloned()); + select.features.all_features = select.features.all_features || self.all_features.unwrap_or(false); + select.features.no_default_features = select.features.no_default_features || self.no_default_features.unwrap_or(false); + + // Merged one field at a time on purpose, so that a count in the file and an index on the + // command line make a whole shard between them — the split every CI matrix wants, since + // the width is shared and the index is not. That the pair must end up whole is checked + // afterwards, by `SelectArgs::shard`, on these effective values rather than on either + // source alone. + select.shard_count = select.shard_count.or(self.shard.count); + select.shard_index = select.shard_index.or(self.shard.index); + + self.validate_effective(select) + } + + /// Re-checks the mutually exclusive settings on the merged values. + /// + /// clap's `conflicts_with` constrains argument *occurrences*, so it sees only what was typed. + /// The file writes into the same structures afterwards and can therefore populate a field clap + /// has already decided must stay empty, and nothing looks again. The two pairs that reach a + /// decision by different routes are re-checked here, on the effective values, where the source + /// of each half no longer matters. + /// + /// `packages` beats `workspace` in `selected_packages`, so a file naming packages silently + /// overrules `--workspace` and mutates a fraction of what was asked for. + /// + /// # Errors + /// + /// Returns a usage error naming both settings and where each came from. + fn validate_effective(&self, select: &SelectArgs) -> Result<()> { + if !select.packages.is_empty() && select.workspace { + return Err(contradiction("packages", !self.packages.is_empty(), "workspace", false)); + } + + Ok(()) + } +} + +/// Builds the usage error for a pair of settings that cannot both apply, naming each one's source. +/// +/// Which source stated what is the whole content of the message: the same two settings from the +/// command line alone are caught by clap, so anyone reading this error has one of them in a file +/// they may have forgotten is there. +fn contradiction(first: &str, first_from_file: bool, second: &str, second_from_file: bool) -> Error { + let source = |from_file: bool| if from_file { RELATIVE_PATH } else { "the command line" }; + + error!( + "`{first}` from {} and `{second}` from {} cannot both apply.\n\ + Drop one of them, or state the one you want on the command line and remove the other from {RELATIVE_PATH}.", + source(first_from_file), + source(second_from_file) + ) + .usage() +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use tempfile::TempDir; + + use super::*; + + fn select_args(dir: &Utf8Path) -> SelectArgs { + SelectArgs { + dir: dir.to_path_buf(), + ..SelectArgs::default() + } + } + + /// `packages` in the file and `--workspace` on the command line both reach `selected_packages`, + /// where `packages` wins — so a committed file quietly reduces `--workspace` to a fraction of + /// the workspace and says nothing. clap cannot see it: `conflicts_with` constrains what was + /// typed, and the file is merged in afterwards. + #[test] + fn a_configured_package_list_contradicts_workspace_on_the_command_line() { + let dir = TempDir::new().expect("temp dir"); + let path = Utf8Path::from_path(dir.path()).expect("utf-8"); + + let config = Config::parse("packages = [\"a\"]\n").expect("a package list parses"); + let mut select = SelectArgs { + workspace: true, + ..select_args(path) + }; + + let failure = config.apply_selection(&mut select).expect_err("the pair cannot both apply"); + let text = failure.to_string(); + + assert!(text.contains("packages"), "{text}"); + assert!(text.contains("workspace"), "{text}"); + assert!(text.contains(RELATIVE_PATH), "{text}"); + assert!(text.contains("the command line"), "{text}"); + } + + /// The oracle half of the same defect, and the worse one: the file narrows which packages' tests + /// judge every mutant while the flag says to widen it, so mutants another package's tests would + /// kill are reported as survivors. + #[test] + fn a_configured_test_package_list_contradicts_test_workspace_on_the_command_line() { + let dir = TempDir::new().expect("temp dir"); + let path = Utf8Path::from_path(dir.path()).expect("utf-8"); + + let config = Config::parse("test-packages = [\"a\"]\n").expect("a test package list parses"); + let mut args = RunArgs { + select: select_args(path), + ..RunArgs::default() + }; + + args.measure.test_workspace = true; + + let failure = config.apply(&mut args).expect_err("the pair cannot both apply"); + let text = failure.to_string(); + + assert!(text.contains("test-packages"), "{text}"); + assert!(text.contains("test-workspace"), "{text}"); + } + + /// Neither pair is a contradiction when only one half is stated, which is the ordinary case and + /// must keep working. + #[test] + fn a_configured_list_on_its_own_is_not_a_contradiction() { + let dir = TempDir::new().expect("temp dir"); + let path = Utf8Path::from_path(dir.path()).expect("utf-8"); + + let config = Config::parse("packages = [\"a\"]\ntest-packages = [\"b\"]\n").expect("both lists parse"); + let mut args = RunArgs { + select: select_args(path), + ..RunArgs::default() + }; + + config.apply(&mut args).expect("one half of each pair is no contradiction"); + + assert_eq!(args.select.packages, vec!["a".to_owned()]); + assert_eq!(args.measure.test_packages, vec!["b".to_owned()]); + } + + /// Every `conflicts_with` in the CLI whose two ids are both config-reachable needs a post-merge + /// check, because the file can set one half after clap has stopped looking at the other. + /// + /// The pairs are listed rather than derived: clap does not expose its conflict graph, and the + /// list is short enough that keeping it beside the checks it justifies is the honest way to + /// notice a new one. `--features`/`--all-features` and the build-timeout pair are deliberately + /// absent — cargo lets one win, and `cargo_options` takes the tighter of the two. + #[test] + fn every_config_reachable_conflicting_pair_is_checked_after_the_merge() { + /// Each pair as the file spells its list half, plus the flag the command line sets. + type Pair = (&'static str, fn(&mut RunArgs)); + + let dir = TempDir::new().expect("temp dir"); + let path = Utf8Path::from_path(dir.path()).expect("utf-8"); + let pairs: [Pair; 2] = [ + ("packages", |args| args.select.workspace = true), + ("test-packages", |args| args.measure.test_workspace = true), + ]; + + for (key, raise) in pairs { + let config = Config::parse(&format!("{key} = [\"a\"]\n")).expect("the list parses"); + let mut args = RunArgs { + select: select_args(path), + ..RunArgs::default() + }; + + raise(&mut args); + + let failure = config.apply(&mut args).expect_err("the pair cannot both apply"); + + assert!(failure.to_string().contains(key), "{key}"); + } + } + + #[test] + fn no_config_wins_over_a_file_that_is_there() { + let dir = TempDir::new().expect("temp dir"); + let root = Utf8Path::from_path(dir.path()).expect("utf-8 path"); + + fs::write(root.join(RELATIVE_PATH), "jobs = 7\n").expect("write"); + + let mut select = select_args(root); + + select.config.no_config = true; + assert_eq!(Config::resolve(&select).expect("resolves").jobs, None); + + select.config.no_config = false; + assert_eq!(Config::resolve(&select).expect("resolves").jobs, Some(7)); + } + + #[test] + fn an_explicit_config_path_is_read_instead_of_the_default_one() { + let dir = TempDir::new().expect("temp dir"); + let root = Utf8Path::from_path(dir.path()).expect("utf-8 path"); + let elsewhere = root.join("elsewhere.toml"); + + fs::write(root.join(RELATIVE_PATH), "jobs = 7\n").expect("write"); + fs::write(&elsewhere, "jobs = 3\n").expect("write"); + + let mut select = select_args(root); + + select.config.path = Some(elsewhere); + assert_eq!(Config::resolve(&select).expect("resolves").jobs, Some(3)); + } + + #[test] + fn an_explicit_config_path_that_is_missing_is_an_error() { + // An absent default file is ordinary; an absent file the user named by hand is a typo, and + // silently running with no configuration would hide it. + let dir = TempDir::new().expect("temp dir"); + let root = Utf8Path::from_path(dir.path()).expect("utf-8 path"); + let mut select = select_args(root); + + select.config.path = Some(root.join("nope.toml")); + let _cause = Config::resolve(&select).unwrap_err(); + } + + #[test] + fn memory_sizes_in_the_file_are_parsed_and_merged_into_the_arguments() { + let config = Config::parse( + "memory = \"enforce\"\nmemory-headroom = \"256MiB\"\nmemory-limit = \"2GiB\"\nbaseline-memory-limit = \"4GiB\"\n", + ) + .expect("parses"); + + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.measure.memory, Some(crate::exec::MemoryControl::Enforce)); + assert_eq!(args.measure.memory_headroom, Some(256 * 1024 * 1024)); + assert_eq!(args.measure.memory_limit, Some(2 * 1024 * 1024 * 1024)); + assert_eq!(args.measure.baseline_memory_limit, Some(4 * 1024 * 1024 * 1024)); + } + + #[test] + fn command_line_memory_limits_override_a_configured_memory_mode() { + let config = Config::parse("memory = \"off\"\n").expect("parses"); + + let mut enforcing = RunArgs::default(); + enforcing.measure.memory_limit = Some(1024); + config.apply(&mut enforcing).expect("merges"); + + assert_eq!(enforcing.measure.memory, Some(crate::exec::MemoryControl::Enforce)); + + let mut measuring = RunArgs::default(); + measuring.measure.baseline_memory_limit = Some(1024); + config.apply(&mut measuring).expect("merges"); + + assert_eq!(measuring.measure.memory, Some(crate::exec::MemoryControl::Measure)); + } + + #[test] + fn an_explicit_command_line_memory_mode_overrides_a_size_flag_implication() { + let config = Config::parse("memory = \"measure\"\n").expect("parses"); + let mut args = RunArgs::default(); + + args.measure.memory = Some(crate::exec::MemoryControl::Off); + args.measure.memory_limit = Some(1024); + config.apply(&mut args).expect("merges"); + + assert_eq!(args.measure.memory, Some(crate::exec::MemoryControl::Off)); + } + + /// The file enables nextest when the command line does not, and yields when it does. + #[test] + fn nextest_in_the_file_is_merged_in_and_adds_to_the_command_line() { + let config = Config::parse("nextest = true\n").expect("parses"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert!(args.measure.nextest); + + let mut chosen = RunArgs::default(); + + chosen.measure.nextest = false; + config + .apply(&mut chosen) + .expect("the merged settings do not contradict one another"); + + assert!(chosen.measure.nextest); + } + + /// The module promises that anything expressible on the command line is expressible here, and + /// `--test-workspace` was the one flag that was not. With `deny_unknown_fields` the key was + /// rejected outright, so a project could not commit its whole-workspace oracle policy and had + /// to edit CI separately — which silently changes which tests judge each mutant. + #[test] + fn test_workspace_in_the_file_selects_the_same_oracle_as_the_flag() { + let config = Config::parse("test-workspace = true\n").expect("the documented equivalence must hold"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert!(args.measure.test_workspace); + } + + /// The file must not be able to switch the oracle off once the command line has asked for it. + #[test] + fn test_workspace_on_the_command_line_survives_a_file_that_does_not_set_it() { + let config = Config::parse("test-workspace = false\n").expect("parses"); + let mut args = RunArgs::default(); + + args.measure.test_workspace = true; + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert!(args.measure.test_workspace); + } + + #[test] + fn whole_test_binaries_in_the_file_selects_the_same_oracle_as_the_flag() { + let config = Config::parse("whole-test-binaries = true\n").expect("the documented equivalence must hold"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert!(args.measure.whole_test_binaries); + } + + #[test] + fn whole_test_binaries_on_the_command_line_survives_a_false_file_setting() { + let config = Config::parse("whole-test-binaries = false\n").expect("parses"); + let mut args = RunArgs::default(); + + args.measure.whole_test_binaries = true; + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert!(args.measure.whole_test_binaries); + } + + #[test] + fn a_memory_size_that_is_not_a_size_is_reported_rather_than_ignored() { + // A ceiling read as a handful of bytes would report every mutant as caught by tests that + // could never have started, which is a far more expensive failure than a rejected file. + let cause = Config::parse("memory-limit = \"lots\"\n").expect_err("must be rejected"); + + assert!(cause.contains("memory-limit"), "{cause}"); + } + + #[test] + fn an_empty_file_is_valid() { + let config = Config::parse("").expect("an empty file is a valid file"); + + assert!(config.mutators.is_none()); + assert!(config.files.is_empty()); + } + + #[test] + fn a_misspelled_key_is_an_error_rather_than_a_silent_no_op() { + // The whole point of `deny_unknown_fields`: a project that believes it has configured + // something and has not is in a worse position than one with no configuration at all. + let cause = Config::parse("exclude-file = [\"src/main.rs\"]\n").expect_err("must be rejected"); + + assert!(cause.contains("unknown field"), "{cause}"); + } + + #[test] + fn a_misspelled_key_in_a_table_is_also_an_error() { + let cause = Config::parse("[shard]\ncount = 4\nidx = 0\n").expect_err("must be rejected"); + + assert!(cause.contains("unknown field"), "{cause}"); + } + + #[test] + fn keys_are_spelled_in_kebab_case() { + let config = + Config::parse("exclude-files = [\"tests/**\"]\ntest-timeout-multiplier = 3.0\n").expect("kebab-case is the file's spelling"); + + assert_eq!(config.exclude_files, vec!["tests/**".to_owned()]); + assert_eq!(config.test_timeout_multiplier, Some(3.0)); + } + + #[test] + fn ops_are_joined_into_the_selector_list_the_flag_parses() { + // One selector per line, with room for a comment, is the reason this is a list. It has to + // arrive at exactly the same parser the flag uses, or the two spellings will drift. + let config = Config::parse("mutators = [\"@arithmetic\", \"!bitwise\"]\n").expect("parses"); + let mut select = SelectArgs::default(); + + config + .apply_selection(&mut select) + .expect("the merged settings do not contradict one another"); + + assert_eq!(select.mutators.as_deref(), Some("@arithmetic,!bitwise")); + } + + #[test] + fn the_command_line_wins_for_scalars() { + let config = Config::parse("mutators = [\"stmt\"]\nmin-score = 10.0\njobs = 1\n").expect("parses"); + let mut args = RunArgs { + select: SelectArgs { + mutators: Some("relational".to_owned()), + ..SelectArgs::default() + }, + min_score: Some(90.0), + ..RunArgs::default() + }; + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.select.mutators.as_deref(), Some("relational")); + assert_eq!(args.min_score, Some(90.0)); + + // A key the command line did not speak to still applies. + assert_eq!(args.measure.jobs, Some(1)); + } + + #[test] + fn lists_concatenate_rather_than_replace() { + // Replacing would mean that adding one exclusion on the command line silently drops every + // exclusion the project has agreed on, which is the opposite of what typing it means. + let config = Config::parse("exclude-files = [\"generated/**\"]\n").expect("parses"); + let mut args = RunArgs { + select: SelectArgs { + exclude_files: vec!["tests/**".to_owned()], + ..SelectArgs::default() + }, + ..RunArgs::default() + }; + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.select.exclude_files, vec!["tests/**".to_owned(), "generated/**".to_owned()]); + } + + #[test] + fn a_configured_flag_turns_on_and_the_command_line_cannot_turn_it_off() { + // Boolean flags have no "off" spelling on the command line, so the configured value can + // only ever add. This is worth a test because it is the one place the precedence rule + // above does not apply, and it is easy to "fix" into a bug. + let config = Config::parse("no-baseline = true\n").expect("parses"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert!(args.no_baseline); + } + + #[test] + fn artifact_directory_comes_from_the_file_when_the_command_line_is_silent() { + let config = Config::parse("artifact-dir = \"out\"\n").expect("parses"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.artifact_dir.as_deref(), Some(Utf8Path::new("out"))); + + args.artifact_dir = Some("cli-out".into()); + config.apply(&mut args).expect("command line wins"); + assert_eq!(args.artifact_dir.as_deref(), Some(Utf8Path::new("cli-out"))); + } + + #[test] + fn individual_report_destinations_are_not_configurable() { + for key in ["html", "json", "sarif", "advice"] { + let text = format!("[reporters]\n{key} = \"out/report\"\n"); + let failure = Config::parse(&text).expect_err("individual destinations are gone"); + + assert!(failure.contains("unknown field"), "{key}: {failure}"); + } + } + + #[test] + fn sharding_can_be_set_entirely_from_the_file() { + let config = Config::parse("[shard]\ncount = 30\nindex = 7\n").expect("parses"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.select.shard().expect("valid sharding"), Some((30, 7))); + } + + /// The split the file exists to support: every job in the matrix agrees on the width, which is + /// committed, and each supplies the slice it is, which is not. + #[test] + fn a_count_from_the_file_and_an_index_from_the_command_line_make_one_shard() { + let config = Config::parse("[shard]\ncount = 8\n").expect("parses"); + let mut args = RunArgs::default(); + + args.select.shard_index = Some(3); + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.select.shard_count, Some(8)); + assert_eq!(args.select.shard().expect("the pair is whole after merging"), Some((8, 3))); + } + + /// And the mirror of it, for a file that pins the index and a command line that says how many. + #[test] + fn an_index_from_the_file_and_a_count_from_the_command_line_make_one_shard() { + let config = Config::parse("[shard]\nindex = 0\n").expect("parses"); + let mut args = RunArgs::default(); + + args.select.shard_count = Some(2); + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.select.shard().expect("the pair is whole after merging"), Some((2, 0))); + } + + /// The command line wins over the file for both halves, as it does for every other scalar. + #[test] + fn a_shard_named_on_the_command_line_overrides_the_file() { + let config = Config::parse("[shard]\ncount = 8\nindex = 7\n").expect("parses"); + let mut args = RunArgs::default(); + + args.select.shard_count = Some(3); + args.select.shard_index = Some(1); + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + assert_eq!(args.select.shard().expect("valid sharding"), Some((3, 1))); + } + + /// A file that names half a shard and a command line that supplies nothing is not "no + /// sharding": it is a setting that cannot be honored, and running the whole population under + /// it would be eight times the work the job asked for, reported as a complete run. + #[test] + fn half_a_shard_in_the_file_and_nothing_on_the_command_line_is_refused() { + let config = Config::parse("[shard]\ncount = 8\n").expect("parses"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + let error = args.select.shard().expect_err("a count with no index is not a shard"); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("--shard-index"), "{error}"); + } + + /// A file that pins only the index is refused the same way, and says which half is missing. + #[test] + fn an_index_alone_in_the_file_is_refused() { + let config = Config::parse("[shard]\nindex = 2\n").expect("parses"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + let error = args.select.shard().expect_err("an index with no count is not a shard"); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("--shard-count"), "{error}"); + } + + /// The bounds are checked on the merged pair too, not only on what the command line carried. + #[test] + fn a_file_count_that_the_command_line_index_falls_outside_is_refused() { + let config = Config::parse("[shard]\ncount = 4\n").expect("parses"); + let mut args = RunArgs::default(); + + args.select.shard_index = Some(4); + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + let error = args.select.shard().expect_err("index 4 of 4 shards does not exist"); + + assert!(error.to_string().contains("out of range"), "{error}"); + } + + /// A zero count in the file is as impossible as one on the command line. + #[test] + fn a_zero_count_in_the_file_is_refused() { + let config = Config::parse("[shard]\ncount = 0\nindex = 0\n").expect("parses"); + let mut args = RunArgs::default(); + + config.apply(&mut args).expect("the merged settings do not contradict one another"); + + let error = args.select.shard().expect_err("zero shards is not a division"); + + assert!(error.to_string().contains("at least 1"), "{error}"); + } + + #[test] + fn a_missing_file_is_not_an_error() { + let dir = TempDir::new().expect("a temporary directory"); + let path = Utf8Path::from_path(dir.path()).expect("path is not UTF-8"); + + let config = Config::load(path).expect("an absent file is the common case"); + + assert!(config.mutators.is_none()); + } + + #[test] + fn a_present_file_is_read() { + let dir = TempDir::new().expect("a temporary directory"); + let path = Utf8Path::from_path(dir.path()).expect("path is not UTF-8"); + + fs::write(path.join(RELATIVE_PATH), "jobs = 3\n").expect("could not write the config"); + + let config = Config::load(path).expect("the file is valid"); + + assert_eq!(config.jobs, Some(3)); + } + + #[test] + fn a_malformed_file_is_a_usage_error_naming_the_path() { + let dir = TempDir::new().expect("a temporary directory"); + let path = Utf8Path::from_path(dir.path()).expect("path is not UTF-8"); + + fs::write(path.join(RELATIVE_PATH), "jobs = \n").expect("could not write the config"); + + let cause = Config::load(path).expect_err("a malformed file must stop the run"); + + assert!(cause.is_usage(), "{cause}"); + assert!(cause.to_string().contains("gamma.toml"), "{cause}"); + } + + #[test] + fn a_foreign_config_file_is_noticed_but_never_read() { + // Reading it would mean another tool's settings quietly changing which mutants are + // suppressed here. Noticing it is what lets the run say so out loud. + let dir = TempDir::new().expect("a temporary directory"); + let path = Utf8Path::from_path(dir.path()).expect("path is not UTF-8"); + + fs::create_dir_all(path.join(".cargo")).expect("could not create .cargo"); + fs::write(path.join(FOREIGN_PATH), "exclude_re = [\"impl Debug\"]\n").expect("could not write the foreign config"); + + assert!(Config::foreign_present(path)); + + let config = Config::load(path).expect("the foreign file must not be parsed as ours"); + + assert!(config.mutators.is_none()); + } + + #[test] + fn a_config_that_cannot_be_read_is_an_error_rather_than_the_defaults() { + // Only an absent file means "this project has no configuration". Anything else — a + // directory in its place, a permission problem — has to be reported, because silently + // falling back to the defaults would run with settings nobody chose. + let dir = TempDir::new().expect("a temporary directory"); + let path = Utf8Path::from_path(dir.path()).expect("path is not UTF-8"); + + fs::create_dir_all(path.join(RELATIVE_PATH)).expect("could not create a directory in the config's place"); + + let error = Config::load(path).expect_err("an unreadable config must not be treated as absent"); + + assert!(error.to_string().contains(RELATIVE_PATH), "{error}"); + } +} diff --git a/crates/cargo-gamma-lib/src/diag/bundle.rs b/crates/cargo-gamma-lib/src/diag/bundle.rs new file mode 100644 index 000000000..9e6927cb2 --- /dev/null +++ b/crates/cargo-gamma-lib/src/diag/bundle.rs @@ -0,0 +1,1191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; +use std::env::consts; + +use serde::{Deserialize, Serialize}; + +use crate::discover::Plan; +use crate::exec::Session; +use crate::model::{Mutant, Outcome, Summary}; + +/// The schema version of the bundle. +/// +/// The same discipline as [`crate::elements`]: a consumer reading a figure whose meaning has +/// silently changed is worse off than one that refuses to read it. Bump this whenever a field's +/// meaning changes or a field is removed; adding one does not need it. +const SCHEMA_VERSION: &str = "3"; + +/// How many rows the ranked tables keep, matching the prose dump. +const TOP: usize = 20; + +/// How much of the identifier hash is kept. +/// +/// Long enough that two packages in one workspace will not collide, short enough that the value +/// reads as an opaque label rather than as something to try to reverse. +const HASH_WIDTH: usize = 12; + +/// What to do with the names of packages, files and binaries. +/// +/// Timing data is entangled with things people cannot share: a path names employees and products, +/// a package name describes an unreleased codebase, and a test name can be as disclosing as the +/// code it covers. None of those are needed to read a cost profile — what is needed is to be able +/// to tell one row from another and to group rows that belong together, which a stable hash gives. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +pub enum Redaction { + /// Replace each identifier with a short stable hash of it. The default. + /// + /// Chosen over omitting because it costs the reader nothing — every question the bundle can + /// answer is about how the cost is distributed, not about what anything is called — and because + /// a hash still lets the person who sent it map a row back to their own tree. + #[default] + Hashed, + + /// Leave the identifiers as they are. + /// + /// For a tree whose names are already public, and for our own runs. Never the default: a user + /// who cannot tell what they are sending will send nothing. + Names, + + /// Drop the identifiers entirely. + /// + /// The rows survive, in order, without labels. For a tree where even the shape of the name + /// space is sensitive. + Omitted, +} + +impl Redaction { + /// Applies the policy to one identifier. + fn apply(self, name: &str) -> Option { + match self { + Self::Names => Some(name.to_owned()), + Self::Omitted => None, + Self::Hashed => { + let digest = blake3::hash(name.as_bytes()).to_hex(); + + Some(digest.get(..HASH_WIDTH).unwrap_or(&digest).to_owned()) + } + } + } +} + +/// Everything a run measured about itself, as a document someone else can read. +/// +/// **No source text, ever.** Not a replacement, not an original, not a line of context. The report +/// carries those because it is read against the tree it describes; this is read by someone who does +/// not have the tree and must not be given it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Bundle { + /// The schema version this document claims to conform to. + pub schema_version: String, + + /// What produced it. + pub tool: Tool, + + /// Which identifier policy was applied, so a reader knows what the labels are. + pub redaction: String, + + /// The machine it ran on, to whatever extent that is knowable and shareable. + pub host: Host, + + /// What the run was asked to do. + pub config: Config, + + /// What the run cost, and how well it used the machine. + pub run: Run, + + /// What was found, before anything was run. + pub population: Population, + + /// What every mutant came to. + pub outcomes: Outcomes, + + /// What the instrumented build cost, and what it withdrew. Absent when nothing was built. + #[serde(skip_serializing_if = "Option::is_none")] + pub build: Option, + + /// Where the run's time went, phase by phase. Absent when nothing was built, so no phase ran. + /// + /// The aggregates above say how much time the run spent; this says where. The copy and the + /// preflight are components of `build.elapsedMs`; the census and the sweep are components of + /// `run.testingMs`. Neither set sums to its aggregate — compiling sits between the copy and the + /// baseline, and bookkeeping between the census and the sweep — because the point is to see + /// which phase a slow run is slow in, not to reconcile the totals. Every phase that did not run + /// is absent rather than zero: `null` means it did not happen, `0` would claim it happened + /// instantly. + #[serde(skip_serializing_if = "Option::is_none")] + pub phases: Option, + + /// The distribution of mutant durations. Absent when nothing was run. + #[serde(skip_serializing_if = "Option::is_none")] + pub durations: Option, + + /// What each test binary cost, most expensive first. + pub binaries: Vec, + + /// What each mutator cost, most expensive first. + /// + /// Never redacted: a mutator name is ours, not the user's, and it is the single most useful + /// axis in the whole document. + pub mutators: Vec, + + /// What each package cost, most expensive first. + pub packages: Vec, +} + +/// What produced the bundle. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Tool { + /// Always `cargo-gamma`. + pub name: String, + + /// The version of the tool that ran. + pub version: String, +} + +/// The machine, to whatever extent it is knowable and shareable. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Host { + /// The target family the tool was built for. + pub os: String, + + /// The target architecture the tool was built for. + pub arch: String, + + /// The parallelism the machine reports as available to this process. + pub cores: usize, + + /// The compiler version string, which is the one host fact that changes a timing most. + #[serde(skip_serializing_if = "Option::is_none")] + pub toolchain: Option, + + /// Whether `RUSTFLAGS` was set, without saying to what. + /// + /// The value can name internal crates, private registries and feature codenames. That it was + /// set at all is the part that explains a build time. + pub rustflags_set: bool, +} + +/// What the run was asked to do. +/// +/// Flags rather than a command line: a command line carries paths, package names and filter +/// patterns, and reconstructing it is not what a cost profile needs. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Config { + /// The mutators that were selected, in registry order. + pub mutators: Vec, + + /// The job count the run was given. + pub jobs: usize, + + /// The silence a mutant was allowed before it was presumed hung, when that was enabled. + #[serde(skip_serializing_if = "Option::is_none")] + pub stall_ms: Option, + + /// Whether this run actually metered memory, which is not always what was configured. + #[serde(skip_serializing_if = "Option::is_none")] + pub metered: Option, + + /// Why memory went unbounded, when it was meant to be bounded and could not be. + #[serde(skip_serializing_if = "Option::is_none")] + pub unbounded: Option, + + /// The shard this run covered, as `index` and `count`, when it was sharded. + #[serde(skip_serializing_if = "Option::is_none")] + pub shard: Option<[u32; 2]>, +} + +/// What the run cost, and how well it used the machine. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Run { + /// Wall time from the first thing the command did to the last. + pub wall_ms: u64, + + /// The part that does not scale with the population: the instrumented build and the baseline. + pub fixed_ms: u64, + + /// The part that does: everything left after the fixed cost. + /// + /// The split is the first question to ask of a slow run, because the two halves want opposite + /// remedies — a faster machine against a smaller population. + pub testing_ms: u64, + + /// The summed time of every mutant that ran. + pub cpu_ms: u64, + + /// CPU over the testing window: how many workers the run really kept busy. + /// + /// A scheduler that is working lands within a fraction of the configured job count, and + /// everything short of that is time spent waiting rather than testing. + pub effective_jobs: f64, + + /// How large the scratch tree was at the end of the run. + /// + /// Only measured when `--diag` was given, because measuring it is a walk of every build + /// artifact the run produced; `null` means it was not asked for, not that it was empty. + #[serde(skip_serializing_if = "Option::is_none")] + pub scratch_bytes: Option, +} + +/// What was found, before anything was run. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Population { + /// How many files were parsed. + pub files: usize, + + /// How many mutants were generated. + pub mutants: usize, + + /// How many workspace packages were involved. + pub packages: usize, + + /// How many mutants a directive suppressed. + pub suppressed: usize, + + /// How many skip directives suppressed nothing. + pub idle_directives: usize, + + /// How many live mutants sharding excluded. + pub sharded_out: usize, + + /// How many an earlier report had already settled. + pub settled_out: usize, +} + +/// What every mutant came to. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Outcomes { + pub killed: u32, + pub survived: u32, + pub timeout: u32, + pub out_of_memory: u32, + pub flaky: u32, + pub unviable: u32, + pub ignored: u32, + pub uncovered: u32, + pub not_built: u32, + pub pending: u32, +} + +/// What the instrumented build cost, and what it withdrew. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Build { + /// How long the whole build took, across every round. + pub elapsed_ms: u64, + + /// How long the baseline suite took. + pub baseline_ms: u64, + + /// How many tests the baseline ran, when a harness announced a count. + #[serde(skip_serializing_if = "Option::is_none")] + pub tests: Option, + + /// How many mutants were withdrawn because they could not compile. + pub withdrawn: usize, + + /// Whether the build had to widen to the whole workspace. + pub widened: bool, + + /// What each round cost and withdrew, oldest first. + /// + /// The first round is what building this workspace costs at all; every round after it exists + /// only because some mutant did not compile, and its time is the price of that mutant. + pub rounds: Vec, + + /// Why the withdrawn mutants were withdrawn, densest pair first. + /// + /// Never redacted: a rustc error code and a mutator name are both ours, and together they are + /// the difference between a mutator that could be taught to look before it mutates and an + /// unavoidable cost of instrumenting the tree at all. + pub withdrawals: Vec, + + /// The largest peak memory any one test binary reached during the baseline. + #[serde(skip_serializing_if = "Option::is_none")] + pub baseline_peak_bytes: Option, + + /// What the stale build-ordering hints did, when the run had any. + /// + /// Omitted entirely when no hint was available, which is different from a run that had hints + /// and found them all wrong — the second is worth knowing about and the first is not. + #[serde(skip_serializing_if = "Option::is_none")] + pub ordering_hints: Option, +} + +/// What front-loading the mutants an earlier run could not compile actually bought. +/// +/// There is deliberately no "rounds saved" here, and its absence is the honest part. That figure is +/// the length of a convergence that never ran, over a mutant population the compiler was never +/// shown in that shape; anything printed for it would be a model of a counterfactual presented as a +/// measurement, and a diagnostic bundle that does that once cannot be trusted anywhere. +/// +/// What is measurable is the trade itself. `offered` is how many mutants the hints put in front of +/// the compiler early; `confirmed` is how many of those the compiler then refused, which is the +/// hints being right; `rounds` is the extra cargo invocations that cost. `confirmed` close to +/// `offered` is a hint set that is paying for its rounds, and `confirmed` near zero over several +/// rounds is one that is not — which is the question a reader actually has. +/// +/// None of it can move a verdict. A mutant named here is built and judged exactly as it would have +/// been without the hint; only the order it met the compiler in changed. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OrderingHints { + /// How many hinted mutants were put in front of the compiler in a probe round. + pub offered: usize, + + /// How many of those the compiler then refused to compile. + pub confirmed: usize, + + /// How many extra build rounds the probes cost. + pub rounds: u32, +} + +/// One round of the instrumented build. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Round { + pub elapsed_ms: u64, + pub withdrew: usize, +} + +/// One group of withdrawn mutants. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Withdrawal { + /// The rustc error code, or empty when the diagnostic carried none. + pub code: String, + + /// The mutator whose mutants drew it, or empty when it could not be attributed. + pub mutator: String, + + /// How many mutants this pair accounts for. + pub mutants: usize, +} + +/// Where the run's time went, phase by phase. +/// +/// The copy and the preflight are components of `build.elapsedMs`; the baseline is `build.baselineMs` +/// restated so the fixed cost reads as a sequence rather than two unrelated numbers; the census and +/// the sweep are components of `run.testingMs`. The value of the split is that whether the per-test +/// census pays for itself — its cost against the launches it saves the sweep — can be read off one +/// run, where before it was folded invisibly inside the build. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Phases { + /// What duplicating the workspace into the scratch tree cost. Part of `build.elapsedMs`. + pub copy: Phase, + + /// What proving the tree compiles at all cost, before any mutant was staged. Part of + /// `build.elapsedMs`. + pub preflight: Phase, + + /// What the baseline suite cost. The same figure as `build.baselineMs`. + pub baseline: Phase, + + /// What the per-test census cost and covered. Absent when `--whole-test-binaries` disabled it. + #[serde(skip_serializing_if = "Option::is_none")] + pub census: Option, + + /// What the sweep cost and launched. Absent when nothing was swept. + #[serde(skip_serializing_if = "Option::is_none")] + pub sweep: Option, +} + +/// What one phase cost, when the cost is the only thing there is to say about it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Phase { + pub elapsed_ms: u64, +} + +/// What the per-test census cost and covered. +/// +/// The census spends one subprocess per test per binary; `walked` is exactly that count, and it is +/// the figure the census's cost has to be weighed against the sweep's launch count to know whether +/// the trade was positive. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CensusPhase { + pub elapsed_ms: u64, + + /// How many test executions the census walked, one subprocess apiece. + pub walked: usize, + + /// How many test binaries the census examined. + pub binaries: usize, +} + +/// What the sweep cost and how it spent its launches. +/// +/// `launches` is what turns the cost model's `build + Σ(launch + prefix)` from a formula into a +/// measurement; `probes` is what says whether the killer hints and the census are earning their +/// keep, since a probe is a launch the run only made because a hint pointed at it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SweepPhase { + pub elapsed_ms: u64, + + /// How many test-binary subprocesses the sweep launched in total. + pub launches: usize, + + /// How many of those launches were hint-directed probes. + pub probes: usize, +} + +/// The distribution of mutant durations. +/// +/// Percentiles rather than every duration: the shape is what a cost profile needs, and a list one +/// entry per mutant would be both enormous and a fingerprint of the tree. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Durations { + pub evaluated: usize, + pub min_ms: u64, + pub p50_ms: u64, + pub p90_ms: u64, + pub p99_ms: u64, + pub max_ms: u64, +} + +/// What one test binary cost the run. +/// +/// A binary's baseline is charged to every mutant that can reach it, so a single slow one is +/// multiplied by the population and is the most leveraged thing in a run. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Binary { + /// The package, subject to the redaction policy. + #[serde(skip_serializing_if = "Option::is_none")] + pub package: Option, + + /// The binary's target name, subject to the redaction policy. + #[serde(skip_serializing_if = "Option::is_none")] + pub target: Option, + + pub baseline_ms: u64, + /// `None` when no cutoff was calibrated, which is what a run with no baseline leaves. + #[serde(skip_serializing_if = "Option::is_none")] + pub budget_ms: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub peak_bytes: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub ceiling_bytes: Option, +} + +/// What one group of mutants cost. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Breakdown { + /// What the group is, subject to the redaction policy where the name is the user's. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + + pub mutants: usize, + pub cpu_ms: u64, + pub survivors: usize, + pub unviable: usize, +} + +/// Everything the bundle needs that is not on the plan or the session. +#[derive(Debug, Clone)] +pub struct Context<'a> { + /// The parallelism the host makes available to this process. + pub cores: usize, + + /// The job count the run was given. + pub jobs: usize, + + /// Wall time for the whole command. + pub wall: Duration, + + /// The mutators that were selected, in registry order. + pub mutators: Vec<&'static str>, + + /// The shard this run covered, as `(count, index)`, when it was sharded. + pub shard: Option<(u32, u32)>, + + /// How large the scratch tree was, when it was worth the walk to find out. + pub scratch_bytes: Option, + + /// What to do with the identifiers. + pub redaction: Redaction, + + /// The tool's version. + pub version: &'a str, +} + +/// Assembles the bundle. +/// +/// `session` is absent when nothing was live, so nothing was built or measured. The population is +/// still worth describing: a run that found no work is exactly the kind that wants explaining. +#[must_use] +pub fn bundle(plan: &Plan, session: Option<&Session>, context: &Context<'_>) -> Bundle { + let summary = Summary::of(&plan.mutants); + let fixed = session.map_or(Duration::ZERO, |session| session.build + session.baseline_wall); + let testing = context.wall.saturating_sub(fixed); + let cpu: Duration = plan.mutants.iter().map(|mutant| Duration::from_millis(mutant.elapsed_ms)).sum(); + let wall_ms = millis(context.wall); + let fixed_ms = millis(fixed); + let testing_ms = wall_ms.saturating_sub(fixed_ms); + + Bundle { + schema_version: SCHEMA_VERSION.to_owned(), + tool: Tool { + name: "cargo-gamma".to_owned(), + version: context.version.to_owned(), + }, + redaction: match context.redaction { + Redaction::Hashed => "hashed", + Redaction::Names => "names", + Redaction::Omitted => "omitted", + } + .to_owned(), + host: Host { + os: consts::OS.to_owned(), + arch: consts::ARCH.to_owned(), + cores: context.cores, + toolchain: redact_toolchain(crate::discover::toolchain(), context.redaction), + rustflags_set: crate::discover::rustflags().is_some(), + }, + config: Config { + mutators: context.mutators.iter().map(|name| (*name).to_owned()).collect(), + jobs: context.jobs, + stall_ms: session.and_then(|session| session.stall).map(millis), + metered: session.map(|session| session.metered), + unbounded: session.and_then(|session| session.unbounded.clone()), + shard: context.shard.map(|(count, index)| [index, count]), + }, + run: Run { + wall_ms, + fixed_ms, + testing_ms, + cpu_ms: millis(cpu), + effective_jobs: effective(cpu, testing), + scratch_bytes: context.scratch_bytes, + }, + population: Population { + files: plan.files.len(), + mutants: plan.mutants.len(), + packages: plan.reach.len(), + suppressed: plan.suppressed, + idle_directives: plan.idle.len(), + sharded_out: plan.sharded_out, + settled_out: plan.settled_out, + }, + outcomes: Outcomes { + killed: summary.killed, + survived: summary.survived, + timeout: summary.timeout, + out_of_memory: summary.out_of_memory, + flaky: summary.flaky, + unviable: summary.unviable, + ignored: summary.ignored, + uncovered: summary.uncovered, + not_built: summary.not_built, + pending: summary.pending, + }, + build: session.map(build_of), + phases: session.map(phases_of), + durations: durations_of(&plan.mutants), + binaries: session.map(|session| binaries_of(session, context.redaction)).unwrap_or_default(), + mutators: breakdown(&plan.mutants, Redaction::Names, |mutant| mutant.mutator.to_string()), + packages: breakdown(&plan.mutants, context.redaction, |mutant| mutant.package.to_string()), + } +} + +/// Applies the bundle's identifier policy to environment-derived tool program names. +/// +/// Version output remains useful and does not contain the program paths used to invoke the tools. +fn redact_toolchain(toolchain: Option, redaction: Redaction) -> Option { + let toolchain = toolchain?; + + Some( + toolchain + .lines() + .filter_map(|line| { + let (key, value) = ["rustc=", "cargo=", "rustc_wrapper=", "rustc_workspace_wrapper="] + .into_iter() + .find_map(|key| line.strip_prefix(key).map(|value| (key, value)))?; + + match redaction { + Redaction::Names => Some(line.to_owned()), + Redaction::Hashed => redaction.apply(value).map(|value| format!("{key}{value}")), + Redaction::Omitted => None, + } + }) + .chain( + toolchain + .lines() + .filter(|line| { + !["rustc=", "cargo=", "rustc_wrapper=", "rustc_workspace_wrapper="] + .into_iter() + .any(|key| line.starts_with(key)) + }) + .map(str::to_owned), + ) + .collect::>() + .join("\n"), + ) +} + +/// Serializes the bundle as pretty-printed JSON. +/// +/// Pretty rather than compact because the first thing anyone should do with it is read it, and a +/// file nobody can read before sending is a file nobody sends. +/// +/// # Errors +/// +/// Returns an error if the bundle cannot be serialized. +pub fn to_json(bundle: &Bundle) -> crate::Result { + serde_json::to_string_pretty(bundle) + .map_err(|cause| crate::error::error!("could not serialize the diagnostics bundle").caused_by(cause)) +} + +/// What the instrumented build cost. +fn build_of(session: &Session) -> Build { + Build { + elapsed_ms: millis(session.build), + baseline_ms: millis(session.baseline_wall), + tests: session.tests, + withdrawn: session.withdrawn, + widened: session.widened, + rounds: session + .rounds_taken + .iter() + .map(|round| Round { + elapsed_ms: millis(round.elapsed), + withdrew: round.withdrew, + }) + .collect(), + withdrawals: session + .census + .iter() + .map(|entry| Withdrawal { + code: entry.code.clone(), + mutator: entry.mutator.clone(), + mutants: entry.mutants, + }) + .collect(), + baseline_peak_bytes: session.peak, + + // A run with no hint at all reports nothing rather than three zeros: zeros would read as + // "the hints found nothing", which is a claim about the hints, and there were none. + ordering_hints: (session.ordering.rounds > 0 || session.ordering.offered > 0).then_some(OrderingHints { + offered: session.ordering.offered, + confirmed: session.ordering.confirmed, + rounds: session.ordering.rounds, + }), + } +} + +/// Where the run's time went, phase by phase. +fn phases_of(session: &Session) -> Phases { + Phases { + copy: Phase { + elapsed_ms: millis(session.phases.copy), + }, + preflight: Phase { + elapsed_ms: millis(session.phases.preflight), + }, + baseline: Phase { + elapsed_ms: millis(session.baseline_wall), + }, + census: session.phases.census.as_ref().map(|census| CensusPhase { + elapsed_ms: millis(census.elapsed), + walked: census.walked, + binaries: census.binaries, + }), + sweep: session.phases.sweep.as_ref().map(|sweep| SweepPhase { + elapsed_ms: millis(sweep.elapsed), + launches: sweep.launches, + probes: sweep.probes, + }), + } +} + +/// The test binaries, most expensive baseline first. +fn binaries_of(session: &Session, redaction: Redaction) -> Vec { + let mut binaries: Vec<&crate::exec::TestBinary> = session.binaries.iter().collect(); + + binaries.sort_by_key(|binary| core::cmp::Reverse(binary.baseline)); + + binaries + .into_iter() + .take(TOP) + .map(|binary| Binary { + package: redaction.apply(&binary.package), + target: redaction.apply(&binary.target), + baseline_ms: millis(binary.baseline), + budget_ms: binary.budget.map(millis), + peak_bytes: binary.peak, + ceiling_bytes: binary.memory, + }) + .collect() +} + +/// The duration distribution, or `None` when nothing ran. +fn durations_of(mutants: &[Mutant]) -> Option { + let mut spent: Vec = mutants + .iter() + .map(|mutant| mutant.elapsed_ms) + .filter(|elapsed| *elapsed > 0) + .collect(); + + if spent.is_empty() { + return None; + } + + spent.sort_unstable(); + + Some(Durations { + evaluated: spent.len(), + min_ms: spent.first().copied().unwrap_or(0), + p50_ms: percentile(&spent, 0.50), + p90_ms: percentile(&spent, 0.90), + p99_ms: percentile(&spent, 0.99), + max_ms: spent.last().copied().unwrap_or(0), + }) +} + +/// One ranked breakdown of the population, most expensive first. +fn breakdown(mutants: &[Mutant], redaction: Redaction, key: impl Fn(&Mutant) -> String) -> Vec { + let mut buckets: crate::HashMap = crate::HashMap::default(); + + for mutant in mutants { + let entry = buckets.entry(key(mutant)).or_insert_with(|| Breakdown { + name: None, + mutants: 0, + cpu_ms: 0, + survivors: 0, + unviable: 0, + }); + + entry.mutants += 1; + entry.cpu_ms = entry.cpu_ms.saturating_add(mutant.elapsed_ms); + + match mutant.outcome { + Outcome::Survived => entry.survivors += 1, + Outcome::CompileError => entry.unviable += 1, + _other => {} + } + } + + let mut rows: Vec<(String, Breakdown)> = buckets.into_iter().collect(); + + rows.sort_by(|(left_name, left), (right_name, right)| right.cpu_ms.cmp(&left.cpu_ms).then_with(|| left_name.cmp(right_name))); + + rows.into_iter() + .take(TOP) + .map(|(name, row)| Breakdown { + name: redaction.apply(&name), + ..row + }) + .collect() +} + +/// CPU over the testing window, to one decimal place. +fn effective(cpu: Duration, testing: Duration) -> f64 { + if testing.is_zero() { + return 0.0; + } + + (cpu.as_secs_f64() / testing.as_secs_f64() * 10.0).round() / 10.0 +} + +/// The value at a percentile of an ascending list. +fn percentile(ascending: &[u64], fraction: f64) -> u64 { + if ascending.is_empty() { + return 0; + } + + #[expect(clippy::cast_precision_loss, reason = "a population that large has other problems")] + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "a fraction of a non-negative length" + )] + let index = ((ascending.len() as f64 - 1.0) * fraction).round() as usize; + + ascending.get(index).copied().unwrap_or(0) +} + +/// A duration in whole milliseconds, which is the resolution everything else here is measured at. +fn millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + use crate::fixtures; + + fn context() -> Context<'static> { + Context { + cores: 3, + jobs: 4, + wall: Duration::from_secs(100), + mutators: vec!["arith.add_to_sub"], + shard: None, + scratch_bytes: None, + redaction: Redaction::default(), + version: "0.0.0", + } + } + + fn plan() -> Plan { + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: "/work/subject".into(), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + } + } + + /// A run that built something, so the bundle has a session and therefore phase timings. The + /// caller supplies the phase profile so a test can say what did and did not run. + fn session_with(phases: crate::exec::Phases) -> Session { + Session { + baseline: Duration::from_millis(1500), + baseline_wall: Duration::from_millis(500), + tests: Some(12), + quiet: Duration::ZERO, + stall: None, + build: Duration::from_secs(7), + peak: None, + metered: false, + unbounded: None, + withdrawn: 0, + census: Vec::new(), + rounds: 1, + rounds_taken: Vec::new(), + binaries: Vec::new(), + scratch: "/work/scratch".into(), + filtered: 0, + widened: false, + ordering: crate::exec::OrderingHints::default(), + phases, + } + } + + /// A phase that ran is timed; one that did not is absent rather than zero, because a zero would + /// read as a phase that happened instantly, and the whole question is whether it happened. + #[test] + fn a_run_that_built_something_carries_its_phase_timings() { + let phases = crate::exec::Phases { + copy: Duration::from_millis(12), + preflight: Duration::from_millis(340), + census: None, + sweep: Some(crate::exec::SweepCost { + elapsed: Duration::from_secs(42), + launches: 47, + probes: 12, + }), + }; + + let built = bundle(&plan(), Some(&session_with(phases)), &context()); + let phases = built.phases.expect("a built run has phase timings"); + + assert_eq!(phases.copy.elapsed_ms, 12); + assert_eq!(phases.preflight.elapsed_ms, 340); + assert_eq!(phases.baseline.elapsed_ms, 500); + assert!(phases.census.is_none(), "no census ran, so the census phase is absent"); + + let sweep = phases.sweep.expect("the sweep ran"); + + assert_eq!(sweep.elapsed_ms, 42_000); + assert_eq!(sweep.launches, 47); + assert_eq!(sweep.probes, 12); + } + + /// Nothing was built, so no phase ran, so there is nothing to profile. + #[test] + fn a_run_that_built_nothing_has_no_phases_at_all() { + assert!(bundle(&plan(), None, &context()).phases.is_none()); + } + + /// The census is the whole subject here, and its figures must appear exactly when it ran. + #[test] + fn the_census_phase_is_present_only_when_a_census_ran() { + let without = bundle(&plan(), Some(&session_with(crate::exec::Phases::default())), &context()); + + assert!(without.phases.expect("phases").census.is_none(), "no census means no census phase"); + + let phases = crate::exec::Phases { + census: Some(crate::exec::CensusCost { + elapsed: Duration::from_secs(8), + walked: 1_681, + binaries: 30, + }), + ..crate::exec::Phases::default() + }; + + let census = bundle(&plan(), Some(&session_with(phases)), &context()) + .phases + .expect("phases") + .census + .expect("a census ran"); + + assert_eq!(census.elapsed_ms, 8_000); + assert_eq!(census.walked, 1_681); + assert_eq!(census.binaries, 30); + } + + /// The one relationship the bundle guarantees by construction: the baseline restated in the + /// phase profile is the very figure the build reports, because both read `session.baseline_wall`. + /// The copy and preflight are subsets of the build and the census and sweep of the testing + /// window, but those are guaranteed by where the clocks sit in a real run, not by this code, so + /// they are not asserted here. + #[test] + fn the_baseline_phase_restates_the_build_baseline_exactly() { + let built = bundle(&plan(), Some(&session_with(crate::exec::Phases::default())), &context()); + + let baseline_phase = built.phases.expect("phases").baseline.elapsed_ms; + let build_baseline = built.build.expect("build").baseline_ms; + + assert_eq!(baseline_phase, build_baseline); + } + + /// The document is read by a stranger, so the shape has to be exactly what old consumers expect: + /// camelCase names, and a phase that did not run omitted rather than serialized as null. + #[test] + fn the_serialized_phases_use_camel_case_and_omit_the_phases_that_did_not_run() { + let phases = crate::exec::Phases { + copy: Duration::from_millis(1), + preflight: Duration::from_millis(2), + census: None, + sweep: Some(crate::exec::SweepCost { + elapsed: Duration::from_millis(3), + launches: 4, + probes: 5, + }), + }; + + let json = to_json(&bundle(&plan(), Some(&session_with(phases)), &context())).expect("json"); + + assert!(json.contains("\"phases\""), "{json}"); + assert!(json.contains("\"elapsedMs\""), "{json}"); + assert!(json.contains("\"launches\""), "{json}"); + assert!(json.contains("\"probes\""), "{json}"); + + // The census did not run, so it must not appear as a key at all. + assert!(!json.contains("\"census\""), "an absent census must be omitted, not null: {json}"); + assert!(!json.contains("\"walked\""), "{json}"); + } + + /// The whole point of the file. Anything a reader has to take on trust before attaching it to a + /// public issue is a reason not to attach it. + #[test] + fn the_bundle_carries_no_absolute_path_and_no_source_text() { + let json = to_json(&bundle(&plan(), None, &context())).expect("json"); + + assert!(!json.contains("/work/subject"), "{json}"); + assert!(!json.contains("projectRoot"), "{json}"); + } + + #[test] + fn hashing_is_the_default_and_hides_the_name() { + assert_eq!(Redaction::default(), Redaction::Hashed); + + let hashed = Redaction::Hashed.apply("secret-product").expect("hashed"); + + assert_ne!(hashed, "secret-product"); + assert_eq!(hashed.len(), HASH_WIDTH); + } + + /// A hash nobody can group by is no better than omitting the name. + #[test] + fn hashing_the_same_name_twice_gives_the_same_label() { + assert_eq!(Redaction::Hashed.apply("subject"), Redaction::Hashed.apply("subject")); + assert_ne!(Redaction::Hashed.apply("subject"), Redaction::Hashed.apply("other")); + } + + #[test] + fn omitting_leaves_no_label_at_all() { + assert_eq!(Redaction::Omitted.apply("secret-product"), None); + assert_eq!(Redaction::Names.apply("secret-product").as_deref(), Some("secret-product")); + } + + #[test] + fn the_bundle_says_which_schema_it_is() { + assert_eq!(bundle(&plan(), None, &context()).schema_version, "3"); + } + + /// The first question to ask of a slow run, so the split has to be right rather than plausible. + #[test] + fn the_fixed_and_testing_split_adds_up_to_the_wall_time() { + let run = bundle(&plan(), None, &context()).run; + + assert_eq!(run.wall_ms, 100_000); + assert_eq!(run.fixed_ms + run.testing_ms, run.wall_ms); + } + + #[test] + fn fractional_milliseconds_keep_the_displayed_duration_split_consistent() { + let mut session = session_with(crate::exec::Phases::default()); + session.build = Duration::from_micros(4_400); + session.baseline_wall = Duration::from_micros(2_500); + let context = Context { + wall: Duration::from_micros(10_100), + ..context() + }; + + let run = bundle(&plan(), Some(&session), &context).run; + + assert_eq!((run.wall_ms, run.fixed_ms, run.testing_ms), (10, 6, 4)); + assert_eq!(run.fixed_ms + run.testing_ms, run.wall_ms); + } + + #[test] + fn redacted_toolchains_hide_environment_derived_program_paths() { + let private = "/opt/acme/private-wrapper"; + let toolchain = Some(format!( + "rustc=/toolchains/rustc\ncargo=/toolchains/cargo\nrustc_wrapper=\nrustc_workspace_wrapper={private}\nrustc 1.90.0\ncargo 1.90.0" + )); + + for redaction in [Redaction::Hashed, Redaction::Omitted] { + let redacted = redact_toolchain(toolchain.clone(), redaction).expect("toolchain"); + + assert!(!redacted.contains(private), "{redaction:?}: {redacted}"); + assert!(redacted.contains("rustc 1.90.0"), "{redaction:?}: {redacted}"); + } + } + + #[test] + fn serialized_redacted_bundles_hide_workspace_wrapper_environment_paths() { + const CHILD: &str = "CARGO_GAMMA_DIAG_REDACTION_CHILD"; + const PRIVATE: &str = "/opt/acme/private-wrapper"; + + if std::env::var_os(CHILD).is_some() { + for redaction in [Redaction::Hashed, Redaction::Omitted] { + let context = Context { redaction, ..context() }; + let json = to_json(&bundle(&plan(), None, &context)).expect("serialized bundle"); + + assert!(!json.contains(PRIVATE), "{redaction:?}: {json}"); + } + + return; + } + + let output = std::process::Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--exact", + "diag::bundle::tests::serialized_redacted_bundles_hide_workspace_wrapper_environment_paths", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RUSTC_WORKSPACE_WRAPPER", PRIVATE) + .output() + .expect("diagnostic child process"); + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn concurrent_baselines_use_wall_time_and_keep_cores_distinct_from_jobs() { + let mut session = session_with(crate::exec::Phases::default()); + session.build = Duration::from_secs(7); + session.baseline = Duration::from_secs(5); + session.baseline_wall = Duration::from_secs(2); + + let context = Context { + cores: 3, + jobs: 7, + wall: Duration::from_secs(20), + ..context() + }; + let bundle = bundle(&plan(), Some(&session), &context); + + assert_eq!( + (bundle.run.wall_ms, bundle.run.fixed_ms, bundle.run.testing_ms), + (20_000, 9_000, 11_000) + ); + assert_eq!(bundle.build.expect("build").baseline_ms, 2_000); + assert_eq!(bundle.phases.expect("phases").baseline.elapsed_ms, 2_000); + assert_eq!(bundle.host.cores, 3); + assert_eq!(bundle.config.jobs, 7); + } + + #[test] + fn effective_jobs_is_cpu_over_the_testing_window() { + assert!((effective(Duration::from_secs(30), Duration::from_secs(10)) - 3.0).abs() < f64::EPSILON); + assert!(effective(Duration::from_secs(1), Duration::ZERO).abs() < f64::EPSILON); + } + + #[test] + fn percentiles_come_from_the_ascending_list() { + let spent = [1_u64, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + assert_eq!(percentile(&spent, 0.50), 6); + assert_eq!(percentile(&spent, 0.90), 9); + assert_eq!(percentile(&spent, 0.0), 1); + assert_eq!(percentile(&[], 0.5), 0); + } + + #[test] + fn a_run_that_measured_nothing_has_no_duration_distribution() { + assert!(durations_of(&[]).is_none()); + } + + /// A mutator name is ours, and it is the most useful axis in the document; hashing it would + /// leave the bundle unable to answer the question it is most often opened for. + #[test] + fn mutator_names_survive_redaction_while_package_names_do_not() { + let mut plan = plan(); + + plan.mutants = vec![mutant("subject", "arith.add_to_sub")]; + + let built = bundle(&plan, None, &context()); + + assert_eq!(built.mutators[0].name.as_deref(), Some("arith.add_to_sub")); + assert_ne!(built.packages[0].name.as_deref(), Some("subject")); + assert!(built.packages[0].name.is_some(), "the row still has to be groupable"); + } + + #[test] + fn the_breakdown_ranks_the_most_expensive_group_first() { + let mut plan = plan(); + let mut cheap = mutant("cheap", "arith.add_to_sub"); + let mut dear = mutant("dear", "relational.lt_to_le"); + + cheap.elapsed_ms = 10; + dear.elapsed_ms = 900; + plan.mutants = vec![cheap, dear]; + + let built = bundle(&plan, None, &context()); + + assert_eq!(built.mutators[0].name.as_deref(), Some("relational.lt_to_le")); + assert_eq!(built.mutators[0].cpu_ms, 900); + } + + fn mutant(package: &str, mutator: &str) -> Mutant { + Mutant { + id: "id".to_owned().into(), + package: package.to_owned().into(), + mutator: mutator.to_owned().into(), + ..fixtures::mutant() + } + } +} diff --git a/crates/cargo-gamma-lib/src/diag/mod.rs b/crates/cargo-gamma-lib/src/diag/mod.rs new file mode 100644 index 000000000..6752ca21b --- /dev/null +++ b/crates/cargo-gamma-lib/src/diag/mod.rs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! What a run cost, said two ways. +//! +//! This is not [`crate::advise`]. Advice is written for someone whose run was slow and who wants +//! to know what to do about it, so it withholds anything they cannot act on. These withhold +//! nothing. +//! +//! [`render`] is the prose dump behind `--diag`: unstable, undocumented, and written for people +//! working on the tool, so that a change to the scheduler, the build sequencing or the mutator +//! catalog can be judged against numbers rather than against how the run felt. It goes to the +//! diagnostic stream, so it composes with piping the results somewhere. +//! +//! [`bundle`] is the same measurements as a versioned document, written for someone else to read. +//! A user reporting that a run was slow has no way to show us why and we have no way to ask for it, +//! so what arrives in an issue is a screenshot or a paraphrase. The bundle is the thing to attach — +//! which is why it carries no source text and hashes the identifiers by default. + +mod bundle; +mod render; + +pub use bundle::{Bundle, Context, Redaction, bundle, to_json}; +pub use render::render; diff --git a/crates/cargo-gamma-lib/src/diag/render.rs b/crates/cargo-gamma-lib/src/diag/render.rs new file mode 100644 index 000000000..5be975782 --- /dev/null +++ b/crates/cargo-gamma-lib/src/diag/render.rs @@ -0,0 +1,1036 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The prose dump. + +use core::cmp::Reverse; +use core::fmt::Write as _; +use core::time::Duration; + +use crate::HashMap; +use crate::advise::human; +use crate::discover::Plan; +use crate::exec::{Session, TestBinary}; +use crate::model::{Mutant, Outcome, Summary}; +use crate::report::quantity; + +/// How many rows the "worst offender" tables keep. +/// +/// Long enough to show a pattern rather than a single outlier, short enough that the whole dump +/// still fits on a screen next to the run that produced it. +const TOP: usize = 10; + +/// One row of a per-something breakdown. +#[derive(Debug, Default)] +struct Bucket { + mutants: usize, + cpu: Duration, + survivors: usize, + unviable: usize, +} + +impl Bucket { + /// Folds one mutant into the tally. + fn absorb(&mut self, mutant: &Mutant) { + self.mutants += 1; + self.cpu += Duration::from_millis(mutant.elapsed_ms); + + match mutant.outcome { + Outcome::Survived => self.survivors += 1, + Outcome::CompileError => self.unviable += 1, + _other => {} + } + } +} + +/// Renders the whole dump. +/// +/// `session` is absent when nothing was live, so nothing was built or measured; the population is +/// still worth reporting, because a run that found no work is exactly the kind that wants +/// explaining. +#[must_use] +pub fn render(plan: &Plan, session: Option<&Session>, jobs: usize, wall: Duration) -> String { + let summary = Summary::of(&plan.mutants); + let mut text = String::new(); + + let _ = writeln!(text, "── diag ──────────────────────────────────────────────"); + + // The build and the baseline are the run's fixed cost, so what is left is the only part that + // scales with the population and the only part worth judging the scheduler on. + let fixed = session.map_or(Duration::ZERO, |session| session.build + session.baseline_wall); + let testing = wall.saturating_sub(fixed); + + let _ = writeln!( + text, + "run wall {}, of which {} testing, {} jobs", + human(wall), + human(testing), + jobs + ); + + let _ = writeln!(text, " root {}", plan.root); + + let _ = writeln!( + text, + "discover {}, {}, {}, {}", + quantity(plan.files.len(), "file"), + quantity(plan.mutants.len(), "mutant"), + quantity(plan.reach.len(), "package"), + quantity(plan.reach.values().map(crate::HashSet::len).sum::(), "reach edge") + ); + + let _ = writeln!( + text, + " withheld: {} suppressed, {} out of shard, {} already settled", + plan.suppressed, plan.sharded_out, plan.settled_out + ); + + // Every outcome is named, including the ones that are zero: this is the dump a user sends when + // they cannot explain a run, and it sits beside a JSON bundle that carries all ten. A line + // listing a hand-picked subset makes the two halves of the same dump disagree, and leaves a + // reader unable to tell a category nobody printed from mutants that went missing. + let counts: Vec = Outcome::ALL + .iter() + .map(|outcome| format!("{} {outcome}", summary.count(*outcome))) + .collect(); + + let _ = writeln!(text, "outcomes {}", counts.join(", ")); + + if let Some(session) = session { + write_session(&mut text, session); + } + + write_throughput(&mut text, &plan.mutants, session, jobs, testing); + write_slowest(&mut text, &plan.mutants); + write_breakdown(&mut text, "mutator", &group(&plan.mutants, |mutant| mutant.mutator.to_string())); + write_breakdown(&mut text, "package", &group(&plan.mutants, |mutant| mutant.package.to_string())); + write_breakdown(&mut text, "file", &group(&plan.mutants, |mutant| mutant.file.to_string())); + + if let Some(session) = session { + write_binaries(&mut text, session); + } + + text +} + +/// Reports what the fixed cost of the run was, and what it bought. +fn write_session(text: &mut String, session: &Session) { + let _ = writeln!( + text, + "build {} over {} rounds, {} withdrawn, selection {}", + human(session.build), + session.rounds, + session.withdrawn, + if session.widened { "widened to the workspace" } else { "kept" } + ); + + write_rounds(text, session); + write_ordering(text, session); + write_census(text, session); + write_phases(text, session); + + let _ = writeln!( + text, + "baseline {} wall, {} cumulative, longest silence {}, stall budget {}", + human(session.baseline_wall), + human(session.baseline), + human(session.quiet), + session.stall.map_or_else(|| "off".to_owned(), human) + ); + + // The one figure that says whether the suite ran at all, which is otherwise only ever said on + // the progress line — and progress resolves to whether a terminal is attached, so on a CI + // runner it is said to nobody. + let _ = writeln!( + text, + "tests {}", + session + .tests + .map_or_else(|| "no harness announced a count".to_owned(), |tests| quantity(tests, "test")) + ); + + let _ = writeln!( + text, + "memory baseline peak {}", + session.peak.map_or_else(|| "not measured".to_owned(), crate::report::bytes) + ); + + // Walked here rather than carried on the session, because it is a stat per build artifact and + // this dump is the only thing that asks for it. A run that fills a CI runner's disk fails at its + // next step with nothing pointing back here, so the path goes out beside the figure. + let _ = writeln!( + text, + "disk {} under {}", + crate::report::bytes(crate::exec::footprint(&session.scratch)), + session.scratch + ); + + // Read from the process rather than the session, because it is a property of the process: a + // reader the drain gave up on outlives the mutant that produced it, which is the whole reason + // it is worth counting. With `jobs` concurrent mutants and two streams apiece an untroubled run + // peaks near twice the job count; far above that means test binaries are leaving descendants + // holding their output pipes open, and the readers waiting on them are never reclaimed. + let _ = writeln!( + text, + "readers {} peak, {} still running", + crate::exec::READERS.peak(), + crate::exec::READERS.live() + ); +} + +/// Reports where the run's time went, phase by phase. +/// +/// The build total folds the copy, the preflight and the compile into one number, and the census's +/// cost hides inside it too — so whether the per-test census pays for itself against the launches it +/// spares the sweep cannot be read from the aggregates alone. These phase timings are what make that +/// trade visible: the census's `walked` count against the sweep's `launches`, with the probe count +/// saying whether the killer hints earn their keep. The copy and the preflight are components of the +/// build, and the census and the sweep of the testing window, so neither set sums to its aggregate. +fn write_phases(text: &mut String, session: &Session) { + let phases = &session.phases; + + let _ = writeln!( + text, + "phases copy {}, preflight {}, baseline {}", + human(phases.copy), + human(phases.preflight), + human(session.baseline_wall) + ); + + if let Some(census) = &phases.census { + let _ = writeln!( + text, + " census {}, {} over {} binaries", + human(census.elapsed), + quantity(census.walked, "test"), + census.binaries + ); + } + + if let Some(sweep) = &phases.sweep { + let _ = writeln!( + text, + " sweep {}, {} launches, {} probes", + human(sweep.elapsed), + sweep.launches, + sweep.probes + ); + } +} + +/// Reports why the withdrawn mutants were withdrawn. +/// +/// A withdrawal count says a number; it does not say whether the number is worth acting on. Grouped +/// by rustc error code and mutator, it does: a mutator that keeps drawing a type error is one that +/// could be taught to look before it mutates, while a spread of borrow-checker codes across every +/// mutator is the cost of the schema and not a bug in anything. Nothing else in the run says this, +/// and deriving it otherwise means patching the tool by hand. +fn write_census(text: &mut String, session: &Session) { + if session.census.is_empty() { + return; + } + + let _ = writeln!(text, "withdrew by rustc error code and mutator"); + + for entry in &session.census { + let _ = writeln!( + text, + " {:<8}{:<28}{}", + if entry.code.is_empty() { "(none)" } else { &entry.code }, + if entry.mutator.is_empty() { "(unknown)" } else { &entry.mutator }, + quantity(entry.mutants, "mutant") + ); + } +} + +/// Reports what each round of the build cost, and what it bought. +/// +/// A build total on its own cannot tell a tree that compiled first time from one that spent most of +/// its time withdrawing mutants that were never going to compile, and those two want opposite +/// remedies: a faster machine against fewer unviable mutants. That is what the split says. The +/// first round is what building this workspace costs at all; every round after it exists only +/// because some mutant did not compile, and its time is the price of that mutant. +fn write_rounds(text: &mut String, session: &Session) { + let Some((first, rest)) = session.rounds_taken.split_first() else { + return; + }; + + let converging: Duration = rest.iter().map(|round| round.elapsed).sum(); + let total = first.elapsed.saturating_add(converging); + + #[expect(clippy::cast_possible_truncation, reason = "a percentage of a ratio in [0, 1]")] + #[expect(clippy::cast_sign_loss, reason = "both durations are non-negative")] + let share = if total.is_zero() { + 0 + } else { + (converging.as_secs_f64() / total.as_secs_f64() * 100.0).round() as u64 + }; + + let _ = writeln!( + text, + "rounds first {}, then {} over {} ({share}% of the build's rounds)", + human(first.elapsed), + human(converging), + quantity(rest.len(), "further round") + ); + + for (index, round) in session.rounds_taken.iter().enumerate() { + let _ = writeln!( + text, + " round {:<4}{:>8} {}", + index.saturating_add(1), + human(round.elapsed), + if round.withdrew == 0 { + "nothing withdrawn".to_owned() + } else { + format!("{} withdrawn", quantity(round.withdrew, "mutant")) + } + ); + } +} + +/// Reports what front-loading the mutants an earlier run could not compile actually bought. +/// +/// Deliberately not phrased as a saving. The rounds a probe avoided are the length of a convergence +/// that never happened, and a number invented for it would be a model of a counterfactual printed +/// as a measurement — in a file whose entire value is that everything in it is something that +/// happened. What is printed is the trade: how many mutants went in front of the compiler early, +/// how many of those it then refused, and what the probes cost in rounds. A confirmation rate near +/// zero over several rounds is a hint set to regenerate or delete, and that is the decision this +/// line exists to support. +fn write_ordering(text: &mut String, session: &Session) { + let hints = session.ordering; + + if hints.rounds == 0 && hints.offered == 0 { + return; + } + + let _ = writeln!( + text, + "hints {} front-loaded over {}, {} confirmed unviable by the compiler", + quantity(hints.offered, "mutant"), + quantity(hints.rounds as usize, "probe round"), + hints.confirmed + ); +} + +/// Reports how well the run kept its workers busy. +/// +/// The number that matters is the effective job count: CPU over wall. A scheduler that is working +/// lands within a fraction of `--jobs`, and everything short of that is time spent waiting for the +/// slowest binary of a batch rather than testing anything. +fn write_throughput(text: &mut String, mutants: &[Mutant], session: Option<&Session>, jobs: usize, testing: Duration) { + let mut spent: Vec = mutants + .iter() + .filter(|mutant| mutant.elapsed_ms > 0) + .map(|mutant| Duration::from_millis(mutant.elapsed_ms)) + .collect(); + + if spent.is_empty() { + let _ = writeln!(text, "mutants nothing was run"); + return; + } + + spent.sort_unstable(); + + let cpu: Duration = spent.iter().sum(); + + let _ = writeln!( + text, + "mutants {} evaluated, {} cpu, {} effective jobs of {jobs}", + spent.len(), + human(cpu), + ratio(cpu, testing) + ); + + let median = percentile(&spent, 0.50); + + let _ = writeln!( + text, + " min {}, p50 {}, p90 {}, p99 {}, max {}", + human(spent[0]), + human(median), + human(percentile(&spent, 0.90)), + human(percentile(&spent, 0.99)), + human(spent[spent.len() - 1]) + ); + + if let Some(session) = session + && let Some(min_budget) = session.binaries.iter().filter_map(|binary| binary.budget).min() + && crowded_timeout(min_budget, median) + { + let _ = writeln!( + text, + "warning test binary timeout {} sits within {NOISE_FACTOR}x the p50 mutant duration {}: \ + timeouts here are scheduling noise, and every one of them is scored as a kill", + human(min_budget), + human(median) + ); + } +} + +/// How much larger than a typical run the timeout has to be before it is a ceiling rather than a +/// coin toss. +/// +/// A mutant's budget is meant to catch a suite that has stopped making progress, not one that took +/// longer than usual. Between the two is a band where whichever mutants happen to land on a busy +/// moment time out — and a timeout counts as undetected, so the noise lowers the score and can fail +/// the run. +const NOISE_FACTOR: u32 = 2; + +/// Whether the derived mutant timeout is close enough to a typical run to be producing noise. +/// +/// A run whose median mutant takes longer than the whole budget divided by [`NOISE_FACTOR`] is one +/// where the timeout is doing something other than what it was set for. +fn crowded_timeout(timeout: Duration, median: Duration) -> bool { + !median.is_zero() && !timeout.is_zero() && timeout < median.saturating_mul(NOISE_FACTOR) +} + +/// Names the mutants that cost the most, which is where a scheduling change shows up first. +fn write_slowest(text: &mut String, mutants: &[Mutant]) { + let mut ranked: Vec<&Mutant> = mutants.iter().filter(|mutant| mutant.elapsed_ms > 0).collect(); + + if ranked.is_empty() { + return; + } + + ranked.sort_unstable_by_key(|mutant| Reverse(mutant.elapsed_ms)); + ranked.truncate(TOP); + + let _ = writeln!(text, "\nslowest mutants"); + + for mutant in ranked { + let _ = writeln!( + text, + " {:>8} {:<9} {}", + human(Duration::from_millis(mutant.elapsed_ms)), + label(mutant.outcome), + mutant.describe() + ); + } +} + +/// Reports one grouping, ranked by the CPU it consumed. +/// +/// Ranked by cost rather than by name because the question this table answers is always "what +/// should be looked at first", and the answer is whatever is at the top. +fn write_breakdown(text: &mut String, noun: &str, buckets: &HashMap) { + if buckets.is_empty() { + return; + } + + let mut rows: Vec<(&String, &Bucket)> = buckets.iter().collect(); + + rows.sort_by(|(left_name, left), (right_name, right)| right.cpu.cmp(&left.cpu).then_with(|| left_name.cmp(right_name))); + + let shown = rows.len().min(TOP); + + let _ = writeln!(text, "\nby {noun} ({shown} of {})", rows.len()); + let _ = writeln!( + text, + " {:>8} {:>7} {:>9} {:>8} {noun}", + "cpu", "mutants", "survivors", "unviable" + ); + + for (name, bucket) in rows.into_iter().take(TOP) { + let _ = writeln!( + text, + " {:>8} {:>7} {:>9} {:>8} {name}", + human(bucket.cpu), + bucket.mutants, + bucket.survivors, + bucket.unviable + ); + } +} + +/// Reports what each test binary cost the run. +/// +/// A binary's baseline is charged to every mutant that can reach it, so a single slow one is +/// multiplied by the population and is the most leveraged thing in a run. +fn write_binaries(text: &mut String, session: &Session) { + if session.binaries.is_empty() { + return; + } + + let mut binaries: Vec<&TestBinary> = session.binaries.iter().collect(); + + binaries.sort_by_key(|binary| Reverse(binary.baseline)); + + let total: Duration = binaries.iter().map(|binary| binary.baseline).sum(); + + let _ = writeln!(text, "\ntest binaries ({}, {} baseline)", binaries.len(), human(total)); + let _ = writeln!( + text, + " {:>8} {:>8} {:>10} {:>10} {:<20} binary", + "baseline", "budget", "peak", "ceiling", "package" + ); + + for binary in binaries.into_iter().take(TOP) { + let _ = writeln!( + text, + " {:>8} {:>8} {:>10} {:>10} {:<20} {}", + human(binary.baseline), + binary.budget.map_or_else(|| "-".to_owned(), human), + binary.peak.map_or_else(|| "-".to_owned(), crate::report::bytes), + binary.memory.map_or_else(|| "-".to_owned(), crate::report::bytes), + binary.package, + binary.path.file_name().unwrap_or(binary.path.as_str()) + ); + } +} + +/// Buckets the population by whatever `key` names. +fn group(mutants: &[Mutant], key: impl Fn(&Mutant) -> String) -> HashMap { + let mut buckets: HashMap = HashMap::default(); + + for mutant in mutants { + buckets.entry(key(mutant)).or_default().absorb(mutant); + } + + buckets +} + +/// The value at `fraction` through an already-sorted list. +fn percentile(sorted: &[Duration], fraction: f64) -> Duration { + if sorted.is_empty() { + return Duration::ZERO; + } + + #[expect(clippy::cast_precision_loss, reason = "a mutant count far exceeds any plausible workspace")] + let position = fraction * (sorted.len() - 1) as f64; + + #[expect(clippy::cast_possible_truncation, reason = "the operand is an index into the list above")] + #[expect(clippy::cast_sign_loss, reason = "the fraction and the length are both non-negative")] + let index = position.round() as usize; + + sorted[index.min(sorted.len() - 1)] +} + +/// How many workers the run actually kept busy, as a printable ratio. +fn ratio(cpu: Duration, wall: Duration) -> String { + if wall.is_zero() { + return "?".to_owned(); + } + + format!("{:.1}", cpu.as_secs_f64() / wall.as_secs_f64()) +} + +/// The unstyled name of an outcome, so the columns line up whatever the terminal is. +const fn label(outcome: Outcome) -> &'static str { + match outcome { + Outcome::Killed => "killed", + Outcome::Survived => "survived", + Outcome::Timeout => "timeout", + Outcome::OutOfMemory => "outofmem", + Outcome::Flaky => "flaky", + Outcome::CompileError => "unviable", + Outcome::Ignored => "ignored", + Outcome::NoCoverage => "uncovered", + Outcome::NotBuilt => "notbuilt", + Outcome::Pending => "pending", + } +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use camino::Utf8PathBuf; + + use super::*; + use crate::exec::Round; + use crate::fixtures; + + fn mutant(file: &str, mutator: &str, outcome: Outcome, ms: u64) -> Mutant { + Mutant { + id: format!("{file}:{mutator}:{ms}").into(), + file: (Utf8PathBuf::from(file)).into(), + mutator: (mutator.to_owned()).into(), + original: "a + b".to_owned().into(), + replacement: "a - b".to_owned().into(), + outcome, + elapsed_ms: ms, + ..fixtures::mutant() + } + } + + fn plan(mutants: Vec) -> Plan { + Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: Utf8PathBuf::from("/w"), + files: Vec::new(), + mutants, + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + } + } + + fn session(binaries: Vec) -> Session { + Session { + census: Vec::new(), + baseline: Duration::from_secs(5), + baseline_wall: Duration::from_secs(2), + tests: Some(12), + quiet: Duration::from_secs(2), + stall: Some(Duration::from_secs(3)), + build: Duration::from_secs(7), + metered: false, + unbounded: None, + withdrawn: 4, + rounds: 2, + rounds_taken: vec![ + Round { + elapsed: Duration::from_secs(5), + withdrew: 4, + }, + Round { + elapsed: Duration::from_secs(2), + withdrew: 0, + }, + ], + binaries, + peak: None, + scratch: Utf8PathBuf::new(), + filtered: 0, + widened: true, + ordering: crate::exec::OrderingHints::default(), + phases: crate::exec::Phases::default(), + } + } + + #[test] + fn a_percentile_never_indexes_past_the_end() { + let sorted = [Duration::from_secs(1), Duration::from_secs(2), Duration::from_secs(3)]; + + assert_eq!(percentile(&sorted, 0.0), Duration::from_secs(1)); + assert_eq!(percentile(&sorted, 1.0), Duration::from_secs(3)); + assert_eq!(percentile(&sorted, 0.5), Duration::from_secs(2)); + } + + #[test] + fn a_percentile_of_nothing_is_zero_rather_than_a_panic() { + assert_eq!(percentile(&[], 0.9), Duration::ZERO); + } + + #[test] + fn effective_jobs_is_cpu_over_wall() { + assert_eq!(ratio(Duration::from_secs(80), Duration::from_secs(10)), "8.0"); + } + + #[test] + fn a_run_with_no_wall_time_reports_no_ratio_rather_than_an_infinite_one() { + assert_eq!(ratio(Duration::from_secs(80), Duration::ZERO), "?"); + } + + #[test] + fn a_breakdown_is_ranked_by_cost() { + let mutants = vec![ + mutant("a.rs", "arith.add_to_sub", Outcome::Killed, 100), + mutant("b.rs", "literal.int_to_zero", Outcome::Survived, 900), + ]; + + let text = render(&plan(mutants), None, 4, Duration::from_secs(1)); + let cheap = text.find("arith.add_to_sub").expect("the cheap family is listed"); + let dear = text.find("literal.int_to_zero").expect("the expensive family is listed"); + + assert!(dear < cheap, "the expensive family must come first:\n{text}"); + } + + #[test] + fn a_run_that_tested_nothing_still_reports_its_population() { + let text = render( + &plan(vec![mutant("a.rs", "arith.add_to_sub", Outcome::Pending, 0)]), + None, + 4, + Duration::ZERO, + ); + + assert!(text.contains("1 mutant,"), "{text}"); + assert!(text.contains("nothing was run"), "{text}"); + + // With no session there is nothing to say about a build that never happened, and inventing + // zeroes for it would read as a build that took no time. + assert!(!text.contains("baseline"), "{text}"); + } + + #[test] + fn a_live_run_reports_the_session_costs_and_scope() { + let text = render(&plan(Vec::new()), Some(&session(Vec::new())), 4, Duration::from_secs(20)); + + // These fixed costs explain how much of the wall clock was not mutant execution. + assert!( + text.contains("build 7.0s over 2 rounds, 4 withdrawn, selection widened to the workspace"), + "{text}" + ); + assert!( + text.contains("baseline 2.0s wall, 5.0s cumulative, longest silence 2.0s, stall budget 3.0s"), + "{text}" + ); + assert!(text.contains("wall 20.0s, of which 11.0s testing"), "{text}"); + } + + /// The disk a run left behind is measured here and nowhere else. + /// + /// It is a stat per build artifact over a directory that holds every object file of every + /// round, so nothing may compute it unless it is going to be printed — which makes this dump, + /// the only thing that prints it, also the only thing that pays for it. + #[test] + fn a_live_run_reports_the_disk_it_left_behind() { + let temporary = tempfile::tempdir().expect("a scratch directory"); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("the scratch path is UTF-8"); + + std::fs::write(base.join("artifact.o").as_std_path(), vec![0u8; 2048]).expect("an artifact"); + + let live = Session { + scratch: base.clone(), + ..session(Vec::new()) + }; + + let text = render(&plan(Vec::new()), Some(&live), 4, Duration::from_secs(20)); + + assert!(text.contains(&format!("disk 2.0 KB under {base}")), "{text}"); + } + + /// The build total folds the copy, the preflight and the census into one number, so the phase + /// line is the only place the dump says where the fixed and testing time actually went — the + /// census's walk against the sweep's launches, with the probe count on whether the hints paid. + #[test] + fn a_live_run_reports_where_each_phase_spent_its_time() { + let live = Session { + phases: crate::exec::Phases { + copy: Duration::from_secs(1), + preflight: Duration::from_secs(2), + census: Some(crate::exec::CensusCost { + elapsed: Duration::from_secs(8), + walked: 1_681, + binaries: 30, + }), + sweep: Some(crate::exec::SweepCost { + elapsed: Duration::from_secs(42), + launches: 47, + probes: 12, + }), + }, + ..session(Vec::new()) + }; + + let text = render(&plan(Vec::new()), Some(&live), 4, Duration::from_mins(1)); + + assert!(text.contains("phases copy 1.0s, preflight 2.0s, baseline 2.0s"), "{text}"); + assert!(text.contains("census 8.0s, 1681 tests over 30 binaries"), "{text}"); + assert!(text.contains("sweep 42.0s, 47 launches, 12 probes"), "{text}"); + } + + /// A census that did not run leaves no census line, because a run that was not asked to census + /// did not census in zero time — it did not census, and inventing a zero would say otherwise. + #[test] + fn a_live_run_without_a_census_prints_no_census_phase() { + let text = render(&plan(Vec::new()), Some(&session(Vec::new())), 4, Duration::from_secs(20)); + + assert!(text.contains("phases copy"), "{text}"); + assert!(!text.contains("census 0"), "an unrun census must not be printed: {text}"); + } + + /// The withdrawal count says a number; only the codes behind it say whether the number is a + /// mutator that could be taught to look before it mutates or an unavoidable cost of the schema. + /// The dump is the only place that answer is available without patching the tool. + #[test] + fn a_live_run_reports_why_its_mutants_were_withdrawn() { + let live = Session { + census: vec![ + crate::exec::Withdrawal { + code: "E0308".to_owned(), + mutator: "lit.true_to_false".to_owned(), + mutants: 12, + }, + crate::exec::Withdrawal { + code: String::new(), + mutator: String::new(), + mutants: 1, + }, + ], + ..session(Vec::new()) + }; + + let text = render(&plan(Vec::new()), Some(&live), 4, Duration::from_secs(20)); + + assert!(text.contains("withdrew by rustc error code and mutator"), "{text}"); + assert!(text.contains("E0308"), "{text}"); + assert!(text.contains("lit.true_to_false"), "{text}"); + assert!(text.contains("12 mutants"), "{text}"); + assert!(text.contains("(none)"), "{text}"); + assert!(text.contains("(unknown)"), "{text}"); + } + + /// A run that withdrew nothing has nothing to say, and a heading over an empty list reads as a + /// missing measurement rather than as an absent one. + #[test] + fn a_run_that_withdrew_nothing_says_nothing_about_why() { + let text = render(&plan(Vec::new()), Some(&session(Vec::new())), 4, Duration::from_secs(20)); + + assert!(!text.contains("withdrew "), "{text}"); + } + + /// The reader count is the measurement that decides whether a bound on stray readers is needed. + /// + /// A reader the drain gave up on holds a thread and a descriptor for the rest of the run, and + /// nothing else in the tool would ever say so — the mutant it belonged to has long since been + /// scored. Reporting the peak is what turns "this might accumulate" into a number, so the line + /// has to actually appear rather than be something a reader is assumed to know to ask for. + #[test] + fn a_live_run_reports_how_many_output_readers_it_needed() { + let text = render(&plan(Vec::new()), Some(&session(Vec::new())), 4, Duration::from_secs(20)); + + assert!(text.contains("readers "), "{text}"); + assert!(text.contains(" peak, "), "{text}"); + assert!(text.contains(" still running"), "{text}"); + } + + /// A build total cannot distinguish a tree that compiled first time from one that converged for + /// most of its time, and those two want opposite remedies. The split has to be stated. + #[test] + fn a_live_run_reports_what_each_build_round_cost() { + let text = render(&plan(Vec::new()), Some(&session(Vec::new())), 4, Duration::from_secs(20)); + + assert!( + text.contains("rounds first 5.0s, then 2.0s over 1 further round (29% of the build's rounds)"), + "{text}" + ); + assert!(text.contains("round 1 5.0s 4 mutants withdrawn"), "{text}"); + assert!(text.contains("round 2 2.0s nothing withdrawn"), "{text}"); + } + + /// A run whose rounds were never recorded says nothing about them rather than claiming a build + /// that took no time at all. + #[test] + fn a_run_with_no_recorded_rounds_says_nothing_about_them() { + let mut session = session(Vec::new()); + session.rounds_taken = Vec::new(); + + let text = render(&plan(Vec::new()), Some(&session), 4, Duration::from_secs(20)); + + assert!(!text.contains("rounds "), "{text}"); + } + + /// + /// Regression, issue-035. The only other place this figure is ever stated is the progress line + /// the baseline closes with, and progress resolves to whether a terminal is attached — so on a + /// CI runner, where "did my suite run at all" is precisely the question this answers, nobody + /// was told. + #[test] + fn a_live_run_reports_how_many_tests_the_baseline_ran() { + let text = render(&plan(Vec::new()), Some(&session(Vec::new())), 4, Duration::from_secs(20)); + + assert!(text.contains("tests 12 tests"), "{text}"); + } + + /// A suite whose harness announced no count says so rather than reporting zero. + /// + /// A `harness = false` target announces nothing, and a run that printed `0 tests` for it would + /// read exactly like a suite that ran none — which is the alarming case this figure exists to + /// make visible. + #[test] + fn a_baseline_no_harness_counted_says_so_rather_than_reporting_zero() { + let mut session = session(Vec::new()); + + session.tests = None; + + let text = render(&plan(Vec::new()), Some(&session), 4, Duration::from_secs(20)); + + assert!(text.contains("tests no harness announced a count"), "{text}"); + } + + /// A timeout no larger than a typical mutant run is called out, because the timeouts it + /// produces are noise scored as detections. + /// + /// Regression, issue-016. A mutant that times out counts as killed, so a budget sitting in the + /// band where scheduling decides the answer inflates the score and sends the reader looking for + /// hangs that were never there. + #[test] + fn a_timeout_close_to_the_typical_mutant_duration_is_called_out() { + let binary = TestBinary { + budget: Some(Duration::from_millis(120)), + ..crate::testing::test_binary("/w/target/debug/deps/a") + }; + let session = session(vec![binary]); + + let text = render( + &plan(vec![ + mutant("a.rs", "m", Outcome::Killed, 100), + mutant("b.rs", "m", Outcome::Killed, 100), + ]), + Some(&session), + 4, + Duration::from_secs(1), + ); + + assert!(text.contains("warning test binary timeout"), "{text}"); + assert!(text.contains("scored as a kill"), "{text}"); + } + + /// A timeout with room above a typical run says nothing, because there is nothing wrong. + #[test] + fn a_timeout_with_room_above_the_typical_mutant_duration_says_nothing() { + let text = render( + &plan(vec![mutant("a.rs", "m", Outcome::Killed, 100)]), + Some(&session(Vec::new())), + 4, + Duration::from_secs(1), + ); + + assert!(!text.contains("warning"), "{text}"); + } + + /// A run that measured no mutant durations cannot say anything about its timeout, and does not + /// invent a comparison against a median of zero. + #[test] + fn a_median_of_nothing_is_not_compared_against_the_timeout() { + assert!(!crowded_timeout(Duration::from_secs(30), Duration::ZERO)); + assert!(!crowded_timeout(Duration::ZERO, Duration::from_secs(30))); + assert!(crowded_timeout(Duration::from_secs(30), Duration::from_secs(20))); + assert!(!crowded_timeout(Duration::from_secs(30), Duration::from_secs(15))); + } + + #[test] + fn a_session_without_a_stall_budget_says_it_is_off() { + let mut session = session(Vec::new()); + + session.stall = None; + session.widened = false; + + let text = render(&plan(Vec::new()), Some(&session), 4, Duration::from_secs(20)); + + // The absence of a stall budget is a real operating mode, not a zero-second timeout. + assert!(text.contains("selection kept"), "{text}"); + assert!(text.contains("stall budget off"), "{text}"); + } + + #[test] + fn a_breakdown_counts_unviable_mutants_separately_from_survivors() { + let text = render( + &plan(vec![ + mutant("a.rs", "arith.add_to_sub", Outcome::CompileError, 10), + mutant("a.rs", "arith.add_to_sub", Outcome::Survived, 20), + ]), + None, + 4, + Duration::from_secs(1), + ); + + // Unviable mutants are withdrawn from the score, but the diagnostic table keeps their cost + // visible to someone improving the mutator. + assert!(text.contains("outcomes 0 killed, 0 timeout, 0 outofmem, 1 survived,"), "{text}"); + assert!(text.contains(", 1 unviable, 0 ignored, 0 notbuilt, 0 pending"), "{text}"); + assert!( + text.contains(" 30ms 2 1 1 arith.add_to_sub"), + "{text}" + ); + } + + #[test] + fn the_outcome_line_accounts_for_every_mutant_in_the_population() { + // The dump is what a user sends when they cannot explain a run, and the JSON bundle beside + // it carries all ten counters. A prose line naming a subset makes the two disagree, and a + // reader cannot tell an unlisted category from a mutant that went missing. Distinct + // multiplicities so that a figure reading the wrong counter fails as loudly as a missing + // one. + let mut mutants = Vec::new(); + + for (index, outcome) in Outcome::ALL.into_iter().enumerate() { + for _ in 0..=index { + mutants.push(mutant("a.rs", "arith.add_to_sub", outcome, 10)); + } + } + + let population = mutants.len(); + let summary = Summary::of(&mutants); + let text = render(&plan(mutants), None, 4, Duration::from_secs(1)); + + let line = text + .lines() + .find(|line| line.starts_with("outcomes ")) + .expect("the dump names the outcomes"); + + let counted: u32 = line + .trim_start_matches("outcomes ") + .split(", ") + .map(|figure| { + figure + .split_once(' ') + .expect("a figure is a count and a name") + .0 + .parse::() + .expect("a figure counts mutants") + }) + .sum(); + + assert_eq!(counted as usize, population, "{text}"); + + for outcome in Outcome::ALL { + assert!( + line.contains(&format!("{} {outcome}", summary.count(outcome))), + "{outcome} is missing from `{line}`" + ); + } + } + + #[test] + fn the_slowest_table_uses_plain_outcome_labels_for_every_non_survivor_result() { + let text = render( + &plan(vec![ + mutant("timeout.rs", "m", Outcome::Timeout, 70), + mutant("unviable.rs", "m", Outcome::CompileError, 60), + mutant("ignored.rs", "m", Outcome::Ignored, 50), + mutant("uncovered.rs", "m", Outcome::NoCoverage, 40), + mutant("pending.rs", "m", Outcome::Pending, 30), + mutant("outofmem.rs", "m", Outcome::OutOfMemory, 20), + mutant("notbuilt.rs", "m", Outcome::NotBuilt, 10), + ]), + None, + 4, + Duration::from_secs(1), + ); + + // The table is intentionally unstyled, so every verdict has to be rendered as text that + // still lines up in a plain diagnostic dump. + for label in ["timeout", "unviable", "ignored", "uncovered", "pending", "outofmem", "notbuilt"] { + assert!(text.contains(label), "{text}"); + } + } + + #[test] + fn test_binaries_are_ranked_by_baseline_cost() { + let text = render( + &plan(Vec::new()), + Some(&session(vec![ + TestBinary { + package: "fast".to_owned(), + baseline: Duration::from_secs(1), + budget: Some(Duration::from_secs(10)), + ..crate::testing::test_binary("/w/target/debug/deps/fast-abc") + }, + TestBinary { + package: "slow".to_owned(), + baseline: Duration::from_secs(3), + budget: Some(Duration::from_secs(30)), + ..crate::testing::test_binary("/w/target/debug/deps/slow-def") + }, + ])), + 4, + Duration::from_secs(20), + ); + + let slow = text.find("slow-def").expect("slow binary is listed"); + let fast = text.find("fast-abc").expect("fast binary is listed"); + + // A slow binary is multiplied by every mutant that reaches it, so the most expensive one + // must be shown first. + assert!(text.contains("test binaries (2, 4.0s baseline)"), "{text}"); + assert!(slow < fast, "{text}"); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/compile_fail.rs b/crates/cargo-gamma-lib/src/discover/compile_fail.rs new file mode 100644 index 000000000..76b762ce1 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/compile_fail.rs @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Recognizing test targets that run the compiler rather than the code. + +use cargo_metadata::{DependencyKind, Metadata}; + +/// The dev-dependencies that give a compile-fail harness away. +/// +/// Both drive rustc once per case and assert on what it says. Neither is detectable from the target +/// list alone, since such a target is an ordinary integration test as far as cargo is concerned. +const HARNESSES: [&str; 2] = ["trybuild", "compiletest_rs"]; + +/// A test target that asserts about compiler output rather than exercising the code under test. +/// +/// Under mutation testing these are ruinous: the harness invokes rustc once per case, every mutant +/// pays that in full, and the oracle almost never gains anything, because what the target checks is +/// that some code fails to compile — which a mutated function body several crates away does not +/// change. A target of this shape has been measured putting a single mutant at one to two minutes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompileFailTarget { + /// The workspace package declaring it. + pub package: String, + + /// The cargo target name, which is what `--exclude-test` is written against. + pub target: String, + + /// The harness crate found in the package's dev-dependencies. + pub harness: String, +} + +/// Finds every test target that appears to assert about compiler output. +/// +/// Detection is two-stage on purpose. The dev-dependency is what makes the search cheap — almost no +/// package declares one, so almost no package is looked at further. Reading the target's root +/// source is what makes the answer specific: a package with a compile-fail target usually has +/// ordinary test targets beside it, and naming those too would send the reader to exclude tests that +/// cost nothing. +/// +/// A target whose source cannot be read is left out rather than guessed at. The whole value of this +/// is that it names a target precisely enough to act on, and a warning about the wrong one is worse +/// than none: `--exclude-test` takes a target out of the oracle, so acting on a bad name silently +/// narrows what can convict a mutant. +pub(super) fn compile_fail_targets(metadata: &Metadata) -> Vec { + let mut found = Vec::new(); + + for package in metadata.workspace_packages() { + let Some(harness) = package + .dependencies + .iter() + .filter(|dependency| dependency.kind == DependencyKind::Development) + .map(|dependency| dependency.name.as_str()) + .find(|name| HARNESSES.contains(name)) + else { + continue; + }; + + for target in package.targets.iter().filter(|target| target.test) { + let Ok(source) = std::fs::read_to_string(&target.src_path) else { + continue; + }; + + if mentions(&source, harness) { + found.push(CompileFailTarget { + package: package.name.as_str().to_owned(), + target: target.name.clone(), + harness: harness.to_owned(), + }); + } + } + } + + found.sort_by(|left, right| left.target.cmp(&right.target)); + found.dedup(); + found +} + +/// Whether a source file names the harness crate. +/// +/// A substring search over the root source, because the shapes in use are `trybuild::TestCases`, +/// `use trybuild;` and `compiletest_rs::run_tests`, and distinguishing those from the same word in a +/// comment would need the file parsed for no gain: a file that mentions the harness at all is a file +/// this warning is right about. +fn mentions(source: &str, harness: &str) -> bool { + source.contains(harness) +} + +/// Renders the warning a run shows when it finds one of these. +/// +/// The flag is spelled out in full because the value of this is that it can be acted on without +/// first learning that `--exclude-test` exists. Nothing is excluded automatically: `trybuild` +/// asserts exact compiler output, so on a proc-macro crate it is often the *primary* oracle — a +/// mutant that corrupts a diagnostic message is caught there and nowhere else. Excluding by default +/// would gut the oracle for the code the technique suits best, and the mutants would come back as +/// survivors rather than as anything visibly wrong. +#[must_use] +pub fn advice(targets: &[CompileFailTarget]) -> Option { + if targets.is_empty() { + return None; + } + + let named: Vec = targets + .iter() + .map(|target| format!("`{}` in {} ({})", target.target, target.package, target.harness)) + .collect(); + + let flags: Vec = targets.iter().map(|target| format!("--exclude-test {}", target.target)).collect(); + + Some(format!( + "{} {} the compiler once per case, so every mutant pays for a full rustc run and almost none are convicted by it: {}.\n\ + If {} is not part of what should be judging these mutants, exclude it with `{}`.\n\ + Left in, it is likely to make this run take hours.", + crate::report::quantity(targets.len(), "test target"), + if targets.len() == 1 { "invokes" } else { "invoke" }, + named.join(", "), + if targets.len() == 1 { "it" } else { "any of them" }, + flags.join(" "), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn target(name: &str) -> CompileFailTarget { + CompileFailTarget { + package: "routerama".to_owned(), + target: name.to_owned(), + harness: "trybuild".to_owned(), + } + } + + #[test] + fn a_source_naming_the_harness_is_recognized() { + assert!(mentions("fn main() { trybuild::TestCases::new(); }", "trybuild")); + assert!(mentions("use compiletest_rs as compiletest;", "compiletest_rs")); + } + + /// An ordinary integration test in the same package must not be named, or the reader is sent to + /// exclude a target that costs nothing and can convict. + #[test] + fn an_ordinary_test_source_is_not_recognized() { + assert!(!mentions("#[test]\nfn routes_are_matched() {}", "trybuild")); + } + + /// Nothing found means nothing said. A warning with no target in it cannot be acted on. + #[test] + fn no_compile_fail_target_produces_no_advice() { + assert_eq!(advice(&[]), None); + } + + /// The whole point is that the flag can be copied out of the message, so it has to be there in + /// full — including the target name, which is what the pattern matches on. + #[test] + fn the_advice_names_the_target_and_the_flag_that_removes_it() { + let text = advice(&[target("router_compile_fail")]).expect("one target is enough to advise about"); + + assert!(text.contains("`router_compile_fail` in routerama (trybuild)"), "{text}"); + assert!(text.contains("--exclude-test router_compile_fail"), "{text}"); + assert!(text.contains("1 test target invokes"), "{text}"); + } + + /// Several targets are one warning with one flag string, since a reader who has to run the + /// command twice will run it once and wonder why the run is still slow. + #[test] + fn several_targets_are_advised_about_together() { + let text = advice(&[target("router_compile_fail"), target("feature_gates")]).expect("two targets are advised about"); + + assert!(text.contains("2 test targets invoke"), "{text}"); + assert!( + text.contains("--exclude-test router_compile_fail --exclude-test feature_gates"), + "{text}" + ); + } + + /// The warning must not exclude anything, only say what would. + #[test] + fn the_advice_says_it_is_the_callers_decision() { + let text = advice(&[target("router_compile_fail")]).expect("one target is enough to advise about"); + + assert!(text.contains("If it is not part of what should be judging"), "{text}"); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/diff.rs b/crates/cargo-gamma-lib/src/discover/diff.rs new file mode 100644 index 000000000..279f55cef --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/diff.rs @@ -0,0 +1,848 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Restricting a run to the lines a unified diff touches. + +use std::fs; +use std::io::{Read, stdin}; + +use camino::{Utf8Path, Utf8PathBuf}; + +use crate::error::error; +use crate::{HashMap, HashSet, Result}; + +/// The lines each file gained in a diff, as a set of line numbers in the new file. +#[derive(Debug, Default)] +pub struct Diff { + touched: HashMap>, + + /// For each key, the path exactly as the diff spelled it. + /// + /// Kept because prefix stripping is a guess, and [`Diff::resolve`] needs the unaltered text to + /// fall back on when the guess turns out not to name anything in the workspace. + sources: HashMap, +} + +impl Diff { + /// Reads a unified diff from a path, or from standard input when the path is `-`. + /// + /// # Errors + /// + /// Returns an error if the diff cannot be read. + pub fn read(path: &Utf8Path) -> Result { + Self::read_from(path, &mut stdin()) + } + + /// Reads a unified diff, taking `-` from `input` rather than from the real standard input. + /// + /// The seam exists so that the `-` path is an ordinary test rather than something that would + /// block on a terminal, which is what reading the process's real standard input would do inside + /// a test binary. + /// + /// # Errors + /// + /// Returns an error if the diff cannot be read. + pub fn read_from(path: &Utf8Path, input: &mut impl Read) -> Result { + let text = if path == "-" { + let mut buffer = String::new(); + + let _read = input + .read_to_string(&mut buffer) + .map_err(|cause| error!("could not read a diff from standard input").caused_by(cause))?; + + buffer + } else { + fs::read_to_string(path).map_err(|cause| error!("could not read the diff `{path}`").caused_by(cause))? + }; + + Ok(Self::parse(&text)) + } + + /// Parses a unified diff. + /// + /// Only added and modified lines count. A deleted line has no position in the new file, so + /// there is nothing there to mutate, and a context line is by definition unchanged. + /// + /// The prefix on each path is worked out from the diff itself rather than assumed to be git's + /// default `b/`. `diff.mnemonicPrefix` writes `i/`, `w/`, `c/` or `o/`, `--dst-prefix` writes + /// whatever it was given, and `diff.noprefix` writes none at all — and a prefix that is not + /// recognized leaves every path naming a file that does not exist, which selects nothing and + /// reads exactly like a change that touched no code. + #[must_use] + pub fn parse(text: &str) -> Self { + let mut touched: HashMap> = HashMap::default(); + let mut sources: HashMap = HashMap::default(); + let mut current: Option<(Utf8PathBuf, Utf8PathBuf)> = None; + let mut line_number = 0_u32; + + // What the `diff --git` header said the post-image prefix is, and what the `---` line + // spelled the pre-image as. Either is enough to recognize the prefix on the `+++` line. + let mut dst_prefix: Option = None; + let mut pre_image: Option = None; + + // The pre- and post-image lines the current hunk still expects, taken from its `@@` header. + // While either is positive the parser is inside the hunk body, where a line that begins like + // a `+++`/`---` file header is really added or removed source text — a raw `+++ x` is an + // added `++ x`, a raw `--- y` a removed `-- y` — and must be counted as content rather than + // consumed as metadata. Only once both are spent is a line read as a header again. + let mut remaining_old = 0_u32; + let mut remaining_new = 0_u32; + + for line in text.lines() { + let marker = line.as_bytes().first().copied(); + let in_body = (remaining_old > 0 || remaining_new > 0) && matches!(marker, Some(b'+' | b'-' | b' ' | b'\\') | None); + + if in_body { + match marker { + Some(b'+') => { + if let Some((path, _raw)) = current.as_ref() { + touched.entry(path.clone()).or_default().push(line_number); + } + line_number = line_number.saturating_add(1); + remaining_new = remaining_new.saturating_sub(1); + } + + // A deleted line has no place in the post-image, so it advances no numbering; it + // does account for one of the pre-image lines the hunk promised. + Some(b'-') => remaining_old = remaining_old.saturating_sub(1), + + // A context line sits in both images and is counted against each. + Some(b' ') | None => { + line_number = line_number.saturating_add(1); + remaining_old = remaining_old.saturating_sub(1); + remaining_new = remaining_new.saturating_sub(1); + } + + // `\ No newline at end of file` belongs to neither image and counts for neither. + _ => {} + } + + continue; + } + + // Outside a hunk body — its declared counts are spent, or this is not a body line — so + // any hunk in progress is finished and the line is read as metadata. + remaining_old = 0; + remaining_new = 0; + + if let Some(rest) = line.strip_prefix("diff --git ") { + dst_prefix = git_prefix(rest); + pre_image = None; + } else if let Some(rest) = line.strip_prefix("+++ ") { + current = new_file_path(rest, dst_prefix.as_deref(), pre_image.as_deref()); + + if let Some((path, raw)) = current.as_ref() { + let _replaced = sources.insert(path.clone(), raw.clone()); + } + } else if let Some(rest) = line.strip_prefix("--- ") { + pre_image = header_path(rest).map(ToOwned::to_owned); + } else if let Some(rest) = line.strip_prefix("@@") { + if let Some((start, old_count, new_count)) = hunk_header(rest) { + line_number = start; + remaining_old = old_count; + remaining_new = new_count; + } else { + current = None; + } + } + } + + sources.retain(|path, _raw| touched.contains_key(path)); + normalize_lines(&mut touched); + + Self { touched, sources } + } + + /// Points every path the diff named at the file in the workspace it refers to. + /// + /// A diff is written from wherever the person who produced it happened to be standing, with + /// whatever prefixes their configuration prefers, so a path in it is a name rather than a + /// location. Each one is matched against the workspace: the path as spelled, then the path with + /// leading directories peeled off, then the workspace file it uniquely ends with. + /// + /// # Errors + /// + /// Returns an error when the diff named a Rust source path that could not be matched, or when + /// it named paths and not one of them could be matched. Neither is an empty change: an empty + /// change names nothing. Both mean the diff was not understood, and continuing would run fewer + /// mutants than the change deserves while reporting a score as though it had run them all. + pub fn resolve(&mut self, root: &Utf8Path, candidates: &[Utf8PathBuf]) -> Result<()> { + let named = core::mem::take(&mut self.touched); + let sources = core::mem::take(&mut self.sources); + let mut resolved: HashMap> = HashMap::default(); + let mut unresolved: Vec = Vec::new(); + + // Built once, because every exact and every peeled candidate is looked up in it and the + // peeling makes that several lookups per path. Scanning the workspace file list instead + // costs a full pass each time, on the path that is advertised as running on every pull + // request — where time to first output is the whole point. + let known: HashSet<&Utf8Path> = candidates.iter().map(Utf8PathBuf::as_path).collect(); + + for (path, lines) in named { + let raw = sources.get(&path).unwrap_or(&path).clone(); + + if let Some(found) = locate(&path, &raw, root, &known, candidates) { + resolved.entry(found).or_default().extend(lines); + } else { + unresolved.push(raw); + } + } + + unresolved.sort(); + + // A path that names Rust source and did not resolve is the dangerous case. `Survey::for_build` + // retains only the files the diff is believed to touch, so every mutant in an unresolved file + // is silently never generated: the population shrinks, the score rises, and nothing says so. + // This is checked before the everything-failed case below because one resolvable `README.md` + // hunk is enough to make that check pass while a whole source file goes missing. + // + // Paths that are not Rust source contribute no mutants whatever happens to them, so an + // unresolved `README.md` is genuinely nothing to report. + let rust: Vec<&str> = unresolved + .iter() + .filter(|path| path.extension() == Some("rs")) + .map(|path| path.as_str()) + .collect(); + + if !rust.is_empty() { + let listed = rust.join(", "); + + return Err(error!( + "the diff names {} of Rust source that this workspace does not contain ({listed}); \ + continuing would silently generate no mutants for them and report a score for a \ + population that never included them", + crate::report::quantity(rust.len(), "path") + ) + .usage()); + } + + if resolved.is_empty() && !unresolved.is_empty() { + let listed = unresolved.iter().map(|path| path.as_str()).collect::>().join(", "); + + return Err(error!( + "the diff names {} that this workspace does not contain ({listed}); \ + it was produced somewhere else, or with path prefixes this workspace cannot resolve", + crate::report::quantity(unresolved.len(), "path") + ) + .usage()); + } + + normalize_lines(&mut resolved); + self.touched = resolved; + + Ok(()) + } + + /// Returns whether a file has any changed line. + #[must_use] + pub fn touches_file(&self, path: &Utf8Path) -> bool { + self.touched.contains_key(path) + } + + /// Returns whether a region of a file overlaps anything the diff changed. + /// + /// A mutation site is matched by its whole extent rather than by its first line, so editing + /// the middle of a multi-line condition still selects the mutants on it. + #[must_use] + pub fn touches(&self, path: &Utf8Path, start: u32, end: u32) -> bool { + if start > end { + return false; + } + + self.touched.get(path).is_some_and(|lines| { + let first = lines.partition_point(|line| *line < start); + + lines.get(first).is_some_and(|line| *line <= end) + }) + } +} + +fn normalize_lines(touched: &mut HashMap>) { + for lines in touched.values_mut() { + lines.sort_unstable(); + lines.dedup(); + } +} + +/// Extracts the path from a `---` or `+++` header, rejecting the one that means "no such file". +fn header_path(rest: &str) -> Option<&str> { + // Trailing tab-separated metadata is part of the format, and git writes a timestamp there. + let path = rest.split('\t').next().unwrap_or(rest).trim(); + + if path.is_empty() || path == "/dev/null" { None } else { Some(path) } +} + +/// Extracts the path from a `+++` header, paired with the text the diff actually wrote. +/// +/// `dst_prefix` is what the `diff --git` header revealed, and `pre_image` is the `---` path this +/// one is paired with. Either identifies the prefix; failing both, git's default `b/` is stripped +/// when it is there, which is what a `diff -u` with no prefixes at all needs left alone. +fn new_file_path(rest: &str, dst_prefix: Option<&str>, pre_image: Option<&str>) -> Option<(Utf8PathBuf, Utf8PathBuf)> { + let raw = header_path(rest)?; + + // The header is the best evidence, the `---` line it is paired with is the next best, and + // git's own default is what is left when a diff carries neither. + let prefix = match (dst_prefix, pre_image) { + (Some(prefix), _pre_image) => prefix, + (None, Some(source)) => derived_prefix(source, raw).unwrap_or("b/"), + (None, None) => "b/", + }; + let stripped = raw.strip_prefix(prefix).unwrap_or(raw); + + if stripped.is_empty() { + return None; + } + + Some((Utf8PathBuf::from(stripped), Utf8PathBuf::from(raw))) +} + +/// Works out the post-image prefix from a `diff --git ` header. +/// +/// The two paths differ only in their prefix whenever the file was not renamed, which is what makes +/// the header self-describing: `diff --git i/x.rs w/x.rs` says the post-image prefix is `w/` as +/// plainly as `diff --git a/x.rs b/x.rs` says it is `b/`, and `diff --git x.rs x.rs` says there is +/// none. Returns `None` for a rename, where the two paths carry no common suffix to compare. +fn git_prefix(rest: &str) -> Option { + // A path may contain spaces, so the split is found by trying each one and keeping the split + // whose halves agree once their first segment is removed, rather than by taking two tokens. + for (index, _matched) in rest.match_indices(' ') { + let left = rest.get(..index)?.trim(); + let right = rest.get(index.saturating_add(1)..)?.trim(); + + if left.is_empty() || right.is_empty() { + continue; + } + + if let Some(prefix) = derived_prefix(left, right) { + return Some(prefix.to_owned()); + } + } + + None +} + +/// The prefix on `dst`, given a `src` that names the same file with a prefix of its own. +/// +/// An empty prefix is a real answer: `diff.noprefix` writes the path twice, unadorned. +fn derived_prefix<'a>(src: &str, dst: &'a str) -> Option<&'a str> { + if src == dst { + return Some(""); + } + + let (_src_prefix, src_rest) = split_first_segment(src)?; + let (dst_prefix, dst_rest) = split_first_segment(dst)?; + + (src_rest == dst_rest && !dst_rest.is_empty()).then_some(dst_prefix) +} + +/// Splits a path into its first segment, separator included, and the rest. +fn split_first_segment(path: &str) -> Option<(&str, &str)> { + let slash = path.find('/')?; + let split = slash.checked_add(1)?; + + Some((path.get(..split)?, path.get(split..)?)) +} + +/// Finds the workspace file a diff path refers to, if any. +fn locate(path: &Utf8Path, raw: &Utf8Path, root: &Utf8Path, known: &HashSet<&Utf8Path>, candidates: &[Utf8PathBuf]) -> Option { + // The set answers first, so the filesystem probe is paid only for a path the workspace file + // list does not already hold — which is what makes a miss cost one syscall rather than one + // syscall per candidate spelling per peeled segment. + let known = |candidate: &Utf8Path| known.contains(candidate) || root.join(candidate).exists(); + + for candidate in [path, raw] { + if known(candidate) { + return Some(candidate.to_owned()); + } + + // A diff produced from a subdirectory, or with a prefix this code did not recognize, names + // the file with extra leading directories. Peeling them one at a time finds the file + // without having to know which of the two it was. + let mut rest = candidate.as_str(); + + while let Some((_first, tail)) = split_first_segment(rest) { + rest = tail; + + if rest.is_empty() { + break; + } + + if known(Utf8Path::new(rest)) { + return Some(Utf8PathBuf::from(rest)); + } + } + } + + // Failing all of that, a workspace file the path uniquely ends with is the file meant. Only a + // single match counts: guessing between two would attribute a change to the wrong file. + // + // This one stays a scan, and has to: a suffix match is not a lookup, and there is no key to + // hash. It is reached only when every exact spelling has already failed. + let mut matched = candidates + .iter() + .filter(|file| ends_with_path(path, file) || ends_with_path(raw, file)); + let found = matched.next()?; + + matched.next().is_none().then(|| found.clone()) +} + +/// Whether `path` ends with `suffix` at a segment boundary. +fn ends_with_path(path: &Utf8Path, suffix: &Utf8Path) -> bool { + let (path, suffix) = (path.as_str(), suffix.as_str()); + + if path == suffix { + return true; + } + + path.strip_suffix(suffix).is_some_and(|head| head.ends_with('/')) +} + +/// Reads a hunk's `@@ -a,b +c,d @@` header into `(new_start, old_count, new_count)`. +/// +/// The counts drive the parser's body/metadata decision, so an unreadable header — one missing +/// either range or its leading digits — yields `None` and abandons the hunk rather than guessing. +/// A range with no `,count` covers a single line, per the unified-diff format. +fn hunk_header(rest: &str) -> Option<(u32, u32, u32)> { + let mut fields = rest.trim_start().split(' '); + let old = fields.next()?.strip_prefix('-')?; + let new = fields.next()?.strip_prefix('+')?; + + let (_old_start, old_count) = parse_range(old)?; + let (new_start, new_count) = parse_range(new)?; + + Some((new_start, old_count, new_count)) +} + +/// Splits a `start` or `start,count` range into its two numbers, defaulting a missing count to 1. +fn parse_range(field: &str) -> Option<(u32, u32)> { + let (start, count) = field.split_once(',').unwrap_or((field, "1")); + + Some((start.parse().ok()?, count.parse().ok()?)) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod fuzz { + use camino::Utf8Path; + + use super::Diff; + use crate::testing::token; + + /// Arbitrary text is parsed without panicking, and never claims a line that is not a line. + /// + /// A diff arrives from whatever tooling the user has: a mail client that rewrapped it, a + /// review tool with its own header, or a file that is not a diff at all. Line numbers are + /// accumulated with `u32` arithmetic driven entirely by the input, so a hunk header claiming a + /// start near the maximum is a real input rather than a hypothetical one. + #[test] + fn arbitrary_text_is_parsed_without_panicking() { + bolero::check!().with_type::().for_each(|text| { + let diff = Diff::parse(text); + + for (path, lines) in &diff.touched { + assert!(diff.touches_file(path), "a touched file is not reported as touched: {path}"); + assert!(!lines.is_empty(), "a file was recorded with no changed line: {path}"); + } + }); + } + + /// An added line is still found when the patch is surrounded by text that is not a diff. + /// + /// Prose above and below a patch is the normal case, not the strange one: it is what an email, + /// a pull-request body and a `git format-patch` cover letter all look like. Losing the hunk to + /// it would silently narrow a `--in-diff` run to nothing and report a perfect score for a + /// population nobody looked at. + #[test] + fn an_added_line_survives_surrounding_prose() { + bolero::check!() + .with_type::<(Vec, String, u16)>() + .for_each(|(noise, name, start)| { + let name = token(name); + let start = u32::from(*start).max(1); + + // Nothing generated may look like a header, or it would open a hunk of its own and + // the assertion below would be about a different file than the one it names. + if noise.iter().any(|line| { + line.lines() + .any(|line| line.starts_with(['+', '-', ' ']) || line.starts_with("@@") || line.starts_with("diff --git")) + }) { + return; + } + + let patch = format!("diff --git a/{name}.rs b/{name}.rs\n--- a/{name}.rs\n+++ b/{name}.rs\n@@ -1 +{start} @@\n+let x = 1;"); + let text = format!("{}\n{patch}\n{}", noise.join("\n"), noise.join("\n")); + + let diff = Diff::parse(&text); + let touched = Utf8Path::new(name.as_str()).with_extension("rs"); + + assert!(diff.touches(&touched, start, start), "the added line was lost in {text:?}"); + }); + } +} + +#[cfg(all(test, not(miri)))] +mod tests { + use std::io; + + use super::*; + + const SAMPLE: &str = "\ +diff --git a/src/lib.rs b/src/lib.rs +index 1234567..89abcde 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -10,6 +10,8 @@ fn existing() { + context one + context two ++added at twelve ++added at thirteen + context three +-removed line + context four +"; + + #[test] + fn added_lines_are_numbered_in_the_new_file() { + let diff = Diff::parse(SAMPLE); + let path = Utf8Path::new("src/lib.rs"); + + assert!(diff.touches(path, 12, 12)); + assert!(diff.touches(path, 13, 13)); + assert!(!diff.touches(path, 11, 11)); + assert!(!diff.touches(path, 14, 20)); + } + + #[test] + fn the_git_prefix_is_stripped() { + assert!(Diff::parse(SAMPLE).touches_file(Utf8Path::new("src/lib.rs"))); + } + + // `diff.mnemonicPrefix` replaces `a/` and `b/` with letters that say which side is which, so a + // parser that knows only `b/` resolves nothing, selects nothing, and reports a perfect score + // for a change it never looked at. The header names both prefixes, so nothing has to be assumed. + #[test] + fn a_mnemonic_prefix_is_taken_from_the_header() { + let text = "diff --git i/x.rs w/x.rs\n--- i/x.rs\n+++ w/x.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let diff = Diff::parse(text); + + assert!(diff.touches_file(Utf8Path::new("x.rs")), "{diff:?}"); + assert!(diff.touches(Utf8Path::new("x.rs"), 2, 2)); + } + + // `--src-prefix` and `--dst-prefix` take arbitrary text, and a multi-segment path behind one of + // them must lose only the prefix rather than its own leading directory. + #[test] + fn an_arbitrary_prefix_is_taken_from_the_header() { + let text = "diff --git old/src/lib.rs new/src/lib.rs\n--- old/src/lib.rs\n+++ new/src/lib.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let diff = Diff::parse(text); + + assert!(diff.touches_file(Utf8Path::new("src/lib.rs")), "{diff:?}"); + } + + // `diff.noprefix` writes the path unadorned on both sides, so nothing may be stripped — a file + // that genuinely lives in `b/` would otherwise be renamed out of existence. + #[test] + fn a_diff_written_without_prefixes_keeps_its_leading_directory() { + let text = "diff --git b/lib.rs b/lib.rs\n--- b/lib.rs\n+++ b/lib.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let diff = Diff::parse(text); + + assert!(diff.touches_file(Utf8Path::new("b/lib.rs")), "{diff:?}"); + } + + // A path containing a space cannot be split on whitespace, so the split is found by the halves + // agreeing about which file they name. + #[test] + fn a_path_with_a_space_still_yields_its_prefix() { + let text = "diff --git i/my file.rs w/my file.rs\n+++ w/my file.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let diff = Diff::parse(text); + + assert!(diff.touches_file(Utf8Path::new("my file.rs")), "{diff:?}"); + } + + // A rename has no common suffix to compare, so the header says nothing; the `---` and `+++` + // pair still does not agree either, and git's default is the only thing left to try. + #[test] + fn a_rename_falls_back_to_the_default_prefix() { + let text = "diff --git a/old.rs b/new.rs\n--- a/old.rs\n+++ b/new.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let diff = Diff::parse(text); + + assert!(diff.touches_file(Utf8Path::new("new.rs")), "{diff:?}"); + } + + // A path that is already a workspace file is kept exactly as it is. + #[test] + fn a_path_that_names_a_workspace_file_resolves_to_it() { + let mut diff = Diff::parse(SAMPLE); + + diff.resolve(Utf8Path::new("/nowhere"), &[Utf8PathBuf::from("src/lib.rs")]) + .expect("the path names a workspace file"); + + assert!(diff.touches(Utf8Path::new("src/lib.rs"), 12, 12)); + } + + // A diff produced from a subdirectory, or with a prefix nothing recognized, names the file with + // leading directories the workspace does not have. The workspace file it uniquely ends with is + // the file meant, and matching it is the difference between running the change and running + // nothing. + #[test] + fn an_unrecognized_prefix_is_resolved_by_the_file_it_ends_with() { + let text = "--- q/z/src/lib.rs\n+++ q/z/src/lib.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let mut diff = Diff::parse(text); + + diff.resolve(Utf8Path::new("/nowhere"), &[Utf8PathBuf::from("src/lib.rs")]) + .expect("the path ends with a workspace file"); + + assert!(diff.touches(Utf8Path::new("src/lib.rs"), 2, 2), "{diff:?}"); + } + + // A path that ends with two different workspace files says nothing about which was meant, and + // attributing the change to the wrong one is worse than not attributing it at all. + #[test] + fn an_ambiguous_suffix_resolves_to_nothing() { + let text = "--- q/lib.rs\n+++ q/lib.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let mut diff = Diff::parse(text); + let candidates = [Utf8PathBuf::from("one/lib.rs"), Utf8PathBuf::from("two/lib.rs")]; + + let error = diff + .resolve(Utf8Path::new("/nowhere"), &candidates) + .expect_err("an ambiguous path resolves to nothing"); + + assert!(error.is_usage(), "{error}"); + } + + // The whole reason resolution exists. A diff that names paths and resolves none of them was not + // understood, and the difference between that and a change that touched no code is the + // difference between a run that failed and a run that tested nothing and said it was fine. + #[test] + fn a_diff_whose_paths_resolve_to_nothing_is_an_error() { + let text = "diff --git w/src/lib.rs i/src/lib.rs\n--- w/src/lib.rs\n+++ i/src/lib.rs\n@@ -1 +1,2 @@\n one\n+two\n"; + let mut diff = Diff::parse(text); + + let error = diff + .resolve(Utf8Path::new("/nowhere"), &[Utf8PathBuf::from("other/file.rs")]) + .expect_err("a diff that names nothing this workspace has must not pass for an empty change"); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("src/lib.rs"), "{error}"); + } + + // A change that named nothing at all is not a misunderstanding, so it stays an empty selection + // rather than becoming an error. + #[test] + fn a_diff_that_names_nothing_resolves_to_nothing_without_failing() { + let mut diff = Diff::parse(""); + + diff.resolve(Utf8Path::new("/nowhere"), &[]).expect("an empty diff is not an error"); + + assert!(diff.touched.is_empty()); + } + + // A pull request usually touches a manifest, a changelog and some documentation alongside the + // code. Those are real paths in the workspace and must count as understood, or every such diff + // would fail. + #[test] + fn a_path_that_is_not_a_source_file_still_counts_as_understood() { + let directory = tempfile::tempdir().expect("could not create a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + fs::write(root.join("README.md"), "hello").expect("could not write the file"); + + let text = "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1,2 @@\n one\n+two\n"; + let mut diff = Diff::parse(text); + + diff.resolve(&root, &[]).expect("a file that exists is a file this diff understood"); + + assert!(diff.touches_file(Utf8Path::new("README.md"))); + } + + // The partial failure, which is the dangerous one. One resolvable path is enough to satisfy the + // "resolved nothing" check above, so without this the unresolved source file is dropped + // silently: `Survey::for_build` keeps only the files the diff is believed to touch, so its + // mutants are never generated and the score rises because the population shrank. + #[test] + fn an_unresolvable_source_path_alongside_a_resolvable_one_is_still_an_error() { + let directory = tempfile::tempdir().expect("could not create a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + fs::write(root.join("README.md"), "hello").expect("could not write the file"); + + let text = concat!( + "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1,2 @@\n one\n+two\n", + "diff --git w/src/lib.rs i/src/lib.rs\n--- w/src/lib.rs\n+++ i/src/lib.rs\n@@ -1 +1,2 @@\n one\n+two\n", + ); + let mut diff = Diff::parse(text); + + let error = diff + .resolve(&root, &[Utf8PathBuf::from("other/file.rs")]) + .expect_err("an unresolved source file must not be hidden by a resolvable README"); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("src/lib.rs"), "{error}"); + + // The README is not the complaint, and naming it would send the reader to the wrong file. + assert!(!error.to_string().contains("README.md"), "{error}"); + } + + // The other half of the same rule: a non-source path that does not resolve cannot cost a single + // mutant, so it must not fail a run that otherwise understood the diff. + #[test] + fn an_unresolvable_path_that_is_not_source_does_not_fail_a_diff_that_resolved_something() { + let directory = tempfile::tempdir().expect("could not create a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + fs::write(root.join("README.md"), "hello").expect("could not write the file"); + + let text = concat!( + "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1,2 @@\n one\n+two\n", + "diff --git a/gone.txt b/gone.txt\n--- a/gone.txt\n+++ b/gone.txt\n@@ -1 +1,2 @@\n one\n+two\n", + ); + let mut diff = Diff::parse(text); + + diff.resolve(&root, &[]) + .expect("an unresolved non-source path is not a misunderstanding"); + + assert!(diff.touches_file(Utf8Path::new("README.md"))); + assert!(!diff.touches_file(Utf8Path::new("gone.txt"))); + } + + #[test] + fn a_deleted_file_contributes_nothing() { + let diff = Diff::parse("--- a/gone.rs\n+++ /dev/null\n@@ -1,2 +0,0 @@\n-one\n-two\n"); + + assert!(diff.touched.is_empty()); + } + + #[test] + fn a_region_spanning_a_changed_line_is_selected() { + let diff = Diff::parse(SAMPLE); + + // A mutation site running from line 10 to line 14 encloses the added lines. + assert!(diff.touches(Utf8Path::new("src/lib.rs"), 10, 14)); + } + + #[test] + fn a_diff_without_git_prefixes_is_understood() { + let diff = Diff::parse("--- old.rs\t2020-01-01\n+++ new.rs\t2020-01-02\n@@ -1 +1,2 @@\n one\n+two\n"); + + assert!(diff.touches(Utf8Path::new("new.rs"), 2, 2)); + } + + #[test] + fn several_hunks_in_one_file_all_count() { + let text = "--- a/x.rs\n+++ b/x.rs\n@@ -1,1 +1,2 @@\n one\n+two\n@@ -50,1 +51,2 @@\n fifty\n+fifty two\n"; + let diff = Diff::parse(text); + let path = Utf8Path::new("x.rs"); + + assert!(diff.touches(path, 2, 2)); + assert!(diff.touches(path, 52, 52)); + assert!(!diff.touches(path, 30, 40)); + } + + #[test] + fn several_files_are_kept_apart() { + let text = "--- a/x.rs\n+++ b/x.rs\n@@ -1 +1,2 @@\n one\n+x change\n--- a/y.rs\n+++ b/y.rs\n@@ -1 +1,2 @@\n one\n+y change\n"; + let diff = Diff::parse(text); + + assert!(diff.touches(Utf8Path::new("x.rs"), 2, 2)); + assert!(diff.touches(Utf8Path::new("y.rs"), 2, 2)); + assert!(!diff.touches_file(Utf8Path::new("z.rs"))); + } + + #[test] + fn an_empty_diff_touches_nothing() { + assert!(Diff::parse("").touched.is_empty()); + } + + // `--in-diff ` is how a pull-request job scopes a run, so reading the diff off disk has + // to produce the same answer as parsing the same bytes in memory. + #[test] + fn a_diff_is_read_from_a_file() { + let dir = tempfile::tempdir().expect("could not create a temporary directory"); + let path = Utf8PathBuf::from_path_buf(dir.path().join("change.patch")).expect("the temporary path is not UTF-8"); + + fs::write(&path, SAMPLE).expect("could not write the diff"); + + let diff = Diff::read(&path).expect("could not read the diff"); + + assert!(!diff.touched.is_empty()); + assert!(diff.touches_file(Utf8Path::new("src/lib.rs"))); + } + + // A diff that does not exist is a usage mistake a caller has to be told about, not an empty + // selection that would silently mutate nothing and report a perfect score. + #[test] + fn a_missing_diff_file_is_an_error_naming_the_path() { + let error = Diff::read(Utf8Path::new("no/such/change.patch")).expect_err("a missing diff must not parse"); + + assert!(error.to_string().contains("no/such/change.patch"), "{error}"); + } + + // `--in-diff -` is the form `git diff | cargo gamma run --in-diff -` uses, and it has to read + // the whole stream rather than the first line of it. + #[test] + fn a_diff_is_read_from_standard_input() { + let mut input = SAMPLE.as_bytes(); + let diff = Diff::read_from(Utf8Path::new("-"), &mut input).expect("could not read the diff"); + + assert!(diff.touches_file(Utf8Path::new("src/lib.rs"))); + } + + // A stream that fails half way through must be reported rather than silently truncated into a + // diff that touches less than the real change did. + #[test] + fn a_failing_standard_input_is_an_error() { + struct Broken; + + impl Read for Broken { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + Err(io::Error::other("the pipe broke")) + } + } + + let error = Diff::read_from(Utf8Path::new("-"), &mut Broken).expect_err("a broken pipe must not parse"); + + assert!(error.to_string().contains("standard input"), "{error}"); + } + + // A hunk header the parser cannot read leaves it with no idea which line the following `+` + // lines are at, so the file is abandoned rather than credited with invented line numbers. + #[test] + fn an_unreadable_hunk_header_abandons_the_file() { + let text = "--- a/x.rs\n+++ b/x.rs\n@@ nonsense @@\n+added\n"; + let diff = Diff::parse(text); + + assert!(diff.touched.is_empty(), "an unparsable hunk must not contribute lines"); + } + + // Real patches carry lines that belong to neither the pre- nor the post-image. They must not + // advance the line counter, or every mutant after them would be attributed to the wrong line. + #[test] + fn a_line_that_is_not_part_of_the_hunk_does_not_advance_the_count() { + let text = "--- a/x.rs\n+++ b/x.rs\n@@ -1,2 +1,2 @@\n one\n-old\n\\ No newline at end of file\n+new\n"; + let diff = Diff::parse(text); + + // `one` is line 1, the deletion has no post-image line, the `\` marker is not a line + // either, so the addition is line 2 rather than line 3. + assert!(diff.touches(Utf8Path::new("x.rs"), 2, 2), "the addition landed on the wrong line"); + } + + // An added line whose own text begins with `+++ ` renders as a raw `+++ ...` line inside the + // hunk. Recognizing `+++` as a file header only outside a hunk body keeps it an ordinary + // addition; treating it as a header would silently drop the change and misread the file path. + #[test] + fn an_added_line_that_looks_like_a_file_header_is_still_an_addition() { + let text = "--- a/x.rs\n+++ b/x.rs\n@@ -1,1 +1,2 @@\n one\n+++ nested;\n"; + let diff = Diff::parse(text); + + // The added source line is `++ nested;`; the leading `+` marks it added, and the parser + // must stay on `x.rs` at line 2 rather than jumping to a file called `nested;`. + assert!( + diff.touches(Utf8Path::new("x.rs"), 2, 2), + "the addition was mistaken for a `+++` header" + ); + assert!( + !diff.touches_file(Utf8Path::new("nested;")), + "a hunk-body line was read as a file header" + ); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/glob.rs b/crates/cargo-gamma-lib/src/discover/glob.rs new file mode 100644 index 000000000..82ba3dc53 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/glob.rs @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Matching source paths against the include and exclude patterns a selection carries. + +use core::cell::RefCell; +use std::borrow::Cow; + +thread_local! { + static SCRATCH: RefCell = RefCell::new(Scratch::default()); +} + +#[derive(Debug, Default)] +struct Scratch { + text: Vec, + next: Vec, + current: Vec, +} + +/// A normalized, tokenized glob that can match many paths without recompiling the pattern. +#[derive(Debug, Clone)] +pub(crate) struct Glob { + tokens: Box<[Token]>, + basename_only: bool, +} + +impl Glob { + #[must_use] + pub(crate) fn new(pattern: &str) -> Self { + let pattern = normalized(pattern); + + Self { + tokens: tokenize(&pattern).into_boxed_slice(), + basename_only: !pattern.contains('/'), + } + } + + #[must_use] + pub(crate) fn matches(&self, path: &str) -> bool { + let path = normalized(path); + let text = if self.basename_only { + path.rsplit('/').next().unwrap_or(&path) + } else { + &path + }; + + SCRATCH.with_borrow_mut(|scratch| glob_match(&self.tokens, text, scratch)) + } +} + +/// Matches a path against a glob pattern supporting `*`, `**` and `?`. +/// +/// A pattern with no separator matches against the file name alone, so `--file lexer.rs` does what +/// it looks like it does regardless of how deep the file is. +/// +/// Both sides are normalised to `/` first. Paths are walked with the platform's own separator, so +/// on Windows they arrive with `\`, whereas patterns are written with `/` — they are typed on a +/// command line, checked into a config file and shared across platforms. Without normalisation +/// every pattern silently matches nothing there, and the run reports zero mutants and succeeds. +#[must_use] +pub fn matches_glob(pattern: &str, path: &str) -> bool { + Glob::new(pattern).matches(path) +} + +/// Rewrites a path or pattern so that `/` is the only separator. +/// +/// Only done on Windows: a backslash is a legal character in a Unix file name, and rewriting it +/// there would make `--file "odd\name.rs"` match a file that does not exist. +pub(super) fn normalize_separators(text: &str) -> String { + normalized(text).into_owned() +} + +fn normalized(text: &str) -> Cow<'_, str> { + if cfg!(windows) { + Cow::Owned(text.replace('\\', "/")) + } else { + Cow::Borrowed(text) + } +} + +#[derive(Debug, Clone, Copy)] +enum Token { + Literal(char), + One, + Star, + RecursiveStar, + RecursivePrefix, +} + +/// Dynamic-programming glob matcher in which `*` stops at a separator and `**` does not. +/// +/// Each token/path-scalar pair is considered at most once, avoiding the exponential suffix +/// retries of recursive backtracking. +fn glob_match(tokens: &[Token], text: &str, scratch: &mut Scratch) -> bool { + scratch.text.clear(); + scratch.text.extend(text.chars()); + let width = scratch.text.len() + 1; + scratch.next.clear(); + scratch.next.resize(width, false); + scratch.current.clear(); + scratch.current.resize(width, false); + scratch.next[scratch.text.len()] = true; + + for token in tokens.iter().rev() { + scratch.current.fill(false); + + match token { + Token::Literal(expected) => { + for (index, character) in scratch.text.iter().enumerate() { + scratch.current[index] = character == expected && scratch.next[index + 1]; + } + } + Token::One => { + for (index, character) in scratch.text.iter().enumerate() { + scratch.current[index] = *character != '/' && scratch.next[index + 1]; + } + } + Token::Star => { + for index in (0..=scratch.text.len()).rev() { + scratch.current[index] = + scratch.next[index] || (index < scratch.text.len() && scratch.text[index] != '/' && scratch.current[index + 1]); + } + } + Token::RecursiveStar => { + for index in (0..=scratch.text.len()).rev() { + scratch.current[index] = scratch.next[index] || (index < scratch.text.len() && scratch.current[index + 1]); + } + } + Token::RecursivePrefix => { + let mut directory_match = false; + + for index in (0..=scratch.text.len()).rev() { + if index < scratch.text.len() && scratch.text[index] == '/' && scratch.next[index + 1] { + directory_match = true; + } + + scratch.current[index] = scratch.next[index] || directory_match; + } + } + } + + core::mem::swap(&mut scratch.next, &mut scratch.current); + } + + scratch.next[0] +} + +fn tokenize(pattern: &str) -> Vec { + let characters: Vec<_> = pattern.chars().collect(); + let mut tokens = Vec::with_capacity(characters.len()); + let mut index = 0; + + while index < characters.len() { + match characters[index] { + '?' => { + tokens.push(Token::One); + index += 1; + } + '*' => { + let start = index; + while index < characters.len() && characters[index] == '*' { + index += 1; + } + + if index - start == 1 { + tokens.push(Token::Star); + } else if characters.get(index) == Some(&'/') { + tokens.push(Token::RecursivePrefix); + index += 1; + } else { + tokens.push(Token::RecursiveStar); + } + } + literal => { + tokens.push(Token::Literal(literal)); + index += 1; + } + } + } + + tokens +} diff --git a/crates/cargo-gamma-lib/src/discover/hints.rs b/crates/cargo-gamma-lib/src/discover/hints.rs new file mode 100644 index 000000000..db910fdf8 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/hints.rs @@ -0,0 +1,707 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The score-neutral part of the record, in a file a workspace can check in. +//! +//! Everything the tool learns about how to be fast lives under `target/`, and CI deletes that on +//! every run. So every CI run is a cold run: the sweep is unguided and the killer map is empty on +//! exactly the runs that cost the most. This is the same knowledge in a form that survives a clean +//! checkout — reviewed, committed, and consulted automatically. +//! +//! Only tiers that cannot move a score are allowed in here, and the file is defined by that +//! property rather than by being a relocated record: +//! +//! - **Probes** — which test caught each mutant. Never believed; the named test is actually run, +//! and a probe that does not convict costs one filtered test process. +//! - **Build order** — which mutants failed to compile for whoever promoted the file. Never +//! believed either: every one of them is compiled and the compiler decides, exactly as it would +//! have without the hint. All it changes is which mutants are offered to the compiler first. +//! +//! A verdict is *not* allowed in, and that is the whole line this file walks. Adopting a carried +//! kill would settle part of the score out of another run's knowledge, which is why local incremental +//! caching requires matching digests; a sidecar that quietly did it would make every reported score unfalsifiable. Unviability +//! is admitted only after being demoted to an ordering hint, because a checked-in envelope will +//! differ from the run reading it almost always — see [`Tier::Ordering`]. + +#[cfg(test)] +use core::cell::RefCell; +use std::fs; +use std::io::ErrorKind; + +use camino::{Utf8Path, Utf8PathBuf}; +use serde::{Deserialize, Serialize}; + +use super::record::{ContextDigest, Killer, RunRecord, Tier}; +use crate::elements::Publication; +use crate::error::error; +use crate::model::{Mutant, MutantId, Outcome}; +use crate::{HashMap, HashSet, Result}; + +/// The artifact's file name. +const FILE: &str = "gamma-hints.json"; + +/// What the artifact format is; a file written by any other version is ignored rather than read. +/// +/// Ignored rather than migrated, deliberately. This file is an optimization that must be safe to +/// delete, so the cost of refusing to read it is time; the cost of reading a format whose fields +/// have changed meaning is a wrong hint, and there is no version of that trade worth taking. +const VERSION: u32 = 1; + +/// Where the artifact lives for a workspace rooted at `root`. +#[must_use] +pub fn path(root: &Utf8Path) -> Utf8PathBuf { + root.join(FILE) +} + +/// The checked-in hints for a workspace. +/// +/// Ordered by source file and then by mutant id, which is not a detail. A population in the tens of +/// thousands, regenerated on a schedule, otherwise produces a diff nobody can review — and an +/// unreviewable file in version control is a liability rather than an asset. Both keys are needed: +/// the file is what makes a diff readable against a change, and the id is what makes the order +/// total. +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Hints { + /// The format this was written in. + version: u32, + + /// What wrote it, which is the provenance a reviewer asks for first. + tool: String, + + /// The build context the promoted record was written under. + /// + /// Provenance and never a gate. Nothing here is read to decide whether a hint applies — the + /// tiers in this file require no term of the context — but a reader comparing two branches, or + /// wondering why a promotion produced a different file, has nothing else to go on. Carrying it + /// costs a handful of hashes and answers "whose machine was this taken on". + context: ContextDigest, + + /// One entry per mutant with something to say about it, ordered by file and then by id. + mutants: Vec, +} + +/// What the artifact remembers about one mutant. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Hint { + /// The workspace-relative file the mutant lives in, which is the artifact's primary sort key. + /// + /// Carried even though nothing reads it at run time — the run keys by mutant id — because a + /// diff of this file is read by people, and a list of content hashes with no file beside them + /// tells a reviewer nothing about what changed. + file: Utf8PathBuf, + + /// The mutant's content-addressed id. + id: MutantId, + + /// The test that caught it, when one did. + #[serde(default, skip_serializing_if = "Option::is_none")] + killer: Option, + + /// Whether it failed to compile for the run that was promoted. + /// + /// A hint about *order*, never a filter. See [`Tier::Ordering`]: the mutant is built and judged + /// exactly as it would have been, and all this decides is that it is offered to the compiler + /// early, where a mutant that does turn out to be unviable costs one round instead of hiding + /// behind another one for several. + #[serde(default, skip_serializing_if = "is_not_set")] + unviable: bool, +} + +/// Whether a flag is at its default, so that the common entry serializes without it. +/// +/// Takes a reference because `skip_serializing_if` hands the field to it by reference and will not +/// accept any other shape. +#[expect(clippy::trivially_copy_pass_by_ref, reason = "the signature is dictated by serde")] +const fn is_not_set(flag: &bool) -> bool { + !*flag +} + +/// What a promotion changed, so the command can say it rather than claim it. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Promotion { + /// How many mutants the artifact now names. + pub mutants: usize, + + /// How many of them carry a killing test. + pub probes: usize, + + /// How many of them are offered to the build as likely to fail. + pub ordering: usize, + + /// Whether the bytes on disk changed. + pub changed: bool, +} + +impl Hints { + /// Reads the artifact for a workspace, or returns an empty one. + /// + /// Every failure is an empty set of hints: missing, unreadable, not JSON, a format from another + /// version, or a file some other tool put at that name. That is the same contract the run + /// record has, and for the same reason — nothing here can move a verdict, so being unable to + /// read it may only ever cost the time it would have saved. A run that failed over it would + /// have turned an optimization into a dependency, which is exactly what a file living in + /// version control must never become. + #[must_use] + pub fn load(root: &Utf8Path) -> Self { + Self::read(&path(root)).unwrap_or_default() + } + + /// Whether this workspace has never had a checked-in hints artifact. + /// + /// An unreadable, corrupt or foreign-version file is present even though it cannot provide + /// hints, so it must not trigger advice to create the file that is already there. + #[must_use] + pub(crate) fn is_missing(root: &Utf8Path) -> bool { + matches!(fs::metadata(path(root).as_std_path()), Err(cause) if cause.kind() == ErrorKind::NotFound) + } + + /// Reads and validates the artifact at `path`, or nothing when it cannot be trusted. + fn read(path: &Utf8Path) -> Option { + let text = fs::read_to_string(path.as_std_path()).ok()?; + let hints = serde_json::from_str::(&text).ok()?; + + (hints.version == VERSION).then_some(hints) + } + + /// The tests to try first, keyed by mutant id. + #[must_use] + pub fn probes(&self) -> HashMap { + self.mutants + .iter() + .filter_map(|hint| hint.killer.clone().map(|killer| (hint.id.clone(), killer))) + .collect() + } + + /// The mutants to offer the compiler first, in the artifact's own order. + #[must_use] + pub fn ordering(&self) -> Vec<&str> { + self.mutants + .iter() + .filter(|hint| hint.unviable) + .map(|hint| hint.id.as_str()) + .collect() + } + + /// Whether it holds nothing at all. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.mutants.is_empty() + } + + /// What this artifact holds, for a caller that wants to report it without writing it. + #[must_use] + pub fn counts(&self) -> Promotion { + self.promotion(false) + } + + /// Builds the artifact from a scratch record and the population as it stands now. + /// + /// The population is what makes this a promotion rather than a copy. A record accumulates + /// entries for mutants that have since been edited away, and committing those would grow the + /// file without bound and fill its diff with ids nobody can locate; joining against the mutants + /// that exist today drops them, and gives every surviving entry the file it lives in, which is + /// what the ordering is for. + /// + /// Verdicts are not consulted. The only knowledge read here cannot move a score: probes are + /// rerun before use, and [`Tier::Ordering`] is unviability demoted to an order to build in. + #[must_use] + pub fn promoted(record: &RunRecord, population: &[Mutant]) -> Self { + let probes = record.probes(); + + // The one place the admission rule is applied, so that widening it means editing a function + // whose name says what it decides. + let unviable: HashSet<&str> = record + .entries() + .filter(|(_id, outcome)| tier_of(*outcome) == Some(Tier::Ordering)) + .map(|(id, _outcome)| id) + .collect(); + + let mut mutants: Vec = population + .iter() + .filter_map(|mutant| { + let hint = Hint { + file: mutant.file.to_path_buf(), + id: mutant.id.clone(), + killer: probes.get(&mutant.id).cloned(), + unviable: unviable.contains(mutant.id.as_str()), + }; + + (hint.killer.is_some() || hint.unviable).then_some(hint) + }) + .collect(); + + // By file, then by id. A population can hold the same id twice only if the same mutant was + // scanned twice, which a shard or an overlapping selection can do, so the duplicates are + // dropped after sorting rather than assumed away. + mutants.sort_by(|left, right| left.file.cmp(&right.file).then_with(|| left.id.cmp(&right.id))); + mutants.dedup_by(|left, right| left.file == right.file && left.id == right.id); + + Self { + version: VERSION, + tool: format!("cargo-gamma {}", env!("CARGO_PKG_VERSION")), + context: record.context().clone(), + mutants, + } + } + + /// Writes the artifact to `path`, reads it back, and conditionally puts the old one back if it did not survive. + /// + /// The write is atomic — staged beside the destination and renamed onto it — so an interrupted + /// promotion leaves the previous file rather than half of a new one. That matters more here + /// than for a scratch file: this one is in version control, and a truncated JSON file that a + /// later run silently treats as "no hints" is a slow run nobody can explain. + /// + /// Reading it back is the same discipline `suppress` applies to the source it edits. Verifying + /// what was written is the only thing that distinguishes "the tool wrote a file" from "the file + /// says what the tool meant", and the cost is one read of a file that was just written. + /// + /// # Errors + /// + /// Returns the reason when the file cannot be written, cannot be read back, does not parse back + /// to what was written, or was already there and could not be read. Unlike every automatic path + /// through this module, a promotion is something somebody asked for, so a failure is reported + /// rather than absorbed. + pub fn write(&self, path: &Utf8Path) -> Result { + let text = self.rendered()?; + let workspace = path.parent().unwrap_or_else(|| Utf8Path::new(".")); + + // Absent and unreadable are different answers, and collapsing them into one `None` is what + // turns the rollback below into a delete: undoing a creation means removing the file, and + // a file that was there all along is not a creation. A file this cannot restore is a file + // it must not replace, so an existing artifact it cannot read stops the promotion outright. + let before = match fs::read_to_string(path.as_std_path()) { + Ok(text) => Some(text), + Err(cause) if cause.kind() == ErrorKind::NotFound => None, + Err(cause) => { + return Err(error!("`{path}` is already there and could not be read, so it must not be replaced").caused_by(cause)); + } + }; + + if before.as_deref() == Some(text.as_str()) { + return Ok(self.promotion(false)); + } + + match crate::elements::write_if_unchanged(workspace, path, before.as_deref(), &text) + .map_err(|cause| error!("could not write `{path}`").caused_by(cause))? + { + Publication::Conflict => { + return Err(error!( + "`{path}` changed while these hints were being promoted; the newer generation was left alone" + )); + } + Publication::Published => {} + // The new hints are visible but the directory entry was not made durable. Keep the + // failure rather than treating a successful read-back as a durable promotion. + Publication::PublishedUndurable(cause) => return Err(cause), + } + + after_publication(path); + + match Self::verified(path, self) { + Ok(()) => Ok(self.promotion(true)), + Err(cause) => Err(restored(workspace, path, before.as_deref(), &text, cause)), + } + } + + /// The artifact as it goes to disk. + fn rendered(&self) -> Result { + let mut text = serde_json::to_string_pretty(self) + .map_err(|cause| error!("the hints could not be serialized; please report this").caused_by(cause))?; + + // A trailing newline, because every other text file in a repository has one and a diff of a + // file without one says "\ No newline at end of file" on every change to its last entry. + text.push('\n'); + + Ok(text) + } + + /// Reads back what was written and checks that it is what was meant. + fn verified(path: &Utf8Path, intended: &Self) -> Result<()> { + let Some(written) = Self::read(path) else { + return Err(error!("`{path}` could not be read back after being written")); + }; + + if &written == intended { + return Ok(()); + } + + Err(error!("`{path}` does not hold what was written to it")) + } + + /// What this artifact would report having promoted. + fn promotion(&self, changed: bool) -> Promotion { + Promotion { + mutants: self.mutants.len(), + probes: self.mutants.iter().filter(|hint| hint.killer.is_some()).count(), + ordering: self.mutants.iter().filter(|hint| hint.unviable).count(), + changed, + } + } +} + +/// Puts back whatever was at `path` before a promotion that could not be verified. +/// +/// The restoration is generation-aware. A successful later promotion must survive a first +/// promotion's failed verification, so this restores only if the path still holds the exact bytes +/// this invocation published. The conditional helpers serialize cooperating processes with the +/// same locked generation protocol as the final comparison; a stale rollback therefore reports +/// its conflict instead of replacing or removing somebody else's artifact. +fn restored( + workspace: &Utf8Path, + path: &Utf8Path, + before: Option<&str>, + published: &str, + cause: crate::error::Error, +) -> crate::error::Error { + let restored = before.map_or_else( + || crate::elements::remove_if_unchanged(workspace, path, published), + |text| crate::elements::write_if_unchanged(workspace, path, Some(published), text), + ); + + match restored { + Ok(Publication::Published) => cause, + Ok(Publication::Conflict) => { + error!("`{path}` changed after this promotion was published, so its later generation was left alone").caused_by(cause) + } + Ok(Publication::PublishedUndurable(failure)) => { + error!("`{path}` was put back after a promotion that could not be verified, but its directory could not be synced ({failure})") + .caused_by(cause) + } + Err(failure) => error!("`{path}` could not be put back after a promotion that could not be verified ({failure})").caused_by(cause), + } +} + +#[cfg(test)] +type PublicationHook = Box; + +#[cfg(test)] +thread_local! { + static AFTER_PUBLICATION: RefCell> = const { RefCell::new(None) }; +} + +/// Runs `hook` after this thread's next hints publication and before its read-back verification. +#[cfg(test)] +fn after_next_publication(hook: impl FnOnce(&Utf8Path) + 'static) { + AFTER_PUBLICATION.with(|next| *next.borrow_mut() = Some(Box::new(hook))); +} + +#[cfg(test)] +fn after_publication(path: &Utf8Path) { + let hook = AFTER_PUBLICATION.with(|next| next.borrow_mut().take()); + + if let Some(hook) = hook { + hook(path); + } +} + +#[cfg(not(test))] +const fn after_publication(_path: &Utf8Path) {} + +/// Whether an outcome is one the artifact is allowed to carry, and as which tier. +/// +/// Stated here rather than at the call site so that widening it is a deliberate edit to a function +/// whose name says what the rule is. The rule is the whole safety argument of this file: a verdict +/// carried into version control and adopted automatically would settle part of a score out of +/// somebody else's run. +#[must_use] +const fn tier_of(outcome: Outcome) -> Option { + match outcome { + Outcome::CompileError => Some(Tier::Ordering), + _other => None, + } +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::super::record; + use super::*; + use crate::fixtures; + use crate::testing::workdir; + + fn mutant(id: &str, file: &str) -> Mutant { + Mutant { + id: id.to_owned().into(), + file: (Utf8PathBuf::from(file)).into(), + ..fixtures::mutant() + } + } + + fn killer(test: &str) -> Killer { + Killer { + package: "subject".to_owned(), + target: "lib".to_owned(), + test: test.to_owned(), + } + } + + /// A workspace holding one source file, and a record base pointing at the same directory. + fn workspace(prefix: &str) -> (tempfile::TempDir, Utf8PathBuf) { + let dir = workdir(prefix); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("the work directory should be UTF-8"); + + fs::create_dir_all(root.join("src")).expect("the source directory should be creatable"); + fs::write(root.join("src/lib.rs"), "fn add() {}").expect("the source should be writable"); + + (dir, root) + } + + /// A build context, which the artifact carries as provenance and never reads as a gate. + fn context_of() -> ContextDigest { + record::context(&record::Context { + toolchain: Some("1.90.0"), + ..record::Context::default() + }) + .expect("a named toolchain gives a context") + } + + /// A record holding one unviable mutant and one probe. + fn recorded(root: &Utf8Path) -> RunRecord { + let mut unviable = mutant("unviable", "src/lib.rs"); + + unviable.outcome = Outcome::CompileError; + + RunRecord::from_run(root, &[unviable], &context_of(), &[root.join("src")]).store(root, root); + RunRecord::store_probes(root, &core::iter::once(("killed".into(), killer("tests::caught"))).collect()); + + RunRecord::load(root) + } + + #[test] + fn a_missing_artifact_is_no_hints_at_all() { + let (_dir, root) = workspace("hints-absent-"); + + assert!(Hints::load(&root).is_empty()); + assert!(Hints::is_missing(&root)); + } + + /// A file that cannot be parsed must cost the run nothing but the speed-up it would have given. + #[test] + fn a_corrupt_artifact_is_no_hints_at_all() { + let (_dir, root) = workspace("hints-corrupt-"); + + fs::write(path(&root).as_std_path(), "{ not json").expect("the artifact should be writable"); + + assert!(Hints::load(&root).is_empty()); + assert!(!Hints::is_missing(&root)); + } + + /// Somebody else's JSON at this name is not this tool's file, and must not be read as one. + #[test] + fn a_foreign_artifact_is_no_hints_at_all() { + let (_dir, root) = workspace("hints-foreign-"); + + fs::write(path(&root).as_std_path(), r#"{"version":99,"tool":"other","mutants":[]}"#).expect("writable"); + + assert!(Hints::load(&root).is_empty()); + assert!(!Hints::is_missing(&root)); + } + + #[test] + fn promotion_carries_the_two_score_neutral_tiers_and_nothing_else() { + let (_dir, root) = workspace("hints-promote-"); + let mut unviable = mutant("unviable", "src/lib.rs"); + + unviable.outcome = Outcome::CompileError; + + let record = { + let mut killed = mutant("killed", "src/lib.rs"); + + killed.outcome = Outcome::Killed; + killed.killed_by = Some("tests::caught".to_owned()); + + RunRecord::from_run(&root, &[unviable, killed], &context_of(), &[root.join("src")]).store(&root, &root); + RunRecord::store_probes(&root, &core::iter::once(("killed".into(), killer("tests::caught"))).collect()); + + RunRecord::load(&root) + }; + + let hints = Hints::promoted(&record, &[mutant("killed", "src/lib.rs"), mutant("unviable", "src/lib.rs")]); + + assert_eq!(hints.probes().get("killed"), Some(&killer("tests::caught"))); + assert_eq!(hints.ordering(), vec!["unviable"]); + + // The kill itself must not be in the file in any form a run could adopt. + let text = hints.rendered().expect("the hints should serialize"); + + assert!(!text.contains("killed\":"), "{text}"); + assert!(!text.contains("outcome"), "a verdict reached the artifact: {text}"); + } + + /// A promoted hint whose mutant no longer exists would grow the file forever and fill its diff + /// with ids nobody can locate. + #[test] + fn promotion_drops_hints_for_mutants_the_population_no_longer_holds() { + let (_dir, root) = workspace("hints-gone-"); + let record = recorded(&root); + + let hints = Hints::promoted(&record, &[mutant("survivor", "src/lib.rs")]); + + assert!(hints.is_empty(), "a hint for a mutant nobody scanned was promoted"); + } + + /// The file is reviewed, so its order has to be one a reviewer can follow, and the same on + /// every machine that regenerates it. + #[test] + fn promotion_orders_by_file_and_then_by_id() { + let (_dir, root) = workspace("hints-order-"); + + fs::write(root.join("src/other.rs"), "fn other() {}").expect("the source should be writable"); + + let population = [ + mutant("zeta", "src/other.rs"), + mutant("alpha", "src/other.rs"), + mutant("mid", "src/lib.rs"), + ]; + + let mut unviable: Vec = population.to_vec(); + + for entry in &mut unviable { + entry.outcome = Outcome::CompileError; + } + + RunRecord::from_run(&root, &unviable, &context_of(), &[root.join("src")]).store(&root, &root); + + let record = RunRecord::load(&root); + let hints = Hints::promoted(&record, &population); + let order: Vec<&str> = hints.mutants.iter().map(|hint| hint.id.as_str()).collect(); + + assert_eq!(order, vec!["mid", "alpha", "zeta"]); + + // Regenerating from the same inputs has to produce the same bytes, or a scheduled + // regeneration commits a diff on every run. + let again = Hints::promoted(&record, &population); + + assert_eq!(hints.rendered().unwrap(), again.rendered().unwrap()); + } + + #[test] + fn a_written_artifact_reads_back_as_what_was_written() { + let (_dir, root) = workspace("hints-write-"); + let record = recorded(&root); + let population = vec![mutant("unviable", "src/lib.rs"), mutant("killed", "src/lib.rs")]; + let hints = Hints::promoted(&record, &population); + let promotion = hints.write(&path(&root)).expect("the artifact should be writable"); + + assert!(promotion.changed); + assert_eq!(promotion.mutants, 2); + assert_eq!(promotion.probes, 1); + assert_eq!(promotion.ordering, 1); + assert_eq!(Hints::load(&root), hints); + assert!(!Hints::is_missing(&root)); + } + + #[test] + fn a_hints_write_uses_the_workspace_lock() { + let (_dir, root) = workspace("hints-lock-"); + let record = recorded(&root); + let hints = Hints::promoted(&record, &[mutant("unviable", "src/lib.rs")]); + let _held = crate::exec::claim_workspace(&root).expect("the workspace lock should be available"); + + let error = hints + .write(&path(&root)) + .expect_err("the existing workspace claim must block the write"); + + assert!(error.to_string().contains("already using"), "{error}"); + } + + /// A directory sync failure happens after the hints file is visible. Promotion still reports + /// the durability failure instead of calling that visible generation a successful promotion. + #[test] + fn a_post_rename_hints_sync_failure_is_reported() { + let (_dir, root) = workspace("hints-sync-failure-"); + let record = recorded(&root); + let hints = Hints::promoted(&record, &[mutant("unviable", "src/lib.rs")]); + + crate::elements::fail_next_directory_sync(); + + let error = hints.write(&path(&root)).expect_err("the post-rename sync fails"); + + assert!(error.to_string().contains("injected directory sync failure"), "{error}"); + assert_eq!(Hints::load(&root), hints, "the published hints must still be readable"); + } + + /// The first writer publishes, then a second writer completes before the first can read back. + /// The first must report its failed verification without restoring or removing the second + /// writer's generation. + #[test] + fn a_failed_hints_rollback_leaves_a_later_successful_promotion_intact() { + let (_dir, root) = workspace("hints-rollback-generation-"); + let record = recorded(&root); + let first = Hints::promoted(&record, &[mutant("unviable", "src/lib.rs")]); + let mut second = first.clone(); + + second.tool = "cargo-gamma second writer".to_owned(); + + let later = second.clone(); + after_next_publication(move |destination| { + assert!(later.write(destination).expect("the later promotion").changed); + }); + + let error = first.write(&path(&root)).expect_err("the later generation changes read-back"); + + assert!(error.to_string().contains("later generation was left alone"), "{error}"); + assert_eq!(Hints::load(&root), second, "the first rollback removed the later promotion"); + } + + /// "There is no file" and "there is a file I cannot read" are different answers, and reading + /// both as absence turns the rollback into a delete of a checked-in artifact. A file this + /// cannot restore is a file it must not replace. + #[test] + fn an_artifact_that_is_there_but_unreadable_is_not_replaced_and_not_deleted() { + let (_dir, root) = workspace("hints-unreadable-"); + let destination = path(&root); + + // Bytes that are not UTF-8: present, readable as bytes, and refused by the text read — + // portable, and needing no permission model to arrange. + fs::create_dir_all(destination.parent().expect("a parent").as_std_path()).expect("the directory"); + fs::write(destination.as_std_path(), [0xff_u8, 0xfe, 0xfd]).expect("the artifact"); + + let record = recorded(&root); + let population = [mutant("unviable", "src/lib.rs")]; + let hints = Hints::promoted(&record, &population); + + let cause = hints.write(&destination).expect_err("an artifact that cannot be read back"); + + assert!(cause.to_string().contains("must not be replaced"), "{cause}"); + assert_eq!( + fs::read(destination.as_std_path()).expect("the artifact afterwards"), + [0xff_u8, 0xfe, 0xfd], + "a promotion that could not read the artifact removed it" + ); + } + + /// Rewriting the same content is not a change, so a regeneration in CI does not look like one. + #[test] + fn writing_the_same_artifact_twice_reports_no_change() { + let (_dir, root) = workspace("hints-idempotent-"); + let record = recorded(&root); + let population = [mutant("unviable", "src/lib.rs")]; + let hints = Hints::promoted(&record, &population); + + assert!(hints.write(&path(&root)).expect("writable").changed); + assert!(!hints.write(&path(&root)).expect("writable").changed); + } + + /// The artifact must never carry a tier that could settle part of a score. + #[test] + fn only_unviability_is_admitted_as_a_tier() { + assert_eq!(tier_of(Outcome::CompileError), Some(Tier::Ordering)); + + for refused in [ + Outcome::Killed, + Outcome::Survived, + Outcome::Timeout, + Outcome::Ignored, + Outcome::NotBuilt, + Outcome::NoCoverage, + Outcome::OutOfMemory, + Outcome::Flaky, + Outcome::Pending, + ] { + assert_eq!(tier_of(refused), None, "{refused:?} would have been carried"); + } + } +} diff --git a/crates/cargo-gamma-lib/src/discover/killers.rs b/crates/cargo-gamma-lib/src/discover/killers.rs new file mode 100644 index 000000000..7934dcb02 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/killers.rs @@ -0,0 +1,328 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Which test functions the workspace's sources declare right now. +//! +//! A kill is a claim about the suite, not about the code: the mutant's identity hashes the file, the +//! item path, the mutator and the replacement, and none of those change when somebody deletes the +//! test that did the killing. Without a check the next run hashes the mutant identically, carries +//! the kill, and reports coverage that no longer exists — silently, permanently, and in the one +//! direction a tool like this must never err in. +//! +//! The names are read out of the sources rather than out of a harness. Asking a harness would mean +//! building the suite, and the decision has to be made before anything is instrumented — which is +//! the whole point of not running these mutants. Parsing is what discovery pays for anyway. +//! +//! That makes the index approximate in exactly one direction. A test generated by a macro this +//! cannot see does not appear here, so the mutant it killed is re-run. Re-running costs time and +//! reaches the same verdict; carrying a kill that is no longer true costs the credibility of every +//! score the tool prints, so the approximation is spent that way deliberately. + +use camino::{Utf8Path, Utf8PathBuf}; +use syn::{Attribute, Item, ItemMod}; + +use crate::discover::record::digest; +use crate::parse::SourceFile; +use crate::{HashMap, HashSet}; + +/// The test functions the current sources declare. +#[derive(Debug, Default)] +pub struct Killers { + /// Whether every requested file was read and parsed. + /// + /// This is deliberately separate from `paths`: a successful scan that finds zero tests proves + /// that recorded tests are gone, while an incomplete scan proves nothing about them. + complete: bool, + + /// Full item paths, such as `tests::rejects_empty`. + paths: HashSet, + + /// Bare function names, such as `rejects_empty`. + /// + /// Harnesses disagree about how much of the path they print, and nextest prefixes the binary, + /// so a name that came back from one of them cannot be matched against a path alone. Deleting + /// a test removes it from here too, which is the case this whole module exists for. + leaves: HashSet, + + /// Mapping from full path to the declaring file. + paths_to_file: HashMap, + + /// Content digest for each scanned file. + file_digests: HashMap, +} + +impl Killers { + /// Reads every `.rs` file under the directories given and indexes the tests they declare. + /// + /// Unparseable and unreadable files are skipped rather than reported. A file that cannot be + /// parsed contributes no names, so every kill attributed to it is re-run — the safe direction — + /// and discovery proper will report the same file with a diagnostic worth reading. + #[cfg(test)] + #[must_use] + pub fn scan(files: &[Utf8PathBuf]) -> Self { + Self::scan_complete(files, true) + } + + /// Reads files whose enclosing walk completed or failed. + /// + /// `Survey` owns the directory walk, so only it can say whether the list is exhaustive. A + /// failed walk makes every recorded kill unconfirmable even when the files it did return parse + /// successfully. + #[must_use] + pub(super) fn scan_complete(files: &[Utf8PathBuf], complete: bool) -> Self { + let mut found = Self { + complete, + ..Self::default() + }; + + for file in files { + let Ok(bytes) = std::fs::read(file.as_std_path()) else { + found.complete = false; + continue; + }; + + let file_digest = digest(&bytes); + let _ = found.file_digests.insert(file.clone(), file_digest); + + let Ok(text) = core::str::from_utf8(&bytes) else { + found.complete = false; + continue; + }; + + let Ok(source) = SourceFile::parse(file.as_str(), text.to_owned()) else { + found.complete = false; + continue; + }; + + found.absorb(&source.ast.items, "", file); + } + + found + } + + /// Whether this index completely scanned the files it was asked to scan. + #[cfg(test)] + #[must_use] + pub const fn is_complete(&self) -> bool { + self.complete + } + + /// Finds the file where the killing test is declared. + #[cfg(test)] + #[must_use] + pub fn file_for(&self, killer: &str) -> Option<&Utf8Path> { + let path = killer.rsplit('$').next().unwrap_or(killer).trim(); + + if path.is_empty() { + return None; + } + + if let Some(file) = self.paths_to_file.get(path) { + return Some(file.as_path()); + } + + let leaf = path.rsplit("::").next().unwrap_or(path); + let mut files = self + .paths + .iter() + .filter(|candidate| candidate.rsplit("::").next() == Some(leaf)) + .filter_map(|candidate| self.paths_to_file.get(candidate)); + let file = files.next()?; + + files.next().is_none().then_some(file.as_path()) + } + + /// Returns the digest of a scanned test file. + #[must_use] + pub fn file_digest(&self, file: &Utf8Path) -> Option<&str> { + self.file_digests.get(file).map(String::as_str) + } + + /// Whether a name an earlier report recorded as the killer still names a test. + /// + /// Nextest spells a test as `package::binary$module::name` and libtest as `module::name`, so + /// the binary is trimmed before the path is compared, and a bare name is accepted on its own. + /// Accepting the bare name means two tests with the same leaf in different modules are not told + /// apart — but the case this guards against is a test that no longer exists anywhere, and a + /// name nothing declares fails both comparisons. + #[cfg(test)] + #[must_use] + pub fn still_there(&self, killer: &str) -> bool { + let path = killer.rsplit('$').next().unwrap_or(killer).trim(); + + if path.is_empty() { + return false; + } + + if self.paths.contains(path) { + return true; + } + + self.leaves.contains(path.rsplit("::").next().unwrap_or(path)) + } + + /// Finds an unambiguous declaring file for a verdict that is about to be carried. + /// + /// The reported name has to match a parsed item path exactly. A root-level test's full item + /// path is itself bare, so this accepts it without treating a nested test's leaf as an + /// identity. The permissive [`Self::still_there`] remains for probes, which are rerun before + /// they can affect a verdict. + #[must_use] + pub fn verdict_file_for(&self, killer: &str) -> Option<&Utf8Path> { + if !self.complete { + return None; + } + + let path = killer.rsplit('$').next().unwrap_or(killer).trim(); + + if path.is_empty() { + return None; + } + + self.paths_to_file.get(path).map(Utf8PathBuf::as_path) + } + + /// Walks one module's items, recording the tests and descending into inline modules. + fn absorb(&mut self, items: &[Item], prefix: &str, file: &Utf8Path) { + for item in items { + match item { + Item::Fn(function) if is_test(&function.attrs) => { + let name = function.sig.ident.to_string(); + let full = if prefix.is_empty() { + name.clone() + } else { + format!("{prefix}::{name}") + }; + + let _added = self.paths.insert(full.clone()); + let _ = self.paths_to_file.insert(full, file.to_path_buf()); + let _also = self.leaves.insert(name.clone()); + } + + Item::Mod(module) => self.descend(module, prefix, file), + + _other => {} + } + } + } + + /// Descends into an inline module, extending the path a test would be named by. + /// + /// A `mod x;` with no body is a file of its own, and that file is walked in its own right by + /// the caller. It contributes nothing here — which loses the module prefix for the tests in it, + /// and costs nothing, because they are still indexed under their own names. + fn descend(&mut self, module: &ItemMod, prefix: &str, file: &Utf8Path) { + let Some((_brace, items)) = module.content.as_ref() else { + return; + }; + + let name = module.ident.to_string(); + let inner = if prefix.is_empty() { name } else { format!("{prefix}::{name}") }; + + self.absorb(items, &inner, file); + } +} + +/// Whether a function's attributes mark it as a test. +/// +/// The last segment is what is compared, so `#[test]`, `#[tokio::test]` and `#[test_log::test]` all +/// count. An attribute macro that expands *into* a test — `#[rstest]`, `#[test_case]` — does not, +/// and the kills those tests made are re-run rather than carried. +fn is_test(attrs: &[Attribute]) -> bool { + attrs + .iter() + .filter_map(|attr| attr.path().segments.last()) + .any(|segment| segment.ident == "test") +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + + fn indexed(text: &str) -> Killers { + let directory = crate::testing::workdir("killers-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("utf8"); + let file = root.join("lib.rs"); + + std::fs::write(file.as_std_path(), text).expect("fixture"); + + Killers::scan(&[file]) + } + + /// The shape the check is for: a test that was there is deleted, and its kill must not survive. + #[test] + fn a_test_that_no_longer_exists_is_not_found() { + let killers = indexed("#[cfg(test)]\nmod tests {\n #[test]\n fn rejects_empty() {}\n}\n"); + + assert!(killers.still_there("tests::rejects_empty")); + assert!(!killers.still_there("tests::rejects_whitespace")); + } + + /// Harnesses disagree about how much of the name they print, and neither spelling may be + /// refused: a kill declined because the tool could not parse its own report is a rerun nobody + /// needed. + #[test] + fn every_spelling_a_harness_uses_finds_the_same_test() { + let killers = indexed("#[cfg(test)]\nmod tests {\n #[tokio::test]\n async fn reads_it() {}\n}\n"); + + assert!(killers.still_there("tests::reads_it"), "libtest"); + assert!(killers.still_there("my-crate::lib$tests::reads_it"), "nextest"); + assert!(killers.still_there("reads_it"), "a bare name"); + assert!(!killers.still_there(""), "and nothing is not a name"); + } + + /// Nested modules are walked, so a deep test is found by its whole path. + /// + /// A path nobody declares still matches when its leaf does, which is the leniency + /// [`Killers::still_there`] documents: what this check is for is a test that exists nowhere, + /// and moving one between modules must not cost a rerun of everything it killed. + #[test] + fn a_test_nested_in_modules_is_indexed_under_its_whole_path() { + let killers = indexed("mod outer {\n mod inner {\n #[test]\n fn deep() {}\n }\n}\n"); + + assert!(killers.still_there("outer::inner::deep")); + assert!(killers.still_there("somewhere::else::deep"), "the leaf is what a moved test keeps"); + assert!( + !killers.still_there("outer::inner::shallow"), + "a name nothing declares is not a match" + ); + } + + /// A function that is not a test never becomes one, or every helper beside a test would keep + /// a deleted test's kill alive. + #[test] + fn a_plain_function_is_not_a_test() { + let killers = indexed("#[cfg(test)]\nmod tests {\n fn helper() {}\n\n #[test]\n fn real() {}\n}\n"); + + assert!(killers.still_there("tests::real")); + assert!(!killers.still_there("tests::helper")); + } + + /// A completed zero-test scan proves that a recorded test is gone. + #[test] + fn scan_completeness_is_separate_from_test_count() { + assert!(!Killers::default().is_complete()); + assert!(indexed("pub fn f() {}\n").is_complete()); + assert!(indexed("#[test]\nfn t() {}\n").is_complete()); + } + + #[test] + fn a_qualified_name_never_falls_back_to_its_leaf_for_a_verdict() { + let killers = indexed("mod left { #[test] fn same() {} }\nmod right { #[test] fn same() {} }\n"); + + assert!(killers.still_there("missing::same"), "hints may still use a leaf"); + assert_eq!(killers.verdict_file_for("missing::same"), None); + assert_eq!(killers.verdict_file_for("same"), None, "duplicate bare names are ambiguous"); + assert!(killers.verdict_file_for("left::same").is_some()); + } + + #[test] + fn a_nested_test_leaf_is_not_an_identity_for_a_verdict() { + let killers = indexed("mod nested { #[test] fn only() {} }\n"); + + assert!(killers.still_there("only"), "a probe may still try the unique leaf"); + assert_eq!(killers.verdict_file_for("only"), None); + assert!(killers.verdict_file_for("nested::only").is_some()); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/mod.rs b/crates/cargo-gamma-lib/src/discover/mod.rs new file mode 100644 index 000000000..3e40ca26f --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/mod.rs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Finding the workspace, its packages, and the source files worth mutating. + +mod compile_fail; +mod diff; +mod glob; +mod hints; +mod killers; +mod modules; +mod order; +mod plan; +mod record; +mod shard; +mod survey; +mod target_file; +mod workspace_snapshot; + +pub use compile_fail::{CompileFailTarget, advice as compile_fail_advice}; +pub use diff::Diff; +pub(crate) use glob::Glob; +pub use glob::matches_glob; +pub use hints::{Hints, Promotion, path as hints_path}; +pub use killers::Killers; +pub(crate) use order::stages; +pub use plan::Plan; +pub(crate) use record::digest; +pub use record::{ + Context as RecordContext, ContextDigest, Killer, RunRecord, Term, Tier, Trust, context as record_context, rustflags, toolchain, +}; +pub use shard::shard_of; +#[cfg(test)] +pub use survey::plan; +pub(crate) use survey::plan_for_build; +pub use survey::{Scanned, Survey, load_metadata}; +pub use target_file::TargetFile; +pub(crate) use workspace_snapshot::WorkspaceSnapshot; diff --git a/crates/cargo-gamma-lib/src/discover/modules.rs b/crates/cargo-gamma-lib/src/discover/modules.rs new file mode 100644 index 000000000..da411af49 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/modules.rs @@ -0,0 +1,564 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Working out which module files the selected configuration does not compile as production code. +//! +//! The collector drops a `#[cfg(test)]` item wherever it sees one, which handles the usual +//! `#[cfg(test)] mod tests { … }` written inline. It cannot handle the other spelling: +//! +//! ```ignore +//! #[cfg(test)] +//! #[path = "reader_tests.rs"] +//! mod tests; +//! ``` +//! +//! The attribute is on the declaration, and the code is in another file. Files are parsed one at a +//! time and independently, so nothing in `reader_tests.rs` says it is test code — and it was +//! therefore mutated, producing a population of mutants inside assertions, which no test can +//! meaningfully catch. +//! +//! Rather than look at one file, this walks the module tree from the crate root and records how +//! each file is reached. A file reached only through a test-only or inactive configuration +//! declaration cannot contribute production mutants, no matter what it contains. + +use camino::{Utf8Path, Utf8PathBuf}; +use syn::{Attribute, Item, Meta}; + +use crate::cfg::{CfgSet, test_gated_for}; +use crate::{HashMap, HashSet}; + +/// One `mod name;` declaration that pulls in another file. +#[derive(Debug, Clone)] +pub(super) struct Declaration { + /// The file it resolves to, absolute. + pub(super) target: Utf8PathBuf, + + /// Whether the selected configuration, or an enclosing module, excludes this declaration. + /// + /// This includes test scaffolding and a `cfg` condition that does not hold. Both kinds of file + /// are walked to carry the exclusion to their children, but neither can contribute mutants + /// unless another live declaration reaches it. + pub(super) excluded: bool, +} + +/// Finds the file declarations a parsed file makes. +/// +/// `path` is the file the declarations were written in, absolute, since a `mod` resolves relative +/// to where it appears. +#[must_use] +pub(super) fn declarations(path: &Utf8Path, ast: &syn::File, cfg: &CfgSet) -> Vec { + let mut found = Vec::new(); + let (Some(directory), Some(beside)) = (owned_directory(path), path.parent()) else { + return found; + }; + + walk(&ast.items, &directory, beside, cfg, false, &mut found); + + found +} + +/// The directory a file's `mod` declarations resolve against. +/// +/// `lib.rs`, `main.rs` and `mod.rs` own the directory they sit in; any other file owns a +/// subdirectory named after it. This is the rule the compiler applies, and getting it wrong would +/// silently resolve a declaration to nothing. +fn owned_directory(path: &Utf8Path) -> Option { + let parent = path.parent()?; + let stem = path.file_stem()?; + + if matches!(stem, "lib" | "main" | "mod") { + return Some(parent.to_owned()); + } + + Some(parent.join(stem)) +} + +/// Walks items, following inline modules so that a declaration nested in one resolves correctly. +/// +/// `directory` is what a plain `mod name;` resolves against. `base` is what a `#[path = "…"]` +/// resolves against, which is not the same place: at the top level of a file, a `#[path]` is +/// relative to the directory that file sits in, while `mod name;` looks in the directory that file +/// owns. For `de/reader_impl_tests.rs` those are `de/` and `de/reader_impl_tests/` respectively, +/// so reading `#[path = "reader_tests.rs"]` against the wrong one finds nothing at all. +fn walk(items: &[Item], directory: &Utf8Path, base: &Utf8Path, cfg: &CfgSet, inherited_exclusion: bool, found: &mut Vec) { + for item in items { + let Item::Mod(module) = item else { continue }; + let excluded = inherited_exclusion || test_gated_for(cfg, &module.attrs) || !cfg.holds_for(&module.attrs); + + // An inline module is a directory rather than a file: `mod outer { mod inner; }` puts + // `inner` under `outer/`, and its own test gating is inherited by everything below it. + if let Some((_brace, items)) = module.content.as_ref() { + let nested = + path_attribute(&module.attrs).map_or_else(|| directory.join(module.ident.to_string()), |relative| base.join(relative)); + + // Inside an inline module both rules point at the same place, so from here down a + // `#[path]` and a plain declaration resolve against the module's own directory. + walk(items, &nested, &nested, cfg, excluded, found); + continue; + } + + let name = module.ident.to_string(); + let candidates = path_attribute(&module.attrs).map_or_else( + || vec![directory.join(format!("{name}.rs")), directory.join(&name).join("mod.rs")], + |relative| vec![base.join(relative)], + ); + + for target in candidates { + if target.as_std_path().is_file() { + found.push(Declaration { target, excluded }); + break; + } + } + } +} + +/// Reads the path a `#[path = "…"]` attribute points at. +fn path_attribute(attrs: &[Attribute]) -> Option { + attrs.iter().find_map(|attr| { + let Meta::NameValue(pair) = &attr.meta else { return None }; + + if !pair.path.is_ident("path") { + return None; + } + + let syn::Expr::Lit(literal) = &pair.value else { return None }; + let syn::Lit::Str(text) = &literal.lit else { return None }; + + Some(text.value()) + }) +} + +/// Works out which files have no live module path under the selected configuration. +/// +/// `roots` are the crate roots — a package's lib and bin entry points — which are never test code. +/// A file reachable from one of them without passing through an excluded declaration is real code, +/// whatever else also points at it; files reached only through excluded declarations are absent +/// from the selected build or exist only for tests. +/// +/// A file nothing points at is left alone rather than assumed to be either. It may be pulled in by +/// `include!`, or by a `mod` behind a `cfg` this cannot evaluate, and dropping mutants on a guess +/// would quietly shrink the population. +#[must_use] +pub(super) fn excluded_files(roots: &[Utf8PathBuf], declared: &[(Utf8PathBuf, Vec)]) -> HashSet { + let edges: HashMap<&Utf8Path, &[Declaration]> = declared.iter().map(|(from, list)| (from.as_path(), list.as_slice())).collect(); + + let mut live: HashSet = HashSet::default(); + let mut queue: Vec<&Utf8Path> = roots.iter().map(Utf8PathBuf::as_path).collect(); + + while let Some(file) = queue.pop() { + if !live.insert(file.to_owned()) { + continue; + } + + for declaration in edges.get(file).copied().unwrap_or(&[]) { + if !declaration.excluded { + queue.push(declaration.target.as_path()); + } + } + } + + // Everything below an excluded module is absent too, so the exclusion follows every edge from + // there rather than stopping at the file the declaration named. + let mut excluded: HashSet = HashSet::default(); + let mut queue: Vec<&Utf8Path> = declared + .iter() + .flat_map(|(_from, list)| list) + .filter(|declaration| declaration.excluded) + .map(|declaration| declaration.target.as_path()) + .collect(); + + while let Some(file) = queue.pop() { + if live.contains(file) || !excluded.insert(file.to_owned()) { + continue; + } + + for declaration in edges.get(file).copied().unwrap_or(&[]) { + queue.push(declaration.target.as_path()); + } + } + + excluded +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + use crate::cfg::test_gated; + + fn parse(text: &str) -> syn::File { + syn::parse_file(text).unwrap() + } + + #[test] + fn a_file_stem_that_is_not_a_module_root_owns_a_subdirectory() { + assert_eq!(owned_directory(Utf8Path::new("/a/src/lib.rs")), Some(Utf8PathBuf::from("/a/src"))); + assert_eq!(owned_directory(Utf8Path::new("/a/src/mod.rs")), Some(Utf8PathBuf::from("/a/src"))); + assert_eq!(owned_directory(Utf8Path::new("/a/src/de.rs")), Some(Utf8PathBuf::from("/a/src/de"))); + } + + #[test] + fn a_cfg_test_declaration_is_recognised() { + // The `fn` is here so that the filter below has a non-module item to reject, which is the + // shape every real source file has. + let ast = parse("fn not_a_module() {}\n#[cfg(test)]\nmod tests;\nmod real;"); + let gated: Vec = ast + .items + .iter() + .filter_map(|item| match item { + Item::Mod(module) => Some(test_gated(&module.attrs)), + _ => None, + }) + .collect(); + + assert_eq!(gated, vec![true, false]); + } + + #[test] + fn a_compound_gate_is_read_all_the_way_down() { + // The parser this replaced looked one level deep, so `all(test, unix)` read as a plain + // `unix` gate and the module below it was surveyed as production code. + let ast = parse(concat!( + "#[cfg(all(test, unix))]\nmod a;\n", + "#[cfg(not(feature = \"x\"))]\n#[cfg(test)]\nmod b;\n", + "#[cfg(all(unix, all(test, feature = \"x\")))]\nmod c;\n", + )); + let gated: Vec = ast + .items + .iter() + .filter_map(|item| match item { + Item::Mod(module) => Some(test_gated(&module.attrs)), + _ => None, + }) + .collect(); + + assert_eq!(gated, vec![true, true, true]); + } + + #[test] + fn a_gate_a_production_build_can_also_satisfy_is_not_test_only() { + // `any(test, …)` holds whenever the other arm does, so the module is compiled into the + // library the run measures. Treating it as test code would drop every mutant in it. + let ast = parse(concat!( + "#[cfg(any(test, feature = \"runtime\"))]\nmod a;\n", + "#[cfg(not(test))]\nmod b;\n", + "#[cfg(any(all(test, unix), windows))]\nmod c;\n", + )); + let gated: Vec = ast + .items + .iter() + .filter_map(|item| match item { + Item::Mod(module) => Some(test_gated(&module.attrs)), + _ => None, + }) + .collect(); + + assert_eq!(gated, vec![false, false, false]); + } + + #[test] + fn a_path_attribute_is_read() { + let ast = parse("#[path = \"reader_tests.rs\"]\nmod tests;"); + let Item::Mod(module) = &ast.items[0] else { + panic!("expected a module") + }; + + assert_eq!(path_attribute(&module.attrs), Some("reader_tests.rs".to_owned())); + } + + #[test] + fn a_path_attribute_resolves_beside_the_file_that_wrote_it() { + // The rule that this got wrong first time. `mod name;` looks in the directory the file + // owns, but `#[path]` looks in the directory the file sits in, and the two differ for + // every file that is not `lib.rs`, `main.rs` or `mod.rs`. + let directory = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).unwrap(); + let declaring = root.join("de").join("reader_impl_tests.rs"); + let target = root.join("de").join("reader_tests.rs"); + + std::fs::create_dir_all(root.join("de").as_std_path()).unwrap(); + std::fs::write(declaring.as_std_path(), "").unwrap(); + std::fs::write(target.as_std_path(), "").unwrap(); + + let ast = parse("#[cfg(test)]\n#[path = \"reader_tests.rs\"]\nmod tests;"); + let found = declarations(&declaring, &ast, &CfgSet::unconditional()); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].target, target); + assert!(found[0].excluded); + } + + #[test] + fn an_active_cfg_attr_marks_an_external_module_as_test_only() { + let directory = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).unwrap(); + let declaring = root.join("lib.rs"); + let target = root.join("tests.rs"); + + std::fs::write(target.as_std_path(), "").unwrap(); + + let ast = parse("#[cfg_attr(unix, cfg(test))]\nmod tests;"); + let active = declarations(&declaring, &ast, &CfgSet::parse("unix\n")); + let inactive = declarations(&declaring, &ast, &CfgSet::parse("windows\n")); + + assert_eq!(active.len(), 1, "{active:?}"); + assert_eq!(active[0].target, target); + assert!(active[0].excluded); + assert_eq!(inactive.len(), 1, "{inactive:?}"); + assert!(!inactive[0].excluded); + } + + #[test] + fn an_active_cfg_attr_excludes_an_external_module() { + let directory = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).unwrap(); + let declaring = root.join("lib.rs"); + let target = root.join("platform.rs"); + + std::fs::write(target.as_std_path(), "").unwrap(); + + let ast = parse("#[cfg_attr(unix, cfg(windows))]\nmod platform;"); + let active = declarations(&declaring, &ast, &CfgSet::parse("unix\n")); + let inactive = declarations(&declaring, &ast, &CfgSet::parse("windows\n")); + let declared = vec![(declaring.clone(), active.clone())]; + + assert_eq!(active.len(), 1, "{active:?}"); + assert!(active[0].excluded); + assert_eq!(inactive.len(), 1, "{inactive:?}"); + assert!(!inactive[0].excluded); + assert!(excluded_files(&[declaring], &declared).contains(&target)); + } + + #[test] + fn a_plain_declaration_resolves_in_the_directory_the_file_owns() { + let directory = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).unwrap(); + let declaring = root.join("de.rs"); + let target = root.join("de").join("raw.rs"); + + std::fs::create_dir_all(root.join("de").as_std_path()).unwrap(); + std::fs::write(target.as_std_path(), "").unwrap(); + + let found = declarations(&declaring, &parse("mod raw;"), &CfgSet::unconditional()); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].target, target); + assert!(!found[0].excluded); + } + + #[test] + fn a_file_reached_only_through_a_test_module_is_excluded() { + let root = Utf8PathBuf::from("/w/src/lib.rs"); + let helper = Utf8PathBuf::from("/w/src/helper.rs"); + let tests = Utf8PathBuf::from("/w/src/reader_tests.rs"); + let declared = vec![( + root.clone(), + vec![ + Declaration { + target: helper.clone(), + excluded: false, + }, + Declaration { + target: tests.clone(), + excluded: true, + }, + ], + )]; + + let excluded = excluded_files(&[root], &declared); + + assert!(excluded.contains(&tests)); + assert!(!excluded.contains(&helper)); + } + + #[test] + fn a_file_a_test_module_shares_with_real_code_is_kept() { + // Reached both ways, so it is real code that tests happen to also pull in. Dropping it + // would silently remove mutants from code the crate actually ships. + let root = Utf8PathBuf::from("/w/src/lib.rs"); + let shared = Utf8PathBuf::from("/w/src/shared.rs"); + let declared = vec![( + root.clone(), + vec![ + Declaration { + target: shared.clone(), + excluded: true, + }, + Declaration { + target: shared, + excluded: false, + }, + ], + )]; + + assert!(excluded_files(&[root], &declared).is_empty()); + } + + #[test] + fn a_file_nothing_declares_is_left_alone() { + let root = Utf8PathBuf::from("/w/src/lib.rs"); + + assert!(excluded_files(&[root], &[]).is_empty()); + } + + #[test] + fn a_module_below_a_test_module_is_excluded_too() { + let root = Utf8PathBuf::from("/w/src/lib.rs"); + let outer = Utf8PathBuf::from("/w/src/outer.rs"); + let inner = Utf8PathBuf::from("/w/src/outer/inner.rs"); + let declared = vec![ + ( + root.clone(), + vec![Declaration { + target: outer.clone(), + excluded: true, + }], + ), + ( + outer.clone(), + vec![Declaration { + target: inner.clone(), + excluded: false, + }], + ), + ]; + + let excluded = excluded_files(&[root], &declared); + + // Both: `inner` is only ever reached by walking through `outer`, which exists for tests, + // so it is test code as surely as its parent is. + assert!(excluded.contains(&outer)); + assert!(excluded.contains(&inner)); + } + + /// A file two live modules both declare is walked once, and stays production code. + #[test] + fn a_file_declared_by_two_live_modules_is_walked_once() { + // `#[path]` lets two modules name the same file. The live walk has to notice it has been + // there before, or a diamond in the module graph becomes an exponential re-walk. + let root = Utf8PathBuf::from("/w/src/lib.rs"); + let left = Utf8PathBuf::from("/w/src/left.rs"); + let right = Utf8PathBuf::from("/w/src/right.rs"); + let shared = Utf8PathBuf::from("/w/src/shared.rs"); + let declared = vec![ + ( + root.clone(), + vec![ + Declaration { + target: left.clone(), + excluded: false, + }, + Declaration { + target: right.clone(), + excluded: false, + }, + ], + ), + ( + left, + vec![Declaration { + target: shared.clone(), + excluded: false, + }], + ), + ( + right, + vec![Declaration { + target: shared.clone(), + excluded: false, + }], + ), + ]; + + let excluded = excluded_files(&[root], &declared); + + assert!(!excluded.contains(&shared), "{excluded:?}"); + } + + #[test] + fn a_cycle_between_two_files_terminates() { + // `#[path]` makes a declaration cycle expressible, and the walk has to notice it has been + // somewhere before rather than following the edge round for ever. + let root = Utf8PathBuf::from("/w/src/lib.rs"); + let first = Utf8PathBuf::from("/w/src/first.rs"); + let second = Utf8PathBuf::from("/w/src/second.rs"); + let declared = vec![ + ( + root.clone(), + vec![Declaration { + target: first.clone(), + excluded: true, + }], + ), + ( + first.clone(), + vec![Declaration { + target: second.clone(), + excluded: false, + }], + ), + ( + second.clone(), + vec![Declaration { + target: first.clone(), + excluded: false, + }], + ), + ]; + + let excluded = excluded_files(&[root], &declared); + + assert!(excluded.contains(&first)); + assert!(excluded.contains(&second)); + } + + #[test] + fn a_path_with_no_directory_declares_nothing() { + // A bare file name has no parent directory to resolve a `mod` against. Returning nothing is + // the honest answer; guessing the current directory would attribute declarations to + // whatever happened to be beside the process. + assert!(declarations(Utf8Path::new(""), &parse("mod raw;"), &CfgSet::unconditional()).is_empty()); + } + + #[test] + fn a_declaration_pointing_at_no_file_is_dropped() { + // A `mod` behind a `cfg` this cannot evaluate, or one generated by a build script, names a + // file that is not on disk. It has to be skipped rather than recorded as a source file the + // run would then fail to open. + let directory = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).unwrap(); + + assert!(declarations(&root.join("lib.rs"), &parse("mod nowhere;"), &CfgSet::unconditional()).is_empty()); + } + + #[test] + fn an_attribute_that_is_neither_cfg_nor_path_is_ignored() { + // Every item carries attributes this does not care about. Reading one as a `#[path]` would + // resolve a module to a doc string. + let ast = parse("#[doc = \"a module\"]\n#[derive(Debug)]\nmod plain;"); + let Item::Mod(module) = &ast.items[0] else { + panic!("expected a module") + }; + + assert_eq!(path_attribute(&module.attrs), None); + assert!(!test_gated(&module.attrs)); + } + + #[test] + fn a_path_attribute_on_an_inline_module_redirects_everything_below_it() { + // `#[path] mod outer { mod inner; }` puts `inner` under the redirected directory, not under + // one named after `outer`. Resolving it against the module's own name would find nothing. + let directory = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).unwrap(); + let target = root.join("custom").join("inner.rs"); + + std::fs::create_dir_all(root.join("custom").as_std_path()).unwrap(); + std::fs::write(target.as_std_path(), "").unwrap(); + + let source = "#[path = \"custom\"]\nmod outer { mod inner; }"; + let found = declarations(&root.join("lib.rs"), &parse(source), &CfgSet::unconditional()); + + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].target, target); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/order.rs b/crates/cargo-gamma-lib/src/discover/order.rs new file mode 100644 index 000000000..83b6dbc7d --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/order.rs @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Ordering workspace packages so that a package is processed after everything it depends on. +//! +//! This order is a constraint, not a preference: a package's dependencies have to be built before +//! it, so there is no freedom here to spend on a hint. The stale unviability a run record carries +//! is consumed one level down, in [`crate::exec`]'s convergence, where mutants within a stage are +//! reordered so that the ones that failed to compile last time are offered to the compiler first. +//! Reordering equal-sized stages here instead would move independent siblings past each other for +//! no gain — every stage runs before the build that the hint is about, whatever order they run in. + +use crate::{HashMap, HashSet}; + +/// A set of packages that have to be processed together, because each can reach the others. +/// +/// Almost every stage holds a single package. A stage holds more than one only when packages are +/// mutually reachable, which happens when a dev-dependency closes a cycle: `a` depends on `b`, and +/// `b`'s integration tests depend on `a`. Neither can be placed before the other, so neither is. +type Stage = Vec; + +/// Orders packages so that every package follows the packages it depends on. +/// +/// The order comes from the sizes of the reach sets rather than from a traversal. `reach` maps a +/// package to itself plus everything it can reach, so a dependency's reach set is a strict subset +/// of its dependent's, and sorting by size ascending is therefore a topological order. Packages +/// with equal reach sets are mutually reachable and share a stage. Ties are broken by name, so the +/// same workspace always produces the same order. +pub(crate) fn stages(packages: &[String], reach: &HashMap>) -> Vec { + let mut ordered: Vec<&String> = packages.iter().collect(); + let size = |name: &str| reach.get(name).map_or(0, HashSet::len); + + ordered.sort_by(|left, right| size(left).cmp(&size(right)).then_with(|| left.cmp(right))); + + let mut stages: Vec = Vec::new(); + + for name in ordered { + // Grouping is by reach-set equality, not sort adjacency. Two mutually reachable packages + // have the same reach set and so the same size, but a different reach set of that same size + // can sort between them by name, so the members of one cycle are not guaranteed to be + // neighbours. Every existing stage is therefore a candidate, and the one whose reach set + // equals this package's — if any — is the stage it belongs in. + let joins = stages.iter_mut().find(|stage| { + stage.first().is_some_and(|first| { + reach + .get(first.as_str()) + .is_some_and(|theirs| reach.get(name.as_str()).is_some_and(|mine| mine == theirs)) + }) + }); + + if let Some(stage) = joins { + stage.push(name.clone()); + } else { + stages.push(vec![name.clone()]); + } + } + + stages +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reach(entries: &[(&str, &[&str])]) -> HashMap> { + entries + .iter() + .map(|(name, reaches)| ((*name).to_owned(), reaches.iter().map(|entry| (*entry).to_owned()).collect())) + .collect() + } + + #[test] + fn a_dependency_is_processed_before_its_dependent() { + let reach = reach(&[("leaf", &["leaf"]), ("mid", &["mid", "leaf"]), ("top", &["top", "mid", "leaf"])]); + let packages = vec!["top".to_owned(), "leaf".to_owned(), "mid".to_owned()]; + + let stages = stages(&packages, &reach); + + assert_eq!(stages, vec![vec!["leaf"], vec!["mid"], vec!["top"]]); + } + + #[test] + fn independent_packages_are_ordered_by_name() { + // Neither can depend on the other, so the only thing left to order them by is their names, + // and the order has to be the same on every run. + let reach = reach(&[("beta", &["beta"]), ("alpha", &["alpha"])]); + let packages = vec!["beta".to_owned(), "alpha".to_owned()]; + + assert_eq!(stages(&packages, &reach), vec![vec!["alpha"], vec!["beta"]]); + } + + #[test] + fn mutually_reachable_packages_share_a_stage() { + // `b`'s tests depend on `a`, which depends on `b`. Neither can be built first, so both are + // built at once rather than one being placed arbitrarily ahead of the other. + let reach = reach(&[("a", &["a", "b"]), ("b", &["a", "b"])]); + let packages = vec!["b".to_owned(), "a".to_owned()]; + + assert_eq!(stages(&packages, &reach), vec![vec!["a", "b"]]); + } + + #[test] + fn a_cycle_survives_an_intervening_equal_sized_reach_set() { + // `a` and `c` are mutually reachable, so they must build together. `b` is independent but + // has a reach set of the same size, and sorts by name between them. Comparing each package + // only against the previous stage would put `b` between the two cycle members and split the + // cycle across two stages; grouping by reach-set equality keeps `a` and `c` together + // however the equal-sized outsider sorts. + let reach = reach(&[("a", &["a", "c", "shared"]), ("c", &["a", "c", "shared"]), ("b", &["b", "x", "y"])]); + let packages = vec!["c".to_owned(), "b".to_owned(), "a".to_owned()]; + + assert_eq!(stages(&packages, &reach), vec![vec!["a", "c"], vec!["b"]]); + } + + #[test] + fn a_package_missing_from_the_reach_map_still_appears() { + // Losing a package here would mean never scanning or building it, which would silently + // drop every mutant it holds. + let stages = stages(&["orphan".to_owned()], &reach(&[])); + + assert_eq!(stages, vec![vec!["orphan"]]); + } + + #[test] + fn packages_are_never_lost_or_duplicated() { + let reach = reach(&[("leaf", &["leaf"]), ("mid", &["mid", "leaf"]), ("other", &["other", "leaf"])]); + let packages = vec!["mid".to_owned(), "other".to_owned(), "leaf".to_owned()]; + + let flattened: Vec = stages(&packages, &reach).into_iter().flatten().collect(); + + assert_eq!(flattened.len(), packages.len()); + + for package in &packages { + assert!(flattened.contains(package), "{package} was dropped"); + } + } +} diff --git a/crates/cargo-gamma-lib/src/discover/plan.rs b/crates/cargo-gamma-lib/src/discover/plan.rs new file mode 100644 index 000000000..011583001 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/plan.rs @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Everything a run needs, worked out before any building happens. + +use camino::{Utf8Path, Utf8PathBuf}; + +use super::target_file::TargetFile; +use crate::model::Mutant; +use crate::suppress::Idle; +use crate::{HashMap, HashSet}; + +/// Everything a run needs, worked out before any building happens. +#[derive(Debug)] +pub struct Plan { + /// Absolute path of the workspace root. + pub root: Utf8PathBuf, + + /// The files that were analyzed. + pub files: Vec, + + /// Every mutant to be tested, with ordinals assigned. + pub mutants: Vec, + + /// Mutants suppressed by a directive. They stay in `mutants`, marked as ignored. + pub suppressed: usize, + + /// Skip directives this run offered a mutant to and which suppressed none of it. + /// + /// See [`crate::suppress::idle`] for what "offered" excludes, and why the distinction matters. + pub idle: Vec, + + /// Mutants excluded by sharding, counted rather than kept. + pub sharded_out: usize, + + /// Mutants an earlier report already settled, carried at their recorded verdict rather than + /// run again. + pub settled_out: usize, + + /// A digest of the normalized source each analyzed file held when its mutants were derived. + /// + /// A leading UTF-8 BOM is omitted just as it is during parsing, so source-edit generation + /// checks compare the representation that supplied their line numbers. + /// + /// The commands that edit source apply line numbers this plan decided, and `suppress` applies + /// them after a whole measured run — hours, on a real workspace. A line number means nothing + /// against text it was not computed from, so the edit compares the file it is about to write + /// against this and refuses rather than deleting or annotating the wrong line. + pub digests: HashMap, + + /// Files found under a mutable target that could not be analyzed, each a complete diagnostic. + /// + /// Carried on the plan rather than reported where it was noticed, because a scan happens once + /// per package and the run assembles the plan from all of them; a message printed per package + /// would arrive interleaved with build output and be read as a build problem. + pub skipped: Vec, + + /// For each workspace package, the workspace packages its test binaries can reach. + /// + /// A test binary can only exercise code it links, so a mutant in a package outside this set is + /// unreachable from that binary no matter what the tests do. Running it anyway is pure cost. + pub reach: HashMap>, + + /// For each workspace package, its manifest directory relative to the root, and its version. + /// + /// See [`Plan::spec`] for why a name on its own will not do. + pub specs: HashMap, +} + +impl Plan { + /// Names one package unambiguously, for a `--package` argument in a tree rooted at `root`. + /// + /// `--package serde` is ambiguous the moment a workspace member shares its name with a crate + /// anywhere in the dependency graph, and a repository that dev-depends on a published version + /// of itself does exactly that. Cargo rejects the whole invocation, and it rejects it before + /// emitting any JSON, so a build that fails this way arrives with nothing to attribute and is + /// indistinguishable from a tree that does not compile. Spelling the package as a path-rooted + /// package ID makes the question unambiguous by construction, so it is asked that way always + /// rather than only once cargo has complained. + /// + /// Falls back to the bare name for a package whose manifest was never located, which is no + /// worse than what came before. + #[must_use] + pub fn spec(&self, root: &Utf8Path, package: &str) -> String { + let Some((directory, version)) = self.specs.get(package) else { + return package.to_owned(); + }; + + let absolute = if directory.as_str().is_empty() { + root.to_owned() + } else { + root.join(directory) + }; + + let normalized = absolute.as_str().replace('\\', "/"); + let root_slash = if normalized.starts_with('/') { "" } else { "/" }; + let source = format!("path+file://{root_slash}{normalized}"); + + format!("{source}#{package}@{version}") + } + + /// A package's directory, relative to the workspace root. + #[must_use] + pub fn directory_of(&self, package: &str) -> Option<&Utf8Path> { + self.specs.get(package).map(|(directory, _version)| directory.as_path()) + } + + /// Folds one package's scan into the plan. + /// + /// A run scans a package, instruments it and builds it before moving to the next, so the plan + /// is assembled a package at a time rather than handed over complete. + pub fn absorb(&mut self, scanned: super::Scanned) { + let super::Scanned { + mutants, + suppressed, + idle, + sharded_out, + settled_out, + skipped, + digests, + } = scanned; + + self.mutants.extend(mutants); + self.suppressed = self.suppressed.saturating_add(suppressed); + self.idle.extend(idle); + self.sharded_out = self.sharded_out.saturating_add(sharded_out); + self.settled_out = self.settled_out.saturating_add(settled_out); + self.skipped.extend(skipped); + self.digests.extend(digests); + } + + /// Puts the mutants in report order, once every package has been absorbed. + pub fn sort(&mut self) { + self.mutants.sort_by(|left, right| { + left.file + .cmp(&right.file) + .then_with(|| left.span.start.cmp(&right.span.start)) + .then_with(|| left.mutator.cmp(&right.mutator)) + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn plan(specs: &[(&str, &str, &str)]) -> Plan { + Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: Utf8PathBuf::from("/w"), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: specs + .iter() + .map(|(name, dir, version)| ((*name).to_owned(), (Utf8PathBuf::from(*dir), (*version).to_owned()))) + .collect(), + } + } + + #[test] + fn a_package_is_named_by_where_its_manifest_is() { + // Cargo rejects an ambiguous `--package` before it emits any JSON, so the build fails with + // nothing to attribute and reads as though the tree does not compile. A path-rooted id + // cannot be ambiguous. + let plan = plan(&[("serde", "serde", "1.0.0")]); + + assert_eq!(plan.spec(Utf8Path::new("/tree"), "serde"), "path+file:///tree/serde#serde@1.0.0"); + } + + #[test] + fn a_single_crate_repository_names_its_root() { + // The manifest sits at the workspace root, so the relative directory is empty and joining + // it blindly would produce the trailing separator that cargo does not match. + let plan = plan(&[("itoa", "", "1.0.18")]); + + assert_eq!(plan.spec(Utf8Path::new("/tree"), "itoa"), "path+file:///tree#itoa@1.0.18"); + } + + #[test] + fn the_spec_follows_the_tree_it_is_asked_about() { + // A run builds in a scratch copy, not in the user's checkout, and a spec naming the wrong + // root would select the wrong package or none at all. + let plan = plan(&[("core", "crates/core", "0.2.0")]); + + assert_eq!( + plan.spec(Utf8Path::new("/scratch/tree"), "core"), + "path+file:///scratch/tree/crates/core#core@0.2.0" + ); + } + + #[cfg(windows)] + #[test] + fn a_windows_package_spec_is_a_file_url() { + // Cargo package IDs are URLs even on Windows. Passing a native `D:\...` path makes Cargo + // parse the drive letter as URL syntax and silently changes the package being named. + let plan = plan(&[("core", "crates/core", "0.2.0")]); + + assert_eq!( + plan.spec(Utf8Path::new(r"D:\scratch\tree"), "core"), + "path+file:///D:/scratch/tree/crates/core#core@0.2.0" + ); + } + + #[test] + fn a_package_with_no_manifest_recorded_keeps_its_bare_name() { + // Erring toward the old behaviour: a bare name builds the right thing whenever it is not + // ambiguous, whereas a malformed id fails every time. + let plan = plan(&[]); + + assert_eq!(plan.spec(Utf8Path::new("/tree"), "lonely"), "lonely"); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/record.rs b/crates/cargo-gamma-lib/src/discover/record.rs new file mode 100644 index 000000000..a1592362b --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/record.rs @@ -0,0 +1,3053 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Build facts and checked hints from the last run. + +use std::process::Command; +use std::{env, fs}; + +use blake3::Hasher; +use camino::{Utf8Path, Utf8PathBuf}; +use serde::{Deserialize, Serialize}; + +use super::Plan; +use super::killers::Killers; +use super::workspace_snapshot::WorkspaceSnapshot; +use crate::cfg::Build; +use crate::model::{Mutant, MutantId, Outcome}; +use crate::{HashMap, HashSet}; + +/// What the cache format is; a file written by any other version is discarded rather than read. +/// +/// A cache is allowed to be thrown away — that is what makes it a cache — so there is no migration +/// path here and there should never be one. Reading a format that has changed meaning is how a +/// cache stops being free. +/// +/// Version 7 records external local Cargo inputs alongside the workspace snapshot. +/// +/// A term added to the context does not move this number, and deliberately. The terms are separate +/// fields and a new one is optional, so an older file is not a foreign format — it is a record that +/// declines to answer the guard on the tiers that ask about that term, and keeps the two that ask +/// nothing. Discarding it instead would throw away the probes and the build order to defend a +/// question they do not depend on. What moves this number is a change of *meaning* in what is +/// already there, which no reader could detect for itself. +const VERSION: u32 = 9; + +/// The file name under the gamma scratch base. +const FILE: &str = "last-gamma-run.json"; + +/// How much of a record a reader is willing to believe. +/// +/// Production runs use [`Self::Free`] and therefore reuse only compiler unviability. Test verdicts +/// are nondeterministic observations and must not contribute to a later run. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum Trust { + /// Only what moves no verdict: the mutants that would not compile. + #[default] + Free, + + /// Test verdicts as well as compiler outcomes. + /// + /// New run records contain no test verdicts, so this cannot add detection credit. + Settled, +} + +/// One term of the build context, digested on its own so that a tier can name what it depends on. +/// +/// Separate terms rather than one digest because the terms do not all matter to the same things. +/// A mutant's compilation and verdict both depend on the compiler, while probes and ordering are +/// never believed. A single digest cannot express that difference without making safe hints cold +/// whenever a developer's machine and CI differ. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Term { + /// The feature selection, including `--all-features` and `--no-default-features`. + Features, + + /// The cargo profile, which decides `debug_assertions` among other things. + Profile, + + /// Whichever of the rustflags variables cargo will actually read. + Rustflags, + + /// The triple the build compiles for, as the command line and the environment ask for it. + /// + /// Separate from [`Self::Extra`] even though a passthrough `--target` is part of both, because + /// the target has a second spelling — `CARGO_BUILD_TARGET` — that no passthrough argument + /// carries. A term fed only by the arguments would let the variable move the whole compilation + /// without moving the key, and unviability found for one architecture would be believed for + /// another. + Target, + + /// The build settings Cargo configuration and workspace files hold, digested together. + /// + /// The complete bytes of every Cargo configuration Cargo reads, together with the + /// configuration's `build.target`, `build.rustflags` and target tables, and the profile bodies + /// of both the configuration and the manifest: everything + /// [`Build::resolve`](crate::cfg::Build::resolve) reads that neither the command line nor the + /// environment carries. One term rather than several because nothing here can be reported more + /// usefully apart — a reader told "the configuration differs" looks at their configuration — + /// and because the set of tables is open, so a term per table could not be a fixed list. + Config, + + /// The arbitrary passthrough build arguments, which can carry `--cfg`, `-C` or `--target`. + Extra, + + /// The compiler, cargo and wrapper identities. + Toolchain, + + /// This tool's own version, which decides what a mutant id denotes. + Tool, + + /// The test filtering and execution arguments. + Tests, + + /// The execution policy that decides how a test result becomes a verdict. + Policy, + + /// Environment inherited by test binaries that gamma does not set itself. + Environment, +} + +impl Term { + /// Every term there is, which is what a tier requiring the whole context asks for. + /// + /// Spelled once, so that adding a term to [`ContextDigest`] and forgetting it here is the only + /// way to weaken an invalidation rule by accident — and the test below pins that this list and + /// the digest agree. + pub const ALL: &'static [Self] = &[ + Self::Features, + Self::Profile, + Self::Rustflags, + Self::Target, + Self::Config, + Self::Extra, + Self::Toolchain, + Self::Tool, + Self::Tests, + Self::Policy, + Self::Environment, + ]; + + /// The term's name, as the diagnostics spell it. + #[must_use] + pub const fn name(self) -> &'static str { + match self { + Self::Features => "features", + Self::Profile => "profile", + Self::Rustflags => "rustflags", + Self::Target => "target", + Self::Config => "config", + Self::Extra => "extra", + Self::Toolchain => "toolchain", + Self::Tool => "tool", + Self::Tests => "tests", + Self::Policy => "policy", + Self::Environment => "environment", + } + } +} + +/// A kind of knowledge the record holds, paired with what a run must match before it may be used. +/// +/// This is the per-tier invalidation rule, and it is deliberately stated as data rather than as a +/// condition spread across the readers: a tier that quietly widened what it accepts would be a +/// mutant dropped from the denominator, and there is no test that could find that in a condition +/// nobody named. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Tier { + /// The mutants that would not compile, used to keep them out of the build entirely. + /// + /// A claim about what compiles, so it depends on everything that decides what compiles. Being + /// wrong here withholds a mutant that might have survived, which turns a real gap in the suite + /// into a better score — the one direction this tool must never be wrong in. + Unviability, + + /// Test verdicts read from a record. + /// + /// The writer does not persist these because matching inputs cannot prove deterministic tests. + Verdict, + + /// Which mutants failed to compile last time, offered to the build as an order to try them in. + /// + /// Depends on nothing either, and for the same shape of reason: being wrong about the order + /// costs the order and nothing else. No mutant is withheld, settled or excluded on this, so a + /// hint from another toolchain, another feature set or another machine is free to be wrong. + Ordering, +} + +impl Tier { + /// The context terms a run has to agree with the record on before it may use this tier. + #[must_use] + pub const fn requires(self) -> &'static [Term] { + match self { + Self::Unviability => &[ + Term::Features, + Term::Profile, + Term::Rustflags, + Term::Target, + Term::Config, + Term::Extra, + Term::Toolchain, + Term::Tool, + Term::Policy, + Term::Environment, + ], + Self::Verdict => &[ + Term::Features, + Term::Profile, + Term::Rustflags, + Term::Target, + Term::Config, + Term::Extra, + Term::Toolchain, + Term::Tool, + Term::Tests, + Term::Policy, + Term::Environment, + ], + Self::Ordering => &[], + } + } + + /// Whether a record written under `recorded` may be read for this tier by a run under `current`. + /// + /// A term neither side states is not agreement. [`ContextDigest::states`] exists because two of + /// the terms are answers only a workspace can give, so a digest built before the workspace was + /// located leaves them open — and reading "neither of us knows" as "we agree" would admit + /// exactly the records this guard is for. + #[must_use] + pub fn admits(self, recorded: &ContextDigest, current: &ContextDigest) -> bool { + self.requires() + .iter() + .all(|term| recorded.states(*term) && current.states(*term) && recorded.term(*term) == current.term(*term)) + } +} + +/// A digest of each thing other than the sources that decides what compiles, kept term by term. +/// +/// Written into the record rather than derived on read, because the question a reader asks is not +/// "what is this run's context" but "did the run that wrote this agree with mine about the terms my +/// tier depends on" — and that can only be answered by a file that states its terms separately. +/// +/// An absent term is an empty string, which is a value like any other: two runs that both passed no +/// rustflags agree about rustflags. An *unstated* term is different and is spelled `None`: the +/// target and the configuration are answers only a located workspace can give, while the inherited +/// environment was not recorded before that term existed. A digest built without one of those +/// answers leaves it open, and a tier that requires it refuses rather than matching on the gap. +/// That is also what an older record — written before a term existed — looks like, which costs it +/// the tiers whose guard it cannot answer and leaves it the two that need no guard. +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct ContextDigest { + /// The feature selection, including the two flags that override it. + features: String, + + /// The cargo profile. + profile: String, + + /// The ambient rustflags cargo will read. + rustflags: String, + + /// The triple the build compiles for, or nothing when this digest does not state one. + #[serde(default)] + target: Option, + + /// Cargo configuration and workspace build settings, or nothing when they had not been read yet. + #[serde(default)] + config: Option, + + /// The passthrough build arguments. + extra: String, + + /// The compiler's version string. + toolchain: String, + + /// This tool's own version. + tool: String, + + /// The test filtering and execution arguments. + #[serde(default)] + tests: String, + + /// The policy that turns test execution into a verdict. + #[serde(default)] + policy: String, + + /// The ambient environment test binaries inherit, or nothing for an older record. + #[serde(default)] + environment: Option, +} + +impl ContextDigest { + /// The digest of one term, which is the empty string for a term this digest does not state. + #[must_use] + pub fn term(&self, term: Term) -> &str { + match term { + Term::Features => &self.features, + Term::Profile => &self.profile, + Term::Rustflags => &self.rustflags, + Term::Target => self.target.as_deref().unwrap_or_default(), + Term::Config => self.config.as_deref().unwrap_or_default(), + Term::Extra => &self.extra, + Term::Toolchain => &self.toolchain, + Term::Tool => &self.tool, + Term::Tests => &self.tests, + Term::Policy => &self.policy, + Term::Environment => self.environment.as_deref().unwrap_or_default(), + } + } + + /// Whether this digest has an answer for a term at all. + /// + /// Asked before [`Self::term`] is compared, because the empty string that an unstated term + /// reads as is also a perfectly good digest of nothing, and the two must not be confused: one + /// says "no rustflags were set", the other says "nobody has looked". + #[must_use] + pub const fn states(&self, term: Term) -> bool { + match term { + Term::Target => self.target.is_some(), + Term::Config => self.config.is_some(), + Term::Environment => self.environment.is_some(), + _stated_by_the_command_line => true, + } + } + + /// The terms this run and `other` disagree about, in [`Term::ALL`] order. + /// + /// Reported rather than merely counted so that a diagnostic can say *which* axis moved. "The + /// record did not apply" sends a reader looking through their whole configuration; "the + /// toolchain differs" is something they can act on, or decide not to. + /// + /// A term one of the two does not state is not a disagreement to report. It is a difference the + /// gate does act on — see [`Tier::admits`] — but naming it here would tell a reader that their + /// target moved when what actually happened is that one of the digests was taken before the + /// workspace was in hand. + #[must_use] + pub fn differences(&self, other: &Self) -> Vec { + Term::ALL + .iter() + .copied() + .filter(|term| self.states(*term) && other.states(*term) && self.term(*term) != other.term(*term)) + .collect() + } + + /// This context with the terms only a located workspace can answer filled in. + /// + /// Separate from [`context`] because the two are known at different moments. The command line + /// settles the features, the profile and the rest before anything has been read from disk, + /// while the target tables and the profile bodies live in files whose location is the workspace + /// root — which the record is handed when it is written and when it is read, and not before. + /// Both sides of every comparison go through here, so the gate compares like with like. + /// + /// Public because the gate is not the only caller that has to compare like with like: + /// [`Self::differences`] skips any term one side leaves unstated, so a caller that reports + /// *why* the cache did not apply and passes an unresolved digest would silently never name the + /// configuration — the one axis a reader is least likely to guess at. + #[must_use] + pub fn resolved_at(&self, root: &Utf8Path) -> Self { + let settings = Build::settings(root); + let parts: Vec<&[u8]> = settings.iter().map(String::as_bytes).collect(); + + Self { + config: Some(term(Term::Config, &parts)), + ..self.clone() + } + } +} + +/// What an earlier run established about each mutant, and what those findings depended on. +/// +/// Of everything a run learns, which mutants failed to compile is both the most expensive to +/// rediscover and the safest to reuse. Expensive because unviability is found by building, blaming +/// and building again, and each round is another rebuild of the instrumented tree. Safe because an +/// unviable mutant is excluded from the score outright, so carrying one forward moves no verdict — +/// unlike a kill, which is a claim about the test suite and is only adopted when asked for. +/// +/// Nothing here is discarded wholesale. The file is read whatever context it was written under, and +/// each [`Tier`] decides for itself whether this run agrees with it about the terms that tier +/// depends on. A toolchain bump therefore costs the unviability and keeps the probes and the build +/// order. +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct RunRecord { + /// The format this was written in. + version: u32, + + /// A digest, term by term, of every policy and toolchain setting that can affect a verdict. + /// + /// Features are the reason this exists. A mutant that cannot compile with one feature set may + /// compile perfectly well with another, so a cache written under `--all-features` says nothing + /// about a default-features run. Kept per term rather than as one digest so that a tier + /// depending on some of the terms is not invalidated by the ones it does not depend on. + context: ContextDigest, + + /// For each source file that holds a recorded mutant, its digest and what was settled there. + files: Vec, + + /// Every workspace input Cargo could have consulted before this run started. + /// + /// This is intentionally wider than the mutable files. A changed helper, manifest, lockfile, + /// build script, test or cargo configuration can change a verdict without changing a mutant's + /// identity, so any difference rejects every carried outcome. + #[serde(default)] + inputs: WorkspaceSnapshot, + + /// Workspace package roots whose contents can affect each package's compilation. + #[serde(default)] + compilation_roots: HashMap>, + + /// What caught each mutant last time, so the sweep can try that test first. + /// + /// Outside [`Self::files`], and deliberately: a verdict is *believed*, so it is gated on the + /// build context and on the digest of the file it was found in. A hint is never believed. Every + /// one of them is a guess the run immediately checks by running the named test, and a guess + /// that does not convict costs one filtered process and is thrown away. Nothing here can move a + /// verdict, so nothing here needs invalidating — and invalidating it would only make the map + /// cold on exactly the runs, after an edit or a feature change, where it is worth the most. + #[serde(default)] + hints: HashMap, +} + +/// The test that caught a mutant, and the binary it lives in. +/// +/// The binary is named by package and target rather than by path because a path is not stable +/// across runs: the binaries a run judges live in a scratch tree that is rebuilt each time, so a +/// recorded path would miss every time and the map would be permanently cold. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Killer { + /// The package whose test binary caught it. + pub package: String, + + /// The cargo target within that package. + pub target: String, + + /// The test the harness named when it failed. + pub test: String, +} + +impl Killer { + /// Whether this names the given binary. + #[must_use] + pub fn names(&self, package: &str, target: &str) -> bool { + self.package == package && self.target == target + } +} + +/// The recorded mutants of one source file, and the digest of the file they were judged in. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct RecordedFile { + /// Workspace-relative path, as the mutants spell it. + path: Utf8PathBuf, + + /// The workspace package this file belongs to. + #[serde(default)] + package: String, + + /// A digest of the file's bytes as they were when these mutants failed to compile. + digest: String, + + /// The file's length in bytes when those mutants failed to compile. + /// + /// A cheap rejection ahead of the digest: a file of a different length is certainly a different + /// file, and saying so costs a `stat` where the digest costs a full read. It can only reject — + /// a matching length still has to be hashed, because two different files of the same length are + /// ordinary rather than exotic, and accepting one would drop a mutant that might have survived + /// out of the denominator. + size: u64, + + /// What this run settled about the mutants in this file. + mutants: Vec, +} + +/// One mutant's settled verdict, as the run that reached it left it. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Entry { + /// The mutant's content-addressed id. + id: MutantId, + + /// The verdict that was reached. + outcome: Outcome, + + /// The test that did the killing, when one did. + /// + /// Written for the same reason the JSON report carries `killedBy`: the mutant's identity hashes + /// the file, the item path, the mutator and the replacement, none of which change when somebody + /// deletes the test that caught it. Without a name to check, a kill cannot be revalidated and + /// must not be carried. + /// + /// Distinct from the entry in [`RunRecord::hints`], which names the same test for a different + /// purpose and on different terms: this one is evidence the verdict is still true, that one is a + /// guess about where to look first. + #[serde(default, skip_serializing_if = "Option::is_none")] + killed_by: Option, + + /// The workspace-relative file that declared `killed_by` before execution. + /// + /// The name alone is insufficient: moving a test with the same qualified name to another file + /// can change which target Cargo builds and what it links. This is present only when the + /// pre-execution scan could identify one unambiguously. + #[serde(default, skip_serializing_if = "Option::is_none")] + killer_file: Option, + + /// What the mutant cost to judge, in milliseconds. + #[serde(default)] + elapsed_ms: u64, +} + +/// Whether an outcome is safe to reuse without executing the mutant. +/// +/// Test outcomes are observations, not proof that the suite is deterministic. Recording only +/// compiler unviability ensures a cached observation can never contribute detection credit. +const fn settled_verdict(outcome: Outcome) -> bool { + matches!(outcome, Outcome::CompileError) +} + +impl RunRecord { + /// Captures the workspace before it is copied or executed. + #[cfg(test)] + #[must_use] + fn snapshot(root: &Utf8Path, scratch_base: &Utf8Path) -> WorkspaceSnapshot { + WorkspaceSnapshot::capture(root, &[scratch_base.to_path_buf()]) + } + + /// Captures a workspace together with local path dependencies Cargo resolves outside it. + #[must_use] + pub(crate) fn snapshot_with_external( + root: &Utf8Path, + scratch_base: &Utf8Path, + external_roots: &[Utf8PathBuf], + untracked_build_script_inputs: bool, + ) -> WorkspaceSnapshot { + WorkspaceSnapshot::capture_with_external(root, &[scratch_base.to_path_buf()], external_roots, untracked_build_script_inputs) + } + + /// Reads the record, or returns an empty one. + /// + /// Every failure is an empty cache rather than an error. A cache that cannot be read has cost + /// the run nothing but the time it takes to rebuild, which is the property that lets it be + /// deleted, corrupted or written by a different version without anyone having to care. + /// + /// The build context is not checked here, because there is no longer one answer to check it + /// against: each [`Tier`] names the terms it depends on, and [`Self::settled`], + /// [`Self::probes`] and [`Self::ordering`] apply their own. A file written under another + /// toolchain is a real record with a smaller usable part, not a foreign one. + #[must_use] + pub fn load(base: &Utf8Path) -> Self { + Self::load_raw(base).unwrap_or_default() + } + + /// Reads the record from disk, or nothing when it is absent, unreadable or a foreign format. + fn load_raw(base: &Utf8Path) -> Option { + let text = fs::read_to_string(base.join(FILE)).ok()?; + let record = serde_json::from_str::(&text).ok()?; + + (record.version == VERSION).then_some(record) + } + + /// What caught each mutant last time, whatever this run's build context is. + /// + /// No context is checked: a probe from another feature set is a wasted test process rather than + /// a wrong verdict, while discarding it would make the map cold on the runs it is worth the most + /// on. + #[must_use] + pub const fn probes(&self) -> &HashMap { + &self.hints + } + + /// Every verdict the record holds, paired with the mutant it belongs to. + /// + /// Offered without any invalidation of its own, because the callers are the tiers that need + /// none: the artifact promotion, which admits only what cannot move a score, and the build + /// order. Anything that *believes* a verdict goes through [`Self::settled`] instead, which + /// applies both the source digest and the tier's context terms. + pub fn entries(&self) -> impl Iterator { + self.files + .iter() + .flat_map(|file| file.mutants.iter()) + .map(|entry| (entry.id.as_str(), entry.outcome)) + } + + /// Whether this record holds anything that the [`Tier::Unviability`] rules govern. + /// + /// Asked so that a run can tell the reader which term of the context cost it the cache, and stay + /// quiet when the record held no unviability to lose in the first place. + #[must_use] + pub fn holds_unviability(&self) -> bool { + self.files + .iter() + .flat_map(|file| file.mutants.iter()) + .any(|entry| entry.outcome == Outcome::CompileError) + } + + /// The mutants this record saw fail to compile, offered as an order to build in. + /// + /// [`Tier::Ordering`] requires no term of the context and no source digest, which is what + /// separates it from [`Self::settled`]: this is not a claim that these mutants are unviable + /// now, only that they are the ones worth compiling first. Nothing downstream may withhold, + /// settle or exclude a mutant on the strength of it — every one of them is built, and the + /// compiler decides, exactly as it would have without the hint. + #[must_use] + pub fn ordering(&self) -> Vec<&str> { + let mut ids: Vec<&str> = self + .entries() + .filter(|(_id, outcome)| *outcome == Outcome::CompileError) + .map(|(id, _outcome)| id) + .collect(); + + // Sorted so that two runs over the same record hand the build the same order. The build + // itself keys by ordinal, but a hint set that iterated a map would still make the reported + // counts and any future tie-break depend on hash order. + ids.sort_unstable(); + ids.dedup(); + + ids + } + + /// The context this record was written under, which is provenance rather than a gate. + #[must_use] + pub const fn context(&self) -> &ContextDigest { + &self.context + } + + /// Replaces the hints in the record on disk, leaving every verdict where it is. + /// + /// Written by the sweep rather than at the end of the run, and written even when the sweep + /// failed: a run that stopped partway still learned which test caught every mutant it got to, + /// and discarding that would make an abandoned run cost the next one as much as it cost this + /// one. + /// + /// A failure is reported as a deferred note rather than failing the run. + pub fn store_probes(base: &Utf8Path, probes: &HashMap) { + let mut record = Self::load_raw(base).unwrap_or_default(); + + record.version = VERSION; + record.hints.clone_from(probes); + + let Ok(text) = serde_json::to_string(&record) else { + return; + }; + + if let Err(failure) = crate::elements::write(&base.join(FILE), &text) { + crate::notes::note(format!("could not save run-record probes: {failure}")); + } + } + + /// What this record settles, given the sources as they are now and how far it is believed. + /// + /// A changed workspace input contributes nothing. The pre-execution snapshot covers every + /// workspace file Cargo can consult, so a changed helper, manifest, lockfile, build script, + /// cargo configuration or test rejects the whole record rather than rebinding an old outcome + /// to new bytes. + /// + /// The other invalidation is the build context, applied per tier rather than to the file as a + /// whole. Both tiers require every applicable term: compiler and execution-policy changes can + /// alter verdicts as surely as source changes can. + /// + /// The run's context is resolved against `root` before either tier is asked, because the terms + /// that describe the workspace's own settings are the ones a caller cannot have filled in: it + /// built its context from a command line, and this is the first place the workspace is known. + /// The recorded side was resolved the same way when it was written. + /// + /// Deliberately conservative in one direction only. A file that cannot be read now is treated + /// as changed, so its mutants are re-tried; the cost of that is time, and the cost of the other + /// choice is signal. + /// + /// Returns the verdicts alongside the number of kills that were refused because the test that + /// did the killing is no longer declared anywhere in the workspace. The file digest cannot + /// catch that: a kill is a claim about the suite, and the suite is not the file the mutant + /// lives in. + #[must_use] + pub fn settled( + &self, + root: &Utf8Path, + trust: Trust, + killers: &Killers, + context: &ContextDigest, + ) -> (HashMap, usize) { + let current_inputs = self.inputs.recapture(root); + + self.settled_against(root, trust, killers, context, ¤t_inputs) + } + + pub(crate) fn settled_against( + &self, + root: &Utf8Path, + trust: Trust, + killers: &Killers, + context: &ContextDigest, + current_inputs: &WorkspaceSnapshot, + ) -> (HashMap, usize) { + let mut settled = HashMap::default(); + let mut declined = 0; + + if !current_inputs.is_complete() { + return (settled, declined); + } + + let workspace_unchanged = self.inputs == *current_inputs; + let current = context.resolved_at(root); + let unviability = Tier::Unviability.admits(&self.context, ¤t); + let verdicts = Tier::Verdict.admits(&self.context, ¤t); + + if !(unviability || verdicts && workspace_unchanged) { + return (settled, declined); + } + + let files_by_path: HashMap<&Utf8Path, &RecordedFile> = self.files.iter().map(|file| (file.path.as_path(), file)).collect(); + + for file in &self.files { + if !is_unchanged(file, current_inputs) { + continue; + } + + for entry in &file.mutants { + let admitted = if entry.outcome == Outcome::CompileError { + unviability + && (workspace_unchanged + || self + .compilation_roots + .get(&file.package) + .is_some_and(|roots| self.inputs.matches_compilation_inputs(current_inputs, roots))) + } else { + verdicts && workspace_unchanged && trust == Trust::Settled + }; + + if !admitted || entry.outcome == Outcome::Timeout { + continue; + } + + if entry.outcome == Outcome::Killed && !still_killed(entry, killers, &files_by_path, root, current_inputs) { + declined += 1; + + continue; + } + + let _previous = settled.insert(entry.id.clone(), entry.outcome); + } + } + + (settled, declined) + } + + /// Captures a record immediately, for callers outside an execution session. + /// + /// The run command uses [`Self::from_snapshot`] instead, with the snapshot captured before the + /// workspace was copied. This constructor serves operations that create synthetic records in + /// one uninterrupted step. + #[cfg(test)] + #[must_use] + pub fn from_run(root: &Utf8Path, mutants: &[Mutant], context: &ContextDigest, _source_dirs: &[Utf8PathBuf]) -> Self { + let inputs = WorkspaceSnapshot::capture(root, &[root.join(FILE)]); + let killers = Killers::scan(&inputs.rust_files(root)); + + Self::from_snapshot(root, mutants, context, inputs, &killers).unwrap_or_default() + } + + /// Builds a record from outcomes bound to their pre-execution workspace snapshot. + /// + /// No current workspace bytes are used to stamp an outcome. If anything Cargo could have read + /// changed after that snapshot, there is no safe attribution and the caller receives `None`. + /// This costs the next run work rather than letting an old execution verdict acquire new + /// provenance. + #[cfg(test)] + #[must_use] + fn from_snapshot( + root: &Utf8Path, + mutants: &[Mutant], + context: &ContextDigest, + inputs: WorkspaceSnapshot, + killers: &Killers, + ) -> Option { + let compilation_roots = mutants + .iter() + .map(|mutant| ((*mutant.package).to_owned(), vec![Utf8PathBuf::new()])) + .collect(); + + Self::from_snapshot_with_roots(root, mutants, context, inputs, killers, compilation_roots) + } + + #[must_use] + pub(crate) fn from_plan_snapshot(plan: &Plan, context: &ContextDigest, inputs: WorkspaceSnapshot, killers: &Killers) -> Option { + let mut compilation_roots = HashMap::default(); + + for package in plan.specs.keys() { + let mut dependencies = plan.reach.get(package).cloned().unwrap_or_default(); + let _self = dependencies.insert(package.clone()); + let mut roots: Vec = dependencies + .iter() + .filter_map(|dependency| plan.specs.get(dependency).map(|(root, _version)| root.clone())) + .collect(); + + roots.sort(); + roots.dedup(); + let _previous = compilation_roots.insert(package.clone(), roots); + } + + Self::from_snapshot_with_roots(&plan.root, &plan.mutants, context, inputs, killers, compilation_roots) + } + + fn from_snapshot_with_roots( + root: &Utf8Path, + mutants: &[Mutant], + context: &ContextDigest, + inputs: WorkspaceSnapshot, + killers: &Killers, + compilation_roots: HashMap>, + ) -> Option { + if !inputs.matches_current(root) { + return None; + } + + let packages: HashMap = mutants + .iter() + .map(|mutant| (mutant.file.to_path_buf(), (*mutant.package).to_owned())) + .collect(); + let mut by_file: HashMap> = inputs + .files + .iter() + .filter(|file| file.path.extension() == Some("rs")) + .map(|file| (file.path.clone(), Vec::new())) + .collect(); + + for mutant in mutants.iter().filter(|mutant| settled_verdict(mutant.outcome)) { + let file = mutant.file.to_path_buf(); + let _known = inputs.file(&file)?; + let killer_file = if mutant.outcome == Outcome::Killed { + mutant.killed_by.as_deref().and_then(|name| { + killers + .verdict_file_for(name) + .and_then(|path| path.strip_prefix(root).ok()) + .map(Utf8Path::to_path_buf) + }) + } else { + None + }; + + by_file.entry(file).or_default().push(Entry { + id: mutant.id.clone(), + outcome: mutant.outcome, + killed_by: mutant.killed_by.clone(), + killer_file, + elapsed_ms: mutant.elapsed_ms, + }); + } + + let mut files: Vec = by_file + .into_iter() + .filter_map(|(path, mut mutants)| { + let input = inputs.file(&path)?; + + mutants.sort_by(|left, right| left.id.cmp(&right.id)); + mutants.dedup_by(|left, right| left.id == right.id); + + Some(RecordedFile { + package: packages.get(&path).cloned().unwrap_or_default(), + path, + digest: input.digest.clone(), + size: input.size, + mutants, + }) + }) + .collect(); + + files.sort_by(|left, right| left.path.cmp(&right.path)); + + Some(Self { + version: VERSION, + context: context.resolved_at(root), + files, + inputs, + compilation_roots, + hints: HashMap::default(), + }) + } + + /// Writes the cache over whatever an earlier run left, keeping what this run did not revisit. + /// + /// A run knows only about the files it looked at. `--in-diff`, a single package and a shard all + /// narrow that to a handful, and a plain overwrite would throw away every entry for everything + /// else — so a sequence of narrow runs, which is exactly the workflow narrowing exists for, + /// would never accumulate a cache at all. + /// + /// This run wins outright for any file it has an entry for: it just built those mutants, so its + /// answer is the current one and the older answer is not evidence against it. Every other file + /// is carried forward only if it is still on disk unchanged, which is the same rule + /// [`RunRecord::settled`] applies when reading — an entry that would not be believed is not + /// worth keeping, and dropping it here is what keeps the union bounded by the files that still + /// exist rather than by every file that ever did. + /// + /// A run that could not write its cache has still produced every verdict it was asked for, so + /// a failure is reported as a deferred note rather than making the optimization a dependency. + pub fn store(&self, base: &Utf8Path, root: &Utf8Path) { + let earlier = Self::load_raw(base).unwrap_or_default(); + let merged = self.absorbing(&earlier); + + if !merged.inputs.matches_current(root) { + return; + } + + let Ok(text) = serde_json::to_string(&merged) else { + return; + }; + + if let Err(failure) = crate::elements::write(&base.join(FILE), &text) { + crate::notes::note(format!("could not save run record: {failure}")); + } + } + + /// This cache, plus the entries of `earlier` for files this run never visited. + fn absorbing(&self, earlier: &Self) -> Self { + let workspace_unchanged = earlier.inputs == self.inputs; + let unviability = Tier::Unviability.admits(&earlier.context, &self.context); + let verdicts = Tier::Verdict.admits(&earlier.context, &self.context) && workspace_unchanged; + let mut carried = Vec::new(); + + for file in &earlier.files { + let compilation_unchanged = earlier + .compilation_roots + .get(&file.package) + .is_some_and(|roots| earlier.inputs.matches_compilation_inputs(&self.inputs, roots)); + let mut admitted = file.clone(); + + admitted.mutants.retain(|entry| { + if entry.outcome == Outcome::CompileError { + unviability && compilation_unchanged + } else { + verdicts + } + }); + + if !admitted.mutants.is_empty() { + carried.push(admitted); + } + } + + let carried_map: HashMap<&Utf8PathBuf, &RecordedFile> = carried.iter().map(|f| (&f.path, f)).collect(); + + let mut seen = HashSet::default(); + let mut files: Vec = Vec::new(); + + for file in &self.files { + let _ = seen.insert(&file.path); + if !file.mutants.is_empty() { + files.push(file.clone()); + } else if let Some(earlier_file) = carried_map.get(&file.path) { + if !earlier_file.mutants.is_empty() && is_unchanged(earlier_file, &self.inputs) { + files.push(RecordedFile { + path: file.path.clone(), + package: earlier_file.package.clone(), + digest: file.digest.clone(), + size: file.size, + mutants: earlier_file.mutants.clone(), + }); + } else { + files.push(file.clone()); + } + } else { + files.push(file.clone()); + } + } + + for file in &carried { + if !seen.contains(&file.path) && is_unchanged(file, &self.inputs) { + files.push(file.clone()); + } + } + + files.sort_by(|left, right| left.path.cmp(&right.path)); + let mut compilation_roots = self.compilation_roots.clone(); + + for file in &files { + if !compilation_roots.contains_key(&file.package) + && let Some(roots) = earlier.compilation_roots.get(&file.package) + { + let _previous = compilation_roots.insert(file.package.clone(), roots.clone()); + } + } + + Self { + version: VERSION, + context: self.context.clone(), + files, + inputs: self.inputs.clone(), + compilation_roots, + hints: if self.hints.is_empty() { + earlier.hints.clone() + } else { + self.hints.clone() + }, + } + } + + /// How many mutants this record holds. + /// + /// Only the tests ask. A run counts what the record actually spared it against the population + /// instead, because an entry whose mutant is no longer there spared nobody anything. + #[cfg(test)] + fn len(&self) -> usize { + self.files.iter().map(|file| file.mutants.len()).sum() + } +} + +/// Whether a kill this record holds is still a kill. +/// +/// A mutant's identity says nothing about the tests, so a kill is accepted only when its recorded +/// killer still has the same identity and contents. +fn still_killed( + entry: &Entry, + killers: &Killers, + files_by_path: &HashMap<&Utf8Path, &RecordedFile>, + root: &Utf8Path, + current_inputs: &WorkspaceSnapshot, +) -> bool { + let Some(name) = entry.killed_by.as_deref() else { + return false; + }; + let Some(recorded_path) = entry.killer_file.as_deref() else { + return false; + }; + let Some(test_file_path) = killers.verdict_file_for(name) else { + return false; + }; + let Ok(current_path) = test_file_path.strip_prefix(root) else { + return false; + }; + + if current_path != recorded_path { + return false; + } + + let Some(recorded) = files_by_path.get(recorded_path) else { + return false; + }; + + killers.file_digest(test_file_path) == Some(recorded.digest.as_str()) && is_unchanged(recorded, current_inputs) +} + +/// Whether the current workspace snapshot has a recorded file exactly as it was when judged. +/// +/// The snapshot already read and hashed every included file. Reusing that result avoids a second +/// content pass while keeping absence or an incomplete capture on the conservative side. +fn is_unchanged(file: &RecordedFile, current_inputs: &WorkspaceSnapshot) -> bool { + current_inputs + .file(&file.path) + .is_some_and(|current| current.size == file.size && current.digest == file.digest) +} + +/// A hex digest of some bytes. +pub(crate) fn digest(bytes: &[u8]) -> String { + let mut hasher = Hasher::new(); + let _ = hasher.update(bytes); + + hasher.finalize().to_hex().to_string() +} + +/// Reports the compiler, cargo and wrapper identities, or nothing if either tool cannot be asked. +/// +/// Failing to name the toolchain is not an error, but it is not harmless either, and the earlier +/// spelling of this — an empty string — got that backwards. An empty answer does not make the cache +/// *less* useful; it removes an invalidation axis entirely, so two runs under different compilers +/// hash the same and the second believes the first. `RUSTC` pointing at a wrapper that does not +/// answer `--version` would make that permanent and silent. `None` says so, and [`context`] turns +/// it into "do not use a cache at all", which costs a run some time instead of a mutant. +#[must_use] +pub fn toolchain() -> Option { + let program = env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let rustc = Command::new(&program) + .arg("-vV") + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())?; + let cargo_program = env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let cargo = Command::new(&cargo_program) + .arg("--version") + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())?; + let wrapper = env::var_os("RUSTC_WRAPPER").unwrap_or_default(); + let workspace_wrapper = env::var_os("RUSTC_WORKSPACE_WRAPPER").unwrap_or_default(); + + Some(format!( + "rustc={}\ncargo={}\nrustc_wrapper={}\nrustc_workspace_wrapper={}\n{rustc}\n{cargo}", + program.to_string_lossy(), + cargo_program.to_string_lossy(), + wrapper.to_string_lossy(), + workspace_wrapper.to_string_lossy() + )) +} + +/// The flags cargo will read from the ambient environment, if any. +/// +/// Cargo takes the global ones from the first variable that has anything to say rather than merging +/// them: `CARGO_ENCODED_RUSTFLAGS`, then `RUSTFLAGS`, then `CARGO_BUILD_RUSTFLAGS`, which is the +/// variable spelling of `build.rustflags` and decides the build exactly as the other two do. Only +/// one of them is ever in force and only that one belongs in the key. A `--cfg` passed any of these +/// ways selects different code entirely — the design notes `--cfg loom` as an expected scenario — +/// so a cache blind to it carries unviability from a tree that was never the tree being built. +/// +/// `CARGO_TARGET__RUSTFLAGS` sits at its own precedence level rather than in that chain, so +/// it is added to the key rather than competing for it. Every such variable is added, not the one +/// naming the triple in force: the triple is not settled at the point this is read, and being +/// over-inclusive costs a cache that could have been kept while being under-inclusive costs a +/// mutant silently withheld from the denominator. +/// +/// The file spelling of the same settings is not here, because it is not in the environment: the +/// configuration's own `build.rustflags` and target tables reach the key through [`Term::Config`]. +#[must_use] +pub fn rustflags() -> Option { + let mut targeted: Vec<(String, String)> = env::vars_os() + .filter_map(|(name, value)| { + let name = name.into_string().ok()?; + + (name.starts_with("CARGO_TARGET_") && name.ends_with("_RUSTFLAGS")) + .then(|| Some((name, value.into_string().ok()?))) + .flatten() + }) + .collect(); + + targeted.sort(); + + rustflags_in( + env::var("CARGO_ENCODED_RUSTFLAGS").ok(), + env::var("RUSTFLAGS").ok(), + env::var("CARGO_BUILD_RUSTFLAGS").ok(), + &targeted, + ) +} + +/// Picks the flags out of the three global variables, in the order cargo consults them, and appends +/// the target-specific ones, which cargo reads independently of that order. +/// +/// Taken as values rather than read here so that a test can vary them without writing the process +/// environment. +fn rustflags_in( + encoded: Option, + plain: Option, + configured: Option, + targeted: &[(String, String)], +) -> Option { + let chosen = encoded.or(plain).or(configured); + + if targeted.is_empty() { + return chosen; + } + + let mut text = chosen.unwrap_or_default(); + + for (name, value) in targeted { + text.push('\n'); + text.push_str(name); + text.push('='); + text.push_str(value); + } + + Some(text) +} + +/// Everything other than the sources that decides whether a mutant compiles. +/// +/// Gathered into one value rather than passed as seven arguments so that adding an axis is a +/// compile error at every construction site, which is the failure this type exists to prevent: the +/// cache is *believed*, so a missing axis is a mutant silently dropped from the denominator — +/// a real gap in the suite reported as a better score. +#[derive(Debug, Default, Clone, Copy)] +#[expect( + clippy::struct_excessive_bools, + reason = "captures discrete boolean CLI and build flags for context digest" +)] +pub struct Context<'a> { + /// The feature arguments, as the command line gave them. + pub features: &'a [String], + + /// Whether every feature was asked for. + pub all_features: bool, + + /// Whether the default features were suppressed. + pub no_default_features: bool, + + /// The cargo profile, which decides `debug_assertions` among other things. + pub profile: Option<&'a str>, + + /// The arbitrary passthrough build arguments, which can carry `--cfg`, `-C` or `--target`. + pub extra: &'a [String], + + /// Whichever of the two rustflags variables cargo will actually read. + pub rustflags: Option<&'a str>, + + /// The compiler, cargo and wrapper identities, or `None` when either tool could not be asked. + pub toolchain: Option<&'a str>, + + /// Test packages filter. + pub test_packages: &'a [String], + + /// Test names to include. + pub include_tests: &'a [String], + + /// Test names to exclude. + pub exclude_tests: &'a [String], + + /// Whether to test the whole workspace. + pub test_workspace: bool, + + /// Whether to run reachable test binaries whole instead of selecting individual test cases. + pub whole_test_binaries: bool, + + /// Whether to run test binaries through `cargo nextest`. + pub nextest: bool, + + /// Arguments added to every test binary with `--cargo-test-arg`. + pub cargo_test_args: &'a [String], + + /// Arguments after the command-line `--` separator. + pub test_args: &'a [String], + + /// Whether the healthy baseline was measured. + pub baseline: bool, + + /// Whether a failing test was confirmed without a mutant. + pub confirm: bool, + + /// Whether silence can produce a stall verdict. + pub stall: bool, + + /// The multiplier for baseline-derived test timeouts. + pub test_timeout_multiplier: Option, + + /// The lower bound for baseline-derived test timeouts. + pub minimum_test_timeout: Option, + + /// Requested memory-control mode. + pub memory: Option, + + /// The baseline-memory multiplier for a mutant's limit. + pub memory_multiplier: Option, + + /// The fixed memory headroom for a mutant's limit. + pub memory_headroom: Option, + + /// An explicit memory limit for a mutant. + pub memory_limit: Option, + + /// An explicit memory limit for the baseline. + pub baseline_memory_limit: Option, + + /// Whether memory-control relaunching was disabled. + pub no_relaunch: bool, + + /// Whether ignored workspace files were copied into the build. + pub copy_ignored: bool, + + /// Requested mutant concurrency, which can affect timeout scheduling. + pub jobs: Option, + + /// Fixed build timeout. + pub build_timeout: Option, + + /// Build timeout multiplier. + pub build_timeout_multiplier: Option, + + /// Maximum rollback build rounds. + pub rollback_rounds: u32, +} + +/// Digests a [`Context`], or returns nothing when it cannot be trusted to be complete. +/// +/// Features first, because they are the axis that actually moves: a workspace routinely has code +/// that only exists under one of them, and a cache that ignored them would carry unviability from a +/// run that never compiled the code in question. +/// +/// The tool's own version is a term because [`Mutant::id`](crate::model::Mutant) — what the cache is +/// keyed by — hashes the item path, the mutator, the normalized site text, the occurrence and the +/// replacement index, every one of which is a property of *this tool*. An upgrade can change what an +/// id denotes while the sources, the features and the toolchain are all unchanged. `VERSION` guards +/// the file's format; this guards the meaning of the keys inside it. +/// +/// Each term is digested on its own rather than folded into one hash, which is what lets a [`Tier`] +/// require some terms and not others. Within a term the parts are length-prefixed, so that no +/// rearrangement of them can produce the same digest as a different one — `["ab", "c"]` and +/// `["a", "bc"]` are different feature selections. +/// +/// A `None` toolchain gives `None` here, which is what makes a compiler that cannot be asked +/// suppress the cache rather than silently drop out of the key. +/// +/// [`Term::Config`] is left unstated, because the settings it covers live in the workspace and this +/// is called before the workspace has been located. `ContextDigest::resolved_at` fills it in +/// where the root is in hand, which is both ends of every comparison the gate makes. +#[must_use] +pub fn context(of: &Context<'_>) -> Option { + let environment = inherited_environment(); + + context_in(of, env::var("CARGO_BUILD_TARGET").ok().as_deref(), &environment) +} + +/// Digests a context against a given `CARGO_BUILD_TARGET`, so a test can vary the one variable +/// this reads for itself. +/// +/// The variable is taken as a value rather than looked up where it is needed because the workspace +/// forbids writing the process environment. Everything else the digest covers arrives through +/// [`Context`] or through the workspace root. +fn context_in(of: &Context<'_>, build_target: Option<&str>, environment: &[(Vec, Vec)]) -> Option { + let toolchain = of.toolchain?; + + let mut features: Vec<&[u8]> = of.features.iter().map(String::as_bytes).collect(); + let flags = [u8::from(of.all_features), u8::from(of.no_default_features)]; + + features.push(&flags); + + let targets = Build::requested_targets(of.extra, build_target); + let named: Vec<&[u8]> = targets.iter().map(String::as_bytes).collect(); + + let mut test_parts: Vec<&[u8]> = Vec::new(); + for pkg in of.test_packages { + test_parts.push(pkg.as_bytes()); + } + test_parts.push(b":inc:"); + for inc in of.include_tests { + test_parts.push(inc.as_bytes()); + } + test_parts.push(b":exc:"); + for exc in of.exclude_tests { + test_parts.push(exc.as_bytes()); + } + // The tag differs from the bit spelling that preceded it so that records written under the + // opposite default are invalidated rather than believed. + test_parts.push(b":case-reachability-default:"); + let test_flags = [u8::from(of.test_workspace), u8::from(of.whole_test_binaries), u8::from(of.nextest)]; + test_parts.push(&test_flags); + + let mut policy_parts: Vec<&[u8]> = Vec::new(); + policy_parts.push(b":cargo-test:"); + policy_parts.extend(of.cargo_test_args.iter().map(String::as_bytes)); + policy_parts.push(b":post--:"); + policy_parts.extend(of.test_args.iter().map(String::as_bytes)); + let policy = format!( + "baseline={};confirm={};stall={};timeout_multiplier={:?};timeout_floor={:?};memory={:?};\ + memory_multiplier={:?};memory_headroom={:?};memory_limit={:?};baseline_memory_limit={:?};\ + no_relaunch={};copy_ignored={};jobs={:?};build_timeout={:?};build_timeout_multiplier={:?};\ + rollback_rounds={}", + of.baseline, + of.confirm, + of.stall, + of.test_timeout_multiplier, + of.minimum_test_timeout, + of.memory, + of.memory_multiplier, + of.memory_headroom, + of.memory_limit, + of.baseline_memory_limit, + of.no_relaunch, + of.copy_ignored, + of.jobs, + of.build_timeout, + of.build_timeout_multiplier, + of.rollback_rounds + ); + policy_parts.push(policy.as_bytes()); + + let mut environment_parts = Vec::with_capacity(environment.len().saturating_mul(2)); + for (name, value) in environment { + environment_parts.push(name.as_slice()); + environment_parts.push(value.as_slice()); + } + + Some(ContextDigest { + features: term(Term::Features, &features), + profile: term(Term::Profile, &[of.profile.unwrap_or_default().as_bytes()]), + rustflags: term(Term::Rustflags, &[of.rustflags.unwrap_or_default().as_bytes()]), + target: Some(term(Term::Target, &named)), + config: None, + extra: term(Term::Extra, &of.extra.iter().map(String::as_bytes).collect::>()), + toolchain: term(Term::Toolchain, &[toolchain.as_bytes()]), + tool: term(Term::Tool, &[env!("CARGO_PKG_VERSION").as_bytes()]), + tests: term(Term::Tests, &test_parts), + policy: term(Term::Policy, &policy_parts), + environment: Some(term(Term::Environment, &environment_parts)), + }) +} + +/// The process environment that reaches a test binary without gamma naming it. +/// +/// Test commands inherit their parent's environment and then overlay only gamma's control +/// variables. Hash every inherited name and value rather than trying to guess which application +/// variables a test, fixture, subprocess or build-produced helper consumes. The digest is sorted +/// and length-prefixed by [`term`], so its value is stable across enumeration order and never +/// serializes the environment's raw contents into the run record. +fn inherited_environment() -> Vec<(Vec, Vec)> { + let mut variables: Vec<(Vec, Vec)> = env::vars_os() + .map(|(name, value)| (name.as_encoded_bytes().to_vec(), value.as_encoded_bytes().to_vec())) + .collect(); + + variables.sort_unstable(); + variables +} + +/// Which term covers each input the build resolution reads, so the two modules cannot drift apart. +/// +/// [`Build::INPUTS`](crate::cfg::Build::INPUTS) is where those inputs are named, beside the code +/// that reads them; this says what the record does about each one. The pair is checked by a test +/// rather than by the type system because the inputs are strings in a configuration format and +/// there is no type that could hold them — but an input added there and not here fails that test, +/// which is the drift this exists to stop. Unviability is withheld from the denominator on the +/// strength of these terms, so an input nobody covers is a mutant nobody counts. +#[cfg(test)] +const COVERAGE: &[(&str, Term)] = &[ + ("CARGO_BUILD_TARGET", Term::Target), + ("CARGO_ENCODED_RUSTFLAGS", Term::Rustflags), + ("RUSTFLAGS", Term::Rustflags), + ("CARGO_BUILD_RUSTFLAGS", Term::Rustflags), + ("CARGO_TARGET__RUSTFLAGS", Term::Rustflags), + // Only ever a way of naming a file. Whatever it selects is read into the settings themselves, + // so the values are covered wherever the home happens to be. + ("CARGO_HOME", Term::Config), + ("cargo config files", Term::Config), + ("build.target", Term::Config), + ("build.rustflags", Term::Config), + ("target.*.rustflags", Term::Config), + ("profile.*", Term::Config), +]; + +/// Digests the parts of one context term. +/// +/// The term's own name goes in first, so two terms that happen to hold the same bytes — an unset +/// profile and unset rustflags are both the empty string — still digest differently. Nothing +/// compares one term's digest against another's, so this changes no decision; it exists so that a +/// reader comparing two envelopes by eye, or a future term that is derived rather than compared +/// field by field, cannot be misled by a coincidence. +fn term(name: Term, parts: &[&[u8]]) -> String { + let mut hasher = Hasher::new(); + + let named = name.name().as_bytes(); + let _ = hasher.update(&(named.len() as u64).to_le_bytes()); + let _ = hasher.update(named); + + for part in parts { + // Length-prefixed, so that no rearrangement of the parts can produce the same digest as a + // different one. + let _ = hasher.update(&(part.len() as u64).to_le_bytes()); + let _ = hasher.update(part); + } + + hasher.finalize().to_hex().to_string() +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::sync::{Arc, Barrier}; + use std::thread; + + use super::*; + use crate::fixtures; + use crate::testing::workdir; + + fn mutant(id: &str, file: &str, outcome: Outcome) -> Mutant { + Mutant { + id: id.to_owned().into(), + file: (Utf8PathBuf::from(file)).into(), + mutator: ("arith".to_owned()).into(), + item_path: ("subject::add".to_owned()).into(), + original: "+".to_owned().into(), + replacement: "-".to_owned().into(), + outcome, + ..fixtures::mutant() + } + } + + fn workspace(prefix: &str, body: &str) -> (tempfile::TempDir, Utf8PathBuf) { + let dir = workdir(prefix); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("the work directory should be UTF-8"); + + fs::create_dir_all(root.join("src")).expect("the source directory should be creatable"); + fs::write(root.join("src/lib.rs"), body).expect("the source should be writable"); + + (dir, root) + } + + /// Indexes the tests declared by a file written into the workspace for the purpose. + fn killers(root: &Utf8PathBuf, body: &str) -> Killers { + let path = root.join("src/tests.rs"); + + fs::write(&path, body).expect("the test source should be writable"); + + Killers::scan(&[path]) + } + + fn killed(id: &str, by: Option<&str>) -> Mutant { + Mutant { + killed_by: by.map(str::to_owned), + elapsed_ms: 42, + ..mutant(id, "src/lib.rs", Outcome::Killed) + } + } + + /// The build context every test writes and reads under unless it is varying one. + fn envelope() -> ContextDigest { + context(&plain()).expect("a named toolchain gives a context") + } + + fn from_run(root: &Utf8Path, mutants: &[Mutant], context: &ContextDigest) -> RunRecord { + RunRecord::from_run(root, mutants, context, &[root.join("src"), root.join("tests")]) + } + + fn package_mutant(id: &str, package: &str, file: &str, outcome: Outcome) -> Mutant { + Mutant { + package: package.to_owned().into(), + ..mutant(id, file, outcome) + } + } + + fn cache_plan(root: &Utf8Path, mutants: Vec, a_depends_on_b: bool) -> Plan { + let mut a_reach = HashSet::default(); + let _inserted = a_reach.insert("a".to_owned()); + if a_depends_on_b { + let _inserted = a_reach.insert("b".to_owned()); + } + let mut b_reach = HashSet::default(); + let _inserted = b_reach.insert("b".to_owned()); + let mut reach = HashMap::default(); + let _previous = reach.insert("a".to_owned(), a_reach); + let _previous = reach.insert("b".to_owned(), b_reach); + let mut specs = HashMap::default(); + let _previous = specs.insert("a".to_owned(), (Utf8PathBuf::from("crates/a"), "0.1.0".to_owned())); + let _previous = specs.insert("b".to_owned(), (Utf8PathBuf::from("crates/b"), "0.1.0".to_owned())); + + Plan { + root: root.to_owned(), + files: Vec::new(), + mutants, + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + digests: HashMap::default(), + skipped: Vec::new(), + reach, + specs, + } + } + + fn cache_workspace(prefix: &str) -> (tempfile::TempDir, Utf8PathBuf) { + let dir = workdir(prefix); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("the work directory should be UTF-8"); + + for package in ["a", "b"] { + fs::create_dir_all(root.join(format!("crates/{package}/src"))).expect("package source directory"); + fs::write( + root.join(format!("crates/{package}/Cargo.toml")), + format!("[package]\nname = \"{package}\"\nversion = \"0.1.0\"\n"), + ) + .expect("package manifest"); + fs::write( + root.join(format!("crates/{package}/src/lib.rs")), + format!("pub fn {package}() -> bool {{ true }}\n"), + ) + .expect("package source"); + } + + fs::write(root.join("Cargo.toml"), "[workspace]\nmembers = [\"crates/a\", \"crates/b\"]\n").expect("workspace manifest"); + fs::write(root.join("Cargo.lock"), "# lock\n").expect("workspace lockfile"); + + (dir, root) + } + + fn from_plan(plan: &Plan) -> RunRecord { + let inputs = WorkspaceSnapshot::capture(&plan.root, &[]); + + RunRecord::from_plan_snapshot(plan, &envelope(), inputs, &Killers::default()) + .expect("an unchanged workspace should produce a record") + } + + /// The same context under a different compiler, and identical in every other term. + fn under_another_toolchain() -> ContextDigest { + context(&Context { + toolchain: Some("1.91.0"), + ..plain() + }) + .expect("a named toolchain gives a context") + } + + /// The same context under a different feature selection, and identical in every other term. + fn under_another_feature_set() -> ContextDigest { + let features = ["extra".to_owned()]; + + context(&Context { + features: &features, + ..plain() + }) + .expect("a named toolchain gives a context") + } + + /// No observed test verdict can become durable evidence about a later run. + #[test] + fn only_compiler_unviability_is_recorded() { + let (_dir, root) = workspace("record-unsettled-", "fn add() {}"); + let population = [ + mutant("killed", "src/lib.rs", Outcome::Killed), + mutant("survivor", "src/lib.rs", Outcome::Survived), + mutant("flake", "src/lib.rs", Outcome::Flaky), + mutant("unbuilt", "src/lib.rs", Outcome::NotBuilt), + mutant("hungry", "src/lib.rs", Outcome::OutOfMemory), + mutant("uncovered", "src/lib.rs", Outcome::NoCoverage), + mutant("pending", "src/lib.rs", Outcome::Pending), + mutant("slow", "src/lib.rs", Outcome::Timeout), + mutant("skipped", "src/lib.rs", Outcome::Ignored), + mutant("unviable", "src/lib.rs", Outcome::CompileError), + ]; + + let record = from_run(&root, &population, &envelope()); + let (settled, _declined) = record.settled(&root, Trust::Settled, &Killers::default(), &envelope()); + + assert_eq!(settled.len(), 1, "{settled:?}"); + assert_eq!(settled.get("unviable"), Some(&Outcome::CompileError)); + } + + #[test] + fn a_cached_mutant_is_settled_when_its_file_is_unchanged() { + let (_dir, root) = workspace("record-hit-", "fn add() {}"); + let cache = from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()); + + assert_eq!( + cache.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.get("abc"), + Some(&Outcome::CompileError) + ); + } + + /// The invalidation that makes this safe. A mutant that could not compile against one version + /// of the surrounding types may compile against the next, and assuming otherwise would drop a + /// mutant that might have survived out of the denominator. + #[test] + fn a_cached_mutant_is_retried_when_its_file_changed() { + let (_dir, root) = workspace("record-miss-", "fn add() {}"); + let cache = from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()); + + fs::write(root.join("src/lib.rs"), "fn add(a: i32) -> i32 { a }").expect("the source should be rewritable"); + + assert!( + cache.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.is_empty(), + "an edited file must not carry its unviability" + ); + } + + /// A file that has been deleted or moved is treated as changed, because the only alternative is + /// to assume something about source nobody can read. + #[test] + fn a_cached_mutant_is_retried_when_its_file_is_gone() { + let (_dir, root) = workspace("record-gone-", "fn add() {}"); + let cache = from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()); + + fs::remove_file(root.join("src/lib.rs")).expect("the source should be removable"); + + assert!(cache.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.is_empty()); + } + + #[test] + fn an_unrelated_package_change_preserves_unviability_but_not_test_verdicts() { + let (_dir, root) = cache_workspace("record-package-input-"); + let unviable = package_mutant("unviable", "a", "crates/a/src/lib.rs", Outcome::CompileError); + let survived = package_mutant("survived", "a", "crates/a/src/lib.rs", Outcome::Survived); + let plan = cache_plan(&root, vec![unviable, survived], false); + let record = from_plan(&plan); + + fs::write(root.join("crates/b/src/lib.rs"), "pub fn b() -> bool { false }\n").expect("unrelated source edit"); + + let settled = record.settled(&root, Trust::Settled, &Killers::default(), &envelope()).0; + + assert_eq!(settled.get("unviable"), Some(&Outcome::CompileError)); + assert!(!settled.contains_key("survived")); + } + + #[test] + fn a_narrow_run_writes_back_safe_unviability_after_an_unrelated_change() { + let (_dir, root) = cache_workspace("record-package-writeback-"); + let unviable = package_mutant("unviable", "a", "crates/a/src/lib.rs", Outcome::CompileError); + let survived = package_mutant("survived", "a", "crates/a/src/lib.rs", Outcome::Survived); + let earlier_plan = cache_plan(&root, vec![unviable, survived], false); + let earlier = from_plan(&earlier_plan); + + fs::write(root.join("crates/b/src/lib.rs"), "pub fn b() -> bool { false }\n").expect("unrelated source edit"); + let fresh = package_mutant("fresh", "b", "crates/b/src/lib.rs", Outcome::CompileError); + let current_plan = cache_plan(&root, vec![fresh], false); + let current = from_plan(¤t_plan); + let merged = current.absorbing(&earlier); + let settled = merged.settled(&root, Trust::Settled, &Killers::default(), &envelope()).0; + + assert_eq!(settled.get("unviable"), Some(&Outcome::CompileError)); + assert_eq!(settled.get("fresh"), Some(&Outcome::CompileError)); + assert!(!settled.contains_key("survived")); + } + + /// The denominator safety property behind fine-grained invalidation: a source edit that makes + /// an old compile failure viable must force that mutant back through compilation. + #[test] + fn a_package_source_change_never_carries_stale_unviability() { + let (_dir, root) = cache_workspace("record-package-source-"); + let unviable = package_mutant("unviable", "a", "crates/a/src/lib.rs", Outcome::CompileError); + let plan = cache_plan(&root, vec![unviable], false); + let record = from_plan(&plan); + + fs::write(root.join("crates/a/src/lib.rs"), "pub fn a() -> bool { false }\n").expect("package source edit"); + + assert!( + record.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.is_empty(), + "a newly viable mutant must remain in the denominator" + ); + } + + #[test] + fn a_workspace_dependency_change_invalidates_dependent_unviability() { + let (_dir, root) = cache_workspace("record-dependency-input-"); + let unviable = package_mutant("unviable", "a", "crates/a/src/lib.rs", Outcome::CompileError); + let plan = cache_plan(&root, vec![unviable], true); + let record = from_plan(&plan); + + fs::write(root.join("crates/b/src/lib.rs"), "pub fn b() -> bool { false }\n").expect("dependency source edit"); + + assert!(record.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.is_empty()); + } + + #[test] + fn every_global_and_package_compilation_input_invalidates_unviability() { + let edits = [ + ("Cargo.toml", "[workspace]\nmembers = []\n"), + ("Cargo.lock", "# changed lock\n"), + (".cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"changed\"]\n"), + ("rust-toolchain.toml", "[toolchain]\nchannel = \"stable\"\n"), + ("rust-toolchain", "1.72.0\n"), + ("crates/a/Cargo.toml", "[package]\nname = \"a\"\nversion = \"0.2.0\"\n"), + ("crates/a/build.rs", "fn main() { println!(\"cargo:rustc-cfg=changed\"); }\n"), + ]; + + for (path, contents) in edits { + let (_dir, root) = cache_workspace("record-compilation-input-"); + let unviable = package_mutant("unviable", "a", "crates/a/src/lib.rs", Outcome::CompileError); + let plan = cache_plan(&root, vec![unviable], false); + let record = from_plan(&plan); + let changed = root.join(path); + + fs::create_dir_all(changed.parent().expect("fixture paths have parents")).expect("fixture parent"); + fs::write(changed, contents).expect("compilation input edit"); + + assert!( + record.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.is_empty(), + "{path} retained stale unviability" + ); + } + } + + #[test] + fn every_workspace_cargo_input_invalidates_a_carried_outcome() { + let edits = [ + ("src/helper.rs", "pub fn answer() -> u32 { 1 }\n", "pub fn answer() -> u32 { 2 }\n"), + ("tests/behaviour.rs", "#[test]\nfn answer() {}\n", "#[test]\nfn answer_now() {}\n"), + ( + "Cargo.toml", + "[package]\nname = \"subject\"\n", + "[package]\nname = \"subject-two\"\n", + ), + ("Cargo.lock", "version = 4\n", "version = 4\n# changed\n"), + ( + ".cargo/config.toml", + "[build]\ntarget-dir = \"build\"\n", + "[build]\ntarget-dir = \"other\"\n", + ), + ( + "build.rs", + "fn main() {}\n", + "fn main() { println!(\"cargo:rerun-if-changed=x\"); }\n", + ), + ]; + + for (path, before, after) in edits { + let (_dir, root) = workspace("record-workspace-input-", "fn add() {}"); + let file = root.join(path); + fs::create_dir_all(file.parent().expect("fixture paths have parents").as_std_path()).expect("fixture parent"); + fs::write(file.as_std_path(), before).expect("fixture input"); + let record = from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()); + + fs::write(file.as_std_path(), after).expect("changed input"); + + assert!( + record.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.is_empty(), + "{path} changed without invalidating the outcome" + ); + } + } + + #[test] + fn an_external_path_dependency_change_carries_neither_verdicts_nor_unviability() { + let directory = workdir("record-external-path-"); + let container = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the work directory should be UTF-8"); + let root = container.join("workspace"); + let dependency = container.join("dependency"); + + fs::create_dir_all(root.join("src")).expect("workspace source directory"); + fs::create_dir_all(dependency.join("src")).expect("dependency source directory"); + fs::write(root.join("src/lib.rs"), "fn add() {}\n").expect("workspace source"); + fs::write(root.join("src/tests.rs"), "#[test]\nfn caught() {}\n").expect("workspace test"); + fs::write( + dependency.join("Cargo.toml"), + "[package]\nname = \"dependency\"\nversion = \"0.1.0\"\n", + ) + .expect("dependency manifest"); + fs::write(dependency.join("src/lib.rs"), "pub fn answer() -> u8 { 1 }\n").expect("dependency source"); + + let inputs = RunRecord::snapshot_with_external(&root, &root.join("target/gamma"), core::slice::from_ref(&dependency), false); + let index = Killers::scan(&[root.join("src/tests.rs")]); + let record = RunRecord::from_snapshot( + &root, + &[ + mutant("unviable", "src/lib.rs", Outcome::CompileError), + killed("killed", Some("caught")), + ], + &envelope(), + inputs, + &index, + ) + .expect("the external dependency belongs to the pre-execution snapshot"); + + fs::write(dependency.join("src/lib.rs"), "pub fn answer() -> u8 { 2 }\n").expect("changed dependency source"); + + assert!( + record.settled(&root, Trust::Settled, &index, &envelope()).0.is_empty(), + "an external dependency change must recompile and retest every recorded outcome" + ); + } + + #[cfg(unix)] + #[test] + fn an_external_symlink_referent_cannot_create_a_reusable_record() { + let directory = workdir("record-external-symlink-"); + let container = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the work directory should be UTF-8"); + let root = container.join("workspace"); + let external = container.join("external.rs"); + + fs::create_dir_all(root.join("src")).expect("workspace source directory"); + fs::write(root.join("src/lib.rs"), "fn add() {}\n").expect("workspace source"); + fs::write(root.join("src/tests.rs"), "#[test]\nfn caught() {}\n").expect("workspace test"); + fs::write(&external, "pub fn external() {}\n").expect("external source"); + std::os::unix::fs::symlink(&external, root.join("src/linked.rs")).expect("external source link"); + + let index = Killers::scan(&[root.join("src/tests.rs")]); + let inputs = RunRecord::snapshot(&root, &root.join("target/gamma")); + + assert!( + RunRecord::from_snapshot( + &root, + &[ + mutant("unviable", "src/lib.rs", Outcome::CompileError), + killed("killed", Some("caught")), + ], + &envelope(), + inputs, + &index, + ) + .is_none(), + "a symlink referent outside the workspace must carry neither a verdict nor an unviability" + ); + } + + #[test] + fn an_edit_after_the_pre_execution_snapshot_records_no_outcome() { + let (_dir, root) = workspace("record-mid-run-edit-", "fn add() {}"); + let snapshot = RunRecord::snapshot(&root, &root.join("target/gamma")); + let barrier = Arc::new(Barrier::new(2)); + let editor_barrier = Arc::clone(&barrier); + let edited = root.join("src/lib.rs"); + + let editor = thread::spawn(move || { + let _waiting = editor_barrier.wait(); + fs::write(edited, "fn add() { panic!() }").expect("changed source"); + let _written = editor_barrier.wait(); + }); + + let _snapshot_taken = barrier.wait(); + let _edit_finished = barrier.wait(); + editor.join().expect("editor thread"); + + assert!( + RunRecord::from_snapshot( + &root, + &[mutant("abc", "src/lib.rs", Outcome::CompileError)], + &envelope(), + snapshot, + &Killers::scan(&[]) + ) + .is_none(), + "post-run bytes must never be stamped onto a pre-edit outcome" + ); + } + + /// The edit-and-revert case, which content alone cannot see. + /// + /// A file changed during the run and put back before it ends is byte-identical to what was + /// captured, so every digest still agrees — yet the outcomes were judged against the + /// intermediate bytes and are worthless. The modification time is what distinguishes the two, + /// because putting the content back is itself a write. + /// + /// The reverted file's time is stamped explicitly rather than left to the write, so that the + /// test measures the snapshot's comparison instead of the host filesystem's timestamp + /// granularity — a real revert advances the time exactly this way, only by an amount the + /// filesystem chooses. + #[test] + fn a_revert_to_the_original_bytes_does_not_restore_the_snapshot() { + let (_dir, root) = workspace("record-aba-edit-", "fn add() {}"); + let source = root.join("src/lib.rs"); + let original = fs::read(source.as_std_path()).expect("fixture source"); + let snapshot = RunRecord::snapshot(&root, &root.join("target/gamma")); + + fs::write(source.as_std_path(), "fn add() { panic!() }").expect("mid-run edit"); + fs::write(source.as_std_path(), &original).expect("revert"); + stamp_later(&source); + + assert_eq!( + fs::read(source.as_std_path()).expect("reverted source"), + original, + "the fixture must end byte-identical, or the test proves nothing new" + ); + assert!( + RunRecord::from_snapshot( + &root, + &[mutant("abc", "src/lib.rs", Outcome::CompileError)], + &envelope(), + snapshot, + &Killers::scan(&[]) + ) + .is_none(), + "a workspace edited and put back was still edited while the outcomes were produced" + ); + } + + /// Moves a file's modification time safely past every time the run could have recorded. + fn stamp_later(path: &Utf8Path) { + let file = fs::File::options() + .write(true) + .open(path.as_std_path()) + .expect("fixture file opens for writing"); + + file.set_modified(std::time::SystemTime::now() + core::time::Duration::from_secs(2)) + .expect("fixture filesystem records modification times"); + } + + #[test] + fn target_and_scratch_artifacts_do_not_invalidate_the_workspace_snapshot() { + let (_dir, root) = workspace("record-artifacts-", "fn add() {}"); + let scratch = root.join(".gamma-work/gamma"); + let snapshot = RunRecord::snapshot(&root, &scratch); + fs::create_dir_all(root.join("target/debug").as_std_path()).expect("target directory"); + fs::create_dir_all(scratch.as_std_path()).expect("scratch directory"); + fs::write(root.join("target/debug/artifact"), "generated").expect("target artifact"); + fs::write(scratch.join("tree"), "generated").expect("scratch artifact"); + + let record = RunRecord::from_snapshot( + &root, + &[mutant("abc", "src/lib.rs", Outcome::CompileError)], + &envelope(), + snapshot, + &Killers::scan(&[]), + ) + .expect("generated artifacts are not workspace inputs"); + + assert_eq!( + record.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.get("abc"), + Some(&Outcome::CompileError) + ); + } + + /// Only unviability is free. A kill is a claim about the test suite and carrying one unasked + /// would inflate the score; a survivor is the finding the whole exercise exists for. + #[test] + fn no_other_verdict_is_ever_carried_for_free() { + let (_dir, root) = workspace("record-only-", "fn add() {}"); + let population = [ + mutant("killed", "src/lib.rs", Outcome::Killed), + mutant("survived", "src/lib.rs", Outcome::Survived), + mutant("timeout", "src/lib.rs", Outcome::Timeout), + mutant("unviable", "src/lib.rs", Outcome::CompileError), + ]; + + let settled = from_run(&root, &population, &envelope()) + .settled(&root, Trust::Free, &Killers::default(), &envelope()) + .0; + + assert_eq!(settled.len(), 1); + assert_eq!(settled.get("unviable"), Some(&Outcome::CompileError)); + } + + #[test] + fn a_timeout_is_not_stored_for_a_later_run() { + let (_dir, root) = workspace("record-timeout-", "fn add() {}"); + let record = from_run(&root, &[mutant("timeout", "src/lib.rs", Outcome::Timeout)], &envelope()); + + assert_eq!(record.len(), 0); + assert!(record.settled(&root, Trust::Settled, &Killers::scan(&[]), &envelope()).0.is_empty()); + } + + #[test] + fn a_cache_survives_a_round_trip_through_the_scratch_directory() { + let (_dir, root) = workspace("record-round-", "fn add() {}"); + + from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + assert_eq!(RunRecord::load(&root).len(), 1); + } + + #[test] + fn every_completed_record_publication_is_parseable() { + let (_dir, root) = workspace("record-atomic-round-", "fn add() {}"); + + for generation in 0..8 { + let id = format!("mutant-{generation}"); + from_run(&root, &[mutant(&id, "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + let stored = fs::read_to_string(root.join(FILE)).expect("completed record"); + let _record: RunRecord = serde_json::from_str(&stored).expect("completed record parses"); + + RunRecord::store_probes( + &root, + &core::iter::once(( + id.into(), + Killer { + package: "subject".to_owned(), + target: "lib".to_owned(), + test: format!("caught_{generation}"), + }, + )) + .collect(), + ); + let stored = fs::read_to_string(root.join(FILE)).expect("completed probe record"); + let _record: RunRecord = serde_json::from_str(&stored).expect("completed probe record parses"); + } + } + + #[test] + fn failed_record_publications_preserve_the_prior_generation() { + let (_dir, root) = workspace("record-atomic-fail-", "fn add() {}"); + + from_run(&root, &[mutant("original", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + let before = fs::read_to_string(root.join(FILE)).expect("original record"); + + crate::elements::before_next_publication(|scratch| { + fs::remove_file(scratch).expect("remove staged record"); + }); + from_run(&root, &[mutant("replacement", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + assert_eq!(fs::read_to_string(root.join(FILE)).expect("prior record"), before); + + crate::elements::before_next_publication(|scratch| { + fs::remove_file(scratch).expect("remove staged probes"); + }); + RunRecord::store_probes( + &root, + &core::iter::once(( + "replacement".into(), + Killer { + package: "subject".to_owned(), + target: "lib".to_owned(), + test: "caught".to_owned(), + }, + )) + .collect(), + ); + assert_eq!(fs::read_to_string(root.join(FILE)).expect("prior record"), before); + } + + /// Unviability written under one feature set says nothing about a run under another: it is a + /// claim about what compiles, and a partial answer there is indistinguishable from a wrong one. + #[test] + fn unviability_from_a_different_context_is_not_settled() { + let (_dir, root) = workspace("record-context-", "fn add() {}"); + + from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + let settled = RunRecord::load(&root) + .settled(&root, Trust::Free, &Killers::default(), &under_another_feature_set()) + .0; + + assert!(settled.is_empty(), "unviability crossed a feature change"); + } + + /// Deleting the cache may only ever cost time. If a missing one could change a verdict it has + /// become state the score depends on. + #[test] + fn a_missing_cache_is_an_empty_one() { + let (_dir, root) = workspace("record-absent-", "fn add() {}"); + + assert_eq!(RunRecord::load(&root).len(), 0); + } + + /// So is a corrupt one, for the same reason. + #[test] + fn a_corrupt_cache_is_an_empty_one() { + let (_dir, root) = workspace("record-corrupt-", "fn add() {}"); + + fs::write(root.join(FILE), "{ this is not json").expect("the cache should be writable"); + + assert_eq!(RunRecord::load(&root).len(), 0); + } + + /// A run that adopted the cache has to write it back, or the cache is warm only every other + /// run: the adopted mutants are never rebuilt, so they are never rediscovered either. + #[test] + fn adopted_mutants_are_written_back() { + let (_dir, root) = workspace("record-writeback-", "fn add() {}"); + let population = [ + mutant("adopted", "src/lib.rs", Outcome::CompileError), + mutant("fresh", "src/lib.rs", Outcome::CompileError), + ]; + + from_run(&root, &population, &envelope()).store(&root, &root); + + assert_eq!(RunRecord::load(&root).len(), 2); + } + + /// A narrowed run keeps what it did not look at, which is the whole point of narrowing. + /// + /// `--in-diff`, a single package and a shard each know about a handful of files. Overwriting the + /// file with only those would leave every later narrow run starting cold, so a sequence of them + /// — the workflow narrowing exists to serve — would never accumulate a cache at all. + #[test] + fn a_narrowed_run_keeps_the_entries_it_never_looked_at() { + let (_dir, root) = workspace("record-narrow-", "fn add() {}"); + + fs::write(root.join("src/other.rs"), "fn other() {}").expect("the source should be writable"); + + from_run(&root, &[mutant("first", "src/other.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + from_run(&root, &[mutant("second", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + let settled = RunRecord::load(&root) + .settled(&root, Trust::Free, &Killers::default(), &envelope()) + .0; + + assert_eq!( + settled.get("first"), + Some(&Outcome::CompileError), + "the earlier run's file was dropped" + ); + assert_eq!(settled.get("second"), Some(&Outcome::CompileError)); + } + + /// A narrowed run cannot bless an older outcome merely by writing a fresh snapshot around it. + /// + /// The helper contains no mutant of either run, which is why checking only each entry's source + /// file would retain `first`; Cargo can still compile it into the test binary and change the + /// outcome. The old pre-execution snapshot must match before any old entry joins the new one. + #[test] + fn a_narrowed_run_drops_prior_entries_after_any_workspace_input_changes() { + let (_dir, root) = workspace("record-narrow-snapshot-", "fn add() {}"); + + fs::write(root.join("src/helper.rs"), "pub fn helper() -> u8 { 1 }").expect("helper source is writable"); + fs::write(root.join("src/other.rs"), "fn other() {}").expect("other source is writable"); + + from_run(&root, &[mutant("first", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + fs::write(root.join("src/helper.rs"), "pub fn helper() -> u8 { 2 }").expect("helper source is writable"); + + from_run(&root, &[mutant("second", "src/other.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + let settled = RunRecord::load(&root) + .settled(&root, Trust::Free, &Killers::default(), &envelope()) + .0; + + assert_eq!(settled.get("first"), None, "the changed helper recertified an old outcome"); + assert_eq!(settled.get("second"), Some(&Outcome::CompileError)); + } + + /// Retention stops exactly where belief does: an entry for a file that has changed is dropped. + /// + /// Carrying it would be carrying a stale compile-error verdict, which takes a mutant that might + /// now survive out of the denominator and turns a real gap in the suite into a better score. + #[test] + fn a_carried_entry_is_dropped_once_its_file_changes() { + let (_dir, root) = workspace("record-stale-", "fn add() {}"); + + fs::write(root.join("src/other.rs"), "fn other() {}").expect("the source should be writable"); + + from_run(&root, &[mutant("first", "src/other.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + fs::write(root.join("src/other.rs"), "fn other() -> usize { 0 }").expect("the source should be writable"); + + from_run(&root, &[mutant("second", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + let reloaded = RunRecord::load(&root); + + assert_eq!(reloaded.len(), 1, "a stale entry was carried forward"); + assert_eq!( + reloaded + .settled(&root, Trust::Free, &Killers::default(), &envelope()) + .0 + .get("second"), + Some(&Outcome::CompileError) + ); + } + + /// A file this run looked at is this run's to describe, however much the earlier one held. + /// + /// The older answer is not evidence against a build that just happened, so it is replaced rather + /// than merged — otherwise a mutant that has since become viable would live in the cache forever + /// under a file whose digest never changed. + #[test] + fn this_runs_answer_replaces_the_earlier_one_for_a_file_it_visited() { + let (_dir, root) = workspace("record-replace-", "fn add() {}"); + let earlier = [ + mutant("gone", "src/lib.rs", Outcome::CompileError), + mutant("kept", "src/lib.rs", Outcome::CompileError), + ]; + + from_run(&root, &earlier, &envelope()).store(&root, &root); + from_run(&root, &[mutant("kept", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + + let settled = RunRecord::load(&root) + .settled(&root, Trust::Free, &Killers::default(), &envelope()) + .0; + + assert_eq!(settled.get("gone"), None, "a mutant this run found viable was carried anyway"); + assert_eq!(settled.get("kept"), Some(&Outcome::CompileError)); + } + + /// A file whose length has changed is rejected without being read. + /// + /// The length is a `stat` where the digest is a full read, so it is checked first — and it may + /// only ever reject, because two different files of the same length are ordinary rather than + /// exotic and accepting one would drop a mutant that might have survived. + #[test] + fn a_file_that_has_grown_is_rejected() { + let (_dir, root) = workspace("record-grown-", "fn add() {}"); + let cache = from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()); + + fs::write(root.join("src/lib.rs"), "fn add() {} // and more").expect("the source should be writable"); + + assert!(cache.settled(&root, Trust::Free, &Killers::default(), &envelope()).0.is_empty()); + } + + /// A context over nothing but a toolchain, for varying one axis at a time. + fn plain() -> Context<'static> { + Context { + toolchain: Some("1.90.0"), + baseline: true, + confirm: true, + stall: true, + ..Context::default() + } + } + + #[test] + fn a_changed_inherited_environment_refuses_carried_unviability() { + let (_dir, root) = workspace("record-unviability-environment-", "fn add() {}"); + let before = + context_in(&plain(), None, &[(b"SUBJECT_BUILD_MODE".to_vec(), b"before".to_vec())]).expect("a named toolchain gives a context"); + let after = + context_in(&plain(), None, &[(b"SUBJECT_BUILD_MODE".to_vec(), b"after".to_vec())]).expect("a named toolchain gives a context"); + let record = from_run(&root, &[unviable("abc")], &before); + + assert_eq!( + record.settled(&root, Trust::Free, &Killers::default(), &before).0.get("abc"), + Some(&Outcome::CompileError), + "the unchanged inherited environment may reuse unviability" + ); + assert!( + record.settled(&root, Trust::Free, &Killers::default(), &after).0.is_empty(), + "a changed inherited environment must recompile an unviable mutant" + ); + } + + /// Every axis that decides whether a mutant compiles has to reach the digest. + /// + /// The cache is believed rather than re-checked, so an axis missing from the key is not a slower + /// run — it is a mutant withheld from the denominator on the strength of a build that never + /// happened, which turns a real gap in the suite into a better score. + /// + /// The `let Context { .. }` below is the point of this test and not decoration. An enumeration + /// of the fields that exist can only ever check what somebody already implemented; a `..` in + /// that pattern, or a table written against the current signature, would let a field be added + /// to `Context` and left out of `context` with every test still green. Destructuring without + /// `..` makes adding an axis a compile error here, so the axis cannot be forgotten — it can + /// only be deliberately dismissed by whoever adds the line that dismisses it. + #[test] + #[expect( + clippy::too_many_lines, + reason = "exhaustive test verifying each context axis distinguishes the digest" + )] + fn the_context_distinguishes_the_things_that_decide_what_compiles() { + let features = ["extra".to_owned()]; + let extra = ["--cfg=loom".to_owned()]; + let test_packages = ["pkg_a".to_owned()]; + let include_tests = ["test_a".to_owned()]; + let exclude_tests = ["test_b".to_owned()]; + let cargo_test_args = ["--nocapture".to_owned()]; + let test_args = ["--skip".to_owned(), "slow".to_owned()]; + + // Adding a field to `Context` fails to compile here until it is given a varied value below. + // The bindings are named rather than wildcards only because a pattern of all wildcards is + // one clippy offers to replace with `..`, which would silently disarm the whole guard. + let Context { + features: _f, + all_features: _a, + no_default_features: _n, + profile: _p, + extra: _e, + rustflags: _r, + toolchain: _t, + test_packages: _tp, + include_tests: _it, + exclude_tests: _et, + test_workspace: _tw, + whole_test_binaries: _wtb, + nextest: _nx, + cargo_test_args: _cta, + test_args: _ta, + baseline: _b, + confirm: _c, + stall: _s, + test_timeout_multiplier: _ttm, + minimum_test_timeout: _mtt, + memory: _m, + memory_multiplier: _mm, + memory_headroom: _mh, + memory_limit: _ml, + baseline_memory_limit: _bml, + no_relaunch: _nr, + copy_ignored: _ci, + jobs: _j, + build_timeout: _bt, + build_timeout_multiplier: _btm, + rollback_rounds: _rr, + } = plain(); + + let varied: &[(&str, Context<'_>)] = &[ + ( + "features", + Context { + features: &features, + ..plain() + }, + ), + ( + "--all-features", + Context { + all_features: true, + ..plain() + }, + ), + ( + "--no-default-features", + Context { + no_default_features: true, + ..plain() + }, + ), + ( + "the profile, which turns debug_assertions off", + Context { + profile: Some("release"), + ..plain() + }, + ), + ( + "passthrough build arguments, which can carry --cfg, -C or --target", + Context { extra: &extra, ..plain() }, + ), + ( + "ambient rustflags, which can select different code entirely", + Context { + rustflags: Some("--cfg loom"), + ..plain() + }, + ), + ( + "the compiler", + Context { + toolchain: Some("1.91.0"), + ..plain() + }, + ), + ( + "test packages", + Context { + test_packages: &test_packages, + ..plain() + }, + ), + ( + "include tests", + Context { + include_tests: &include_tests, + ..plain() + }, + ), + ( + "exclude tests", + Context { + exclude_tests: &exclude_tests, + ..plain() + }, + ), + ( + "test workspace", + Context { + test_workspace: true, + ..plain() + }, + ), + ( + "whole test binaries", + Context { + whole_test_binaries: true, + ..plain() + }, + ), + ("nextest", Context { nextest: true, ..plain() }), + ( + "cargo test arguments", + Context { + cargo_test_args: &cargo_test_args, + ..plain() + }, + ), + ( + "post-separator test arguments", + Context { + test_args: &test_args, + ..plain() + }, + ), + ( + "baseline mode", + Context { + baseline: false, + ..plain() + }, + ), + ("confirmation mode", Context { confirm: false, ..plain() }), + ("stall detection", Context { stall: false, ..plain() }), + ( + "test timeout multiplier", + Context { + test_timeout_multiplier: Some(1.5), + ..plain() + }, + ), + ( + "minimum test timeout", + Context { + minimum_test_timeout: Some(30.0), + ..plain() + }, + ), + ( + "memory mode", + Context { + memory: Some(crate::exec::MemoryControl::Off), + ..plain() + }, + ), + ( + "memory multiplier", + Context { + memory_multiplier: Some(3.0), + ..plain() + }, + ), + ( + "memory headroom", + Context { + memory_headroom: Some(1024), + ..plain() + }, + ), + ( + "memory limit", + Context { + memory_limit: Some(2048), + ..plain() + }, + ), + ( + "baseline memory limit", + Context { + baseline_memory_limit: Some(4096), + ..plain() + }, + ), + ( + "memory relaunch", + Context { + no_relaunch: true, + ..plain() + }, + ), + ( + "ignored-file copying", + Context { + copy_ignored: true, + ..plain() + }, + ), + ("parallel scheduling", Context { jobs: Some(2), ..plain() }), + ( + "build timeout", + Context { + build_timeout: Some(60.0), + ..plain() + }, + ), + ( + "build timeout multiplier", + Context { + build_timeout_multiplier: Some(2.0), + ..plain() + }, + ), + ( + "rollback rounds", + Context { + rollback_rounds: 1, + ..plain() + }, + ), + ]; + + let base = context(&plain()).expect("a named toolchain gives a context"); + let mut digests = HashSet::default(); + + let _ = digests.insert(base.clone()); + + for (axis, varied) in varied { + let digest = context(varied).expect("a named toolchain gives a context"); + + assert_ne!(base, digest, "{axis}"); + + // Pairwise as well as against the base: two axes that hashed the same as each other + // would each pass the check above and still leave one of them unable to invalidate the + // other's cache. + assert!(digests.insert(digest), "{axis} digests the same as another axis"); + } + + assert_eq!(base, context(&plain()).unwrap(), "the same context digests the same"); + } + + /// The tool's own version is an axis too, and it is the one no test can vary. + /// + /// A mutant id hashes the item path, the mutator, the normalized site text, the occurrence and + /// the replacement index — every one of them a property of *this tool* — so an upgrade can + /// change what an id denotes while the sources, the features and the toolchain are all + /// unchanged. It cannot be varied from a test, because it is an `env!` read at compile time, so + /// what is asserted instead is that it is a term of its own: a context that spells the version + /// out among its passthrough arguments must not digest the same as one that does not. + #[test] + fn the_tool_version_is_a_term_of_the_digest_rather_than_loose_bytes() { + let spelled = [env!("CARGO_PKG_VERSION").to_owned()]; + + assert_ne!( + context(&plain()).unwrap(), + context(&Context { + extra: &spelled, + ..plain() + }) + .unwrap() + ); + } + + /// A compiler that cannot be asked its version suppresses the cache rather than being hashed as + /// an empty string. + /// + /// Hashing the absence would not make the cache less useful; it would remove the compiler from + /// the key, so two runs under different toolchains would match and the second would believe the + /// first. `RUSTC` pointing at a wrapper that does not answer `--version` would make that + /// permanent and silent. + #[test] + fn a_toolchain_that_cannot_be_named_yields_no_context_at_all() { + assert_eq!( + context(&Context { + toolchain: None, + ..plain() + }), + None + ); + } + + /// Rearranging the parts must not collide, which is what the length prefixes buy. + #[test] + fn two_different_contexts_cannot_digest_the_same_by_running_their_parts_together() { + let split = ["ab".to_owned(), "c".to_owned()]; + let joined = ["a".to_owned(), "bc".to_owned()]; + + assert_ne!( + context(&Context { + features: &split, + ..plain() + }), + context(&Context { + features: &joined, + ..plain() + }) + ); + } + + /// The whole of F2 in one assertion: a compiler bump costs unviability and nothing else. + /// + /// The old envelope discarded the file wholesale, which made a record shared between a + /// developer's machine and CI deliver nothing while appearing to work — silently, which is + /// worse than an absent file. + #[test] + fn a_toolchain_change_discards_unviability_and_keeps_the_probes() { + let (_dir, root) = workspace("record-tiers-", "fn add() {}"); + let index = killers(&root, "#[test]\nfn caught() {}\n"); + + from_run(&root, &[mutant("abc", "src/lib.rs", Outcome::CompileError)], &envelope()).store(&root, &root); + RunRecord::store_probes( + &root, + &core::iter::once(( + "abc".into(), + Killer { + package: "subject".to_owned(), + target: "lib".to_owned(), + test: "caught".to_owned(), + }, + )) + .collect(), + ); + + let record = RunRecord::load(&root); + let settled = record.settled(&root, Trust::Free, &index, &under_another_toolchain()).0; + + assert!(settled.is_empty(), "unviability survived a compiler it was never checked against"); + assert_eq!( + RunRecord::load(&root).probes().get("abc").map(|killer| killer.test.as_str()), + Some("caught"), + "the probe was thrown away with the unviability" + ); + assert_eq!(record.ordering(), vec!["abc"], "the demoted unviability was thrown away too"); + } + + /// A compiler change can alter test behaviour, so it invalidates verdicts as well as builds. + #[test] + fn a_toolchain_change_rechecks_the_verdicts() { + let (_dir, root) = workspace("record-verdict-toolchain-", "fn add() {}"); + let index = killers(&root, "#[test]\nfn caught() {}\n"); + + let record = from_run(&root, &[killed("abc", Some("caught"))], &envelope()); + let settled = record.settled(&root, Trust::Settled, &index, &under_another_toolchain()).0; + + assert!(settled.is_empty()); + } + + /// Every execution axis invalidates a verdict, `--profile` most obviously: a `debug_assert!` + /// that caught a mutant in one profile does not exist in the other. + #[test] + fn every_required_term_invalidates_a_verdict() { + let (_dir, root) = workspace("record-verdict-terms-", "fn add() {}"); + let index = killers(&root, "#[test]\nfn caught() {}\n"); + let record = from_run(&root, &[killed("abc", Some("caught"))], &envelope()); + + let extra = ["--cfg=loom".to_owned()]; + let test_packages = ["pkg_b".to_owned()]; + let varied: &[(&str, Context<'_>)] = &[ + ( + "the profile", + Context { + profile: Some("release"), + ..plain() + }, + ), + ( + "rustflags", + Context { + rustflags: Some("--cfg loom"), + ..plain() + }, + ), + ("passthrough arguments", Context { extra: &extra, ..plain() }), + ( + "test filtering", + Context { + test_packages: &test_packages, + ..plain() + }, + ), + ]; + + for (axis, context_of) in varied { + let digest = context(context_of).expect("a named toolchain gives a context"); + + assert!( + record.settled(&root, Trust::Settled, &index, &digest).0.is_empty(), + "a verdict crossed a change of {axis}" + ); + } + + assert!( + record + .settled(&root, Trust::Settled, &index, &under_another_feature_set()) + .0 + .is_empty(), + "a verdict crossed a feature change" + ); + } + + /// Every term a run digests has to be a term a tier can require, or a tier requiring "all of + /// them" would silently stop covering the one that was added. + #[test] + fn every_digested_term_is_one_a_tier_can_require() { + // Resolved against a workspace, so that every term holds a digest: an unstated one reads as + // the empty string, which would let two of them collide with each other unnoticed. + let (_dir, root) = workspace("record-terms-", "fn add() {}"); + let digest = envelope().resolved_at(&root); + let named: HashSet<&str> = Term::ALL.iter().map(|term| digest.term(*term)).collect(); + + // Each term is digested from different bytes, so a term missing from `Term::ALL` shows up + // as a value nobody can name. The digest is a struct, so the compiler enforces the other + // direction — `term` cannot answer for a field that does not exist. + assert_eq!(named.len(), Term::ALL.len(), "two terms digest the same, so one cannot invalidate"); + + assert_eq!( + Tier::Unviability.requires(), + &[ + Term::Features, + Term::Profile, + Term::Rustflags, + Term::Target, + Term::Config, + Term::Extra, + Term::Toolchain, + Term::Tool, + Term::Policy, + Term::Environment, + ] + ); + assert!(Tier::Ordering.requires().is_empty()); + assert!( + Tier::Verdict.requires().contains(&Term::Toolchain), + "verdicts are rechecked after a compiler change" + ); + assert!( + Tier::Verdict.requires().contains(&Term::Tests), + "the verdict tier depends on the tests" + ); + assert!( + Tier::Verdict.requires().contains(&Term::Policy), + "verdicts depend on their execution policy" + ); + assert!( + Tier::Verdict.requires().contains(&Term::Environment), + "verdicts depend on what test processes inherit" + ); + } + + /// An unviable mutant, which is what [`Tier::Unviability`] governs. + fn unviable(id: &str) -> Mutant { + mutant(id, "src/lib.rs", Outcome::CompileError) + } + + /// What a record settles for a run that trusts it as far as it is free to. + fn free(record: &RunRecord, root: &Utf8Path, context: &ContextDigest) -> HashMap { + record.settled(root, Trust::Free, &Killers::default(), context).0 + } + + /// `CARGO_BUILD_TARGET` is the spelling of the target that no passthrough argument carries, and + /// the one a record could cross without noticing: every source digest matches, no rustflags are + /// set on either side, and the mutants would come back for an architecture they were never + /// compiled against. + #[test] + fn a_record_written_for_one_target_is_not_read_for_another() { + let (_dir, root) = workspace("record-target-", "fn add() {}"); + let here = context_in(&plain(), None, &[]).expect("a named toolchain gives a context"); + let elsewhere = context_in(&plain(), Some("x86_64-unknown-linux-musl"), &[]).expect("a named toolchain gives a context"); + + let record = from_run(&root, &[unviable("abc")], &here); + + assert_eq!( + free(&record, &root, &here).get("abc"), + Some(&Outcome::CompileError), + "the record has to apply to the run that wrote it" + ); + assert!( + free(&record, &root, &elsewhere).is_empty(), + "unviability crossed a change of target, so a mutant is out of the denominator for a build it never saw" + ); + } + + /// The same for the target's other spelling, which reaches the digest through the arguments the + /// run passes cargo rather than through the environment. + #[test] + fn a_record_written_for_one_passthrough_target_is_not_read_for_another() { + let (_dir, root) = workspace("record-target-argument-", "fn add() {}"); + let extra = ["--target".to_owned(), "wasm32-unknown-unknown".to_owned()]; + + let here = context_in(&plain(), None, &[]).expect("a named toolchain gives a context"); + let elsewhere = context_in(&Context { extra: &extra, ..plain() }, None, &[]).expect("a named toolchain gives a context"); + + let record = from_run(&root, &[unviable("abc")], &here); + + assert!( + free(&record, &root, &elsewhere).is_empty(), + "unviability crossed a change of target" + ); + assert_ne!( + here.term(Term::Target), + elsewhere.term(Term::Target), + "the target term is what has to notice, so that the diagnostic can name it" + ); + } + + /// The scenario `rustflags` names as expected — a `--cfg` added to the build — written in the + /// file rather than in the variable. The record is loaded from disk in between, because that is + /// the shape of the run this defends: the flag is edited between two invocations. + #[test] + fn a_record_written_before_a_configured_rustflag_is_not_read_after_it() { + let (_dir, root) = workspace("record-configured-rustflags-", "fn add() {}"); + let envelope = envelope(); + + from_run(&root, &[unviable("abc")], &envelope).store(&root, &root); + + assert_eq!( + free(&RunRecord::load(&root), &root, &envelope).get("abc"), + Some(&Outcome::CompileError), + "the record has to apply while nothing has changed" + ); + + fs::create_dir_all(root.join(".cargo").as_std_path()).expect("the configuration directory should be creatable"); + fs::write( + root.join(".cargo/config.toml").as_std_path(), + "[build]\nrustflags = [\"--cfg\", \"loom\"]\n", + ) + .expect("the configuration should be writable"); + + assert!( + free(&RunRecord::load(&root), &root, &envelope).is_empty(), + "unviability crossed a configured rustflag, which selects different code entirely" + ); + } + + /// Every setting the workspace holds has to move the key, or the list of them below is a claim + /// nobody checks. + /// + /// One case per file-borne input in [`Build::INPUTS`], because the digest is taken over a + /// rendering of those settings and a rendering that dropped one of them would still look like a + /// digest. + #[test] + fn a_change_to_any_setting_the_workspace_holds_moves_the_config_term() { + let edits: &[(&str, &str, &str)] = &[ + ( + "build.target", + ".cargo/config.toml", + "[build]\ntarget = \"wasm32-unknown-unknown\"\n", + ), + ( + "build.rustflags", + ".cargo/config.toml", + "[build]\nrustflags = [\"--cfg\", \"loom\"]\n", + ), + ( + "target.*.rustflags", + ".cargo/config.toml", + "[target.'cfg(unix)']\nrustflags = [\"--cfg\", \"loom\"]\n", + ), + ("profile.*", "Cargo.toml", "[profile.dev]\ndebug-assertions = false\n"), + ]; + + for (input, file, body) in edits { + let (_dir, root) = workspace("record-settings-", "fn add() {}"); + let path = root.join(file); + let before = envelope().resolved_at(&root); + + fs::create_dir_all(path.parent().expect("every fixture path has a parent").as_std_path()) + .expect("the directory should be creatable"); + fs::write(path.as_std_path(), *body).expect("the setting should be writable"); + + assert_ne!( + before.term(Term::Config), + envelope().resolved_at(&root).term(Term::Config), + "a change to {input} left the key where it was" + ); + } + } + + /// The two modules that decide what compiles must not drift apart. + /// + /// `cfg::build` is where a build input is read and [`Build::INPUTS`] is where it is named; + /// [`COVERAGE`] is what this module does about each one. An input the resolution reads and no + /// term covers is unviability carried across a change of build, which is a mutant dropped from + /// the denominator — so the two lists are checked against each other rather than trusted to + /// stay in step. + #[test] + fn every_input_the_build_resolution_reads_is_covered_by_a_term() { + let uncovered: Vec<&str> = Build::INPUTS + .iter() + .copied() + .filter(|input| !COVERAGE.iter().any(|(name, _term)| name == input)) + .collect(); + + assert!( + uncovered.is_empty(), + "these decide what compiles and no term covers them: {uncovered:?}" + ); + + let unrequired: Vec<&str> = COVERAGE + .iter() + .filter(|(_name, term)| !Term::ALL.contains(term)) + .map(|(name, _term)| *name) + .collect(); + + assert!( + unrequired.is_empty(), + "these are covered by a term no tier can require, so the cover is not one: {unrequired:?}" + ); + + // The other direction, which catches the cover that outlived the input: a name here that + // `cfg::build` no longer reads is either a stale entry or, worse, a rename that left the + // real input uncovered while this list still looked complete. + let stale: Vec<&str> = COVERAGE + .iter() + .filter(|(name, _term)| !Build::INPUTS.contains(name)) + .map(|(name, _term)| *name) + .collect(); + + assert!(stale.is_empty(), "these are covered and nothing reads them: {stale:?}"); + } + + /// A record written before these terms existed states neither of them, which is the case the + /// format number does not move for: it loses the tiers whose guard it cannot answer and keeps + /// the two that ask nothing, where discarding it would have cost all three. + #[test] + fn a_record_that_states_none_of_the_newer_terms_keeps_only_the_tiers_that_ask_nothing() { + let (_dir, root) = workspace("record-older-terms-", "fn add() {}"); + + from_run(&root, &[unviable("abc")], &envelope()).store(&root, &root); + + let text = fs::read_to_string(root.join(FILE).as_std_path()).expect("the record should be readable"); + let mut written: serde_json::Value = serde_json::from_str(&text).expect("the record is JSON"); + let context = written + .get_mut("context") + .and_then(serde_json::Value::as_object_mut) + .expect("the record states its context"); + + let _target = context.remove("target"); + let _config = context.remove("config"); + + fs::write( + root.join(FILE).as_std_path(), + serde_json::to_string(&written).expect("the record serializes"), + ) + .expect("the record should be writable"); + + let older = RunRecord::load(&root); + + assert!( + free(&older, &root, &envelope()).is_empty(), + "unviability was adopted from a record that cannot say what build it was written under" + ); + assert_eq!(older.ordering(), ["abc"], "the tiers that require no term still apply"); + } + + /// A term neither side has an answer for is not agreement, and the tier that requires it has to + /// refuse. A record written before the term existed looks exactly like this. + #[test] + fn a_term_nobody_states_admits_nothing() { + let unresolved = envelope(); + + assert!(!unresolved.states(Term::Config)); + assert!( + !Tier::Unviability.admits(&unresolved, &unresolved), + "an unstated term must not match itself" + ); + assert!( + unresolved.differences(&unresolved).is_empty(), + "a term neither side states is not an axis that moved" + ); + assert!( + Tier::Ordering.admits(&unresolved, &unresolved), + "the tiers that require nothing are unaffected by what nobody states" + ); + } + + /// Cargo reads the three global rustflags variables in order and takes the first that says + /// anything, so the key holds whichever one is actually in force. `CARGO_BUILD_RUSTFLAGS` is the + /// variable spelling of `build.rustflags` and decides the build exactly as the other two do. + #[test] + fn the_flags_in_the_key_are_the_ones_cargo_will_read() { + let of = |value: &str| Some(value.to_owned()); + + assert_eq!(rustflags_in(None, None, of("--cfg loom"), &[]).as_deref(), Some("--cfg loom")); + assert_eq!(rustflags_in(None, of("plain"), of("configured"), &[]).as_deref(), Some("plain")); + assert_eq!( + rustflags_in(of("encoded"), of("plain"), of("configured"), &[]).as_deref(), + Some("encoded") + ); + assert_eq!(rustflags_in(None, None, None, &[]), None); + } + + /// A target-specific variable is its own precedence level, so it is added to the key rather + /// than competing with the global three for it. + /// + /// Cargo applies `CARGO_TARGET__RUSTFLAGS` where the global variables are unset, so a + /// key that only ever held the first global that spoke would be identical across a change to it + /// — and unviability would then be carried across a flag that decides what compiles. + #[test] + fn a_target_specific_rustflags_variable_moves_the_key() { + let of = |value: &str| Some(value.to_owned()); + let targeted = [( + "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS".to_owned(), + "--cfg live".to_owned(), + )]; + + let alone = rustflags_in(None, None, None, &targeted); + + assert_ne!(alone, None, "a target-specific flag on its own said nothing"); + assert_ne!( + alone, + rustflags_in(None, None, None, &[]), + "the key did not move when the only rustflag in force was a target-specific one" + ); + + assert_ne!( + rustflags_in(of("encoded"), None, None, &targeted), + rustflags_in(of("encoded"), None, None, &[]), + "a global variable hid a target-specific one that cargo would also apply" + ); + + assert_ne!( + rustflags_in(None, None, None, &targeted), + rustflags_in(None, None, None, &[(targeted[0].0.clone(), "--cfg other".to_owned())]), + "a change to the value of a target-specific variable left the key where it was" + ); + } + + /// A record from the previous format is ignored rather than half-believed. /// + /// Version 3 hashed every term into one digest, so nothing in it can answer the question a tier + /// asks — "did you agree with me about the profile" — and inventing an answer would be adopting + /// knowledge under a rule it was never written under. + #[test] + fn a_record_from_the_previous_format_is_ignored() { + let (_dir, root) = workspace("record-old-format-", "fn add() {}"); + + fs::write( + root.join(FILE), + r#"{"version":3,"context":"deadbeef","files":[{"path":"src/lib.rs","digest":"x","size":11,"mutants":[{"id":"abc","outcome":"unviable"}]}],"hints":{}}"#, + ) + .expect("the cache should be writable"); + + let record = RunRecord::load(&root); + + assert_eq!(record.len(), 0); + assert!(record.ordering().is_empty()); + assert!(RunRecord::load(&root).probes().is_empty()); + } + + /// The ordering tier answers whatever the context is, because being wrong about an order costs + /// the order and nothing else. + #[test] + fn the_ordering_tier_survives_every_term_and_is_stable() { + let (_dir, root) = workspace("record-ordering-", "fn add() {}"); + let population = [ + mutant("zeta", "src/lib.rs", Outcome::CompileError), + mutant("alpha", "src/lib.rs", Outcome::CompileError), + mutant("killed", "src/lib.rs", Outcome::Killed), + ]; + + let record = from_run(&root, &population, &envelope()); + + assert_eq!( + record.ordering(), + vec!["alpha", "zeta"], + "only unviability orders, and it is sorted" + ); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/shard.rs b/crates/cargo-gamma-lib/src/discover/shard.rs new file mode 100644 index 000000000..869761e31 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/shard.rs @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Assigning a mutant to one shard of a split run, stably across runs and shard counts. + +/// Assigns a mutant to a shard. +/// +/// This uses jump consistent hashing rather than `hash % count`, because the two behave very +/// differently when the shard count changes. With a modulus, bumping a nightly job from 8 shards +/// to 9 reshuffles roughly 8/9 of all mutants into different shards; with jump consistent hashing +/// only the fraction that must move does. Shard membership is therefore something a team can +/// reason about across a config change instead of a fresh random assignment each time. +#[must_use] +pub fn shard_of(id: &str, count: u32) -> u32 { + if count <= 1 { + return 0; + } + + let mut key = fnv1a(id.as_bytes()); + let mut candidate: i64 = -1; + let mut next: i64 = 0; + + while next < i64::from(count) { + candidate = next; + key = key.wrapping_mul(2_862_933_555_777_941_757).wrapping_add(1); + + #[expect(clippy::cast_precision_loss, reason = "only the leading bits steer the choice")] + let divisor = ((key >> 33).wrapping_add(1)) as f64; + + #[expect(clippy::cast_precision_loss, reason = "the operand is a small shard ordinal")] + let scaled = ((candidate + 1) as f64) * (f64::from(1_u32 << 31) / divisor); + + #[expect(clippy::cast_possible_truncation, reason = "the value is bounded by the shard count")] + { + next = scaled as i64; + } + } + + u32::try_from(candidate.max(0)).unwrap_or(0) +} + +/// FNV-1a, used only to turn an id into a shard key. +const fn fnv1a(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + let mut index = 0; + + while index < bytes.len() { + hash ^= bytes[index] as u64; + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + index += 1; + } + + hash +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn one_shard_holds_everything() { + for id in ["a", "b", "deadbeef1234"] { + assert_eq!(shard_of(id, 1), 0); + } + } + + #[test] + fn shards_are_always_in_range() { + for count in 1_u32..=16 { + for index in 0..500_u32 { + let id = format!("mutant{index:04}"); + let shard = shard_of(&id, count); + + assert!(shard < count, "{id} landed in shard {shard} of {count}"); + } + } + } + + #[test] + fn sharding_is_deterministic() { + assert_eq!(shard_of("abc123def456", 7), shard_of("abc123def456", 7)); + } + + #[test] + fn every_mutant_lands_in_exactly_one_shard() { + let ids: Vec = (0..300).map(|index| format!("mutant{index:04}")).collect(); + + for count in [2_u32, 5, 7, 16] { + let total: usize = (0..count) + .map(|shard| ids.iter().filter(|id| shard_of(id, count) == shard).count()) + .sum(); + + assert_eq!(total, ids.len(), "shard count {count} lost or duplicated mutants"); + } + } + + #[test] + fn shards_are_reasonably_balanced() { + let ids: Vec = (0..2000).map(|index| format!("mutant{index:05}")).collect(); + let count = 8_u32; + let expected = ids.len() / count as usize; + + for shard in 0..count { + let size = ids.iter().filter(|id| shard_of(id, count) == shard).count(); + + assert!( + size > expected / 2 && size < expected * 2, + "shard {shard} holds {size}, expected around {expected}" + ); + } + } + + #[test] + fn growing_the_shard_count_moves_few_mutants() { + // The whole reason for jump consistent hashing: a team that raises its nightly shard count + // should keep most of its coverage history, not reshuffle everything. + let ids: Vec = (0..2000).map(|index| format!("mutant{index:05}")).collect(); + let moved = ids.iter().filter(|id| shard_of(id, 8) != shard_of(id, 9)).count(); + let total = ids.len(); + + // A modulus would move about 8/9 of them. + assert!(moved < total / 4, "{moved} of {total} mutants moved when growing 8 -> 9"); + } + + #[test] + fn different_ids_can_land_in_different_shards() { + let ids: Vec = (0..100).map(|index| format!("mutant{index:04}")).collect(); + let distinct: crate::HashSet = ids.iter().map(|id| shard_of(id, 4)).collect(); + + assert!(distinct.len() > 1, "sharding put everything in one shard"); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/survey.rs b/crates/cargo-gamma-lib/src/discover/survey.rs new file mode 100644 index 000000000..dbff2f6ce --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/survey.rs @@ -0,0 +1,3877 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Walking the workspace to decide which files to parse and which mutants to plan. + +use core::iter::once; +use core::mem; +use core::num::NonZero; +use core::panic::AssertUnwindSafe; +use core::sync::atomic::{AtomicUsize, Ordering}; +use std::ffi::OsStr; +use std::panic::{catch_unwind, resume_unwind}; +use std::sync::{Barrier, Mutex, OnceLock, PoisonError}; +use std::thread; + +use camino::{Utf8Path, Utf8PathBuf}; +use cargo_metadata::{CargoOpt, Metadata, MetadataCommand, Target, TargetKind}; +use walkdir::WalkDir; + +use super::compile_fail::{CompileFailTarget, compile_fail_targets}; +use super::glob::{Glob, normalize_separators}; +use super::killers::Killers; +use super::shard::shard_of; +use super::{Diff, Plan, TargetFile, modules}; +use crate::cfg::{CfgSet, Cfgs, features}; +use crate::commands::{FeatureArgs, SelectArgs}; +use crate::error::{Error, error}; +use crate::exec::CargoOptions; +use crate::model::{Interner, Mutant, MutantId, Outcome}; +use crate::ops::collect; +use crate::ops::registry::Selection; +use crate::parse::SourceFile; +use crate::{HashMap, HashSet, Result, suppress}; + +/// Builds the plan for a run. +/// +/// `notify` is called with a short human-readable message, so the caller can drive a progress +/// display without this module knowing anything about terminals. It is called once with the size +/// of the job before the files are parsed, and once per package afterwards carrying what that +/// package actually yielded — the counts do not exist until the parse is done. +#[cfg(test)] +pub fn plan(args: &SelectArgs, selection: &Selection, shard: Option<(u32, u32)>, notify: &mut impl FnMut(&str)) -> Result { + let survey = Survey::new(args, shard)?; + + plan_survey(survey, selection, notify) +} + +/// Builds a plan for the Cargo options the caller has already resolved. +/// +/// Commands that have merged `gamma.toml` into their run arguments pass the same options +/// here for every discovery pass. Re-reading the file after an edit has begun could mix the first +/// pass's plan with a later configuration generation. +pub(crate) fn plan_for_build( + args: &SelectArgs, + selection: &Selection, + shard: Option<(u32, u32)>, + cargo: &CargoOptions, + notify: &mut impl FnMut(&str), +) -> Result { + let survey = Survey::for_build(args, shard, cargo)?; + + plan_survey(survey, selection, notify) +} + +/// Scans one resolved survey into a plan. +fn plan_survey(survey: Survey, selection: &Selection, notify: &mut impl FnMut(&str)) -> Result { + // Parsing is the expensive half of discovery and says nothing while it runs, so its size is + // announced before it starts rather than leaving the display silent. + notify(&format!("{} for mutants", crate::report::quantity(survey.files.len(), "file"))); + + let mut ordinals = 0; + let scanned = survey.scan(None, selection, &mut ordinals)?; + + report_by_package(&survey.files, &scanned.mutants, notify); + + Ok(survey.into_plan(scanned)) +} + +/// The workspace, its files and its shape, worked out without parsing a line of source. +/// +/// Discovery divides in two. Working out which files are worth mutating costs a cargo metadata +/// call and a directory walk; parsing them costs far more. Splitting the two lets a run copy the +/// workspace and then scan, instrument and build one package at a time, rather than parsing +/// everything before anything else can start. +#[derive(Debug)] +pub struct Survey { + /// Absolute path of the workspace root. + pub root: Utf8PathBuf, + + /// Every file worth mutating, sorted by path. + pub files: Vec, + + /// Every mutable source file walked for each selected package, including files excluded from + /// mutation by `--file` or `--in-diff`. + /// + /// Module declarations in an excluded parent can still prove that an included child exists + /// only under `#[cfg(test)]`, so the declaration graph must be wider than the population. + declaration_files: HashMap>, + + /// For each workspace package, the workspace packages its test binaries can reach. + pub reach: HashMap>, + + /// The packages cargo itself would act on here, sorted. + /// + /// This is the whole of `--package`, `--workspace` and the invocation directory resolved to a + /// package list, and it is deliberately wider than [`Self::packages`]: a `--file` or + /// `--in-diff` filter narrows what gets mutated without narrowing what cargo would build and + /// test. It is the ceiling on the oracle, so it has to be the set cargo would select rather + /// than the set that turned out to hold mutants, or + /// mutating one file would quietly withdraw the rest of the package's own tests. + pub selected: Vec, + + /// Every test target the workspace declares, by name, sorted and deduplicated. + /// + /// This is the population `--include-test` and `--exclude-test` are checked against, and it is + /// deliberately wider than the set of binaries a given run builds. A target gated behind + /// `required-features` is declared here and compiled only when those features are on, so + /// checking against what was built would reject a pattern in `gamma.toml` on every run that + /// did not happen to enable them — which is precisely the run the pattern exists to survive. + /// Collected across every workspace member, since `--package` chooses what to mutate while + /// these patterns choose what judges it. + pub tests: Vec, + + /// Every test target that appears to run the compiler rather than the code under test. + /// + /// Kept beside `tests` because it is a property of what the workspace declares rather than of + /// what this run happens to build, and because the run that most needs to hear about one is the + /// run that has not yet paid for it. + pub compile_fail: Vec, + + /// For each package, the positions in `files` that belong to it, in `files` order. + /// + /// A real run never scans the whole workspace at once — it scans a package at a time, across + /// every dependency stage — so selecting a package's files by filtering the flat list would + /// walk every file in the workspace once per package. That term is packages times files, which + /// is invisible on a small tree and quadratic on a monorepo. + /// + /// Positions rather than a second copy of the files, and positions in `files` order rather than + /// any order of their own, because the deterministic path order is what makes two scans of the + /// same workspace produce the same population. + by_package: HashMap>, + + /// For each package, its crate roots — the lib and bin entry points the module tree hangs off. + roots: HashMap>, + + /// For each package, where its manifest sits relative to the workspace root, and its version. + /// + /// A bare `--package name` is ambiguous whenever a workspace member shares its name with a + /// crate in the dependency graph, which happens routinely: a crate that dev-depends on a + /// published version of itself, or two members of a graph that both vendor a common name. + /// Cargo then refuses the build, and refuses it *before* producing any JSON, so the failure + /// arrives with no diagnostics to attribute and looks like the tree simply not compiling. + /// Keeping the manifest location lets every build name its packages exactly. + specs: HashMap, + + /// For each package, the configuration predicates that hold when it is built. + /// + /// Code the compiler will strip produces no mutants, because a guard there is never compiled + /// and no test could activate it. Resolved once here rather than per file, since a `rustc` + /// call and a feature closure per source file would dominate discovery. + cfgs: Cfgs, + + /// Every directory a workspace target has sources in, deduplicated and sorted. + /// + /// Wider than [`Self::files`] on purpose: it includes the targets this run will never mutate, + /// integration tests above all, because that is where a great many of the tests that convict + /// mutants actually live. Only [`Self::killers`] reads it, and only under `--incremental`. + pub(super) source_dirs: Vec, + + /// Local path dependencies Cargo can read outside the workspace root. + /// + /// They are not mutation candidates, but their bytes can change what a workspace package + /// compiles or what its tests observe. The record snapshot captures them separately. + external_inputs: Vec, + + /// Whether a build script may have read an external path Cargo does not report. + untracked_build_script_inputs: bool, + + diff: Option, + shard: Option<(u32, u32)>, + settled: HashMap, +} + +/// What scanning some part of the workspace yielded. +#[derive(Debug, Default)] +pub struct Scanned { + /// The mutants found, with ordinals already assigned to the live ones. + pub mutants: Vec, + + /// How many were suppressed by a directive. They stay in `mutants`, marked as ignored. + pub suppressed: usize, + + /// The skip directives that suppressed nothing, though this run offered them the chance. + pub idle: Vec, + + /// How many live mutants sharding excluded. These are counted rather than kept. + pub sharded_out: usize, + + /// How many mutants an earlier report had already settled. + /// + /// They are kept, carrying the verdict that report gave them, so the score is over the whole + /// population rather than over the part of it this run happened to retry. + pub settled_out: usize, + + /// A digest of the normalized source each analyzed file held when its mutants were derived. + /// + /// A leading UTF-8 BOM is omitted just as it is during parsing, so source-edit generation + /// checks compare the representation that supplied their line numbers. + /// + /// A command that edits source works from line numbers this scan decided, and applies them + /// later — for `suppress`, after a whole measured run, which can be hours. Whoever owns the + /// tree may have written to it in between, and a line number means nothing against text it was + /// not computed from. This is what lets the edit refuse rather than delete the wrong line. + pub digests: HashMap, + + /// Files that were found but could not be analyzed, each already a complete diagnostic. + /// + /// These contributed no mutants, so they are missing from both halves of the score's fraction + /// and the score is silently a claim about less code than the caller asked about. Reporting + /// them is not optional: a skipped file that nobody mentions is indistinguishable from a file + /// with nothing worth mutating in it. + pub skipped: Vec, +} + +impl Survey { + /// Finds the workspace and the files worth mutating, without parsing any of them. + /// + /// The build this describes is the one the configuration file and the environment ask for. + /// A caller that has already settled the run's cargo options — the run itself does, from + /// flags the selection arguments do not carry — should use [`Survey::for_build`] instead, so + /// that the predicates discovery evaluates are the ones the compiler will. + /// + /// # Errors + /// + /// Returns an error if configuration or cargo metadata cannot be read, a named package does + /// not exist, or the diff cannot be parsed. + #[cfg(test)] + pub fn new(args: &SelectArgs, shard: Option<(u32, u32)>) -> Result { + let config = crate::config::Config::resolve(args)?; + let cargo = config.cargo_options(); + + Self::for_build(args, shard, &cargo) + } + + /// Finds the files worth mutating for a build with these cargo options. + /// + /// Discovery and the build have to agree about what is compiled: a file surveyed under one + /// target, profile or set of `--cfg` flags and compiled under another produces guards in code + /// the compiler never sees, and drops mutants from code it does. The options the build will + /// use are therefore what the configuration predicates are derived from; see + /// [`CargoOptions::cfg_build`]. + /// + /// The feature selection is read from both places it can be written. A selector passed through + /// `-C`/`--cargo-arg`, or configured as a `gamma.toml` `cargo_args` entry, reaches the cargo + /// this run invokes just as gamma's own `--features` does, so the closure and the metadata are + /// resolved under the union of the two; see [`features::from_extra`]. + /// + /// # Errors + /// + /// Returns an error if cargo metadata cannot be read, a named package does not exist, or the + /// diff cannot be parsed. + pub fn for_build(args: &SelectArgs, shard: Option<(u32, u32)>, cargo: &CargoOptions) -> Result { + Self::for_build_with_cache_inputs(args, shard, cargo, false) + } + + #[expect(clippy::too_many_lines, reason = "workspace selection is one ordered metadata pass")] + pub(crate) fn for_build_with_cache_inputs( + args: &SelectArgs, + shard: Option<(u32, u32)>, + cargo: &CargoOptions, + cache_inputs: bool, + ) -> Result { + cargo.validate()?; + + let features = features::from_extra(&args.features, &cargo.extra); + let metadata = load_metadata(&args.dir, &features)?; + let root = Utf8PathBuf::from(metadata.workspace_root.as_str()); + let external_inputs = if cache_inputs { + external_path_inputs(&args.dir, &features, &root)? + } else { + ExternalPathInputs::default() + }; + let enabled = features::enabled(&metadata, &features); + let mut files: Vec = Vec::new(); + let mut seen: HashSet = HashSet::default(); + let mut declaration_files: HashMap> = HashMap::default(); + let mut declaration_seen: HashSet = HashSet::default(); + let mut roots: HashMap> = HashMap::default(); + let mut specs: HashMap = HashMap::default(); + let mut source_dirs: HashSet = HashSet::default(); + + let mut diff = args.in_diff.as_ref().map(|path| Diff::read(path)).transpose()?; + let patterns = FilePatterns::new(args); + + // Only collected when there is something to check, since it is one string per Rust file in + // the workspace and the overwhelmingly common case has no patterns at all. A diff needs the + // same list, to say which workspace file each path it names refers to. + let checking_patterns = !args.files.is_empty() || !args.exclude_files.is_empty(); + let collecting_walked = checking_patterns || diff.is_some(); + let mut walked: Vec = Vec::new(); + + if let Some(named) = unknown_packages(&metadata, args) { + return Err(error!("no package named `{named}` in this workspace").usage()); + } + + let selected = selected_packages(&metadata, args); + + for package in metadata.workspace_packages() { + let mutating = selected.contains(package.name.as_str()); + + if let Some(directory) = Utf8Path::new(package.manifest_path.as_str()).parent() { + let relative = directory + .strip_prefix(&root) + .unwrap_or_else(|_outside| Utf8Path::new("")) + .to_owned(); + let _replaced = specs.insert(package.name.to_string(), (relative, package.version.to_string())); + } + + // A package this run does not mutate is still walked when patterns need checking. The + // patterns usually live in `gamma.toml` and are written once for the whole workspace, + // whereas `--package` narrows a single run; validating them against the narrowed set + // would reject a correct config on every run that happened to select another package. + if !mutating && !checking_patterns { + continue; + } + + for target in &package.targets { + // Before the mutability filter, so that the tests which judge this workspace are + // indexed whether or not their own target is one this run would mutate. + if let Some(directory) = Utf8Path::new(target.src_path.as_str()).parent() { + let _added = source_dirs.insert(directory.to_owned()); + } + + if !is_mutable_target(target, enabled.get(package.name.as_str())) { + continue; + } + + let source_root = Utf8Path::new(target.src_path.as_str()); + + // The module tree is walked from here to find the files that exist only for tests, + // so a root is recorded whether or not it survives the filters below: a crate root + // excluded from mutation still says what the rest of the crate is. + if mutating { + roots.entry(package.name.to_string()).or_default().push(source_root.to_owned()); + } + + let Some(directory) = source_root.parent() else { + continue; + }; + + for absolute in walk_rust_files(directory)? { + let relative = placed_under(&root, &absolute, &package.name)?; + + if collecting_walked { + walked.push(relative.clone()); + } + + if !mutating { + continue; + } + + if declaration_seen.insert(absolute.clone()) { + declaration_files + .entry(package.name.to_string()) + .or_default() + .push(absolute.clone()); + } + + if !patterns.includes(&relative) { + continue; + } + + // A lib and a bin target in one package usually share a source directory, so + // the same file is walked more than once. A set rather than a scan of what is + // already held, because a large workspace makes that scan quadratic. + if !seen.insert(absolute.clone()) { + continue; + } + + files.push(TargetFile { + path: relative, + absolute, + package: package.name.to_string(), + }); + } + } + } + + files.sort_by(|left, right| left.path.cmp(&right.path)); + for paths in declaration_files.values_mut() { + paths.sort(); + } + + // The diff names files; only the workspace can say which files those are. Resolving it + // needs the walk to have happened, and it fails rather than selects nothing when not one + // of the paths it names turns out to be a file here. + if let Some(diff) = diff.as_mut() { + diff.resolve(&root, &walked)?; + + // A file the diff does not mention cannot contain a changed line, so it is dropped + // before it is parsed rather than after, which is most of what makes `--in-diff` fast + // enough to run on every pull request. + files.retain(|file| diff.touches_file(&file.path)); + } + + if let Some(pattern) = patterns.unmatched(&walked) { + return Err(error!( + "no source file matches `{pattern}`; patterns are relative to the workspace root and use `/` on every platform" + ) + .usage()); + } + + let mut by_package: HashMap> = HashMap::default(); + + for (position, file) in files.iter().enumerate() { + by_package.entry(file.package.clone()).or_default().push(position); + } + + let cfgs = configuration(&root, cargo, &enabled); + + Ok(Self { + root, + files, + declaration_files, + by_package, + reach: reachable(&metadata), + selected: sorted(selected), + tests: test_targets(&metadata), + compile_fail: compile_fail_targets(&metadata), + roots, + specs, + cfgs, + diff, + shard, + settled: HashMap::default(), + source_dirs: sorted(source_dirs), + external_inputs: external_inputs.roots, + untracked_build_script_inputs: external_inputs.has_build_scripts, + }) + } + + /// Indexes the test functions this workspace declares right now. + /// + /// Only worth building for a run that is about to carry an earlier report's kills forward; see + /// [`Killers`] for why the names are read out of the sources rather than out of a harness. + /// + /// A directory that cannot be walked contributes nothing here rather than failing the run, + /// which is the opposite of what the same walk does when it builds the population — and for + /// the opposite reason. A file missing from *this* index can only make a recorded kill + /// unconfirmable, and an unconfirmed kill is re-run rather than believed, so the error costs + /// time. A file missing from the population is a mutant nobody measures and a score nobody can + /// see is wrong. + #[must_use] + pub fn killers(&self) -> Killers { + let mut files = Vec::new(); + let mut complete = true; + + for directory in &self.source_dirs { + match walk_rust_files(directory) { + Ok(found) => files.extend(found), + Err(_failure) => complete = false, + } + } + + Killers::scan_complete(&files, complete) + } + + /// Local path dependency roots a cache snapshot must include. + #[must_use] + pub(crate) fn external_inputs(&self) -> &[Utf8PathBuf] { + &self.external_inputs + } + + /// Whether cache reuse must be disabled because a build script can read untracked paths. + #[must_use] + pub(crate) const fn has_untracked_build_script_inputs(&self) -> bool { + self.untracked_build_script_inputs + } + + /// Adopts the verdicts an earlier report already settled, before ordinals are handed out. + /// + /// A settled mutant is not run again — it takes no ordinal, is never instrumented and is never + /// announced as work — but it keeps its place in the population wearing the verdict it earned. + /// Dropping it instead would make the run report on the subset it retried: the score would be + /// computed over a handful of survivors, `--min-score` would be judged against that, and the + /// report written out could not be fed to the next iteration because it no longer describes the + /// whole population. + pub fn settle(&mut self, settled: HashMap) { + self.settled = settled; + } + + /// The workspace packages that have files worth mutating, in a stable order. + #[must_use] + pub fn packages(&self) -> Vec { + let mut order: Vec = Vec::new(); + let mut seen: HashSet<&str> = HashSet::default(); + + for file in &self.files { + if seen.insert(file.package.as_str()) { + order.push(file.package.clone()); + } + } + + order + } + + /// An empty plan for this workspace, to be filled in a package at a time. + #[must_use] + pub fn skeleton(&self) -> Plan { + Plan { + root: self.root.clone(), + files: self.files.clone(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + skipped: Vec::new(), + digests: HashMap::default(), + reach: self.reach.clone(), + specs: self.specs.clone(), + } + } + + /// Parses and mutates one package's files, or every file when `package` is `None`. + /// + /// `ordinals` carries the last ordinal handed out, so that packages scanned one after another + /// number their mutants continuously. Ordinals name the live mutants to the guard runtime, so + /// they have to be unique across the whole run, not within a package. + /// + /// # Errors + /// + /// Returns an error if a file cannot be read or parsed. + pub fn scan(&self, package: Option<&str>, selection: &Selection, ordinals: &mut u32) -> Result { + let files: Vec<&TargetFile> = package.map_or_else( + || self.files.iter().collect(), + |wanted| { + self.by_package + .get(wanted) + .map(|positions| positions.iter().filter_map(|&position| self.files.get(position)).collect()) + .unwrap_or_default() + }, + ); + + let roots: Vec = package.map_or_else( + || self.roots.values().flatten().cloned().collect(), + |wanted| self.roots.get(wanted).cloned().unwrap_or_default(), + ); + let declaration_files: Vec<(&Utf8Path, &CfgSet)> = package.map_or_else( + || { + self.declaration_files + .iter() + .flat_map(|(package, paths)| { + let cfg = self.cfgs.for_package(package); + + paths.iter().map(move |path| (path.as_path(), cfg)) + }) + .collect() + }, + |wanted| { + let cfg = self.cfgs.for_package(wanted); + + self.declaration_files + .get(wanted) + .map(|paths| paths.iter().map(|path| (path.as_path(), cfg)).collect()) + .unwrap_or_default() + }, + ); + let Scan { + mut mutants, + suppressed, + idle, + skipped, + digests, + } = scan(&files, &declaration_files, &roots, selection, &self.cfgs)?; + + // Within a file the diff still has the last word: a changed line usually sits among many + // that were not touched, and mutating those would report on code the change never went + // near. A mutant is selected by its whole extent, from the line its site starts on to the + // one it ends on, so editing an interior line of a multi-line site still selects it. + if let Some(diff) = self.diff.as_ref() { + mutants.retain(|mutant| { + let start = u32::try_from(mutant.line).unwrap_or(u32::MAX); + let end = u32::try_from(mutant.end_line).unwrap_or(u32::MAX); + + diff.touches(&mutant.file, start, end) + }); + } + + // A mutant an earlier run already settled takes the verdict that run gave it and stops + // being work: no ordinal, no shard slot, nothing built for it. It stays in the population, + // because the score is a claim about the population and not about whichever part of it this + // run had reason to retry. + let mut settled_out = 0_usize; + + if !self.settled.is_empty() { + for mutant in &mut mutants { + if let Some(outcome) = self.settled.get(&mutant.id).copied() { + mutant.outcome = outcome; + settled_out = settled_out.saturating_add(1); + } + } + } + + // Suppressed and already-settled mutants are kept but never run, so they take no part in + // sharding: letting them occupy shard slots would make one night's shard cheaper than + // another for no reason, and would hide how much of the population is actually being + // exercised. + let is_live = |mutant: &Mutant| mutant.outcome == Outcome::Pending; + let before = mutants.iter().filter(|mutant| is_live(mutant)).count(); + + if let Some((count, index)) = self.shard { + mutants.retain(|mutant| !is_live(mutant) || shard_of(&mutant.id, count) == index); + } + + // Each file shared its own strings as it produced its mutants, but a mutator name and a + // package name repeat across every file in the workspace. This is the first point that has + // the whole population, and so the first that can collapse those onto one copy each. + Interner::default().share(&mut mutants); + + let mut live = 0_usize; + + for mutant in &mut mutants { + if is_live(mutant) { + live = live.saturating_add(1); + *ordinals = ordinals.saturating_add(1); + mutant.ordinal = *ordinals; + } + } + + Ok(Scanned { + mutants, + suppressed, + idle, + sharded_out: before - live, + settled_out, + skipped, + digests, + }) + } + + /// Turns a scan of the whole workspace into the plan a run works from. + #[must_use] + pub fn into_plan(self, scanned: Scanned) -> Plan { + let mut plan = Plan { + root: self.root, + files: self.files, + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + skipped: Vec::new(), + digests: HashMap::default(), + reach: self.reach, + specs: self.specs, + }; + + plan.absorb(scanned); + plan.sort(); + + plan + } +} + +/// Reports what each package yielded, once the counts exist. +/// +/// Package order follows the files, which are already sorted by path, so the same workspace always +/// reports in the same order. A package that produced no mutants is still named: a crate that +/// silently contributes nothing to a run is worth noticing, and its absence from the list would +/// look like it had simply not been looked at. +fn report_by_package(files: &[TargetFile], mutants: &[Mutant], notify: &mut impl FnMut(&str)) { + let mut order: Vec<&str> = Vec::new(); + let mut counts: HashMap<&str, (usize, usize)> = HashMap::default(); + + for file in files { + let entry = counts.entry(file.package.as_str()).or_insert_with(|| { + order.push(file.package.as_str()); + + (0, 0) + }); + + entry.0 += 1; + } + + for mutant in mutants { + if let Some(entry) = counts.get_mut(&*mutant.package) { + entry.1 += 1; + } + } + + for package in order { + let (files, mutants) = counts.get(package).copied().unwrap_or((0, 0)); + + notify(&format!( + "{package}, {} in {}", + crate::report::quantity(mutants, "mutant"), + crate::report::quantity(files, "file") + )); + } +} + +/// Turns one parsed file into its mutants, with the suppressions it declares already applied. +fn mutate(file: &TargetFile, source: &SourceFile, selection: &Selection, cfgs: &Cfgs, defaults: &collect::Defaults) -> Result { + // Taken from the tree that was parsed for mutants anyway, so knowing which files exist only for + // tests costs a walk over the top-level items rather than a second parse of everything. + let cfg = cfgs.for_package(&file.package); + let declared = modules::declarations(&file.absolute, &source.ast, cfg); + + // Before anything is collected, because a stated value that cannot be honoured is a hint the + // author believes is working. Reporting it is worth more than the mutants of the file it sits + // in, and reporting it first means the message is about the attribute rather than about + // whatever the file happened to yield without it. + // + // Checked and collected in one call, which runs the stated-value audit and the numeric/import + // indexes in a single walk of the syntax tree rather than the two separate ones a standalone + // `check_stated` followed by `collect_with` would need. + let candidates = collect::check_stated_and_collect_with(source, selection, cfg, defaults)?; + let mut found = collect::into_mutants(source, &file.package, candidates); + let directives = suppress::directives_for(source, cfg)?; + let suppressed = suppress::suppress(&mut found, &directives); + + // Asked here, before the diff and the shard have had their say, because those narrow the + // population within a file that was scanned in full. A directive whose mutants all fall + // outside `--in-diff` has not stopped earning its place, and saying so would make every + // incremental run condemn most of the tree. + let idle = suppress::idle(&file.path, &found, &directives, selection); + + Ok(Parsed { + mutants: found, + suppressed, + idle, + declared, + digest: crate::discover::digest(source.text.as_bytes()), + }) +} + +/// Reads, parses and mutates every file, returning the population and how much of it was suppressed. +/// +/// Parsing is what discovery actually spends its time on, so the files are divided across the +/// available cores. Work is claimed one file at a time rather than in fixed blocks, since files +/// vary enormously in size and a static split leaves the machine waiting on whichever worker drew +/// the largest ones. Results are put back in file order afterwards, so the population does not +/// depend on how the work happened to land. +/// +/// It happens in two phases, because one file's mutants depend on what the others declare: +/// `Default::default()` is only worth offering for a type that has a `Default`, and the definition +/// that settles that is usually in a different file. So every file is parsed first, every syntax +/// tree is held, and the index over them is complete before any mutant is emitted. +/// +/// A worker keeps the trees it parsed and mutates those same files in the second phase. A syntax +/// tree is not `Send` — `syn` spans carry a handle that only the thread that made them may touch — +/// so the trees cannot be pooled and redistributed. What crosses the barrier between the phases is +/// the index, which is only names. That is also why the two phases share one scope rather than +/// running as two consecutive ones: the trees cannot outlive the thread that built them. +/// +/// The barrier's party count is fixed at spawn, so *every* worker must reach both waits on every +/// path, including the ones this code did not plan for. Failures are recorded and fall through; +/// panics are caught, held, and resumed once both waits are behind them. +fn scan( + files: &[&TargetFile], + declaration_files: &[(&Utf8Path, &CfgSet)], + roots: &[Utf8PathBuf], + selection: &Selection, + cfgs: &Cfgs, +) -> Result { + let workers = thread::available_parallelism().map_or(1, NonZero::get).min(files.len().max(1)); + let shared = Shared { + next: AtomicUsize::new(0), + partials: Mutex::new(Vec::new()), + skipped: Mutex::new(Vec::new()), + barrier: Barrier::new(workers), + defaults: OnceLock::new(), + }; + + let mut collected: Vec<(usize, Parsed)> = thread::scope(|scope| { + let handles: Vec<_> = (0..workers) + .map(|_worker| { + let shared = &shared; + + scope.spawn(move || work(files, shared, selection, cfgs)) + }) + .collect(); + + let mut collected = Vec::new(); + let mut failure: Option<(usize, Error)> = None; + + for handle in handles { + // A panic in a worker is a bug in this crate, not something a user can act on, so it + // is propagated rather than turned into a diagnostic that blames their code. It can + // only get here because the worker held it until both barriers were behind it. + match handle.join().unwrap_or_else(|payload| resume_unwind(payload)) { + Ok(mine) => collected.extend(mine), + + // Several files can be unreadable or unparseable at once, and which worker noticed + // first is a race. The earliest in file order is reported so the message does not + // change between runs. + Err((at, error)) => { + if failure.as_ref().is_none_or(|(seen, _)| at < *seen) { + failure = Some((at, error)); + } + } + } + } + + match failure { + Some((_at, error)) => Err(error), + None => Ok(collected), + } + })?; + + collected.sort_by_key(|(index, _parsed)| *index); + + // A file reachable only through a test-only or inactive module declaration is absent from the + // production population, whatever it looks like from the inside, and its mutants are dropped + // rather than reported. A mutated assertion is a broken test, not a gap in one, and inactive + // code is not built for any test to catch. + let mut declared: Vec<(Utf8PathBuf, Vec)> = collected + .iter() + .map(|(index, parsed)| { + let path = files.get(*index).map_or_else(Utf8PathBuf::new, |file| file.absolute.clone()); + + (path, parsed.declared.clone()) + }) + .collect(); + let selected: HashSet<&Utf8Path> = files.iter().map(|file| file.absolute.as_path()).collect(); + + let extra_decl_files: Vec<(&Utf8Path, &CfgSet)> = declaration_files + .iter() + .filter(|&&(path, _cfg)| !selected.contains(path)) + .map(|&(path, cfg)| (path, cfg)) + .collect(); + + if !extra_decl_files.is_empty() { + let extra_declared = parse_declarations_parallel(&extra_decl_files)?; + declared.extend(extra_declared); + } + + let excluded = modules::excluded_files(roots, &declared); + let total = collected.iter().map(|(_index, parsed)| parsed.mutants.len()).sum(); + let mut mutants = Vec::with_capacity(total); + let mut digests: HashMap = HashMap::default(); + let mut suppressed = 0; + let mut idle = Vec::new(); + + for (index, parsed) in collected { + if files.get(index).is_some_and(|file| excluded.contains(&file.absolute)) { + continue; + } + + if let Some(file) = files.get(index) { + let _replaced = digests.insert(file.path.clone(), parsed.digest); + } + + mutants.extend(parsed.mutants); + suppressed += parsed.suppressed; + idle.extend(parsed.idle); + } + + // Sorted by file order for the same reason the earliest failure is the one reported: which + // worker claimed which file is a race, and a diagnostic that reorders itself between runs is + // one nobody can diff. + let mut unanalyzable = shared.skipped.into_inner().unwrap_or_else(PoisonError::into_inner); + + unanalyzable.sort_by_key(|(at, _message)| *at); + + Ok(Scan { + mutants, + suppressed, + idle, + skipped: unanalyzable.into_iter().map(|(_at, message)| message).collect(), + digests, + }) +} + +type DeclarationParse = (usize, Result<(Utf8PathBuf, Vec)>); + +/// Parses declaration-only files with bounded parallelism. +/// +/// Each file is read and parsed solely to extract module declarations — no mutation is performed. +/// The parallelism is bounded by `available_parallelism` to avoid exceeding system thread limits. +/// Results are returned in input order. +/// +/// # Errors +/// +/// Returns the first file-read or parse error encountered, in path order. +fn parse_declarations_parallel(files: &[(&Utf8Path, &CfgSet)]) -> Result)>> { + if files.is_empty() { + return Ok(Vec::new()); + } + + let workers = thread::available_parallelism().map_or(1, NonZero::get).min(files.len()); + + if workers <= 1 { + let mut results = Vec::with_capacity(files.len()); + for &(path, cfg) in files { + let source = SourceFile::read(path)?; + results.push(((*path).to_owned(), modules::declarations(path, &source.ast, cfg))); + } + return Ok(results); + } + + let next = AtomicUsize::new(0); + + let mut results: Vec = thread::scope(|scope| { + let handles: Vec<_> = (0..workers) + .map(|_| { + let next = &next; + scope.spawn(move || { + let mut mine = Vec::new(); + loop { + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(&(path, cfg)) = files.get(index) else { + break; + }; + match SourceFile::read(path) { + Ok(source) => { + let decls = modules::declarations(path, &source.ast, cfg); + mine.push((index, Ok(((*path).to_owned(), decls)))); + } + Err(e) => { + mine.push((index, Err(e.into()))); + } + } + } + mine + }) + }) + .collect(); + + let mut all = Vec::new(); + for handle in handles { + all.extend(handle.join().unwrap_or_else(|payload| resume_unwind(payload))); + } + all + }); + + results.sort_by_key(|(index, _result)| *index); + + // Check for errors after restoring input order, so scheduling cannot choose the diagnostic. + let mut ok_results: Vec<(Utf8PathBuf, Vec)> = Vec::with_capacity(results.len()); + + for (_index, result) in results { + ok_results.push(result?); + } + + // Declaration consumers use path order, independent of how the caller ordered its pairs. + ok_results.sort_by(|a, b| a.0.cmp(&b.0)); + + Ok(ok_results) +} + +/// What parsing and mutating a set of files produced. +struct Scan { + mutants: Vec, + suppressed: usize, + idle: Vec, + skipped: Vec, + digests: HashMap, +} + +/// Everything the workers of one scan share, so that adding a channel does not widen every +/// signature between here and the loop that uses it. +struct Shared { + /// The next file index to claim, so work is taken one file at a time rather than in blocks. + next: AtomicUsize, + + /// Each worker's index of what the files it parsed declare, merged by the leader. + partials: Mutex>, + + /// Files stepped over, each with the index that orders it and the diagnostic that names it. + skipped: Mutex>, + + /// Where every worker meets between the two phases, since one file's mutants depend on what the + /// others declare. + barrier: Barrier, + + /// The merged index, set once by the leader and read by all of them. + defaults: OnceLock, +} + +/// What one file yielded when it was parsed. +/// One survey worker: parses whatever files it can claim, then mutates the ones it parsed. +/// +/// Split out of `scan` because it is the whole of a worker's life and reads better whole, and +/// because the barrier discipline it implements is the point of the function rather than a detail +/// of the loop that spawns it. +/// +/// The `usize` in the error is the index of the offending file, so `scan` can report the earliest +/// in file order rather than whichever worker happened to notice first. +fn work(files: &[&TargetFile], shared: &Shared, selection: &Selection, cfgs: &Cfgs) -> Result, (usize, Error)> { + let Shared { + next, + partials, + skipped, + barrier, + defaults, + } = shared; + + let mut mine: Vec<(usize, SourceFile)> = Vec::new(); + let mut index = collect::Defaults::default(); + let mut failure: Option<(usize, Error)> = None; + + // Phase one is guarded because the two waits below are not optional. `Barrier` has a fixed party + // count and no poisoning, so a worker that unwound past them would leave every other worker + // blocked in `wait` forever, `thread::scope` blocked joining those workers, and the panic that + // should have been re-raised never raised at all — a silent hang, after the run has already paid + // for discovery. The read-failure path below takes the same shape for the same reason; this is + // that discipline extended to the failures the code cannot see coming. + let unwound = catch_unwind(AssertUnwindSafe(|| { + loop { + let at = next.fetch_add(1, Ordering::Relaxed); + let Some(file) = files.get(at) else { break }; + + #[cfg(test)] + #[cfg(not(miri))] + tests::panic_probe(&file.path); + + match SourceFile::read(&file.absolute) { + Ok(mut source) => { + // Report paths relative to the workspace root; that is what a user can act on + // and what a suppression or an expectation is keyed by. + source.path = file.path.clone(); + index.absorb(collect::Defaults::of_in(&source.ast, cfgs.for_package(&file.package))); + mine.push((at, source)); + } + + // A file this tool cannot analyze but `rustc` can build is not a reason to + // refuse the workspace; the rest of it is still worth measuring, and the file is + // named so the score is read knowing what is missing from it. Every other read + // failure stops the run, because it says the tree is not what it claims to be. + Err(error) if error.is_skippable() => { + skipped.lock().unwrap_or_else(PoisonError::into_inner).push((at, error.to_string())); + } + + Err(error) => { + failure = Some((at, error.into())); + break; + } + } + } + })) + .err(); + + partials.lock().unwrap_or_else(PoisonError::into_inner).push(mem::take(&mut index)); + + // Guarded for the same reason, and more urgently: the leader has one more wait to reach, and it + // is the only thread that can release the others. + let merged = if barrier.wait().is_leader() { + catch_unwind(AssertUnwindSafe(|| { + let mut merged = collect::Defaults::default(); + + for partial in partials.lock().unwrap_or_else(PoisonError::into_inner).drain(..) { + merged.absorb(partial); + } + + let _first = defaults.set(merged); + })) + .err() + } else { + None + }; + + let _released = barrier.wait(); + + // Nobody can be left waiting now, so the panic goes back to being a panic and reaches the `join` + // in `scan`, which is where a bug in this crate belongs. + if let Some(payload) = unwound.or(merged) { + resume_unwind(payload); + } + + if let Some(failed) = failure { + return Err(failed); + } + + // Unset only if the leader unwound out of the merge, in which case the leader is resuming that + // panic right now and it will reach `join`. There is nothing for this worker to mutate against, + // and nothing useful for it to say. + let Some(defaults) = defaults.get() else { + return Ok(Vec::new()); + }; + + let mut parsed = Vec::with_capacity(mine.len()); + + for (at, source) in &mine { + let Some(file) = files.get(*at) else { continue }; + + match mutate(file, source, selection, cfgs, defaults) { + Ok(one) => parsed.push((*at, one)), + Err(error) => return Err((*at, error)), + } + } + + Ok(parsed) +} + +struct Parsed { + mutants: Vec, + suppressed: usize, + idle: Vec, + declared: Vec, + + /// A digest of the exact bytes this file's mutants were derived from. + digest: String, +} + +/// Drains a set into the sorted vector the survey stores, so two scans agree on order. +fn sorted(values: HashSet) -> Vec { + let mut ordered: Vec = values.into_iter().collect(); + + ordered.sort_unstable(); + ordered +} + +/// Works out which workspace packages each workspace package can reach. +/// +/// Built from the declared dependencies rather than from a resolved graph, so it costs nothing +/// beyond the metadata already loaded. Dependencies of every kind count, including dev: an +/// integration test links its package's dev-dependencies, and being over-inclusive here can only +/// cost time, never correctness — the reverse would silently skip a test that really does reach the +/// mutated code and turn a survivor into a false clean bill of health. +/// +/// The metadata is loaded with `--no-deps`, so a dependency that is not itself a workspace member +/// has no entry to walk into. A registry dependency cannot lead back into the workspace and can be +/// ignored, but a *path* dependency outside the workspace can: `app -> facade -> core` is a real +/// chain that this graph cannot see. Rather than skip a test binary that does reach the mutated +/// code, a package with such a dependency reaches everything — and so does every package that can +/// reach it, which is both the same fail-open argument and what keeps a dependency's reach set a +/// subset of its dependent's, the property [`stages`](super::stages) sorts on. +/// +/// Members are assigned integer IDs so graph traversal avoids allocating package names. +/// +/// Reachability itself is computed once for the whole graph rather than once per starting package: +/// [`reachable_ids`] collapses the graph into strongly connected components and gives each +/// component's closure a single bitset, built in one bottom-up pass over the condensation instead +/// of a fresh breadth-first search per member. A workspace of `n` packages and `e` dependency edges +/// used to cost `O(n * (n + e))`; collapsing first costs `O(n + e)` to find the components and +/// `O(c^2 / 64)` words to union their closures, where `c <= n` is the component count — a real +/// saving whenever `c` is smaller than `n`, and never worse, since a graph with no cycles has +/// `c == n` and the bitset union pass still costs only `O(n^2 / 64)` words rather than `O(n^2)` +/// pointer-chasing queue operations. +fn reachable(metadata: &Metadata) -> HashMap> { + let member_packages: Vec<&str> = metadata.workspace_packages().iter().map(|p| p.name.as_str()).collect(); + let member_count = member_packages.len(); + + let name_to_id: HashMap<&str, usize> = member_packages.iter().enumerate().map(|(id, name)| (*name, id)).collect(); + + let mut edges: Vec> = vec![Vec::new(); member_count]; + let mut opaque: Vec = vec![false; member_count]; + + for package in metadata.workspace_packages() { + let Some(&src_id) = name_to_id.get(package.name.as_str()) else { + continue; + }; + + for dependency in &package.dependencies { + if let Some(&dst_id) = name_to_id.get(dependency.name.as_str()) { + edges[src_id].push(dst_id); + } else if dependency.path.is_some() { + opaque[src_id] = true; + } + } + } + + reachable_ids(&edges, &opaque) + .into_iter() + .enumerate() + .map(|(id, reached)| { + let reachable_set: HashSet = reached + .into_iter() + .map(|reached_id| member_packages[reached_id].to_owned()) + .collect(); + + (member_packages[id].to_owned(), reachable_set) + }) + .collect() +} + +/// A fixed-size, word-packed set of small integers. +/// +/// Used to hold one strongly connected component's reach set: a workspace has too few members to +/// justify pulling in a bitset crate, but the closures computed below are unioned often enough that +/// per-word operations matter more than the allocation they would need to avoid. +#[derive(Clone)] +struct Bitset { + words: Vec, +} + +impl Bitset { + fn new(len: usize) -> Self { + Self { + words: vec![0; len.div_ceil(64)], + } + } + + fn set(&mut self, index: usize) { + self.words[index / 64] |= 1 << (index % 64); + } + + /// Unions another set's bits into this one. + fn or_with(&mut self, other: &Self) { + for (word, other_word) in self.words.iter_mut().zip(&other.words) { + *word |= *other_word; + } + } + + fn iter_set(&self) -> Vec { + let mut out = Vec::new(); + + for (word_index, word) in self.words.iter().enumerate() { + let mut remaining = *word; + + while remaining != 0 { + let bit = remaining.trailing_zeros(); + out.push(word_index * 64 + usize::try_from(bit).unwrap_or(0)); + remaining &= remaining - 1; + } + } + + out + } +} + +/// One call frame of the iterative Tarjan walk, replacing the recursive call [`strongly_connected_components`] +/// would otherwise need one stack frame per node for. +/// +/// A workspace's dependency graph can have a long linear chain (`a -> b -> c -> ...`), and a +/// recursive implementation would need one native stack frame per link in it. Driving the descent +/// through an explicit stack removes that risk entirely, at the cost of tracking, per node +/// currently open, which of its edges has already been followed. +struct Frame { + node: usize, + edge_at: usize, +} + +/// Collapses a directed graph into its strongly connected components, using Tarjan's algorithm. +/// +/// Returns each node's component id. Components are numbered in the order they finish (are fully +/// popped off Tarjan's stack), which guarantees that every edge crossing from component `c` to a +/// different component `d` satisfies `d < c`: nothing a component points to can finish, and so be +/// numbered, after it does. [`reachable_ids`] relies on this to compute every component's closure +/// in one forward pass over increasing ids, without a separate topological sort. +fn strongly_connected_components(edges: &[Vec]) -> Vec { + let node_count = edges.len(); + let mut index_of: Vec> = vec![None; node_count]; + let mut low_link: Vec = vec![0; node_count]; + let mut on_stack: Vec = vec![false; node_count]; + let mut comp_of: Vec = vec![usize::MAX; node_count]; + let mut stack: Vec = Vec::new(); + let mut next_index: u32 = 0; + let mut next_component: usize = 0; + + for start in 0..node_count { + if index_of[start].is_some() { + continue; + } + + let mut call_stack: Vec = vec![Frame { node: start, edge_at: 0 }]; + index_of[start] = Some(next_index); + low_link[start] = next_index; + next_index += 1; + stack.push(start); + on_stack[start] = true; + + while let Some(frame) = call_stack.last_mut() { + let node = frame.node; + + if frame.edge_at < edges[node].len() { + let neighbor = edges[node][frame.edge_at]; + frame.edge_at += 1; + + if let Some(neighbor_index) = index_of[neighbor] { + if on_stack[neighbor] { + low_link[node] = low_link[node].min(neighbor_index); + } + } else { + index_of[neighbor] = Some(next_index); + low_link[neighbor] = next_index; + next_index += 1; + stack.push(neighbor); + on_stack[neighbor] = true; + call_stack.push(Frame { + node: neighbor, + edge_at: 0, + }); + } + } else { + call_stack.pop(); + + // Guarded by pushing `start` above and never popping below it in this loop, so a + // parent frame is always here to read the child's finished low-link from. + if let Some(parent) = call_stack.last() { + low_link[parent.node] = low_link[parent.node].min(low_link[node]); + } + + if low_link[node] == index_of[node].expect("just indexed above, on the way in") { + loop { + let member = stack.pop().expect("root of this component pushed before this loop started"); + on_stack[member] = false; + comp_of[member] = next_component; + + if member == node { + break; + } + } + + next_component += 1; + } + } + } + } + + comp_of +} + +/// The pure graph half of [`reachable`]: every node's id-keyed reachable set, from an edge list and +/// an opaque flag per node. +/// +/// Split out from [`reachable`] so the graph algorithm can be tested directly against small, +/// hand-built and randomized graphs, without needing a real `cargo_metadata::Metadata` to drive it. +fn reachable_ids(edges: &[Vec], opaque: &[bool]) -> Vec> { + let node_count = edges.len(); + let comp_of = strongly_connected_components(edges); + let component_count = comp_of.iter().copied().max().map_or(0, |max| max + 1); + + let mut comp_edges: Vec> = vec![HashSet::default(); component_count]; + let mut comp_opaque: Vec = vec![false; component_count]; + let mut comp_members: Vec> = vec![Vec::new(); component_count]; + + for node in 0..node_count { + let comp = comp_of[node]; + comp_members[comp].push(node); + + if opaque[node] { + comp_opaque[comp] = true; + } + + for &neighbor in &edges[node] { + let neighbor_comp = comp_of[neighbor]; + + if neighbor_comp != comp { + let _added = comp_edges[comp].insert(neighbor_comp); + } + } + } + + // Every component's own closure over components: itself, plus every successor's closure. Built + // in increasing id order, which the finish-order numbering above guarantees is also a + // reachability order — by the time `comp` is processed, everything smaller it points to already + // has its own finished closure to union in. + let mut comp_closure: Vec = Vec::with_capacity(component_count); + + for (comp, successors) in comp_edges.iter().enumerate() { + let mut closure = Bitset::new(component_count); + closure.set(comp); + + for &successor in successors { + let successor_closure = comp_closure[successor].clone(); + closure.or_with(&successor_closure); + } + + comp_closure.push(closure); + } + + // A component reaches everything once any component in its own closure is opaque: the missing + // edges that make a component's own graph incomplete make it impossible to prove it does *not* + // reach a package, and the same is true of anything that can reach that component. + let all_nodes: HashSet = (0..node_count).collect(); + let effective_opaque: Vec = (0..component_count) + .map(|comp| comp_closure[comp].iter_set().into_iter().any(|reached| comp_opaque[reached])) + .collect(); + + (0..node_count) + .map(|node| { + let comp = comp_of[node]; + + if effective_opaque[comp] { + all_nodes.clone() + } else { + comp_closure[comp] + .iter_set() + .into_iter() + .flat_map(|reached_comp| comp_members[reached_comp].iter().copied()) + .collect() + } + }) + .collect() +} + +/// Works out which configuration predicates hold for each package of the workspace. +/// +/// Asking `rustc` is one process for the whole run, and the feature closure is arithmetic over +/// metadata that has already been loaded, so this is cheap enough to do unconditionally. +/// +/// The predicates describe the build cargo will run rather than a bare host compile: its target, +/// its profile's `debug_assertions` and whatever `--cfg` its flags carry all decide which code is +/// compiled, and evaluating a different build would classify code the compiler does see as absent +/// and drop its mutants. +/// +/// A `rustc` that cannot be run, or a build no single set of predicates describes, leaves every set +/// unconditional, which is exactly how the tool behaved before it evaluated predicates at all: +/// nothing is stripped, and a user on an unusual toolchain gets a noisier report rather than a +/// failed run. +fn configuration(root: &Utf8Path, cargo: &CargoOptions, enabled: &HashMap>) -> Cfgs { + let build = cargo.cfg_build(root); + + let Ok(target) = crate::cfg::for_build(&build) else { + return Cfgs::unconditional(); + }; + + Cfgs::new(&target, enabled) +} + +/// Places a package's source file relative to the workspace root, refusing one that lies outside it. +/// +/// `Utf8Path::join` with an absolute argument replaces the base, so an absolute `TargetFile::path` +/// bypasses every containment check in the filesystem layer and the write lands in the user's real +/// source tree rather than the scratch copy. Cargo accepts a member outside the root when the member +/// names its workspace, so this is reachable, and it is refused rather than repaired into a path +/// that means something else. +/// +/// # Errors +/// +/// Returns an error if `absolute` is not under `root`. +fn placed_under(root: &Utf8Path, absolute: &Utf8Path, package: &str) -> Result { + let Ok(inside) = absolute.strip_prefix(root) else { + return Err(error!( + "package `{package}` has the source file `{absolute}`, which lies outside the workspace root `{root}`; \ + gamma cannot mutate a file it cannot place inside its scratch copy" + )); + }; + + Ok(Utf8PathBuf::from(normalize_separators(inside.as_str()))) +} + +/// Loads cargo metadata for the tree at `dir`. +/// +/// The feature selection has to match the one the build will use. Metadata decides which targets +/// exist and which files are walked, so discovering under one feature set and compiling under +/// another would place guards in files the compiler never sees. +pub fn load_metadata(dir: &Utf8Path, features: &FeatureArgs) -> Result { + let mut command = MetadataCommand::new(); + + let _builder = command.current_dir(dir).no_deps(); + + if features.all_features { + let _builder = command.features(CargoOpt::AllFeatures); + } + + if features.no_default_features { + let _builder = command.features(CargoOpt::NoDefaultFeatures); + } + + if !features.features.is_empty() { + // Cargo accepts a comma-separated list in one argument and repetition across several, so + // the entries are split apart here and handed over as the flat list they denote. + let named: Vec = features + .features + .iter() + .flat_map(|entry| entry.split([',', ' '])) + .filter(|entry| !entry.is_empty()) + .map(ToOwned::to_owned) + .collect(); + + let _builder = command.features(CargoOpt::SomeFeatures(named)); + } + + command + .exec() + .map_err(|cause| error!("could not read cargo metadata for `{dir}`").caused_by(cause)) +} + +/// The local filesystem inputs Cargo's dependency graph exposes. +#[derive(Default)] +struct ExternalPathInputs { + roots: Vec, + has_build_scripts: bool, +} + +/// Finds local packages Cargo resolves outside the workspace root. +/// +/// The ordinary metadata pass intentionally uses `--no-deps`, because discovery only needs +/// workspace targets. Cache provenance needs the opposite answer as well: a path dependency's +/// source is a filesystem input even though it is not a workspace target and can change without a +/// workspace file moving. Registry and git packages are identified by their locked source and are +/// covered by the lockfile; only local packages have no source identifier to carry that change. +/// Build scripts are different: any package's script can read arbitrary paths that metadata does +/// not enumerate, so their presence makes a snapshot incomplete. +fn external_path_inputs(dir: &Utf8Path, features: &FeatureArgs, root: &Utf8Path) -> Result { + let mut command = MetadataCommand::new(); + let _builder = command.current_dir(dir); + + if features.all_features { + let _builder = command.features(CargoOpt::AllFeatures); + } + + if features.no_default_features { + let _builder = command.features(CargoOpt::NoDefaultFeatures); + } + + if !features.features.is_empty() { + let named: Vec = features + .features + .iter() + .flat_map(|entry| entry.split([',', ' '])) + .filter(|entry| !entry.is_empty()) + .map(ToOwned::to_owned) + .collect(); + + let _builder = command.features(CargoOpt::SomeFeatures(named)); + } + + let metadata = command + .exec() + .map_err(|cause| error!("could not read the external Cargo inputs for `{dir}`").caused_by(cause))?; + let workspace = crate::paths::physical(root)?; + let mut roots = Vec::new(); + let mut has_build_scripts = false; + + for package in metadata.packages { + has_build_scripts |= package.targets.iter().any(|target| target.kind.contains(&TargetKind::CustomBuild)); + + if package.source.is_some() { + continue; + } + + let Some(directory) = Utf8Path::new(package.manifest_path.as_str()).parent() else { + return Err(error!( + "Cargo reported a package manifest without a parent at `{}`", + package.manifest_path + )); + }; + let directory = crate::paths::physical(directory)?; + + if !directory.starts_with(&workspace) { + roots.push(directory); + } + } + + roots.sort(); + roots.dedup(); + + Ok(ExternalPathInputs { roots, has_build_scripts }) +} + +/// Names the packages a run mutates, following cargo's own selection rules. +/// +/// Cargo acts on the package that owns the directory it was invoked from, and on the whole +/// workspace only when asked. Mutating every member by default would be a much worse surprise than +/// it is for `cargo build`: a run over an unasked-for member costs a full build and test suite per +/// mutant, in code the caller may not own, and a failure anywhere ends the run. +/// +/// At the workspace root there is no owning package, so the default members decide, exactly as they +/// do for a bare `cargo test`. `workspace_default_members` needs cargo 1.71, and dereferencing it on +/// anything older panics, so an unavailable list falls back to the owning package, or to every +/// member — which is what cargo itself does for a virtual manifest that declares no `default-members`. +fn selected_packages(metadata: &Metadata, args: &SelectArgs) -> HashSet { + let members = || { + metadata + .workspace_packages() + .iter() + .map(|package| package.name.to_string()) + .collect() + }; + let defaults = || -> Option> { + metadata.workspace_default_members.is_available().then(|| { + metadata + .workspace_default_packages() + .iter() + .map(|package| package.name.to_string()) + .collect() + }) + }; + + if !args.packages.is_empty() { + return args.packages.iter().cloned().collect(); + } + + if args.workspace { + return members(); + } + + // At the workspace root cargo's own default members are the answer, and that is the one place + // `owning_package` disagrees: a root which is *itself* a package and also declares + // `default-members` resolves to the root package alone, so gamma mutates one crate while a bare + // `cargo test` in the same directory runs another. The oracle is capped by this same selection, + // so the divergence changes verdicts rather than only cost. + if at_workspace_root(metadata, &args.dir) + && let Some(selected) = defaults() + { + return selected; + } + + if let Some(owner) = owning_package(metadata, &args.dir) { + return once(owner).collect(); + } + + defaults().unwrap_or_else(members) +} + +/// Returns whether `dir` is the workspace root itself rather than somewhere beneath it. +fn at_workspace_root(metadata: &Metadata, dir: &Utf8Path) -> bool { + let resolve = |path: &Utf8Path| path.canonicalize_utf8().unwrap_or_else(|_unresolved| path.to_owned()); + + resolve(dir) == resolve(Utf8Path::new(metadata.workspace_root.as_str())) +} + +/// Returns the workspace member whose directory contains `dir`, if any. +/// +/// The deepest match wins, since a package may be nested inside another package's directory and the +/// inner one is the one cargo would act on. A directory that resolves to no member — the root of a +/// virtual manifest, or a directory outside the workspace entirely — has no owner. +fn owning_package(metadata: &Metadata, dir: &Utf8Path) -> Option { + let absolute = dir.canonicalize_utf8().unwrap_or_else(|_unresolved| dir.to_owned()); + let mut best: Option<(usize, String)> = None; + + for package in metadata.workspace_packages() { + let Some(home) = Utf8Path::new(package.manifest_path.as_str()).parent() else { + continue; + }; + + let home = home.canonicalize_utf8().unwrap_or_else(|_unresolved| home.to_owned()); + + if !absolute.starts_with(&home) { + continue; + } + + let depth = home.components().count(); + + if best.as_ref().is_none_or(|(deepest, _name)| depth > *deepest) { + best = Some((depth, package.name.to_string())); + } + } + + best.map(|(_depth, name)| name) +} + +/// Returns the first `--package` name that no workspace member answers to. +/// +/// A misspelled package name would otherwise select nothing and report a clean run over an empty +/// population, which reads exactly like a workspace with no gaps in its tests. +fn unknown_packages<'args>(metadata: &Metadata, args: &'args SelectArgs) -> Option<&'args str> { + let known: HashSet<&str> = metadata.workspace_packages().iter().map(|package| package.name.as_str()).collect(); + + args.packages.iter().map(String::as_str).find(|wanted| !known.contains(wanted)) +} + +/// Returns whether a target contains code worth mutating. +/// +/// Test, bench and example targets are excluded: mutating a test measures the tests' tests, and +/// mutating an example measures nothing at all, since examples are usually not run by the suite. +/// +/// Proc-macro targets are excluded for a sharper reason: no mutant of one can ever be killed. A +/// proc macro runs inside `rustc`, while some *other* crate is being compiled, but a run builds the +/// tree once and only then selects one mutant per test process. By the time a test is watching, the +/// macro has long since finished its work, so every mutant of it survives however good the suite +/// is. Including them would charge a project the full cost of building and testing each one and +/// then hand back a pile of survivors that say nothing about its tests — which is exactly the kind +/// of unearned noise that teaches people to stop reading the score. +/// +/// The way to get a proc macro under mutation is to keep its logic in an ordinary library that the +/// macro crate delegates to, which is what this project does with `cargo-gamma-attrs-impl`. +/// +/// The guard runtime is excluded for a third reason, and it is the plainest of the three: a mutant +/// is only ever reached through a call to the runtime that decides whether it is active, so +/// mutating the runtime would place that call inside the very crate that defines it. The result +/// does not compile, and no amount of test quality changes that. This only bites a workspace that +/// vendors or develops the runtime itself — which is to say, this one — but the exclusion belongs +/// here rather than in a local ignore file, because it follows from how mutants are switched on +/// rather than from anything particular to this repository. +/// +/// A binary whose `required-features` are not all enabled is excluded too, because cargo will not +/// build it. Surveying it costs the run a target's worth of mutants that cannot compile, reported +/// as unviable and retried through the rollback loop, all of it about a target the build never had. +/// `enabled` is what the feature closure worked out for the target's own package, and `None` — a +/// package the closure never described — keeps the target, since proving a gate unmet is what +/// having no answer makes impossible. +/// +/// This decides targets rather than files, and the walk that follows a surviving target takes its +/// whole source directory: a gated `[[bin]]` whose file sits beside `lib.rs` is still reached +/// through the library. That is the same over-inclusion the walk carries throughout, and the safe +/// direction to err in. +fn is_mutable_target(target: &Target, enabled: Option<&Vec>) -> bool { + if target.name == crate::exec::RUNTIME_CRATE { + return false; + } + + let kind = target + .kind + .iter() + .any(|kind| matches!(kind.to_string().as_str(), "lib" | "rlib" | "cdylib" | "bin")); + + // Library targets carry no `required-features` — cargo rejects a manifest that gives them + // any — so this is the binary gate and nothing else. + kind && enabled.is_none_or(|on| { + target.required_features.iter().all(|required| { + // A `dependency/feature` requirement is about another package's feature table, which + // this closure does not key by, so it is left as satisfied rather than guessed at. + required.contains('/') || on.iter().any(|feature| feature == required) + }) + }) +} + +/// Names every test target the workspace declares. +/// +/// The name is what cargo reports as `target.name` for the binary it builds, and so is what a +/// `--include-test` or `--exclude-test` pattern is written against. Bench and example targets are +/// listed when they carry `test = true`, because cargo builds and runs those as test binaries too, +/// and a run that cannot name them cannot take them out of the oracle. +/// +/// Names are deduplicated because two workspace members may each have a `tests/integration.rs`, +/// and a pattern naming it means both. That is the honest reading: these patterns select targets, +/// and `--test-package` is what selects by package. +fn test_targets(metadata: &Metadata) -> Vec { + let mut names: Vec = metadata + .workspace_packages() + .iter() + .flat_map(|package| package.targets.iter()) + .filter(|target| target.test) + .map(|target| target.name.clone()) + .collect(); + + names.sort(); + names.dedup(); + names +} + +/// Returns the first `--file` or `--exclude-file` pattern that matches no source file. +/// +/// A pattern that matches nothing is nearly always a mistake — a typo, a stale path after a move, +/// or separators written for the wrong platform. Left alone it produces an empty run that reports +/// no mutants and exits successfully, which reads in CI exactly like a clean bill of health. The +/// same reasoning already makes an unmatched `--mutators` selector an error. +/// +/// `walked` names every mutable source file in the workspace, not only the files this run selects, +/// so a workspace-wide pattern stays valid on a run narrowed with `--package`. +struct FilePatterns<'args> { + include: Vec<(&'args str, Glob)>, + exclude: Vec<(&'args str, Glob)>, +} + +impl<'args> FilePatterns<'args> { + fn new(args: &'args SelectArgs) -> Self { + let compile = |patterns: &'args [String]| patterns.iter().map(|pattern| (pattern.as_str(), Glob::new(pattern))).collect(); + + Self { + include: compile(&args.files), + exclude: compile(&args.exclude_files), + } + } + + fn unmatched(&self, walked: &[Utf8PathBuf]) -> Option<&'args str> { + self.include + .iter() + .chain(&self.exclude) + .find(|(_pattern, compiled)| !walked.iter().any(|path| compiled.matches(path.as_str()))) + .map(|(pattern, _compiled)| *pattern) + } + + fn includes(&self, path: &Utf8Path) -> bool { + let text = path.as_str(); + + if self.exclude.iter().any(|(_pattern, compiled)| compiled.matches(text)) { + return false; + } + + self.include.is_empty() || self.include.iter().any(|(_pattern, compiled)| compiled.matches(text)) + } +} + +/// Returns whether a file passes the include and exclude patterns. +#[cfg(test)] +fn is_included(path: &Utf8Path, args: &SelectArgs) -> bool { + FilePatterns::new(args).includes(path) +} + +/// Lists every `.rs` file under a directory, in a deterministic order. +/// +/// Every failure is reported rather than skipped, which is the opposite of what a walk usually +/// does. This is the sole producer of the candidate file list for the whole population, and a walk +/// error is per *entry*: an unreadable subdirectory yields one error and then simply produces no +/// descendants, so a swallow deletes that entire subtree from the population. Nothing downstream +/// can notice — inclusion filtering, target mutability and the score denominator are all computed +/// over whatever survives — and the run reports a *higher* score with no warning and no non-zero +/// exit, which is the one direction this tool must never fail in. +/// +/// A path that is not UTF-8 is refused only when it names a Rust source file. Such a file would +/// have been mutated and now cannot even be named, while a file of any other kind was never part +/// of the population and its spelling is nobody's business here. +fn walk_rust_files(directory: &Utf8Path) -> Result> { + let mut found: Vec = Vec::new(); + + for entry in WalkDir::new(directory) { + let entry = entry.map_err(|cause| error!("could not list the source files under `{directory}`").caused_by(cause))?; + + if entry.file_type().is_dir() { + continue; + } + + let rust = entry.path().extension() == Some(OsStr::new("rs")); + + match Utf8PathBuf::from_path_buf(entry.into_path()) { + Ok(path) if rust => found.push(path), + Ok(_other) => {} + Err(path) if rust => { + return Err(error!( + "the source file `{}` under `{directory}` has a path that is not UTF-8, so it cannot be mutated", + path.display() + )); + } + Err(_other) => {} + } + } + + found.sort(); + + Ok(found) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + use crate::testing::discover_fixture::counting_mutant; + + /// The name a test gives a file it wants a phase-1 survey worker to panic on. + /// + /// Injected by name rather than by an armed flag so that nothing has to be disarmed and no + /// other test running at the same time can trip it. + const PANIC_PROBE: &str = "gamma_panic_probe.rs"; + + /// Panics if this is the file a test planted to make a survey worker die mid-phase. + pub(super) fn panic_probe(path: &Utf8Path) { + assert!( + !path.as_str().ends_with(PANIC_PROBE), + "a survey worker panicking during phase one, on purpose" + ); + } + + #[test] + fn excludes_beat_includes() { + let args = SelectArgs { + files: vec!["src/**/*.rs".to_owned()], + exclude_files: vec!["generated.rs".to_owned()], + ..SelectArgs::default() + }; + + assert!(is_included(Utf8Path::new("src/lexer.rs"), &args)); + assert!(!is_included(Utf8Path::new("src/generated.rs"), &args)); + } + + #[test] + fn no_include_patterns_means_everything() { + assert!(is_included(Utf8Path::new("anything.rs"), &SelectArgs::default())); + } + + #[test] + fn a_package_reaches_itself() { + // Otherwise every mutant in a leaf crate would be reported as unreachable by its own tests. + let metadata = load_metadata(Utf8Path::new(env!("CARGO_MANIFEST_DIR")), &FeatureArgs::default()).expect("metadata"); + let reach = reachable(&metadata); + + for (package, reachable_from) in &reach { + assert!(reachable_from.contains(package), "{package} does not reach itself"); + } + } + + #[test] + fn a_dependent_reaches_what_it_depends_on() { + let metadata = load_metadata(Utf8Path::new(env!("CARGO_MANIFEST_DIR")), &FeatureArgs::default()).expect("metadata"); + let reach = reachable(&metadata); + + // The binary crate is deliberately thin and defers everything to the library, so it must + // reach it; the reverse must not hold, or the filter would never exclude anything here. + let from_binary = reach.get("cargo-gamma").expect("the binary crate is a workspace member"); + + assert!(from_binary.contains("cargo-gamma-lib"), "{from_binary:?}"); + + let from_library = reach.get("cargo-gamma-lib").expect("the library is a workspace member"); + + assert!(!from_library.contains("cargo-gamma"), "{from_library:?}"); + } + + /// Exhaustively checks [`reachable_ids`] against the same breadth-first search the previous + /// implementation ran per starting node, so the collapse-then-close rewrite is only trusted + /// once it is shown to answer identically to the algorithm it replaced. + fn brute_force_reachable(edges: &[Vec], opaque: &[bool]) -> Vec> { + let node_count = edges.len(); + + (0..node_count) + .map(|start| { + let mut seen = vec![false; node_count]; + let mut queue = std::collections::VecDeque::from([start]); + seen[start] = true; + let mut hits_opaque = false; + + while let Some(current) = queue.pop_front() { + if opaque[current] { + hits_opaque = true; + } + + for &neighbor in &edges[current] { + if !seen[neighbor] { + seen[neighbor] = true; + queue.push_back(neighbor); + } + } + } + + if hits_opaque { + (0..node_count).collect() + } else { + seen.iter() + .enumerate() + .filter(|(_id, reached)| **reached) + .map(|(id, _)| id) + .collect() + } + }) + .collect() + } + + /// A minimal, deterministic pseudo-random generator, so the randomized graph test below needs + /// no external `rand` dependency and reproduces the same graphs on every run. + fn xorshift(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state + } + + #[test] + fn a_chain_gives_every_node_a_strictly_smaller_id_than_its_predecessor() { + // 0 -> 1 -> 2 -> 3, no cycles: every node is its own singleton component, and the finish + // order of a depth-first walk numbers a leaf before the node that points to it. + let edges = vec![vec![1], vec![2], vec![3], vec![]]; + let comp = strongly_connected_components(&edges); + + assert_ne!(comp[0], comp[1]); + assert_ne!(comp[1], comp[2]); + assert_ne!(comp[2], comp[3]); + assert!(comp[3] < comp[2]); + assert!(comp[2] < comp[1]); + assert!(comp[1] < comp[0]); + } + + #[test] + fn a_full_cycle_collapses_into_one_component_that_reaches_itself_and_its_own_dependency() { + // 0 -> 1 -> 2 -> 0 is one strongly connected component; 2 -> 3 means the cycle, as a + // whole, also reaches the dependency any one of its members can walk to. + let edges = vec![vec![1], vec![2], vec![0, 3], vec![]]; + let opaque = [false; 4]; + let reach = reachable_ids(&edges, &opaque); + + for member in [0, 1, 2] { + assert_eq!(reach[member], [0, 1, 2, 3].into_iter().collect::>(), "member {member}"); + } + + assert_eq!(reach[3], std::iter::once(3).collect::>()); + } + + #[test] + fn a_dense_graph_matches_the_brute_force_oracle() { + // A handful of criss-crossing edges and no cycle, curated rather than random so a failure + // here is easy to reason about by hand. + let edges = vec![ + vec![1, 2], + vec![3], + vec![3, 4], + vec![5], + vec![5], + std::iter::once(6).collect(), + vec![], + ]; + let opaque = [false; 7]; + + assert_eq!(reachable_ids(&edges, &opaque), brute_force_reachable(&edges, &opaque)); + } + + #[test] + fn an_opaque_node_taints_itself_and_everything_that_can_reach_it() { + // 0 -> 1 -> 2, and 2 has an unresolvable path dependency: 2 must reach everything, and so + // must 0 and 1, since both can reach 2. + let edges = vec![vec![1], vec![2], vec![]]; + let opaque = [false, false, true]; + let reach = reachable_ids(&edges, &opaque); + + for member in [0, 1, 2] { + assert_eq!(reach[member], [0, 1, 2].into_iter().collect::>(), "member {member}"); + } + } + + #[test] + fn an_opaque_member_of_a_cycle_taints_the_whole_component() { + // 0 <-> 1 form one component; only 1 is directly opaque, but the collapse means neither + // member can be told apart from the other, so both fall back to reaching every workspace + // member, exactly as a single opaque node does on its own. Node 2 is untouched: nothing in + // its own graph, nor anything that can reach it, is opaque. + let edges = vec![vec![1], vec![0], vec![]]; + let opaque = [false, true, false]; + let reach = reachable_ids(&edges, &opaque); + + assert_eq!(reach[0], [0, 1, 2].into_iter().collect::>()); + assert_eq!(reach[1], [0, 1, 2].into_iter().collect::>()); + assert_eq!(reach[2], std::iter::once(2).collect::>()); + } + + #[test] + fn randomized_graphs_with_cycles_and_opaque_nodes_match_the_brute_force_oracle() { + let mut state: u64 = 0x9E37_79B9_7F4A_7C15; + + for trial in 0..200 { + let node_count = 1 + usize::try_from(xorshift(&mut state) % 12).unwrap_or(0); + let mut edges: Vec> = vec![Vec::new(); node_count]; + + for (from, out_edges) in edges.iter_mut().enumerate() { + for to in 0..node_count { + if from != to && xorshift(&mut state).is_multiple_of(3) { + out_edges.push(to); + } + } + } + + let opaque: Vec = (0..node_count).map(|_| xorshift(&mut state).is_multiple_of(10)).collect(); + + assert_eq!( + reachable_ids(&edges, &opaque), + brute_force_reachable(&edges, &opaque), + "trial {trial} with {node_count} nodes: {edges:?}, opaque {opaque:?}" + ); + } + } + + #[test] + fn external_path_dependencies_and_local_build_scripts_are_cache_inputs() { + let directory = crate::testing::workdir("survey-external-inputs-"); + let container = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("UTF-8 path"); + let root = container.join("workspace"); + let dependency = container.join("dependency"); + + fs::create_dir_all(root.join("src")).expect("workspace source"); + fs::create_dir_all(dependency.join("src")).expect("dependency source"); + fs::write( + root.join("Cargo.toml"), + "[workspace]\n\n[package]\nname = \"workspace\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\ndependency = { path = \"../dependency\" }\n", + ) + .expect("workspace manifest"); + fs::write(root.join("src/lib.rs"), "pub fn workspace() {}\n").expect("workspace source"); + fs::write( + dependency.join("Cargo.toml"), + "[package]\nname = \"dependency\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ) + .expect("dependency manifest"); + fs::write(dependency.join("src/lib.rs"), "pub fn dependency() {}\n").expect("dependency source"); + fs::write(dependency.join("build.rs"), "fn main() {}\n").expect("dependency build script"); + + let inputs = external_path_inputs(&root, &FeatureArgs::default(), &root).expect("metadata"); + + assert_eq!(inputs.roots, vec![crate::paths::physical(&dependency).expect("dependency path")]); + assert!( + inputs.has_build_scripts, + "a local build script can read external paths Cargo metadata does not enumerate" + ); + } + + #[test] + fn a_registry_build_script_makes_the_snapshot_uncacheable() { + let root = Utf8Path::new(env!("CARGO_MANIFEST_DIR")); + let mut command = MetadataCommand::new(); + let _builder = command.current_dir(root); + let metadata = command.exec().expect("workspace metadata"); + let blake3 = metadata + .packages + .iter() + .find(|package| package.name == "blake3") + .expect("the workspace resolves blake3"); + + assert!(blake3.source.is_some(), "blake3 must be a registry package"); + assert!( + blake3.targets.iter().any(|target| target.kind.contains(&TargetKind::CustomBuild)), + "blake3 must expose its build script in metadata" + ); + + let workspace = Utf8Path::new(metadata.workspace_root.as_str()); + let inputs = external_path_inputs(workspace, &FeatureArgs::default(), workspace).expect("metadata"); + + assert!( + inputs.has_build_scripts, + "a registry build script can read uncaptured inputs, so its snapshot must be incomplete" + ); + } + + #[test] + fn discovery_refuses_inline_and_file_cargo_configuration_before_metadata() { + let (_directory, root) = workspace(); + let args = SelectArgs { + dir: root, + ..SelectArgs::default() + }; + + for extra in [ + vec!["--config".to_owned(), "build.target = \"wasm32-wasip1\"".to_owned()], + vec!["--config=outside.toml".to_owned()], + ] { + let failure = Survey::for_build( + &args, + None, + &CargoOptions { + extra, + ..CargoOptions::default() + }, + ) + .expect_err("unmodelled Cargo configuration must stop discovery"); + + assert!(failure.is_usage(), "{failure}"); + assert!(failure.to_string().contains("--config"), "{failure}"); + } + } + + #[test] + fn discovery_propagates_a_configuration_failure() { + let (_directory, root) = workspace(); + let config = root.join("gamma.toml"); + + fs::create_dir_all(config.parent().expect("configuration parent")).expect("configuration parent"); + fs::write(&config, "cargo-args = [\n").expect("malformed configuration"); + let failure = Survey::new( + &SelectArgs { + dir: root, + ..SelectArgs::default() + }, + None, + ) + .expect_err("configuration failure must not become default discovery settings"); + + assert!(failure.is_usage(), "{failure}"); + assert!(failure.to_string().contains("gamma.toml"), "{failure}"); + } + + #[test] + fn a_narrow_survey_keeps_unselected_dependency_roots() { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write(&root, "Cargo.toml", "[workspace]\nmembers = [\"a\", \"b\"]\nresolver = \"3\"\n"); + write( + &root, + "a/Cargo.toml", + "[package]\nname = \"a\"\nversion = \"0.1.0\"\nedition = \"2024\"\n[dependencies]\nb = { path = \"../b\" }\n", + ); + write(&root, "a/src/lib.rs", "pub fn a() -> bool { b::b() }\n"); + write( + &root, + "b/Cargo.toml", + "[package]\nname = \"b\"\nversion = \"0.1.0\"\nedition = \"2024\"\n", + ); + write(&root, "b/src/lib.rs", "pub fn b() -> bool { true }\n"); + + let survey = Survey::for_build( + &SelectArgs { + dir: root, + packages: vec!["a".to_owned()], + ..SelectArgs::default() + }, + None, + &CargoOptions::default(), + ) + .expect("the narrowed workspace should survey"); + + assert_eq!(survey.selected, ["a"]); + assert_eq!(survey.specs.get("a").map(|(path, _version)| path.as_str()), Some("a")); + assert_eq!(survey.specs.get("b").map(|(path, _version)| path.as_str()), Some("b")); + } + + #[test] + fn a_package_that_depends_through_a_non_member_reaches_the_whole_workspace() { + // Regression, issue-006. `cargo metadata --no-deps` does not list packages outside the + // workspace, so an `app -> facade -> core` chain through a path dependency that is not a + // member is invisible. Concluding "app does not reach core" from a graph with a hole in it + // means never running app's tests against a mutant in core, and scoring that mutant + // uncovered when a test does in fact cover it. Reaching everything is the fail-open answer. + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write( + &root, + "Cargo.toml", + "[workspace]\nmembers = [\"core\", \"app\"]\nexclude = [\"facade\"]\nresolver = \"3\"\n", + ); + write( + &root, + "core/Cargo.toml", + "[package]\nname = \"core\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(&root, "core/src/lib.rs", "pub fn add(a: i32, b: i32) -> i32 { a + b }\n"); + + // Outside the workspace on purpose: this is the package the metadata cannot see through. + write( + &root, + "facade/Cargo.toml", + "[package]\nname = \"facade\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [dependencies]\ncore = { path = \"../core\" }\n\n[workspace]\n", + ); + write(&root, "facade/src/lib.rs", "pub use core::add;\n"); + + write( + &root, + "app/Cargo.toml", + "[package]\nname = \"app\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [dependencies]\nfacade = { path = \"../facade\" }\n", + ); + write(&root, "app/src/lib.rs", "pub fn go() -> i32 { facade::add(1, 2) }\n"); + + let metadata = load_metadata(&root, &FeatureArgs::default()).expect("metadata"); + let reach = reachable(&metadata); + let from_app = reach.get("app").expect("app is a workspace member"); + + assert!(from_app.contains("core"), "{from_app:?}"); + + // The package with no such dependency keeps its exact reach: fail-open must not become + // "everything reaches everything", which would run every binary for every mutant. + let from_core = reach.get("core").expect("core is a workspace member"); + + assert!(!from_core.contains("app"), "{from_core:?}"); + } + + /// `stages()` orders packages by the size of their reach sets, which is a topological order + /// only because a dependency's reach set is a subset of its dependent's. An opaque package + /// reaching every member breaks that on its own: a dependent that cannot reach some third + /// member has the *smaller* set and sorts ahead of the package it depends on. So the union + /// travels to the dependents too, and the sort keeps its premise. + #[test] + fn a_dependent_of_an_opaque_package_is_never_ordered_before_it() { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write( + &root, + "Cargo.toml", + "[workspace]\nmembers = [\"opaque\", \"dependent\", \"unrelated\"]\nexclude = [\"outside\"]\nresolver = \"3\"\n", + ); + + // Outside the workspace, so the metadata cannot see through it and `opaque` reaches + // everything rather than risk missing a path back into the workspace. + write( + &root, + "outside/Cargo.toml", + "[package]\nname = \"outside\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n", + ); + write(&root, "outside/src/lib.rs", "pub fn f() {}\n"); + + write( + &root, + "opaque/Cargo.toml", + "[package]\nname = \"opaque\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [dependencies]\noutside = { path = \"../outside\" }\n", + ); + write(&root, "opaque/src/lib.rs", "pub fn f() {}\n"); + + write( + &root, + "dependent/Cargo.toml", + "[package]\nname = \"dependent\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [dependencies]\nopaque = { path = \"../opaque\" }\n", + ); + write(&root, "dependent/src/lib.rs", "pub fn f() {}\n"); + + write( + &root, + "unrelated/Cargo.toml", + "[package]\nname = \"unrelated\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(&root, "unrelated/src/lib.rs", "pub fn f() {}\n"); + + let metadata = load_metadata(&root, &FeatureArgs::default()).expect("metadata"); + let reach = reachable(&metadata); + let of = |name: &str| reach.get(name).cloned().unwrap_or_default(); + + assert!(of("dependent").is_superset(&of("opaque")), "{reach:?}"); + + let packages = vec!["opaque".to_owned(), "dependent".to_owned(), "unrelated".to_owned()]; + let stages = crate::discover::stages(&packages, &reach); + let place = |name: &str| { + stages + .iter() + .position(|stage| stage.iter().any(|member| member == name)) + .expect("every package is placed somewhere") + }; + + assert!(place("opaque") <= place("dependent"), "{stages:?}"); + } + + /// Cargo acts on the package that owns the directory it was invoked from, and mutation testing + /// has far more reason to follow that rule than `cargo build` does: an unasked-for member costs + /// a full build and test suite per mutant. + #[test] + fn a_run_from_inside_a_member_selects_that_member_alone() { + let home = TempDir::new().expect("could not create a temporary directory"); + let root = Utf8Path::from_path(home.path()).expect("the temporary path is not UTF-8"); + + write( + root, + "Cargo.toml", + "[workspace]\nmembers = [\"alpha\", \"beta\"]\nresolver = \"2\"\n", + ); + write( + root, + "alpha/Cargo.toml", + "[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(root, "alpha/src/lib.rs", "pub fn a() {}\n"); + write( + root, + "beta/Cargo.toml", + "[package]\nname = \"beta\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(root, "beta/src/lib.rs", "pub fn b() {}\n"); + + let metadata = load_metadata(root, &FeatureArgs::default()).expect("metadata"); + + let inside = SelectArgs { + dir: root.join("alpha"), + ..SelectArgs::default() + }; + + assert_eq!(selected_packages(&metadata, &inside), once("alpha".to_owned()).collect()); + + // A subdirectory of a member is still that member, which is where anyone actually stands. + let deeper = SelectArgs { + dir: root.join("alpha/src"), + ..SelectArgs::default() + }; + + assert_eq!(selected_packages(&metadata, &deeper), once("alpha".to_owned()).collect()); + } + + /// `--workspace` is what widens the run, and `--package` still names the selection outright. + #[test] + fn the_whole_workspace_is_selected_only_when_asked_for() { + let home = TempDir::new().expect("could not create a temporary directory"); + let root = Utf8Path::from_path(home.path()).expect("the temporary path is not UTF-8"); + + write( + root, + "Cargo.toml", + "[workspace]\nmembers = [\"alpha\", \"beta\"]\nresolver = \"2\"\n", + ); + write( + root, + "alpha/Cargo.toml", + "[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(root, "alpha/src/lib.rs", "pub fn a() {}\n"); + write( + root, + "beta/Cargo.toml", + "[package]\nname = \"beta\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(root, "beta/src/lib.rs", "pub fn b() {}\n"); + + let metadata = load_metadata(root, &FeatureArgs::default()).expect("metadata"); + let both: HashSet = ["alpha".to_owned(), "beta".to_owned()].into_iter().collect(); + + let whole = SelectArgs { + dir: root.join("alpha"), + workspace: true, + ..SelectArgs::default() + }; + + assert_eq!(selected_packages(&metadata, &whole), both); + + let named = SelectArgs { + dir: root.join("alpha"), + packages: vec!["beta".to_owned()], + ..SelectArgs::default() + }; + + assert_eq!(selected_packages(&metadata, &named), once("beta".to_owned()).collect()); + + // At the root of a virtual manifest no package owns the directory, and with no + // `default-members` declared cargo means all of them. + let outside = SelectArgs { + dir: root.to_owned(), + ..SelectArgs::default() + }; + + assert_eq!(selected_packages(&metadata, &outside), both); + } + + /// At the workspace root the default members decide, exactly as they do for a bare `cargo test`. + #[test] + fn the_workspace_root_honours_the_default_members() { + let home = TempDir::new().expect("could not create a temporary directory"); + let root = Utf8Path::from_path(home.path()).expect("the temporary path is not UTF-8"); + + write( + root, + "Cargo.toml", + "[workspace]\nmembers = [\"alpha\", \"beta\"]\ndefault-members = [\"alpha\"]\nresolver = \"2\"\n", + ); + write( + root, + "alpha/Cargo.toml", + "[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(root, "alpha/src/lib.rs", "pub fn a() {}\n"); + write( + root, + "beta/Cargo.toml", + "[package]\nname = \"beta\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(root, "beta/src/lib.rs", "pub fn b() {}\n"); + + let metadata = load_metadata(root, &FeatureArgs::default()).expect("metadata"); + + let at_root = SelectArgs { + dir: root.to_owned(), + ..SelectArgs::default() + }; + + assert_eq!(selected_packages(&metadata, &at_root), once("alpha".to_owned()).collect()); + } + + /// The sibling of the test above, with a root that is itself a package. Cargo's own + /// `default_members` answers `["alpha"]` here too, but an `owning_package` lookup consulted + /// first finds the root package and pre-empts it — so a bare `cargo gamma` mutates only the + /// root while a bare `cargo test` runs `alpha`. + #[test] + fn a_root_package_workspace_honours_the_default_members() { + let home = TempDir::new().expect("could not create a temporary directory"); + let root = Utf8Path::from_path(home.path()).expect("the temporary path is not UTF-8"); + + write( + root, + "Cargo.toml", + "[package]\nname = \"host\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [workspace]\nmembers = [\"alpha\"]\ndefault-members = [\"alpha\"]\nresolver = \"2\"\n", + ); + write(root, "src/lib.rs", "pub fn h() {}\n"); + write( + root, + "alpha/Cargo.toml", + "[package]\nname = \"alpha\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ); + write(root, "alpha/src/lib.rs", "pub fn a() {}\n"); + + let metadata = load_metadata(root, &FeatureArgs::default()).expect("metadata"); + + let at_root = SelectArgs { + dir: root.to_owned(), + ..SelectArgs::default() + }; + + assert_eq!(selected_packages(&metadata, &at_root), once("alpha".to_owned()).collect()); + } + + /// Cargo accepts a member outside the workspace root. `TargetFile::path` is documented as + /// relative, and the filesystem layer's containment rests on that: `Utf8Path::join` with an + /// absolute argument *replaces* the base, so an absolute path here sends every instrumented + /// write into the user's real source tree instead of the scratch copy. + #[test] + fn a_member_outside_the_workspace_root_is_refused_rather_than_given_an_absolute_path() { + let home = TempDir::new().expect("could not create a temporary directory"); + let base = Utf8Path::from_path(home.path()).expect("the temporary path is not UTF-8"); + let root = base.join("w"); + + write(&root, "Cargo.toml", "[workspace]\nmembers = [\"../sibling\"]\nresolver = \"2\"\n"); + write( + base, + "sibling/Cargo.toml", + "[package]\nname = \"sibling\"\nversion = \"0.0.0\"\nedition = \"2024\"\nworkspace = \"../w\"\n", + ); + write(base, "sibling/src/lib.rs", "pub fn s() -> i32 { 1 + 2 }\n"); + + let args = SelectArgs { + dir: root, + ..SelectArgs::default() + }; + + let failure = Survey::new(&args, None).expect_err("a member outside the root must be refused"); + + assert!(failure.to_string().contains("outside the workspace root"), "{failure}"); + } + + #[test] + fn a_registry_dependency_does_not_make_a_package_opaque() { + // Regression, issue-006. Only a *path* dependency can lead back into the workspace. Marking + // a package opaque for an ordinary crates.io dependency would make almost every real + // workspace reach everything, undoing the scoping this whole graph exists for. + let metadata = load_metadata(Utf8Path::new(env!("CARGO_MANIFEST_DIR")), &FeatureArgs::default()).expect("metadata"); + let reach = reachable(&metadata); + let from_library = reach.get("cargo-gamma-lib").expect("the library is a workspace member"); + + assert!( + !from_library.contains("cargo-gamma"), + "this crate has many registry dependencies: {from_library:?}" + ); + } + + /// A mutant whose package never had a file walked for it — because the mutant's own package + /// name is wrong, or the two lists have simply drifted apart — must not silently inflate + /// another package's counts. If it landed anywhere, the per-package report would blame the + /// wrong crate for a mutant it never produced. + #[test] + fn a_mutant_from_a_package_no_file_was_walked_for_does_not_inflate_anothers_count() { + let files = [TargetFile { + path: Utf8PathBuf::from("core/src/lib.rs"), + absolute: Utf8PathBuf::from("/tree/core/src/lib.rs"), + package: "core".to_owned(), + }]; + let mutants = [counting_mutant("core"), counting_mutant("ghost")]; + + let mut lines = Vec::new(); + report_by_package(&files, &mutants, &mut |line: &str| lines.push(line.to_owned())); + + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("core, 1 mutant in 1"), "{lines:?}"); + } + + /// + /// `core` is a plain library, `app` is a binary that depends on it and also carries an example + /// and an integration test — which is what makes it useful here, since those are exactly the + /// target kinds the survey has to walk past. + fn workspace() -> (TempDir, Utf8PathBuf) { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write( + &root, + "Cargo.toml", + "[workspace]\nmembers = [\"core\", \"app\"]\nresolver = \"3\"\n", + ); + + write( + &root, + "core/Cargo.toml", + "[package]\nname = \"core\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[features]\nextra = []\n", + ); + write(&root, "core/src/lib.rs", "pub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n"); + write(&root, "core/src/generated.rs", "pub fn scale(x: i32) -> i32 {\n x * 2\n}\n"); + + write( + &root, + "app/Cargo.toml", + "[package]\nname = \"app\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\ncore = { path = \"../core\" }\n", + ); + write(&root, "app/src/main.rs", "fn main() {\n let _ = 1 + 1;\n}\n"); + write(&root, "app/examples/demo.rs", "fn main() {\n let _ = 2 + 2;\n}\n"); + write(&root, "app/tests/it.rs", "#[test]\nfn works() {\n assert_eq!(1 + 1, 2);\n}\n"); + + (directory, root) + } + + /// Scanning package by package must reach exactly the population one whole-workspace scan does. + /// + /// The per-package selection is an index rather than a filter, because a filter walks every + /// file in the workspace once per package. An index can disagree with the list it indexes in + /// two ways a filter cannot — by missing a file, and by ordering them differently — and the + /// second is the silent one, because path order is what makes the population deterministic. + #[test] + fn scanning_a_package_at_a_time_reaches_the_same_population_as_scanning_the_workspace() { + let (_directory, root) = wide_workspace(); + let survey = survey(&root, SelectArgs::default()); + let selection = Selection::parse("all").expect("every mutator resolves"); + + let mut ordinals = 0; + let whole = survey.scan(None, &selection, &mut ordinals).expect("the fixture must scan"); + + let mut ordinals = 0; + let mut piecemeal: Vec = Vec::new(); + + for package in survey.packages() { + let scanned = survey + .scan(Some(&package), &selection, &mut ordinals) + .expect("the fixture must scan"); + + piecemeal.extend(scanned.mutants.iter().map(|mutant| mutant.id.clone())); + } + + let mut expected: Vec = whole.mutants.iter().map(|mutant| mutant.id.clone()).collect(); + let mut found = piecemeal.clone(); + + expected.sort(); + found.sort(); + + assert_eq!(found, expected, "package-by-package scanning reached a different population"); + + // A package the workspace does not have is empty rather than the whole workspace, which is + // what a lookup gets wrong if it falls back to the unfiltered list. + let mut ordinals = 0; + let absent = survey + .scan(Some("nothing-by-this-name"), &selection, &mut ordinals) + .expect("scanning nothing is not an error"); + + assert!(absent.mutants.is_empty(), "{:?}", absent.mutants); + } + + /// A workspace whose shapes exercise the deduplication and graph-walking paths. + /// + /// `core` has both a library and a binary rooted in the same directory, so its files are + /// walked twice; `mid` and `app` both depend on `core`, so the reachability walk meets it + /// twice; and `core` reaches a module only through `#[cfg(test)]`. + fn wide_workspace() -> (TempDir, Utf8PathBuf) { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write( + &root, + "Cargo.toml", + "[workspace]\nmembers = [\"core\", \"mid\", \"app\"]\nresolver = \"3\"\n", + ); + + write( + &root, + "core/Cargo.toml", + "[package]\nname = \"core\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[[bin]]\nname = \"core-cli\"\npath = \"src/main.rs\"\n", + ); + write( + &root, + "core/src/lib.rs", + "#[cfg(test)]\nmod helpers;\n\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ); + write(&root, "core/src/main.rs", "fn main() {\n let _ = 1 + 1;\n}\n"); + write(&root, "core/src/helpers.rs", "pub fn double(x: i32) -> i32 {\n x * 2\n}\n"); + + write( + &root, + "mid/Cargo.toml", + "[package]\nname = \"mid\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\ncore = { path = \"../core\" }\n", + ); + write(&root, "mid/src/lib.rs", "pub fn triple(x: i32) -> i32 {\n x * 3\n}\n"); + + write( + &root, + "app/Cargo.toml", + "[package]\nname = \"app\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\ncore = { path = \"../core\" }\nmid = { path = \"../mid\" }\n", + ); + write(&root, "app/src/main.rs", "fn main() {\n let _ = 2 + 2;\n}\n"); + + (directory, root) + } + + /// A workspace wide enough that the scan's parallelism and its collision handling both matter. + /// + /// The filler files are byte-identical on purpose. Identical text in different files is what + /// exercises the duplicate screen, whose winner depends on visit order, and thirty-two of them + /// is enough that the workers finish in a different order from one run to the next. + fn repeatable_workspace() -> (TempDir, Utf8PathBuf) { + let (directory, root) = wide_workspace(); + + for at in 0..32 { + write( + &root, + &format!("core/src/filler{at}.rs"), + "pub fn value(x: i32) -> i32 {\n x + 1\n}\n", + ); + } + + // One suppressed site, so that the withheld set is part of what is being compared rather + // than being uniformly empty. + write( + &root, + "mid/src/quiet.rs", + "#[gamma::skip(reason = \"the population has to be stable including what was withheld\")]\npub fn quiet(x: i32) -> i32 {\n x - 1\n}\n", + ); + + (directory, root) + } + + /// Everything about a mutant that two runs over an unchanged tree have to agree on. + fn fingerprint(mutant: &Mutant) -> String { + format!( + "{}|{}|{}|{}|{:?}|{}|{}|{}|{}|{:?}|{:?}|{:?}", + mutant.ordinal, + mutant.id, + mutant.file, + mutant.package, + mutant.span, + mutant.line, + mutant.column, + mutant.mutator, + mutant.item_path, + mutant.shape, + mutant.outcome, + mutant.replacement, + ) + } + + /// Two scans of an unchanged tree produce the same population, in the same order. + /// + /// Ordering is stabilized within a file and identifiers are stable for identical input, but + /// neither of those covers the two places order is decided across files: the per-worker partial + /// results are merged in completion order, and the duplicate screen resolves a collision in + /// favour of whichever site it visited first. Both are load-bearing — ordinals are assigned + /// from the merged order, and `--iterate`, sharding and every run-to-run comparison the tool + /// invites are built on the population being the same population. + /// + /// The whole plan is compared, not just the identifiers: an ordinal, a span or a replacement + /// that moved would leave the identifier set intact and still make two reports incomparable. + /// + /// Three scans rather than two, because a merge that depends on completion order does not + /// necessarily disagree on the first attempt. + #[test] + fn scanning_an_unchanged_tree_twice_produces_the_same_population() { + let (_directory, root) = repeatable_workspace(); + let selection = Selection::parse("all").expect("every mutator resolves"); + + let mut populations = Vec::new(); + let mut counts = Vec::new(); + + for _ in 0..3 { + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + let scanned = survey.scan(None, &selection, &mut ordinals).expect("the fixture must scan"); + let plan = survey.into_plan(scanned); + + counts.push(( + plan.files.iter().map(|file| file.path.to_string()).collect::>(), + plan.suppressed, + plan.sharded_out, + plan.settled_out, + )); + populations.push(plan.mutants.iter().map(fingerprint).collect::>()); + } + + assert!( + !populations[0].is_empty(), + "the fixture produced no mutants, so this proves nothing" + ); + assert!(counts[0].1 > 0, "the fixture suppressed nothing, so the withheld set is untested"); + + assert_eq!(populations[1], populations[0], "the second scan found a different population"); + assert_eq!(populations[2], populations[0], "the third scan found a different population"); + assert_eq!(counts[1], counts[0], "the second scan disagreed about the files or the counts"); + assert_eq!(counts[2], counts[0], "the third scan disagreed about the files or the counts"); + } + + /// A workspace wide enough to keep several survey workers busy, with one file that kills the + /// worker unlucky enough to claim it. + fn probe_workspace() -> (TempDir, Utf8PathBuf) { + let (directory, root) = wide_workspace(); + + for at in 0..64 { + write( + &root, + &format!("core/src/filler{at}.rs"), + "pub fn value(x: i32) -> i32 {\n x + 1\n}\n", + ); + } + + write(&root, &format!("core/src/{PANIC_PROBE}"), "pub fn nothing() {}\n"); + + (directory, root) + } + + /// A worker that dies while parsing still has two barrier waits ahead of it, and every other + /// worker is blocked on them. If it unwinds straight past, they wait forever, `thread::scope` + /// waits on them, and the panic is never re-raised — the run hangs with no diagnosis. + /// + /// Run under the shared watchdog, because the failure this guards against is a hang and an + /// unbounded test would take the whole suite down with it rather than reporting. + #[test] + fn a_worker_that_panics_while_parsing_does_not_wedge_the_others_at_the_barrier() { + let (directory, root) = probe_workspace(); + + let panicked = crate::testing::within(crate::testing::WATCHDOG, "the scan", move || { + catch_unwind(AssertUnwindSafe(|| { + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + + survey + .scan(None, &Selection::parse("@all").expect("selection"), &mut ordinals) + .map(|scanned| scanned.mutants.len()) + })) + .is_err() + }); + + assert!(panicked, "the worker's panic must reach the caller, not be swallowed"); + + // Kept alive until here rather than dropped at the top: the fixture tree has to outlive the + // scan that reads it, and the watchdog's body owns everything it touches. + drop(directory); + } + + fn write(root: &Utf8Path, relative: &str, text: &str) { + let path = root.join(relative); + + fs::create_dir_all(path.parent().expect("every fixture path has a parent").as_std_path()) + .expect("could not create the fixture directory"); + fs::write(path.as_std_path(), text).expect("could not write the fixture file"); + } + + /// The walk is the sole producer of the candidate file list, and a walk error is per entry: a + /// directory that cannot be read yields one error and then no descendants at all. Swallowing it + /// takes that whole subtree out of the population and out of the denominator, and reports a + /// better score for it. + #[test] + fn a_directory_that_cannot_be_walked_is_reported_rather_than_passed_off_as_empty() { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + let missing = root.join("vanished"); + let error = walk_rust_files(&missing).expect_err("a directory that is not there cannot have been walked"); + + assert!(error.to_string().contains("vanished"), "{error}"); + + // The same walk over a directory that is really there still lists what it holds. + write(&root, "here/one.rs", "pub fn f() {}\n"); + write(&root, "here/notes.txt", "not a source file\n"); + + let listed = walk_rust_files(&root.join("here")).expect("a readable directory walks"); + + assert_eq!(listed, vec![root.join("here/one.rs")]); + } + + /// A source file whose path cannot be spelled as UTF-8 is one this run would have mutated and + /// now cannot name, so it is refused rather than quietly left out of the population. A file of + /// any other kind was never in the population and its spelling decides nothing. + #[cfg(unix)] + #[test] + fn a_source_file_with_a_path_that_is_not_utf8_is_reported_rather_than_dropped() { + use std::os::unix::ffi::OsStrExt; + + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write(&root, "src/lib.rs", "pub fn f() {}\n"); + + let odd = directory.path().join("src").join(OsStr::from_bytes(b"lat\xffin.rs")); + + fs::write(&odd, "pub fn g() {}\n").expect("the fixture file is written"); + + let error = walk_rust_files(&root.join("src")).expect_err("a source file nobody can name is not skipped"); + + assert!(error.to_string().contains("not UTF-8"), "{error}"); + + // Renamed to something that is not a source file, it is out of the population anyway. + fs::rename(&odd, directory.path().join("src").join(OsStr::from_bytes(b"lat\xffin.txt"))).expect("renamed"); + + let listed = walk_rust_files(&root.join("src")).expect("a file that is not Rust says nothing about the walk"); + + assert_eq!(listed, vec![root.join("src/lib.rs")]); + } + + fn survey(root: &Utf8Path, args: SelectArgs) -> Survey { + Survey::new( + &SelectArgs { + dir: root.to_owned(), + ..args + }, + None, + ) + .expect("the fixture workspace must survey") + } + + /// A malformed suppression directive is a mistake in the user's own source, and it has to stop + /// the scan with the same named error `suppress::directives` itself would report, rather than + /// being swallowed by the parallel scan and the mutant simply going unsuppressed with nothing + /// to explain why: the directive is why the file was written, and a typo in it should not + /// silently disable itself. + #[test] + fn an_unknown_suppression_directive_fails_the_scan_rather_than_being_ignored() { + let (_directory, root) = workspace(); + + write( + &root, + "core/src/lib.rs", + "#[gamma::note]\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ); + + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + + let error = survey + .scan(None, &Selection::parse("all").expect("every mutator resolves"), &mut ordinals) + .expect_err("the directive names no known intent"); + + assert!(error.to_string().contains("unknown directive"), "{error}"); + } + + /// A file this tool cannot walk is left out and named, rather than taking the workspace with it. + /// + /// The nesting guard exists because a recursive descent over deep enough source runs out of + /// stack, and `rustc` builds files far deeper than this tool can walk — a generated parser + /// table or a macro-expanded literal is the ordinary case. So this is the one parse failure + /// that says nothing about whether the workspace is sound, and refusing the run over it makes + /// a valid workspace unmeasurable. The file is skipped, and the skip reaches the plan so that + /// the score is read knowing which code is not in it: a file dropped in silence is + /// indistinguishable from a file with nothing worth mutating. + #[test] + fn a_file_too_deep_to_walk_is_left_out_by_name_rather_than_stopping_the_scan() { + let (_directory, root) = workspace(); + + // Deliberately a number rather than the limit itself: what is under test is that a file + // this tool refuses is stepped over, not where the refusal starts, and a test that reads + // the constant would still pass if the constant were raised past anything real. + let depth = 512; + let deep = format!("pub fn deep() -> i32 {{\n {}1{}\n}}\n", "(".repeat(depth), ")".repeat(depth)); + + write(&root, "core/src/deep.rs", &deep); + write( + &root, + "core/src/lib.rs", + "mod deep;\n\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ); + + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + + let scanned = survey + .scan(None, &Selection::parse("all").expect("every mutator resolves"), &mut ordinals) + .expect("a file this tool cannot walk does not make the workspace unmeasurable"); + + assert_eq!(scanned.skipped.len(), 1, "{:?}", scanned.skipped); + + let note = scanned.skipped.first().expect("one file was skipped"); + + assert!(note.contains("deep.rs"), "the skip does not name the file: {note}"); + assert!(note.contains("nests deeper"), "the skip does not say why: {note}"); + + // The rest of the package is still measured, which is the whole point of skipping rather + // than refusing. + assert!( + scanned.mutants.iter().any(|mutant| mutant.file.as_str().ends_with("lib.rs")), + "{:?}", + scanned.mutants.iter().map(|mutant| mutant.file.as_str()).collect::>() + ); + } + + /// A stated value the tool cannot honour stops the scan for the same reason a misspelled + /// suppression does: the author wrote it believing it was working. The compiler catches this + /// too, but not until a mutant is built, and a run that quietly collected the site's guessed + /// values in the meantime would report a mutation score computed from mutants the author had + /// already said were the wrong question. + #[test] + fn a_stated_value_that_cannot_be_honoured_fails_the_scan_rather_than_being_ignored() { + let (_directory, root) = workspace(); + + write( + &root, + "core/src/lib.rs", + "#[gamma::value(1, 2)]\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ); + + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + + let error = survey + .scan(None, &Selection::parse("all").expect("every mutator resolves"), &mut ordinals) + .expect_err("two expressions are not one value"); + + assert!(error.to_string().contains("states one value"), "{error}"); + assert!(error.to_string().contains("core/src/lib.rs:1"), "{error}"); + } + + /// A stated value that is well formed is collected rather than reported. + /// + /// The other half of the check: a rule that fires on correct source is worse than no rule, + /// because the only way out of it is to stop using the feature. + #[test] + fn a_stated_value_reaches_the_population_as_a_mutant_of_its_own() { + let (_directory, root) = workspace(); + + write( + &root, + "core/src/lib.rs", + "#[gamma::value(a - b)]\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ); + + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + let scanned = survey + .scan(None, &Selection::parse("fn_value").expect("every mutator resolves"), &mut ordinals) + .expect("a well-formed stated value is not an error"); + + let stated: Vec<&Mutant> = scanned + .mutants + .iter() + .filter(|mutant| &*mutant.mutator == "fn_value.stated") + .collect(); + + assert_eq!(stated.len(), 1, "{:?}", scanned.mutants); + assert_eq!(stated[0].replacement, "a - b"); + } + + /// A file walked once per target still appears once, and test-only modules never appear. + #[test] + fn files_reached_twice_are_listed_once_and_test_only_modules_are_dropped() { + let (_directory, root) = wide_workspace(); + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + let scanned = survey + .scan(None, &Selection::parse("@all").expect("selection"), &mut ordinals) + .expect("scan"); + let plan = survey.into_plan(scanned); + + let listed: Vec<&Utf8PathBuf> = plan.files.iter().map(|file| &file.path).collect(); + let core_lib = Utf8PathBuf::from("core/src/lib.rs"); + + assert_eq!(listed.iter().filter(|path| ****path == core_lib).count(), 1, "{listed:?}"); + + // `helpers.rs` is only ever reached through `#[cfg(test)] mod helpers;`, so it is test + // code however ordinary it looks, and none of its mutants belong in the population. + assert!( + !plan.mutants.iter().any(|mutant| mutant.file.as_str().ends_with("helpers.rs")), + "{:?}", + plan.mutants.iter().map(|mutant| &mutant.file).collect::>() + ); + } + + /// A package two others depend on is visited once, however many paths lead to it. + #[test] + fn a_package_reached_by_two_paths_is_walked_once() { + let (_directory, root) = wide_workspace(); + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + let scanned = survey + .scan(None, &Selection::parse("@all").expect("selection"), &mut ordinals) + .expect("scan"); + let plan = survey.into_plan(scanned); + + // `app` reaches `core` directly and again through `mid`; the graph walk has to terminate + // and report it once rather than loop or double-count. + let reached = plan.reach.get("app").expect("app should have a reachable set"); + + assert!(reached.contains("core"), "{reached:?}"); + assert!(reached.contains("mid"), "{reached:?}"); + } + + /// The names `--include-test` and `--exclude-test` are checked against. + #[test] + fn every_test_target_in_the_workspace_is_named() { + let (_directory, root) = workspace(); + let survey = survey(&root, SelectArgs::default()); + + // `it` is the integration target under `app/tests`, and the lib and bin targets are named + // because their own unit tests build into binaries of the same name. `demo` is an example, + // which cargo does not build as a test unless the manifest says so. + assert!(survey.tests.contains(&"it".to_owned()), "{:?}", survey.tests); + assert!(survey.tests.contains(&"core".to_owned()), "{:?}", survey.tests); + assert!(survey.tests.contains(&"app".to_owned()), "{:?}", survey.tests); + assert!(!survey.tests.contains(&"demo".to_owned()), "{:?}", survey.tests); + } + + /// A proc macro's code runs inside `rustc` while another crate is compiled, but a run builds + /// once and only then selects a mutant per test process. No mutant of one can therefore ever be + /// killed, so mutating it would charge for the work and hand back guaranteed survivors. An + /// ordinary library in the same workspace must still be mutated, or this would be a way to lose + /// real coverage. + #[test] + fn a_proc_macro_target_yields_no_mutants_while_its_neighbours_still_do() { + let (_directory, root) = workspace(); + + write( + &root, + "Cargo.toml", + "[workspace]\nmembers = [\"core\", \"app\", \"macros\"]\nresolver = \"3\"\n", + ); + write( + &root, + "macros/Cargo.toml", + "[package]\nname = \"macros\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[lib]\nproc-macro = true\n", + ); + write(&root, "macros/src/lib.rs", "pub fn widen(x: i32) -> i32 {\n x + 1\n}\n"); + + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + let scanned = survey + .scan(None, &Selection::parse("all").expect("every mutator resolves"), &mut ordinals) + .expect("the fixture must scan"); + + let mutated: HashSet<&str> = scanned.mutants.iter().map(|mutant| mutant.file.as_str()).collect(); + + assert!( + !mutated.iter().any(|file| file.contains("macros")), + "a proc-macro target must not be mutated: {mutated:?}" + ); + assert!( + mutated.iter().any(|file| file.contains("core")), + "ordinary libraries must still be mutated: {mutated:?}" + ); + } + + /// Every mutant is reached through a call to the guard runtime, so mutating the runtime itself + /// would put that call inside the crate that defines it and the tree would stop compiling. This + /// only arises for a workspace that builds the runtime — this one, when it runs on itself — but + /// the exclusion is keyed on the library's name, so a fixture declaring the same name proves it + /// without needing the real crate. Its neighbours must still be mutated. + #[test] + fn the_guard_runtime_yields_no_mutants_while_its_neighbours_still_do() { + let (_directory, root) = workspace(); + + write( + &root, + "Cargo.toml", + "[workspace]\nmembers = [\"core\", \"app\", \"rt\"]\nresolver = \"3\"\n", + ); + write( + &root, + "rt/Cargo.toml", + &format!( + "[package]\nname = \"rt\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[lib]\nname = \"{}\"\n", + crate::exec::RUNTIME_CRATE + ), + ); + write(&root, "rt/src/lib.rs", "pub fn widen(x: i32) -> i32 {\n x + 1\n}\n"); + + let survey = survey(&root, SelectArgs::default()); + let mut ordinals = 0; + let scanned = survey + .scan(None, &Selection::parse("all").expect("every mutator resolves"), &mut ordinals) + .expect("the fixture must scan"); + + let mutated: HashSet<&str> = scanned.mutants.iter().map(|mutant| mutant.file.as_str()).collect(); + + assert!( + !mutated.iter().any(|file| file.contains("rt")), + "the guard runtime must not be mutated: {mutated:?}" + ); + assert!( + mutated.iter().any(|file| file.contains("core")), + "ordinary libraries must still be mutated: {mutated:?}" + ); + } + + /// Test targets are collected across the whole workspace, since `--package` says what to + /// mutate while these patterns say what judges it. + #[test] + fn test_targets_are_named_even_for_packages_left_unmutated() { + let (_directory, root) = workspace(); + let args = SelectArgs { + dir: root, + packages: vec!["core".to_owned()], + ..SelectArgs::default() + }; + let survey = Survey::new(&args, None).expect("survey"); + + assert!(survey.tests.contains(&"it".to_owned()), "{:?}", survey.tests); + } + + #[test] + fn a_file_filter_still_recognizes_a_test_only_module_declared_outside_the_filter() { + let (_directory, root) = workspace(); + write( + &root, + "core/src/lib.rs", + "#[cfg(test)]\n#[path = \"reader_tests.rs\"]\nmod tests;\n", + ); + write( + &root, + "core/src/reader_tests.rs", + "#[test]\nfn reads() {\n assert_eq!(1 + 2, 3);\n}\n", + ); + + let survey = survey( + &root, + SelectArgs { + files: vec!["core/src/reader_tests.rs".to_owned()], + ..SelectArgs::default() + }, + ); + let mut ordinals = 0; + let scanned = survey + .scan(None, &Selection::parse("@all").expect("selection"), &mut ordinals) + .expect("scan"); + + assert!(scanned.mutants.is_empty(), "{:?}", scanned.mutants); + } + + #[test] + fn a_package_that_is_not_in_the_workspace_is_a_usage_error() { + // Silently surveying nothing would report a perfect score for a package name that was + // simply mistyped, which is the worst possible way to learn about a typo. + let (_directory, root) = workspace(); + let args = SelectArgs { + dir: root, + packages: vec!["nosuch".to_owned()], + ..SelectArgs::default() + }; + + let error = Survey::new(&args, None).expect_err("an unknown package must not survey"); + + assert!(error.to_string().contains("nosuch"), "{error}"); + assert!(error.is_usage(), "{error}"); + } + + #[test] + fn naming_one_package_leaves_the_others_alone() { + let (_directory, root) = workspace(); + let plan = survey( + &root, + SelectArgs { + packages: vec!["core".to_owned()], + ..SelectArgs::default() + }, + ); + + assert!(plan.files.iter().all(|file| file.package == "core"), "{:?}", plan.files); + } + + /// A workspace-wide exclusion must survive a run narrowed to one package. + /// + /// The patterns are written once, in `gamma.toml`, for the whole workspace, while `--package` + /// narrows a single run. Checking them against only the narrowed files makes a correct config + /// fail outright the moment someone runs a single package — which is what anyone iterating on + /// one crate does, so the config that works in CI breaks on every local run. + #[test] + fn a_pattern_naming_another_package_is_still_matched_when_one_package_is_named() { + let (_directory, root) = workspace(); + let args = SelectArgs { + dir: root, + packages: vec!["core".to_owned()], + exclude_files: vec!["app/**".to_owned()], + ..SelectArgs::default() + }; + + let plan = Survey::new(&args, None).expect("a pattern naming an unselected package must not be an error"); + + assert!(plan.files.iter().all(|file| file.package == "core"), "{:?}", plan.files); + } + + /// The wider walk must not weaken the check that catches a genuine typo. + #[test] + fn a_pattern_matching_nothing_in_the_whole_workspace_is_still_an_error() { + let (_directory, root) = workspace(); + let args = SelectArgs { + dir: root, + packages: vec!["core".to_owned()], + exclude_files: vec!["nosuch/**".to_owned()], + ..SelectArgs::default() + }; + + let error = Survey::new(&args, None).expect_err("a pattern matching nothing must not survey"); + + assert!(error.to_string().contains("nosuch/**"), "{error}"); + assert!(error.is_usage(), "{error}"); + } + + #[test] + fn a_test_or_example_target_is_never_mutated() { + // Mutating a test measures the tests' tests, and mutating an example measures nothing at + // all, since the suite does not run examples. + let (_directory, root) = workspace(); + let plan = survey(&root, SelectArgs::default()); + let paths: Vec<&str> = plan.files.iter().map(|file| file.path.as_str()).collect(); + + assert!(!paths.iter().any(|path| path.contains("examples")), "{paths:?}"); + assert!(!paths.iter().any(|path| path.contains("tests")), "{paths:?}"); + assert!(paths.iter().any(|path| path.ends_with("main.rs")), "{paths:?}"); + } + + #[test] + fn an_excluded_file_is_dropped_before_it_is_parsed() { + let (_directory, root) = workspace(); + let plan = survey( + &root, + SelectArgs { + exclude_files: vec!["**/generated.rs".to_owned()], + ..SelectArgs::default() + }, + ); + + assert!( + !plan.files.iter().any(|file| file.path.as_str().ends_with("generated.rs")), + "{:?}", + plan.files + ); + } + + /// A diff path that cannot be read at all has to fail the survey up front, naming the diff, + /// rather than falling through to a scan that silently behaves as though `--in-diff` had never + /// been given: a run over the wrong set of lines because a diff path was mistyped is far harder + /// to notice than an error at start-up. + #[test] + fn a_diff_that_cannot_be_read_fails_the_survey_rather_than_being_ignored() { + let (_directory, root) = workspace(); + + let error = Survey::new( + &SelectArgs { + dir: root.clone(), + in_diff: Some(root.join("no-such.patch")), + ..SelectArgs::default() + }, + None, + ) + .expect_err("the diff file does not exist"); + + assert!(error.to_string().contains("no-such.patch"), "{error}"); + } + + #[test] + fn a_file_the_diff_never_mentions_is_dropped_before_it_is_parsed() { + // This is most of what makes `--in-diff` affordable on a pull request: the files the change + // did not touch are skipped without ever being read, let alone parsed. + let (_directory, root) = workspace(); + + write( + &root, + "change.patch", + "--- a/core/src/lib.rs\n+++ b/core/src/lib.rs\n@@ -1,2 +1,2 @@\n one\n+ a + b\n", + ); + + let plan = survey( + &root, + SelectArgs { + in_diff: Some(root.join("change.patch")), + ..SelectArgs::default() + }, + ); + + assert_eq!(plan.files.len(), 1, "{:?}", plan.files); + assert!(plan.files[0].path.as_str().ends_with("lib.rs"), "{:?}", plan.files); + } + + /// `diff.mnemonicPrefix` is a setting a developer turns on once and forgets, and every diff + /// they produce afterwards carries `i/` and `w/` instead of `a/` and `b/`. Selecting nothing + /// from such a diff is indistinguishable from a change that touched no code. + #[test] + fn a_diff_written_with_mnemonic_prefixes_selects_the_same_files() { + let (_directory, root) = workspace(); + + write( + &root, + "change.patch", + "diff --git i/core/src/lib.rs w/core/src/lib.rs\n--- i/core/src/lib.rs\n+++ w/core/src/lib.rs\n@@ -1,2 +1,2 @@\n one\n+ a + b\n", + ); + + let plan = survey( + &root, + SelectArgs { + in_diff: Some(root.join("change.patch")), + ..SelectArgs::default() + }, + ); + + assert_eq!(plan.files.len(), 1, "{:?}", plan.files); + assert!(plan.files[0].path.as_str().ends_with("lib.rs"), "{:?}", plan.files); + } + + /// A diff naming files this workspace does not have was not understood, and a survey that + /// shrugged and selected nothing would hand the run an empty population and a perfect score. + #[test] + fn a_diff_that_names_nothing_in_this_workspace_fails_the_survey() { + let (_directory, root) = workspace(); + + write( + &root, + "change.patch", + "--- a/elsewhere/src/lib.rs\n+++ b/elsewhere/src/lib.rs\n@@ -1,2 +1,2 @@\n one\n+ a + b\n", + ); + + let error = Survey::new( + &SelectArgs { + dir: root.clone(), + in_diff: Some(root.join("change.patch")), + ..SelectArgs::default() + }, + None, + ) + .expect_err("a diff that names no file here must not pass for an empty change"); + + assert!(error.is_usage(), "{error}"); + assert!(error.to_string().contains("elsewhere/src/lib.rs"), "{error}"); + } + + /// A diff that touched only files this run does not mutate is understood perfectly well, and + /// selecting nothing is the right answer to it. + #[test] + fn a_diff_that_touches_no_source_file_selects_nothing_without_failing() { + let (_directory, root) = workspace(); + + write( + &root, + "change.patch", + "--- a/Cargo.toml\n+++ b/Cargo.toml\n@@ -1,2 +1,2 @@\n one\n+two\n", + ); + + let plan = survey( + &root, + SelectArgs { + in_diff: Some(root.join("change.patch")), + ..SelectArgs::default() + }, + ); + + assert!(plan.files.is_empty(), "{:?}", plan.files); + } + + #[test] + fn a_changed_file_still_only_yields_mutants_on_the_changed_lines() { + // A changed line usually sits among many that were not touched. Mutating the whole file + // would report on code the change never went near. + let (_directory, root) = workspace(); + + write( + &root, + "core/src/lib.rs", + "pub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\npub fn sub(a: i32, b: i32) -> i32 {\n a - b\n}\n", + ); + write( + &root, + "change.patch", + "--- a/core/src/lib.rs\n+++ b/core/src/lib.rs\n@@ -1,2 +1,2 @@\n head\n+ a + b\n", + ); + + let plan = survey( + &root, + SelectArgs { + in_diff: Some(root.join("change.patch")), + packages: vec!["core".to_owned()], + ..SelectArgs::default() + }, + ); + + let mut ordinals = 0; + let scanned = plan + .scan(None, &Selection::parse("all").expect("every mutator resolves"), &mut ordinals) + .expect("the fixture must scan"); + + assert!(!scanned.mutants.is_empty(), "the changed line yields nothing"); + // Whole-extent matching: every yielded mutant must span the touched line 2 -- `add`'s + // whole-body value mutants run lines 1..3 and its `a + b` mutants sit on line 2 -- while + // nothing from `sub`, four lines below and never touched, is pulled in. + assert!( + scanned.mutants.iter().all(|mutant| mutant.line <= 2 && mutant.end_line >= 2), + "{:?}", + scanned.mutants + ); + assert!( + scanned.mutants.iter().all(|mutant| &*mutant.item_path == "add"), + "{:?}", + scanned.mutants + ); + } + + #[test] + fn a_mutant_is_selected_by_its_whole_extent_not_just_its_first_line() { + // A mutation site can span several lines -- a call, match, or binary expression broken + // across them. A change that edits an interior line of such a site has changed that site, + // so its mutants must be selected even though the line the diff touched is neither the one + // the site starts on nor the one the report will name. + let (_directory, root) = workspace(); + + write( + &root, + "core/src/lib.rs", + "pub fn add(a: i32, b: i32, c: i32) -> i32 {\n a\n + b\n + c\n}\n", + ); + // The only added line is line 3, `+ b`, an interior line of the `a + b + c` expression + // whose site runs from line 2 to line 4. + write( + &root, + "change.patch", + "--- a/core/src/lib.rs\n+++ b/core/src/lib.rs\n@@ -2,3 +2,3 @@\n a\n- + x\n+ + b\n + c\n", + ); + + let plan = survey( + &root, + SelectArgs { + in_diff: Some(root.join("change.patch")), + packages: vec!["core".to_owned()], + ..SelectArgs::default() + }, + ); + + let mut ordinals = 0; + let scanned = plan + .scan(None, &Selection::parse("all").expect("every mutator resolves"), &mut ordinals) + .expect("the fixture must scan"); + + // Selecting a site by only its first line would attribute every mutant to line 2 and, since + // the diff never touched line 2, drop them all. + assert!(!scanned.mutants.is_empty(), "a site edited on an interior line was dropped"); + assert!( + scanned.mutants.iter().any(|mutant| mutant.line == 2 && mutant.end_line >= 3), + "no multi-line site reached the touched interior line: {:?}", + scanned.mutants + ); + // Every surviving mutant must genuinely span the touched line 3, start to end. + assert!( + scanned.mutants.iter().all(|mutant| mutant.line <= 3 && mutant.end_line >= 3), + "{:?}", + scanned.mutants + ); + } + + #[test] + fn a_mutant_an_earlier_report_settled_is_kept_with_its_verdict_and_never_run() { + // Incremental execution exists so a second run costs only the mutants that were still open, and a + // settled mutant therefore takes no ordinal and no shard slot. It stays in the population + // wearing the verdict it earned, because the score is a claim about the population: a run + // that dropped them would report on the subset it retried and call that the score. + let (_directory, root) = workspace(); + let mut plan = survey( + &root, + SelectArgs { + packages: vec!["core".to_owned()], + ..SelectArgs::default() + }, + ); + + let selection = Selection::parse("all").expect("every mutator resolves"); + let mut ordinals = 0; + let first = plan.scan(None, &selection, &mut ordinals).expect("the fixture must scan"); + let settled: HashMap = first.mutants.iter().map(|mutant| (mutant.id.clone(), Outcome::Killed)).collect(); + + assert!(!settled.is_empty(), "the fixture yielded no mutants to settle"); + + plan.settle(settled.clone()); + + let mut ordinals = 0; + let second = plan.scan(None, &selection, &mut ordinals).expect("the fixture must scan"); + + assert_eq!(second.mutants.len(), first.mutants.len()); + assert_eq!(second.settled_out, settled.len()); + assert_eq!(ordinals, 0, "a settled mutant is not work and must take no ordinal"); + assert!( + second.mutants.iter().all(|mutant| mutant.outcome == Outcome::Killed), + "{:?}", + second.mutants + ); + assert_eq!(crate::model::Summary::of(&second.mutants).scored(), Some(100.0)); + } + + /// Only the mutants the report actually settled are carried; the rest are work again, and the + /// population the second run scores is the same one the first run faced. + #[test] + fn an_unsettled_mutant_is_still_work_in_an_iterative_run() { + let (_directory, root) = workspace(); + let mut plan = survey( + &root, + SelectArgs { + packages: vec!["core".to_owned()], + ..SelectArgs::default() + }, + ); + + let selection = Selection::parse("all").expect("every mutator resolves"); + let mut ordinals = 0; + let first = plan.scan(None, &selection, &mut ordinals).expect("the fixture must scan"); + let live: Vec<&Mutant> = first.mutants.iter().filter(|mutant| mutant.ordinal > 0).collect(); + + assert!(live.len() > 1, "the fixture needs more than one live mutant"); + + let settled: HashMap = live.iter().skip(1).map(|mutant| (mutant.id.clone(), Outcome::Killed)).collect(); + + plan.settle(settled.clone()); + + let mut ordinals = 0; + let second = plan.scan(None, &selection, &mut ordinals).expect("the fixture must scan"); + + assert_eq!(second.mutants.len(), first.mutants.len()); + assert_eq!(second.settled_out, settled.len()); + assert_eq!(ordinals, 1, "only the mutant that is still open is work"); + } + + #[test] + fn a_feature_selection_is_carried_into_the_metadata_it_surveys() { + // Discovering under one feature set and compiling under another would place guards in files + // the compiler never sees, so every form of feature selection has to reach cargo. + let (_directory, root) = workspace(); + + for features in [ + FeatureArgs { + all_features: true, + ..FeatureArgs::default() + }, + FeatureArgs { + no_default_features: true, + ..FeatureArgs::default() + }, + FeatureArgs { + features: vec!["core/extra, ".to_owned()], + ..FeatureArgs::default() + }, + ] { + let metadata = load_metadata(&root, &features).expect("the fixture workspace must produce metadata"); + + assert_eq!(metadata.workspace_packages().len(), 2, "{features:?}"); + } + } + + /// A workspace whose binary target only exists when a feature turns it on. + /// + /// The gated binary sits in a directory of its own, because the walk that follows a target is + /// by directory: a `[[bin]]` beside `lib.rs` has its file reached through the library's own + /// walk whether or not cargo builds the binary. + fn gated_workspace() -> (TempDir, Utf8PathBuf) { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write(&root, "Cargo.toml", "[workspace]\nmembers = [\"gated\"]\nresolver = \"3\"\n"); + write( + &root, + "gated/Cargo.toml", + concat!( + "[package]\nname = \"gated\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n", + "[features]\ncli = []\n\n", + "[lib]\nname = \"gated\"\npath = \"src/lib.rs\"\n\n", + "[[bin]]\nname = \"gated-cli\"\npath = \"cli/main.rs\"\nrequired-features = [\"cli\"]\n\n", + "[[bin]]\nname = \"gated-plain\"\npath = \"plain/main.rs\"\n", + ), + ); + write( + &root, + "gated/src/lib.rs", + concat!( + "pub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n", + "#[cfg(feature = \"cli\")]\npub fn only_with_cli(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ), + ); + write(&root, "gated/cli/main.rs", "fn main() {\n let _ = 1 + 1;\n}\n"); + write(&root, "gated/plain/main.rs", "fn main() {\n let _ = 2 + 2;\n}\n"); + + (directory, root) + } + + /// Cargo does not build a binary whose `required-features` are off, so neither does the survey. + /// + /// Every mutant in such a target is unviable: the file is never compiled, so nothing can switch + /// a guard in it on and nothing can kill it. They arrive as build errors instead, and the + /// rollback loop pays for a round of them before withdrawing the lot. + #[test] + fn a_binary_whose_required_features_are_off_is_not_surveyed() { + let (_directory, root) = gated_workspace(); + let survey = survey(&root, SelectArgs::default()); + let listed: Vec<&Utf8PathBuf> = survey.files.iter().map(|file| &file.path).collect(); + + assert!(listed.contains(&&Utf8PathBuf::from("gated/src/lib.rs")), "{listed:?}"); + assert!(listed.contains(&&Utf8PathBuf::from("gated/plain/main.rs")), "{listed:?}"); + assert!(!listed.contains(&&Utf8PathBuf::from("gated/cli/main.rs")), "{listed:?}"); + } + + /// The same target, with the feature it asks for turned on, is built and therefore surveyed. + #[test] + fn a_binary_whose_required_features_are_on_is_surveyed_like_any_other() { + let (_directory, root) = gated_workspace(); + let survey = survey( + &root, + SelectArgs { + features: FeatureArgs { + all_features: true, + ..FeatureArgs::default() + }, + ..SelectArgs::default() + }, + ); + let listed: Vec<&Utf8PathBuf> = survey.files.iter().map(|file| &file.path).collect(); + + assert!(listed.contains(&&Utf8PathBuf::from("gated/src/lib.rs")), "{listed:?}"); + assert!(listed.contains(&&Utf8PathBuf::from("gated/plain/main.rs")), "{listed:?}"); + assert!(listed.contains(&&Utf8PathBuf::from("gated/cli/main.rs")), "{listed:?}"); + } + + /// A feature selector reaches cargo through `-C`/`--cargo-arg` and through `gamma.toml`'s + /// `cargo_args`, and cargo then really builds with it — so a closure that reads only the typed + /// arguments describes a build nobody runs. The gated binary leaves the population whole and + /// every item behind the feature is judged absent, both of which raise the score. + #[test] + fn a_feature_selector_in_the_passthrough_arguments_reaches_the_closure() { + let (_directory, root) = gated_workspace(); + let selected = CargoOptions { + extra: vec!["--features".to_owned(), "cli".to_owned()], + ..CargoOptions::default() + }; + + let survey = Survey::for_build( + &SelectArgs { + dir: root.clone(), + ..SelectArgs::default() + }, + None, + &selected, + ) + .expect("the fixture workspace must survey"); + let listed: Vec<&Utf8PathBuf> = survey.files.iter().map(|file| &file.path).collect(); + + assert!(listed.contains(&&Utf8PathBuf::from("gated/cli/main.rs")), "{listed:?}"); + + let items = mutated_items(&root, &selected); + + assert!(items.iter().any(|item| item.contains("only_with_cli")), "{items:?}"); + + // The control: with no selector anywhere, cargo builds neither, and neither does the survey. + let plain = mutated_items(&root, &CargoOptions::default()); + + assert!(!plain.iter().any(|item| item.contains("only_with_cli")), "{plain:?}"); + } + + /// A `dependency/feature` requirement is not something the per-package closure can answer, so + /// the target is kept rather than dropped on a guess. Losing a target that is built is worse + /// than keeping one that is not: the first silently shrinks the population, the second costs a + /// rollback round. + #[test] + fn a_required_feature_decides_a_binary_only_when_the_closure_can_answer() { + let (_directory, root) = gated_workspace(); + let metadata = load_metadata(&root, &FeatureArgs::default()).expect("the fixture workspace must produce metadata"); + let packages = metadata.workspace_packages(); + let package = packages.first().expect("the fixture workspace has one member"); + let gated = package + .targets + .iter() + .find(|target| target.name == "gated-cli") + .expect("the fixture declares the gated binary"); + + assert!( + !is_mutable_target(gated, Some(&Vec::new())), + "cargo does not build it with the feature off" + ); + assert!(is_mutable_target(gated, Some(&vec!["cli".to_owned()]))); + + // A package the closure never described leaves the requirement unproven, and an unproven + // requirement does not take a target out of the population. + assert!(is_mutable_target(gated, None)); + + let mut cross = (*gated).clone(); + + cross.required_features = vec!["serde/derive".to_owned()]; + + assert!(is_mutable_target(&cross, Some(&Vec::new()))); + } + + /// A workspace whose items are gated on the things the build decides. + /// + /// One item per facet: the profile's `debug_assertions`, its negation, the target, and a + /// predicate that only a `--cfg` in the build's flags can satisfy. A survey that probed a bare + /// host compile in the default profile would answer the first correctly and the rest wrong. + fn predicate_workspace() -> (TempDir, Utf8PathBuf) { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + + write(&root, "Cargo.toml", "[workspace]\nmembers = [\"probe\"]\nresolver = \"3\"\n"); + write( + &root, + "probe/Cargo.toml", + "[package]\nname = \"probe\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ); + write( + &root, + "probe/src/lib.rs", + concat!( + "#[cfg(debug_assertions)]\npub fn debug_only(a: i32, b: i32) -> i32 {\n a + b\n}\n\n", + "#[cfg(not(debug_assertions))]\npub fn release_only(a: i32, b: i32) -> i32 {\n a + b\n}\n\n", + "#[cfg(target_os = \"solaris\")]\npub fn solaris_only(a: i32, b: i32) -> i32 {\n a + b\n}\n\n", + "#[cfg(gamma_probe)]\npub fn custom_only(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ), + ); + + (directory, root) + } + + /// The items that yield a mutant when this workspace is surveyed for this build. + fn mutated_items(root: &Utf8Path, cargo: &CargoOptions) -> Vec { + mutated_items_with_assertions(root, cargo, None) + } + + fn mutated_items_with_assertions(root: &Utf8Path, cargo: &CargoOptions, debug_assertions: Option) -> Vec { + let mut survey = Survey::for_build( + &SelectArgs { + dir: root.to_owned(), + ..SelectArgs::default() + }, + None, + cargo, + ) + .expect("the fixture workspace must survey"); + + if let Some(debug_assertions) = debug_assertions { + let mut features = HashMap::default(); + + let _old = features.insert("probe".to_owned(), Vec::new()); + survey.cfgs = Cfgs::new( + &crate::cfg::for_build(&crate::cfg::Build { + debug_assertions: Some(debug_assertions), + ..crate::cfg::Build::default() + }) + .expect("the host compiler must describe its cfg predicates"), + &features, + ); + } + + let mut ordinals = 0; + let scanned = survey + .scan(None, &Selection::parse("arith.add_to_sub").expect("selection"), &mut ordinals) + .expect("the fixture must scan"); + + scanned.mutants.iter().map(|mutant| mutant.item_path.to_string()).collect() + } + + /// The profile decides `debug_assertions`, so it decides which half of that gate is compiled. + #[test] + fn the_profile_decides_which_half_of_a_debug_assertions_gate_is_surveyed() { + let (_directory, root) = predicate_workspace(); + let debug = mutated_items_with_assertions(&root, &CargoOptions::default(), Some(true)); + let release = mutated_items_with_assertions( + &root, + &CargoOptions { + profile: Some("release".to_owned()), + ..CargoOptions::default() + }, + Some(false), + ); + + assert!(debug.iter().any(|item| item.contains("debug_only")), "{debug:?}"); + assert!(!debug.iter().any(|item| item.contains("release_only")), "{debug:?}"); + assert!(release.iter().any(|item| item.contains("release_only")), "{release:?}"); + assert!(!release.iter().any(|item| item.contains("debug_only")), "{release:?}"); + } + + /// A build for another target compiles that target's code, and so surveys it. + /// + /// The triple is one every toolchain knows how to describe and no CI host runs, so the item is + /// absent by default and present only because the build asked for it. Nothing is compiled here: + /// `rustc --print cfg` answers for a target whose standard library is not installed. + #[test] + fn a_cross_target_build_surveys_the_code_that_target_compiles() { + let (_directory, root) = predicate_workspace(); + let host = mutated_items(&root, &CargoOptions::default()); + let elsewhere = mutated_items( + &root, + &CargoOptions { + extra: vec!["--target".to_owned(), "x86_64-pc-solaris".to_owned()], + ..CargoOptions::default() + }, + ); + + assert!(!host.iter().any(|item| item.contains("solaris_only")), "{host:?}"); + assert!(elsewhere.iter().any(|item| item.contains("solaris_only")), "{elsewhere:?}"); + } + + /// A predicate that only the build's own flags define is present exactly when they define it. + #[test] + fn a_custom_predicate_from_the_build_flags_is_surveyed_as_present() { + const CHILD: &str = "CARGO_GAMMA_CONFIG_RUSTFLAGS_TEST_CHILD"; + + if std::env::var_os(CHILD).is_none() { + let status = std::process::Command::new(std::env::current_exe().expect("the test executable has a path")) + .args([ + "--exact", + "discover::survey::tests::a_custom_predicate_from_the_build_flags_is_surveyed_as_present", + "--nocapture", + ]) + .env(CHILD, "1") + .env_remove("CARGO_ENCODED_RUSTFLAGS") + .env_remove("RUSTFLAGS") + .env_remove("CARGO_BUILD_RUSTFLAGS") + .status() + .expect("the isolated test process runs"); + + assert!(status.success(), "{status}"); + return; + } + + let (_directory, root) = predicate_workspace(); + let plain = mutated_items(&root, &CargoOptions::default()); + + assert!(!plain.iter().any(|item| item.contains("custom_only")), "{plain:?}"); + + write(&root, ".cargo/config.toml", "[build]\nrustflags = [\"--cfg\", \"gamma_probe\"]\n"); + + let flagged = mutated_items(&root, &CargoOptions::default()); + + assert!(flagged.iter().any(|item| item.contains("custom_only")), "{flagged:?}"); + } + + #[test] + fn metadata_that_cannot_be_read_names_the_directory() { + let directory = TempDir::new().expect("a temporary directory"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the temporary path is not UTF-8"); + let error = load_metadata(&root, &FeatureArgs::default()).expect_err("a directory with no manifest has no metadata"); + + assert!(error.to_string().contains(root.as_str()), "{error}"); + } +} diff --git a/crates/cargo-gamma-lib/src/discover/target_file.rs b/crates/cargo-gamma-lib/src/discover/target_file.rs new file mode 100644 index 000000000..69049f5c4 --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/target_file.rs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! A source file that will be analyzed. + +use camino::Utf8PathBuf; + +/// A source file that will be analyzed. +#[derive(Debug, Clone)] +pub struct TargetFile { + /// Path relative to the workspace root, with forward slashes. + pub path: Utf8PathBuf, + + /// Absolute path on disk. + pub absolute: Utf8PathBuf, + + /// The package the file belongs to. + pub package: String, +} diff --git a/crates/cargo-gamma-lib/src/discover/workspace_snapshot.rs b/crates/cargo-gamma-lib/src/discover/workspace_snapshot.rs new file mode 100644 index 000000000..e9705e6ba --- /dev/null +++ b/crates/cargo-gamma-lib/src/discover/workspace_snapshot.rs @@ -0,0 +1,461 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The workspace inputs a run had before it copied or executed anything. + +use core::sync::atomic::{AtomicBool, Ordering}; +use std::fs; +use std::sync::Mutex; + +use camino::{Utf8Path, Utf8PathBuf}; +use ignore::{WalkBuilder, WalkState}; +use serde::{Deserialize, Serialize}; + +use super::record::digest; + +/// A file in the workspace input snapshot. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(super) struct SnapshotFile { + pub(super) path: Utf8PathBuf, + pub(super) digest: String, + pub(super) size: u64, + + /// The file's modification time, in nanoseconds since the epoch, where the platform reports one. + /// + /// Content alone cannot answer the question the snapshot is asked. A file edited during the run + /// and restored to its original bytes before the run ends is byte-identical to what was + /// captured, so a comparison of digests says the workspace never moved — while the outcomes + /// were produced against the intermediate bytes. The modification time is an identity an + /// edit-and-revert does not forge: restoring the content writes the file again, and writing it + /// moves the time forward. + /// + /// Optional because not every platform and filesystem reports one, and a snapshot that refused + /// to be taken there would disable the cache rather than protect it. Where it is absent on both + /// sides the comparison degrades to the content check it always was. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) modified: Option, +} + +/// One local Cargo dependency whose source lies outside the workspace. +/// +/// Paths below this root use the same relative spelling as workspace files, while the root itself +/// remains in the record so a later capture can re-read the same external input. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct ExternalInput { + root: Utf8PathBuf, + files: Vec, +} + +/// The inputs a run had before it copied or executed the workspace. +/// +/// Cargo input discovery is intentionally conservative: every regular workspace file is an input +/// unless it is generated under `target`, under gamma's scratch base, or under `.git`. This covers +/// source and test modules, manifests, lockfiles, build scripts and workspace cargo configuration +/// without hashing artifacts the build produced itself. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct WorkspaceSnapshot { + /// A failed walk, read or non-UTF-8 path makes the snapshot unfit for carrying outcomes. + #[serde(default)] + complete: bool, + + /// Workspace-relative scratch paths that were excluded when this snapshot was taken. + #[serde(default)] + excluded: Vec, + + /// Every included regular file, sorted by path. + #[serde(default)] + pub(super) files: Vec, + + /// Local path dependencies outside the workspace. + #[serde(default)] + external: Vec, +} + +impl WorkspaceSnapshot { + /// Captures a complete conservative workspace input set. + #[cfg(test)] + pub(super) fn capture(root: &Utf8Path, excluded: &[Utf8PathBuf]) -> Self { + Self::capture_with_external(root, excluded, &[], false) + } + + /// Captures the workspace and every local path dependency Cargo can compile against. + pub(super) fn capture_with_external( + root: &Utf8Path, + excluded: &[Utf8PathBuf], + external_roots: &[Utf8PathBuf], + untracked_build_script_inputs: bool, + ) -> Self { + let mut snapshot = Self { + // Cargo build scripts can read arbitrary filesystem paths. Unless their inputs have + // been reported and captured, retaining outcomes would make an external file change + // invisible; no cache is safer than treating the script source as its whole input. + complete: !untracked_build_script_inputs, + excluded: exclusions(root, excluded), + files: Vec::new(), + external: Vec::new(), + }; + snapshot.files = capture_tree(root, &snapshot.excluded, &mut snapshot.complete); + + let workspace = match crate::paths::physical(root) { + Ok(path) => path, + Err(_unresolved) => { + snapshot.complete = false; + return snapshot; + } + }; + let mut roots = Vec::new(); + + for root in external_roots { + match crate::paths::physical(root) { + Ok(root) if !root.starts_with(&workspace) => roots.push(root), + Ok(_inside_workspace) => {} + Err(_unresolved) => snapshot.complete = false, + } + } + + roots.sort(); + roots.dedup(); + + for root in roots { + let mut excluded = exclusions(&root, &[]); + + // A path dependency may be an ancestor of the workspace. Its snapshot must not walk + // back down into the workspace (and its scratch artifacts) a second time; those bytes + // are already captured by the workspace half under its own exclusions. + if let Ok(relative) = workspace.strip_prefix(&root) + && !relative.as_str().is_empty() + { + excluded.push(relative.to_path_buf()); + excluded.sort(); + excluded.dedup(); + } + + let files = capture_tree(&root, &excluded, &mut snapshot.complete); + + snapshot.external.push(ExternalInput { root, files }); + } + + snapshot + } + + /// Whether the workspace remains exactly as it was before the run. + /// + /// Equality covers each file's path, content, size and modification time, so an edit that is + /// reverted before the run ends is still refused: reverting rewrites the file, and rewriting it + /// moves its modification time past the one recorded. Content equality alone would have called + /// such a workspace unchanged and carried forward outcomes that were produced against the + /// intermediate bytes. + /// + /// Two gaps remain, both accepted. A filesystem whose modification times have coarse + /// granularity — one second is the classic case — hides an edit-and-revert cycle that begins + /// and ends inside a single tick. And a platform that reports no modification time at all + /// leaves the comparison as the content check it always was. + pub(super) fn matches_current(&self, root: &Utf8Path) -> bool { + self.complete && *self == self.recapture(root) + } + + pub(super) fn recapture(&self, root: &Utf8Path) -> Self { + let external_roots: Vec = self.external.iter().map(|input| input.root.clone()).collect(); + + Self::capture_with_external(root, &self.excluded, &external_roots, false) + } + + pub(super) const fn is_complete(&self) -> bool { + self.complete + } + + pub(super) fn matches_compilation_inputs(&self, current: &Self, roots: &[Utf8PathBuf]) -> bool { + self.complete + && current.complete + && self.external == current.external + && self + .files + .iter() + .filter(|file| Self::is_compilation_input(&file.path, roots)) + .eq(current.files.iter().filter(|file| Self::is_compilation_input(&file.path, roots))) + } + + /// The pre-execution metadata for one workspace-relative file. + pub(super) fn file(&self, path: &Utf8Path) -> Option<&SnapshotFile> { + let index = self.files.binary_search_by(|file| file.path.as_path().cmp(path)).ok()?; + + self.files.get(index) + } + + fn is_compilation_input(path: &Utf8Path, roots: &[Utf8PathBuf]) -> bool { + matches!( + path.as_str(), + "Cargo.toml" | "Cargo.lock" | "rust-toolchain" | "rust-toolchain.toml" + ) || path.starts_with(".cargo") + || roots.iter().any(|root| root.as_str().is_empty() || path.starts_with(root)) + } + + /// Rust files known before the run, for the declaring-file index. + #[cfg(test)] + pub(super) fn rust_files(&self, root: &Utf8Path) -> Vec { + self.files + .iter() + .filter(|file| file.path.extension() == Some("rs")) + .map(|file| root.join(&file.path)) + .collect() + } +} + +/// Captures every regular file below one independently tracked source root. +/// +/// The walk fans out over `ignore`'s worker pool with every filter it would otherwise apply +/// turned off, so the set of files visited is exactly the one a single-threaded walk would visit. +/// Each worker computes its own digests and reads its own metadata without holding any lock; the +/// only shared state is the completion flag, set with a relaxed store because the parallel walk +/// joins every worker thread before this function reads it back, and the results list, locked only +/// long enough to push one already-computed [`SnapshotFile`]. +fn capture_tree(root: &Utf8Path, excluded: &[Utf8PathBuf], complete: &mut bool) -> Vec { + let boundary = fs::canonicalize(root.as_std_path()) + .ok() + .and_then(|path| Utf8PathBuf::from_path_buf(path).ok()); + if boundary.is_none() { + *complete = false; + } + + let complete_flag = AtomicBool::new(*complete); + let files: Mutex> = Mutex::new(Vec::new()); + let excluded = excluded.to_vec(); + let root_owned = root.to_owned(); + + let mut builder = WalkBuilder::new(root.as_std_path()); + + let _builder = builder + // Every regular file below the root is a candidate input; none of `ignore`'s conventions + // for hidden files, parent-directory ignore files or `.git`-relative rules apply here. + .hidden(false) + .parents(false) + .require_git(false) + .git_ignore(false) + .git_exclude(false) + .git_global(false) + .ignore(false) + // A link is recorded by its own spelling below, never followed. + .follow_links(false); + + builder.build_parallel().run(|| { + let root = root_owned.clone(); + let excluded = excluded.clone(); + let boundary = boundary.clone(); + let complete_flag = &complete_flag; + let files = &files; + + Box::new(move |entry| { + let Ok(entry) = entry else { + complete_flag.store(false, Ordering::Relaxed); + return WalkState::Continue; + }; + let Ok(path) = Utf8PathBuf::from_path_buf(entry.path().to_path_buf()) else { + complete_flag.store(false, Ordering::Relaxed); + return WalkState::Continue; + }; + let relative = path.strip_prefix(&root).unwrap_or(&path); + let is_dir = entry.file_type().is_some_and(|file_type| file_type.is_dir()); + + if excluded.iter().any(|excluded| relative.starts_with(excluded)) { + return if is_dir { WalkState::Skip } else { WalkState::Continue }; + } + + if is_dir { + return WalkState::Continue; + } + + if entry.path_is_symlink() { + let Ok(target) = fs::read_link(path.as_std_path()) else { + complete_flag.store(false, Ordering::Relaxed); + return WalkState::Continue; + }; + let bytes = target.as_os_str().as_encoded_bytes(); + + // A link inside the captured tree can point at a file Cargo reads outside it. The + // target spelling alone says nothing about that file's bytes, so do not reuse any + // outcome when it happens. Dangling links have no referent bytes yet; their + // spelling and own timestamp are the complete input until one appears. + if let Some(boundary) = &boundary { + match fs::canonicalize(path.as_std_path()) { + Ok(referent) => match Utf8PathBuf::from_path_buf(referent) { + Ok(referent) if referent.starts_with(boundary) => { + let relative = referent.strip_prefix(boundary).expect("the prefix was checked"); + + if excluded.iter().any(|excluded| relative.starts_with(excluded)) { + complete_flag.store(false, Ordering::Relaxed); + } + } + Ok(_) | Err(_) => complete_flag.store(false, Ordering::Relaxed), + }, + Err(cause) if cause.kind() == std::io::ErrorKind::NotFound => {} + Err(_unresolved) => complete_flag.store(false, Ordering::Relaxed), + } + } + + let file = SnapshotFile { + path: relative.to_path_buf(), + digest: digest(bytes), + size: bytes.len() as u64, + modified: symlink_modified_at(&path), + }; + + files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(file); + + return WalkState::Continue; + } + + match fs::metadata(path.as_std_path()) { + Ok(metadata) if metadata.is_file() => {} + _other => { + complete_flag.store(false, Ordering::Relaxed); + return WalkState::Continue; + } + } + + let Ok(bytes) = fs::read(path.as_std_path()) else { + complete_flag.store(false, Ordering::Relaxed); + return WalkState::Continue; + }; + + let file = SnapshotFile { + path: relative.to_path_buf(), + digest: digest(&bytes), + size: bytes.len() as u64, + // Read after the bytes rather than before them, so that a write landing between + // the two leaves a time ahead of the content it describes rather than behind it. + // The stale direction would be dangerous: a recorded time older than the bytes + // recorded beside it would match a later capture of those same bytes. + modified: modified_at(&path), + }; + + files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(file); + + WalkState::Continue + }) + }); + + // Every worker thread has joined by the time `run` returns, so this ordinary load sees every + // relaxed store a worker made; no stronger ordering is needed for a flag that only ever moves + // from true to false. + *complete = complete_flag.load(Ordering::Relaxed); + + let mut files = files.into_inner().unwrap_or_else(std::sync::PoisonError::into_inner); + + files.sort_by(|left, right| left.path.cmp(&right.path)); + files +} + +/// A file's modification time in nanoseconds since the epoch, where the platform reports one. +/// +/// Every failure — an unsupported filesystem, a time before the epoch, a value past what fits — +/// answers "unknown" rather than a wrong number, because a wrong time is worse than none: it would +/// either invalidate a record that is sound or, in the other direction, be equal across a change. +fn modified_at(path: &Utf8Path) -> Option { + modified(&fs::metadata(path.as_std_path()).ok()?) +} + +/// A symlink's own modification time, without following its target. +fn symlink_modified_at(path: &Utf8Path) -> Option { + modified(&fs::symlink_metadata(path.as_std_path()).ok()?) +} + +fn modified(metadata: &fs::Metadata) -> Option { + let since = metadata.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?; + + u64::try_from(since.as_nanos()).ok() +} + +/// Workspace-relative paths the snapshot never traverses. +fn exclusions(root: &Utf8Path, excluded: &[Utf8PathBuf]) -> Vec { + let mut paths = vec![Utf8PathBuf::from("target"), Utf8PathBuf::from(".git")]; + + for path in excluded { + if path.is_relative() { + paths.push(path.clone()); + } else if let Ok(relative) = path.strip_prefix(root) + && !relative.as_str().is_empty() + { + paths.push(relative.to_path_buf()); + } + } + + paths.sort(); + paths.dedup(); + paths +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::fs; + + use camino::Utf8Path; + + use super::WorkspaceSnapshot; + + #[cfg(unix)] + #[test] + fn directory_and_broken_symlinks_form_a_repeatable_snapshot() { + let directory = crate::testing::workdir("snapshot-symlinks-"); + let root = Utf8Path::from_path(directory.path()).expect("the temporary path is UTF-8"); + fs::create_dir(root.join("real")).expect("directory"); + std::os::unix::fs::symlink("real", root.join("directory-link")).expect("directory symlink"); + std::os::unix::fs::symlink("missing", root.join("broken-link")).expect("broken symlink"); + + let snapshot = WorkspaceSnapshot::capture(root, &[]); + + assert!(snapshot.matches_current(root)); + assert_eq!( + snapshot.files.iter().map(|file| file.path.as_str()).collect::>(), + ["broken-link", "directory-link"] + ); + } + + #[cfg(unix)] + #[test] + fn a_symlink_to_an_external_referent_makes_reuse_ineligible() { + let directory = crate::testing::workdir("snapshot-external-link-"); + let container = Utf8Path::from_path(directory.path()).expect("the temporary path is UTF-8"); + let root = container.join("workspace"); + let external = container.join("dependency.rs"); + + fs::create_dir(&root).expect("workspace"); + fs::write(&external, "pub fn dependency() {}\n").expect("external input"); + std::os::unix::fs::symlink(&external, root.join("linked.rs")).expect("external link"); + + let snapshot = WorkspaceSnapshot::capture(&root, &[]); + + assert!( + !snapshot.is_complete(), + "a link whose referent is outside the captured roots cannot certify a cache entry" + ); + } + + #[test] + fn an_ancestor_path_dependency_does_not_recapture_the_workspace() { + let directory = crate::testing::workdir("snapshot-ancestor-dependency-"); + let container = Utf8Path::from_path(directory.path()).expect("the temporary path is UTF-8"); + let workspace = container.join("workspace"); + let dependency = container.join("dependency.rs"); + + fs::create_dir(&workspace).expect("workspace"); + fs::write(workspace.join("source.rs"), "pub fn source() {}\n").expect("workspace input"); + fs::write(&dependency, "pub fn dependency() {}\n").expect("dependency input"); + + let snapshot = WorkspaceSnapshot::capture_with_external(&workspace, &[], &[container.to_path_buf()], false); + + assert!(snapshot.is_complete()); + assert!( + snapshot.external[0].files.iter().all(|file| !file.path.starts_with("workspace")), + "the external half must not double-capture the workspace or its scratch outputs" + ); + assert!(snapshot.matches_current(&workspace)); + + fs::write(&dependency, "pub fn dependency() { panic!() }\n").expect("changed dependency"); + + assert!( + !snapshot.matches_current(&workspace), + "a change outside the workspace must invalidate reuse" + ); + } +} diff --git a/crates/cargo-gamma-lib/src/docs.rs b/crates/cargo-gamma-lib/src/docs.rs new file mode 100644 index 000000000..4c7adadea --- /dev/null +++ b/crates/cargo-gamma-lib/src/docs.rs @@ -0,0 +1,393 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![cfg_attr( + not(feature = "internals"), + expect( + dead_code, + reason = "this module exists for `tests/docs.rs`, which is what keeps the README's generated tables honest; nothing the tool does at run time reads them, so without the feature that opens the facade the module has no caller" + ) +)] + +//! Reference tables for the documentation, rendered from the registry that defines them. +//! +//! The mutator catalog is the tool's public vocabulary: the same names appear on `--mutators`, in every +//! suppression directive, in the report, in SARIF rule identifiers and in configuration. A +//! reference that drifts from the registry is therefore worse than no reference at all, because a +//! reader who copies a name out of it gets a usage error and no clue that the document was wrong. +//! +//! So the tables are generated here and checked against the README by a test. Adding a +//! mutator fails that test until the document is regenerated, which is the only arrangement that +//! keeps a hand-written catalog honest as the catalog grows. + +use core::fmt::Write as _; + +use crate::ops::registry::{PRESETS, REGISTRY, families}; + +/// The marker that opens a generated block in a documentation file. +/// +/// The blocks are delimited rather than owning the whole file so that the prose explaining what a +/// family is *for* can live beside the table listing what it contains. A reference that is only a +/// table tells a reader what exists without telling them when to reach for it. +pub const BEGIN: &str = ""; + +/// Renders the block named `name`, or `None` when no such block exists. +#[must_use] +pub fn block(name: &str) -> Option { + match name { + "mutators" => Some(mutators()), + "presets" => Some(presets()), + "families" => Some(family_summary()), + "commands" => Some(commands()), + "options" => Some(options()), + _ => None, + } +} + +/// One row per subcommand, with the one-line summary clap prints. +fn commands() -> String { + let command = ::command(); + let mut out = String::new(); + + let _ = writeln!(out, "| Command | What it does |"); + let _ = writeln!(out, "| --- | --- |"); + + for sub in command.get_subcommands() { + let about = sub.get_about().map(ToString::to_string).unwrap_or_default(); + + let _ = writeln!( + out, + "| [`gamma {}`](#gamma-{}) | {} |", + sub.get_name(), + sub.get_name(), + escape(&about) + ); + } + + out.trim_end().to_owned() +} + +/// Every option of every subcommand, grouped by the help heading it carries. +/// +/// Rendered from the same `clap` definition that answers `--help`, so an option cannot be added +/// without appearing here, and the categories a reader sees in the terminal are the categories +/// they see in the reference. +fn options() -> String { + let command = ::command(); + let mut out = String::new(); + + // The globals are declared on the root rather than on any subcommand, so walking the + // subcommands alone would silently omit the two options that apply to all of them. + let _ = writeln!(out, "### Accepted by every subcommand\n"); + + for (heading, arguments) in grouped(&command) { + let _ = writeln!(out, "**{heading}**\n"); + let _ = writeln!(out, "| Option | Value | What it does |"); + let _ = writeln!(out, "| --- | --- | --- |"); + + for argument in arguments { + let _ = writeln!( + out, + "| {} | {} | {} |", + spelling(argument), + value_of(argument), + escape(&help_of(argument)) + ); + } + + out.push('\n'); + } + + for sub in command.get_subcommands() { + let _ = writeln!(out, "### `gamma {}`\n", sub.get_name()); + + if let Some(about) = sub.get_about() { + let _ = writeln!(out, "{about}\n"); + } + + let _ = writeln!(out, "```text\n{}\n```\n", usage(sub)); + + for (heading, arguments) in grouped(sub) { + let _ = writeln!(out, "**{heading}**\n"); + let _ = writeln!(out, "| Option | Value | What it does |"); + let _ = writeln!(out, "| --- | --- | --- |"); + + for argument in arguments { + let _ = writeln!( + out, + "| {} | {} | {} |", + spelling(argument), + value_of(argument), + escape(&help_of(argument)) + ); + } + + out.push('\n'); + } + } + + out.trim_end().to_owned() +} + +/// The usage line clap would print for `sub`, spelled the way a user types it. +fn usage(sub: &clap::Command) -> String { + let mut sub = sub.clone(); + let rendered = sub.render_usage().to_string().replace("Usage: ", ""); + + format!("cargo gamma {rendered}") +} + +/// The arguments of `sub`, grouped by help heading in the order the headings first appear. +/// +/// `--help` and `--version` are dropped: they are on every command, they are not what a reference +/// is consulted for, and a row for each would be nine rows of noise. +fn grouped(sub: &clap::Command) -> Vec<(String, Vec<&clap::Arg>)> { + let mut groups: Vec<(String, Vec<&clap::Arg>)> = Vec::new(); + + for argument in sub.get_arguments() { + if matches!(argument.get_id().as_str(), "help" | "version") || argument.is_hide_set() { + continue; + } + + let heading = if argument.is_positional() { + "Arguments".to_owned() + } else { + argument.get_help_heading().map_or_else(|| "Options".to_owned(), ToOwned::to_owned) + }; + + if let Some(slot) = groups.iter_mut().find(|(name, _)| *name == heading) { + slot.1.push(argument); + } else { + groups.push((heading, vec![argument])); + } + } + + groups +} + +/// How an argument is written on the command line, shorts included. +fn spelling(argument: &clap::Arg) -> String { + let Some(long) = argument.get_long() else { + return format!("`<{}>`", argument.get_id().as_str().to_uppercase()); + }; + + argument + .get_short() + .map_or_else(|| format!("`--{long}`"), |short| format!("`-{short}`, `--{long}`")) +} + +/// The placeholder an argument takes, or a blank cell for a flag. +fn value_of(argument: &clap::Arg) -> String { + if matches!(argument.get_action(), clap::ArgAction::SetTrue | clap::ArgAction::SetFalse) { + return String::new(); + } + + argument + .get_value_names() + .and_then(<[clap::builder::Str]>::first) + .map_or_else(String::new, |name| format!("`<{name}>`")) +} + +/// The one-line help for an argument, with the default appended when there is one. +/// +/// clap strips the full stop from the end of a doc comment when it renders short help, so it is +/// put back before anything is appended — otherwise the default runs straight into the sentence. +fn help_of(argument: &clap::Arg) -> String { + let mut text = argument.get_help().map(ToString::to_string).unwrap_or_default().replace('\n', " "); + + if !text.is_empty() && !text.ends_with(['.', '!', '?']) { + text.push('.'); + } + + let defaults = argument.get_default_values(); + + if defaults.is_empty() { + return text; + } + + let shown = defaults + .iter() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>() + .join(", "); + + format!("{text} Defaults to `{shown}`.") +} + +/// Every mutator, grouped by family, with its alias and default state. +fn mutators() -> String { + let mut out = String::new(); + + for family in families() { + let members: Vec<_> = REGISTRY + .iter() + .filter(|mutator| mutator.name.split('.').next() == Some(family)) + .collect(); + + let _ = writeln!(out, "#### `{family}`\n"); + let _ = writeln!(out, "| Mutator | What it does | Alias | Default |"); + let _ = writeln!(out, "| --- | --- | --- | --- |"); + + for mutator in members { + let aliases = if mutator.aliases.is_empty() { + String::new() + } else { + format!("`{}`", mutator.aliases.join("`, `")) + }; + + let _ = writeln!( + out, + "| `{}` | {} | {} | {} |", + mutator.name, + escape(mutator.description), + aliases, + if mutator.default_on { "yes" } else { "no" } + ); + } + + out.push('\n'); + } + + out.trim_end().to_owned() +} + +/// Every mutator preset, with the selectors it expands to. +fn presets() -> String { + let mut out = String::new(); + + let _ = writeln!(out, "| Mutator preset | What it selects | Expands to |"); + let _ = writeln!(out, "| --- | --- | --- |"); + + for preset in PRESETS { + let members = preset + .members + .iter() + .map(|member| format!("`{member}`")) + .collect::>() + .join(", "); + + let _ = writeln!(out, "| `@{}` | {} | {members} |", preset.name, escape(preset.description)); + } + + out.trim_end().to_owned() +} + +/// One row per family, with how many mutators it holds. +fn family_summary() -> String { + let mut out = String::new(); + + let _ = writeln!(out, "| Family | Mutators | What it asks |"); + let _ = writeln!(out, "| --- | ---: | --- |"); + + for family in families() { + let count = REGISTRY + .iter() + .filter(|mutator| mutator.name.split('.').next() == Some(family)) + .count(); + + let _ = writeln!(out, "| [`{family}`](#{family}) | {count} | {} |", question(family)); + } + + let _ = writeln!(out, "| **Total** | **{}** | |", REGISTRY.len()); + + out.trim_end().to_owned() +} + +/// The question a family exists to ask, in the reader's terms rather than the mutator's. +/// +/// A description of the transform — "replace `<` with `<=`" — says what the tool does, which the +/// per-mutator table already covers. What a reader choosing between families needs is what a +/// survivor in that family would mean about their tests, which is a different sentence. +fn question(family: &str) -> &'static str { + match family { + "fn_value" => "Does anything check what this function returns?", + "relational" => "Is this comparison's boundary the right one?", + "arith" => "Does this calculation's operator matter?", + "bitwise" => "Is this mask or flag combination correct?", + "shift" => "Is this shift's direction load-bearing?", + "assign" => "Does this compound assignment's operator matter?", + "assign_value" => "Is the value assigned here ever read in a way that would notice?", + "logical" => "Is this `&&` really an `&&`?", + "cond" => "Does anything depend on this branch being taken?", + "match_guard" => "Does anything depend on this guard being right?", + "match_arm" => "Is this arm reachable, and does anything notice when it stops matching?", + "loop" => "Does this `break` or `continue` carry the loop's meaning?", + "range" => "Is this bound inclusive on purpose?", + "literal" => "Does this constant's exact value matter?", + "expr" => "Would an off-by-one here be caught?", + "unary" => "Does this negation or complement matter?", + "stmt" => "Does this statement's side effect matter?", + "struct_field" => "Does this field's value matter, or is the default good enough?", + "option" => "Is the present case distinguished from the absent one?", + "result" => "Is success distinguished from failure?", + "iter" => "Does anything observe that this was ordered, deduplicated, or taken from one end?", + "string" => "Does the prefix, the case, or the trimmed end actually matter?", + "collection" => "Does every element of this literal earn its place?", + _ => "", + } +} + +/// Escapes the characters that would otherwise end a table cell. +fn escape(text: &str) -> String { + text.replace('|', "\\|") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_registered_mutator_appears_in_the_mutator_table() { + // The table is the tool's published vocabulary. A name missing from it is a feature the + // user cannot discover, and a name in it that the registry does not have is worse: it + // reads as usable and produces a usage error. + let rendered = mutators(); + + for mutator in REGISTRY { + assert!( + rendered.contains(mutator.name), + "`{}` is missing from the mutator table", + mutator.name + ); + } + } + + #[test] + fn every_family_is_given_a_question_to_ask() { + // A blank cell in the summary would be the one row a reader skips, and it would be skipped + // for the newest family — the one most in need of an explanation. + for family in families() { + assert!( + !question(family).is_empty(), + "family `{family}` has no question in the summary table" + ); + } + } + + #[test] + fn a_description_containing_a_pipe_cannot_break_the_table() { + // `bitwise.or_to_and` and friends describe themselves with `|`, which would otherwise end + // the cell and silently shift every column after it. + assert_eq!(escape("replace | with &"), "replace \\| with &"); + } + + #[test] + fn an_unrecognized_family_name_is_given_no_question_rather_than_a_guess() { + // `question` is keyed by family name, not driven by the registry, so a name that never + // shipped as a family — a typo, or a check performed before the family exists — must fall + // through to an empty string instead of panicking or fabricating a plausible-sounding + // question that would mislead a reader. + assert_eq!(question("nonesuch"), ""); + } + + #[test] + fn an_unknown_block_name_is_refused_rather_than_rendered_empty() { + // A misspelled marker that produced an empty block would delete a whole table from the + // documentation and pass every check that follows. + assert!(block("mutators").is_some()); + assert!(block("nonesuch").is_none()); + } +} diff --git a/crates/cargo-gamma-lib/src/elements/digest.rs b/crates/cargo-gamma-lib/src/elements/digest.rs new file mode 100644 index 000000000..d8750d85c --- /dev/null +++ b/crates/cargo-gamma-lib/src/elements/digest.rs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The verdict-only view of a mutation-testing report. + +use std::borrow::Cow; + +use serde::Deserialize; + +use super::report::{ + FLAKY_PREFIX, FRAMEWORK_NAME, NOT_BUILT_PREFIX, OUT_OF_MEMORY_PREFIX, SUPPORTED_SCHEMA_MAJOR, TIMEOUT_PREFIX, supported_schema_version, +}; +use crate::model::Outcome; +use crate::{HashMap, HashSet}; + +/// The slice of a report a cross-run reader needs. +/// +/// Serde skips embedded source fields without allocating them. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Digest<'text> { + #[serde(borrow)] + pub schema_version: Cow<'text, str>, + #[serde(borrow)] + pub framework: FrameworkDigest<'text>, + #[serde(borrow)] + pub files: HashMap, FileDigest<'text>>, +} + +/// The report producer. +#[derive(Debug, Deserialize)] +pub struct FrameworkDigest<'text> { + #[serde(borrow)] + pub name: Cow<'text, str>, +} + +impl Digest<'_> { + /// Checks that this tool can safely reuse the document's verdicts. + /// + /// # Errors + /// + /// Returns an error for another producer or an unsupported schema version. + pub fn ensure_ours(&self) -> Result<(), String> { + if self.framework.name != FRAMEWORK_NAME { + return Err(format!( + "the report was written by `{}` rather than by {FRAMEWORK_NAME}, so its verdicts are not this tool's to carry forward", + self.framework.name + )); + } + + if supported_schema_version(&self.schema_version) { + Ok(()) + } else { + Err(format!( + "the report claims schema version `{}`, which this build does not understand; \ + versions 1 to {SUPPORTED_SCHEMA_MAJOR} are supported", + self.schema_version + )) + } + } +} + +/// The mutants recorded for one file. +#[derive(Debug, Deserialize)] +pub struct FileDigest<'text> { + #[serde(borrow)] + pub mutants: Vec>, +} + +/// One recorded verdict. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MutantDigest<'text> { + #[serde(borrow)] + pub id: Cow<'text, str>, + #[serde(borrow)] + pub status: Cow<'text, str>, + #[serde(borrow, default)] + pub status_reason: Option>, +} + +impl MutantDigest<'_> { + /// Maps a verdict onto the outcome it settles. + #[must_use] + pub fn settled_outcome(&self) -> Option { + settled_verdict(&self.status, self.status_reason.as_deref()) + } +} + +/// Reads the IDs of mutants an earlier report settled. +/// +/// # Errors +/// +/// Returns an error if the text is not a compatible report from this tool. +#[cfg_attr( + not(feature = "internals"), + allow( + dead_code, + reason = "kept to verify that writer and incremental reader agree about settled verdicts" + ) +)] +pub fn settled_mutants(text: &str) -> Result, String> { + let report: Digest<'_> = serde_json::from_str(text).map_err(|cause| cause.to_string())?; + + report.ensure_ours()?; + + Ok(report + .files + .values() + .flat_map(|file| file.mutants.iter()) + .filter(|mutant| mutant.settled_outcome().is_some()) + .map(|mutant| mutant.id.clone().into_owned()) + .collect()) +} + +pub(super) fn settled_verdict(status: &str, reason: Option<&str>) -> Option { + let reason_is = |prefix: &str| reason.is_some_and(|reason| reason.starts_with(prefix)); + + match status { + "Survived" if reason_is(OUT_OF_MEMORY_PREFIX) => None, + "Survived" if reason_is(TIMEOUT_PREFIX) => Some(Outcome::Timeout), + "Timeout" if reason.is_none() || reason_is(OUT_OF_MEMORY_PREFIX) => None, + "Ignored" if reason.is_none() || reason_is(NOT_BUILT_PREFIX) || reason_is(FLAKY_PREFIX) => None, + "Killed" => Some(Outcome::Killed), + "Timeout" => Some(Outcome::Timeout), + "CompileError" => Some(Outcome::CompileError), + "Ignored" => Some(Outcome::Ignored), + _other => None, + } +} diff --git a/crates/cargo-gamma-lib/src/elements/mod.rs b/crates/cargo-gamma-lib/src/elements/mod.rs new file mode 100644 index 000000000..4f639eba7 --- /dev/null +++ b/crates/cargo-gamma-lib/src/elements/mod.rs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The `mutation-testing-elements` report schema. +//! +//! This is the interchange format the Stryker report viewers consume, and emitting it is what +//! gives cargo-gamma a report UI, an Azure DevOps extension and a GitHub integration without +//! writing any of them. The schema is a published artifact of another project, so the mapping is +//! spelled out here rather than left implicit: drift is silent and shows up as a blank page in +//! someone's browser rather than as a failing build. + +mod digest; +mod publication; +mod report; + +pub use digest::{Digest, FileDigest, FrameworkDigest, MutantDigest, settled_mutants}; +pub(crate) use publication::{Publication, remove_if_unchanged, write_if_unchanged, write_streamed}; +#[cfg(test)] +pub(crate) use publication::{before_next_publication, fail_next_directory_sync, next_scratch_path}; +pub use publication::{publish, write}; +pub use report::{ + FileResult, Framework, Location, MergeProvenance, MutantResult, Position, Report, RunInfo, ShardInfo, SourceProvenance, Thresholds, + VerdictProvenance, build, to_json, write_json, +}; +pub(crate) use report::{supported_schema_version, validate_schema}; diff --git a/crates/cargo-gamma-lib/src/elements/publication.rs b/crates/cargo-gamma-lib/src/elements/publication.rs new file mode 100644 index 000000000..3fa5d9eba --- /dev/null +++ b/crates/cargo-gamma-lib/src/elements/publication.rs @@ -0,0 +1,1008 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Writing a file atomically: whole or not at all. +//! +//! Reports, the generated configuration and the source files `suppress` and `unsuppress` rewrite +//! are all published through here, so a failed or interrupted write can never leave a truncated +//! file under the destination name. + +#[cfg(test)] +use core::cell::{Cell, RefCell}; +use core::hash::{BuildHasher as _, Hasher as _}; +use core::sync::atomic::{AtomicU64, Ordering}; +use std::collections::hash_map::RandomState; +use std::fs::{self, File}; +use std::io::{self, ErrorKind, Write as _}; + +use camino::{Utf8Path, Utf8PathBuf}; + +use crate::Result; +use crate::error::error; + +/// Writes a file whole or not at all, creating parent directories as needed. +/// +/// The contents go to a sibling of the destination and are renamed onto it once they are all +/// there and durable, and the rename itself is made durable in turn, so a crash, a kill or a full +/// disk part-way through leaves the previous file rather than a truncated one — and a machine that +/// loses power after this returns comes back with the new file rather than with neither. That +/// matters because the reader these outputs exist for is a CI +/// job parsing the JSON report, which cannot tell a truncated file from a short one — it sees a +/// name it expects, holding something that is not valid JSON, and reports a broken run instead of +/// a failed write. It matters more for the source files `suppress` and `unsuppress` rewrite, where +/// the truncated file is not a report nobody wrote by hand but the user's own code. +/// +/// The temporary is a sibling rather than a file in the system temporary directory so that the +/// rename stays within one filesystem, which is what makes it atomic; and it carries a suffix that +/// is unique to each write across the whole filesystem, so that overlapping runs cannot collide. +/// See `scratch_path` for why the process id alone is not enough. +/// +/// A destination that already exists keeps the permissions it has when the contents are finished, +/// and a destination reached through a symlink is replaced through the link rather than in place of +/// it. Both are decisions the user made about their own file, and replacing its contents is no +/// reason to undo either. The permissions are a snapshot taken immediately before the rename rather +/// than a lock: nothing here excludes a concurrent `chmod`, so a mode set between that snapshot and +/// the rename is still lost. +/// +/// # Errors +/// +/// Returns the reason when the directory, the temporary file, the rename or the durability of +/// either fails. A temporary left behind by a failed write is removed, because the alternative is +/// litter beside the report with no one to clean it up. +pub fn write(path: &Utf8Path, contents: &str) -> Result<()> { + // Resolved before anything is staged, so that the rename lands on what the name points at and + // the staging file is a sibling of the file the rename will actually touch. A dangling link + // still has a destination: creating through it must preserve the link rather than replace it. + let destination = crate::paths::physical(path)?; + create_parents(&destination)?; + + replace(path, &destination, write_bytes(contents)) +} + +/// Writes a file whole or not at all by streaming it through `fill`, creating parent directories as +/// needed. +/// +/// The streaming counterpart to [`write`], for a document large enough that holding the finished +/// string on top of what produced it is the memory spike worth avoiding — the report and the +/// self-contained page. `fill` writes into the staging file; the atomic write-then-rename contract, +/// the durability barrier and the error context are exactly [`write`]'s. +/// +/// # Errors +/// +/// Returns the reason when the directory, the temporary file, `fill`, the rename or the durability +/// of either fails. A temporary left behind by a failed write is removed. +pub(crate) fn write_streamed(path: &Utf8Path, fill: impl FnOnce(&mut dyn io::Write) -> io::Result<()>) -> Result<()> { + let destination = crate::paths::physical(path)?; + create_parents(&destination)?; + + replace(path, &destination, fill) +} + +/// Atomically replaces `path` only when it still holds `expected`. +/// +/// The lock belongs to the original workspace's stable cargo-gamma cache, so every command editing +/// that workspace shares one lock domain even when its reusable build state was redirected with +/// `--cache-dir`. +/// +/// [`Publication::Conflict`] means the destination did not hold `expected` at the final comparison; +/// no destination bytes were replaced. That comparison happens after staging. Portable Rust has no +/// inode- or generation-aware compare-and-replace primitive, so a non-cooperating process can +/// replace the destination in the syscall interval between the comparison and rename and be +/// overwritten. The directory lock closes that interval only for writers using this API. +pub(crate) fn write_if_unchanged(workspace: &Utf8Path, path: &Utf8Path, expected: Option<&str>, contents: &str) -> Result { + let destination = crate::paths::physical(path)?; + create_parents(&destination)?; + let _lock = crate::exec::claim_workspace(workspace)?; + let scratch = scratch_path(&destination); + + stage(&scratch, write_bytes(contents), Some(&destination)).map_err(|cause| discard(&scratch, path, cause))?; + before_publication(&scratch); + + if !matches_contents(&destination, expected).map_err(|cause| error!("could not check `{path}` before replacing it").caused_by(cause))? { + remove_staging(&scratch, path)?; + return Ok(Publication::Conflict); + } + + after_comparison(&destination); + fs::rename(scratch.as_std_path(), destination.as_std_path()).map_err(|cause| discard(&scratch, path, cause))?; + + match published(&destination) { + Ok(()) => Ok(Publication::Published), + Err(cause) => Ok(Publication::PublishedUndurable(error!("could not write `{path}`").caused_by(cause))), + } +} + +/// Removes `path` only when it still holds `expected`. +/// +/// This is the rollback counterpart to [`write_if_unchanged`]. It shares the same lock and final +/// generation check, so a failed transaction cannot delete a later successful writer's generation. +pub(crate) fn remove_if_unchanged(workspace: &Utf8Path, path: &Utf8Path, expected: &str) -> Result { + let destination = crate::paths::physical(path)?; + let _lock = crate::exec::claim_workspace(workspace)?; + + before_publication(&destination); + + if !matches_contents(&destination, Some(expected)) + .map_err(|cause| error!("could not check `{path}` before removing it").caused_by(cause))? + { + return Ok(Publication::Conflict); + } + + fs::remove_file(destination.as_std_path()).map_err(|cause| error!("could not remove `{path}`").caused_by(cause))?; + + match published(&destination) { + Ok(()) => Ok(Publication::Published), + Err(cause) => Ok(Publication::PublishedUndurable( + error!("could not remove `{path}`").caused_by(cause), + )), + } +} + +/// Whether a conditional publication reached the destination name. +/// +/// A directory sync follows the rename, so it can fail after the new generation is visible. +/// Callers that compensate source edits must record that generation before propagating +/// [`Self::PublishedUndurable`], otherwise their rollback loses track of an edit that happened. +#[derive(Debug)] +pub(crate) enum Publication { + /// The validated generation was gone before the rename or removal. + Conflict, + + /// The new generation was published and its directory was synced. + Published, + + /// The new generation was published but its directory could not be synced. + PublishedUndurable(crate::error::Error), +} + +/// Replaces the resolved destination after all checks that protect this publication have passed. +fn replace(path: &Utf8Path, destination: &Utf8Path, fill: impl FnOnce(&mut dyn io::Write) -> io::Result<()>) -> Result<()> { + let scratch = scratch_path(destination); + + stage(&scratch, fill, Some(destination)).map_err(|cause| discard(&scratch, path, cause))?; + before_publication(&scratch); + + fs::rename(scratch.as_std_path(), destination.as_std_path()).map_err(|cause| discard(&scratch, path, cause))?; + + published(destination).map_err(|cause| error!("could not write `{path}`").caused_by(cause)) +} + +/// Writes a file at a name that must be free, whole or not at all, creating parent directories as +/// needed. +/// +/// Returns whether it was written: `false` means the name was already taken and nothing on disk +/// was touched, which is a decision for the caller rather than a failure — the file that is there +/// may be one somebody wrote by hand. +/// +/// Publishing is a link from the staged copy rather than a rename onto the name, because a rename +/// replaces whatever is already there and the whole point here is that it must not. It is also not +/// a create-then-write into the final name: a write that fails part-way through would leave a +/// corrupt file under the name a retry then refuses to overwrite, which is the failure this exists +/// to rule out. Linking publishes bytes that are already complete and already durable, in one step +/// that either takes the free name or reports that it was not free — and a failure before that +/// step leaves no file under the final name at all. The new directory entry is then made durable +/// in its own right, because this is the path that prints "Wrote `gamma.toml`" and a file +/// the user was told about has to still be there after a power cut — on Unix, which is as far as +/// `published` can promise. +/// +/// # Errors +/// +/// Returns the reason when the directory, the temporary file, the link or the durability of the +/// link fails. +pub fn publish(path: &Utf8Path, contents: &str) -> Result { + create_parents(path)?; + + let scratch = scratch_path(path); + + stage(&scratch, write_bytes(contents), None).map_err(|cause| discard(&scratch, path, cause))?; + + let linked = match fs::hard_link(scratch.as_std_path(), path.as_std_path()) { + Ok(()) => true, + + Err(cause) if cause.kind() == ErrorKind::AlreadyExists => false, + + Err(cause) => return Err(discard(&scratch, path, cause)), + }; + + let durability = linked.then(|| published(path)).transpose(); + + // Said out loud rather than swallowed: the file under the final name is correct either way, so + // this is not a failed publish, but the leftover sits beside it under a hidden name that + // nothing else will ever mention. + if let Err(cause) = fs::remove_file(scratch.as_std_path()) { + crate::notes::note(format!("`{scratch}` was left behind and could not be removed: {cause}")); + } + + if let Err(cause) = durability { + return Err(error!("could not write `{path}`").caused_by(cause)); + } + + Ok(linked) +} + +/// Creates the directories `path` will be written into. +fn create_parents(path: &Utf8Path) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent.as_std_path()).map_err(|cause| error!("could not create `{parent}`").caused_by(cause))?; + } + + Ok(()) +} + +/// Whether the path still names exactly the bytes its caller validated. +fn matches_contents(path: &Utf8Path, expected: Option<&str>) -> io::Result { + match fs::read(path.as_std_path()) { + Ok(actual) => Ok(expected.is_some_and(|expected| actual == expected.as_bytes())), + Err(cause) if cause.kind() == ErrorKind::NotFound => Ok(expected.is_none()), + Err(cause) => Err(cause), + } +} + +/// Removes an unpublishable staged file, preserving the conflict as the caller's result. +fn remove_staging(scratch: &Utf8Path, path: &Utf8Path) -> Result<()> { + match fs::remove_file(scratch.as_std_path()) { + Ok(()) => Ok(()), + Err(cause) if cause.kind() == ErrorKind::NotFound => Ok(()), + Err(cause) => Err(error!("`{path}` changed before it could be replaced, and `{scratch}` could not be removed").caused_by(cause)), + } +} + +/// Fills the staging file and makes it durable, taking `mode`'s permissions when there are any. +/// +/// The contents are flushed to the device before the caller publishes them. Without that, the +/// rename or the link can reach the disk ahead of the bytes it names, and a machine that loses +/// power in between comes back with the new name over an empty file — the outcome staging exists +/// to rule out, reached by a different route. +/// +/// The staging file is created exclusively rather than truncated into. A name that is already +/// taken is a name this call has no business writing through: it is either another process's +/// staging file — which truncating would corrupt from both ends — or something planted under a +/// predictable name for this process to write through, which `O_EXCL` also refuses to follow. +/// +/// The destination's permissions are read and applied *after* the contents are written. Reading +/// them last is what keeps the snapshot as close to the rename as it can be without a lock, and +/// applying them last keeps the file out of the state Windows enforces on the next handle open +/// rather than retroactively — where marking the staging file read-only before writing it is a +/// hazard rather than the caution it is on Unix. +/// +/// The contents are produced by `fill` writing into the staging file rather than handed over as a +/// finished string, so a caller with a large document — a report and the page that embeds it — can +/// stream it through a buffered writer instead of materializing the whole thing first. The buffer +/// is flushed before the permissions and the durability barrier, so every byte `fill` wrote is on +/// the file by the time it is synced. +fn stage(scratch: &Utf8Path, fill: impl FnOnce(&mut dyn io::Write) -> io::Result<()>, mode: Option<&Utf8Path>) -> io::Result<()> { + let mut staging = File::create_new(scratch.as_std_path())?; + + { + let mut writer = io::BufWriter::new(&mut staging); + + fill(&mut writer)?; + writer.flush()?; + } + + if let Some(permissions) = mode + .and_then(|path| fs::metadata(path.as_std_path()).ok()) + .map(|metadata| metadata.permissions()) + { + staging.set_permissions(permissions)?; + } + + staging.sync_all() +} + +/// Writes `contents` straight through, the [`stage`] filler for a caller that already holds the +/// finished bytes. +fn write_bytes(contents: &str) -> impl FnOnce(&mut dyn io::Write) -> io::Result<()> + '_ { + move |writer| writer.write_all(contents.as_bytes()) +} + +/// Makes the appearance of `path` under its own name durable, not merely its contents. +/// +/// `sync_all` on the staging file covers the bytes; the rename or the link that gives them their +/// final name is a change to the *parent directory*, and on XFS, on btrfs and on ext4 mounted +/// `data=writeback` that entry can be lost across a power cut even though every byte it names was +/// synced. The caller can report a successful write before that loss, so the directory entry is +/// part of the durability guarantee too. +/// +/// **The guarantee is Unix-only, deliberately, and this is a no-op elsewhere.** Windows offers no +/// equivalent barrier that would cover the operations used here: +/// +/// - `FlushFileBuffers` needs a handle opened for writing, and a directory cannot be opened that +/// way, so the Unix move of syncing the parent has no spelling on Windows at all. +/// - `MoveFileEx` has `MOVEFILE_WRITE_THROUGH`, which does not return "until the file is actually +/// moved on the disk" — but `std::fs::rename` calls `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING` +/// alone, so nothing here requests it, and reaching it would mean an FFI rename of our own. +/// - Even that would cover only [`replace`]. [`publish`] takes its name with `CreateHardLink` and +/// the removal path with `DeleteFile`, and neither has a write-through flag. A barrier on one of +/// the three would leave the guarantee looking whole while two paths still lacked it, which is +/// worse than the honest gap. +/// +/// So on Windows the bytes are durable and the directory entry naming them is at NTFS's discretion. +/// NTFS journals metadata, so the entry is far more likely to survive than on the Unix filesystems +/// named above, but that is a property of the filesystem rather than a promise this code obtained. +#[cfg(unix)] +fn published(path: &Utf8Path) -> io::Result<()> { + #[cfg(test)] + if take_directory_sync_failure() { + return Err(io::Error::other("injected directory sync failure")); + } + + let parent = path.parent().filter(|parent| !parent.as_str().is_empty()); + + File::open(parent.unwrap_or_else(|| Utf8Path::new(".")).as_std_path())?.sync_all() +} + +/// Always succeeds off Unix: see the Unix spelling for why there is nothing to ask for there. +#[cfg(all(not(unix), test))] +fn published(_path: &Utf8Path) -> io::Result<()> { + if take_directory_sync_failure() { + return Err(io::Error::other("injected directory sync failure")); + } + + Ok(()) +} + +#[cfg(all(not(unix), not(test)))] +#[expect(clippy::unnecessary_wraps, reason = "the Unix spelling is fallible, and the two must agree")] +const fn published(_path: &Utf8Path) -> io::Result<()> { + Ok(()) +} + +/// Removes the staging file and reports why the write it held could not be completed. +/// +/// A failure to remove it is folded into the message rather than dropped. The leftover is beside +/// the destination under a name nothing else mentions, so a caller told only about the first +/// failure would never learn it is there. +fn discard(scratch: &Utf8Path, path: &Utf8Path, cause: io::Error) -> crate::error::Error { + let failed = error!("could not write `{path}`").caused_by(cause); + + match fs::remove_file(scratch.as_std_path()) { + Ok(()) => failed, + + Err(removal) if removal.kind() == ErrorKind::NotFound => failed, + + Err(removal) => error!("{failed}; and `{scratch}` could not be removed either: {removal}"), + } +} + +/// Where the contents of `path` are staged before being published onto it. +/// +/// A per-process counter gives each call a different name, while random entropy separates pid +/// namespaces sharing a filesystem. A pid is unique only within its namespace, and two containers +/// sharing a bind-mounted workspace can both start their agent at pid 1. The exclusive create in +/// [`stage`] is a final defence against the astronomically unlikely cross-process collision. +pub(crate) fn scratch_path(path: &Utf8Path) -> Utf8PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + + #[cfg(test)] + if let Some(scratch) = TEST_SCRATCH.with(|next| next.borrow_mut().take()) { + return scratch; + } + + let name = path.file_name().unwrap_or("report"); + let invocation = NEXT.fetch_add(1, Ordering::Relaxed); + + // The counter gives every call in this process a distinct name. The random portion separates + // pid namespaces sharing a filesystem, where two unrelated processes can both be pid 1 and + // make the same sequence of calls. + let mut hasher = RandomState::new().build_hasher(); + + hasher.write_u32(std::process::id()); + hasher.write_u64(invocation); + + path.with_file_name(format!(".{name}.{}.{invocation}.{:016x}.tmp", std::process::id(), hasher.finish())) +} + +#[cfg(test)] +type PublicationHook = Box; + +#[cfg(test)] +thread_local! { + static BEFORE_PUBLICATION: RefCell> = const { RefCell::new(None) }; + static AFTER_COMPARISON: RefCell> = const { RefCell::new(None) }; + static TEST_SCRATCH: RefCell> = const { RefCell::new(None) }; + static DIRECTORY_SYNC_FAILURE: Cell = const { Cell::new(false) }; +} + +/// Runs `hook` after this thread has staged its next replacement and before it is published. +/// +/// Kept thread-local so concurrent tests can force an interleaving without becoming an +/// interleaving themselves. Production builds have no hook. +#[cfg(test)] +pub(crate) fn before_next_publication(hook: impl FnOnce(&Utf8Path) + 'static) { + BEFORE_PUBLICATION.with(|next| *next.borrow_mut() = Some(Box::new(hook))); +} + +/// Runs `hook` after the next conditional comparison and before its rename. +#[cfg(test)] +fn after_next_comparison(hook: impl FnOnce(&Utf8Path) + 'static) { + AFTER_COMPARISON.with(|next| *next.borrow_mut() = Some(Box::new(hook))); +} + +/// Uses `scratch` for this thread's next staged write. +/// +/// This is an error-path seam: a directory at that path makes exclusive creation fail without +/// relying on permissions or free disk space. +#[cfg(test)] +pub(crate) fn next_scratch_path(scratch: Utf8PathBuf) { + TEST_SCRATCH.with(|next| *next.borrow_mut() = Some(scratch)); +} + +/// Makes the next post-rename directory sync fail on this test thread. +/// +/// The rename still completes, which lets command tests exercise the distinction between a +/// published generation and a durability failure without depending on a filesystem fault. +#[cfg(test)] +pub(crate) fn fail_next_directory_sync() { + DIRECTORY_SYNC_FAILURE.with(|next| next.set(true)); +} + +#[cfg(test)] +fn take_directory_sync_failure() -> bool { + DIRECTORY_SYNC_FAILURE.with(|next| next.replace(false)) +} + +#[cfg(test)] +fn before_publication(path: &Utf8Path) { + let hook = BEFORE_PUBLICATION.with(|next| next.borrow_mut().take()); + + if let Some(hook) = hook { + hook(path); + } +} + +#[cfg(not(test))] +const fn before_publication(_path: &Utf8Path) {} + +#[cfg(test)] +fn after_comparison(path: &Utf8Path) { + let hook = AFTER_COMPARISON.with(|next| next.borrow_mut().take()); + + if let Some(hook) = hook { + hook(path); + } +} + +#[cfg(not(test))] +const fn after_comparison(_path: &Utf8Path) {} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::process::Command; + use std::sync::{Arc, Barrier, Mutex}; + use std::thread; + + use super::*; + + const EDITOR_PATH: &str = "CARGO_GAMMA_PUBLICATION_EDITOR_PATH"; + + fn edit_in_child(path: &Utf8Path) { + let status = Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--exact", + "elements::publication::tests::conditional_publication_has_a_deterministic_external_editor_boundary", + "--nocapture", + ]) + .env(EDITOR_PATH, path) + .status() + .expect("external editor process"); + + assert!(status.success(), "{status}"); + } + + #[test] + fn writing_a_report_creates_parent_directories() { + let directory = crate::testing::workdir("elements-write"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + let parent = root.join("nested").join("deeper"); + let path = parent.join("report.json"); + + // Report paths are often nested artifact locations; the caller should not have to create + // the directory tree separately. + write(&path, "{}").expect("write report"); + + assert!(parent.is_dir(), "the parent directory was not created"); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("read report"), "{}"); + } + + /// The streaming writer publishes exactly the bytes its filler produced, so a report streamed + /// through it reaches its name whole rather than in the pieces the filler wrote it in. + #[test] + fn a_streamed_write_publishes_the_streamed_bytes_whole() { + let directory = crate::testing::workdir("elements-stream-ok-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + let path = root.join("nested").join("report.json"); + + write_streamed(&path, |writer| { + writer.write_all(b"{\"schemaVersion\":")?; + writer.write_all(b"\"2\"}") + }) + .expect("stream report"); + + assert_eq!( + fs::read_to_string(path.as_std_path()).expect("published bytes"), + "{\"schemaVersion\":\"2\"}" + ); + } + + /// A filler that fails part-way is the streaming form of a staging failure: the previous file is + /// left byte for byte, no partial document is published under its name, and the staging sibling + /// is cleaned up. This is the atomic contract [`write`] upholds, reached through the writer form. + #[test] + fn a_streamed_write_that_fails_midway_leaves_the_previous_file_and_no_staging_litter() { + let directory = crate::testing::workdir("elements-stream-fail-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + let path = root.join("report.json"); + + fs::write(path.as_std_path(), "the original").expect("seed the destination"); + + let error = write_streamed(&path, |writer| { + writer.write_all(b"half a document")?; + + Err(io::Error::other("the filler gave up")) + }) + .expect_err("a filler failure must surface"); + + assert!(error.to_string().contains("could not write"), "{error}"); + assert!(error.to_string().contains("the filler gave up"), "{error}"); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("previous bytes"), "the original"); + + let entries: Vec = fs::read_dir(root.as_std_path()) + .expect("read the directory") + .map(|entry| entry.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + + assert_eq!(entries, ["report.json"], "the staging sibling must be cleaned up: {entries:?}"); + } + + /// A failed directory sync is not a failed publication: the bytes already reached their final + /// name. Callers need that distinction to register source rollback before returning the + /// durability error. + #[test] + fn a_conditional_replacement_reports_post_rename_sync_failure_as_published() { + let directory = crate::testing::workdir("elements-conditional-sync-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("utf-8"); + let path = root.join("source.rs"); + + fs::write(path.as_std_path(), "before").expect("source"); + fail_next_directory_sync(); + + let publication = write_if_unchanged(&root, &path, Some("before"), "after").expect("the rename itself succeeds"); + + assert!(matches!(publication, Publication::PublishedUndurable(_)), "{publication:?}"); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("published bytes"), "after"); + assert!( + !root.join(".source.rs.cargo-gamma.lock").exists(), + "publication locks belong in the external workspace cache" + ); + } + + /// Removing the final name has the same post-publication durability boundary as replacing it. + #[test] + fn a_conditional_removal_reports_post_unlink_sync_failure_as_published() { + let directory = crate::testing::workdir("elements-conditional-remove-sync-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("utf-8"); + let path = root.join("source.rs"); + + fs::write(path.as_std_path(), "before").expect("source"); + fail_next_directory_sync(); + + let publication = remove_if_unchanged(&root, &path, "before").expect("the removal itself succeeds"); + + assert!(matches!(publication, Publication::PublishedUndurable(_)), "{publication:?}"); + assert!(!path.exists(), "the name was removed before the directory sync failed"); + } + + #[test] + fn conditional_publication_has_a_deterministic_external_editor_boundary() { + if let Some(path) = std::env::var_os(EDITOR_PATH) { + fs::write(path, "editor").expect("external editor writes"); + return; + } + + let directory = crate::testing::workdir("elements-external-editor-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("utf-8"); + let path = root.join("source.rs"); + + fs::write(path.as_std_path(), "before").expect("source"); + let edited = path.clone(); + before_next_publication(move |_| edit_in_child(&edited)); + + let conflict = write_if_unchanged(&root, &path, Some("before"), "gamma").expect("comparison"); + + assert!(matches!(conflict, Publication::Conflict), "{conflict:?}"); + assert_eq!(fs::read_to_string(&path).expect("editor bytes"), "editor"); + + fs::write(path.as_std_path(), "before").expect("reset source"); + let edited = path.clone(); + after_next_comparison(move |_| edit_in_child(&edited)); + + let published = write_if_unchanged(&root, &path, Some("before"), "gamma").expect("publication"); + + assert!(matches!(published, Publication::Published), "{published:?}"); + assert_eq!( + fs::read_to_string(&path).expect("published bytes"), + "gamma", + "the API must not claim to preserve non-cooperating edits made after its comparison" + ); + } + + #[test] + fn a_publish_sync_failure_removes_its_staging_file() { + let directory = crate::testing::workdir("elements-publish-sync-failure-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("utf-8"); + let path = root.join("gamma.toml"); + + fail_next_directory_sync(); + + let error = publish(&path, "jobs = 2\n").expect_err("directory sync must fail"); + let entries: Vec<_> = fs::read_dir(&root) + .expect("directory") + .map(|entry| entry.expect("entry").file_name()) + .collect(); + + assert!(error.to_string().contains("injected directory sync failure"), "{error}"); + assert_eq!(fs::read_to_string(&path).expect("published destination"), "jobs = 2\n"); + assert_eq!(entries, vec![path.file_name().expect("name")]); + } + + /// The visible file changes from one whole version to the next, with nothing partial in + /// between and nothing left beside it. + /// + /// A CI job parsing the JSON report cannot tell a truncated file from a short one, so a write + /// interrupted part-way has to leave the previous report rather than a broken one. Staging in + /// a sibling and renaming is what buys that; this asserts the observable half of it — the + /// destination is never the staging file, and the staging file does not survive the write. + #[test] + fn a_report_is_written_whole_rather_than_streamed_into_its_destination() { + let dir = tempfile::TempDir::new().expect("temp"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("nested").join("report.json"); + + write(&path, "{\"first\":true}").expect("first write"); + write(&path, "{\"second\":true}").expect("second write"); + + assert_eq!(fs::read_to_string(path.as_std_path()).expect("read"), "{\"second\":true}"); + + let leftovers: Vec = fs::read_dir(path.parent().expect("parent").as_std_path()) + .expect("read dir") + .map(|entry| entry.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + + assert_eq!(leftovers, vec!["report.json".to_owned()], "the staging file must not survive"); + } + + /// A write that cannot be completed leaves the previous file exactly as it was. + /// + /// The rename is the only step that can be observed, so making it fail — by pointing it at a + /// directory, which no platform will replace with a file — is what exercises the failure path + /// that the old truncate-then-stream writer had no answer for. + #[test] + fn a_write_that_cannot_be_completed_leaves_the_previous_file_alone() { + let dir = tempfile::TempDir::new().expect("temp"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("report.json"); + + fs::create_dir(path.as_std_path()).expect("a destination the rename cannot replace"); + + assert!(write(&path, "{}").is_err(), "the failure must be reported rather than swallowed"); + assert!(path.as_std_path().is_dir(), "the destination must be untouched"); + + let leftovers = fs::read_dir(root.as_std_path()).expect("read dir").count(); + + assert_eq!(leftovers, 1, "a failed rename must not leave its staging file behind"); + } + + #[test] + fn a_path_with_no_parent_at_all_skips_directory_creation() { + // An empty path has nothing above it to create, and asking `create_dir_all` to create + // nothing would either do nothing useful or fail for a reason that has nothing to do with + // the report itself. Skipping straight to the write is what keeps that irrelevant failure + // out of the caller's way. + let path = Utf8PathBuf::new(); + + assert!(path.parent().is_none()); + assert!(write(&path, "{}").is_err()); + } + + /// A write that fails before it publishes leaves the previous contents byte for byte. + /// + /// Staging is the only step that can be made to fail from a test without a full disk, and + /// blocking it stands in for every way the bytes can fail to arrive — which is the case the old + /// truncate-then-stream writer answered by destroying the file first and hoping. + #[test] + fn a_write_that_cannot_be_staged_leaves_the_previous_contents_alone() { + let dir = crate::testing::workdir("elements-staging-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("report.json"); + + fs::write(path.as_std_path(), "{\"original\":true}").expect("the original"); + let scratch = root.join(".blocked-stage"); + fs::create_dir(scratch.as_std_path()).expect("something the staging file cannot be"); + next_scratch_path(scratch); + + let error = write(&path, "{\"replacement\":true}").expect_err("the write must fail"); + + assert!(error.to_string().contains("report.json"), "{error}"); + assert_eq!( + fs::read_to_string(path.as_std_path()).expect("read"), + "{\"original\":true}", + "the previous contents were not left alone" + ); + } + + /// A file this tool rewrites is the user's, and so are its permissions: a mode of 0o600 on a + /// source file is a decision, and editing the file is no reason to hand it back as 0o644. + #[cfg(unix)] + #[test] + fn a_replaced_file_keeps_the_permissions_it_had() { + use std::os::unix::fs::PermissionsExt as _; + + let dir = crate::testing::workdir("elements-permissions-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("source.rs"); + + fs::write(path.as_std_path(), "fn f() {}\n").expect("the original"); + fs::set_permissions(path.as_std_path(), fs::Permissions::from_mode(0o640)).expect("mode"); + + write(&path, "fn g() {}\n").expect("write"); + + let mode = fs::metadata(path.as_std_path()).expect("metadata").permissions().mode() & 0o777; + + assert_eq!(mode, 0o640, "the file came back with different permissions"); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("read"), "fn g() {}\n"); + } + + /// Replacing the contents of a linked file must replace what the link points at. Renaming onto + /// the link itself would delete the link and leave an unrelated file in its place — and if the + /// link crosses a filesystem, the rename would not even be permitted. + #[cfg(unix)] + #[test] + fn a_write_through_a_symlink_replaces_what_it_points_at() { + let dir = crate::testing::workdir("elements-symlink-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let target = root.join("real.rs"); + let link = root.join("link.rs"); + + fs::write(target.as_std_path(), "fn f() {}\n").expect("the original"); + std::os::unix::fs::symlink(target.as_std_path(), link.as_std_path()).expect("symlink"); + + write(&link, "fn g() {}\n").expect("write"); + + assert!( + fs::symlink_metadata(link.as_std_path()).expect("metadata").file_type().is_symlink(), + "the link was replaced by a file" + ); + assert_eq!(fs::read_to_string(target.as_std_path()).expect("read"), "fn g() {}\n"); + } + + /// A dangling link still names the destination that a write must create. Falling back to the + /// unresolved link name would replace the link itself during the final rename. + #[cfg(unix)] + #[test] + fn a_write_through_a_dangling_symlink_preserves_the_link_and_creates_its_target() { + let dir = crate::testing::workdir("elements-dangling-symlink-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let target = root.join("created").join("real.rs"); + let link = root.join("link.rs"); + + std::os::unix::fs::symlink("created/real.rs", link.as_std_path()).expect("symlink"); + + write(&link, "fn g() {}\n").expect("write"); + + assert!( + fs::symlink_metadata(link.as_std_path()).expect("metadata").file_type().is_symlink(), + "the dangling link was replaced" + ); + assert_eq!(fs::read_to_string(target.as_std_path()).expect("target"), "fn g() {}\n"); + } + + /// Publishing takes a free name and reports, rather than takes, a name that is not free. + #[test] + fn publishing_takes_a_free_name_and_leaves_a_taken_one_alone() { + let dir = crate::testing::workdir("elements-publish-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("nested").join("gamma.toml"); + + assert!(publish(&path, "jobs = 2\n").expect("publish"), "a free name must be taken"); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("read"), "jobs = 2\n"); + + assert!( + !publish(&path, "jobs = 9\n").expect("publish"), + "a name that is taken must be reported, not overwritten" + ); + assert_eq!( + fs::read_to_string(path.as_std_path()).expect("read"), + "jobs = 2\n", + "the file that was already there was overwritten" + ); + + let leftovers: Vec = fs::read_dir(path.parent().expect("parent").as_std_path()) + .expect("read dir") + .map(|entry| entry.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + + assert_eq!(leftovers, vec!["gamma.toml".to_owned()], "the staging file must not survive"); + } + + /// A publish that fails leaves nothing at all under the final name. + /// + /// This is the whole reason the bytes are staged first. A file created under the final name and + /// then written into would, on a failure, leave a corrupt file that the retry refuses to + /// overwrite — the migration would be permanently stuck behind its own wreckage. + #[test] + fn a_publish_that_cannot_be_staged_leaves_no_file_to_block_the_retry() { + let dir = crate::testing::workdir("elements-publish-failure-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("gamma.toml"); + let scratch = root.join(".blocked-stage"); + + fs::create_dir(scratch.as_std_path()).expect("something the staging file cannot be"); + next_scratch_path(scratch.clone()); + + let error = publish(&path, "jobs = 2\n").expect_err("the publish must fail"); + + assert!(error.to_string().contains("gamma.toml"), "{error}"); + assert!(!path.as_std_path().exists(), "a failed publish left the final name taken"); + + // And once whatever blocked it is gone, the same call succeeds: nothing about the failure + // is sticky. + fs::remove_dir(scratch.as_std_path()).expect("clear the obstruction"); + assert!(publish(&path, "jobs = 2\n").expect("retry")); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("read"), "jobs = 2\n"); + } + + /// A staging file that cannot be removed is named, because nothing else will ever mention it. + #[test] + fn a_staging_file_that_cannot_be_removed_is_reported_with_the_failure() { + let dir = crate::testing::workdir("elements-discard-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("report.json"); + let scratch = root.join(".unremovable-stage"); + + // A directory cannot be removed by `remove_file`, so this stands in for a staging file the + // process cannot clean up after itself. + fs::create_dir(scratch.as_std_path()).expect("an unremovable staging path"); + next_scratch_path(scratch.clone()); + + let error = write(&path, "{}").expect_err("the write must fail"); + + assert!(error.to_string().contains("could not be removed"), "{error}"); + assert!(error.to_string().contains(scratch.file_name().expect("name")), "{error}"); + } + + /// A staging name that is already taken is refused rather than written through. + /// + /// The name is a sibling of the destination and is derived rather than negotiated, so the only + /// thing standing between two writers of one destination is that the name distinguishes them. + /// If it ever does not — two pid namespaces over one bind mount, or a name planted in a + /// world-writable directory — truncating into it publishes one writer's bytes over the other's + /// under a name a CI job then parses. Refusing costs a failed write; taking it over costs a + /// report that is not valid JSON. + #[test] + fn a_staging_name_that_is_already_taken_is_refused_rather_than_written_through() { + let dir = crate::testing::workdir("elements-exclusive-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("report.json"); + + fs::write(path.as_std_path(), "{\"original\":true}").expect("the original"); + let scratch = root.join(".taken-stage"); + fs::write(scratch.as_std_path(), "somebody else's staging file").expect("the staging file"); + next_scratch_path(scratch); + + let error = write(&path, "{\"replacement\":true}").expect_err("the write must fail"); + + assert!(error.to_string().contains("report.json"), "{error}"); + assert_eq!( + fs::read_to_string(path.as_std_path()).expect("read"), + "{\"original\":true}", + "the destination was published from a staging file this call did not create" + ); + } + + /// The staging name carries entropy of its own, not only the process id. + /// + /// A pid is unique within a pid namespace; the name has to be unique on a filesystem, and two + /// containers sharing a bind-mounted workspace both start at pid 1. + #[test] + fn a_staging_name_is_not_derived_from_the_process_id_alone() { + let path = Utf8Path::new("/w/report.json"); + let first = scratch_path(path).file_name().expect("a name").to_owned(); + let second = scratch_path(path).file_name().expect("a name").to_owned(); + + assert_ne!(first, second, "two invocations must not share a staging path"); + assert_ne!(first, format!(".report.json.{}.tmp", std::process::id())); + assert!(first.contains(&std::process::id().to_string()), "{first}"); + assert!(second.contains(&std::process::id().to_string()), "{second}"); + } + + /// Three writers stop after staging, so every staging path is simultaneously live. A shared + /// path would make the second writer remove the first's staged bytes on its error path; unique + /// names let all three publish only bytes they staged themselves. + #[test] + fn concurrent_writers_neither_remove_nor_publish_another_writers_staging_file() { + let dir = crate::testing::workdir("elements-concurrent-staging-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("report.json"); + let barrier = Arc::new(Barrier::new(3)); + let staged = Arc::new(Mutex::new(Vec::new())); + let mut writers = Vec::new(); + + for contents in ["first", "second", "third"] { + let barrier = Arc::clone(&barrier); + let staged = Arc::clone(&staged); + let path = path.clone(); + + writers.push(thread::spawn(move || { + before_next_publication(move |scratch| { + assert_eq!(fs::read_to_string(scratch).expect("staged bytes"), contents); + staged.lock().expect("staged paths").push(scratch.to_path_buf()); + let _ = barrier.wait(); + }); + + write(&path, contents) + })); + } + + for writer in writers { + writer.join().expect("writer panicked").expect("write"); + } + + let mut staged = staged.lock().expect("staged paths").clone(); + + staged.sort(); + staged.dedup(); + + assert_eq!(staged.len(), 3, "every invocation must own its staging path"); + assert!(["first", "second", "third"].contains(&fs::read_to_string(path).expect("published bytes").as_str())); + } + + /// The rename is only as durable as the directory that records it. + /// + /// The crash itself cannot be staged from a test, so what is pinned here is that the parent is + /// opened and synced at all, and that a parent it cannot open is reported rather than passed + /// off as a completed write. + #[cfg(unix)] + #[test] + fn the_directory_a_published_name_lives_in_is_synced_too() { + let dir = crate::testing::workdir("elements-durable-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("report.json"); + + write(&path, "{}").expect("write"); + published(&path).expect("the parent of a written file is syncable"); + + assert!(published(&root.join("absent").join("report.json")).is_err()); + + // A bare file name has a parent of `""`, which names no directory at all: the process's own + // directory is what its rename touched. + published(Utf8Path::new("report.json")).expect("a relative name syncs the working directory"); + } + + /// A file this tool rewrites keeps the permissions it has when the contents are finished. + /// + /// The mode is read as late as it can be, so that a change made during the write is carried + /// rather than reverted. It is applied after the bytes rather than before them, which on Unix + /// costs a microsecond of exposure under the default creation mode and on Windows is the + /// difference between a read-only marking that the open handle predates and one it does not. + #[cfg(unix)] + #[test] + fn a_read_only_destination_is_replaced_and_stays_read_only() { + use std::os::unix::fs::PermissionsExt as _; + + let dir = crate::testing::workdir("elements-read-only-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + let path = root.join("report.json"); + + fs::write(path.as_std_path(), "{\"original\":true}").expect("the original"); + fs::set_permissions(path.as_std_path(), fs::Permissions::from_mode(0o444)).expect("mode"); + + write(&path, "{\"replacement\":true}").expect("write"); + + let mode = fs::metadata(path.as_std_path()).expect("metadata").permissions().mode() & 0o777; + + assert_eq!(mode, 0o444, "the file came back with different permissions"); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("read"), "{\"replacement\":true}"); + } +} diff --git a/crates/cargo-gamma-lib/src/elements/report.rs b/crates/cargo-gamma-lib/src/elements/report.rs new file mode 100644 index 000000000..d6d37065a --- /dev/null +++ b/crates/cargo-gamma-lib/src/elements/report.rs @@ -0,0 +1,1837 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The `mutation-testing-elements` report document. +//! +//! The schema is a published artifact of another project, so the mapping is spelled out rather +//! than left implicit: drift is silent and shows up as a blank page in someone's browser rather +//! than as a failing build. + +use core::fmt::{Display, Write as _}; +use std::collections::BTreeMap; +use std::{fs, io}; + +use camino::Utf8Path; +use compact_str::CompactString; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::discover::Plan; +use crate::error::error; +use crate::model::{Mutant, Outcome}; +use crate::parse::SourceFile; +use crate::{HashMap, HashSet, Result}; + +/// The schema version we emit. +/// +/// The version string validates against `^([1-2])(\.(([1-9]\d*)|0)){0,2}$` — major 1 and 2 only — +/// even though the npm package that defines it is at 3.x. Emitting "3" fails validation for a +/// reason that looks like version skew and is not. +const SCHEMA_VERSION: &str = "2"; + +/// The highest schema major version a cross-run reader will act on. +/// +/// Kept separate from [`SCHEMA_VERSION`] because the two answer different questions: one is what we +/// write, the other is what we are willing to read. A build that starts emitting a newer version +/// still has to read the reports its predecessors wrote. +pub(super) const SUPPORTED_SCHEMA_MAJOR: u32 = 2; + +/// The `framework.name` this tool writes, and the only one a cross-run reader accepts. +pub(super) const FRAMEWORK_NAME: &str = "cargo-gamma"; + +type SchemaResult = core::result::Result; + +/// A whole mutation test result, the root of the report document. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Report { + /// The schema version this document claims to conform to. + pub schema_version: String, + + /// The score bands the viewer colors by. + pub thresholds: Thresholds, + + /// Absolute path the file keys are relative to. + #[serde(skip_serializing_if = "Option::is_none")] + pub project_root: Option, + + /// What produced the report. + pub framework: Framework, + + /// One entry per mutated file, keyed by workspace-relative path. + pub files: HashMap, + + /// Free-form run metadata. + /// + /// The schema declares this "free-format", which is what makes it the right home for the shard + /// identity and the run time. `merge` needs both, and inventing a sidecar file for them would + /// mean a report artifact that is only half the story. + #[serde(skip_serializing_if = "Option::is_none")] + pub config: Option, +} + +/// What `merge` needs to know about the run that produced a report. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RunInfo { + /// When the run started, in seconds since the Unix epoch. + /// + /// Seconds rather than a formatted timestamp because every use is arithmetic — freshness, + /// ordering, windowing — and parsing a date format back is a step that can only lose. + pub started_at: u64, + + /// Whether this document combines reports rather than describing one run. + /// + /// A merged document is not a complete population snapshot even though it has no shard + /// identity: it can combine an incomplete rotation and reports from different revisions. + #[serde(default, skip_serializing_if = "is_false")] + pub merged: bool, + + /// The shard this run covered, when it was sharded. + #[serde(skip_serializing_if = "Option::is_none")] + pub shard: Option, + + /// How many tests the baseline ran, when a harness announced a count. + /// + /// Carried in the report because it is the figure that says whether the suite ran at all, and + /// a score computed over a suite that ran nothing is the most dangerous number this tool can + /// produce. The console shows it too, but the console is not there in CI. + #[serde(skip_serializing_if = "Option::is_none")] + pub tests: Option, + + /// How many mutants belonged to a build this run gave up on. + /// + /// These export as `Ignored`, which they share with deliberate suppressions, because the + /// statuses that would name them better — `NoCoverage`, `RuntimeError` — sit outside the + /// schema's denominator and would make the viewer's score disagree with the printed one. That + /// is the disagreement the whole `NotBuilt` outcome exists to remove, so the distinction is + /// carried here instead of being pushed into the status. + /// + /// A reader wanting to know which *particular* mutants they were has `NOT_BUILT_PREFIX`; this + /// is so that they can be counted without walking every mutant in the report. Filled in by + /// [`build`], never by the caller, so it cannot drift from the mutants beside it. + #[serde(skip_serializing_if = "Option::is_none")] + pub not_built: Option, + + /// Test packages the preflight dropped so that the tree would check at all. + /// + /// Empty in the ordinary run, and omitted from the document when it is. When it is not, some + /// package the caller never asked to mutate does not compile, and rather than refuse to run, + /// the tool narrowed the build to the packages being mutated and went ahead. + /// + /// It is in the report because it changes how the report reads. Those packages' test targets + /// were neither built nor run, so a mutant one of them would have killed appears here as a + /// survivor — a gap in this run's oracle wearing the clothes of a gap in the suite. Anyone + /// comparing this run against one taken over the whole workspace needs to know that before + /// they compare the scores. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dropped_test_packages: Vec, + + /// The original verdict and source generations retained by a merged report. + /// + /// A merged report can carry verdicts from several runs. Keeping only its own publication time + /// would make a later merge treat every one of them as equally fresh, so this free-form config + /// extension preserves the information the merger needs without extending the interchange + /// schema's mutant objects. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub merge_provenance: Option, +} + +/// Per-item provenance retained by a merged report. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MergeProvenance { + /// The report that supplied each rendered file's source and language. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub sources: BTreeMap, + + /// The run that established each retained verdict. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub verdicts: BTreeMap, +} + +/// The source generation a merged file renders. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SourceProvenance { + /// When the source report started. + pub started_at: u64, + + /// The source report's stable merge tie-breaker. + pub origin: String, + + /// A deterministic identity for this source generation. + /// + /// Older merged reports omit this field. The merger derives an identity for those documents + /// from their rendered source so they remain readable. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub lineage: String, +} + +/// The run generation that established a merged mutant's status. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VerdictProvenance { + /// When the verdict's run started. + pub started_at: u64, + + /// The verdict report's stable merge tie-breaker. + pub origin: String, + + /// A deterministic identity for this verdict generation. + /// + /// Older merged reports omit this field. The merger derives an identity for those documents + /// from their retained mutant result so they remain readable. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub lineage: String, +} + +/// Identifies one shard of a rotation. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ShardInfo { + /// Which shard this was, from zero. + pub index: u32, + + /// How many shards the population was divided into. + pub count: u32, +} + +#[expect(clippy::trivially_copy_pass_by_ref, reason = "serde requires a predicate over a reference")] +const fn is_false(value: &bool) -> bool { + !*value +} + +/// The score bands the viewer colors by. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Thresholds { + /// At or above this score the viewer shows green. + pub high: u32, + + /// Below this score the viewer shows red. + pub low: u32, +} + +impl Default for Thresholds { + fn default() -> Self { + Self { high: 80, low: 60 } + } +} + +/// Identifies the tool, which the viewer shows in its header. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Framework { + /// The tool name. + pub name: String, + + /// The tool version. + pub version: String, +} + +/// One mutated file: its full source, and every mutant generated in it. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileResult { + /// The complete source text. The viewer renders this with the mutants overlaid, which is why + /// the report is self-contained and can be opened without the repository. + pub source: String, + + /// The language, used to pick syntax highlighting. + pub language: String, + + /// Every mutant in this file. + pub mutants: Vec, +} + +/// One mutant, in the schema's vocabulary. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MutantResult { + /// Our content-addressed identity. + /// + /// Stryker uses a within-run integer here; the field is a free-form string, and an identity + /// that survives edits elsewhere in the file is strictly more useful in a report someone may + /// compare against last week's. + pub id: CompactString, + + /// The registry name of the mutator, such as `relational.lt_to_le`. + /// + /// The viewer groups and filters by this string, so the naming scheme becomes the UI's facet + /// list at no extra cost. + pub mutator_name: CompactString, + + /// Where the mutated construct is. + pub location: Location, + + /// The verdict, in the schema's closed `PascalCase` vocabulary. + /// + /// Owned rather than `&'static str` so a written report can be read back — which is what + /// `merge` does. The closed-enum guarantee is kept at the only place it can be violated, the + /// mapping in `status_of`, and asserted against the vendored schema by a conformance test. + pub status: CompactString, + + /// The replacement source text. + #[serde(skip_serializing_if = "Option::is_none")] + pub replacement: Option, + + /// A human sentence describing the change. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + + /// Why the status is what it is: a suppression reason, or the test that killed it. + #[serde(skip_serializing_if = "Option::is_none")] + pub status_reason: Option, + + /// Wall time spent on this mutant, in milliseconds. + /// + /// The interchange schema permits any JSON number, rather than only whole milliseconds. + /// Keeping that precision is necessary when a report produced by another implementation is + /// merged and written again. + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + + /// The test that killed it, when one did. + #[serde(skip_serializing_if = "Option::is_none")] + pub killed_by: Option>, +} + +/// A half-open source range, in the schema's one-based line and column terms. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Location { + /// Inclusive start. + pub start: Position, + + /// Exclusive end. + pub end: Position, +} + +/// A one-based line and column. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Position { + /// One-based line. + pub line: usize, + + /// One-based column. + pub column: usize, +} + +/// Opens the `statusReason` of a mutant its memory ceiling stopped. +/// +/// The schema has no out-of-memory status, so the verdict is exported as `Survived` and this +/// prefix preserves the resource outcome. `statusReason` is a free-form string in the schema, so +/// carrying the distinction here costs the schema nothing, where a field of our own invention +/// would be an extension a strict validator could reject. +pub(super) const OUT_OF_MEMORY_PREFIX: &str = "out of memory: "; + +/// Marks a timed-out mutant exported as schema `Survived`. +/// +/// The schema treats its `Timeout` status as detected. Gamma does not: a resource limit observed +/// the mutant, but no assertion rejected it. The prefix preserves the distinct verdict while the +/// `Survived` status keeps schema consumers' score aligned with Gamma's. +pub(super) const TIMEOUT_PREFIX: &str = "timed out: "; + +/// Marks a mutant the build gave up on in `statusReason`, for the same reason +/// [`OUT_OF_MEMORY_PREFIX`] exists. +/// +/// `NotBuilt` exports as `Ignored`, which is also what a deliberate suppression exports as, and the +/// two ask a reader to do opposite things: one is work somebody chose not to do, the other is work +/// this run could not do. The per-mutant count is in `config.notBuilt`; this is what lets a reader +/// tell which of the two *one* mutant was, and what stops incremental execution from freezing a mutant nobody +/// ever judged into every later run as though it had been suppressed. +pub(super) const NOT_BUILT_PREFIX: &str = "not built: "; + +/// Marks a flake in `statusReason`, for the same reason [`OUT_OF_MEMORY_PREFIX`] exists. +/// +/// A flake exports as `Ignored`, which it shares with suppressions and with mutants the build never +/// compiled. Those three ask the reader to do quite different things, and this prefix is what tells +/// them apart once the report has left the machine that produced it. +pub(super) const FLAKY_PREFIX: &str = "flaky: "; + +/// Maps a verdict onto the schema's closed status enum. +/// +/// The schema treats `Timeout` as detected, so Gamma exports both resource-exhaustion outcomes as +/// `Survived` with reason prefixes. They remain in the denominator without entering the numerator, +/// and the prefixes preserve the verdicts the schema cannot represent directly. +const fn status_of(outcome: Outcome) -> &'static str { + match outcome { + Outcome::Pending => "Pending", + Outcome::Killed => "Killed", + Outcome::Survived | Outcome::Timeout | Outcome::OutOfMemory => "Survived", + Outcome::CompileError => "CompileError", + // A mutant the build never compiled is `Ignored` rather than `NoCoverage`: both are real + // options here, but `NoCoverage` is in the schema's denominator and would lower the score + // the viewer shows relative to the printed one, which is the disagreement this whole + // outcome exists to remove. + // A flake exports as `Ignored` for the same reason `NotBuilt` does: the run established + // nothing about this mutant, and every other status the schema offers would state something + // it did not find. The reason string carries the test that failed both ways. + Outcome::Ignored | Outcome::NotBuilt | Outcome::Flaky => "Ignored", + Outcome::NoCoverage => "NoCoverage", + } +} + +/// Explains a verdict in one sentence, for the viewer's detail pane. +fn reason_for(mutant: &Mutant) -> Option { + if let Some(suppression) = mutant.suppression.as_ref() { + let mut text = format!("suppressed by {}", suppression.channel.as_str()); + + if let Some(reason) = suppression.reason.as_ref() { + text.push_str(": "); + text.push_str(reason); + } + + if let Some(tag) = suppression.tag.as_ref() { + let _ = write!(text, " [#{tag}]"); + } + + return Some(text); + } + + match mutant.outcome { + Outcome::Killed => mutant + .killed_by + .as_ref() + .map(|test| format!("failed `{test}`")) + .or_else(|| mutant.note.clone()), + Outcome::CompileError => Some("the mutant does not compile".to_owned()), + // `NotBuilt` and a suppression both export as `Ignored`, so without this the reader cannot + // tell a mutant the run gave up on from one that was deliberately skipped. + Outcome::NotBuilt => Some(format!( + "{NOT_BUILT_PREFIX}{}", + mutant + .note + .clone() + .unwrap_or_else(|| "the build this mutant belonged to could not be converged".to_owned()) + )), + Outcome::Timeout => Some(format!( + "{TIMEOUT_PREFIX}{}", + mutant.note.clone().unwrap_or_else(|| "the test run exceeded its budget".to_owned()) + )), + // Exports as `Ignored`, like a suppression and like `NotBuilt`, so without the note a + // reader of the report cannot tell a flake from a mutant somebody deliberately skipped — + // and the note is where the test to fix is named. + Outcome::Flaky => Some(format!( + "{FLAKY_PREFIX}{}", + mutant + .note + .clone() + .unwrap_or_else(|| "a test failed with no mutant active as well as with one".to_owned()) + )), + // The sweep already built a note saying how far past the ceiling the run went, and dropping + // it here would leave the reader a bare `Survived` with nothing to explain the distinct + // resource outcome that was reached. + Outcome::OutOfMemory => Some(format!( + "{OUT_OF_MEMORY_PREFIX}{}", + mutant + .note + .clone() + .unwrap_or_else(|| "the test run exceeded the memory this run allowed it".to_owned()) + )), + Outcome::NoCoverage => Some( + "no selected runtime test reached this mutation site; coverage reports may exclude this code or include other configurations" + .to_owned(), + ), + _ => None, + } +} + +/// Builds the report document for a completed plan. +/// +/// Every mutated file's full source is embedded, because a report that needs the repository beside +/// it to be readable cannot be attached to a CI run or mailed to someone. +pub fn build(plan: &Plan, thresholds: Thresholds, run: Option) -> Result { + let mut files: HashMap = HashMap::default(); + + // Grouped once rather than rescanned per file: a workspace with many files has many mutants + // too, so the pairing is quadratic in exactly the case it needs not to be. + let mut grouped: HashMap<&Utf8Path, Vec<&Mutant>> = HashMap::default(); + + for mutant in &plan.mutants { + grouped.entry(&*mutant.file).or_default().push(mutant); + } + + for file in &plan.files { + let Some(mutants) = grouped.get(file.path.as_path()) else { + continue; + }; + + let original = fs::read_to_string(file.absolute.as_std_path()) + .map_err(|cause| error!("could not read `{}`", file.absolute).caused_by(cause))?; + let has_bom = original.starts_with('\u{feff}'); + let source = SourceFile::parse(file.absolute.clone(), original.clone())?; + let rendered = mutants + .iter() + .map(|mutant| render_with_first_line_offset(mutant, &source, usize::from(has_bom))) + .collect(); + + let _ = files.insert( + file.path.to_string(), + FileResult { + source: original, + language: "rust".to_owned(), + mutants: rendered, + }, + ); + } + + let not_built = plan.mutants.iter().filter(|mutant| mutant.outcome == Outcome::NotBuilt).count(); + + Ok(Report { + schema_version: SCHEMA_VERSION.to_owned(), + thresholds, + project_root: Some(plan.root.to_string()), + framework: Framework { + name: FRAMEWORK_NAME.to_owned(), + version: env!("CARGO_PKG_VERSION").to_owned(), + }, + files, + config: run.map(|run| RunInfo { + not_built: (not_built > 0).then_some(not_built), + ..run + }), + }) +} + +/// Whether a schema version has the form and major version this crate supports. +#[must_use] +pub(crate) fn supported_schema_version(version: &str) -> bool { + let mut parts = version.split('.'); + let Some(major) = parts.next() else { + return false; + }; + + if !matches!(major, "1" | "2") { + return false; + } + + let mut components = 1; + + for part in parts { + components += 1; + + if components > 3 || part.is_empty() || (part.len() > 1 && part.starts_with('0')) || !part.bytes().all(|byte| byte.is_ascii_digit()) + { + return false; + } + } + + true +} + +/// Validates a document against the adopted mutation-testing-elements schema. +/// +/// The schema permits extension fields, so this checks every field it defines and deliberately +/// leaves unknown fields alone. That lets an external producer add metadata without letting a +/// malformed known field silently become valid merely because this crate does not use it. +pub(crate) fn validate_schema(document: &Value) -> SchemaResult<()> { + let report = object(document, "report")?; + + validate_schema_version(required(report, "schemaVersion", "report")?, "report.schemaVersion")?; + validate_thresholds(required(report, "thresholds", "report")?, "report.thresholds")?; + validate_files(required(report, "files", "report")?, "report.files")?; + + if let Some(config) = report.get("config") { + let _config = object(config, "report.config")?; + } + + if let Some(root) = report.get("projectRoot") { + let _ = string(root, "report.projectRoot")?; + } + + if let Some(framework) = report.get("framework") { + validate_framework(framework, "report.framework")?; + } + + if let Some(performance) = report.get("performance") { + validate_performance(performance, "report.performance")?; + } + + if let Some(test_files) = report.get("testFiles") { + validate_test_files(test_files, "report.testFiles")?; + } + + if let Some(system) = report.get("system") { + validate_system(system, "report.system")?; + } + + Ok(()) +} + +fn object(value: &Value, path: impl Display) -> SchemaResult<&serde_json::Map> { + value.as_object().ok_or_else(|| format!("{path} must be an object")) +} + +fn array(value: &Value, path: impl Display) -> SchemaResult<&Vec> { + value.as_array().ok_or_else(|| format!("{path} must be an array")) +} + +fn required<'value>(object: &'value serde_json::Map, name: &str, path: impl Display) -> SchemaResult<&'value Value> { + object.get(name).ok_or_else(|| format!("{path} is missing required field `{name}`")) +} + +fn string(value: &Value, path: impl Display) -> SchemaResult<&str> { + value.as_str().ok_or_else(|| format!("{path} must be a string")) +} + +fn number(value: &Value, path: impl Display) -> SchemaResult<()> { + if value.is_number() { + Ok(()) + } else { + Err(format!("{path} must be a number")) + } +} + +fn boolean(value: &Value, path: impl Display) -> SchemaResult<()> { + if value.is_boolean() { + Ok(()) + } else { + Err(format!("{path} must be a boolean")) + } +} + +fn integer(value: &Value, path: impl Display) -> SchemaResult { + value.as_u64().ok_or_else(|| format!("{path} must be a non-negative integer")) +} + +fn validate_schema_version(value: &Value, path: &str) -> SchemaResult<()> { + let version = string(value, path)?; + + if supported_schema_version(version) { + Ok(()) + } else { + Err(format!("schema version `{version}` at {path} must match the supported pattern")) + } +} + +fn validate_thresholds(value: &Value, path: &str) -> SchemaResult<()> { + let thresholds = object(value, path)?; + + for name in ["high", "low"] { + let field = format!("{path}.{name}"); + let threshold = integer(required(thresholds, name, path)?, &field)?; + + if threshold > 100 { + return Err(format!("{field} must be at most 100")); + } + } + + Ok(()) +} + +fn validate_files(value: &Value, path: &str) -> SchemaResult<()> { + for (name, file) in object(value, path)? { + validate_file(file, &format!("{path}[{name:?}]"))?; + } + + Ok(()) +} + +fn validate_file(value: &Value, path: &str) -> SchemaResult<()> { + let file = object(value, path)?; + + let _ = string(required(file, "language", path)?, format_args!("{path}.language"))?; + let _ = string(required(file, "source", path)?, format_args!("{path}.source"))?; + validate_mutants(required(file, "mutants", path)?, &format!("{path}.mutants")) +} + +fn validate_mutants(value: &Value, path: &str) -> SchemaResult<()> { + let mutants = array(value, path)?; + let mut unique = HashSet::default(); + let mut encoded = Vec::new(); + + for (index, mutant) in mutants.iter().enumerate() { + let path = format!("{path}[{index}]"); + encoded.clear(); + serde_json::to_writer(&mut encoded, mutant).map_err(|cause| format!("{path} could not be compared: {cause}"))?; + + if !unique.insert(blake3::hash(&encoded)) { + return Err(format!("{path} duplicates another mutant")); + } + + validate_mutant(mutant, &path)?; + } + + Ok(()) +} + +fn validate_mutant(value: &Value, path: &str) -> SchemaResult<()> { + let mutant = object(value, path)?; + + let id = string(required(mutant, "id", path)?, format_args!("{path}.id"))?; + let _ = string(required(mutant, "mutatorName", path)?, format_args!("{path}.mutatorName"))?; + validate_location(required(mutant, "location", path)?, format_args!("{path}.location"), false)?; + + let status = string(required(mutant, "status", path)?, format_args!("{path}.status"))?; + + if !matches!( + status, + "Killed" | "Survived" | "NoCoverage" | "CompileError" | "RuntimeError" | "Timeout" | "Ignored" | "Pending" + ) { + return Err(format!("{path} mutant `{id}` has unknown schema status `{status}`")); + } + + for name in ["description", "replacement", "statusReason"] { + if let Some(value) = mutant.get(name) { + let _ = string(value, format_args!("{path}.{name}"))?; + } + } + + for name in ["duration", "testsCompleted"] { + if let Some(value) = mutant.get(name) { + number(value, format_args!("{path}.{name}"))?; + } + } + + for name in ["coveredBy", "killedBy"] { + if let Some(value) = mutant.get(name) { + validate_strings(value, format_args!("{path}.{name}"))?; + } + } + + if let Some(value) = mutant.get("static") { + boolean(value, format_args!("{path}.static"))?; + } + + Ok(()) +} + +fn validate_strings(value: &Value, path: impl Display + Copy) -> SchemaResult<()> { + for (index, value) in array(value, path)?.iter().enumerate() { + let _ = string(value, format_args!("{path}[{index}]"))?; + } + + Ok(()) +} + +fn validate_location(value: &Value, path: impl Display + Copy, open_end: bool) -> SchemaResult<()> { + let location = object(value, path)?; + validate_position(required(location, "start", path)?, format_args!("{path}.start"))?; + + if open_end { + if let Some(end) = location.get("end") { + validate_position(end, format_args!("{path}.end"))?; + } + } else { + validate_position(required(location, "end", path)?, format_args!("{path}.end"))?; + } + + Ok(()) +} + +fn validate_position(value: &Value, path: impl Display + Copy) -> SchemaResult<()> { + let position = object(value, path)?; + + for name in ["line", "column"] { + if integer(required(position, name, path)?, format_args!("{path}.{name}"))? == 0 { + return Err(format!("{path}.{name} must be at least 1")); + } + } + + Ok(()) +} + +fn validate_test_files(value: &Value, path: &str) -> SchemaResult<()> { + for (name, file) in object(value, path)? { + let file_path = format!("{path}[{name:?}]"); + let file = object(file, &file_path)?; + + if let Some(source) = file.get("source") { + let _ = string(source, format_args!("{file_path}.source"))?; + } + + for (index, test) in array(required(file, "tests", &file_path)?, format_args!("{file_path}.tests"))? + .iter() + .enumerate() + { + let test_path = format!("{file_path}.tests[{index}]"); + let test = object(test, &test_path)?; + + let _ = string(required(test, "id", &test_path)?, format_args!("{test_path}.id"))?; + let _ = string(required(test, "name", &test_path)?, format_args!("{test_path}.name"))?; + + if let Some(location) = test.get("location") { + validate_location(location, format_args!("{test_path}.location"), true)?; + } + } + } + + Ok(()) +} + +fn validate_performance(value: &Value, path: &str) -> SchemaResult<()> { + let performance = object(value, path)?; + + for name in ["setup", "initialRun", "mutation"] { + number(required(performance, name, path)?, format_args!("{path}.{name}"))?; + } + + Ok(()) +} + +fn validate_framework(value: &Value, path: &str) -> SchemaResult<()> { + let framework = object(value, path)?; + + let _ = string(required(framework, "name", path)?, format_args!("{path}.name"))?; + + if let Some(version) = framework.get("version") { + let _ = string(version, format_args!("{path}.version"))?; + } + + if let Some(branding) = framework.get("branding") { + let branding_path = format!("{path}.branding"); + let branding = object(branding, &branding_path)?; + let homepage = string( + required(branding, "homepageUrl", &branding_path)?, + format_args!("{branding_path}.homepageUrl"), + )?; + + if !is_uri(homepage) { + return Err(format!("{branding_path}.homepageUrl must be a URI")); + } + + if let Some(image) = branding.get("imageUrl") { + let _ = string(image, format_args!("{branding_path}.imageUrl"))?; + } + } + + if let Some(dependencies) = framework.get("dependencies") { + let dependencies_path = format!("{path}.dependencies"); + + for (name, version) in object(dependencies, &dependencies_path)? { + let _ = string(version, format_args!("{dependencies_path}[{name:?}]"))?; + } + } + + Ok(()) +} + +fn is_uri(value: &str) -> bool { + let Some((scheme, rest)) = value.split_once(':') else { + return false; + }; + + if scheme.is_empty() + || !scheme + .bytes() + .enumerate() + .all(|(index, byte)| byte.is_ascii_alphabetic() || (index > 0 && (byte.is_ascii_digit() || matches!(byte, b'+' | b'-' | b'.')))) + { + return false; + } + + let bytes = rest.as_bytes(); + let mut index = 0; + + while let Some(&byte) = bytes.get(index) { + if byte == b'%' { + if !bytes.get(index + 1).is_some_and(u8::is_ascii_hexdigit) || !bytes.get(index + 2).is_some_and(u8::is_ascii_hexdigit) { + return false; + } + + index += 3; + } else { + if byte.is_ascii_control() || matches!(byte, b' ' | b'"' | b'<' | b'>' | b'\\' | b'^' | b'`' | b'{' | b'|' | b'}') { + return false; + } + + index += 1; + } + } + + true +} + +fn validate_system(value: &Value, path: &str) -> SchemaResult<()> { + let system = object(value, path)?; + boolean(required(system, "ci", path)?, format_args!("{path}.ci"))?; + + if let Some(os) = system.get("os") { + let os_path = format!("{path}.os"); + let os = object(os, &os_path)?; + let _ = string(required(os, "platform", &os_path)?, format_args!("{os_path}.platform"))?; + + for name in ["description", "version"] { + if let Some(value) = os.get(name) { + let _ = string(value, format_args!("{os_path}.{name}"))?; + } + } + } + + if let Some(cpu) = system.get("cpu") { + let cpu_path = format!("{path}.cpu"); + let cpu = object(cpu, &cpu_path)?; + number(required(cpu, "logicalCores", &cpu_path)?, format_args!("{cpu_path}.logicalCores"))?; + + if let Some(clock) = cpu.get("baseClock") { + number(clock, format_args!("{cpu_path}.baseClock"))?; + } + + if let Some(model) = cpu.get("model") { + let _ = string(model, format_args!("{cpu_path}.model"))?; + } + } + + if let Some(ram) = system.get("ram") { + let ram_path = format!("{path}.ram"); + let ram = object(ram, &ram_path)?; + number(required(ram, "total", &ram_path)?, format_args!("{ram_path}.total"))?; + } + + Ok(()) +} + +/// Converts one mutant into its schema form. +#[cfg(test)] +fn render(mutant: &Mutant, source: &SourceFile) -> MutantResult { + render_with_first_line_offset(mutant, source, 0) +} + +#[expect( + clippy::cast_precision_loss, + reason = "the interchange schema represents durations as JSON numbers, while a run measures milliseconds in u64" +)] +fn render_with_first_line_offset(mutant: &Mutant, source: &SourceFile, first_line_offset: usize) -> MutantResult { + let (start_line, start_column) = source.location(mutant.span.start); + let (end_line, end_column) = source.location(mutant.span.end); + + MutantResult { + id: mutant.id.clone(), + mutator_name: CompactString::new(&mutant.mutator), + location: Location { + start: Position { + line: start_line, + column: start_column + usize::from(start_line == 1) * first_line_offset, + }, + end: Position { + line: end_line, + column: end_column + usize::from(end_line == 1) * first_line_offset, + }, + }, + status: CompactString::new(status_of(mutant.outcome)), + replacement: Some(mutant.replacement.clone()), + description: Some(mutant.summary()), + status_reason: reason_for(mutant), + duration: (mutant.elapsed_ms > 0).then_some(mutant.elapsed_ms as f64), + killed_by: mutant.killed_by.clone().map(|test| vec![test]), + } +} + +/// Serializes the report as pretty-printed JSON. +pub fn to_json(report: &Report) -> Result { + let document = serde_json::to_value(report).map_err(|cause| error!("could not serialize the report").caused_by(cause))?; + + validate_schema(&document).map_err(|cause| error!("could not serialize the report: {cause}"))?; + + serde_json::to_string_pretty(&document).map_err(|cause| error!("could not serialize the report").caused_by(cause)) +} + +/// Writes the report to `path` as pretty-printed JSON. +/// +/// Streams through a validated `serde_json::Value` for deterministic key ordering, then +/// directly to the writer without building a full `String` in memory. +pub fn write_json(report: &Report, path: &Utf8Path) -> Result<()> { + let document = serde_json::to_value(report).map_err(|cause| error!("could not serialize the report").caused_by(cause))?; + + validate_schema(&document).map_err(|cause| error!("could not serialize the report: {cause}"))?; + + crate::elements::write_streamed(path, |writer| { + serde_json::to_writer_pretty(writer, &document).map_err(io::Error::from) + }) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use core::ops::Range; + use std::borrow::Cow; + + use camino::Utf8PathBuf; + + use super::super::digest::{Digest, settled_mutants, settled_verdict}; + use super::*; + use crate::discover::TargetFile; + use crate::fixtures; + use crate::model::{Channel, Suppression}; + + fn mutant(outcome: Outcome, span: Range) -> Mutant { + Mutant { + id: "abc123abc123".to_owned().into(), + span, + original: "a < b".to_owned().into(), + replacement: "(a) <= (b)".to_owned().into(), + outcome, + ..fixtures::mutant() + } + } + + #[test] + fn every_verdict_maps_onto_the_closed_enum() { + // The schema's status list is closed, so a verdict we invent a name for would fail + // validation in the viewer rather than here. + const VALID: [&str; 7] = [ + "Killed", + "Survived", + "NoCoverage", + "CompileError", + "RuntimeError", + "Timeout", + "Ignored", + ]; + + // Every non-`Pending` verdict is pinned to the *exact* schema status it exports as, not + // merely to some valid one: a membership check survives swapping `Killed`→`Survived` or + // `NoCoverage`→`Ignored`, because the replacement is itself valid. `OutOfMemory`, `Flaky`, + // and `NotBuilt` have no status of their own and are folded onto the closest one the schema + // does have; getting any of these wrong silently changes the score a reader sees in the + // viewer relative to the printed one. + for (outcome, status) in [ + (Outcome::Killed, "Killed"), + (Outcome::Survived, "Survived"), + (Outcome::Timeout, "Survived"), + (Outcome::OutOfMemory, "Survived"), + (Outcome::CompileError, "CompileError"), + (Outcome::NoCoverage, "NoCoverage"), + (Outcome::Ignored, "Ignored"), + (Outcome::Flaky, "Ignored"), + (Outcome::NotBuilt, "Ignored"), + ] { + assert_eq!(status_of(outcome), status, "{outcome} maps onto the wrong status"); + assert!(VALID.contains(&status), "{status} is not one of the schema's statuses"); + } + + // A never-judged mutant carries the schema's `Pending`, which sits outside the closed set + // of resolved statuses above. + assert_eq!(status_of(Outcome::Pending), "Pending"); + } + + #[test] + fn the_schema_version_is_in_the_supported_range() { + // The npm package is at 3.x but the schema only validates major 1 and 2. + assert_eq!(SCHEMA_VERSION, "2"); + } + + #[test] + fn branding_homepage_uris_follow_the_schema_format() { + for uri in [ + "https://example.test/gamma%20report", + "data:image/png;base64,AAAA", + "mailto:maintainers@example.test", + ] { + assert!(is_uri(uri), "{uri} should be a URI"); + } + + for value in ["not a URI", "https://example.test/a space", "https://example.test/%xz"] { + assert!(!is_uri(value), "{value} must not be a URI"); + } + } + + #[test] + fn a_span_becomes_a_one_based_half_open_location() { + let source = SourceFile::parse("src/lib.rs", "fn f() {\n a < b\n}\n".to_owned()).expect("parses"); + let start = source.text.find("a <").expect("present"); + let rendered = render(&mutant(Outcome::Survived, start..start + 5), &source); + + assert_eq!(rendered.location.start.line, 2); + assert_eq!(rendered.location.start.column, 5); + assert_eq!(rendered.location.end.line, 2); + assert_eq!(rendered.location.end.column, 10); + } + + #[test] + fn a_bom_embedded_in_report_source_offsets_first_line_columns() { + let source = SourceFile::parse("src/lib.rs", "\u{feff}fn f() {}".to_owned()).expect("parses"); + let rendered = render_with_first_line_offset(&mutant(Outcome::Survived, 0..2), &source, 1); + + assert_eq!(rendered.location.start.column, 2); + assert_eq!(rendered.location.end.column, 4); + } + + #[test] + fn compact_report_fields_remain_json_strings() { + let source = SourceFile::parse("src/lib.rs", "fn f() { a < b; }".to_owned()).expect("parses"); + let rendered = render(&mutant(Outcome::Survived, 9..14), &source); + let json = serde_json::to_value(rendered).expect("serializes"); + + assert_eq!(json["id"], "abc123abc123"); + assert_eq!(json["mutatorName"], "relational.lt_to_le"); + assert_eq!(json["status"], "Survived"); + assert_eq!(json["replacement"], "(a) <= (b)"); + } + + #[test] + fn a_killing_test_is_named_in_the_status_reason() { + let mut subject = mutant(Outcome::Killed, 0..1); + + subject.killed_by = Some("tests::the_boundary".to_owned()); + + assert_eq!(reason_for(&subject), Some("failed `tests::the_boundary`".to_owned())); + assert_eq!( + render(&subject, &SourceFile::parse("src/lib.rs", "fn f() {}".to_owned()).expect("parses")).killed_by, + Some(vec!["tests::the_boundary".to_owned()]) + ); + } + + /// A killed mutant whose killer arrived only as a `note` still explains itself. + /// + /// A merged or foreign report can carry a kill with no structured `killed_by` — the name of the + /// failing test survives only in the free-text note. The `.or_else` fallback in `reason_for` is + /// what keeps that explanation; deleting it leaves the viewer a bare "Killed" with an empty + /// `statusReason`, and the reader with no test to look at. + #[test] + fn a_killed_mutant_without_a_named_test_falls_back_to_its_note() { + let mut subject = mutant(Outcome::Killed, 0..1); + + subject.killed_by = None; + subject.note = Some("failed X".to_owned()); + + assert_eq!(reason_for(&subject), Some("failed X".to_owned())); + } + + #[test] + fn an_uncovered_mutant_explains_why_coverage_can_still_be_complete() { + let reason = reason_for(&mutant(Outcome::NoCoverage, 0..1)).expect("a reason"); + + assert!(reason.contains("no selected runtime test"), "{reason}"); + assert!(reason.contains("coverage reports"), "{reason}"); + } + + #[test] + fn unviable_and_timeout_mutants_explain_their_status() { + let mut timed_out = mutant(Outcome::Timeout, 0..1); + + // These verdicts are not self-explanatory in the report viewer, so the reason field + // distinguishes a compile failure from a budget overrun. + assert_eq!( + reason_for(&mutant(Outcome::CompileError, 0..1)), + Some("the mutant does not compile".to_owned()) + ); + assert_eq!( + reason_for(&timed_out), + Some(format!("{TIMEOUT_PREFIX}the test run exceeded its budget")) + ); + + timed_out.note = Some("stalled, last test named was `slow_case`".to_owned()); + + assert_eq!( + reason_for(&timed_out), + Some(format!("{TIMEOUT_PREFIX}stalled, last test named was `slow_case`")) + ); + } + + #[test] + fn a_suppression_carries_its_reason_and_tag_into_the_report() { + // This is what makes suppressions auditable at a glance in the viewer, rather than a + // silent hole in the population. + let mut subject = mutant(Outcome::Ignored, 0..1); + + subject.suppression = Some(Suppression { + channel: Channel::Comment, + reason: Some("fixed-point math".to_owned()), + tag: Some("perf".to_owned()), + line: Some(4), + }); + + assert_eq!( + reason_for(&subject), + Some("suppressed by comment: fixed-point math [#perf]".to_owned()) + ); + } + + #[test] + fn a_suppression_with_no_reason_or_tag_still_names_its_channel() { + // A suppression directive is never required to carry a reason or a tag, and the report + // must not invent either one: naming the channel alone is the honest answer when that is + // all the directive said. + let mut subject = mutant(Outcome::Ignored, 0..1); + + subject.suppression = Some(Suppression { + channel: Channel::Comment, + reason: None, + tag: None, + line: Some(4), + }); + + assert_eq!(reason_for(&subject), Some("suppressed by comment".to_owned())); + } + + #[test] + fn an_untimed_mutant_omits_its_duration() { + // Emitting `"duration": 0` for a mutant that never ran would show up in the viewer as a + // suspiciously fast result rather than as no result. + let source = SourceFile::parse("src/lib.rs", "fn f() {}".to_owned()).expect("parses"); + + assert_eq!(render(&mutant(Outcome::Ignored, 0..1), &source).duration, None); + + let mut timed = mutant(Outcome::Killed, 0..1); + + timed.elapsed_ms = 12; + + assert_eq!(render(&timed, &source).duration, Some(12.0)); + } + + #[test] + fn only_settled_mutants_are_carried_forward() { + // A survivor has to be retried, because the next run's tests may kill it. A killed mutant + // never will be, so rerunning it is pure cost. + let text = r#"{ + "schemaVersion": "2", + "thresholds": { "high": 80, "low": 60 }, + "framework": { "name": "cargo-gamma", "version": "0.1.0" }, + "files": { + "src/lib.rs": { + "language": "rust", + "source": "src/lib.rs", + "mutants": [ + { "id": "a", "mutatorName": "m", "status": "Killed", "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } } }, + { "id": "b", "mutatorName": "m", "status": "Survived", "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } } }, + { "id": "c", "mutatorName": "m", "status": "Timeout", "statusReason": "the test run exceeded its budget", "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } } }, + { "id": "d", "mutatorName": "m", "status": "NoCoverage", "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } } } + ] + } + } + }"#; + + let settled = settled_mutants(text).expect("parses"); + let mut ids: Vec<&str> = settled.iter().map(String::as_str).collect(); + + ids.sort_unstable(); + assert_eq!(ids, vec!["a", "c"]); + } + + /// A status whose meaning lives in a reason the document does not carry settles nothing. + /// + /// `Ignored` is written for a suppression, for a flake, and for a mutant the build gave up on; + /// `Timeout` is written for both the clock and the memory ceiling. The reason is the only thing + /// that tells them apart, so a verdict without one has not said which it is — and settling it + /// would be choosing whichever reading happens to close the mutant. + #[test] + fn a_status_with_no_reason_to_disambiguate_it_is_not_settled() { + assert_eq!(settled_verdict("Ignored", None), None); + assert_eq!(settled_verdict("Timeout", None), None); + + assert_eq!(settled_verdict("Ignored", Some("suppressed by attribute")), Some(Outcome::Ignored)); + assert_eq!( + settled_verdict("Timeout", Some("the test run exceeded its budget")), + Some(Outcome::Timeout) + ); + + // Every `Ignored` and `Timeout` this tool writes carries a reason, so the rule costs a + // report of our own nothing. + for outcome in [Outcome::Ignored, Outcome::Timeout, Outcome::NotBuilt, Outcome::Flaky] { + let mut subject = mutant(outcome, 0..1); + + if outcome == Outcome::Ignored { + subject.suppression = Some(Suppression { + channel: Channel::Attribute, + reason: None, + tag: None, + line: None, + }); + } + + assert!( + reason_for(&subject).is_some(), + "{outcome:?} exports with no reason to disambiguate it" + ); + } + } + + /// A report another tool wrote, or one from a schema this build predates, is refused. + /// + /// The format is shared, so a foreign document parses into [`Digest`] perfectly well and its + /// statuses are then read with *this* tool's meanings — meanings that rest on a `statusReason` + /// convention no schema imposes. + #[test] + fn a_report_this_tool_did_not_write_settles_nothing() { + let document = |framework: &str, version: &str| { + format!( + r#"{{ + "schemaVersion": "{version}", + "thresholds": {{ "high": 80, "low": 60 }}, + "framework": {{ "name": "{framework}", "version": "0.1.0" }}, + "files": {{ + "src/lib.rs": {{ + "language": "rust", + "source": "fn f() {{}}", + "mutants": [ + {{ "id": "a", "mutatorName": "m", "status": "Killed", "location": {{ "start": {{ "line": 1, "column": 1 }}, "end": {{ "line": 1, "column": 2 }} }} }} + ] + }} + }} + }}"# + ) + }; + + let foreign = settled_mutants(&document("stryker-js", SCHEMA_VERSION)).expect_err("a foreign producer"); + assert!(foreign.contains("stryker-js"), "{foreign}"); + + let ahead = settled_mutants(&document(FRAMEWORK_NAME, "3")).expect_err("an unsupported schema"); + assert!(ahead.contains('3'), "{ahead}"); + + for version in ["later", "2.bad", "2.0.0.0"] { + let unreadable = settled_mutants(&document(FRAMEWORK_NAME, version)).expect_err("an unreadable schema"); + assert!(unreadable.contains(version), "{unreadable}"); + } + + // A minor and a patch component are additive, so our own future reports still read. + let ours = settled_mutants(&document(FRAMEWORK_NAME, "2.1.3")).expect("our own report"); + assert_eq!(ours.len(), 1); + } + + /// A report missing the fields that identify it is refused rather than trusted by default. + #[test] + fn a_report_that_does_not_identify_itself_settles_nothing() { + let text = r#"{ + "files": { + "src/lib.rs": { + "language": "rust", + "source": "fn f() {}", + "mutants": [ + { "id": "a", "mutatorName": "m", "status": "Killed", "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } } } + ] + } + } + }"#; + + let _cause = settled_mutants(text).expect_err("a document that names neither its schema nor its producer"); + } + + /// A cross-run reader borrows what it needs and never copies the source a report embeds. + /// + /// The embedded source is the overwhelming majority of a report — the complete text of every + /// mutated file, kept so a viewer can render it without the repository — and both readers that + /// carry knowledge between runs want only the verdicts. This is the assertion that keeps it that + /// way: decoding through [`Report`] again would compile and pass every other test, while + /// silently allocating the whole document. + #[test] + fn a_digest_borrows_its_verdicts_and_leaves_the_embedded_source_alone() { + let text = r#"{ + "schemaVersion": "2", + "thresholds": { "high": 80, "low": 60 }, + "framework": { "name": "cargo-gamma", "version": "0" }, + "files": { + "src/lib.rs": { + "source": "fn f() {}", + "language": "rust", + "mutants": [ + { "id": "a", "mutatorName": "m", "status": "Killed", + "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } }, + "killedBy": ["tests::one"] } + ] + } + } + }"#; + + let digest: Digest<'_> = serde_json::from_str(text).expect("parses"); + let file = digest.files.get("src/lib.rs").expect("the file"); + let mutant = file.mutants.first().expect("the mutant"); + + assert!( + matches!(mutant.id, Cow::Borrowed(_)), + "an unescaped id was copied out of the document" + ); + assert_eq!(mutant.settled_outcome(), Some(Outcome::Killed)); + } + + #[test] + fn an_out_of_memory_mutant_keeps_its_note_in_the_status_reason() { + // `Survived` keeps the schema's score aligned with Gamma's, while the reason preserves the + // resource outcome the schema cannot represent. + let mut starved = mutant(Outcome::OutOfMemory, 0..1); + let source = SourceFile::parse("src/lib.rs", "fn f() {}".to_owned()).expect("parses"); + + starved.note = Some("`unit-abc` reached 300.0 MB, past the 256.0 MB this run allowed it".to_owned()); + + let rendered = render(&starved, &source); + let json = serde_json::to_string(&rendered).expect("serializes"); + let read: MutantResult = serde_json::from_str(&json).expect("round-trips"); + let reason = read.status_reason.clone().expect("a reason"); + + assert_eq!(read.status, "Survived"); + assert!(reason.starts_with(OUT_OF_MEMORY_PREFIX), "{reason}"); + assert!(reason.contains("256.0 MB"), "{reason}"); + assert_eq!( + settled_verdict(&read.status, read.status_reason.as_deref()), + None, + "a mutant the ceiling stopped was never judged, so a rerun could change it" + ); + } + + /// A flake keeps the test to fix in its status reason, and is told apart from a suppression. + /// + /// The schema has no flaky status, so it exports as `Ignored` — the same status a deliberate + /// skip gets. Without the prefix and the note, a reader of the report would see a mutant + /// somebody chose to ignore, when in fact a test in their suite is unreliable. + #[test] + fn a_flaky_mutant_keeps_the_test_to_fix_in_its_status_reason() { + let mut flake = mutant(Outcome::Flaky, 0..1); + let source = SourceFile::parse("src/lib.rs", "fn f() {}".to_owned()).expect("parses"); + + flake.note = Some("test `a::b` in `unit-abc` fails with no mutant active as well as with one".to_owned()); + + let rendered = render(&flake, &source); + let json = serde_json::to_string(&rendered).expect("serializes"); + let read: MutantResult = serde_json::from_str(&json).expect("round-trips"); + let reason = read.status_reason.clone().expect("a reason"); + + assert_eq!(read.status, "Ignored"); + assert!(reason.starts_with(FLAKY_PREFIX), "{reason}"); + assert!(reason.contains("test `a::b`"), "{reason}"); + assert_eq!( + settled_verdict(&read.status, read.status_reason.as_deref()), + None, + "a flake was never judged, so a rerun could change it" + ); + } + + /// A flake is re-tested by the next run rather than carried forward as settled. + /// + /// `Ignored` is otherwise the most settled status there is, so without this one unreliable test + /// would permanently exclude a mutant that was never judged at all — and the next run is + /// exactly the thing that might judge it. A genuine suppression stays settled. + #[test] + fn a_flake_is_not_settled_but_a_real_suppression_is() { + let mut flake = mutant(Outcome::Flaky, 0..1); + let source = SourceFile::parse("src/lib.rs", "fn f() {}".to_owned()).expect("parses"); + + flake.note = Some("test `a::b` in `unit-abc` fails with no mutant active as well as with one".to_owned()); + + let read: MutantResult = + serde_json::from_str(&serde_json::to_string(&render(&flake, &source)).expect("serializes")).expect("round-trips"); + + assert_eq!( + settled_verdict(&read.status, read.status_reason.as_deref()), + None, + "a flake was never judged, so a rerun could change it" + ); + + let skipped = MutantResult { + status_reason: Some("suppressed by an attribute".to_owned()), + ..read + }; + + assert_eq!( + settled_verdict(&skipped.status, skipped.status_reason.as_deref()), + Some(Outcome::Ignored) + ); + } + + /// A mutant the build gave up on is neither counted as suppressed nor carried forward as one. + /// + /// `Ignored` covers both "we chose not to test this" and "we could not", and the two ask a + /// reader to do opposite things. Without the distinction, incremental execution would freeze a mutant + /// nobody ever judged into every later run as though somebody had decided to skip it. + #[test] + fn a_mutant_the_build_gave_up_on_is_not_settled_but_a_real_suppression_is() { + let mut abandoned = mutant(Outcome::NotBuilt, 0..1); + let source = SourceFile::parse("src/lib.rs", "fn f() {}".to_owned()).expect("parses"); + + abandoned.note = Some("the build this mutant belonged to could not be converged".to_owned()); + + let rendered = render(&abandoned, &source); + + assert_eq!(rendered.status, "Ignored", "the status must stay in the schema's denominator"); + + let read: MutantResult = serde_json::from_str(&serde_json::to_string(&rendered).expect("serializes")).expect("round-trips"); + + assert!( + read.status_reason + .as_deref() + .is_some_and(|reason| reason.starts_with(NOT_BUILT_PREFIX)), + "{:?}", + read.status_reason + ); + assert_eq!( + settled_verdict(&read.status, read.status_reason.as_deref()), + None, + "nothing was established about this mutant" + ); + + let skipped = MutantResult { + status_reason: Some("suppressed by an attribute".to_owned()), + ..read + }; + + assert_eq!( + settled_verdict(&skipped.status, skipped.status_reason.as_deref()), + Some(Outcome::Ignored) + ); + } + + /// A reader can count the mutants a run gave up on without walking every mutant in the report. + #[test] + fn a_report_counts_the_mutants_its_run_never_built() { + let source = "pub fn f(a: i32, b: i32) -> bool { a < b } +"; + let dir = crate::testing::workdir("elements-not-built-"); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf-8"); + + fs::write(root.join("lib.rs").as_std_path(), source).expect("source"); + + let plan = Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: root.clone(), + files: vec![TargetFile { + path: Utf8PathBuf::from("lib.rs"), + absolute: root.join("lib.rs"), + package: "subject".to_owned(), + }], + mutants: vec![ + Mutant { + file: (Utf8PathBuf::from("lib.rs")).into(), + ..mutant(Outcome::NotBuilt, 35..40) + }, + Mutant { + id: "def456def456".to_owned().into(), + file: (Utf8PathBuf::from("lib.rs")).into(), + ..mutant(Outcome::Killed, 35..40) + }, + ], + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + }; + let info = RunInfo { + started_at: 0, + merged: false, + shard: None, + tests: None, + not_built: None, + dropped_test_packages: Vec::new(), + merge_provenance: None, + }; + + let report = build(&plan, Thresholds::default(), Some(info)).expect("a report"); + + assert_eq!(report.config.expect("config").not_built, Some(1)); + + // Nothing to report is left out rather than written as a zero, so an ordinary run's report + // is unchanged. + let clean = Plan { + mutants: vec![Mutant { + file: (Utf8PathBuf::from("lib.rs")).into(), + ..mutant(Outcome::Killed, 35..40) + }], + ..plan + }; + let report = build( + &clean, + Thresholds::default(), + Some(RunInfo { + started_at: 0, + merged: false, + shard: None, + tests: None, + not_built: None, + dropped_test_packages: Vec::new(), + merge_provenance: None, + }), + ) + .expect("a report"); + + assert_eq!(report.config.expect("config").not_built, None); + } + + #[test] + fn an_out_of_memory_mutant_with_no_note_still_says_what_stopped_it() { + let reason = reason_for(&mutant(Outcome::OutOfMemory, 0..1)).expect("a reason"); + + assert!(reason.starts_with(OUT_OF_MEMORY_PREFIX), "{reason}"); + assert!(reason.len() > OUT_OF_MEMORY_PREFIX.len(), "{reason}"); + } + + #[test] + fn a_timeout_exports_as_undetected_without_losing_its_verdict() { + let source = SourceFile::parse("src/lib.rs", "fn f() {}".to_owned()).expect("parses"); + let rendered = render(&mutant(Outcome::Timeout, 0..1), &source); + + assert_eq!(rendered.status, "Survived"); + assert!( + rendered + .status_reason + .as_deref() + .is_some_and(|reason| reason.starts_with(TIMEOUT_PREFIX)) + ); + assert_eq!( + settled_verdict(&rendered.status, rendered.status_reason.as_deref()), + Some(Outcome::Timeout) + ); + } + + #[test] + fn an_out_of_memory_verdict_is_not_settled_but_a_real_timeout_is() { + // The ceiling is inferred from a sampled peak rather than observed, so a spurious verdict + // must not become permanent. A stall is observed, so it stays settled. + let text = r#"{ + "schemaVersion": "2", + "thresholds": { "high": 80, "low": 60 }, + "framework": { "name": "cargo-gamma", "version": "0.1.0" }, + "files": { + "src/lib.rs": { + "language": "rust", + "source": "src/lib.rs", + "mutants": [ + { "id": "stalled", "mutatorName": "m", "status": "Timeout", "statusReason": "stalled, last test named was `slow_case`", "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } } }, + { "id": "starved", "mutatorName": "m", "status": "Timeout", "statusReason": "out of memory: `unit-abc` reached 300.0 MB, past the 256.0 MB this run allowed it", "location": { "start": { "line": 1, "column": 1 }, "end": { "line": 1, "column": 2 } } } + ] + } + } + }"#; + + let settled = settled_mutants(text).expect("parses"); + + assert!(settled.contains("stalled"), "{settled:?}"); + assert!(!settled.contains("starved"), "{settled:?}"); + } + + #[test] + fn a_file_without_mutants_is_left_out_of_the_report() { + let plan = Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: Utf8PathBuf::from("/w"), + files: vec![TargetFile { + path: Utf8PathBuf::from("src/lib.rs"), + absolute: Utf8PathBuf::from("/w/src/lib.rs"), + package: "subject".to_owned(), + }], + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + }; + + let report = build(&plan, Thresholds::default(), None).expect("report builds"); + + // Embedding every selected file would bloat empty reports and try to read files that have + // no result to show. + assert!(report.files.is_empty()); + } + + #[test] + fn an_unparsable_prior_report_is_reported_rather_than_ignored() { + let _cause = settled_mutants("not json").unwrap_err(); + } + + #[test] + fn the_document_serializes_with_the_schema_field_names() { + let report = Report { + schema_version: SCHEMA_VERSION.to_owned(), + thresholds: Thresholds::default(), + project_root: None, + framework: Framework { + name: "cargo-gamma".to_owned(), + version: "0.1.0".to_owned(), + }, + files: HashMap::default(), + config: None, + }; + let json = to_json(&report).expect("serializes"); + + assert!(json.contains("\"schemaVersion\": \"2\""), "{json}"); + assert!(json.contains("\"thresholds\""), "{json}"); + assert!(json.contains("\"files\""), "{json}"); + assert!(!json.contains("projectRoot"), "{json}"); + } + + #[test] + fn serialization_refuses_reports_outside_the_adopted_schema() { + let mut threshold = fixtures::report(); + threshold.thresholds.high = 101; + + assert!(to_json(&threshold).is_err(), "an out-of-range threshold must not be emitted"); + + let mut position = fixtures::report(); + let mut mutant = fixtures::mutant_result(); + mutant.location.start.line = 0; + let _ = position.files.insert( + "src/lib.rs".to_owned(), + FileResult { + source: "fn f() {}\n".to_owned(), + language: "rust".to_owned(), + mutants: vec![mutant], + }, + ); + + assert!(to_json(&position).is_err(), "a zero source position must not be emitted"); + } + + /// The streamed JSON writer emits exactly the bytes the string form produces and publishes them + /// atomically, so moving the report path to streaming cannot change the document a reader gets. + #[test] + fn streamed_json_matches_the_string_form() { + let report = fixtures::report(); + let directory = crate::testing::workdir("elements-stream-json-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + let path = root.join("report.json"); + + write_json(&report, &path).expect("streams the report"); + + let expected = to_json(&report).expect("serializes"); + assert_eq!(fs::read_to_string(path.as_std_path()).expect("published bytes"), expected); + } + + /// A report the schema rejects is refused before a byte reaches the destination, matching the + /// string form's validation and leaving no partial file behind. + #[test] + fn streamed_json_refuses_an_out_of_schema_report_without_writing() { + let mut threshold = fixtures::report(); + threshold.thresholds.high = 101; + let directory = crate::testing::workdir("elements-stream-invalid-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + let path = root.join("report.json"); + + assert!( + write_json(&threshold, &path).is_err(), + "an out-of-range threshold must not be emitted" + ); + assert!(!path.exists(), "a rejected report must leave no file behind"); + } + + /// Every discovered mutant has to reach the report, because the score's denominator is the + /// population the report holds. + /// + /// `build` groups mutants by file and then walks the plan's file list, skipping any group whose + /// file the plan does not list. Nothing else asserts that what came out equals what went in, so + /// emitting one mutant per file — or breaking out of the loop after the first — passes every + /// other test in this module, and the failure it hides is the one the design says must never + /// happen quietly: a smaller denominator reads as a better score. + #[test] + fn every_discovered_mutant_reaches_the_report() { + let directory = crate::testing::workdir("elements-conservation"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + + let sources = [ + ("a.rs", "fn a() { x(a < b); }"), + ("b.rs", "fn b() { y(a < b); }"), + ("c.rs", "fn c() { z(a < b); }"), + ]; + + for (name, text) in sources { + fs::write(root.join(name).as_std_path(), text).expect("source"); + } + + let files: Vec = sources + .iter() + .map(|(name, _text)| TargetFile { + path: Utf8PathBuf::from(*name), + absolute: root.join(*name), + package: "subject".to_owned(), + }) + .collect(); + + // Two mutants in one file, one in each of the others, so that a loop emitting a single + // mutant per file and a loop stopping after the first file both fail. + let placements = [("a.rs", "m1"), ("a.rs", "m2"), ("b.rs", "m3"), ("c.rs", "m4")]; + + let plan = Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root, + files, + mutants: placements + .iter() + .map(|(file, id)| Mutant { + id: (*id).to_owned().into(), + file: (Utf8PathBuf::from(*file)).into(), + ..mutant(Outcome::Killed, 11..16) + }) + .collect(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + }; + + let report = build(&plan, Thresholds::default(), None).expect("a report"); + + let mut emitted: Vec<(String, String)> = report + .files + .iter() + .flat_map(|(path, file)| file.mutants.iter().map(move |result| (path.clone(), result.id.to_string()))) + .collect(); + + emitted.sort(); + + let mut expected: Vec<(String, String)> = placements.iter().map(|(file, id)| ((*file).to_owned(), (*id).to_owned())).collect(); + + expected.sort(); + + assert_eq!(emitted, expected); + } + + /// A mutant in a file the plan does not list would vanish from the denominator without a word. + /// + /// The skip is deliberate — a report cannot embed the source of a file the survey never + /// recorded — but it is exactly the shape of the loss the test above guards against, so the + /// case is pinned rather than left to be discovered as a wrong score. + #[test] + fn a_mutant_in_an_unlisted_file_is_the_only_thing_the_report_drops() { + let directory = crate::testing::workdir("elements-unlisted"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + + fs::write(root.join("a.rs").as_std_path(), "fn a() { x(a < b); }").expect("source"); + + let plan = Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: root.clone(), + files: vec![TargetFile { + path: Utf8PathBuf::from("a.rs"), + absolute: root.join("a.rs"), + package: "subject".to_owned(), + }], + mutants: vec![ + Mutant { + id: "listed".to_owned().into(), + file: (Utf8PathBuf::from("a.rs")).into(), + ..mutant(Outcome::Killed, 11..16) + }, + Mutant { + id: "unlisted".to_owned().into(), + file: (Utf8PathBuf::from("gone.rs")).into(), + ..mutant(Outcome::Killed, 11..16) + }, + ], + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + }; + + let report = build(&plan, Thresholds::default(), None).expect("a report"); + let emitted: Vec<&str> = report + .files + .values() + .flat_map(|file| file.mutants.iter().map(|result| result.id.as_str())) + .collect(); + + assert_eq!(emitted, vec!["listed"]); + assert!(!report.files.contains_key("gone.rs")); + } + + /// Streaming `write_json` produces the same bytes as the in-memory `to_json`. + #[test] + fn write_json_matches_to_json_output() { + let report = fixtures::report_with(Some((0, 2)), 100, vec![fixtures::mutant_result()]); + let expected = to_json(&report).expect("to_json succeeds"); + + let dir = crate::testing::workdir("write_json_matches"); + let path = Utf8PathBuf::from_path_buf(dir.path().join("report.json")).expect("utf8"); + + write_json(&report, &path).expect("write_json succeeds"); + + let written = fs::read_to_string(&path).expect("read back"); + assert_eq!(written, expected, "streaming write must produce identical bytes"); + } +} diff --git a/crates/cargo-gamma-lib/src/error.rs b/crates/cargo-gamma-lib/src/error.rs new file mode 100644 index 000000000..bf3b93a06 --- /dev/null +++ b/crates/cargo-gamma-lib/src/error.rs @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The error type used throughout the tool. + +use core::error::Error as StdError; +use core::fmt::{self, Display, Formatter}; +use std::io; + +/// An error carrying a human-readable message and an optional cause. +/// +/// Messages are written for the person who ran the command, not for a log aggregator: they say +/// what was being attempted and what to do about it. +#[derive(Debug)] +pub struct Error { + message: String, + cause: Option>, + usage: bool, + skippable: bool, +} + +impl Error { + /// Creates an error with the given message. + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + cause: None, + usage: false, + skippable: false, + } + } + + /// Marks this as a usage error: something the user typed or configured, not something that + /// went wrong while running. + /// + /// The distinction is the whole point of the exit-code scheme. A CI script needs to tell "you + /// invoked me wrongly" from "I ran and could not proceed", and collapsing the two forces it to + /// parse the message text to find out which happened. + #[must_use] + pub const fn usage(mut self) -> Self { + self.usage = true; + self + } + + /// Returns whether this is a usage error. + #[must_use] + pub const fn is_usage(&self) -> bool { + self.usage + } + + /// Marks this as an error the caller may step over: one file could not be handled, and the + /// work the caller is doing is still worth finishing without it. + /// + /// Only the *producer* of an error knows whether its subject is the whole job or one item of + /// it, and only the *consumer* knows whether stepping over an item is acceptable there. This + /// flag is how the first tells the second, instead of the second matching on message text. It + /// is never permission to be quiet: a caller that skips must say what it skipped, because a + /// file dropped from a mutation population silently raises the score. + #[must_use] + pub const fn skippable(mut self) -> Self { + self.skippable = true; + self + } + + /// Returns whether the caller may step over this error and carry on with the rest of the job. + #[must_use] + pub const fn is_skippable(&self) -> bool { + self.skippable + } + + /// Attaches an underlying cause. + #[must_use] + pub fn caused_by(mut self, cause: impl StdError + Send + Sync + 'static) -> Self { + self.cause = Some(Box::new(cause)); + self + } + + /// Returns the message, without the cause chain. + #[must_use] + pub fn message(&self) -> &str { + &self.message + } +} + +impl Display for Error { + #[expect(clippy::renamed_function_params, reason = "`f` is less clear than `formatter`")] + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.message)?; + + if let Some(cause) = &self.cause { + write!(formatter, ": {cause}")?; + } + + Ok(()) + } +} + +impl StdError for Error { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + self.cause.as_ref().map(|cause| &**cause as &(dyn StdError + 'static)) + } +} + +impl From for Error { + fn from(value: io::Error) -> Self { + Self::new("I/O error").caused_by(value) + } +} + +impl From for Error { + fn from(value: cargo_gamma_engine::Error) -> Self { + let (message, cause, usage, skippable) = value.into_parts(); + + Self { + message, + cause, + usage, + skippable, + } + } +} + +/// Creates an [`Error`] from a format string. +macro_rules! error { + ($($arg:tt)*) => { $crate::error::Error::new(format!($($arg)*)) }; +} + +pub(crate) use error; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn message_is_preserved() { + let error = Error::new("could not read the manifest"); + + assert_eq!(error.message(), "could not read the manifest"); + assert_eq!(error.to_string(), "could not read the manifest"); + } + + #[test] + fn cause_is_appended_to_the_display_form() { + let cause = io::Error::new(io::ErrorKind::NotFound, "no such file"); + let error = Error::new("could not read the manifest").caused_by(cause); + + assert_eq!(error.to_string(), "could not read the manifest: no such file"); + } + + #[test] + fn source_is_exposed_for_the_error_trait() { + use core::error::Error as _; + + let cause = io::Error::new(io::ErrorKind::NotFound, "no such file"); + let error = Error::new("outer").caused_by(cause); + + assert!(error.source().is_some()); + assert!(Error::new("outer").source().is_none()); + } + + #[test] + fn errors_are_not_usage_errors_by_default() { + assert!(!Error::new("something went wrong").is_usage()); + } + + #[test] + fn usage_errors_are_marked() { + // The exit-code scheme depends on this: a caller must be able to tell "you invoked me + // wrongly" from "I ran and could not proceed" without parsing the message. + assert!(Error::new("bad selector").usage().is_usage()); + } + + #[test] + fn marking_a_usage_error_preserves_the_message_and_cause() { + let cause = io::Error::new(io::ErrorKind::NotFound, "no such file"); + let error = Error::new("outer").caused_by(cause).usage(); + + assert_eq!(error.to_string(), "outer: no such file"); + assert!(error.is_usage()); + } + + #[test] + fn io_errors_convert() { + let error: Error = io::Error::new(io::ErrorKind::PermissionDenied, "denied").into(); + + assert!(error.to_string().contains("denied")); + } + + #[test] + fn engine_errors_preserve_classification_and_causes() { + let engine = cargo_gamma_engine::Error::new("could not read source") + .caused_by(io::Error::new(io::ErrorKind::PermissionDenied, "denied")) + .usage() + .skippable(); + let error = Error::from(engine); + + assert_eq!(error.to_string(), "could not read source: denied"); + assert!(error.is_usage()); + assert!(error.is_skippable()); + assert!(error.source().is_some()); + } +} diff --git a/crates/cargo-gamma-lib/src/estimate.rs b/crates/cargo-gamma-lib/src/estimate.rs new file mode 100644 index 000000000..6ddf05193 --- /dev/null +++ b/crates/cargo-gamma-lib/src/estimate.rs @@ -0,0 +1,405 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Projecting what a run will cost, before paying for it. +//! +//! The failure mode this exists to prevent is discovering a four-hour job four hours in. Everything +//! here is derived from measurements a run has already taken by the time the first mutant would +//! start: the build really built, the baseline really ran, and unviable mutants were really +//! withdrawn. Only one quantity is genuinely unknown before mutants execute — how much of the suite +//! a killed mutant gets through before something fails — and the projection says which assumption +//! it made about it rather than folding it silently into a single confident number. + +use core::time::Duration; + +use crate::advise::human; +use crate::model::{Mutant, Outcome}; +use crate::report::quantity; + +/// The share of the suite a mutant that fails or survives is assumed to reach before it is judged. +/// +/// A killed mutant almost never runs the whole suite: something fails, and with a fail-fast binary +/// the rest is never reached. Assuming the full baseline for every mutant would overestimate badly +/// on a healthy codebase, which is the failure mode that makes an estimate useless — nobody plans +/// against a number they have learned is always too big. A survivor does run the whole suite, so +/// this is a blend across both rather than a claim about either. +const KILLED_SHARE: f64 = 0.60; + +/// The share of mutants assumed to hang, for the middle of the range. +/// +/// This is the one quantity that decides what a run costs, and the one nothing measured before the +/// mutants execute can supply. It matters far more than its size suggests: a mutant that hangs is +/// stopped by a budget with a floor under it, then re-run to confirm, so on a suite that finishes +/// in a moment one hang can cost as much as several thousand mutants that do not. Turning a loop +/// counter into an infinite loop is an ordinary mutation, not an exotic one, so assuming none is +/// not the safe choice — it is the choice that produced an estimate off by two orders of magnitude. +const STALL_SHARE: f64 = 0.05; + +/// The share of mutants assumed to hang, for the top of the range, as a percentage. +/// +/// Held as a percentage because it is printed as one. Deriving the fraction from the number that is +/// displayed keeps the projection and its explanation from ever disagreeing. +const STALL_PERCENT_HIGH: u32 = 15; + +/// The share of mutants assumed to hang, for the top of the range. +fn stall_share_high() -> f64 { + f64::from(STALL_PERCENT_HIGH) / 100.0 +} + +/// What testing every live mutant once would cost, summed over the mutants. +/// +/// Every field is a serial total over the live mutants, counting for each one only the test +/// binaries that can actually reach its package. Projecting from the whole baseline instead — as +/// though every mutant ran the entire suite — overestimates a loosely coupled workspace by roughly +/// the number of crates in it, which is exactly the shape of estimate nobody plans against because +/// they have learned it is always too big. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Workload { + /// What testing them all once would cost. + pub suite: Duration, + + /// What it would cost if every one of them ran every binary out of time. + pub budget: Duration, + + /// What it would cost if every one of them ran a single average binary out of time. + pub single: Duration, +} + +/// A projection of what a run will cost. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Estimate { + /// Mutants that would actually be tested. + pub live: usize, + + /// Mutants withdrawn during the build because they could not compile. + pub withdrawn: usize, + + /// Measured: the instrumented build. + pub build: Duration, + + /// Measured: wall time for the concurrently run suite with no mutant active. + pub baseline: Duration, + + /// Projected: testing every live mutant, at the configured parallelism, assuming the usual + /// share of them hang. + pub mutants: Duration, + + /// Projected: the same, assuming none of them hang. + pub settled: Duration, + + /// Projected: the same, assuming an unusually large share of them hang. + pub stalling: Duration, + + /// How many mutants are tested at once. + pub jobs: usize, + + /// Projected: every live mutant running out the budget of every binary that can reach it. + pub worst: Duration, +} + +impl Estimate { + /// The lower end of the range: nothing hangs. + /// + /// The width of this range is not a confidence interval dressed up as one. It is the single + /// thing that decides what the run costs and cannot be measured before it starts, so a run + /// whose ends are far apart is telling the reader something true: the cost is going to be + /// decided by how many mutants hang, not by how many there are. + #[must_use] + pub fn low(&self) -> Duration { + self.build + self.baseline + self.settled + } + + /// The upper end of the range. + /// + /// Capped at the projected ceiling, which is the point where every mutant has already been + /// given every second of test time it will ever get: a range whose top is above that is + /// describing time that cannot be spent on tests. + #[must_use] + pub fn high(&self) -> Duration { + (self.build + self.baseline + self.stalling).min(self.worst_case()) + } + + /// The ceiling on test time: every mutant hits its timeout and is then confirmed. + /// + /// A ceiling on the time spent *running tests*, and not on the time a run takes. It counts the + /// confirmation run, which is what makes it a ceiling on that time rather than a guess at it: a + /// mutant that exhausts its budget is not believed on the first try, so the path that costs the + /// most costs several times its timeout rather than one of them. + /// + /// What it does not count is the fixed cost of putting each mutant through the machine — + /// launching its processes, scheduling it onto a lane, and writing down what happened. Nothing + /// measured before the first mutant runs prices that, so it is left out and said to be left + /// out, rather than approximated into a number that would look like a bound and not be one. On + /// a suite that finishes in a moment the fixed cost is the larger share, so a run can and does + /// overshoot this; [`render`] says so on the line it is printed on. + #[must_use] + pub fn worst_case(&self) -> Duration { + self.build + self.baseline + self.worst + } +} + +/// Projects a run from what the build and baseline already measured. +#[must_use] +pub fn project(mutants: &[Mutant], work: Workload, baseline: Duration, build: Duration, jobs: usize) -> Estimate { + let live = mutants + .iter() + .filter(|mutant| mutant.ordinal > 0 && mutant.outcome == Outcome::Pending) + .count(); + + let withdrawn = mutants.iter().filter(|mutant| mutant.outcome == Outcome::CompileError).count(); + let lanes = u32::try_from(jobs.max(1)).unwrap_or(1); + + Estimate { + live, + withdrawn, + build, + baseline, + mutants: spend(work, lanes, STALL_SHARE), + settled: spend(work, lanes, 0.0), + stalling: spend(work, lanes, stall_share_high()), + jobs, + worst: work.budget.saturating_mul(1 + crate::exec::CONFIRM_FACTOR) / lanes, + } +} + +/// What testing every live mutant comes to, if the given share of them hang. +/// +/// The two outcomes are priced separately because they differ by orders of magnitude rather than by +/// a little. A mutant that is judged pays for the tests it reached. A mutant that hangs pays for +/// everything it got through, then for a whole budget it never finishes, and then for the +/// confirmation run that budget is not believed without — and that budget has a floor under it, so +/// on a quick suite it is not a multiple of the tests but a constant far larger than all of them. +fn spend(work: Workload, lanes: u32, stalling: f64) -> Duration { + let judged = work.suite.mul_f64(KILLED_SHARE * (1.0 - stalling)); + let hung = (work.suite + work.single.saturating_mul(1 + crate::exec::CONFIRM_FACTOR)).mul_f64(stalling); + + (judged + hung) / lanes +} + +/// Renders a projection as the single line printed once the fixed cost is paid. +/// +/// One line, because it is printed in the middle of a run whose build and baseline timings are +/// already on the screen directly above it; repeating them would be padding. What is left is the +/// only thing the reader cannot already see: how long the remaining wait is, and how bad it could +/// get. +/// +/// The ends of the range are labelled with the assumption that produces each, rather than left to +/// look like a margin of error. A reader who sees a wide range and is told what widens it can do +/// something about it — lower the timeout floor, or find the mutants that hang — where a reader +/// shown a bare interval can only distrust it. +/// +/// The ceiling is labelled with what it leaves out for the same reason. It bounds the time spent +/// running tests and nothing else: the fixed cost of launching, scheduling and recording each +/// mutant is not measured before the first one runs, so it is not in the number. On a suite that +/// finishes in a moment that cost is the larger share and a run will exceed the figure, which is +/// exactly the sort of surprise a CI budget is planned around. +#[must_use] +pub fn render(estimate: &Estimate) -> String { + format!( + "{} if none hang, {} if {}% do, for {} at {}; {} worst case for test time, before per-mutant overhead", + human(estimate.low()), + human(estimate.high()), + STALL_PERCENT_HIGH, + quantity(estimate.live, "mutant"), + quantity(estimate.jobs, "job"), + human(estimate.worst_case()) + ) +} + +#[cfg(test)] +mod tests { + use camino::Utf8PathBuf; + + use super::*; + use crate::fixtures; + + fn mutant(ordinal: u32, outcome: Outcome) -> Mutant { + Mutant { + id: format!("m{ordinal}{outcome}").into(), + ordinal, + package: ("p".to_owned()).into(), + file: (Utf8PathBuf::from("a.rs")).into(), + outcome, + ..fixtures::mutant() + } + } + + fn population() -> Vec { + let mut mutants: Vec = (1..=100).map(|index| mutant(index, Outcome::Pending)).collect(); + + mutants.push(mutant(0, Outcome::Ignored)); + mutants.push(mutant(101, Outcome::CompileError)); + mutants + } + + /// A serial workload of `secs` seconds, and a worst case ten times as bad. + fn work(secs: u64) -> Workload { + Workload { + suite: Duration::from_secs(secs), + budget: Duration::from_secs(secs * 10), + single: Duration::from_secs(secs * 5), + } + } + + /// A workload whose only cost is the suite, so a projection reduces to the settled case. + fn suite_only(secs: u64) -> Workload { + Workload { + suite: Duration::from_secs(secs), + ..Workload::default() + } + } + + #[test] + fn only_mutants_that_would_run_are_counted() { + let estimate = project(&population(), work(100), Duration::ZERO, Duration::from_secs(5), 1); + + assert_eq!(estimate.live, 100); + assert_eq!(estimate.withdrawn, 1); + } + + #[test] + fn parallelism_divides_the_projection() { + let one = project(&population(), work(1000), Duration::ZERO, Duration::from_secs(50), 1); + let eight = project(&population(), work(1000), Duration::ZERO, Duration::from_secs(50), 8); + + assert_eq!(one.mutants / 8, eight.mutants); + assert_eq!(one.worst / 8, eight.worst); + } + + #[test] + fn zero_jobs_does_not_divide_by_zero() { + let estimate = project(&population(), work(1000), Duration::ZERO, Duration::from_secs(50), 0); + + assert!(estimate.mutants > Duration::ZERO); + } + + #[test] + fn only_the_binaries_a_mutant_reaches_are_charged_for_it() { + // The caller sums the reachable suites; a mutant that can only be seen by a tenth of the + // workspace must cost a tenth of what one visible to all of it costs. + let narrow = project(&population(), suite_only(100), Duration::ZERO, Duration::ZERO, 1); + let wide = project(&population(), suite_only(1000), Duration::ZERO, Duration::ZERO, 1); + + assert_eq!(narrow.mutants * 10, wide.mutants); + } + + #[test] + fn the_error_bar_brackets_the_estimate() { + let estimate = project(&population(), work(1000), Duration::from_secs(3), Duration::from_secs(50), 4); + + assert!(estimate.low() < estimate.high()); + assert!(estimate.build + estimate.baseline <= estimate.low()); + } + + #[test] + fn the_error_bar_never_widens_the_part_that_was_measured() { + // The build really happened; the projection has no business being uncertain about it. + let estimate = project(&[], Workload::default(), Duration::ZERO, Duration::from_secs(30), 4); + + assert_eq!(estimate.low(), Duration::from_secs(30)); + assert_eq!(estimate.high(), Duration::from_secs(30)); + } + + #[test] + fn the_worst_case_exceeds_the_estimate() { + let estimate = project(&population(), work(1000), Duration::from_secs(3), Duration::from_secs(50), 4); + + assert!(estimate.worst_case() > estimate.high()); + } + + #[test] + fn the_worst_case_pays_for_confirming_every_timeout() { + // A mutant that runs out its budget is made to prove it, so a ceiling that counts one + // timeout apiece is one a real run can walk straight past. + let load = work(1000); + let estimate = project(&population(), load, Duration::ZERO, Duration::ZERO, 1); + + assert_eq!(estimate.worst_case(), load.budget.saturating_mul(1 + crate::exec::CONFIRM_FACTOR)); + } + + #[test] + fn the_projected_range_never_reaches_past_the_ceiling() { + // Above the worst case there is no time left to spend: every mutant has already been given + // every second it will ever get. + let load = Workload { + suite: Duration::from_secs(1000), + ..work(1) + }; + let estimate = project(&population(), load, Duration::ZERO, Duration::ZERO, 1); + + assert_eq!(estimate.high(), estimate.worst_case()); + } + + #[test] + fn the_rendering_is_one_line_carrying_the_range_the_population_and_the_worst_case() { + let estimate = project(&population(), work(1000), Duration::from_secs(3), Duration::from_secs(50), 4); + let rendered = render(&estimate); + + assert_eq!(rendered.lines().count(), 1, "{rendered}"); + assert!(rendered.contains("100 mutants"), "{rendered}"); + assert!(rendered.contains("4 jobs"), "{rendered}"); + assert!(rendered.contains("worst case"), "{rendered}"); + } + + /// Each end of the range is labelled with the assumption that produces it, because a reader + /// shown a bare interval learns only that the tool is unsure, where a reader told that the + /// width is hanging mutants can go and do something about them. + #[test] + fn the_rendering_says_what_widens_the_range() { + let estimate = project(&population(), work(1000), Duration::from_secs(3), Duration::from_secs(50), 4); + let rendered = render(&estimate); + + assert!(rendered.contains("if none hang"), "{rendered}"); + assert!(rendered.contains("if 15% do"), "{rendered}"); + } + + /// The ceiling counts test time and nothing else. Saying so is the whole fix: a figure that + /// omits the per-mutant launch, scheduling and reporting cost is routinely walked past by a + /// real run at the timeout floor, and a reader planning a CI budget against a bare "worst + /// case" has no way to know that. + #[test] + fn the_rendering_says_what_the_ceiling_leaves_out() { + let estimate = project(&population(), work(1000), Duration::from_secs(3), Duration::from_secs(50), 4); + let rendered = render(&estimate); + + assert!(rendered.contains("worst case for test time"), "{rendered}"); + assert!(rendered.contains("before per-mutant overhead"), "{rendered}"); + } + + /// The whole point of the rework: a population that hangs must be projected as costing more + /// than the same population that does not. The old model had no term for it at all, and was + /// measured two orders of magnitude optimistic on a suite whose mutants hung. + #[test] + fn hanging_mutants_cost_more_than_mutants_that_are_judged() { + let load = Workload { + suite: Duration::from_secs(10), + budget: Duration::from_secs(4000), + single: Duration::from_secs(2000), + }; + let estimate = project(&population(), load, Duration::ZERO, Duration::ZERO, 1); + + assert!(estimate.stalling > estimate.settled.saturating_mul(100), "{estimate:?}"); + assert!(estimate.mutants > estimate.settled, "{estimate:?}"); + } + + /// A timeout budget is spent in the one binary that hangs, not in every binary that could have + /// reached the mutant, so a workload measuring only the total across binaries would price a + /// hang at several times what one costs. + #[test] + fn a_hang_is_charged_for_one_binary_rather_than_all_of_them() { + let all = Workload { + suite: Duration::ZERO, + budget: Duration::from_secs(400), + single: Duration::from_secs(400), + }; + let one = Workload { + single: Duration::from_secs(100), + ..all + }; + + let costly = project(&population(), all, Duration::ZERO, Duration::ZERO, 1); + let cheaper = project(&population(), one, Duration::ZERO, Duration::ZERO, 1); + + assert_eq!(cheaper.stalling * 4, costly.stalling); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/baseline.rs b/crates/cargo-gamma-lib/src/exec/baseline.rs new file mode 100644 index 000000000..3594503f1 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/baseline.rs @@ -0,0 +1,636 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::sync::atomic::{AtomicUsize, Ordering}; +use core::time::Duration; +use std::sync::mpsc; +use std::thread; +use std::time::Instant; + +use cargo_gamma_process::MemoryRequest; + +use super::stall::Stall; +use super::test_binary::TestBinary; +use super::verdict::{Attempt, Observation, Only, Verdict, observe}; +use super::workspace::Workspace; +use crate::Result; +use crate::error::error; + +/// What the baseline measured. +#[derive(Debug, Clone, Copy)] +pub(super) struct Baseline { + /// How long the suite took, as the sum of what each binary in it took. + /// + /// The sum rather than the wall clock, because the binaries are measured many at a time while + /// a mutant runs its own binaries one after another: what a mutant's budget has to cover is + /// every binary it may have to visit, so the figure that budget is derived from has to be the + /// same total. Wall clock over a concurrent measurement would be that total divided by however + /// many workers happened to overlap, and every budget derived from it short by the same factor. + pub(super) elapsed: Duration, + + /// How long the concurrent baseline measurement took on the wall clock. + pub(super) wall: Duration, + + /// The longest the suite legitimately went quiet, which calibrates the stall budget. + pub(super) quiet: Duration, + + /// How many tests ran, or `None` if no harness said. + /// + /// This is what ran, not what exists: `--test-package`, `--test-workspace` and any filter + /// passed through to the harness all narrow it. That is the useful figure, since it is exactly + /// the set of tests that will pass judgement on every mutant. + pub(super) tests: Option, + + /// The largest peak memory any one binary reached, when the run asked for a measurement. + /// + /// The whole suite's peaks are not added up, because each binary is metered inside its own + /// accounting boundary and later judged against a ceiling of its own: what a ceiling has to + /// admit is the most any single one of them needed, not the sum of what all of them needed at + /// different moments. + pub(super) peak: Option, +} + +/// How long one binary gets to produce a baseline before the run gives up on it. +/// +/// Generous rather than calibrated, because there is nothing to calibrate from yet: this is the +/// measurement every later budget is derived from. It only ever fires on a suite that has hung. +const BASELINE_BUDGET: Duration = Duration::from_mins(10); + +/// Runs the suite with no mutant active and returns how long it took. +/// +/// `jobs` is the sweep's own concurrency, and the measurement is taken at exactly that width. A +/// baseline measured one binary at a time on an otherwise idle machine describes a situation that +/// never occurs again in the run: the timeouts derived from it are then spent by `jobs` workers +/// contending for the same cores, so they are systematically too tight, and the mutants that lose +/// that race are recorded as timeouts — which count as kills, and inflate the score with detections +/// the suite never made. Calibrating and spending under the same load makes every derived quantity +/// correct by construction rather than by a fudge factor. +pub(super) fn measure_baseline( + work: &Workspace, + binaries: &mut [TestBinary], + request: MemoryRequest, + jobs: usize, + completed: impl FnMut(), +) -> Result { + measure_within_reporting(work, binaries, BASELINE_BUDGET, request, jobs, completed) +} + +/// Measures the baseline under an explicit budget. +/// +/// The budget is a parameter so that the paths a hung or failing suite takes can be exercised +/// without waiting out the real one. +#[cfg(all(test, unix))] +fn measure_within( + work: &Workspace, + binaries: &mut [TestBinary], + budget: Duration, + request: MemoryRequest, + jobs: usize, +) -> Result { + measure_within_reporting(work, binaries, budget, request, jobs, || {}) +} + +fn measure_within_reporting( + work: &Workspace, + binaries: &mut [TestBinary], + budget: Duration, + request: MemoryRequest, + jobs: usize, + completed: impl FnMut(), +) -> Result { + let started = Instant::now(); + let measured = sweep_binaries(work, binaries, budget, request, jobs, completed); + let wall = started.elapsed(); + let mut elapsed = Duration::ZERO; + let mut quiet = Duration::ZERO; + let mut tests: Option = None; + let mut peak: Option = None; + + // Folded in the binaries' own order rather than in the order the workers happened to finish, so + // that which failure a red suite reports does not depend on the scheduler. + for (entry, taken) in binaries.iter_mut().zip(measured) { + let Some((took, observed)) = taken else { + continue; + }; + + entry.baseline = took; + entry.peak = observed.peak; + entry.tests = observed.tests; + elapsed = elapsed.saturating_add(took); + quiet = quiet.max(observed.quiet); + + if let Some(measured) = observed.peak { + peak = Some(peak.unwrap_or(0).max(measured)); + } + + // A binary with no harness contributes nothing rather than turning the total into a + // guess, but one binary reporting is enough for the total to be worth stating. + if let Some(counted) = observed.tests { + tests = Some(tests.unwrap_or(0).saturating_add(counted)); + } + + match observed.verdict { + Verdict::Passed => {} + Verdict::Failed(name) => { + let which = name.map_or_else(|| "a test".to_owned(), |test| format!("test `{test}`")); + let path = &entry.path; + + return Err(error!( + "the baseline is not green: {which} in `{path}` fails before any mutant is applied.\n\ + Every verdict in a run is a comparison against the baseline, so there is nothing to \ + measure until the suite passes." + )); + } + Verdict::TestEnumerationFailed(_) => { + return Err(error!( + "the baseline could not be measured because `cargo nextest` could not enumerate \ + its tests.\nNothing was measured, so the run stops here rather than judge every \ + mutant against a baseline it never took." + )); + } + Verdict::TimedOut | Verdict::Stalled(_) => { + return Err(baseline_timeout_error(&entry.path)); + } + Verdict::MemoryLimit { peak, limit } => { + return Err(baseline_memory_error(&entry.path, peak, limit)); + } + // The baseline runs with no mutant active, so there is nothing to re-run it without + // and no confirmation is attempted. A flake here is an ordinary red baseline. + Verdict::Flaky(name) => { + let which = name.map_or_else(|| "a test".to_owned(), |test| format!("test `{test}`")); + let path = &entry.path; + + return Err(error!( + "the baseline is not green: {which} in `{path}` fails before any mutant is applied.\n\ + Every verdict in a run is a comparison against the baseline, so there is nothing to \ + measure until the suite passes." + )); + } + Verdict::Unmetered(reason) => { + return Err(error!( + "the baseline could not be measured as this run was configured: {reason}.\n\ + Nothing was measured, so the run stops here rather than judge every mutant \ + against a baseline it never took." + )); + } + + // Fatal here where it is not during the sweep: there is no mutant to record it against, + // and a baseline binary that went unmeasured leaves every mutant that binary covers + // without a budget to be judged against. + Verdict::Unjudged(reason) => { + return Err(error!( + "the baseline could not be measured because the machine would not run it: {reason}.\n\ + Nothing was measured, so the run stops here rather than judge every mutant \ + against a baseline it never took." + )); + } + } + } + + Ok(Baseline { + elapsed, + wall, + quiet, + tests, + peak, + }) +} + +/// Runs every binary once with no mutant active, `jobs` of them at a time. +/// +/// Returns what each one produced, positionally, so the caller can fold the results in the order +/// the binaries were given rather than the order they finished. +fn sweep_binaries( + work: &Workspace, + binaries: &[TestBinary], + budget: Duration, + request: MemoryRequest, + jobs: usize, + completed: impl FnMut(), +) -> Vec> { + sweep_binaries_with(work, binaries, budget, request, jobs, observe, completed) +} + +fn sweep_binaries_with( + work: &Workspace, + binaries: &[TestBinary], + budget: Duration, + request: MemoryRequest, + jobs: usize, + observer: O, + mut completed: impl FnMut(), +) -> Vec> +where + O: Fn(&Workspace, &TestBinary, Attempt<'_>) -> Observation + Sync, +{ + let mut measured: Vec> = (0..binaries.len()).map(|_index| None).collect(); + let next = AtomicUsize::new(0); + let (sender, receiver) = mpsc::channel::<(usize, Duration, Observation)>(); + let notes = crate::notes::current(); + + thread::scope(|scope| { + for _worker in 0..jobs.max(1) { + let sender = sender.clone(); + let next = &next; + let notes = notes.clone(); + let observer = &observer; + + let _handle = scope.spawn(move || { + let _notes = crate::notes::enter(notes.as_ref()); + + loop { + let index = next.fetch_add(1, Ordering::Relaxed); + + let Some(binary) = binaries.get(index) else { + break; + }; + + let began = Instant::now(); + let observation = observer( + work, + binary, + Attempt { + active: None, + timeout: Some(budget), + stall: Stall::NONE, + request, + only: Only::All, + census: None, + }, + ); + + // A closed receiver means the calling thread is gone, which cannot happen while + // the scope is open. + let _sent = sender.send((index, began.elapsed(), observation)); + } + }); + } + + // The workers hold the only remaining senders, so the drain ends when the last one finishes. + drop(sender); + + for (index, took, observed) in receiver { + if let Some(slot) = measured.get_mut(index) { + *slot = Some((took, observed)); + } + + completed(); + } + }); + + measured +} + +fn baseline_timeout_error(binary: &camino::Utf8Path) -> crate::error::Error { + error!("the baseline run of `{binary}` did not finish within ten minutes") +} + +/// Explains a baseline that hit the explicit ceiling put around the calibration itself. +/// +/// A ceiling derived from the baseline cannot protect the machine from a baseline that is itself +/// runaway, so a ceiling may be placed around the calibration. When that one fires, no mutant is +/// involved: the suite needs more memory than the run was told to allow it, and the number to +/// change is the one the user supplied. +fn baseline_memory_error(binary: &camino::Utf8Path, peak: Option, limit: u64) -> crate::error::Error { + let reached = peak.map_or_else(String::new, |peak| format!(", reaching {}", crate::report::bytes(peak))); + + error!( + "the baseline run of `{binary}` exceeded the {} it was allowed{reached}.\n\ + Every mutant is judged against this run, so there is nothing to measure until the suite \ + fits within the ceiling `--baseline-memory-limit` set, or that ceiling is raised.", + crate::report::bytes(limit) + ) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use camino::Utf8Path; + + use super::*; + + #[test] + fn a_partial_stream_note_from_a_baseline_worker_reaches_the_parent() { + crate::notes::alone(|| { + let (_directory, work) = crate::testing::helper_workspace("baseline-worker-note", &[]); + let binary = crate::testing::helper(); + + let measured = sweep_binaries_with( + &work, + &[binary], + Duration::from_secs(30), + MemoryRequest::default(), + 1, + |_work, _binary, _attempt| { + crate::notes::note("baseline binary produced a partial stream"); + + Observation { + verdict: Verdict::Passed, + quiet: Duration::ZERO, + tests: Some(1), + peak: None, + } + }, + || {}, + ); + + assert_eq!(measured.len(), 1); + assert!( + crate::notes::drain().iter().any(|note| note.contains("partial")), + "the worker's partial-stream diagnostic was lost" + ); + }); + } + + /// Wraps an existing directory as a workspace whose "test binary" is `/bin/sh`. + /// + /// The script is passed as an argument rather than written to disk: a file made executable + /// while other threads are forking can be refused with `ETXTBSY`, which would make these + /// tests intermittently fail for a reason that has nothing to do with what they assert. + #[cfg(unix)] + fn harness(body: &str) -> (tempfile::TempDir, Workspace, Vec) { + let (directory, work) = crate::testing::shell_workspace("baseline", body); + let binaries = vec![TestBinary { + package: "subject".to_owned(), + ..crate::testing::test_binary("/bin/sh") + }]; + + (directory, work, binaries) + } + + /// A suite that hangs before any mutant is applied stops the run. + #[test] + #[cfg(unix)] + fn a_baseline_that_never_finishes_stops_the_run() { + let (_directory, work, mut binaries) = harness("sleep 30"); + let failure = measure_within(&work, &mut binaries, Duration::from_millis(50), MemoryRequest::default(), 1) + .expect_err("the baseline must fail"); + + // Continuing would time out every mutant against a suite that never finishes and report a + // perfect score built entirely out of false detections. + assert!(failure.to_string().contains("did not finish"), "{failure}"); + } + + /// A suite that is already failing stops the run, naming the test. + #[test] + #[cfg(unix)] + fn a_red_baseline_stops_the_run_and_names_the_failing_test() { + let (_directory, work, mut binaries) = harness("echo 'test a::b ... FAILED'\nexit 101"); + let failure = + measure_within(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 1).expect_err("the baseline must fail"); + + // Every verdict is a comparison against the baseline, so a red one makes every mutant + // look killed by a failure that was there before mutation started. + assert!(failure.to_string().contains("test `a::b`"), "{failure}"); + } + + /// A green suite yields the elapsed time and the harness's own test count. + #[test] + #[cfg(unix)] + fn a_green_baseline_reports_the_elapsed_time_and_the_test_count() { + let (_directory, work, mut binaries) = harness("echo 'running 3 tests'\necho 'test a::b ... ok'\nexit 0"); + let baseline = + measure_within(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 1).expect("the baseline must pass"); + + // The per-binary baseline is what apportions each mutant's budget, so it has to be + // written back rather than merely totalled. + assert_eq!(baseline.tests, Some(3)); + assert_eq!(binaries[0].tests, Some(3)); + assert!(binaries[0].baseline > Duration::ZERO); + } + + /// A suite of several binaries is measured with as many running at once as the sweep will use, + /// and the total it reports is the sum of the parts rather than the wall clock over them. + /// + /// Regression, issue-016. Measured one at a time on an idle machine, every budget derived from + /// the baseline describes a machine the run never sees again — the sweep spends those budgets + /// with `jobs` binaries contending for the same cores, and the mutants that lose the race are + /// recorded as timeouts, which count as kills and inflate the score. Wall clock over a + /// concurrent measurement would be wrong in the same direction for a different reason: a mutant + /// runs its own binaries one after another, so the budget has to cover their sum. + #[test] + #[cfg(unix)] + fn a_baseline_is_measured_at_the_concurrency_the_sweep_will_use() { + let (_directory, work) = crate::testing::shell_workspace("baseline-jobs", "sleep 0.4\nexit 0"); + let mut binaries: Vec = (0..4) + .map(|_index| TestBinary { + package: "subject".to_owned(), + ..crate::testing::test_binary("/bin/sh") + }) + .collect(); + + let began = Instant::now(); + let mut completed = 0; + let baseline = measure_within_reporting(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 4, || { + completed += 1; + }) + .expect("the baseline passes"); + let wall = began.elapsed(); + + let summed: Duration = binaries.iter().map(|binary| binary.baseline).sum(); + + assert_eq!(baseline.elapsed, summed); + assert_eq!(completed, binaries.len(), "each completed binary advances progress exactly once"); + assert!(binaries.iter().all(|binary| binary.baseline > Duration::ZERO)); + + // Four four-hundred-millisecond sleeps run four at a time cannot take the 1.6 seconds a + // serial measurement would; the reported total nevertheless has to be that sum. + assert!(wall < summed, "wall {wall:?} against summed {summed:?}"); + assert!( + baseline.wall < summed, + "recorded wall {:?} against summed {summed:?}", + baseline.wall + ); + assert!( + baseline.wall <= wall, + "recorded wall {:?} against caller wall {wall:?}", + baseline.wall + ); + } + + #[test] + fn baseline_progress_advances_once_for_every_finished_binary() { + let (_directory, work) = crate::testing::helper_workspace("baseline-progress", &["exit:0"]); + let mut binaries: Vec = (0..3) + .map(|_index| TestBinary { + package: "subject".to_owned(), + ..crate::testing::helper() + }) + .collect(); + let mut completed = 0; + + let _baseline = measure_within_reporting(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 2, || { + completed += 1; + }) + .expect("the baseline passes"); + + assert_eq!(completed, binaries.len()); + } + + /// A red binary is reported whichever worker happened to reach it first. + /// + /// With more than one binary in flight the order results arrive in is the scheduler's business, + /// so the failure a run reports is folded in the binaries' own order instead — otherwise the + /// same red suite would name a different test from run to run. + #[test] + #[cfg(unix)] + fn a_red_binary_is_reported_whatever_order_the_workers_finished_in() { + let (_directory, work) = crate::testing::shell_workspace("baseline-order", "echo 'test a::b ... FAILED'\nexit 101"); + let mut binaries: Vec = (0..4) + .map(|_index| TestBinary { + package: "subject".to_owned(), + ..crate::testing::test_binary("/bin/sh") + }) + .collect(); + + let failure = + measure_within(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 4).expect_err("the baseline must fail"); + + assert!(failure.to_string().contains("test `a::b`"), "{failure}"); + } + + /// A binary whose harness announced no tests records that, so nothing later mistakes it for a + /// binary that could have convicted something. + /// + /// Regression, issue-011. Cargo emits a unit-test binary for every lib target whether or not it + /// holds a test, so the existence of a binary says nothing; the announced count is the only + /// evidence the run has that a package has no tests at all. + #[test] + #[cfg(unix)] + fn a_binary_that_announced_no_tests_records_the_zero_rather_than_nothing() { + let (_directory, work, mut binaries) = harness("echo 'running 0 tests'\nexit 0"); + let baseline = + measure_within(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 1).expect("the baseline passes"); + + assert_eq!(binaries[0].tests, Some(0)); + assert_eq!(baseline.tests, Some(0)); + } + + /// A binary whose harness announces nothing still contributes its time. + #[test] + #[cfg(unix)] + fn a_baseline_with_no_harness_count_still_measures_the_time() { + let (_directory, work, mut binaries) = harness("echo 'custom harness'\nexit 0"); + let baseline = + measure_within(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 1).expect("the baseline must pass"); + + // A custom harness is not a broken one; inventing a count would be worse than omitting it. + assert_eq!(baseline.tests, None); + } + + /// A suite that fails without ever printing a recognisable `test ... FAILED` line still stops + /// the run, just without naming a specific test. + /// + /// A custom harness that exits non-zero has failed just as surely as one that names a test, + /// and inventing a name for it would be worse than admitting the run only knows "a test" broke; + /// a baseline stage that instead pressed on would judge every mutant against a suite that was + /// already red for a reason nobody could see. + #[test] + #[cfg(unix)] + fn a_baseline_that_fails_without_naming_a_test_still_stops_the_run() { + let (_directory, work, mut binaries) = harness("exit 1"); + let failure = + measure_within(&work, &mut binaries, Duration::from_secs(30), MemoryRequest::default(), 1).expect_err("the baseline must fail"); + + assert!(failure.to_string().contains("a test"), "{failure}"); + } + + /// A baseline binary that outgrows the ceiling placed around the calibration itself is + /// reported as such, not merely timed out or silently truncated. + /// + /// The baseline has no mutant to blame, so if this path were not wired up a runaway baseline + /// would either hang the whole run past the timeout or, worse, get judged as if it were an + /// ordinary passing suite while quietly starving the machine of memory. + #[test] + #[cfg(unix)] + fn a_baseline_binary_that_outgrows_its_ceiling_is_reported_as_such() { + if crate::testing::without_memory_support("a baseline measuring the suite's memory") { + return; + } + + let fill = format!("/dev/shm/gamma-baseline.{}", std::process::id()); + let (_directory, work, mut binaries) = harness(&format!("dd if=/dev/zero of={fill} bs=1M count=512 2>/dev/null")); + let request = MemoryRequest { + meter: true, + limit: Some(32 * 1024 * 1024), + }; + + let failure = measure_within(&work, &mut binaries, Duration::from_mins(1), request, 1) + .expect_err("a baseline that outgrows its ceiling must be reported, not merely timed out"); + + let _removed = std::fs::remove_file(&fill); + + assert!(failure.to_string().contains("--baseline-memory-limit"), "{failure}"); + } + + /// A host that cannot install the memory accounting a metered baseline asked for stops the + /// run rather than measure the suite unprotected. + /// + /// Reporting a "green" baseline that was never actually metered would mean every mutant + /// afterwards is silently compared against an unprotected run, hiding the exact failure mode + /// the ceiling was meant to catch. + #[test] + #[cfg(unix)] + fn an_undelegated_host_stops_a_metered_baseline_rather_than_measure_it_unprotected() { + if crate::exec::memory::support().is_ok() { + return; + } + + let (_directory, work, mut binaries) = harness("exit 0"); + let request = MemoryRequest { meter: true, limit: None }; + + let failure = measure_within(&work, &mut binaries, Duration::from_secs(30), request, 1) + .expect_err("a host that cannot meter memory must not measure a baseline unprotected"); + + // The wrapper has to say the baseline could not be measured as configured, and it has to + // carry the underlying cause through rather than replacing it. + assert!(failure.to_string().contains("as this run was configured"), "{failure}"); + assert!(failure.to_string().contains("cgroup"), "{failure}"); + } + + /// A metered baseline writes each binary's peak back, which is what a ceiling is derived from. + #[test] + #[cfg(unix)] + fn a_metered_baseline_records_what_each_binary_used() { + if crate::testing::without_memory_support("a baseline measuring the suite's memory") { + return; + } + + let (_directory, work, mut binaries) = harness("dd if=/dev/zero of=/dev/null bs=1M count=32 2>/dev/null\nexit 0"); + let request = MemoryRequest { meter: true, limit: None }; + let baseline = measure_within(&work, &mut binaries, Duration::from_secs(30), request, 1).expect("the baseline must pass"); + + // A ceiling is derived per binary, so the per-binary figure has to be written back and not + // merely totalled; a run that only kept the total would bound every binary by the largest. + assert!(binaries[0].peak.is_some(), "{:?}", binaries[0].peak); + assert_eq!(baseline.peak, binaries[0].peak); + } + + /// A baseline that outgrows its own explicit ceiling stops the run and says which number to move. + #[test] + fn a_baseline_that_outgrows_its_ceiling_names_the_ceiling() { + let cause = baseline_memory_error( + Utf8Path::new("/workspace/target/debug/deps/unit"), + Some(300 * 1024 * 1024), + 256 * 1024 * 1024, + ) + .to_string(); + + // The user set this ceiling themselves, and no mutant is involved, so the message has to + // point at the flag rather than read like a mutant was caught. + assert!(cause.contains("unit"), "{cause}"); + assert!(cause.contains("--baseline-memory-limit"), "{cause}"); + assert!(cause.contains("256.0 MB"), "{cause}"); + assert!(cause.contains("300.0 MB"), "{cause}"); + } + + #[test] + fn a_baseline_timeout_names_the_binary_that_stopped_progress() { + let cause = baseline_timeout_error(Utf8Path::new("/workspace/target/debug/deps/unit")).to_string(); + + // A timeout before mutants run is a property of the fixed suite, so the message must point + // at the binary the user can run directly. + assert!(cause.contains("unit"), "{cause}"); + assert!(cause.contains("ten minutes"), "{cause}"); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/build.rs b/crates/cargo-gamma-lib/src/exec/build.rs new file mode 100644 index 000000000..23612079c --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/build.rs @@ -0,0 +1,1241 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; +use std::time::Instant; + +use camino::Utf8PathBuf; + +use super::cargo_options::BuildLimits; +use super::events::Events; +use super::test_binary::{TestBinary, test_binaries}; +use super::verdict::tail; +use super::workspace::Workspace; +use crate::discover::Plan; +use crate::error::{Error, error}; +use crate::model::{Mutant, Outcome}; +use crate::schema::Guard; +use crate::{HashMap, HashSet, Result}; + +mod blame; +mod complaints; +mod invoke; +mod messages; +mod splices; + +#[cfg(all(test, not(miri)))] +mod tests; + +use blame::blame; +use complaints::{DIAGNOSTIC_LIMIT, complaints, diagnostics, leading, manifests_of, prioritize}; +use invoke::run_cargo; +use messages::compiled_sources; +use splices::Splices; + +/// Where each live mutant's guard landed, by ordinal, paired with the file it landed in. +type Guards = HashMap; + +/// A test-only stand-in for one proof build: a verdict on which spliced ordinals fail to compile. +/// +/// Mirrors [`Converger::subset_fails`]'s own return: `Some(true)` failed, `Some(false)` compiled, +/// `None` could not be told (a timeout). +#[cfg(test)] +type SubsetOracle = fn(&HashSet) -> Option; + +/// What the stale build-ordering hints actually did, counted rather than modelled. +/// +/// Every figure here is something that happened. There is deliberately no "rounds saved", because +/// that number does not exist: it is the length of a convergence that was never run, over a mutant +/// population that was never offered to the compiler in that shape, and any figure printed for it +/// would be a model of a counterfactual dressed up as a measurement. What *can* be measured is how +/// many mutants the hints put in front of the compiler early and how many of those the compiler +/// then refused, and those two together say whether the hints are worth their round: `offered` +/// close to `confirmed` is a hint set that is paying, and `confirmed` near zero is one that is +/// costing a build per stage and buying nothing. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct OrderingHints { + /// How many hinted mutants were put in front of the compiler in a probe round. + pub offered: usize, + + /// How many of those the compiler then blamed, which is the hint turning out to be right. + /// + /// A hinted mutant that compiles is not an error and is not withheld: it stays live and is + /// judged by the run exactly as if it had never been hinted. This counts only the ones the + /// compiler independently refused. + pub confirmed: usize, + + /// How many probe rounds the run spent, which is the cost side of the trade. + pub rounds: u32, +} + +/// How many hinted mutants make a probe round worth the build it costs. +/// +/// A probe round is one extra cargo invocation. It repays that by putting the mutants likeliest to +/// fail in front of the compiler with nothing else to mask them, so they are blamed together +/// instead of a few per wave. Below a handful there is nothing to unmask — the ordinary rounds +/// would have found them just as fast — and the build would be spent for nothing, so the round is +/// simply not taken. The number is a judgement rather than a measurement, which is exactly why the +/// run reports what the probes offered and confirmed instead of claiming a saving. +const PROBE_FLOOR: usize = 4; + +/// What one round of a build cost, and what it bought. +/// +/// A run reports the series rather than a total because the two ends mean opposite things. The +/// first round is what compiling this workspace costs at all, which no amount of mutant selection +/// will avoid; every round after it exists only because some mutant did not compile, and its time +/// is the price of that mutant. A total conflates the two and so points at the wrong remedy. +#[derive(Debug, Clone)] +pub struct Round { + /// How long the round's cargo invocation took. + pub elapsed: Duration, + + /// How many mutants the round withdrew, which is zero for the round that finally compiled. + pub withdrew: usize, +} + +/// How many mutants one rustc error code withdrew from one mutator. +/// +/// The pairing is what makes the figure actionable. A code on its own says what kind of code the +/// instrumented tree contained; a mutator on its own says which mutator is expensive; the two +/// together say *which mutator emits which mistake*, which is the form a heuristic can be written +/// against. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Withdrawal { + /// The rustc error code, or empty for a diagnostic that carried none. + pub code: String, + + /// The mutator whose mutants the code was reported against. + pub mutator: String, + + /// How many mutants — not how many diagnostics — the pair accounts for. + pub mutants: usize, +} + +/// What a build round produced. +#[derive(Debug, Default)] +pub(super) struct Build { + /// What each round of this run's builds cost, oldest first. + pub(super) history: Vec, + + pub(super) binaries: Vec, + pub(super) withdrawn: usize, + + /// How many rollback rounds the whole run spent, summed over its builds. + pub(super) rounds: u32, + + /// Whether a narrowed build was abandoned and the whole workspace built instead. + pub(super) widened: bool, + + /// Why the withdrawn mutants were withdrawn, densest pair first. + pub(super) census: Vec, + + /// The population given up on when this build could not be made to compile at all. + /// + /// `None` for a build that converged, which is the ordinary case. When it is set the binaries + /// are empty — there is nothing to run a mutant against — and the run reports what it knows + /// rather than exiting with nothing at all. + pub(super) stuck: Option, + + /// What the stale build-ordering hints put in front of the compiler, and what came of it. + pub(super) ordering: OrderingHints, +} + +/// What one build arrived at. +/// +/// A build that cannot be made to compile is not the same kind of event as a build that could not +/// be started. The first is a fact about this tree and these mutants, which a run can record, +/// report and carry on from; the second is a fact about the machine, which nothing downstream can +/// say anything useful about. Only the first is modelled here — everything else stays an `Err`. +#[derive(Debug)] +enum Convergence { + /// The build succeeded, carrying cargo's JSON stream from the round that succeeded. + Built(String), + + /// The build failed and no further mutant can be withdrawn to change that. + Stuck(Error), +} + +/// What a proof build isolated after rustc's spans could not identify a cause. +enum Isolation { + /// One or more mutants failed even without any other mutant from their item. + Blamed(Vec), + + /// Mutants in one item only fail in combination, so none can honestly be blamed alone. + Item(Vec), +} + +/// The mutants a build gave up on, and why. +/// +/// The reason is the text of the error that would otherwise abort the whole run. It is preserved +/// word for word — the rollback-limit advice about a falling or flat withdrawal series, the excerpt +/// of what cargo said — because that text is the only thing that tells a reader whether the answer +/// is to raise a limit, to fix a build script, or to look somewhere else entirely. +#[derive(Debug)] +pub(super) struct Abandoned { + /// The diagnostic explaining why the build could not be converged. + pub(super) reason: String, + + /// The ordinals of the mutants that will never be run because of it, ascending. + pub(super) ordinals: Vec, +} + +/// Drives the build, withdrawing mutants that cannot compile until what is asked for compiles. +/// +/// A run converges the workspace one stage at a time and then once as a whole, and every one of +/// those builds shares the same withdrawal set and budget reference. Sharing the withdrawal set is +/// what lets a stage inherit what earlier stages already ruled out: a mutant already known to be +/// unbuildable stays withdrawn for the rest of the run. +/// +/// The round counter is not shared. `--rollback-rounds` caps the rounds one build may spend +/// converging, so it is reset for each build; a cumulative counter would let early stages spend the +/// budget and leave the build that decides the run with no chance to converge at all. +#[derive(Debug, Default)] +pub(super) struct Converger { + withdrawn: HashSet, + + /// The subset of `withdrawn` that was given up on rather than blamed. + /// + /// A withdrawn mutant is one the compiler pointed at: it is unviable, and saying so is a + /// verdict. A mutant in here was never accused of anything — the build it belonged to could + /// not be made to compile, so its whole population was taken out of the tree to let the run + /// carry on. Conflating the two would report a mutant the tool never judged as one the tool + /// judged unbuildable, which is the exact confusion this run is trying to avoid. + abandoned: HashSet, + + /// How many rounds the build currently converging has spent, reset at the start of each one. + rounds: u32, + + /// How many rounds the whole run has spent, which is what the run reports. + total_rounds: u32, + + /// How many mutants each failed round of the current build blamed, oldest first. + /// + /// Kept so that a build which hits the limit can say whether it was converging, which is the + /// only thing that decides whether raising the limit would have helped. Reset with `rounds`, + /// so that the advice describes the build that just failed rather than the ones before it. + /// + /// Counts what a round blamed rather than what it withdrew, because the two differ in exactly + /// the round the reader is being told about: the round that hits the limit blames mutants and + /// is then stopped before it can withdraw them. Recording the withdrawal would leave that round + /// out of its own diagnostic, and the trend the advice reads is a trend in what the rounds are + /// finding. + per_round: Vec, + + /// What every round of every build in this run cost, oldest first. + /// + /// Unlike `per_round`, this is never reset: it describes the whole run, because what a reader + /// wants to know is where the run's build time went, not where one stage's did. + history: Vec, + + /// How long the first build of the run took, which every later budget is scaled from. + /// + /// Set once and never reset. A stage builds a fraction of the workspace, so letting a small + /// stage set this reference would leave every later stage — and the whole-workspace build that + /// follows them — with a budget derived from a build that was never comparable. + first_round: Option, + + /// What the tree already holds, so a round rewrites only the files it changed. + splices: Splices, + + /// Source files named by successful staged and final builds. + /// + /// The final test-target build does not necessarily compile a package's default target: a + /// library with `test = false` and no integration tests is one example. Staged default-target + /// builds still prove those sources compiled, so their dep-info must remain part of the final + /// inventory rather than letting the test-target artifact stream erase them. + compiled: Option>, + + /// The rustc error code that first named each withdrawn mutant. + /// + /// The count of withdrawals says whether the number is large; only the codes say whether it is + /// worth acting on. A run dominated by `E0308` is one where a mutator produces ill-typed code + /// and could be taught not to, while one dominated by the borrow checker is a cost of the + /// schema itself. Distinguishing them by hand meant patching this file every time, which is why + /// it is kept rather than derived on demand. + census: HashMap, + + /// Mutants that failed to compile for some earlier run whose build context no longer matches. + /// + /// Held by content id rather than by ordinal because ordinals are handed out stage by stage as + /// the run scans, so most of them do not exist yet when this is set. + /// + /// This is evidence about *order* and nothing else. Not one mutant in here is withheld, + /// excluded, settled or scored on the strength of it: every one is spliced into the tree and + /// offered to the compiler, and the only thing the hint decides is that it is offered early, + /// on its own, where a mutant that really is unviable is blamed with nothing else masking it. + /// A hint that turns out to be wrong costs exactly the round it was probed in, and the mutant + /// goes on to be built and judged as if it had never been named. That is what makes the tier + /// safe under a context that no longer matches, which filtering would not be. + hinted: HashSet, + + /// Ordinals already put through a probe round, so no build pays for the same probe twice. + probed: HashSet, + + /// What the probe rounds offered and what the compiler made of it. + ordering: OrderingHints, + + /// Whether every build this run makes must compile the whole workspace. + /// + /// Set when the preflight check could only be made to pass by widening: the tree the run works + /// in compiles under cargo's feature unification over every member and does not compile under + /// any subset of them. A narrowed build after that is a build already known to fail, and its + /// failure would be attributed to whichever mutants the rollback loop happened to blame — so + /// the same feature unification the preflight proved is what every later build asks for. + whole_workspace: bool, + + /// A test-only stand-in for the proof build in [`Self::subset_fails`]. + /// + /// Reaching [`Isolation::Item`] needs a subset that compiles alone but fails only in + /// combination, which no cheap real mutation fixture produces. When set, each proof build asks + /// this function — a pure verdict on which ordinals are spliced — instead of invoking cargo, so + /// a test can drive isolation to any branch deterministically without a real interaction bug. + #[cfg(test)] + subset_oracle: Option, +} + +/// What a preflight check settled: the scope it needed, and what it cost to pass at all. +#[derive(Debug)] +pub(super) struct Preflight { + /// Whether only a whole-workspace build was shown to compile. + /// + /// Carried rather than discarded because a narrowed build after a wide-only success is a build + /// already known to fail — and its failure would be blamed on mutants, settling valid ones as + /// unbuildable and quietly shrinking the population the score is taken over. + pub(super) whole_workspace: bool, + + /// The packages the check had to give up on, empty in the ordinary case. + pub(super) dropped: Vec, +} + +impl Preflight { + /// The result of a check that passed in the scope it was asked about. + const fn narrow(dropped: Vec) -> Self { + Self { + whole_workspace: false, + dropped, + } + } +} + +impl Converger { + /// A converger that front-loads the mutants an out-of-context record says would not compile. + /// + /// `hinted` holds mutant content ids. Passing ids that name nothing in this run is harmless: + /// they resolve to no ordinal and no probe round is taken for them. + pub(super) fn guided(hinted: HashSet) -> Self { + Self { hinted, ..Self::default() } + } + + /// Records that only a whole-workspace build has been shown to compile. + /// + /// Called with the preflight's own answer, so that the scope which proved the tree sound is the + /// scope every later build uses. See [`Self::whole_workspace`]. + pub(super) const fn require_whole_workspace(&mut self) { + self.whole_workspace = true; + } + + /// Invalidates position-based splice indexes after the plan is sorted. + pub(super) fn plan_reordered(&mut self) { + self.splices.plan_reordered(); + } + + /// The package selection a build may actually use. + /// + /// Every narrowing goes through here, so there is one place that can answer "may this build be + /// narrowed at all" and no build can be narrowed by forgetting to ask. + const fn scoped<'names>(&self, select: Option<&'names [String]>) -> Option<&'names [String]> { + if self.whole_workspace { None } else { select } + } + + /// Instruments the tree and builds it until it compiles, withdrawing whatever stands in the way. + /// + /// `select` names the packages to build, or is `None` for the whole workspace. `verb` is the + /// cargo command and its flags. + /// + /// Returns cargo's JSON stream from the build that finally succeeded, or the diagnostic for a + /// build that could not be made to compile at all. That second case is returned rather than + /// raised because it is a result: the run can withdraw the population it belongs to, keep every + /// verdict it has already reached, and still produce a report. + fn converge( + &mut self, + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + verb: &[&str], + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result { + // The round budget is per build: what earlier stages spent converging is not this build's + // to answer for, and the withdrawal series the limit error reads has to describe the build + // that failed. The withdrawal set is deliberately left alone — a mutant already known not + // to compile stays withdrawn for the rest of the run. + self.rounds = 0; + self.per_round.clear(); + + // Before the first ordinary round, and only ever before it. Whatever the probe withdraws is + // withdrawn by the compiler's own accusation in a real build, so the loop below starts from + // a tree the compiler has already ruled on rather than from a guess. + self.probe(work, plan, select, verb, limits, events)?; + + loop { + self.rounds = self.rounds.saturating_add(1); + self.total_rounds = self.total_rounds.saturating_add(1); + + let guards = self.splices.instrument(work, plan, &self.withdrawn)?; + + let started = Instant::now(); + let outcome = run_cargo(work, plan, verb, select, limits, self.first_round, events)?; + let elapsed = started.elapsed(); + + let Some(stdout) = outcome.stdout else { + let budget = limits.budget(self.first_round).unwrap_or(elapsed); + + return Err(Self::build_timeout_error(budget)); + }; + + if self.first_round.is_none() { + self.first_round = Some(elapsed); + } + + if outcome.succeeded { + self.history.push(Round { elapsed, withdrew: 0 }); + + return Ok(Convergence::Built(stdout)); + } + + let blamed = blame(&stdout, &work.root, &guards); + + if blamed.is_empty() { + if let Some(isolated) = self.isolate(work, plan, select, verb, limits, events)? { + let ordinals = match &isolated { + Isolation::Blamed(ordinals) | Isolation::Item(ordinals) => ordinals, + }; + + self.history.push(Round { + elapsed, + withdrew: ordinals.len(), + }); + + for ordinal in ordinals { + let _ = self.withdrawn.insert(*ordinal); + let _ = self.census.entry(*ordinal).or_default(); + } + + if let Isolation::Item(ordinals) = isolated { + self.abandoned.extend(ordinals); + } + + continue; + } + + self.history.push(Round { elapsed, withdrew: 0 }); + + return Ok(Convergence::Stuck(Self::unattributed_build_error(work, &stdout, &outcome.stderr))); + } + + // This round joins the series before the limit is checked, because the round the limit + // stops is the one the diagnostic is about: it blamed these mutants and was refused the + // chance to withdraw them. Reading the series without it leaves a one-round budget with + // nothing to report and the advice saying the last round found nothing, which is the + // opposite of what happened. + self.per_round.push(blamed.len()); + + if self.rounds >= limits.rounds() { + let error = Self::rollback_limit_error(self.rounds, limits.rounds(), &self.per_round, work, &stdout); + + // Nothing was withdrawn: `history` is what the run reports its build time against, + // and this round ended without applying its blame. + self.history.push(Round { elapsed, withdrew: 0 }); + + return Ok(Convergence::Stuck(error)); + } + + self.history.push(Round { + elapsed, + withdrew: blamed.len(), + }); + + for (ordinal, code) in blamed { + let _ = self.withdrawn.insert(ordinal); + let _ = self.census.entry(ordinal).or_insert(code); + } + } + } + + /// Uses proof builds to isolate a failure whose diagnostic spans name no guard. + /// + /// The pristine stage is tried first so a linker, build script or native dependency failure is + /// never blamed on whichever mutant happens to be bisected last. A real schema failure is then + /// narrowed by enclosing item and finally by ordinal. The extra builds are rare, warm, and + /// logarithmic for the ordinary single-mutant case; their cost is preferable to discarding a + /// package's entire population. + fn isolate( + &mut self, + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + verb: &[&str], + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result> { + let mut candidates: Vec<&Mutant> = plan + .mutants + .iter() + .filter(|mutant| { + mutant.ordinal > 0 + && !self.withdrawn.contains(&mutant.ordinal) + && select.is_none_or(|packages| packages.iter().any(|package| package.as_str() == &*mutant.package)) + }) + .collect(); + + if candidates.is_empty() || self.subset_fails(work, plan, select, verb, limits, events, &candidates, &[])? != Some(false) { + return Ok(None); + } + + candidates.sort_by(|left, right| left.item_path.cmp(&right.item_path).then_with(|| left.ordinal.cmp(&right.ordinal))); + let population = candidates.clone(); + + let mut items: Vec> = Vec::new(); + + for mutant in candidates { + if items + .last() + .and_then(|item| item.first()) + .is_some_and(|first| first.item_path == mutant.item_path) + { + items + .last_mut() + .unwrap_or_else(|| unreachable!("the item was just observed")) + .push(mutant); + } else { + items.push(vec![mutant]); + } + } + + while items.len() > 1 { + let middle = items.len() / 2; + let left = items[..middle].concat(); + let right = items[middle..].concat(); + + if self.subset_fails(work, plan, select, verb, limits, events, &population, &left)? == Some(true) { + items.truncate(middle); + continue; + } + + if self.subset_fails(work, plan, select, verb, limits, events, &population, &right)? == Some(true) { + drop(items.drain(..middle)); + continue; + } + + // Neither half fails alone, so the failure is an interaction. Remove one item at a + // time from the failing set and keep the first removal proven to restore the build. + for item in &items { + let active: Vec<&Mutant> = population + .iter() + .copied() + .filter(|candidate| !item.iter().any(|removed| removed.ordinal == candidate.ordinal)) + .collect(); + + if self.subset_fails(work, plan, select, verb, limits, events, &population, &active)? == Some(false) { + return Ok(Some(Isolation::Item(item.iter().map(|mutant| mutant.ordinal).collect()))); + } + } + + return Ok(None); + } + + let item = items.pop().unwrap_or_default(); + let mut narrowed = item.clone(); + + while narrowed.len() > 1 { + let middle = narrowed.len() / 2; + let left = &narrowed[..middle]; + let right = &narrowed[middle..]; + + if self.subset_fails(work, plan, select, verb, limits, events, &population, left)? == Some(true) { + narrowed.truncate(middle); + } else if self.subset_fails(work, plan, select, verb, limits, events, &population, right)? == Some(true) { + drop(narrowed.drain(..middle)); + } else { + return Ok(Some(Isolation::Item(item.iter().map(|mutant| mutant.ordinal).collect()))); + } + } + + Ok(Some(Isolation::Blamed(narrowed.iter().map(|mutant| mutant.ordinal).collect()))) + } + + /// Builds one chosen subset of a candidate population. + #[expect( + clippy::too_many_arguments, + reason = "a proof build needs the same complete context as convergence" + )] + fn subset_fails( + &mut self, + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + verb: &[&str], + limits: BuildLimits, + events: &mut dyn Events, + population: &[&Mutant], + active: &[&Mutant], + ) -> Result> { + let active: HashSet = active.iter().map(|mutant| mutant.ordinal).collect(); + + #[cfg(test)] + if let Some(oracle) = self.subset_oracle { + return Ok(oracle(&active)); + } + + let mut withdrawn = self.withdrawn.clone(); + + for mutant in population { + if !active.contains(&mutant.ordinal) { + let _ = withdrawn.insert(mutant.ordinal); + } + } + + let _guards = self.splices.instrument(work, plan, &withdrawn)?; + let started = Instant::now(); + let outcome = run_cargo(work, plan, verb, select, limits, self.first_round, events)?; + let elapsed = started.elapsed(); + + self.total_rounds = self.total_rounds.saturating_add(1); + self.history.push(Round { elapsed, withdrew: 0 }); + + Ok(outcome.stdout.map(|_stdout| !outcome.succeeded)) + } + + /// Builds only the mutants an out-of-context record expects to fail, before anything else. + /// + /// This is the whole of what a stale unviability tier is allowed to do. It does not withhold a + /// mutant, settle one, exclude one, or touch the population in any way — every live mutant in + /// the selection is still built and still judged. What it changes is the *order* the compiler + /// meets them in: the hinted ones go first, alone, so that a genuinely unviable mutant is + /// blamed with no other mutant's error masking it, and the ordinary convergence below starts + /// with them already out of the tree instead of discovering them a wave at a time. + /// + /// Everything it withdraws is withdrawn on the compiler's evidence, in a real build, exactly as + /// an ordinary round withdraws. A hint that was wrong simply produces a mutant that compiles: + /// it stays live, it is spliced back in by the next round, and it is judged as if it had never + /// been hinted at all. That is the property that makes the tier safe when the context no longer + /// matches, and it is why the probe is allowed to run without any envelope check. + /// + /// Best-effort throughout. A probe that times out, that cannot be attributed, or that fails for + /// reasons no mutant can be blamed for is abandoned without a word: the ordinary convergence + /// that follows asks the same question properly and is the one whose answer the run reports. + fn probe( + &mut self, + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + verb: &[&str], + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result<()> { + let (candidates, deferred) = self.probe_sets(plan, select); + + if candidates.len() < PROBE_FLOOR { + return Ok(()); + } + + events.build_progress(&format!( + "probing {} that did not compile for an earlier run before building the rest", + crate::report::quantity(candidates.len(), "mutant") + )); + + // Marked before the build rather than after it, so a probe that fails in any of the ways + // below is still never repeated: the cost of a wasted round is bounded at one per mutant + // for the whole run. + self.probed.extend(candidates.iter().copied()); + self.ordering.offered = self.ordering.offered.saturating_add(candidates.len()); + self.ordering.rounds = self.ordering.rounds.saturating_add(1); + + let guards = self.splices.instrument(work, plan, &deferred)?; + + let started = Instant::now(); + let outcome = run_cargo(work, plan, verb, select, limits, self.first_round, events)?; + let elapsed = started.elapsed(); + + // Counted as a round of this run's build time because that is what it is, and hiding it + // would make the reported build total disagree with the clock. It is deliberately not + // charged against `--rollback-rounds`, which caps how many times a build may withdraw + // before it is declared unconvergeable: the probe is an extra round the run chose to + // spend, and letting it eat that budget could turn a build that would have converged into + // an abandoned population. + self.total_rounds = self.total_rounds.saturating_add(1); + + let Some(stdout) = outcome.stdout else { + self.history.push(Round { elapsed, withdrew: 0 }); + + return Ok(()); + }; + + // Deliberately not allowed to set `first_round`, which every later build timeout is scaled + // from. A probe compiles a fraction of the mutants and usually stops on the first errors, + // so its elapsed time is not what a full round of this build costs — adopting it as the + // reference would set every later budget from a build that was never comparable, and the + // run would start timing out builds that are merely honest about their size. + + if outcome.succeeded { + // Every hint was wrong: nothing here is unviable now. Nothing is withdrawn and nothing + // is recorded against these mutants — the round bought only the knowledge that it did + // not need to be taken, which `offered` against `confirmed` is what reports. + self.history.push(Round { elapsed, withdrew: 0 }); + + return Ok(()); + } + + let blamed = blame(&stdout, &work.root, &guards); + + self.history.push(Round { + elapsed, + withdrew: blamed.len(), + }); + + self.ordering.confirmed = self.ordering.confirmed.saturating_add(blamed.len()); + + for (ordinal, code) in blamed { + let _ = self.withdrawn.insert(ordinal); + let _ = self.census.entry(ordinal).or_insert(code); + } + + Ok(()) + } + + /// The mutants to probe, and the exclusion set that leaves only them in the tree. + /// + /// Restricted to the packages this build actually compiles. A mutant outside the selection + /// contributes no diagnostic to this build, so deferring it would buy nothing and would rewrite + /// a file some later stage is about to want instrumented again — churn that costs rebuilds + /// without answering anything. + fn probe_sets(&self, plan: &Plan, select: Option<&[String]>) -> (Vec, HashSet) { + let mine = |mutant: &Mutant| select.is_none_or(|names| names.iter().any(|name| name.as_str() == &*mutant.package)); + + let mut candidates: Vec = Vec::new(); + let mut deferred = self.withdrawn.clone(); + + for mutant in &plan.mutants { + if mutant.ordinal == 0 || self.withdrawn.contains(&mutant.ordinal) || !mine(mutant) { + continue; + } + + if self.hinted.contains(&mutant.id) && !self.probed.contains(&mutant.ordinal) { + candidates.push(mutant.ordinal); + } else { + let _ = deferred.insert(mutant.ordinal); + } + } + + // Sorted so that the probe a run takes depends only on the plan and the hints, never on the + // iteration order of a set. The build keys by ordinal, but the reported counts and any + // future tie-break would otherwise vary between two runs over an identical tree. + candidates.sort_unstable(); + + (candidates, deferred) + } + + fn build_timeout_error(budget: Duration) -> Error { + error!( + "the build was still running after {budget:.0?} and was stopped. A run builds once, so a \ + build that does not finish costs the whole run; raise --build-timeout if this one is simply slow." + ) + } + + fn unattributed_build_error(work: &Workspace, stdout: &str, stderr: &str) -> Error { + let diagnostics = diagnostics(stdout); + + // A build that produced no diagnostics at all did not fail the way this message assumes. + // The compiler was never reached — a build script panicked, a native library is missing, a + // dependency would not resolve, a package spec was ambiguous — and every one of those is + // explained on stderr and nowhere else. Saying "does not compile" here would send the + // reader hunting for a broken mutant that was never generated. + if diagnostics.is_empty() { + return error!( + "the instrumented tree failed to build, and the compiler reported nothing, so no \ + mutant can be blamed for it. The cause is usually something cargo hit before it \ + reached the code — a build script, a missing native dependency, a bad invocation — \ + and it is almost always in what cargo said:\n\n{}\n\n{}", + tail(&complaints(stderr), 30), + work.inspect_hint() + ); + } + + error!( + "the instrumented tree does not compile and the failure could not be attributed to a mutant.\n\ + {}\n\n{}", + work.inspect_hint(), + leading(&diagnostics, DIAGNOSTIC_LIMIT) + ) + } + + /// Explains a build that ran out of rollback rounds. + /// + /// `per_round` is what each round of this build blamed, oldest first, and it includes the round + /// the limit stopped. Every entry is non-zero: a round that blames nothing is not a rollback + /// failure at all and is reported by `unattributed_build_error` instead. + fn rollback_limit_error(rounds: u32, limit: u32, per_round: &[usize], work: &Workspace, stdout: &str) -> Error { + let blamed: usize = per_round.iter().sum(); + + // Whether the rounds were still making progress is the one thing that decides what to do + // next, and it is invisible from a total. A falling tail means the cap was simply too low + // for this tree; a flat one means each round is uncovering as much as the last, and more + // rounds will not help. + let recent: Vec = per_round.iter().rev().take(5).rev().map(usize::to_string).collect(); + + error!( + "the instrumented tree still does not compile after {rounds} of the {limit} rollback rounds \ + this build is allowed, having blamed unviable mutants in each of them ({blamed} blamed \ + during this build, the last round's among them not withdrawn because the limit stopped it).\n\ + Mutants blamed in the last rounds of this build: {}.\n\ + If those counts are falling, the tree was converging and --rollback-rounds is simply too \ + low for it. If they are flat, each round is uncovering as much as the last and raising \ + the limit will only make the failure slower.\n\ + {}\n\n{}", + recent.join(", "), + work.inspect_hint(), + leading(&diagnostics(stdout), DIAGNOSTIC_LIMIT) + ) + } + + fn missing_guard_error(missing: &Mutant) -> Error { + error!( + "internal error: no guard was emitted for the mutant at {}:{}, so it could not \ + be tested. Please report this.\n {}", + missing.file, + missing.line, + missing.describe() + ) + } + + /// Checks that the copied tree compiles before a single mutant is applied to it. + /// + /// This is what makes every later compiler error attributable. The tree that the staged builds + /// and the baseline compile is this same tree with guards written into it, so once this passes, + /// an error that appears afterwards was introduced by a mutant and nothing else. Without it, + /// gamma cannot tell a broken mutant from code that never compiled, and reports the second as + /// though it were the first — which sends the reader hunting through their own source for a + /// fault that was there before the tool arrived. + /// + /// It runs `cargo check` rather than a build because it is a question about the code and not + /// about artifacts: no codegen, no linking, and nothing it produces is kept. What it cannot see + /// is exactly what `check` never reaches — link failures and post-monomorphization errors — so + /// passing here is a strong precondition rather than a total one, and the later builds still + /// report a failure they cannot pin on any mutant instead of absorbing it. + /// + /// `--tests` is not optional. The baseline build compiles test targets, so leaving them out + /// here would clear the libraries and let a broken test target fail later, unattributably, in + /// the middle of a run that had already paid for instrumentation. + pub(super) fn preflight( + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + mutating: &[String], + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result { + match Self::check(work, plan, select, mutating, limits, events) { + Ok(()) => Ok(Preflight::narrow(Vec::new())), + + // A narrowed check is not a smaller version of the whole one: cargo unifies features + // over the packages it is told to build, so a target that only compiles because some + // package outside the selection switches a feature on fails here through no fault of + // the tree. Accusing the caller's code of not compiling on that evidence would be the + // very mistake this check exists to prevent, so the selection is abandoned and the + // question asked again of the whole workspace, which is how `finish` treats the same + // trap. + Err(narrow) if select.is_some() => { + events.build_progress("the selected packages alone did not build, checking the whole workspace instead"); + + // The wider check is asked one question only: does the narrow failure survive real + // feature unification? Its own diagnostics are not reported, because a workspace + // this size usually has something broken in a package the caller never mentioned, + // and answering "your tree does not compile" with errors from a crate they did not + // choose to mutate sends them to fix the wrong thing. What they can act on is the + // failure in their own selection. + // + // Which scope answered is carried back rather than discarded. A wide success is not + // "the tree compiles"; it is "the tree compiles when cargo unifies features over + // every member", and a later build that narrowed again would reproduce the very + // failure this branch has just proved is not any mutant's doing. + if Self::check(work, plan, None, mutating, limits, events).is_ok() { + return Ok(Preflight { + whole_workspace: true, + dropped: Vec::new(), + }); + } + + Self::retreat(work, plan, select, mutating, limits, events, narrow) + } + + Err(error) => Err(error), + } + } + + /// The last attempt: check only the packages this run is actually mutating. + /// + /// Both wider questions have now failed, and neither answers the one that matters. The + /// selection is everything that can reach a mutant, and the workspace is everything at all, so + /// a single package that does not compile — a `sys` crate without its native library, a + /// sibling broken by somebody else's commit — fails both while saying nothing about the code + /// the caller asked to measure. Refusing to run on that evidence turns a workspace's unrelated + /// breakage into a tool that cannot be used at all, when narrowing the scope by hand would + /// have worked: a flag the caller had no reason to know they needed. + /// + /// Succeeding here is not free, and the cost is not the tool's to hide. The packages dropped + /// are the ones whose *tests* can no longer convict anything, so a mutant one of them would + /// have killed now survives — a survivor that reads as a gap in the suite and is nothing of + /// the kind. They are returned so the run can narrow its oracle to match what it checked, and + /// so the report can name them. + /// + /// The narrow error is what a failure here reports, not this attempt's own. Features are not + /// unified the way the real build unifies them over a selection this small, so its diagnostics + /// are the least trustworthy of the three, while the narrow ones are about the selection the + /// builds will genuinely compile. + fn retreat( + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + mutating: &[String], + limits: BuildLimits, + events: &mut dyn Events, + narrow: Error, + ) -> Result { + let dropped: Vec = select + .unwrap_or_default() + .iter() + .filter(|package| !mutating.contains(package)) + .cloned() + .collect(); + + // The selection was already nothing but the mutated packages, so this attempt is the first + // one again and would fail again. There is no narrower run to retreat to. + if dropped.is_empty() { + return Err(narrow); + } + + events.build_progress("the whole workspace did not build either, checking only the packages being mutated"); + + Self::check(work, plan, Some(mutating), mutating, limits, events).map_err(|_last| narrow)?; + + Ok(Preflight::narrow(dropped)) + } + + /// Runs one preflight check over the packages named, or the whole workspace when none are. + fn check( + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + mutating: &[String], + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result<()> { + let outcome = run_cargo(work, plan, &["check", "--tests", "--keep-going"], select, limits, None, events)?; + + let Some(stdout) = outcome.stdout else { + return Err(Self::build_timeout_error(limits.budget(None).unwrap_or_default())); + }; + + if outcome.succeeded { + return Ok(()); + } + + let mut diagnostics = diagnostics(&stdout); + + prioritize(&mut diagnostics, &manifests_of(plan, &work.root, mutating)); + + // Nothing on the JSON stream means the compiler was never reached, which is the same class + // of failure `unattributed_build_error` describes and wants the same explanation. Saying + // "does not compile" over an empty diagnostic list would be a lie about a build script or a + // missing native library. + if diagnostics.is_empty() { + return Err(error!( + "the tree could not be checked, and the compiler reported nothing, so the cause is \ + something cargo hit before it reached the code — a build script, a missing native \ + dependency, a bad invocation:\n\n{}", + tail(&complaints(&outcome.stderr), 30) + )); + } + + Err(error!( + "this tree does not compile before any mutation is applied, so there is nothing to \ + measure against.\n\ + These are the compiler's own errors, on the unmodified sources. Note that `cargo build` \ + alone would not show them, because it does not build test targets; `cargo check --tests` \ + reproduces them. A feature selection that leaves a test target's dependencies switched \ + off is the usual cause.\n\n{}", + leading(&diagnostics, DIAGNOSTIC_LIMIT) + )) + } + + /// Compiles one stage's libraries, so its mutants are ruled on before anything downstream. + /// + /// Only libraries and binaries are built, never test targets. Every mutant lives in one of + /// those, so this sees every diagnostic a mutant can cause, and it avoids the one thing a + /// subset build cannot reproduce: cargo resolves features over the packages being built, so a + /// test target that relies on a feature some other package switches on does not compile on its + /// own. Those targets are left to the whole-workspace build, where the features are the real + /// ones. + /// A stage that cannot be made to compile does not stop the run. Its own mutants are taken out + /// of the tree — which restores exactly the sources the preflight check already proved compile + /// — and what it gave up on is returned so the run can report it. See [`Self::abandon`]. + pub(super) fn stage( + &mut self, + work: &Workspace, + plan: &mut Plan, + packages: &[String], + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result> { + // Nothing is narrated from inside a stage: the stage reports what it found and what it + // withdrew as one line when it is done, and a round-by-round commentary underneath that + // would bury the sequence the whole arrangement exists to show. + match self.converge(work, plan, self.scoped(Some(packages)), &["build", "--keep-going"], limits, events)? { + Convergence::Built(stdout) => { + self.remember_compiled(&stdout, &work.root); + Ok(None) + } + Convergence::Stuck(reason) => Ok(Some(self.abandon(plan, Some(packages), &reason))), + } + } + + /// Takes a population out of the run because the build holding it could not be made to compile. + /// + /// `packages` names whose mutants to give up on, or is `None` for every one still live. + /// + /// Each mutant is recorded as [`Outcome::NotBuilt`] rather than [`Outcome::CompileError`]: the + /// compiler never accused it of anything, and reporting a mutant nobody judged as unviable + /// would be inventing a verdict. They are added to the withdrawal set as well, so the next + /// build instruments the tree without them — that is what makes carrying on possible, since a + /// tree with every one of them withdrawn is the pristine tree the preflight check cleared. + fn abandon(&mut self, plan: &mut Plan, packages: Option<&[String]>, reason: &Error) -> Abandoned { + let mut ordinals = Vec::new(); + + for mutant in &mut plan.mutants { + let mine = packages.is_none_or(|packages| packages.iter().any(|package| package.as_str() == &*mutant.package)); + + if mutant.ordinal == 0 || !mine || self.withdrawn.contains(&mutant.ordinal) { + continue; + } + + mutant.outcome = Outcome::NotBuilt; + mutant.note = Some("the build this mutant belongs to could not be made to compile, so it was never run".to_owned()); + + ordinals.push(mutant.ordinal); + } + + self.withdrawn.extend(ordinals.iter().copied()); + self.abandoned.extend(ordinals.iter().copied()); + ordinals.sort_unstable(); + + Abandoned { + reason: reason.to_string(), + ordinals, + } + } + + /// Writes the withdrawal verdicts back onto the plan. + /// + /// Only mutants the compiler actually blamed are called unviable; what was abandoned wholesale + /// already carries [`Outcome::NotBuilt`] from [`Self::abandon`] and keeps it. + pub(super) fn settle(&self, plan: &mut Plan) { + for mutant in &mut plan.mutants { + if self.abandoned.contains(&mutant.ordinal) { + mutant.outcome = Outcome::NotBuilt; + mutant.note = + Some("the instrumented forms in this item could not compile together, so its mutants were not run".to_owned()); + } else if self.withdrawn.contains(&mutant.ordinal) { + mutant.outcome = Outcome::CompileError; + } + } + } + + /// Compiles the test targets of `select`, or of the whole workspace when it is `None`. + /// + /// Returns cargo's JSON stream, whose artifact messages name the test binaries. + fn compile( + &mut self, + work: &Workspace, + plan: &Plan, + select: Option<&[String]>, + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result { + // `cargo build --tests` emits the same compiler-artifact executable messages consumed by + // `test_binaries`, while `--keep-going` lets convergence collect diagnostics from siblings + // after one target fails. Reusing this stream avoids a second cache-hit Cargo invocation. + self.converge( + work, + plan, + select, + &["build", "--tests", "--examples", "--keep-going"], + limits, + events, + ) + } + + /// Builds the whole workspace, which is what decides the run. + /// + /// The staged builds before this one are a way of ruling on mutants early and of saying what is + /// happening while it happens; this is the build whose feature resolution matches the one + /// `cargo test` would use, and the binaries come from it for that reason. + /// + /// A build that cannot be made to compile leaves no test binary to judge anything with, so + /// every mutant still live is abandoned and the returned [`Build`] says so. The run reports + /// what it has rather than exiting with nothing. + pub(super) fn finish( + mut self, + work: &Workspace, + plan: &mut Plan, + select: Option<&[String]>, + limits: BuildLimits, + events: &mut dyn Events, + ) -> Result { + let select = self.scoped(select); + let mut widened = false; + + let converged = match self.compile(work, plan, select, limits, events)? { + Convergence::Built(stdout) => Convergence::Built(stdout), + + // A narrowed build is not merely a smaller version of the whole one: cargo unifies + // features over the packages it is told to build, so a test target that only compiles + // because a package left out of the selection switches a feature on will fail here and + // will fail in a way no mutant can be blamed for. That is a wrong answer to the + // question the run is asking, so the selection is abandoned rather than reported. + Convergence::Stuck(narrow) if select.is_some() => { + widened = true; + + match self.compile(work, plan, None, limits, events)? { + Convergence::Built(stdout) => Convergence::Built(stdout), + Convergence::Stuck(_whole) => Convergence::Stuck(narrow), + } + } + + Convergence::Stuck(reason) => Convergence::Stuck(reason), + }; + + let stdout = match converged { + Convergence::Built(stdout) => stdout, + + Convergence::Stuck(reason) => { + let stuck = self.abandon(plan, None, &reason); + + self.settle(plan); + + return Ok(Build { + history: self.history.clone(), + census: self.tally(plan), + binaries: Vec::new(), + withdrawn: self.withdrawn.len().saturating_sub(self.abandoned.len()), + rounds: self.total_rounds, + widened, + stuck: Some(stuck), + ordering: self.ordering, + }); + } + }; + + self.settle(plan); + self.remember_compiled(&stdout, &work.root); + + // Runs after the withdrawal above so that a mutant which genuinely failed to compile keeps + // that more specific verdict; see [`withdraw_uncompiled`] for why the set is only trusted + // when it agrees with the survey at all. + if let Some(compiled) = &self.compiled { + withdraw_uncompiled(plan, compiled); + } + + Ok(Build { + history: self.history.clone(), + census: self.tally(plan), + binaries: test_binaries(&stdout), + withdrawn: self.withdrawn.len().saturating_sub(self.abandoned.len()), + rounds: self.total_rounds, + widened, + stuck: None, + ordering: self.ordering, + }) + } + + /// How many mutants have been withdrawn so far. + pub(super) fn withdrawn(&self) -> usize { + self.withdrawn.len() + } + + /// Adds one successful Cargo invocation's dep-info to the run-wide source inventory. + fn remember_compiled(&mut self, stdout: &str, root: &camino::Utf8Path) { + if let Some(found) = compiled_sources(stdout, root) { + self.compiled.get_or_insert_with(HashSet::default).extend(found); + } + } + + /// Groups the withdrawals by rustc error code and mutator, densest pair first. + /// + /// Counts distinct ordinals, because a diagnostic is not a mutant: one unviable mutant can draw + /// a four-figure count of follow-on complaints, so anything tallying rows rather than mutants + /// overstates the answer by an order of magnitude. + fn tally(&self, plan: &Plan) -> Vec { + let mut mutators: HashMap = HashMap::default(); + + for mutant in &plan.mutants { + let _ = mutators.insert(mutant.ordinal, &mutant.mutator); + } + + let mut counts: HashMap<(&str, &str), usize> = HashMap::default(); + + for (ordinal, code) in &self.census { + let mutator = mutators.get(ordinal).copied().unwrap_or(""); + + *counts.entry((code.as_str(), mutator)).or_default() += 1; + } + + let mut census: Vec = counts + .into_iter() + .map(|((code, mutator), mutants)| Withdrawal { + code: code.to_owned(), + mutator: mutator.to_owned(), + mutants, + }) + .collect(); + + // Descending by weight, then by name, so that the line worth reading is the first one and + // two runs over the same tree print the same thing. + census.sort_by(|left, right| { + right + .mutants + .cmp(&left.mutants) + .then_with(|| left.code.cmp(&right.code)) + .then_with(|| left.mutator.cmp(&right.mutator)) + }); + + census + } +} + +/// Marks every pending mutant whose file the compiler never read as not built. +/// +/// A mutant in a file no compilation opened cannot be judged by any test, so it is taken out of the +/// run here rather than left to be reported as a survivor later. +/// +/// The agreement check in front of the loop exists because the failure this could otherwise cause +/// is silent and expensive: if dep-info ever spelled its paths differently from the way the survey +/// spells them, nothing would match, every mutant would be excused, and the run would report a +/// flattering score with no sign that anything had gone wrong. A set that names not one file the +/// survey found is a set we do not understand, so nothing is concluded from it. +/// +/// The check is deliberately whole-set and cannot be tightened to a per-file one: "this file is +/// missing from the compiled set" is exactly the question being asked, so a per-file guard would +/// answer it with itself. That makes the check blind to a spelling difference that affects only +/// *some* paths, which is why [`messages::compiled_sources`] has to decode the dep-info escaping +/// correctly rather than rely on being caught here. +fn withdraw_uncompiled(plan: &mut Plan, compiled: &HashSet) { + if !plan.files.iter().any(|file| compiled.contains(&file.path)) { + return; + } + + for mutant in &mut plan.mutants { + if mutant.outcome == Outcome::Pending && !compiled.contains(&*mutant.file) { + mutant.outcome = Outcome::NotBuilt; + } + } +} diff --git a/crates/cargo-gamma-lib/src/exec/build/blame.rs b/crates/cargo-gamma-lib/src/exec/build/blame.rs new file mode 100644 index 000000000..17efac9fa --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/build/blame.rs @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Which mutant a compiler error belongs to. + +use core::ops::{Range, RangeInclusive}; + +use camino::Utf8Path; + +use super::Guards; +use super::messages::{CompilerMessage, Span, cargo_message}; +use crate::schema::{Guard, Position}; +use crate::{HashMap, HashSet}; + +/// Works out which mutants to blame for a failed build. +/// +/// Guard positions come from the instrumented text rather than from the mutants' source lines, +/// because a guard emits the original text alongside the mutated one and so shifts every later +/// line. Only primary spans are considered: a diagnostic's notes routinely point at the innocent +/// declaration a mutated expression happened to misuse. +/// +/// A diagnostic landing in some guard's mutated branch names its cause exactly, since that branch +/// is the only text in the tree that is not a copy of the original and no two of them overlap. +/// Failing that — a mutant can break code it merely encloses, and a deletion has no replacement +/// text to land in — the innermost guarded site containing the diagnostic is blamed instead. +/// Mutants sharing a site are withdrawn together, which can retire one that would have compiled; +/// it is reported as unviable rather than dropped. +#[expect(clippy::too_many_lines, reason = "the attribution tiers share one parsed diagnostic walk")] +pub(super) fn blame(stdout: &str, root: &Utf8Path, guards: &Guards) -> HashMap { + let mut blamed: HashMap = HashMap::default(); + + // A failing build reports many diagnostics and a large workspace has many guards, so pairing + // them off one at a time is quadratic. Grouping by file first makes the common case a lookup. + let mut by_path: HashMap<&Utf8Path, Vec<(u32, &Guard)>> = HashMap::default(); + + for (ordinal, (file, guard)) in guards { + by_path.entry(file.as_path()).or_default().push((*ordinal, guard)); + } + + for line in stdout.lines() { + let Some(message) = cargo_message(line) else { + continue; + }; + + if message.reason != "compiler-message" { + continue; + } + + let Some(diagnostic) = message.message else { + continue; + }; + + if diagnostic.level != "error" { + continue; + } + + let primary: Vec<&Span<'_>> = diagnostic.spans.iter().filter(|span| span.is_primary).collect(); + let considered = if primary.is_empty() { + diagnostic.spans.iter().collect() + } else { + primary + }; + let mut exact = HashSet::default(); + let mut enclosing: Option<(u32, HashSet)> = None; + let mut contained: Option<(u32, HashSet)> = None; + + for span in considered { + let Some(file_name) = span.file_name.as_deref() else { + continue; + }; + + let relative = Utf8Path::new(file_name) + .strip_prefix(root.as_str()) + .unwrap_or_else(|_ignored| Utf8Path::new(file_name)); + + let Some(reported) = position_range(span) else { + continue; + }; + + // The exact relative path is the normal case; the scan is the fallback for a diagnostic + // whose path is spelled differently, and only runs when the lookup found nothing. + let matched = by_path.get(relative).map(Vec::as_slice).unwrap_or_default(); + let scanned; + + let here = if matched.is_empty() { + scanned = by_path + .iter() + .filter(|(file, _found)| file_name.ends_with(file.as_str())) + .flat_map(|(_file, found)| found.iter().copied()) + .collect::>(); + + scanned.as_slice() + } else { + matched + }; + + for (ordinal, guard) in here.iter().copied() { + if guard.mutated.as_ref().is_some_and(|mutated| covers(mutated, &reported)) { + let _ = exact.insert(ordinal); + } else if covers(&guard.site, &reported) { + let width = guard.site.end.line.saturating_sub(guard.site.start.line); + + match &mut enclosing { + Some((best, ordinals)) if *best == width => { + let _ = ordinals.insert(ordinal); + } + Some((best, _ordinals)) if *best < width => {} + _ => enclosing = Some((width, HashSet::from_iter([ordinal]))), + } + } else if covers(&reported, &guard.site) { + // The diagnostic encloses the guard rather than the other way round, which is + // what a borrow checker error looks like: the guard makes some subexpression + // non-constant and the complaint lands on the whole construct that depended on + // it. Every guard inside the smallest such region is a candidate, because + // nothing narrower distinguishes them. + let width = reported.end.line.saturating_sub(reported.start.line); + + match &mut contained { + Some((best, ordinals)) if *best == width => { + let _ = ordinals.insert(ordinal); + } + Some((best, _ordinals)) if *best < width => {} + _ => contained = Some((width, HashSet::from_iter([ordinal]))), + } + } + } + } + + // rustc sometimes puts the consequence of a mutation in the primary span and the cause in + // a secondary "expected because of this" span. A secondary span is too broad a basis for + // enclosing-site attribution—it routinely names innocent declarations—but intersection + // with a mutated branch is exact evidence: that text exists only because gamma emitted it. + if exact.is_empty() && enclosing.is_none() && contained.is_none() { + for span in diagnostic.spans.iter().filter(|span| !span.is_primary) { + let Some(file_name) = span.file_name.as_deref() else { + continue; + }; + let relative = Utf8Path::new(file_name) + .strip_prefix(root.as_str()) + .unwrap_or_else(|_ignored| Utf8Path::new(file_name)); + let Some(reported) = position_range(span) else { + continue; + }; + let matched = by_path.get(relative).map(Vec::as_slice).unwrap_or_default(); + + for (ordinal, guard) in matched.iter().copied() { + if guard.mutated.as_ref().is_some_and(|mutated| overlaps(mutated, &reported)) { + let _ = exact.insert(ordinal); + } + } + } + } + + // Preference runs from the most specific attribution to the least. The last is a blunt + // instrument and can retire mutants that would have compiled, but the alternative is a + // diagnostic nothing can be blamed for, which loses the entire run rather than a few + // mutants that are then reported as unviable. + let ordinals = if exact.is_empty() { + enclosing.or(contained).map(|(_width, ordinals)| ordinals) + } else { + Some(exact) + }; + + let ordinals = ordinals.or_else(|| diverted(&diagnostic, root, &by_path)); + + if let Some(ordinals) = ordinals { + // The first diagnostic to name a mutant is the one kept. A single unviable mutant can + // draw a thousand follow-on complaints, and the later ones describe the wreckage rather + // than the cause; the census is only worth reading if each mutant contributes the one + // error that explains it. + let code = diagnostic.code.as_ref().map_or("", |code| code.code.as_ref()); + + for ordinal in ordinals { + let _ = blamed.entry(ordinal).or_insert_with(|| code.to_owned()); + } + } + } + + blamed +} + +/// The rustc error codes whose diagnostics need not point anywhere near their cause. +/// +/// Every one of these comes from a flow-sensitive analysis — borrow checking, move checking, +/// initialization tracking — which reasons about paths through a function rather than about the +/// text of an expression. Such an analysis reports at the point where the consequence becomes +/// visible, which can be an arbitrary distance from the change that made it reachable, and in a +/// span that need not contain the change at all. Every other class of error rustc reports is +/// positional: a type error lands on the expression whose type is wrong. +/// +/// The list is a gate rather than a hint. What it guards is a fallback that blames mutants a +/// diagnostic does not name, and applying that to an error whose position *is* meaningful would +/// withdraw innocent mutants for a fault they did not cause. +pub(super) const FLOW_SENSITIVE: &[&str] = &[ + "E0381", // used binding is possibly-uninitialized + "E0382", // use of moved value + "E0383", // partial reinitialization of an uninitialized structure + "E0384", // cannot assign twice to immutable variable + "E0499", // cannot borrow as mutable more than once + "E0502", // cannot borrow as mutable because also borrowed as immutable + "E0503", // cannot use value because it was mutably borrowed + "E0505", // cannot move out of value because it is borrowed + "E0506", // cannot assign to value because it is borrowed + "E0507", // cannot move out of borrowed content + "E0508", // cannot move out of type, a non-copy array + "E0509", // cannot move out of type which implements Drop + "E0510", // cannot mutate place in this match guard + "E0594", // cannot assign to borrowed content + "E0596", // cannot borrow as mutable + "E0597", // borrowed value does not live long enough + "E0716", // temporary value dropped while borrowed +]; + +/// Blames a flow-sensitive diagnostic on the mutants that could have changed which paths exist. +/// +/// The three positional tiers have already found nothing, which for these error codes says little: +/// the cause is not required to be inside any span the diagnostic reports. What is left is to look +/// at the region the diagnostic talks about as a whole — every span it names, including the ones on +/// its notes, which is where "value moved here" and "this reinitialization might get skipped" live — +/// and to ask which mutants inside that region could have changed reachability at all. +/// +/// Deletion mutants are preferred, and they are the reason this works. A deletion is recorded with +/// no `mutated` range, and deleting a statement is the only edit that can make a path statically +/// reachable that was not before: guard a `continue` or a `return` and the code after it is suddenly +/// live, so a value moved earlier is now seen to be used again. A substitution changes what a value +/// is, never where control goes. +/// +/// Falling back to every guard in the region is blunt and retires mutants that would have compiled. +/// It is still the right trade, because the alternative is an unattributable error, which loses the +/// entire run rather than a handful of mutants — and those are reported as unviable rather than +/// silently dropped. +pub(super) fn diverted( + diagnostic: &CompilerMessage<'_>, + root: &Utf8Path, + by_path: &HashMap<&Utf8Path, Vec<(u32, &Guard)>>, +) -> Option> { + let code = diagnostic.code.as_ref()?; + + if !FLOW_SENSITIVE.contains(&code.code.as_ref()) { + return None; + } + + let mut deletions = HashSet::default(); + let mut all = HashSet::default(); + + for (file, region) in regions(diagnostic) { + let relative = Utf8Path::new(file.as_str()) + .strip_prefix(root.as_str()) + .unwrap_or_else(|_ignored| Utf8Path::new(file.as_str())); + + let here = by_path.get(relative).map_or_else( + || { + by_path + .iter() + .filter(|(known, _found)| file.ends_with(known.as_str())) + .flat_map(|(_known, found)| found.iter().copied()) + .collect::>() + }, + Clone::clone, + ); + + for (ordinal, guard) in here { + if !region.contains(&guard.site.start.line) { + continue; + } + + let _ = all.insert(ordinal); + + if guard.mutated.is_none() { + let _ = deletions.insert(ordinal); + } + } + } + + if !deletions.is_empty() { + return Some(deletions); + } + + if all.is_empty() { None } else { Some(all) } +} + +/// The line span each file contributes to a diagnostic, notes and all. +/// +/// Children are walked because a flow-sensitive diagnostic keeps the interesting half of what it +/// knows in them: the primary span is where the error surfaced, and the note spans are where the +/// value was moved, borrowed or conditionally skipped. The cause sits between those points far more +/// often than it sits in any one of them. +pub(super) fn regions(diagnostic: &CompilerMessage<'_>) -> HashMap> { + let mut spread: HashMap> = HashMap::default(); + let mut pending = vec![diagnostic]; + + while let Some(node) = pending.pop() { + for span in &node.spans { + let (Some(file), Some(from), Some(to)) = (span.file_name.as_deref(), span.line_start, span.line_end) else { + continue; + }; + + let from = clamped(from); + let to = clamped(to); + + let _widened = spread + .entry(file.to_owned()) + .and_modify(|known| *known = (*known.start()).min(from)..=(*known.end()).max(to)) + .or_insert(from..=to); + } + + pending.extend(&node.children); + } + + spread +} + +/// Narrows one of cargo's line or column numbers, saturating rather than refusing it. +pub(super) fn clamped(number: u64) -> u32 { + u32::try_from(number).unwrap_or(u32::MAX) +} + +/// Reads the region a compiler diagnostic points at out of its JSON span. +pub(super) fn position_range(span: &Span<'_>) -> Option> { + let at = |line: Option, column: Option| { + Some(Position { + line: clamped(line?), + column: clamped(column?), + }) + }; + + Some(at(span.line_start, span.column_start)?..at(span.line_end, span.column_end)?) +} + +/// Reports whether a guard's region wholly contains the one a diagnostic points at. +pub(super) fn covers(range: &Range, reported: &Range) -> bool { + range.start <= reported.start && range.end >= reported.end +} + +/// Whether two non-empty source regions share any text. +fn overlaps(left: &Range, right: &Range) -> bool { + left.start < right.end && right.start < left.end +} diff --git a/crates/cargo-gamma-lib/src/exec/build/complaints.rs b/crates/cargo-gamma-lib/src/exec/build/complaints.rs new file mode 100644 index 000000000..c9561e3d0 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/build/complaints.rs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! What a build failure quotes back to the reader. + +use camino::Utf8Path; + +use super::invoke::is_progress; +use super::messages::{cargo_message, normalize_separators}; +use crate::HashSet; +use crate::discover::Plan; + +/// How many compiler errors a build failure quotes before it starts counting instead. +/// +/// A tree that does not compile usually does so for one reason, restated by every crate that +/// depended on it, so quoting everything buries the first error under its own consequences. +pub(super) const DIAGNOSTIC_LIMIT: usize = 5; + +/// Keeps the part of cargo's stderr that explains a failure. +/// +/// Cargo narrates its progress on the same stream it reports failures on, and a cold build narrates +/// thousands of lines. Handing all of that to someone whose build just failed buries the two lines +/// that matter, so the progress verbs are dropped and everything else is kept — including the +/// indented `Caused by` blocks and build-script output, which is usually where the real cause is. +/// +/// Segments are split on carriage returns as well as newlines, and the progress bar is recognised +/// by `is_progress` rather than by a list of its own. Cargo redraws the bar by returning to the +/// start of the line, so a whole build's worth of redraws is a single newline-terminated line; +/// splitting only on newlines would hand the reader that entire bar as one enormous complaint. +pub(super) fn complaints(stderr: &str) -> String { + /// The verbs cargo uses to narrate work it is doing rather than trouble it has hit. + const PROGRESS: [&str; 11] = [ + "Compiling", + "Building", + "Checking", + "Downloading", + "Downloaded", + "Updating", + "Locking", + "Adding", + "Finished", + "Fresh", + "Running", + ]; + + let mut kept = String::new(); + + // A carriage return separates redraws of the same line, so its segments are lines in their own + // right; the empty ones a redraw leaves behind are an artifact of the drawing rather than + // something cargo said, and only a genuinely blank line is worth keeping. + let segments = stderr + .lines() + .flat_map(|line| line.split('\r').filter(move |segment| !segment.is_empty() || line.is_empty())); + + for line in segments { + let trimmed = line.trim_start(); + + if is_progress(line) { + continue; + } + + if PROGRESS.iter().any(|verb| { + trimmed + .strip_prefix(verb) + .is_some_and(|rest| rest.starts_with(' ') || rest.is_empty()) + }) { + continue; + } + + if trimmed.is_empty() && kept.is_empty() { + continue; + } + + kept.push_str(line); + kept.push('\n'); + } + + if kept.trim().is_empty() { + return "cargo said nothing on stderr either.".to_owned(); + } + + kept +} + +/// One error-level compiler message, kept with the package it came from. +/// +/// The package matters because a failing preflight quotes only the first few errors, and the ones +/// worth quoting are those in the code the caller chose to mutate. A reverse dependency dragged in +/// because its tests form part of the oracle can produce dozens of errors that are consequences of +/// a single problem in the caller's own crate, and quoting those instead sends them to fix a +/// package they never mentioned. +pub(super) struct Diagnostic { + /// The manifest of the package the compiler was building, when cargo said. + pub(super) manifest: Option, + + /// The compiler's own rendering, snippet and underlines and all. + pub(super) rendered: String, +} + +/// Extracts the human-readable compiler diagnostics from cargo's JSON output. +/// +/// With `--message-format=json` the diagnostics arrive on stdout as structured messages and +/// stderr carries only a summary, so a failure report built from stderr would not say what went wrong. +pub(super) fn diagnostics(stdout: &str) -> Vec { + let mut rendered = Vec::new(); + + for line in stdout.lines() { + let Some(message) = cargo_message(line) else { + continue; + }; + + if message.reason != "compiler-message" { + continue; + } + + let manifest = message.manifest_path.as_deref().map(normalize_separators); + + let Some(diagnostic) = message.message else { + continue; + }; + + if diagnostic.level != "error" { + continue; + } + + if let Some(text) = diagnostic.rendered { + rendered.push(Diagnostic { + manifest, + rendered: text.into_owned(), + }); + } + } + + rendered +} + +/// Moves the diagnostics from `manifests` to the front, keeping each group in the order the +/// compiler emitted it. +/// +/// A stable partition rather than a filter: the other packages' errors are still worth having when +/// there is room for them, and when the caller's own package turns out to be clean they are the +/// whole story. +pub(super) fn prioritize(found: &mut [Diagnostic], manifests: &HashSet) { + found.sort_by_key(|diagnostic| !diagnostic.manifest.as_ref().is_some_and(|path| manifests.contains(path))); +} + +/// The manifest paths of the named packages inside the copied tree. +pub(super) fn manifests_of(plan: &Plan, root: &Utf8Path, packages: &[String]) -> HashSet { + packages + .iter() + .filter_map(|package| plan.directory_of(package)) + .map(|directory| { + let absolute = if directory.as_str().is_empty() { + root.to_owned() + } else { + root.join(directory) + }; + + normalize_separators(absolute.join("Cargo.toml").as_str()) + }) + .collect() +} + +/// Renders the first few diagnostics whole, saying how many were left out. +/// +/// From the front, and by whole diagnostics rather than by line. A compiler's first error is the +/// one to fix — the rest are frequently consequences of it — and a limit measured in lines cuts +/// one in half, which produces a report that opens partway through a snippet with no error line +/// above it to say what is being pointed at. +pub(super) fn leading(found: &[Diagnostic], limit: usize) -> String { + let shown: String = found.iter().take(limit).map(|diagnostic| diagnostic.rendered.as_str()).collect(); + let omitted = found.len().saturating_sub(limit); + + if omitted == 0 { + return shown; + } + + format!("{shown}\n(and {} not shown)\n", crate::report::quantity(omitted, "further error")) +} diff --git a/crates/cargo-gamma-lib/src/exec/build/invoke.rs b/crates/cargo-gamma-lib/src/exec/build/invoke.rs new file mode 100644 index 000000000..7a8bc8177 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/build/invoke.rs @@ -0,0 +1,648 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Running cargo, and turning its two output streams into something a reader can follow. + +use core::time::Duration; +use std::io::{self, Read}; +use std::process::{Command, Output, Stdio}; +use std::sync::mpsc::{self, Receiver, SyncSender}; +use std::thread::{self, JoinHandle}; +use std::time::Instant; + +use cargo_gamma_process::{MemoryRequest, ProcessTree, prepare}; + +use super::super::cargo_options::BuildLimits; +use super::super::events::Events; +#[cfg(test)] +use super::super::faults::{self, Fault}; +use super::super::workspace::Workspace; +use super::messages::cargo_message; +use crate::Result; +use crate::discover::Plan; +use crate::error::{Error, error}; + +/// Explains a failed cargo spawn. +/// +/// Platforms do not agree on which error a missing working directory produces, so the directory +/// is probed to tell that case apart from a missing program. Naming the program matters because it +/// is not always `cargo` from `PATH`: an inherited `CARGO` wins, and a stale one left over from an +/// earlier shell points at a binary that no longer exists. +pub(super) fn spawn_failure(program: &str, work: &Workspace, cause: io::Error) -> Error { + if !work.root.as_std_path().is_dir() { + return error!("the scratch tree at `{}` disappeared while it was being built", work.root).caused_by(cause); + } + + error!( + "could not run `{program}` in `{}`. Cargo is taken from the `CARGO` environment variable when it is set, and from `PATH` otherwise", + work.root + ) + .caused_by(cause) +} + +/// Runs one cargo build, stopping it if it outstays its budget. +/// +/// Returns `None` when the budget ran out, having killed the build and everything it started. +/// Waiting for cargo to finish and complaining afterwards would report a slow build accurately and +/// a hung one never: the whole run rests on this single compile, and there is no test harness +/// behind it to notice that nothing is happening. +/// +/// The pipes are drained on their own threads. A build produces megabytes of JSON, and a pipe +/// holds about sixty-four kilobytes, so a caller that waits for exit before reading deadlocks +/// against a compiler blocked writing to it — which would look exactly like the hang this is meant +/// to catch. +pub(super) fn compile(work: &Workspace, args: &[String], budget: Option, events: &mut dyn Events) -> Result> { + let mut command = work.cargo(); + + let _command = command + .env("CARGO_TERM_PROGRESS_WHEN", "always") + .env("CARGO_TERM_PROGRESS_WIDTH", PROGRESS_WIDTH.to_string()) + .args(args) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()); + + supervise(&mut command, work, budget, events) +} + +/// Runs an already-configured cargo command under containment, and collects what it said. +/// +/// Split from [`compile`] so that a test can supervise a command of its own choosing. The cargo +/// that `compile` builds comes from the environment, and the behaviour worth testing here — what +/// happens to a build's descendants when its budget runs out — needs a program that deliberately +/// leaves one behind, which no real cargo invocation can be asked to do. +/// +/// The build is contained for the same reason a test binary is. Cargo is the root of a tree: +/// `rustc`, build scripts, and whatever those start in turn. Killing cargo alone leaves that tree +/// compiling — burning the cores the next attempt needs, writing into the scratch tree the run is +/// about to reuse, and, because every one of them inherited the two pipes below, holding open the +/// write ends that the readers are waiting to see closed. A build killed for hanging would then be +/// followed by a collection that hangs, which is the failure the budget exists to prevent, arriving +/// one step later. +/// +/// No accounting is asked for. This is the boundary that can be killed, and nothing here reads a +/// peak: the memory a build uses is `rustc`'s business, and a ceiling on it would fail builds the +/// user never asked to be bounded. +pub(super) fn supervise( + command: &mut Command, + work: &Workspace, + budget: Option, + events: &mut dyn Events, +) -> Result> { + supervise_with_limits(command, work, budget, events, OUTPUT_LIMITS) +} + +/// Runs a build with explicit limits for retained and narrated output. +pub(super) fn supervise_with_limits( + command: &mut Command, + work: &Workspace, + budget: Option, + events: &mut dyn Events, + limits: OutputLimits, +) -> Result> { + let program = command.get_program().to_string_lossy().into_owned(); + + let guard = prepare(command, MemoryRequest::default()) + .map_err(|reason| error!("the cargo build in `{}` could not be contained: {reason}", work.root))?; + + let child = command.spawn().map_err(|cause| spawn_failure(&program, work, cause))?; + + let mut subtree = match ProcessTree::adopt(child, guard) { + Ok(subtree) => subtree, + Err(reason) => { + events.build_finished(); + + return Err(error!("the cargo build in `{}` could not be contained: {reason}", work.root)); + } + }; + + let (sender, lines) = mpsc::sync_channel(limits.backlog); + + let stdout = subtree + .take_stdout() + .map(|pipe| read_pipe_with_limits(pipe, Stream::Json, &sender, limits)); + let stderr = subtree + .take_stderr() + .map(|pipe| read_pipe_with_limits(pipe, Stream::Prose, &sender, limits)); + + // The senders the reader threads hold are the only ones that matter; this one would keep the + // channel open forever after they finish. + drop(sender); + + let deadline = budget.map(|budget| Instant::now() + budget); + + let outcome = loop { + let _narrated = narrate(&lines, events); + + match subtree.observe() { + Ok(Some(status)) => { + break Ok(Some(status)); + } + + Ok(None) => { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + collect(&mut subtree); + + break Ok(None); + } + + thread::sleep(BUILD_POLL_INTERVAL); + } + + Err(cause) => { + // Nothing more can be asked of this child, and dropping it would leave the whole + // build tree running with no handle on it at all. + collect(&mut subtree); + + break Err(error!("could not wait for cargo in `{}`", work.root).caused_by(cause)); + } + } + }; + + // Bounded, and reached on every path out of the loop above: the build and everything it + // started have been killed by now, so the pipes close and the readers finish at once — unless + // something escaped the containment, in which case waiting for them without a limit would hang + // the run on the exact process the kill was supposed to have collected. + // + // Released first on every one of those paths, since this wait is the long one and a slot still + // naming a reaped pid names whatever the kernel hands that id to next. + debug_assert!(subtree.released(), "the containment is released before the output is drained"); + + let grace = Instant::now() + DRAIN_GRACE; + let (said, printed) = finish_readers(stdout, stderr, &lines, events, grace); + + events.build_finished(); + + let Some(status) = outcome? else { + return Ok(None); + }; + + // Not reported as a build that produced no diagnostics, which is what an empty stream reads as + // to everything downstream: a partial JSON stream loses artifacts, and a run that lost the + // artifacts of a build reports the tests it could not find as ones that do not exist. Both ways + // of losing it are refused here — a reader still blocked on a pipe something else holds open, + // and one that stopped on a read that failed — because the difference is invisible in the bytes. + let (Some(stdout), Some(stderr)) = (said, printed) else { + return Err(error!( + "cargo in `{}` finished, but its output could not be read to the end, so what it built could not be read", + work.root + )); + }; + + if !stdout.complete || !stderr.complete { + return Err(error!( + "cargo in `{}` finished, but its output could not be read to the end, so what it built could not be read", + work.root + )); + } + + if !stdout.within_limits || !stderr.within_limits { + return Err(error!( + "cargo in `{}` exceeded the configured {}-byte retained or {}-byte per-line build-output limit, so its truncated output could not be trusted", + work.root, limits.retained, limits.line + )); + } + + Ok(Some(Output { + status, + stdout: stdout.text, + stderr: stderr.text, + })) +} + +/// Ends a build that will not be waited out, and reaps it. +/// +/// The order is the whole content of this function, which is why it is one. The group is signalled +/// while its leader is still unreaped, so the id being signalled cannot yet have been handed to +/// another spawn; the watch slot is given back next, while the pid it names is still this child's; +/// and only then is the pid freed. Reaping first would put an allocation the run does not control +/// between the two, and a stray `killpg` in a run this wide lands on a test binary and is scored as +/// a kill that never happened. +/// +/// The ordinary-exit path uses the same order through [`ProcessTree::observe`], which observes an exit +/// without reaping it before sweeping and finally waiting. +fn collect(subtree: &mut ProcessTree) { + #[cfg(unix)] + debug_assert!( + !subtree.released(), + "the subtree is signalled while it still holds its leader and its watch slot" + ); + + let _reaped = subtree.terminate(); +} + +/// How wide cargo is asked to draw its progress bar. +/// +/// Fixed rather than taken from the terminal, because cargo is writing to a pipe and has no +/// terminal to measure. The display this is handed to truncates to the real width. +pub(super) const PROGRESS_WIDTH: usize = 100; + +/// Turns cargo's two streams into the handful of things a reader needs told while a build runs. +/// +/// Compiler diagnostics go to `build_output`, which is the escape valve's channel and is silent +/// unless `--show-build` asked for it. That is deliberate: during an instrumented build a +/// compiler error is the mechanism rather than a fault. The tree was checked before any mutant +/// was applied, so an error here was introduced by one, and the rollback loop is already about +/// to withdraw it and rebuild. Showing it would present the tool's normal operation as a +/// failure — and it reads as one, because these arrive in the first seconds of a run and name +/// files the caller never chose to mutate. +/// +/// They are rendered whole rather than condensed, because the only reason to ask for them is to +/// read them: `--show-build` exists to reproduce what a bare cargo invocation would have shown, +/// and cargo puts the snippet and the underlines in `rendered`. Nothing else carries them — +/// under `--message-format=json` the prose stream has only cargo's own summary lines — so +/// without this the escape valve would show a build with no diagnostics in it at all. +/// +/// A call consumes at most [`NARRATION_BATCH`] lines. That keeps an output flood from monopolizing +/// the supervising thread and postponing its build deadline; the bounded channel retains the rest +/// until the next pass. +pub(super) fn narrate(lines: &Receiver<(Stream, String)>, events: &mut dyn Events) -> usize { + let wanted = events.wants_build_output(); + let mut narrated = 0; + + while narrated < NARRATION_BATCH { + let Ok((stream, line)) = lines.try_recv() else { + break; + }; + + narrated += 1; + + match stream { + Stream::Prose if is_progress(&line) => events.build_progress(line.trim_end()), + Stream::Prose => events.build_output(line.trim_end()), + + Stream::Json => { + if !wanted { + continue; + } + + if let Some(rendered) = rendered_diagnostic(&line) { + events.build_output(rendered.trim_end()); + } + } + } + } + + narrated +} + +/// Whether a line is cargo's progress bar rather than something it wanted to say. +/// +/// Matched on the gauge rather than on the word, so that the check does not rest on cargo's choice +/// of verb. A line that is not recognised is treated as prose, which shows it only when asked for; +/// the failure is a bar that does not appear, never a garbled display. +/// +/// The styling is stripped first because cargo colors this line when it is told to, which puts an +/// escape sequence in front of the verb and between the verb and the gauge. +pub(super) fn is_progress(line: &str) -> bool { + let plain = crate::report::unstyled(line); + let trimmed = plain.trim_start(); + + trimmed.starts_with("Building [") || trimmed.starts_with("Compiling [") +} + +/// The compiler's own rendering of a diagnostic, for a line of cargo's JSON stream that carries one. +/// +/// Returned whole, snippet and underlines and all, because the only caller shows it to someone who +/// asked to see the build. Warnings are included for the same reason: `--show-build` is meant to +/// reproduce what running cargo directly would have printed, and cargo prints them. +pub(super) fn rendered_diagnostic(line: &str) -> Option { + let message = cargo_message(line)?; + + if message.reason != "compiler-message" { + return None; + } + + let rendered = message.message?.rendered?; + + if rendered.trim().is_empty() { + None + } else { + Some(rendered.into_owned()) + } +} + +/// How often a running build is checked for having finished. +/// +/// A build is measured in seconds at best, so a coarse poll costs nothing and keeps an otherwise +/// idle thread from spinning against a compiler that wants the core. +pub(super) const BUILD_POLL_INTERVAL: Duration = Duration::from_millis(20); + +/// Bounds every allocation a build's two output readers can retain or queue. +#[derive(Clone, Copy, Debug)] +pub(super) struct OutputLimits { + /// Maximum bytes retained from one stream for parsing after the build. + pub(super) retained: usize, + + /// Maximum bytes copied into one narrated line. + pub(super) line: usize, + + /// Maximum narrated lines awaiting the build supervisor. + pub(super) backlog: usize, +} + +/// The normal build-output budget, per stream. +const OUTPUT_LIMITS: OutputLimits = OutputLimits { + retained: 4 * 1024 * 1024, + line: 64 * 1024, + backlog: 64, +}; + +/// The number of lines one supervision pass narrates before it re-checks the build. +const NARRATION_BATCH: usize = 128; + +/// What one output reader finished with. +#[derive(Debug)] +pub(super) struct Pipe { + pub(super) text: Vec, + pub(super) complete: bool, + pub(super) within_limits: bool, +} + +/// Drains one pipe on its own thread, publishing each line as it arrives. +/// +/// Bytes are retained only up to [`OutputLimits::retained`]. Everything downstream needs a whole +/// stream, so crossing that limit marks the build output unusable rather than passing a prefix to +/// artifact discovery. The channel is a second view for narration, and its synchronous capacity +/// supplies backpressure instead of retaining an unbounded queue. +/// +/// Lines are split on carriage returns as well as newlines: cargo redraws its progress bar by +/// returning to the start of the line, so a bar that has been redrawn a thousand times is one +/// newline-terminated line and is worth nothing to a reader who only sees it at the end. +/// +/// Whether the stream really ended and stayed within its limits travels with the bytes. A read +/// failure or limit crossing leaves a prefix indistinguishable from a quiet build to artifact +/// discovery, so [`supervise`] refuses both rather than parsing a truncated stream. +#[cfg(test)] +pub(super) fn read_pipe( + pipe: R, + stream: Stream, + sink: &SyncSender<(Stream, String)>, +) -> io::Result> { + read_pipe_with_limits(pipe, stream, sink, OUTPUT_LIMITS) +} + +/// Drains one pipe with a caller-selected output limit. +pub(super) fn read_pipe_with_limits( + mut pipe: R, + stream: Stream, + sink: &SyncSender<(Stream, String)>, + limits: OutputLimits, +) -> io::Result> { + let sink = sink.clone(); + + #[cfg(test)] + if faults::fired(Fault::Thread) { + return Err(io::Error::other("the reader thread a test asked to fail")); + } + + thread::Builder::new().name("cargo-gamma-build-output".to_owned()).spawn(move || { + let mut text = Vec::with_capacity(limits.retained); + let mut buffer = [0_u8; 8192]; + let mut line = Vec::with_capacity(limits.line); + let mut complete = true; + let mut within_limits = true; + let mut line_limited = false; + + loop { + let read = match pipe.read(&mut buffer) { + Ok(0) => break, + Ok(read) => read, + Err(cause) => { + // Retried rather than given up on, since an interrupted read has not lost + // anything: nothing was taken out of the pipe, and the next read gets it. + if cause.kind() == io::ErrorKind::Interrupted { + continue; + } + + complete = false; + + break; + } + }; + + let room = limits.retained.saturating_sub(text.len()); + let kept = read.min(room); + + text.extend_from_slice(&buffer[..kept]); + + if kept < read { + within_limits = false; + } + + for byte in &buffer[..read] { + if matches!(*byte, b'\n' | b'\r') { + if !line_limited + && !line.is_empty() + && let Ok(line) = str::from_utf8(&line) + { + // The bounded synchronous channel makes a fast compiler wait for the + // supervisor to narrate output, rather than retaining an unbounded queue. + let _sent = sink.send((stream, line.to_owned())); + } + + line.clear(); + line_limited = false; + + continue; + } + + if line.len() < limits.line { + line.push(*byte); + } else { + // A line's owned copy is part of the queue bound. Refuse a result built from + // output that cannot be held within that bound, while continuing to drain. + line_limited = true; + within_limits = false; + } + } + } + + if !line_limited + && !line.is_empty() + && let Ok(line) = str::from_utf8(&line) + { + let _ = sink.send((stream, line.to_owned())); + } + + Pipe { + text, + complete, + within_limits, + } + }) +} + +/// Which of cargo's two streams a line arrived on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Stream { + /// The JSON stream, carrying compiler messages and artifacts. + Json, + + /// Cargo's own prose: what it is compiling, and how far along it is. + Prose, +} + +/// Waits for one pipe reader to finish with the whole of its stream, but not past `deadline`. +/// +/// `None` when it did not, which happens two ways. The thread may still be running: the pipe is +/// held by something that outlived the kill — a descendant that left the process group, or one the +/// platform could not reach — and the thread is left where it is, blocked in a read on a handle it +/// owns, with no way to make it return short of closing the pipe underneath it. Or it finished on a +/// read that failed, having taken only a prefix of what was written. +/// +/// Both are the same fact to the caller: what came back is not the stream. Abandoning one thread is +/// the lesser cost, and the caller reports the loss rather than passing off a truncated stream as +/// the build's output. +#[cfg(test)] +pub(super) fn drained(handle: Option>, deadline: Instant) -> Option> { + let Some(handle) = handle else { + return Some(Vec::new()); + }; + + while !handle.is_finished() { + if Instant::now() >= deadline { + return None; + } + + thread::sleep(BUILD_POLL_INTERVAL); + } + + // A reader that panicked has no bytes at all, and one that stopped on a failed read has only a + // prefix; neither is the stream, and the caller's guard treats them alike. + let Ok(pipe) = handle.join() else { + return None; + }; + + (pipe.complete && pipe.within_limits).then_some(pipe.text) +} + +/// Narrates while waiting for bounded output readers to finish. +/// +/// A reader can legitimately be blocked on the synchronous channel when a compiler is noisy. +/// Draining that channel here is therefore part of joining it: waiting for the thread without +/// narrating would deadlock at a full, deliberately bounded backlog. +pub(super) fn finish_readers( + stdout: Option>>, + stderr: Option>>, + lines: &Receiver<(Stream, String)>, + events: &mut dyn Events, + deadline: Instant, +) -> (Option, Option) { + let finished = |reader: &Option>>| { + reader + .as_ref() + .is_none_or(|reader| reader.as_ref().is_err() || reader.as_ref().is_ok_and(JoinHandle::is_finished)) + }; + + while !finished(&stdout) || !finished(&stderr) { + let _narrated = narrate(lines, events); + + if Instant::now() >= deadline { + return (None, None); + } + + thread::sleep(BUILD_POLL_INTERVAL); + } + + while narrate(lines, events) == NARRATION_BATCH {} + + let stdout = stdout.map(|handle| { + let handle = handle.ok()?; + handle.join().ok() + }); + let stderr = stderr.map(|handle| { + let handle = handle.ok()?; + handle.join().ok() + }); + + ( + stdout.unwrap_or_else(|| { + Some(Pipe { + text: Vec::new(), + complete: true, + within_limits: true, + }) + }), + stderr.unwrap_or_else(|| { + Some(Pipe { + text: Vec::new(), + complete: true, + within_limits: true, + }) + }), + ) +} + +/// How long the readers are given once the build and everything it started have been killed. +/// +/// Long enough that a machine under load still closes its pipes in time, and short enough that a +/// descendant which escaped the containment costs the run seconds rather than the whole build. +const DRAIN_GRACE: Duration = Duration::from_secs(5); + +/// What one cargo invocation produced. +#[derive(Debug)] +pub(super) struct Compiled { + pub(super) succeeded: bool, + + /// Cargo's JSON stream, or `None` if the build was stopped for outstaying its budget. + pub(super) stdout: Option, + + /// What cargo said on stderr. + /// + /// Kept because the JSON stream only carries what the *compiler* said, and a build can fail + /// without the compiler ever being reached: a build script that panics, a missing native + /// library, an unresolvable dependency, an ambiguous package. Those failures are explained on + /// stderr and nowhere else, and discarding it left the reader with a build that failed for no + /// stated reason. + pub(super) stderr: String, +} + +/// Runs one cargo command in the tree under the build budget. +pub(super) fn run_cargo( + work: &Workspace, + plan: &Plan, + verb: &[&str], + select: Option<&[String]>, + limits: BuildLimits, + first_round: Option, + events: &mut dyn Events, +) -> Result { + let mut args: Vec = verb.iter().map(|arg| (*arg).to_owned()).collect(); + + args.push("--message-format=json".to_owned()); + + match select { + Some(packages) => { + for package in packages { + args.push("--package".to_owned()); + args.push(plan.spec(&work.root, package)); + } + } + None => args.push("--workspace".to_owned()), + } + + work.cargo.extend_build_args(&mut args); + + let Some(output) = compile(work, &args, limits.budget(first_round), events)? else { + return Ok(Compiled { + succeeded: false, + stdout: None, + stderr: String::new(), + }); + }; + + // Cargo's JSON stream can run to many megabytes, and it is valid UTF-8 in every case that + // matters, so the bytes are taken over rather than copied. + let stdout = match String::from_utf8(output.stdout) { + Ok(text) => text, + Err(invalid) => String::from_utf8_lossy(invalid.as_bytes()).into_owned(), + }; + + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + + Ok(Compiled { + succeeded: output.status.success(), + stdout: Some(stdout), + stderr, + }) +} diff --git a/crates/cargo-gamma-lib/src/exec/build/messages.rs b/crates/cargo-gamma-lib/src/exec/build/messages.rs new file mode 100644 index 000000000..84179249f --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/build/messages.rs @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The shapes cargo's JSON output is read back as. + +use std::borrow::Cow; +use std::fs; + +use camino::{Utf8Path, Utf8PathBuf}; +use serde::Deserialize; + +use crate::{HashMap, HashSet}; + +/// One line of cargo's `--message-format=json` stream, decoded to the fields this tool reads. +/// +/// A borrowed decode rather than a `serde_json::Value`: the stream runs to megabytes, four separate +/// consumers walk it, and between them they read eight fields. Decoding into a shape that names +/// those fields lets serde skip everything else without building it, and lets the strings that are +/// only ever compared point into the line instead of onto the heap. +/// +/// `Cow` rather than `&str` because a rendered diagnostic is full of escapes, and a borrowed `&str` +/// cannot represent a JSON string that needed unescaping. Every field is optional or defaulted: +/// cargo emits several kinds of message on this stream, most lines carry only some of these, and a +/// line this tool has no interest in must be skipped rather than fail the parse. +#[derive(Deserialize)] +pub(super) struct CargoMessage<'line> { + /// What kind of message this is: `compiler-message`, `compiler-artifact`, and others. + #[serde(borrow, default)] + pub(super) reason: Cow<'line, str>, + + /// The diagnostic itself, on a `compiler-message`. + #[serde(borrow, default)] + pub(super) message: Option>, + + /// The artifacts produced, on a `compiler-artifact`. + #[serde(borrow, default)] + pub(super) filenames: Vec>, + + /// The manifest of the package being built, used to put the caller's own errors first. + #[serde(borrow, default)] + pub(super) manifest_path: Option>, +} + +/// A compiler diagnostic, and the children that carry the rest of what it knows. +#[derive(Deserialize)] +pub(super) struct CompilerMessage<'line> { + /// `error`, `warning`, and the rest of rustc's levels. + #[serde(borrow, default)] + pub(super) level: Cow<'line, str>, + + /// The compiler's own rendering, snippet and underlines and all. + #[serde(borrow, default)] + pub(super) rendered: Option>, + + /// The error code, when the diagnostic has one. + #[serde(borrow, default)] + pub(super) code: Option>, + + /// Where in the source the diagnostic points. + #[serde(borrow, default)] + pub(super) spans: Vec>, + + /// The notes and helps hung off this diagnostic, which carry spans of their own. + #[serde(borrow, default)] + pub(super) children: Vec, +} + +/// The `E0382`-style code of a diagnostic, which arrives wrapped in an object of its own. +#[derive(Deserialize)] +pub(super) struct DiagnosticCode<'line> { + /// The code itself. + #[serde(borrow, default)] + pub(super) code: Cow<'line, str>, +} + +/// One region of source a diagnostic points at. +/// +/// The line and column numbers are `u64` rather than `u32` because they are read from JSON that +/// this tool does not produce: a number too large for a `u32` must clamp, exactly as the field-by- +/// field reading it replaced did, rather than fail the whole message and lose the diagnostic. +#[derive(Deserialize)] +pub(super) struct Span<'line> { + /// The file the span is in, as the compiler spelled it. + #[serde(borrow, default)] + pub(super) file_name: Option>, + + /// First line of the region, 1-based. + #[serde(default)] + pub(super) line_start: Option, + + /// Last line of the region, 1-based. + #[serde(default)] + pub(super) line_end: Option, + + /// First column of the region. + #[serde(default)] + pub(super) column_start: Option, + + /// Last column of the region. + #[serde(default)] + pub(super) column_end: Option, + + /// Whether this is the span the diagnostic is really about. + #[serde(default)] + pub(super) is_primary: bool, +} + +/// Decodes one line of cargo's JSON stream, or `None` when it is not one. +/// +/// Cargo interleaves its own prose on this stream when it feels like it, and a line that does not +/// decode is one this tool has nothing to say about rather than an error. +pub(super) fn cargo_message(line: &str) -> Option> { + serde_json::from_str(line).ok() +} + +/// Names every source file the compiler actually read, according to cargo's dep-info. +/// +/// Returns `None` when no dep-info could be read at all, which has to mean "do not draw any +/// conclusion" rather than "nothing was compiled": treating an unreadable scratch tree as an empty +/// set would condemn every mutant in the run as unbuilt. +/// +/// Cargo writes a `.d` file beside each artifact listing the sources that went into it, in the +/// makefile format `target: dep dep dep`. That list is the compiler's own account of what it read, +/// which is the only thing that answers the question honestly. Evaluating `#[cfg]` predicates +/// ourselves would mean reimplementing feature resolution, target detection and every other +/// predicate cargo and rustc already agreed on, and being subtly wrong about it in the cases that +/// matter most. +/// +/// Which `.d` files belong to *this* build is decided from the build's own artifact messages +/// rather than by looking at what is on disk. The scratch target directory is deliberately kept +/// between runs so that builds are incremental, so it accumulates dep-info from every earlier run +/// as well — including runs with a different feature set. Reading all of it would union today's +/// answer with a previous one and quietly conclude that everything was compiled, which is the +/// wrong answer in exactly the case this is here to catch. +pub(super) fn compiled_sources(stdout: &str, root: &Utf8Path) -> Option> { + let mut compiled: HashSet = HashSet::default(); + let mut read_any = false; + + for dep_file in dep_files(stdout) { + let Ok(text) = fs::read_to_string(dep_file.as_std_path()) else { + continue; + }; + + read_any = true; + + for line in text.lines() { + let Some((_artifact, list)) = line.split_once(": ") else { + continue; + }; + + for path in dependencies(list) { + let path = Utf8Path::new(&path); + let relative = path.strip_prefix(root).unwrap_or(path); + + let _added = compiled.insert(Utf8PathBuf::from(normalize_separators(relative.as_str()))); + } + } + } + + read_any.then_some(compiled) +} + +/// Splits the dependency half of a dep-info line into the paths it names. +/// +/// The `.d` file is a makefile fragment, so the separator is unescaped whitespace and a path that +/// contains a space is written with the space escaped: `src/my\ file.rs` is one dependency, not +/// two. Splitting on whitespace alone yields two fragments, neither of which is a path the survey +/// knows about, and the file then looks as though the compiler never read it — which is how every +/// mutant inside it comes to be excused as unbuilt. +/// +/// Only whitespace is treated as escapable, rather than "a backslash escapes whatever follows". +/// Both emitters — rustc's `escape_dep_filename` and cargo's — escape the space and nothing else, +/// so a backslash before anything other than whitespace is a Windows path separator and has to +/// survive to reach [`normalize_separators`]. A general makefile unescape would eat those +/// separators and turn `C:\src\lib.rs` into `C:srclib.rs`, trading a rare defect for a universal +/// one on that platform. +fn dependencies(list: &str) -> Vec { + let mut paths = Vec::new(); + let mut path = String::new(); + let mut characters = list.chars().peekable(); + + while let Some(character) = characters.next() { + match character { + '\\' if characters.peek().is_some_and(|next| next.is_whitespace()) => { + if let Some(escaped) = characters.next() { + path.push(escaped); + } + } + + character if character.is_whitespace() => { + if !path.is_empty() { + paths.push(core::mem::take(&mut path)); + } + } + + character => path.push(character), + } + } + + if !path.is_empty() { + paths.push(path); + } + + paths +} + +/// Names the dep-info file for every unit in one build, from cargo's JSON artifact messages. +/// +/// Cargo does not list the `.d` file among an artifact's `filenames`, but it does name it after +/// the same unit hash, so an artifact at `deps/libfoo-9a3f.rmeta` is described by `deps/foo-9a3f.d`. +/// Deriving the name from the hash rather than from the file stem avoids having to reproduce +/// cargo's own rules about which artifact kinds carry a `lib` prefix. +/// +/// Uplifted copies such as `debug/libfoo.rlib` are skipped: they carry no hash, so their dep-info +/// is overwritten by whichever run last built that package under any feature set, which is the +/// staleness this is avoiding. +pub(super) fn dep_files(stdout: &str) -> Vec { + let mut wanted: HashMap> = HashMap::default(); + + for line in stdout.lines() { + let Some(message) = cargo_message(line) else { + continue; + }; + + if message.reason != "compiler-artifact" { + continue; + } + + for filename in &message.filenames { + let path = Utf8Path::new(filename.as_ref()); + + let Some((directory, stem)) = path.parent().zip(path.file_stem()) else { + continue; + }; + + let Some((_name, hash)) = stem.rsplit_once('-') else { + continue; + }; + + let _added = wanted.entry(directory.to_owned()).or_default().insert(hash.to_owned()); + } + } + + let mut found = Vec::new(); + + for (directory, hashes) in wanted { + let Ok(entries) = fs::read_dir(directory.as_std_path()) else { + continue; + }; + + for entry in entries.flatten() { + let Ok(path) = Utf8PathBuf::from_path_buf(entry.path()) else { + continue; + }; + + if path.extension() != Some("d") { + continue; + } + + let matched = path + .file_stem() + .and_then(|stem| stem.rsplit_once('-')) + .is_some_and(|(_name, hash)| hashes.contains(hash)); + + if matched { + found.push(path); + } + } + } + + found +} + +/// Rewrites `\` to `/` so that a dep-info path compares equal to a discovered one on Windows. +pub(super) fn normalize_separators(path: &str) -> String { + path.replace('\\', "/") +} diff --git a/crates/cargo-gamma-lib/src/exec/build/splices.rs b/crates/cargo-gamma-lib/src/exec/build/splices.rs new file mode 100644 index 000000000..81895b35f --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/build/splices.rs @@ -0,0 +1,298 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The edits one build round writes into the scratch tree. + +use std::fs; + +use camino::Utf8PathBuf; + +use super::super::workspace::Workspace; +use super::{Converger, Guards}; +use crate::discover::{Plan, TargetFile}; +use crate::error::error; +use crate::parse::{BOM, strip_bom}; +use crate::schema::{self, Guard}; +use crate::{HashMap, HashSet, Result}; + +/// What each file of the copied tree was last instrumented with, so a round can skip the rest. +/// +/// The rollback loop instruments the whole tree once per round, but between two rounds only the +/// files whose mutants were withdrawn can differ — every other file would be read from disk, +/// re-spliced, and found to be byte-identical to what is already there. With a rollback ceiling in +/// the hundreds and a large tree that product is the cost, so what a round produced is remembered +/// and the next round rewrites only what it changed. +/// +/// Only the text of files that carry live mutants is kept. A file with none is written back to its +/// original once and then never touches this again, so caching it would hold the whole tree in +/// memory to serve a case that does not recur. +#[derive(Debug, Default)] +pub(super) struct Splices { + /// The tree these splices describe, so a workspace swapped underneath is not trusted. + pub(super) root: Utf8PathBuf, + + /// The original text of each instrumented file, read once rather than once per round. + pub(super) sources: HashMap, + + /// The live ordinals last spliced into each file, and the guards that resulted. + /// + /// The ordinals are the identity of the splice: two rounds that put the same live mutants into + /// a file produce the same text and therefore the same guards, so the recorded ones can be + /// handed back instead of being recomputed. + pub(super) placed: HashMap, HashMap)>, + + /// Maps file paths to positions in the growing plan. + file_index: HashMap, + + /// Mutant positions grouped once as stages extend the plan. + mutants_by_file: HashMap>, + + /// The file containing each live ordinal, for withdrawal deltas. + file_by_ordinal: HashMap, + + indexed_files: usize, + indexed_mutants: usize, + plan_identity: Option, + withdrawn: HashSet, +} + +#[derive(Debug)] +pub(super) struct Original { + parsed: String, + serialized: String, +} + +impl Original { + fn instrumented(&self, parsed: String) -> String { + if self.serialized.starts_with(BOM) { + let mut serialized = String::with_capacity(BOM.len_utf8() + parsed.len()); + serialized.push(BOM); + serialized.push_str(&parsed); + serialized + } else { + parsed + } + } +} + +impl Splices { + /// Drops indexes whose positions were invalidated by reordering the plan's mutant vector. + /// + /// Instrumented text and guards remain valid because they are keyed by file and ordinal, not + /// by vector position. + pub(super) fn plan_reordered(&mut self) { + self.file_index.clear(); + self.mutants_by_file.clear(); + self.file_by_ordinal.clear(); + self.indexed_files = 0; + self.indexed_mutants = 0; + self.plan_identity = None; + self.withdrawn.clear(); + } + + /// Writes the instrumented form of every mutated file into the copied tree. + /// + /// Returns where each live mutant's guard landed, which is what attributes a compiler diagnostic + /// back to the mutant responsible. + /// + /// The implementation visits only files whose live ordinals actually changed since the prior + /// round ("dirty" files). Every other file's cached guards are returned from `self.placed` + /// without re-reading, re-splicing or re-writing the file. + pub(super) fn instrument(&mut self, work: &Workspace, plan: &Plan, withdrawn: &HashSet) -> Result { + if self.root != work.root { + self.root = work.root.clone(); + self.sources.clear(); + self.placed.clear(); + self.file_index.clear(); + self.mutants_by_file.clear(); + self.file_by_ordinal.clear(); + self.indexed_files = 0; + self.indexed_mutants = 0; + self.withdrawn.clear(); + } + + self.restore_removed_files(work, plan)?; + let dirty = self.refresh_index(plan, withdrawn); + let mut guards = Guards::default(); + + for (path, (_ordinals, found)) in &self.placed { + if !dirty.contains(path) { + for (ordinal, guard) in found { + let _ = guards.insert(*ordinal, (path.clone(), guard.clone())); + } + } + } + + let mut dirty: Vec = dirty.iter().filter_map(|path| self.file_index.get(path).copied()).collect(); + dirty.sort_unstable(); + + for position in dirty { + let Some(file) = plan.files.get(position) else { + continue; + }; + let live: Vec<_> = self + .mutants_by_file + .get(&file.path) + .into_iter() + .flatten() + .filter_map(|position| plan.mutants.get(*position)) + .filter(|mutant| mutant.ordinal > 0 && !withdrawn.contains(&mutant.ordinal)) + .collect(); + let ordinals: Vec = live.iter().map(|mutant| mutant.ordinal).collect(); + + if let Some((placed, found)) = self.placed.get(&file.path) + && *placed == ordinals + { + for (ordinal, guard) in found { + let _ = guards.insert(*ordinal, (file.path.clone(), guard.clone())); + } + + continue; + } + + let original = self.original(file)?; + + // A file whose every mutant has been withdrawn still has to be rewritten, back to the + // original, or the previous round's instrumented copy would survive its own withdrawal + // and the rollback loop could never converge. + let (instrumented, found) = if live.is_empty() { + (original.serialized.clone(), HashMap::default()) + } else { + let (parsed, found) = schema::instrument_with_guards(&original.parsed, &live)?; + (original.instrumented(parsed), found) + }; + + for (ordinal, guard) in &found { + let _ = guards.insert(*ordinal, (file.path.clone(), guard.clone())); + } + + // A live mutant with no guard would still be run — with nothing in the tree to make it + // behave differently — and its verdict recorded as a survivor. That is a wrong answer + // rather than a missing one, and nothing downstream could tell the difference, so the + // invariant is checked rather than assumed. + if let Some(missing) = live.iter().find(|mutant| !guards.contains_key(&mutant.ordinal)) { + return Err(Converger::missing_guard_error(missing)); + } + + // Rewriting a file with the text it already holds would make cargo rebuild its crate, so + // an unchanged file is left alone and its mtime with it. + let destination = work.root.join(&file.path); + let _written = Workspace::overwrite(&work.root, &destination, &instrumented)?; + + let _replaced = self.placed.insert(file.path.clone(), (ordinals, found)); + + // A file back at its original text will not be spliced again unless its mutants come + // back, which they cannot: withdrawal is permanent for the rest of the run. + if live.is_empty() { + let _dropped = self.sources.remove(&file.path); + } + } + + Ok(guards) + } + + fn restore_removed_files(&mut self, work: &Workspace, plan: &Plan) -> Result<()> { + let identity = core::ptr::from_ref(plan) as usize; + if self.plan_identity.is_none_or(|previous| previous == identity) { + return Ok(()); + } + + let current: HashSet<&camino::Utf8Path> = plan.files.iter().map(|file| file.path.as_path()).collect(); + let removed: Vec = self + .placed + .keys() + .filter(|path| !current.contains(path.as_path())) + .cloned() + .collect(); + + for path in removed { + if let Some(original) = self.sources.get(&path) { + let destination = work.root.join(&path); + let _written = Workspace::overwrite(&work.root, &destination, &original.serialized)?; + } + let _placed = self.placed.remove(&path); + let _source = self.sources.remove(&path); + } + + Ok(()) + } + + fn refresh_index(&mut self, plan: &Plan, withdrawn: &HashSet) -> HashSet { + let mut dirty = HashSet::default(); + let plan_identity = core::ptr::from_ref(plan) as usize; + + if self.plan_identity != Some(plan_identity) || self.indexed_files > plan.files.len() || self.indexed_mutants > plan.mutants.len() { + dirty.extend(self.file_index.keys().cloned()); + self.file_index.clear(); + self.mutants_by_file.clear(); + self.file_by_ordinal.clear(); + self.indexed_files = 0; + self.indexed_mutants = 0; + dirty.extend(plan.files.iter().map(|file| file.path.clone())); + } + self.plan_identity = Some(plan_identity); + + for (position, file) in plan.files.iter().enumerate().skip(self.indexed_files) { + let _previous = self.file_index.insert(file.path.clone(), position); + let _new = dirty.insert(file.path.clone()); + } + self.indexed_files = plan.files.len(); + + for (position, mutant) in plan.mutants.iter().enumerate().skip(self.indexed_mutants) { + if mutant.ordinal > 0 { + self.mutants_by_file.entry(mutant.file.to_path_buf()).or_default().push(position); + let _previous = self.file_by_ordinal.insert(mutant.ordinal, mutant.file.to_path_buf()); + let _new = dirty.insert(mutant.file.to_path_buf()); + } + } + self.indexed_mutants = plan.mutants.len(); + + if self.withdrawn.is_subset(withdrawn) { + for ordinal in withdrawn.difference(&self.withdrawn) { + if let Some(path) = self.file_by_ordinal.get(ordinal) { + let _new = dirty.insert(path.clone()); + } + } + } else { + dirty.extend(self.file_index.keys().cloned()); + } + self.withdrawn.clone_from(withdrawn); + + dirty + } + + /// The file's original text, read from disk the first time a round needs it and kept after. + /// + /// Read here rather than taken from the survey's `SourceFile`, so the byte-order mark has to be + /// dropped here too: mutant spans index the text `syn` saw, which is the text after the mark. + pub(super) fn original(&mut self, file: &TargetFile) -> Result<&Original> { + if !self.sources.contains_key(&file.path) { + let serialized = fs::read_to_string(file.absolute.as_std_path()) + .map_err(|cause| error!("could not read `{}`", file.absolute).caused_by(cause))?; + let parsed = strip_bom(&serialized).to_owned(); + + let _stored = self.sources.insert(file.path.clone(), Original { parsed, serialized }); + } + + Ok(self.sources.get(&file.path).unwrap_or_else(|| unreachable!("just inserted"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn instrumented_text_retains_the_original_byte_order_mark() { + let parsed = "fn f() {}\n"; + let original = Original { + parsed: parsed.to_owned(), + serialized: format!("{BOM}{parsed}"), + }; + + assert_eq!( + original.instrumented("fn f() { gamma(); }\n".to_owned()), + format!("{BOM}fn f() {{ gamma(); }}\n") + ); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/build/tests.rs b/crates/cargo-gamma-lib/src/exec/build/tests.rs new file mode 100644 index 000000000..895b1ab94 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/build/tests.rs @@ -0,0 +1,3015 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::fmt::Write as _; +use core::ops::Range; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::{fs, io, thread}; + +use camino::Utf8Path; +use serde_json::Value; + +use super::complaints::Diagnostic; +use super::invoke::{ + OutputLimits, Stream, compile, drained, finish_readers, is_progress, read_pipe, read_pipe_with_limits, rendered_diagnostic, + spawn_failure, supervise, supervise_with_limits, +}; +use super::messages::dep_files; +use super::*; +use crate::discover::TargetFile; +use crate::schema::Position; + +/// The ordinals [`blame`] attributed, without the error codes it attributed them under. +/// +/// Every one of these tests predates the census and asks only which mutants were named, so +/// they say so directly rather than each spelling out a code they do not care about. +fn ordinals_blamed(stdout: &str, root: &Utf8Path, guards: &Guards) -> HashSet { + blame(stdout, root, guards).into_keys().collect() +} + +fn artifact_message(path: &Utf8Path) -> String { + serde_json::json!({ + "reason": "compiler-artifact", + "filenames": [path.as_str()], + }) + .to_string() +} + +/// The bar is what a reader watches during the long silences, so it has to be recognised. +#[test] +fn cargos_progress_bar_is_told_apart_from_the_things_it_wanted_to_say() { + assert!(is_progress(" Building [====> ] 4/17: serde_core, quote")); + assert!(is_progress(" Compiling [=> ] 1/9: syn")); + assert!(is_progress("\u{1b}[1;36m Building\u{1b}[0m [====> ] 4/17: serde_core, quote")); + + assert!(!is_progress(" Compiling serde v1.0.229")); + assert!(!is_progress( + " Finished `dev` profile [unoptimized + debuginfo] target(s) in 6.14s" + )); + assert!(!is_progress("")); +} + +/// The escape valve exists to reproduce what running cargo directly would have shown, so it +/// passes the compiler's own rendering through untouched, warnings included. +#[test] +fn a_diagnostic_is_surfaced_the_way_the_compiler_rendered_it() { + let message = |level: &str| { + format!( + r#"{{"reason":"compiler-message","message":{{"level":"{level}","message":"mismatched types","rendered":"error[E0308]: mismatched types\n --> src/lib.rs:2:5\n"}}}}"# + ) + }; + + assert_eq!( + rendered_diagnostic(&message("error")).as_deref(), + Some("error[E0308]: mismatched types\n --> src/lib.rs:2:5\n") + ); + + assert!(rendered_diagnostic(&message("warning")).is_some()); +} + +/// Everything else on the stream is cargo talking about its own progress, not a diagnostic. +#[test] +fn a_line_that_is_not_a_compiler_message_yields_no_diagnostic() { + assert!(rendered_diagnostic(r#"{"reason":"compiler-artifact","target":{"name":"serde"}}"#).is_none()); + assert!(rendered_diagnostic(" Building [====> ] 4/17: serde").is_none()); + assert!(rendered_diagnostic("").is_none()); +} + +/// A message cargo did not render carries nothing a reader could act on. +#[test] +fn a_message_without_a_rendering_yields_nothing() { + let bare = r#"{"reason":"compiler-message","message":{"level":"error","message":"could not find `nope`","spans":[]}}"#; + + assert!(rendered_diagnostic(bare).is_none()); + assert!(rendered_diagnostic(r#"{"reason":"compiler-message","message":{"level":"error","rendered":" "}}"#).is_none()); +} + +/// Cargo draws its bar with carriage returns and no newlines, so a reader that split only on +/// newlines would see one enormous line at the very end and nothing at all while it mattered. +#[test] +fn the_reader_splits_on_carriage_returns_as_well_as_newlines() { + let (sender, lines) = mpsc::sync_channel(64); + let text = "one\rtwo\nthree\r\nfour"; + + let collected = read_pipe(io::Cursor::new(text), Stream::Prose, &sender) + .expect("spawn reader") + .join() + .expect("reader"); + + drop(sender); + + let seen: Vec = lines.into_iter().map(|(_, line)| line).collect(); + + assert_eq!(seen, ["one", "two", "three", "four"]); + assert_eq!(collected.text, text.as_bytes()); + assert!(collected.complete, "a stream read to its end is the whole of it"); + assert!(collected.within_limits, "short output stays within the normal output limit"); +} + +/// A pipe that fails part way through is not a stream that ended. +/// +/// `supervise` refuses a build whose readers did not finish, because a truncated JSON stream loses +/// artifacts and a run that lost them reports the test binaries it could not find as ones that do +/// not exist, and excuses the mutants in files it could not see as never built. A reader that +/// stopped on an `EIO` and handed back what it had would walk in through the one door that guard +/// does not watch: the thread is finished, so the bytes look like the whole of what cargo said. +#[test] +fn a_pipe_that_fails_part_way_is_not_read_as_the_end_of_the_stream() { + /// Yields one chunk and then refuses, the way a pipe whose writer died mid-stream does. + struct Faltering(bool); + + impl io::Read for Faltering { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.0 { + return Err(io::Error::other("the read a test asked to fail")); + } + + self.0 = true; + + let said = b"{\"reason\":\"compiler-artifact\"}\n"; + buf[..said.len()].copy_from_slice(said); + + Ok(said.len()) + } + } + + let (sender, _lines) = mpsc::sync_channel(64); + let reader = read_pipe(Faltering(false), Stream::Json, &sender).expect("spawn reader"); + + assert!( + drained(Some(reader), Instant::now() + Duration::from_secs(5)).is_none(), + "a reader that stopped on an error must not pass its bytes off as the whole stream" + ); +} + +/// A small configurable limit lets the failure path be tested without allocating megabytes. +#[test] +fn a_build_past_its_output_limit_fails_without_retaining_the_excess() { + let directory = crate::testing::workdir("build-output-limit-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("utf8"); + let work = Workspace::adopt(root.clone(), root.join("target")); + let mut command = Command::new(crate::testing::helper_binary_path().as_std_path()); + + let _configured = command + .arg(crate::testing::directive("print:the compiler said more than sixteen bytes")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let error = supervise_with_limits( + &mut command, + &work, + Some(Duration::from_secs(30)), + &mut crate::testing::Recorder::default(), + OutputLimits { + retained: 16, + line: 64, + backlog: 1, + }, + ) + .expect_err("output above the configured limit is not parsed as a complete build"); + + assert!( + error + .to_string() + .contains("configured 16-byte retained or 64-byte per-line build-output limit"), + "{error}" + ); +} + +/// A small retained cap records truncation while continuing to drain the pipe. +#[test] +fn an_over_limit_pipe_keeps_only_its_configured_prefix() { + let (sender, _lines) = mpsc::sync_channel(4); + let pipe = read_pipe_with_limits( + io::Cursor::new("0123456789\n"), + Stream::Prose, + &sender, + OutputLimits { + retained: 4, + line: 16, + backlog: 4, + }, + ) + .expect("spawn reader") + .join() + .expect("reader"); + + assert_eq!(pipe.text, b"0123"); + assert!(pipe.complete, "the pipe was still drained to EOF"); + assert!(!pipe.within_limits, "the capped output is explicitly marked incomplete"); +} + +#[test] +fn reader_thread_creation_failure_is_reported_as_incomplete_output() { + let (sender, lines) = mpsc::sync_channel(1); + let _refused = crate::exec::faults::arm(crate::exec::faults::Fault::Thread); + + let failure = read_pipe(io::Cursor::new("cargo output"), Stream::Prose, &sender).expect_err("thread creation was asked to fail"); + drop(sender); + let mut events = crate::testing::Recorder::default(); + let (stdout, _stderr) = finish_readers( + Some(Err(failure)), + None, + &lines, + &mut events, + Instant::now() + Duration::from_secs(1), + ); + + assert!(stdout.is_none(), "a reader that could not be created was treated as complete"); +} + +/// Finishing readers continues narration, so a bounded channel never turns collection into a +/// post-build deadlock. +#[test] +fn finishing_readers_drains_a_backpressured_narration_channel() { + let (sender, lines) = mpsc::sync_channel(0); + let stdout = read_pipe_with_limits( + io::Cursor::new("first\nsecond\n"), + Stream::Prose, + &sender, + OutputLimits { + retained: 64, + line: 64, + backlog: 0, + }, + ); + let mut events = crate::testing::Recorder::default(); + + let (stdout, stderr) = finish_readers(Some(stdout), None, &lines, &mut events, Instant::now() + Duration::from_secs(5)); + + assert_eq!(stdout.expect("the reader was drained").text, b"first\nsecond\n"); + assert!(stderr.is_some(), "an absent stderr pipe is an empty complete stream"); +} + +use crate::ops::collect::Shape; + +/// The diagnostic from a build that could not be made to compile, or a panic if it did. +fn stuck_reason(convergence: Convergence) -> String { + match convergence { + Convergence::Built(_stdout) => panic!("the build was expected not to compile"), + Convergence::Stuck(reason) => reason.to_string(), + } +} + +/// A workspace holding one trivial crate, so a real cargo invocation is cheap. +fn trivial_workspace(prefix: &str) -> (tempfile::TempDir, Workspace) { + let dir = crate::testing::workdir(prefix); + let root = Utf8PathBuf::from_path_buf(dir.path().join("src")).expect("utf8"); + + fs::create_dir_all(root.join("src").as_std_path()).expect("src"); + fs::write( + root.join("Cargo.toml").as_std_path(), + "[package]\nname = \"trivial\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n", + ) + .expect("manifest"); + fs::write(root.join("src/lib.rs").as_std_path(), "pub const A: i32 = 1;\n").expect("lib"); + + let target = Utf8PathBuf::from_path_buf(dir.path().join("target")).expect("utf8"); + let work = Workspace::adopt(root, target); + + (dir, work) +} + +/// A build that fails for a reason no guard explains stops rather than looping forever. +#[test] +fn a_build_that_no_guard_explains_stops_with_the_compiler_output() { + let (_dir, work) = trivial_workspace("build-unattributed-"); + + // Broken source that has nothing to do with instrumentation: no mutant can be withdrawn + // to make it compile, so withdrawing forever would be an infinite loop. + fs::write( + work.root.join("src/lib.rs").as_std_path(), + "pub const A: i32 = \"not an integer\";\n", + ) + .expect("lib"); + + let plan = empty_plan(&work); + let limits = BuildLimits::default(); + let convergence = Converger::default() + .converge( + &work, + &plan, + None, + &["build", "--tests", "--keep-going"], + limits, + &mut crate::testing::Recorder::default(), + ) + .expect("the build ran, however badly"); + + assert!(stuck_reason(convergence).contains("could not be attributed")); +} + +/// A build that fails before the compiler is reached says so, rather than blaming a mutant. +/// +/// The two ways a build can fail unattributably are not the same failure, and the message that +/// serves one misleads about the other. `a_build_that_no_guard_explains_stops_with_the_compiler +/// _output` covers the case where rustc ran and complained about something no guard explains. +/// This one covers the case where rustc was never reached at all: cargo gave up on a build +/// script, so the JSON stream carries no diagnostics and there is nothing to attribute. Telling +/// the reader the tree "does not compile" there sends them hunting for a broken mutant that was +/// never generated. +#[test] +fn a_build_that_never_reached_the_compiler_says_so_rather_than_blaming_the_tree() { + let (_dir, work) = trivial_workspace("build-no-diagnostics-"); + + // A build script that panics fails the build with an exit status and a message on stderr, + // and with no rustc diagnostic anywhere in the JSON stream. + fs::write( + work.root.join("build.rs").as_std_path(), + "fn main() { panic!(\"the build script refused\"); }\n", + ) + .expect("build script"); + + let plan = empty_plan(&work); + let convergence = Converger::default() + .converge( + &work, + &plan, + None, + &["build", "--tests", "--keep-going"], + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build ran, however badly"); + let reason = stuck_reason(convergence); + + assert!(reason.contains("the compiler reported nothing"), "{reason}"); + assert!( + !reason.contains("does not compile"), + "the message for a tree rustc rejected must not be reused here: {reason}" + ); + assert!( + reason.contains("the build script refused"), + "cargo's own words are the only account of this failure: {reason}" + ); +} + +/// A narrowed build that fails is retried across the whole workspace before giving up. +#[test] +fn a_narrowed_build_that_fails_is_retried_across_the_whole_workspace() { + let (_dir, work) = trivial_workspace("build-widen-"); + let mut plan = empty_plan(&work); + let select = vec!["no-such-package".to_owned()]; + + // Cargo rejects the selection outright, which is exactly the shape of failure the widen + // path exists for: the narrowing is at fault, not the code being built. + let build = Converger::default() + .finish( + &work, + &mut plan, + Some(&select), + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("widening to the whole workspace must succeed"); + + assert!(build.widened, "the build should have reported that it widened"); +} + +/// Examples remain part of the compilation oracle even though they are never test binaries. +#[test] +fn the_final_build_still_compiles_examples() { + let (_dir, work) = trivial_workspace("build-example-"); + + fs::create_dir_all(work.root.join("examples").as_std_path()).expect("examples"); + fs::write( + work.root.join("examples/broken.rs").as_std_path(), + "fn main() { let _: i32 = \"not an integer\"; }\n", + ) + .expect("example"); + + let mut plan = empty_plan(&work); + let build = Converger::default() + .finish( + &work, + &mut plan, + Some(&["trivial".to_owned()]), + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build reports the example failure"); + + assert!( + build.stuck.is_some(), + "a broken example must not disappear from the compilation oracle" + ); + assert!(build.binaries.is_empty(), "a failed compilation produces no runnable oracle"); +} + +/// A workspace of two members, one of which does not compile and is not being mutated. +fn split_workspace(prefix: &str) -> (tempfile::TempDir, Workspace) { + let dir = crate::testing::workdir(prefix); + let root = Utf8PathBuf::from_path_buf(dir.path().join("src")).expect("utf8"); + + fs::create_dir_all(root.join("good/src").as_std_path()).expect("good"); + fs::create_dir_all(root.join("broken/src").as_std_path()).expect("broken"); + fs::write( + root.join("Cargo.toml").as_std_path(), + "[workspace]\nmembers = [\"good\", \"broken\"]\nresolver = \"3\"\n", + ) + .expect("workspace manifest"); + + for member in ["good", "broken"] { + fs::write( + root.join(member).join("Cargo.toml").as_std_path(), + format!("[package]\nname = \"{member}\"\nversion = \"0.0.0\"\nedition = \"2024\"\n"), + ) + .expect("member manifest"); + } + + fs::write(root.join("good/src/lib.rs").as_std_path(), "pub const A: i32 = 1;\n").expect("good lib"); + fs::write( + root.join("broken/src/lib.rs").as_std_path(), + "pub const B: i32 = \"gamma-broken-marker\";\n", + ) + .expect("broken lib"); + + let target = Utf8PathBuf::from_path_buf(dir.path().join("target")).expect("utf8"); + let work = Workspace::adopt(root, target); + + (dir, work) +} + +/// A workspace whose selected member compiles only when cargo unifies features over every member. +/// +/// `leaf` guards its only item behind an optional feature that nothing in `leaf` activates; `app` +/// depends on `leaf` and turns it on. A build of `leaf` alone therefore fails, and a build of both +/// succeeds — which is exactly the trap a narrowed build falls into and no mutant is responsible +/// for. +fn unified_workspace(prefix: &str) -> (tempfile::TempDir, Workspace) { + let dir = crate::testing::workdir(prefix); + let root = Utf8PathBuf::from_path_buf(dir.path().join("src")).expect("utf8"); + + fs::create_dir_all(root.join("app/src").as_std_path()).expect("app"); + fs::create_dir_all(root.join("leaf/src").as_std_path()).expect("leaf"); + fs::write( + root.join("Cargo.toml").as_std_path(), + "[workspace]\nmembers = [\"app\", \"leaf\"]\nresolver = \"3\"\n", + ) + .expect("workspace manifest"); + + fs::write( + root.join("app/Cargo.toml").as_std_path(), + "[package]\nname = \"app\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [dependencies]\nleaf = { path = \"../leaf\", features = [\"wide\"] }\n", + ) + .expect("app manifest"); + fs::write( + root.join("leaf/Cargo.toml").as_std_path(), + "[package]\nname = \"leaf\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[features]\nwide = []\n", + ) + .expect("leaf manifest"); + + fs::write( + root.join("leaf/src/lib.rs").as_std_path(), + "#[cfg(feature = \"wide\")]\npub const WIDE: i32 = 1;\n\n\ + pub fn value() -> i32 {\n WIDE\n}\n", + ) + .expect("leaf lib"); + fs::write( + root.join("app/src/lib.rs").as_std_path(), + "pub fn value() -> i32 {\n leaf::value()\n}\n", + ) + .expect("app lib"); + + let target = Utf8PathBuf::from_path_buf(dir.path().join("target")).expect("utf8"); + let work = Workspace::adopt(root, target); + + (dir, work) +} + +/// The preflight says which scope proved the tree, not merely that some scope did. +/// +/// A narrow failure that the whole workspace survives is a statement about the *selection*: the +/// tree compiles under cargo's feature unification and does not compile under a subset of it. +/// Answering `Ok(())` and dropping which scope answered leaves the run free to narrow again. +#[test] +fn a_check_that_only_the_whole_workspace_passes_says_so() { + let (_dir, work) = unified_workspace("build-preflight-unified-"); + let plan = empty_plan(&work); + let select = vec!["leaf".to_owned()]; + + let cleared = Converger::preflight( + &work, + &plan, + Some(&select), + &select, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the workspace compiles when its features are unified"); + + assert!( + cleared.whole_workspace, + "the check widened to succeed and then reported nothing about it" + ); + assert!(cleared.dropped.is_empty(), "{:?}", cleared.dropped); +} + +/// A converger told the tree only builds whole does not narrow again. +/// +/// Without this, the staged build reruns the failure the preflight already attributed to the +/// selection, the rollback loop blames whichever mutants it can, and valid mutants are settled as +/// `NotBuilt` — withheld from the denominator and quietly raising the score. +#[test] +fn a_whole_workspace_requirement_survives_into_the_staged_builds() { + let (_dir, work) = unified_workspace("build-stage-unified-"); + let mut plan = empty_plan(&work); + let stage = vec!["leaf".to_owned()]; + + let mut narrowing = Converger::default(); + + assert!( + narrowing + .stage( + &work, + &mut plan, + &stage, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build runs") + .is_some(), + "the fixture is meant to be one a narrowed build cannot compile" + ); + + let mut whole = Converger::default(); + + whole.require_whole_workspace(); + + assert!( + whole + .stage( + &work, + &mut plan, + &stage, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build runs") + .is_none(), + "the scope the preflight validated was not carried into the staged build" + ); +} + +/// A package nobody asked to mutate, and which does not compile, must not stop the whole run. +/// +/// Both wider checks fail on it and neither says anything about the code the caller asked +/// about. Refusing to run on that evidence turns somebody else's broken crate into a tool that +/// cannot be used at all, when narrowing by hand would have worked — which is a flag the caller +/// had no reason to know they needed. +#[test] +fn a_broken_package_nobody_is_mutating_is_dropped_rather_than_failing_the_run() { + let (_dir, work) = split_workspace("build-preflight-retreat-"); + let plan = empty_plan(&work); + let select = vec!["broken".to_owned(), "good".to_owned()]; + let mutating = vec!["good".to_owned()]; + + let dropped = Converger::preflight( + &work, + &plan, + Some(&select), + &mutating, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the package being mutated compiles on its own"); + + // Named, because the run is about to stop building and running their tests, and a mutant + // they would have killed will be reported as a survivor. + assert_eq!(dropped.dropped, vec!["broken".to_owned()]); + assert!( + !dropped.whole_workspace, + "a retreat narrows the scope, so it cannot be reporting that only the whole workspace built" + ); +} + +/// The retreat is not a way to run over code that does not compile. +/// +/// A failure in the package being mutated survives every narrowing there is, so the run stops +/// exactly as it did before — otherwise the preflight would stop being the thing that makes +/// later compiler errors attributable to a mutant. +#[test] +fn a_broken_package_that_is_being_mutated_still_stops_the_run() { + let (_dir, work) = split_workspace("build-preflight-noretreat-"); + + fs::write( + work.root.join("good/src/lib.rs").as_std_path(), + "pub const A: i32 = \"gamma-selected-marker\";\n", + ) + .expect("good lib"); + + let plan = empty_plan(&work); + let select = vec!["broken".to_owned(), "good".to_owned()]; + let mutating = vec!["good".to_owned()]; + + let error = Converger::preflight( + &work, + &plan, + Some(&select), + &mutating, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect_err("no narrowing makes the selected package compile"); + + assert!(error.to_string().contains("gamma-selected-marker"), "{error}"); +} + +/// When both preflight checks fail, the caller hears about the packages they selected. +/// +/// The wider check is only ever asked whether feature unification explains the narrow failure. +/// Reporting its diagnostics instead would answer "your tree does not compile" with errors from +/// a crate the caller never chose to mutate, which is the wrong thing to send them to fix. +#[test] +fn a_preflight_that_fails_both_ways_reports_the_selected_packages_errors() { + let (_dir, work) = trivial_workspace("build-preflight-both-"); + + fs::write( + work.root.join("src/lib.rs").as_std_path(), + "pub const A: i32 = \"gamma-narrow-marker\";\n", + ) + .expect("lib"); + + let plan = empty_plan(&work); + let select = vec!["trivial".to_owned()]; + + let error = Converger::preflight( + &work, + &plan, + Some(&select), + &select, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect_err("neither check can succeed"); + + assert!(error.to_string().contains("gamma-narrow-marker"), "{error}"); +} + +/// A whole-workspace build that fails is reported as it stands, with nothing to widen to. +#[test] +fn a_whole_workspace_build_that_fails_is_reported_rather_than_retried() { + let (_dir, work) = trivial_workspace("build-nowiden-"); + + fs::write( + work.root.join("src/lib.rs").as_std_path(), + "pub const A: i32 = \"not an integer\";\n", + ) + .expect("lib"); + + let mut plan = empty_plan(&work); + + // There was no narrowing to blame, so there is no second build to try: retrying the same + // command would only spend the time again to reach the same answer. + let build = Converger::default() + .finish( + &work, + &mut plan, + None, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build ran, however badly"); + + let stuck = build.stuck.expect("the build could not be made to compile"); + + assert!(stuck.reason.contains("could not be attributed"), "{}", stuck.reason); + assert!(build.binaries.is_empty(), "a build that never compiled has nothing to run"); +} + +/// A rollback loop that keeps hitting compile errors it can attribute has to stop somewhere, +/// or an author who mistypes `--rollback-rounds` far too low would watch the tool spin without +/// ever explaining why: the limit is reported by name, with the withdrawal history that led to +/// it, rather than the run simply hanging. +#[test] +fn hitting_the_rollback_round_limit_is_reported_rather_than_retried_forever() { + let (_dir, work) = trivial_workspace("build-rollback-limit-"); + + // A guard around a `const` initializer cannot compile: the compiler cannot call a function + // while evaluating a constant, whatever the guard would have chosen between. That failure + // is squarely inside the guard's own span, so it is exactly the kind of round this loop is + // meant to withdraw and retry — which is what makes it a safe way to force a limit to bite + // deterministically rather than relying on a build that never converges at all. + let text = "pub const A: i32 = 1;\n"; + let start = text.find('1').expect("the literal is in the fixture"); + let mutant = Mutant { + span: start..start + 1, + replacement: "2".to_owned().into(), + ..mutant() + }; + let mut plan = empty_plan(&work); + plan.files.push(target_file(&work.root, "src/lib.rs")); + plan.mutants.push(mutant); + + let limits = BuildLimits { + timeout: None, + multiplier: None, + rollback_rounds: 1, + }; + + let convergence = Converger::default() + .converge( + &work, + &plan, + None, + &["build", "--keep-going"], + limits, + &mut crate::testing::Recorder::default(), + ) + .expect("a guard around a const initializer cannot compile"); + + assert!(stuck_reason(convergence).contains("rollback")); +} + +/// A stage that cannot be converged gives up on its own mutants instead of ending the run, and +/// records them as never built rather than as unviable. +/// +/// The two are not the same claim. `unviable` says the compiler looked at this mutant and +/// refused it; `notbuilt` says nobody ever asked. A run that could not converge has made the +/// second claim about every mutant in the failing build, and dressing that up as the first +/// would put a verdict on mutants the tool never judged. +#[test] +fn a_stage_that_cannot_be_converged_abandons_its_mutants_as_never_built() { + let (_dir, work) = trivial_workspace("build-stage-abandons-"); + let mut plan = unguardable_plan(&work, 1); + + plan.mutants[0].package = ("trivial".to_owned()).into(); + + // One round only, so the guard the first round blames exhausts the budget rather than being + // withdrawn and retried. That is the rollback-limit half of giving up. + let limits = BuildLimits { + timeout: None, + multiplier: None, + rollback_rounds: 1, + }; + let mut converger = Converger::default(); + + let abandoned = converger + .stage( + &work, + &mut plan, + &["trivial".to_owned()], + limits, + &mut crate::testing::Recorder::default(), + ) + .expect("the stage ran, however badly") + .expect("a budget of one round cannot survive a failing round"); + + assert_eq!(abandoned.ordinals, vec![1]); + + // The advice that would otherwise reach the user as the run's dying words has to reach them as a + // diagnostic instead: whether the counts were falling or flat is the only thing that + // decides whether raising `--rollback-rounds` would have helped. With a budget of one + // round the series is this one round, so it has to be there rather than being omitted for + // never having been withdrawn. + assert!(abandoned.reason.contains("rollback rounds"), "{}", abandoned.reason); + assert!( + abandoned.reason.contains("Mutants blamed in the last rounds of this build: 1"), + "{}", + abandoned.reason + ); + + assert_eq!(plan.mutants[0].outcome, Outcome::NotBuilt); + assert!(plan.mutants[0].note.is_some(), "the mutant should say why it never ran"); + + // And the count the run reports as unviable does not swallow them: nothing here was + // blamed on a mutant. + converger.settle(&mut plan); + + assert_eq!(plan.mutants[0].outcome, Outcome::NotBuilt, "settling must not overwrite it"); +} + +/// The run carries on after a stage it could not converge, because giving up on that stage's +/// mutants puts its sources back exactly as the preflight check found them. +/// +/// This is what makes continuing defensible rather than optimistic: the next build is asked of +/// a tree with no guards left in the offending package, which is the tree that was already +/// proved to compile. +#[test] +fn a_stage_the_run_gave_up_on_leaves_a_tree_the_next_build_can_still_compile() { + let (_dir, work) = trivial_workspace("build-stage-carries-on-"); + let mut plan = unguardable_plan(&work, 1); + + plan.mutants[0].package = ("trivial".to_owned()).into(); + + let limits = BuildLimits { + timeout: None, + multiplier: None, + rollback_rounds: 1, + }; + let mut converger = Converger::default(); + let packages = ["trivial".to_owned()]; + + let _abandoned = converger + .stage(&work, &mut plan, &packages, limits, &mut crate::testing::Recorder::default()) + .expect("the stage ran, however badly") + .expect("a budget of one round cannot survive a failing round"); + + let again = converger + .stage(&work, &mut plan, &packages, limits, &mut crate::testing::Recorder::default()) + .expect("the stage ran"); + + assert!(again.is_none(), "the tree compiles once the abandoned mutants are out of it"); +} + +#[test] +fn an_unattributed_failure_is_isolated_to_the_mutant_that_provably_breaks_the_build() { + let (_dir, work) = trivial_workspace("build-isolates-unattributed-"); + let text = "pub fn bad() -> i32 { 1 }\n\ + pub fn good(x: i32) -> i32 { x + 1 }\n"; + let pristine = work.root.parent().expect("tree parent").join("pristine"); + + fs::create_dir_all(pristine.as_std_path()).expect("pristine"); + fs::write(work.root.join("src/lib.rs").as_std_path(), text).expect("working source"); + fs::write(pristine.join("lib.rs").as_std_path(), text).expect("pristine source"); + + let bad_start = text.find("{ 1 }").expect("bad body") + 2; + let good_start = text.find("x + 1").expect("addition"); + let mut bad = mutant(); + bad.ordinal = 1; + bad.span = bad_start..bad_start + 1; + bad.original = "1".into(); + bad.replacement = "()".into(); + bad.item_path = ("trivial::bad".to_owned()).into(); + bad.package = ("trivial".to_owned()).into(); + let mut good = mutant(); + good.ordinal = 2; + good.span = good_start..good_start + "x + 1".len(); + good.original = "x + 1".into(); + good.replacement = "x - 1".into(); + good.item_path = ("trivial::good".to_owned()).into(); + good.package = ("trivial".to_owned()).into(); + + let mut plan = empty_plan(&work); + plan.files.push(TargetFile { + path: Utf8PathBuf::from("src/lib.rs"), + absolute: pristine.join("lib.rs"), + package: "trivial".to_owned(), + }); + plan.mutants.extend([bad, good]); + fs::create_dir_all(work.root.join("gamma-rt/src").as_std_path()).expect("runtime source"); + fs::write( + work.root.join("gamma-rt/Cargo.toml").as_std_path(), + "[package]\nname = \"gamma_rt\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ) + .expect("runtime manifest"); + fs::write( + work.root.join("gamma-rt/src/lib.rs").as_std_path(), + "pub const fn a(_: u32) -> bool { false }\n", + ) + .expect("runtime library"); + work.link_runtime("trivial", &plan.files).expect("runtime linked"); + + let isolated = Converger::default() + .isolate( + &work, + &plan, + Some(&["trivial".to_owned()]), + &["build", "--keep-going"], + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("isolation builds ran") + .expect("the bad mutant was isolated"); + + assert!(matches!(isolated, Isolation::Blamed(ordinals) if ordinals == vec![1])); +} + +/// Mutants that compile alone but fail only together settle to `NotBuilt`, not `CompileError`. +/// +/// When `isolate` narrows an unattributed failure to an interaction inside one item it returns +/// [`Isolation::Item`], and convergence abandons those ordinals so [`Converger::settle`] records +/// them as never judged rather than accusing honest mutants of being unviable. No cheap real +/// fixture makes two mutations compile apart and fail together, so the proof build is scripted +/// through the test-only `subset_oracle`: each mutant compiles on its own, only the pair fails. +#[test] +fn an_item_only_interaction_settles_its_mutants_to_not_built() { + #[expect( + clippy::unnecessary_wraps, + reason = "the oracle matches subset_fails's Option verdict, where None is an indeterminate build" + )] + fn only_the_pair_fails(active: &HashSet) -> Option { + Some(active.contains(&1) && active.contains(&2)) + } + + let (_dir, work) = trivial_workspace("build-isolates-interaction-"); + + let mut plan = empty_plan(&work); + plan.mutants = vec![Mutant { ordinal: 1, ..mutant() }, Mutant { ordinal: 2, ..mutant() }]; + + let mut converger = Converger { + subset_oracle: Some(only_the_pair_fails), + ..Converger::default() + }; + + let isolated = converger + .isolate( + &work, + &plan, + None, + &["build", "--keep-going"], + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the scripted proof builds ran") + .expect("the interaction was isolated to its item"); + + assert!( + matches!(&isolated, Isolation::Item(ordinals) if *ordinals == vec![1, 2]), + "an interaction with no single culprit must be an item isolation, not a blamed one" + ); + + // Convergence withdraws every isolated ordinal but abandons only an `Item` one, and abandoning + // is what makes `settle` record it as never built rather than as compiler-rejected. + let ordinals = match &isolated { + Isolation::Blamed(ordinals) | Isolation::Item(ordinals) => ordinals.clone(), + }; + for ordinal in &ordinals { + let _ = converger.withdrawn.insert(*ordinal); + } + if let Isolation::Item(ordinals) = &isolated { + converger.abandoned.extend(ordinals.iter().copied()); + } + + converger.settle(&mut plan); + + assert_eq!( + plan.mutants[0].outcome, + Outcome::NotBuilt, + "an item-only interaction is never judged" + ); + assert_eq!( + plan.mutants[1].outcome, + Outcome::NotBuilt, + "an item-only interaction is never judged" + ); +} + +/// A plan holding one mutant of the trivial fixture's `const`, which cannot be guarded and so +/// costs exactly one rollback round to withdraw. +fn unguardable_plan(work: &Workspace, ordinal: u32) -> Plan { + let text = "pub const A: i32 = 1;\n"; + let start = text.find('1').expect("the literal is in the fixture"); + + // Each round instruments the tree from `absolute`, so the pristine copy has to sit outside + // the tree the rounds rewrite: reading it back out of the scratch tree would have a round + // treat the previous round's guards as the original source, and nothing would converge. + let pristine = work.root.parent().expect("the tree has a parent").join("pristine"); + + fs::create_dir_all(pristine.as_std_path()).expect("pristine"); + fs::write(pristine.join("lib.rs").as_std_path(), text).expect("pristine source"); + + let mut plan = empty_plan(work); + + plan.files.push(TargetFile { + path: Utf8PathBuf::from("src/lib.rs"), + absolute: pristine.join("lib.rs"), + package: "trivial".to_owned(), + }); + plan.mutants.push(Mutant { + ordinal, + span: start..start + 1, + replacement: "2".to_owned().into(), + ..mutant() + }); + + plan +} + +/// `--rollback-rounds` caps the rounds one build may spend converging, and a run performs +/// several builds. Charging them all against one counter means a stage that converged normally +/// can leave the build that actually decides the run with no budget at all — which surfaces as +/// a rollback-limit failure on a tree that was converging perfectly well. +#[test] +fn each_build_gets_the_whole_round_budget_rather_than_what_earlier_builds_left() { + let (_dir, work) = trivial_workspace("build-budget-per-build-"); + let limits = BuildLimits { + timeout: None, + multiplier: None, + rollback_rounds: 2, + }; + let mut converger = Converger::default(); + let first = unguardable_plan(&work, 1); + + let first_built = converger + .converge( + &work, + &first, + None, + &["build", "--keep-going"], + limits, + &mut crate::testing::Recorder::default(), + ) + .expect("one withdrawal is enough to make the tree compile"); + + assert!(matches!(first_built, Convergence::Built(_)), "the tree must converge"); + + assert_eq!(converger.rounds, 2, "one failed round and one that succeeded"); + + // A second build, with a mutant the first one never saw, costs a round of its own. The + // whole budget has to be available to it. + let second = unguardable_plan(&work, 2); + + let second_built = converger + .converge( + &work, + &second, + None, + &["build", "--keep-going"], + limits, + &mut crate::testing::Recorder::default(), + ) + .expect("the second build must get a budget of its own"); + + assert!(matches!(second_built, Convergence::Built(_)), "the tree must converge"); + + assert_eq!(converger.rounds, 2, "the counter is per build, not per run"); + assert_eq!(converger.total_rounds, 4, "the run still reports what it spent in total"); + + // The withdrawal set is the one thing that is deliberately shared: a mutant already known + // not to compile stays withdrawn for the rest of the run. + assert_eq!(converger.withdrawn(), 2, "withdrawals carry across builds"); +} + +/// The limit error reads a series of withdrawal counts and gives falling-or-flat advice from +/// it, so the series has to describe the build that just failed. Counts left over from earlier +/// builds would have it describe work the reader is not being told about. +#[test] +fn the_rollback_limit_error_describes_only_the_build_that_hit_it() { + let (_dir, work) = trivial_workspace("build-limit-series-"); + let mut converger = Converger { + rounds: 9, + total_rounds: 9, + per_round: vec![41], + ..Converger::default() + }; + let limits = BuildLimits { + timeout: None, + multiplier: None, + rollback_rounds: 1, + }; + let plan = unguardable_plan(&work, 1); + + let error = stuck_reason( + converger + .converge( + &work, + &plan, + None, + &["build", "--keep-going"], + limits, + &mut crate::testing::Recorder::default(), + ) + .expect("a budget of one round cannot survive a failing round"), + ); + + assert!(error.contains("1 of the 1 rollback rounds"), "{error}"); + assert!(!error.contains("41"), "{error}"); + assert!(!error.contains("10 of"), "{error}"); +} + +/// A cargo that cannot even be spawned fails inside `run_cargo`, and that failure has to climb +/// back out through `converge` rather than being swallowed as an ordinary build failure to +/// withdraw mutants over: retrying a build that can never start would spin forever, and the +/// person running it deserves to be told cargo itself could not be launched, not that every +/// mutant in the tree is somehow unviable. +#[test] +fn converging_when_the_tree_is_missing_reports_the_failure_rather_than_looping() { + let work = Workspace::adopt( + Utf8PathBuf::from("/gamma/definitely/not/a/directory"), + Utf8PathBuf::from("/gamma/definitely/not/a/directory/target"), + ); + let plan = empty_plan(&work); + + let error = Converger::default() + .converge( + &work, + &plan, + None, + &["build", "--keep-going"], + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect_err("cargo cannot be spawned in a directory that does not exist"); + + assert!(error.to_string().contains("disappeared"), "{error}"); +} + +/// A mutant whose span no longer fits the file it names is an internal error whether it is +/// discovered by calling the instrumenter directly or, as here, by driving a whole round of +/// `converge`: the round has to stop and report it rather than treating the file as though it +/// simply failed to compile, or the bug would surface as a baffling rollback instead of the +/// named internal error it actually is. +#[test] +fn converging_a_mutant_whose_span_no_longer_fits_the_file_reports_the_internal_error() { + let (_dir, work) = trivial_workspace("build-converge-missing-guard-"); + let text = fs::read_to_string(work.root.join("src/lib.rs").as_std_path()).expect("lib"); + + let mutant = Mutant { + span: text.len() + 10..text.len() + 11, + ..mutant() + }; + let mut plan = empty_plan(&work); + plan.files.push(target_file(&work.root, "src/lib.rs")); + plan.mutants.push(mutant); + + let error = Converger::default() + .converge( + &work, + &plan, + None, + &["build", "--keep-going"], + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect_err("the span is out of range"); + + assert!(error.to_string().contains("no guard was emitted"), "{error}"); +} + +/// `stage` builds only what one stage needs, and a failure there has to surface exactly like +/// any other build failure would: this function had never been called by a unit test at all, +/// so nothing stood between it compiling and it actually reporting the right thing when the +/// tree it is given does not build. +#[test] +fn a_stage_that_fails_to_compile_reports_the_failure() { + let (_dir, work) = trivial_workspace("build-stage-fails-"); + + fs::write( + work.root.join("src/lib.rs").as_std_path(), + "pub const A: i32 = \"not an integer\";\n", + ) + .expect("lib"); + + let mut plan = empty_plan(&work); + let abandoned = Converger::default() + .stage( + &work, + &mut plan, + &["trivial".to_owned()], + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the stage ran, however badly") + .expect("the package does not compile"); + + assert!(abandoned.reason.contains("could not be attributed"), "{}", abandoned.reason); +} + +/// Narrowing to a package is abandoned when even the widened, whole-workspace build fails: the +/// narrowing was never the problem, the code itself does not compile, and reporting the +/// original narrow-build error rather than the widened one keeps the message about the +/// selection that was actually asked for instead of an unrelated whole-workspace retry. +#[test] +fn widening_to_the_whole_workspace_that_also_fails_reports_the_original_narrow_error() { + let (_dir, work) = trivial_workspace("build-widen-fails-"); + + fs::write( + work.root.join("src/lib.rs").as_std_path(), + "pub const A: i32 = \"not an integer\";\n", + ) + .expect("lib"); + + let mut plan = empty_plan(&work); + let select = vec!["trivial".to_owned()]; + + let stuck = Converger::default() + .finish( + &work, + &mut plan, + Some(&select), + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build ran, however badly") + .stuck + .expect("neither the narrowed nor the widened build compiles"); + + assert!(stuck.reason.contains("could not be attributed"), "{}", stuck.reason); +} + +/// A mutant whose span no longer fits the file it names is dropped by the instrumenter rather +/// than spliced at a wrong offset, but it is still "live" as far as withdrawal bookkeeping is +/// concerned. If that mismatch were left unchecked, the mutant would run with nothing in the +/// tree to make it behave differently and be recorded as a survivor — a wrong verdict rather +/// than a missing one — so this invariant is caught and reported as the internal error it is. +#[test] +fn a_live_mutant_whose_span_no_longer_fits_the_file_is_an_internal_error_rather_than_a_silent_survivor() { + let (_dir, work) = trivial_workspace("build-missing-guard-"); + let text = fs::read_to_string(work.root.join("src/lib.rs").as_std_path()).expect("lib"); + + let mutant = Mutant { + span: text.len() + 10..text.len() + 11, + ..mutant() + }; + let mut plan = empty_plan(&work); + plan.files.push(target_file(&work.root, "src/lib.rs")); + plan.mutants.push(mutant); + + let error = Splices::default() + .instrument(&work, &plan, &HashSet::default()) + .expect_err("the span is out of range"); + + assert!(error.to_string().contains("no guard was emitted"), "{error}"); +} + +/// Every file the survey found is read while instrumenting, whether or not it has a live +/// mutant of its own, because a withdrawn file still has to be rewritten back to its original +/// text. A file that has since become unreadable — moved, deleted, permissions revoked between +/// the survey and the build — must stop the round with a named error rather than silently +/// dropping the file from the instrumented tree, which would leave stale instrumented text +/// behind for a mutant nothing further ever rewrites. +#[test] +fn a_survey_file_that_can_no_longer_be_read_stops_instrumentation_with_a_named_error() { + let (_dir, work) = trivial_workspace("build-unreadable-file-"); + let mut plan = empty_plan(&work); + + plan.files.push(target_file(&work.root, "src/gone.rs")); + + let error = Splices::default() + .instrument(&work, &plan, &HashSet::default()) + .expect_err("the file was never created"); + + assert!(error.to_string().contains("could not read"), "{error}"); +} + +/// Two live mutants whose spans overlap without one nesting inside the other cannot both be +/// spliced into the same text, and the instrumenter reports that ambiguity as an error rather +/// than guessing an order. That failure has to be visible from `instrument_tree`'s own call +/// site, not merely from the schema module's tests, because a caller reading a stack trace here +/// needs to see the tree walk that actually produced it. +#[test] +fn overlapping_mutant_spans_are_reported_as_an_ambiguous_splice() { + let (_dir, work) = trivial_workspace("build-overlap-"); + let text = "fn f(a: i32, b: i32, c: i32) -> i32 { a + b + c }"; + + fs::write(work.root.join("src/lib.rs").as_std_path(), text).expect("lib"); + + let left = text.find("a + b").expect("the fixture contains this text"); + let right = text.find("b + c").expect("the fixture contains this text"); + + let mut plan = empty_plan(&work); + plan.files.push(target_file(&work.root, "src/lib.rs")); + plan.mutants.push(Mutant { + ordinal: 1, + span: left..left + "a + b".len(), + ..mutant() + }); + plan.mutants.push(Mutant { + ordinal: 2, + span: right..right + "b + c".len(), + ..mutant() + }); + + let error = Splices::default() + .instrument(&work, &plan, &HashSet::default()) + .expect_err("the spans overlap without nesting"); + + assert!(error.to_string().contains("overlap"), "{error}"); +} + +/// Staged discovery appends mutants in package order, then the baseline sorts them into report +/// order. The splice cache stores vector positions, so the sort has to invalidate those positions +/// before a changed withdrawal set makes any file dirty again. +#[test] +fn sorting_a_staged_plan_reindexes_mutants_before_the_baseline_splice() { + let (_dir, work) = trivial_workspace("build-sort-reindexes-"); + let a_text = "pub fn a() -> bool { true }\n"; + let b_text = "pub fn b() -> bool { true }\n"; + + fs::write(work.root.join("src/a.rs").as_std_path(), a_text).expect("a"); + fs::write(work.root.join("src/b.rs").as_std_path(), b_text).expect("b"); + + let mut plan = empty_plan(&work); + plan.files + .extend([target_file(&work.root, "src/a.rs"), target_file(&work.root, "src/b.rs")]); + + let mut b = mutant(); + b.file = Utf8PathBuf::from("src/b.rs").into(); + b.span = b_text.find("true").expect("true")..b_text.find("true").expect("true") + 4; + plan.mutants.push(b); + + let mut splices = Splices::default(); + let _guards = splices + .instrument(&work, &plan, &HashSet::default()) + .expect("the first stage is instrumented"); + + let mut a = mutant(); + a.ordinal = 2; + a.file = Utf8PathBuf::from("src/a.rs").into(); + a.span = a_text.find("true").expect("true")..a_text.find("true").expect("true") + 4; + plan.mutants.push(a); + + let _guards = splices + .instrument(&work, &plan, &HashSet::default()) + .expect("the second stage is instrumented"); + + plan.sort(); + splices.plan_reordered(); + + let guards = splices + .instrument(&work, &plan, &HashSet::from_iter([1])) + .expect("the baseline withdrawal reindexes the sorted plan"); + + assert_eq!(guards.get(&2).map(|(path, _guard)| path.as_str()), Some("src/a.rs")); + assert!(!guards.contains_key(&1), "the withdrawn b mutant has no guard"); +} + +/// A mutant's file has to already exist in the scratch tree for its instrumented text to be +/// written back over it: the copy step is what puts it there, so a `TargetFile` naming a path +/// the copy never created is a bug in the survey rather than something worth silently creating +/// a brand new file for, which is why the write is refused with the same error `overwrite` +/// reports for that case anywhere else it is called from. +#[test] +fn a_mutants_file_the_copy_never_created_reports_the_write_failure() { + let (_dir, work) = trivial_workspace("build-uncopied-destination-"); + let mut plan = empty_plan(&work); + + // The source is genuinely readable, so the read at the top of the loop succeeds; only the + // later write, against a destination the copy never produced, is meant to fail here. + plan.files.push(TargetFile { + path: Utf8PathBuf::from("src/never_copied.rs"), + absolute: work.root.join("src/lib.rs"), + package: "trivial".to_owned(), + }); + + let error = Splices::default() + .instrument(&work, &plan, &HashSet::default()) + .expect_err("the destination was never copied"); + + assert!(error.to_string().contains("which the copy did not create"), "{error}"); +} + +/// A plan with no mutants and no files, rooted in the given workspace. +fn target_file(root: &Utf8Path, path: &str) -> TargetFile { + TargetFile { + path: Utf8PathBuf::from(path), + absolute: root.join(path), + package: "trivial".to_owned(), + } +} + +fn empty_plan(work: &Workspace) -> Plan { + Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: work.root.clone(), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + } +} + +/// A build that outstays its budget is killed rather than waited out. +#[test] +fn a_build_that_outstays_its_budget_is_stopped() { + let (_dir, work) = trivial_workspace("build-budget-"); + + // Zero budget: the deadline has passed before the first poll, so the child is killed on the + // very first pass through the wait loop. + let outcome = compile( + &work, + &["check".to_owned()], + Some(Duration::ZERO), + &mut crate::testing::Recorder::default(), + ) + .expect("spawn"); + + assert!(outcome.is_none(), "a build past its budget should report no output"); +} + +/// A build stopped by its budget reports no output at all, so the caller can say so. +#[test] +fn a_build_stopped_by_its_budget_reports_no_stdout() { + let (_dir, work) = trivial_workspace("build-nostdout-"); + let limits = BuildLimits { + timeout: Some(Duration::ZERO), + multiplier: None, + rollback_rounds: 0, + }; + + let compiled = run_cargo( + &work, + &empty_plan(&work), + &["check"], + None, + limits, + None, + &mut crate::testing::Recorder::default(), + ) + .expect("spawn"); + + assert!(!compiled.succeeded); + assert!(compiled.stdout.is_none()); +} + +/// And converging on such a build stops with an error naming the budget rather than looping. +#[test] +fn converging_on_a_build_that_never_finishes_stops_with_the_budget() { + let (_dir, work) = trivial_workspace("build-converge-budget-"); + let plan = Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: work.root.clone(), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + }; + let limits = BuildLimits { + timeout: Some(Duration::ZERO), + multiplier: None, + rollback_rounds: 0, + }; + + let error = Converger::default() + .converge(&work, &plan, None, &["check"], limits, &mut crate::testing::Recorder::default()) + .expect_err("the build never finishes"); + + assert!(error.to_string().contains("was still running"), "{error}"); + assert!(error.to_string().contains("--build-timeout"), "{error}"); +} + +/// A build inside its budget is collected through the same polling wait. +#[test] +fn a_build_inside_its_budget_is_collected() { + let (_dir, work) = trivial_workspace("build-collected-"); + + let outcome = compile( + &work, + &["--version".to_owned()], + Some(Duration::from_mins(2)), + &mut crate::testing::Recorder::default(), + ) + .expect("spawn") + .expect("cargo should finish well inside two minutes"); + + assert!(outcome.status.success(), "{outcome:?}"); +} + +/// A build stopped for outstaying its budget takes everything it started with it. +/// +/// Cargo is the root of a tree, not a process: `rustc`, build scripts, and whatever those start. +/// A kill aimed at cargo alone leaves that tree compiling against the same scratch directory the +/// run is about to reuse, on the cores the next attempt needs — and holding the write ends of the +/// pipes it inherited, which is what turns a build the budget cut off into a collection that never +/// ends. +/// +/// The descendant here is the shape that survives: spawned by the build, holding its pipes, and +/// due to write a file well after the budget runs out. Reaching that write is what a surviving +/// subtree looks like from outside. +#[test] +fn a_build_stopped_by_its_budget_takes_its_descendants_with_it() { + crate::testing::within(crate::testing::WATCHDOG, "a build with a descendant", || { + let started = crate::testing::workdir("build-descendant-"); + let root = Utf8PathBuf::from_path_buf(started.path().to_path_buf()).expect("a UTF-8 scratch path"); + let (running, survived) = (root.join("running"), root.join("survived")); + + let mut command = Command::new(crate::testing::helper_binary_path().as_std_path()); + + let _configured = command + .arg(crate::testing::directive(format_args!( + "spawn:touch:{running}|sleep:3000|touch:{survived}" + ))) + .arg(crate::testing::directive("sleep:30000")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let work = Workspace::adopt(root.clone(), root.join("target")); + + let outcome = supervise( + &mut command, + &work, + Some(Duration::from_millis(750)), + &mut crate::testing::Recorder::default(), + ) + .expect("the build was supervised"); + + assert!(outcome.is_none(), "a build past its budget reports no output"); + assert!( + running.as_std_path().exists(), + "the descendant never started, so this test proves nothing about killing it" + ); + + // Past the descendant's sleep, so anything still alive has had its chance to write. + thread::sleep(Duration::from_millis(3500)); + + assert!( + !survived.as_std_path().exists(), + "the build's descendant outlived the kill that was aimed at the build" + ); + }); +} + +/// A build that finished is collected without waiting for whatever it left running. +/// +/// Cargo exiting says nothing about what it started: a build script that leaves a daemon behind +/// leaves it holding the two pipes this run is reading, and end of file arrives when the *last* +/// holder lets go. Joining the readers first would wait for the survivor rather than for the +/// build, which is a hang with a build that succeeded sitting in it. +/// +/// That the output arrives at all is the proof: a pipe still held by a live descendant cannot +/// reach end of file, so a reader that finished is one whose survivors are gone. +#[test] +fn a_finished_build_is_collected_without_waiting_for_its_survivors() { + crate::testing::within(crate::testing::WATCHDOG, "a build with a survivor", || { + let started = crate::testing::workdir("build-survivor-"); + let root = Utf8PathBuf::from_path_buf(started.path().to_path_buf()).expect("a UTF-8 scratch path"); + let running = root.join("running"); + + let mut command = Command::new(crate::testing::helper_binary_path().as_std_path()); + + let _configured = command + .arg(crate::testing::directive(format_args!("spawn:touch:{running}|sleep:20000"))) + .arg(crate::testing::directive(format_args!("wait-file:{running}|5000"))) + .arg(crate::testing::directive("print:the build said this")) + .arg(crate::testing::directive("exit:0")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let work = Workspace::adopt(root.clone(), root.join("target")); + let began = Instant::now(); + + let output = supervise( + &mut command, + &work, + Some(Duration::from_secs(45)), + &mut crate::testing::Recorder::default(), + ) + .expect("the build was supervised") + .expect("the build finished well inside its budget"); + + assert!(output.status.success(), "{output:?}"); + assert!( + running.as_std_path().exists(), + "the survivor never started, so this test proves nothing about outliving it" + ); + assert!( + began.elapsed() < Duration::from_secs(10), + "collecting the build waited for the survivor holding its pipes" + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("the build said this"), + "what the build itself said has to survive the sweep: {output:?}" + ); + }); +} + +/// A cargo that cannot be spawned at all names the tree it was to run in. +#[test] +fn a_missing_scratch_tree_is_named_rather_than_blamed_on_cargo() { + let work = Workspace::adopt( + Utf8PathBuf::from("/gamma/definitely/not/a/directory"), + Utf8PathBuf::from("/gamma/definitely/not/a/directory/target"), + ); + + let error = compile(&work, &["--version".to_owned()], None, &mut crate::testing::Recorder::default()).expect_err("no such directory"); + + assert!(error.to_string().contains("disappeared"), "{error}"); +} + +#[test] +fn a_cargo_that_cannot_be_found_names_the_program_and_where_it_came_from() { + let root = tempfile::TempDir::new().expect("temp"); + let root = Utf8PathBuf::from_path_buf(root.path().to_path_buf()).expect("utf8"); + let work = Workspace::adopt(root.clone(), root.join("target")); + + let cause = io::Error::from(io::ErrorKind::NotFound); + let error = spawn_failure("gamma-no-such-cargo-binary", &work, cause); + + assert!(error.to_string().contains("gamma-no-such-cargo-binary"), "{error}"); + assert!(error.to_string().contains("CARGO"), "{error}"); +} + +fn at(line: u32, column: u32) -> Position { + Position { line, column } +} + +fn guard(site: Range, mutated: Option>) -> Guard { + Guard { site, mutated } +} + +fn span(file: &str, line_start: u32, column_start: u32, line_end: u32, column_end: u32, primary: bool) -> Value { + serde_json::json!({ + "file_name": file, + "line_start": line_start, + "column_start": column_start, + "line_end": line_end, + "column_end": column_end, + "is_primary": primary, + }) +} + +fn compiler_message(spans: &[Value]) -> String { + serde_json::json!({ + "reason": "compiler-message", + "message": { + "level": "error", + "rendered": "error: boom\n", + "spans": spans, + }, + }) + .to_string() +} + +/// A diagnostic that names a code and keeps half of what it knows in its notes, which is the +/// shape every borrow-checker error has. +fn coded_message(code: &str, primary: &[Value], notes: &[Value]) -> String { + serde_json::json!({ + "reason": "compiler-message", + "message": { + "level": "error", + "code": { "code": code }, + "rendered": "error: boom\n", + "spans": primary, + "children": [{ "level": "note", "spans": notes }], + }, + }) + .to_string() +} + +fn mutant() -> Mutant { + Mutant { + id: "deadbeefcafe".to_owned().into(), + ordinal: 1, + file: (Utf8PathBuf::from("src/lib.rs")).into(), + package: ("pkg".to_owned()).into(), + span: 0..1, + line: 7, + end_line: 7, + column: 3, + mutator: ("lit.true_to_false".to_owned()).into(), + item_path: ("pkg::f".to_owned()).into(), + occurrence: 0, + replacement_index: 0, + original: "true".to_owned().into(), + replacement: "false".to_owned().into(), + shape: Shape::Expr, + outcome: Outcome::Pending, + suppression: None, + expectation: None, + test_timeout_multiplier: None, + elapsed_ms: 0, + killed_by: None, + note: None, + } +} + +/// The failure this tier exists for, taken from a real tree: a deleted `continue` makes a path +/// that could not be reached statically reachable, so a value moved earlier is now seen to be +/// used again — and rustc reports that at the use, at the move and at the reinitialization, +/// none of which is where the guard sits. +#[test] +fn a_move_error_is_blamed_on_the_deletion_that_changed_which_paths_exist() { + let mut guards = Guards::default(); + + // The deleted `continue`, at line 396: inside the region the diagnostic talks about, but + // named by none of its spans. + let _ = guards.insert(7, (Utf8PathBuf::from("src/codegen.rs"), guard(at(396, 9)..at(396, 17), None))); + + // A substitution in the same region. It cannot have changed reachability, so it is not the + // one to withdraw while a deletion is available. + let _ = guards.insert( + 8, + ( + Utf8PathBuf::from("src/codegen.rs"), + guard(at(400, 5)..at(400, 9), Some(at(400, 5)..at(400, 9))), + ), + ); + + let stdout = coded_message( + "E0382", + &[span("src/codegen.rs", 432, 9, 432, 13, true)], + &[ + span("src/codegen.rs", 372, 5, 372, 20, false), + span("src/codegen.rs", 383, 9, 383, 13, false), + ], + ); + + let blamed = ordinals_blamed(&stdout, Utf8Path::new(""), &guards); + + assert_eq!(blamed, HashSet::from_iter([7]), "the deletion is the only reachability change"); +} + +/// A run can report how many mutants would not compile; without the code it cannot say whether +/// that number is a mutator emitting ill-typed code or the borrow checker objecting to the +/// schema, and those want opposite remedies. +#[test] +fn a_withdrawal_remembers_the_error_code_that_caused_it() { + let mut guards = Guards::default(); + + let _ = guards.insert( + 7, + ( + Utf8PathBuf::from("src/lib.rs"), + guard(at(10, 5)..at(10, 9), Some(at(10, 5)..at(10, 9))), + ), + ); + + let stdout = coded_message("E0308", &[span("src/lib.rs", 10, 5, 10, 9, true)], &[]); + + assert_eq!( + blame(&stdout, Utf8Path::new(""), &guards).get(&7).map(String::as_str), + Some("E0308") + ); +} + +#[test] +fn a_secondary_span_intersecting_generated_text_blames_that_mutant() { + let mut guards = Guards::default(); + let _ = guards.insert( + 7, + ( + Utf8PathBuf::from("src/lib.rs"), + guard(at(10, 5)..at(10, 45), Some(at(10, 20)..at(10, 25))), + ), + ); + + let stdout = compiler_message(&[span("src/lib.rs", 12, 5, 12, 20, true), span("src/lib.rs", 10, 5, 10, 40, false)]); + + assert_eq!(ordinals_blamed(&stdout, Utf8Path::new(""), &guards), HashSet::from_iter([7])); +} + +#[test] +fn a_secondary_span_on_innocent_original_text_blames_nothing() { + let mut guards = Guards::default(); + let _ = guards.insert( + 7, + ( + Utf8PathBuf::from("src/lib.rs"), + guard(at(10, 5)..at(10, 45), Some(at(10, 20)..at(10, 25))), + ), + ); + + let stdout = compiler_message(&[span("src/lib.rs", 12, 5, 12, 20, true), span("src/lib.rs", 10, 30, 10, 40, false)]); + + assert!(ordinals_blamed(&stdout, Utf8Path::new(""), &guards).is_empty()); +} + +/// One unviable mutant can draw a four-figure count of follow-on diagnostics, so a census that +/// tallies rows rather than distinct mutants overstates its answer by an order of magnitude — +/// and the pair that leads the list is the one a reader will act on. +#[test] +fn the_census_counts_mutants_rather_than_diagnostics_and_leads_with_the_densest_pair() { + let mut converger = Converger::default(); + let plan = plan_of(&[(1, "lit.true_to_false"), (2, "lit.true_to_false"), (3, "expr.delete")]); + + let _ = converger.census.insert(1, "E0308".to_owned()); + let _ = converger.census.insert(2, "E0308".to_owned()); + let _ = converger.census.insert(3, "E0382".to_owned()); + + // A second sighting of a mutant already in the census is what a follow-on diagnostic looks + // like, and must not count twice. + let _ = converger.census.insert(1, "E0308".to_owned()); + + assert_eq!( + converger.tally(&plan), + vec![ + Withdrawal { + code: "E0308".to_owned(), + mutator: "lit.true_to_false".to_owned(), + mutants: 2, + }, + Withdrawal { + code: "E0382".to_owned(), + mutator: "expr.delete".to_owned(), + mutants: 1, + }, + ] + ); +} + +/// A plan with one mutant per ordinal and mutator, for the census. +fn plan_of(entries: &[(u32, &str)]) -> Plan { + Plan { + skipped: Vec::new(), + digests: HashMap::default(), + root: Utf8PathBuf::new(), + files: Vec::new(), + mutants: entries + .iter() + .map(|(ordinal, mutator)| Mutant { + ordinal: *ordinal, + mutator: ((*mutator).to_owned()).into(), + ..mutant() + }) + .collect(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: HashMap::default(), + specs: HashMap::default(), + } +} + +/// The gate is the whole safety of the tier. An error whose position means something must not +/// reach it, or a tree that simply does not compile would have innocent mutants withdrawn from +/// it one round at a time. +#[test] +fn an_error_that_is_not_flow_sensitive_is_never_blamed_by_region() { + let mut guards = Guards::default(); + + let _ = guards.insert(7, (Utf8PathBuf::from("src/codegen.rs"), guard(at(396, 9)..at(396, 17), None))); + + // An unresolved import is positional, and it is exactly what a tree with a broken feature + // selection reports. Nothing here is a mutant's doing. + let stdout = coded_message("E0432", &[span("src/codegen.rs", 432, 9, 432, 13, true)], &[]); + + assert!(ordinals_blamed(&stdout, Utf8Path::new(""), &guards).is_empty()); + + // Neither is an error with no code at all, which carries nothing to gate on. + let uncoded = compiler_message(&[span("src/codegen.rs", 432, 9, 432, 13, true)]); + + assert!(ordinals_blamed(&uncoded, Utf8Path::new(""), &guards).is_empty()); +} + +/// With no deletion in the region there is nothing better to go on, and losing a few mutants +/// that would have compiled is a far smaller loss than losing the run. +#[test] +fn a_region_with_no_deletion_falls_back_to_every_guard_in_it() { + let mut guards = Guards::default(); + + let _ = guards.insert( + 3, + ( + Utf8PathBuf::from("src/codegen.rs"), + guard(at(400, 5)..at(400, 9), Some(at(400, 5)..at(400, 9))), + ), + ); + + // Outside the region the diagnostic spans, so it is not a candidate however blunt the + // fallback is. + let _ = guards.insert( + 4, + ( + Utf8PathBuf::from("src/codegen.rs"), + guard(at(900, 5)..at(900, 9), Some(at(900, 5)..at(900, 9))), + ), + ); + + let stdout = coded_message( + "E0499", + &[span("src/codegen.rs", 432, 9, 432, 13, true)], + &[span("src/codegen.rs", 383, 9, 383, 13, false)], + ); + + assert_eq!(ordinals_blamed(&stdout, Utf8Path::new(""), &guards), HashSet::from_iter([3])); +} + +/// The positional tiers are still the better answer when they have one, since they name a +/// single mutant rather than a region's worth of them. +#[test] +fn a_diagnostic_that_lands_on_a_guard_is_still_blamed_on_that_guard_alone() { + let mut guards = Guards::default(); + + let _ = guards.insert( + 5, + ( + Utf8PathBuf::from("src/codegen.rs"), + guard(at(432, 9)..at(432, 13), Some(at(432, 9)..at(432, 13))), + ), + ); + + let _ = guards.insert(6, (Utf8PathBuf::from("src/codegen.rs"), guard(at(396, 9)..at(396, 17), None))); + + let stdout = coded_message( + "E0382", + &[span("src/codegen.rs", 432, 9, 432, 13, true)], + &[span("src/codegen.rs", 383, 9, 383, 13, false)], + ); + + assert_eq!(ordinals_blamed(&stdout, Utf8Path::new(""), &guards), HashSet::from_iter([5])); +} + +#[test] +fn diagnostics_are_read_from_the_json_stream() { + // Diagnostics arrive on stdout as JSON; stderr holds only a summary, so a failure report + // built from stderr would say nothing about what actually went wrong. + let stdout = concat!( + r#"{"reason":"compiler-message","message":{"level":"error","rendered":"error[E0308]: boom"}}"#, + "\n", + r#"{"reason":"compiler-message","message":{"level":"warning","rendered":"just a warning"}}"#, + "\n", + r#"{"reason":"compiler-message"}"#, + "\n", + r#"{"reason":"compiler-message","message":{}}"#, + "\n", + r#"{"reason":"compiler-artifact","profile":{"test":true},"executable":"/tmp/x"}"#, + "\n", + // Cargo interleaves its own plain-text chatter with the JSON stream. + "warning: unused manifest key\n", + ); + let rendered = diagnostics(stdout).into_iter().map(|found| found.rendered).collect::(); + + assert!(rendered.contains("E0308")); + assert!(!rendered.contains("just a warning")); + assert!(!rendered.contains("unused manifest key")); +} + +/// An error-level compiler message with no `rendered` field is not something real `rustc` +/// emits, but a future cargo could add an error kind that omits it, and treating a missing +/// field as an empty string rather than skipping it silently would make the caller's report +/// look like `rustc` said nothing at all, which is worse than the diagnostic being absent. +#[test] +fn an_error_message_with_no_rendered_text_contributes_nothing_rather_than_panicking() { + let stdout = r#"{"reason":"compiler-message","message":{"level":"error"}}"#; + + assert!(diagnostics(stdout).is_empty()); +} + +/// The report quotes whole diagnostics from the front. Keeping the *last* N lines instead would +/// open the report partway through a snippet, with no error line above it to say what the +/// underlines were pointing at. +#[test] +fn a_long_diagnostic_list_keeps_the_first_errors_whole_and_counts_the_rest() { + let rendered: Vec = (0..8) + .map(|index| reported(None, &format!("error[E{index:04}]: something\n --> src/lib.rs:{index}\n"))) + .collect(); + + let shown = leading(&rendered, 3); + + assert!(shown.starts_with("error[E0000]"), "{shown}"); + assert!(shown.contains("error[E0002]"), "{shown}"); + assert!(!shown.contains("error[E0003]"), "{shown}"); + assert!(shown.contains("and 5 further errors not shown"), "{shown}"); +} + +#[test] +fn a_diagnostic_list_within_the_limit_is_quoted_whole_and_counts_nothing() { + let rendered = vec![reported(None, "error[E0001]: something\n")]; + + assert_eq!(leading(&rendered, 3), "error[E0001]: something\n"); +} + +/// The errors quoted first are the ones in the code the caller chose to mutate. A reverse +/// dependency pulled in because its tests form part of the oracle can bury the caller's own +/// crate under errors that are consequences of it. +#[test] +fn errors_in_the_mutated_packages_are_quoted_before_anyone_elses() { + let mut found = vec![ + reported(Some("/w/other/Cargo.toml"), "error: in the reverse dependency\n"), + reported(Some("/w/mine/Cargo.toml"), "error: in my own crate\n"), + reported(None, "error: from nowhere in particular\n"), + ]; + + prioritize(&mut found, &core::iter::once("/w/mine/Cargo.toml".to_owned()).collect()); + + assert!(leading(&found, 1).starts_with("error: in my own crate\n"), "{}", leading(&found, 1)); + + // The rest keep the order the compiler emitted them in. + assert!( + leading(&found, 3).contains("reverse dependency\nerror: from nowhere"), + "{}", + leading(&found, 3) + ); +} + +fn reported(manifest: Option<&str>, rendered: &str) -> Diagnostic { + Diagnostic { + manifest: manifest.map(ToOwned::to_owned), + rendered: rendered.to_owned(), + } +} + +#[test] +fn diagnostic_spans_inside_mutated_text_name_that_mutant_exactly() { + let mut guards = Guards::default(); + let file = Utf8PathBuf::from("src/lib.rs"); + + let _old = guards.insert(1, (file.clone(), guard(at(10, 1)..at(20, 1), Some(at(12, 5)..at(12, 10))))); + let _old = guards.insert(2, (file, guard(at(10, 1)..at(20, 1), None))); + + // Exact mutated-branch hits are trusted before the broader guarded site, or shared sites + // would withdraw innocent neighbors. + let blamed = ordinals_blamed( + &compiler_message(&[span("src/lib.rs", 12, 6, 12, 8, true)]), + Utf8Path::new("/work"), + &guards, + ); + + assert_eq!(blamed, HashSet::from_iter([1])); +} + +#[test] +fn a_diagnostic_inside_a_guard_blames_the_innermost_enclosing_site() { + let mut guards = Guards::default(); + + for ordinal in [1, 2, 3] { + let site = if ordinal == 1 { at(1, 1)..at(50, 1) } else { at(10, 1)..at(15, 1) }; + let _old = guards.insert(ordinal, (Utf8PathBuf::from("src/lib.rs"), guard(site, None))); + } + + // When a deletion or type change breaks copied text inside the guard, the narrowest site + // is the least destructive rollback candidate, with ties kept together. + let blamed = ordinals_blamed( + &compiler_message(&[span("src/lib.rs", 12, 2, 12, 4, true)]), + Utf8Path::new("/work"), + &guards, + ); + + assert_eq!(blamed, HashSet::from_iter([2, 3])); +} + +#[test] +fn a_diagnostic_that_encloses_guards_keeps_only_the_smallest_reported_region() { + let mut guards = Guards::default(); + + let _old = guards.insert(1, (Utf8PathBuf::from("src/lib.rs"), guard(at(20, 1)..at(21, 1), None))); + let _old = guards.insert(2, (Utf8PathBuf::from("src/lib.rs"), guard(at(40, 1)..at(41, 1), None))); + + let stdout = compiler_message(&[span("src/lib.rs", 1, 1, 100, 1, false), span("src/lib.rs", 35, 1, 45, 1, false)]); + + // Borrow-checker errors can land on a construct containing a guard; the smallest such + // diagnostic is blamed so a broad fallback does not mask a narrower one. + let blamed = ordinals_blamed(&stdout, Utf8Path::new("/work"), &guards); + + assert_eq!(blamed, HashSet::from_iter([2])); +} + +/// When a wider enclosing diagnostic is processed after a narrower one has already claimed a +/// guard, the wider one must not overwrite that narrower attribution: preferring the smaller +/// region is the whole point of keeping "the smallest reported region", and a widen-then-keep +/// bug here would silently blame — and withdraw — the wrong, unrelated guard instead. +#[test] +fn a_wider_enclosing_diagnostic_processed_after_a_narrower_one_does_not_replace_its_attribution() { + let mut guards = Guards::default(); + + let _old = guards.insert(1, (Utf8PathBuf::from("src/lib.rs"), guard(at(10, 1)..at(11, 1), None))); + let _old = guards.insert(2, (Utf8PathBuf::from("src/lib.rs"), guard(at(60, 1)..at(61, 1), None))); + + // The narrow span (covering only guard 1) comes first in the message; the much wider span + // (covering both guards) comes second. Message spans are read in order, so this + // deterministically exercises the case where a later, wider candidate must be discarded + // rather than take over from an earlier, narrower one. + let stdout = compiler_message(&[span("src/lib.rs", 5, 1, 15, 1, false), span("src/lib.rs", 1, 1, 100, 1, false)]); + + let blamed = ordinals_blamed(&stdout, Utf8Path::new("/work"), &guards); + + assert_eq!(blamed, HashSet::from_iter([1])); +} + +#[test] +fn diagnostics_are_matched_by_suffix_when_cargo_spells_paths_differently() { + let mut guards = Guards::default(); + + let _old = guards.insert( + 1, + ( + Utf8PathBuf::from("crates/pkg/src/lib.rs"), + guard(at(5, 1)..at(6, 1), Some(at(5, 5)..at(5, 10))), + ), + ); + + // Cargo can report an absolute or otherwise differently-rooted path; suffix matching keeps + // those diagnostics attributable rather than losing the whole run. + let blamed = ordinals_blamed( + &compiler_message(&[span("/elsewhere/crates/pkg/src/lib.rs", 5, 6, 5, 8, true)]), + Utf8Path::new("/scratch/tree"), + &guards, + ); + + assert_eq!(blamed, HashSet::from_iter([1])); +} + +#[test] +fn non_error_messages_and_malformed_spans_are_ignored_for_ordinals_blamed() { + let mut guards = Guards::default(); + + let _old = guards.insert(1, (Utf8PathBuf::from("src/lib.rs"), guard(at(1, 1)..at(2, 1), None))); + let stdout = [ + "not json".to_owned(), + serde_json::json!({"reason": "compiler-artifact"}).to_string(), + serde_json::json!({"reason": "compiler-message", "message": {"level": "warning"}}).to_string(), + serde_json::json!({"reason": "compiler-message", "message": {"level": "error"}}).to_string(), + serde_json::json!({ + "reason": "compiler-message", + "message": {"level": "error", "spans": [{}, {"file_name": "src/lib.rs"}]}, + }) + .to_string(), + ] + .join("\n"); + + // Ignoring incomplete compiler output is safer than guessing and withdrawing unrelated + // mutants. + assert!(ordinals_blamed(&stdout, Utf8Path::new("/work"), &guards).is_empty()); +} + +/// A span with a line but no column is exactly as unusable as one missing everything: reading a +/// half-formed position and treating it as real would blame a guard at some arbitrary column, +/// which is worse than simply not attributing the diagnostic at all. +#[test] +fn a_span_with_a_line_but_no_column_is_not_read_as_a_position() { + let mut guards = Guards::default(); + + let _old = guards.insert(1, (Utf8PathBuf::from("src/lib.rs"), guard(at(1, 1)..at(2, 1), None))); + let span = serde_json::json!({ + "file_name": "src/lib.rs", + "line_start": 1, + "line_end": 1, + "column_end": 2, + "is_primary": true, + }); + let stdout = compiler_message(&[span]); + + assert!(ordinals_blamed(&stdout, Utf8Path::new("/work"), &guards).is_empty()); +} + +/// A span whose start is complete but whose end is missing a field is just as unreadable as one +/// missing its start: the diagnostic names no coherent region at all, so it must be ignored +/// rather than blamed against whatever guard happens to sit near its start. +#[test] +fn a_span_with_a_complete_start_but_no_end_is_not_read_as_a_position() { + let mut guards = Guards::default(); + + let _old = guards.insert(1, (Utf8PathBuf::from("src/lib.rs"), guard(at(1, 1)..at(2, 1), None))); + let span = serde_json::json!({ + "file_name": "src/lib.rs", + "line_start": 1, + "column_start": 1, + "line_end": 1, + "is_primary": true, + }); + let stdout = compiler_message(&[span]); + + assert!(ordinals_blamed(&stdout, Utf8Path::new("/work"), &guards).is_empty()); +} + +#[test] +fn build_errors_are_formatted_without_running_cargo() { + let timeout = Converger::build_timeout_error(Duration::from_secs(2)).to_string(); + let stdout = compiler_message(&[span("src/lib.rs", 1, 1, 1, 2, true)]); + let (_dir, mut work) = trivial_workspace("build-errors-"); + + work.leak = true; + + let unattributed = Converger::unattributed_build_error(&work, &stdout, "").to_string(); + let limited = Converger::rollback_limit_error(32, 32, &[9, 5, 2], &work, &stdout).to_string(); + let missing = Converger::missing_guard_error(&mutant()).to_string(); + + // These messages are the user's only explanation of build failures that happen before any + // test binary can run, so the pure formatting paths are kept under test. + assert!(timeout.contains("after 2s")); + assert!(unattributed.contains("could not be attributed")); + assert!(limited.contains("32 of the 32 rollback rounds"), "{limited}"); + assert!(limited.contains("16 blamed during this build"), "{limited}"); + assert!(limited.contains("9, 5, 2"), "{limited}"); + + // The series ends with the round the limit stopped, so a budget of one round still has a + // count to report and still gets the falling-or-flat advice. Reading the series without + // that round would leave this message empty and claim the last round found nothing. + let single = Converger::rollback_limit_error(1, 1, &[41], &work, &stdout).to_string(); + + assert!(single.contains("1 of the 1 rollback rounds"), "{single}"); + assert!(single.contains("41 blamed during this build"), "{single}"); + assert!(single.contains("blamed in the last rounds of this build: 41"), "{single}"); + assert!(single.contains("If those counts are falling"), "{single}"); + assert!(missing.contains("no guard was emitted")); + assert!(missing.contains("src/lib.rs:7")); + + // A leaked tree is still there to be read, so the message names it. + assert!(unattributed.contains(work.root.as_str()), "{unattributed}"); + + // One that was not leaked is gone by the time the message is read, and sending someone to + // a path that does not exist reads as a second bug on top of the one being reported. + work.leak = false; + + let swept = Converger::unattributed_build_error(&work, &stdout, "").to_string(); + + assert!(swept.contains("--leak-dirs"), "{swept}"); + assert!(!swept.contains(work.root.as_str()), "{swept}"); +} + +#[test] +fn a_build_that_reported_nothing_is_not_described_as_a_compile_failure() { + // Cargo rejects a bad invocation — an ambiguous `--package`, an unknown feature — before it + // compiles anything, so the failure arrives with an empty JSON stream. Calling that "does + // not compile" sends the reader hunting for a broken mutant that was never generated. + let (_dir, work) = trivial_workspace("silent-build-"); + let stderr = " Compiling tonic v0.14.0\n\ + error: failed to run custom build command for `codegen v0.1.0`\n\n\ + Caused by:\n \ + process didn't exit successfully: exit status: 101\n"; + let message = Converger::unattributed_build_error(&work, "", stderr).to_string(); + + assert!(message.contains("the compiler reported nothing"), "{message}"); + assert!(!message.contains("does not compile"), "{message}"); + + // The whole point: the cause is on stderr and nowhere else, so it has to be shown. + assert!(message.contains("failed to run custom build command"), "{message}"); + assert!(message.contains("Caused by"), "{message}"); + + // Cargo narrates its progress on the same stream, and a cold build narrates thousands of + // lines. Those must not bury the two that matter. + assert!(!message.contains("Compiling tonic"), "{message}"); +} + +#[test] +fn a_failure_with_nothing_on_either_stream_says_so_rather_than_showing_a_blank() { + // Printing an empty block reads as though the message itself is broken. + let (_dir, work) = trivial_workspace("silent-both-"); + let message = Converger::unattributed_build_error(&work, "", " Compiling x v0.1.0\n").to_string(); + + assert!(message.contains("cargo said nothing on stderr either"), "{message}"); +} + +#[test] +fn cargo_progress_is_told_apart_from_a_word_that_merely_starts_the_same_way() { + // `Compiling` is progress; `Compilation failed` is not, and dropping it would hide the + // only line that explains the failure. + let kept = complaints(" Compiling x v0.1.0\nCompilationfailed for a reason\n"); + + assert!(kept.contains("Compilationfailed"), "{kept}"); + assert!(!kept.contains("Compiling x"), "{kept}"); +} + +/// A progress verb is only progress when it is followed by a space or by nothing at all; a line +/// that merely shares the verb's letters as a prefix, with real text immediately butted up +/// against it, is not cargo narrating a step and has to be kept — dropping it on the strength of +/// the prefix alone would risk hiding a genuine complaint that happened to start the same way. +#[test] +fn a_word_sharing_a_progress_verbs_prefix_but_continuing_without_a_space_is_kept() { + let kept = complaints("Compilingx and y do not unify\n"); + + assert!(kept.contains("Compilingx and y do not unify"), "{kept}"); +} + +/// Cargo draws its progress bar with carriage returns and no newlines, so a whole build's worth +/// of redraws arrives as a single newline-terminated line. Splitting only on newlines would +/// hand someone whose build just failed one enormous bar with the error buried inside it. +#[test] +fn a_carriage_return_progress_bar_does_not_bury_the_error_it_was_drawn_around() { + let bar = " Building [==> ] 2/17: serde_core\r\ + \u{1b}[1;36m Building\u{1b}[0m [====> ] 4/17: quote, syn\r\ + \u{1b}[1;36m Compiling\u{1b}[0m [======> ] 9/17: trivial\r"; + let stderr = + format!(" Compiling serde v1.0.229\n{bar}\rerror: linking with `cc` failed: exit status: 1\r{bar}\n Building [=>] 1/2\r"); + + let kept = complaints(&stderr); + + assert_eq!(kept, "error: linking with `cc` failed: exit status: 1\n", "{kept}"); +} + +/// Cargo pads its own diagnostics with a blank leading line; keeping it would show the reader +/// an empty first line before the actual complaint, which reads as though the message itself +/// is truncated or broken. +#[test] +fn a_leading_blank_line_on_stderr_is_dropped_rather_than_kept() { + let kept = complaints("\nerror: something is wrong\n"); + + assert_eq!(kept, "error: something is wrong\n"); +} + +/// Dep-info from an earlier run with different features is the whole reason this is keyed off +/// the build's own artifact messages, so it has to be shown that the stale file is ignored. +#[test] +fn only_the_dep_info_belonging_to_this_build_is_read() { + let dir = crate::testing::workdir("dep-files-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "x: src/kept.rs\n").expect("mine"); + fs::write(deps.join("stale-bbbb.d").as_std_path(), "x: src/gone.rs\n").expect("stale"); + + let stdout = artifact_message(&deps.join("libmine-aaaa.rmeta")); + + let compiled = compiled_sources(&stdout, Utf8Path::new("/nowhere")).expect("read something"); + + assert!(compiled.contains(Utf8Path::new("src/kept.rs")), "{compiled:?}"); + assert!(!compiled.contains(Utf8Path::new("src/gone.rs")), "{compiled:?}"); +} + +/// A dependency path with an escaped space is one path, not two. +/// +/// `.d` files are makefile fragments, so a space inside a path is written `\ `. Read as a +/// separator it produces two fragments that match nothing the survey found, the source looks as +/// though the compiler never opened it, and every mutant in it is excused as unbuilt — silently, +/// because the set still agrees with the survey about every space-free file beside it. +#[test] +fn a_dependency_path_containing_an_escaped_space_is_read_as_one_path() { + let dir = crate::testing::workdir("dep-spaces-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "x.rmeta: src/lib.rs src/my\\ file.rs\n").expect("mine"); + + let stdout = artifact_message(&deps.join("libmine-aaaa.rmeta")); + + let compiled = compiled_sources(&stdout, Utf8Path::new("/nowhere")).expect("read something"); + + assert!(compiled.contains(Utf8Path::new("src/lib.rs")), "{compiled:?}"); + assert!(compiled.contains(Utf8Path::new("src/my file.rs")), "{compiled:?}"); + assert_eq!(compiled.len(), 2, "the escaped space split the path: {compiled:?}"); +} + +/// A backslash that is not escaping whitespace is a Windows separator and stays in the path. +/// +/// The two emitters escape the space and nothing else, so unescaping every backslash would turn +/// `C:\src\lib.rs` into `C:srclib.rs` and lose every dependency on that platform — a far larger +/// hole than the one closed by handling the space at all. +#[test] +fn a_backslash_that_is_not_an_escape_survives_into_the_path() { + let dir = crate::testing::workdir("dep-separators-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "x.rmeta: src\\my\\ file.rs\n").expect("mine"); + + let stdout = artifact_message(&deps.join("libmine-aaaa.rmeta")); + + let compiled = compiled_sources(&stdout, Utf8Path::new("/nowhere")).expect("read something"); + + // The separator is normalised the way every other dep-info path is, so what a Windows run + // compares against the survey is the same shape as what a Unix run does. + assert!(compiled.contains(Utf8Path::new("src/my file.rs")), "{compiled:?}"); +} + +/// A mutant in a file whose path contains a space is not excused as never built. +/// +/// This is the consequence the tokenising exists to prevent, and the reason it cannot be caught +/// downstream: the whole-set agreement check passes as soon as any one space-free source is +/// recognised, so a misread path is withdrawn one file at a time with nothing in the report saying +/// so. `NotBuilt` is excluded from the score, so those mutants leave the denominator silently. +#[test] +fn a_mutant_in_a_file_whose_path_contains_a_space_is_not_excused_as_unbuilt() { + let dir = crate::testing::workdir("withdraw-spaces-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "x.rmeta: src/lib.rs src/my\\ file.rs\n").expect("mine"); + + let stdout = artifact_message(&deps.join("libmine-aaaa.rmeta")); + let compiled = compiled_sources(&stdout, Utf8Path::new("/nowhere")).expect("read something"); + + let target = |path: &str| TargetFile { + path: Utf8PathBuf::from(path), + absolute: Utf8PathBuf::from("/nowhere").join(path), + package: "pkg".to_owned(), + }; + + let mutant_in = |path: &str, ordinal: u32| Mutant { + ordinal, + file: (Utf8PathBuf::from(path)).into(), + ..mutant() + }; + + let mut plan = plan_of(&[]); + + plan.files = vec![target("src/lib.rs"), target("src/my file.rs"), target("src/absent.rs")]; + plan.mutants = vec![ + mutant_in("src/lib.rs", 1), + mutant_in("src/my file.rs", 2), + mutant_in("src/absent.rs", 3), + ]; + + withdraw_uncompiled(&mut plan, &compiled); + + assert_eq!(plan.mutants[0].outcome, Outcome::Pending, "the plain file was withdrawn"); + assert_eq!(plan.mutants[1].outcome, Outcome::Pending, "the spaced file was withdrawn"); + + // The withdrawal still has to happen for a file the compiler genuinely never opened, or the + // test would pass just as well against a version that concluded nothing at all. + assert_eq!(plan.mutants[2].outcome, Outcome::NotBuilt, "an uncompiled file was kept"); +} + +/// A compiled set that names not one surveyed file makes `withdraw_uncompiled` conclude nothing. +/// +/// The whole-set agreement guard is what stands between a dep-info spelling regression and a +/// flattering perfect score: if `compiled_sources`/`normalize_separators`/escaping ever spelled +/// paths differently from the survey, every path would miss, every pending mutant would be excused +/// as `NotBuilt`, and the run would report approximately 100% with no sign anything went wrong. When the +/// compiled set is wholly disjoint from `plan.files`, the tool does not understand it, so it must +/// keep its hands off rather than withdraw the lot. Deleting the guard flips both mutants to +/// `NotBuilt`. +#[test] +fn a_compiled_set_disjoint_from_the_survey_withdraws_nothing() { + let target = |path: &str| TargetFile { + path: Utf8PathBuf::from(path), + absolute: Utf8PathBuf::from("/nowhere").join(path), + package: "pkg".to_owned(), + }; + let mutant_in = |path: &str, ordinal: u32| Mutant { + ordinal, + file: (Utf8PathBuf::from(path)).into(), + ..mutant() + }; + + let mut plan = plan_of(&[]); + + plan.files = vec![target("src/lib.rs"), target("src/main.rs")]; + plan.mutants = vec![mutant_in("src/lib.rs", 1), mutant_in("src/main.rs", 2)]; + + // Every path names something the survey never produced, so nothing intersects `plan.files`. + let mut compiled = HashSet::default(); + let _ = compiled.insert(Utf8PathBuf::from("build/generated.rs")); + let _ = compiled.insert(Utf8PathBuf::from("vendor/other.rs")); + + withdraw_uncompiled(&mut plan, &compiled); + + assert_eq!( + plan.mutants[0].outcome, + Outcome::Pending, + "a disjoint compiled set must not excuse a mutant" + ); + assert_eq!( + plan.mutants[1].outcome, + Outcome::Pending, + "a disjoint compiled set must not excuse a mutant" + ); +} + +/// An unreadable tree must yield no opinion at all. Returning an empty set instead would mark +/// every mutant in the run as unbuilt and report a perfect score for a run that tested nothing. +#[test] +fn dep_info_that_cannot_be_read_yields_no_conclusion() { + let stdout = r#"{"reason":"compiler-artifact","filenames":["/nowhere/deps/libmine-aaaa.rmeta"]}"#; + + assert!(compiled_sources(stdout, Utf8Path::new("/nowhere")).is_none()); +} + +/// A `.d` path that cannot be read as text — because it turned out to be a directory, not a +/// file — must not abort discovery of every other unit's dep-info; only that one entry's +/// contribution is lost. +#[test] +fn a_dep_info_path_that_cannot_be_read_as_text_is_skipped() { + let dir = crate::testing::workdir("dep-not-a-file-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + + // A directory happens to share the name a dep-info file would have; reading it as text + // fails the same way an unreadable file would. + fs::create_dir_all(deps.join("mine-aaaa.d").as_std_path()).expect("a directory standing in for the dep file"); + + let stdout = artifact_message(&deps.join("libmine-aaaa.rmeta")); + let compiled = compiled_sources(&stdout, Utf8Path::new("/nowhere")); + + // Nothing could be read at all, so the honest answer is "no opinion", not an empty set. + assert!(compiled.is_none(), "{compiled:?}"); +} + +/// A dep-info line naming no rule at all — no `:` in it — carries nothing this can act on; +/// tolerating it keeps the rest of the file's entries readable instead of failing the whole +/// parse on one stray or corrupted line. +#[test] +fn a_dep_info_line_without_a_rule_separator_is_ignored() { + let dir = crate::testing::workdir("dep-no-colon-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "just some text with no colon\n").expect("mine"); + + let stdout = artifact_message(&deps.join("libmine-aaaa.rmeta")); + let compiled = compiled_sources(&stdout, Utf8Path::new("/nowhere")).expect("the file was read"); + + assert!(compiled.is_empty(), "{compiled:?}"); +} + +/// Cargo's JSON stream can be interleaved with plain text on some platforms; a line that fails +/// to parse must be skipped rather than aborting discovery of every artifact that follows it. +#[test] +fn a_line_that_is_not_json_does_not_stop_dep_file_discovery() { + let dir = crate::testing::workdir("dep-files-bad-json-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "x: src/kept.rs\n").expect("mine"); + + let stdout = format!("not json at all\n{}\n", artifact_message(&deps.join("libmine-aaaa.rmeta"))); + + assert_eq!(dep_files(&stdout), vec![deps.join("mine-aaaa.d")]); +} + +/// Only `compiler-artifact` messages name a compiled unit; a message of a different reason — +/// cargo emits many kinds on the same stream — carries no dep-info to look for and must not be +/// mistaken for one. +#[test] +fn messages_with_a_different_reason_are_not_read_as_artifacts() { + let stdout = r#"{"reason":"build-script-executed","filenames":["/gamma/deps/libmine-aaaa.rmeta"]}"#; + + assert!(dep_files(stdout).is_empty()); +} + +/// A filename with no file stem — such as one that is only an extension — cannot be paired +/// with a hash, so it has to be skipped instead of panicking or fabricating one. +#[test] +fn a_filename_with_no_parent_or_stem_is_skipped() { + let stdout = r#"{"reason":"compiler-artifact","filenames":[".rlib"]}"#; + + assert!(dep_files(stdout).is_empty()); +} + +/// An entirely empty filename has neither a parent directory nor a name to hash from; cargo +/// would never really emit one, but a malformed or truncated JSON stream should still be +/// tolerated rather than panicking on a path with nothing in it. +#[test] +fn a_completely_empty_filename_is_skipped() { + let stdout = r#"{"reason":"compiler-artifact","filenames":[""]}"#; + + assert!(dep_files(stdout).is_empty()); +} + +/// A directory that vanished between the build finishing and this scan running must not abort +/// the whole discovery pass; the rest of the units still have dep-info worth reading. +#[test] +fn a_dep_info_directory_that_does_not_exist_is_skipped() { + let stdout = r#"{"reason":"compiler-artifact","filenames":["/gamma/definitely/not/a/directory/libmine-aaaa.rmeta"]}"#; + + assert!(dep_files(stdout).is_empty()); +} + +/// A filesystem is not obliged to hand back valid UTF-8 names, even though cargo's own output +/// always is; an entry that fails that conversion has to be skipped rather than making the +/// whole discovery pass panic partway through a directory listing. +#[test] +#[cfg(unix)] +fn a_directory_entry_with_a_non_utf8_name_is_skipped_without_panicking() { + use std::os::unix::ffi::OsStrExt; + + let dir = crate::testing::workdir("dep-files-non-utf8-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "x: src/kept.rs\n").expect("mine"); + + let invalid_name = std::ffi::OsStr::from_bytes(b"mine-\xFF\xFE.d"); + fs::write(dir.path().join("deps").join(invalid_name), "x: src/ignored.rs\n").expect("non-utf8 entry"); + + let stdout = artifact_message(&deps.join("libmine-aaaa.rmeta")); + let found = dep_files(&stdout); + + assert_eq!(found, vec![deps.join("mine-aaaa.d")], "{found:?}"); +} + +/// Dep-info spells its paths absolutely for some units and relatively for others, and mutants +/// are only ever named relative to the tree, so both spellings have to arrive the same way. +#[test] +fn absolute_and_relative_dependency_paths_both_land_relative_to_the_tree() { + let dir = crate::testing::workdir("dep-paths-"); + let deps = Utf8PathBuf::from_path_buf(dir.path().join("deps")).expect("utf8"); + + fs::create_dir_all(deps.as_std_path()).expect("deps"); + fs::write(deps.join("mine-aaaa.d").as_std_path(), "x: /tree/src/one.rs src/two.rs\n").expect("mine"); + + let stdout = artifact_message(&deps.join("mine-aaaa.rlib")); + let compiled = compiled_sources(&stdout, Utf8Path::new("/tree")).expect("read something"); + + assert!(compiled.contains(Utf8Path::new("src/one.rs")), "{compiled:?}"); + assert!(compiled.contains(Utf8Path::new("src/two.rs")), "{compiled:?}"); +} + +/// A diagnostic full of escapes must survive the borrowed decode intact. +/// +/// The decode names the fields it wants so serde can skip the rest of a megabyte-scale stream +/// without building it, and the strings that are only ever compared point into the line. A +/// `rendered` diagnostic is the one field that cannot: it is all newlines and quotes, so it +/// needs unescaping and a plain `&str` could not hold it. This is what pins the `Cow`. +#[test] +fn a_rendered_diagnostic_survives_the_escapes_it_arrives_with() { + let line = "{\"reason\": \"compiler-message\", \"message\": {\"level\": \"error\", \"rendered\": \"error[E0308]: mismatched types\\n --> src/x.rs:1:5\\n |\\n = note: \\\"quoted\\\"\\n\"}}"; + + let rendered = rendered_diagnostic(line).expect("the line carries a rendered diagnostic"); + + assert!(rendered.contains('\n'), "the newlines are real ones: {rendered:?}"); + assert!(rendered.contains("= note: \"quoted\""), "the quotes came through: {rendered:?}"); + assert!(rendered.starts_with("error[E0308]")); +} + +/// A second round rewrites the files it changed, and leaves every other file exactly as it was. +/// +/// Between two rollback rounds only the files whose mutants were withdrawn can differ, so +/// re-reading and re-splicing the rest produces byte-identical text at a cost that multiplies +/// by the round count. The proof has to be observable rather than incidental: a sentinel is +/// written over an untouched file's instrumented copy between the rounds, and it is still there +/// afterwards precisely because the second round never wrote that file. The file whose mutant +/// was withdrawn is rewritten in the same round, which is what stops this from being a test +/// that a round does nothing at all. +#[test] +fn a_second_round_rewrites_only_the_files_it_withdrew_from() { + let dir = crate::testing::workdir("build-incremental-splice-"); + let origin = Utf8PathBuf::from_path_buf(dir.path().join("origin")).expect("utf8"); + let root = Utf8PathBuf::from_path_buf(dir.path().join("copy")).expect("utf8"); + + let source = "pub const A: i32 = 1;\n"; + + for base in [&origin, &root] { + fs::create_dir_all(base.join("src").as_std_path()).expect("src"); + fs::write(base.join("src/a.rs").as_std_path(), source).expect("a"); + fs::write(base.join("src/b.rs").as_std_path(), source).expect("b"); + } + + let target = Utf8PathBuf::from_path_buf(dir.path().join("target")).expect("utf8"); + let work = Workspace::adopt(root.clone(), target); + let mut plan = empty_plan(&work); + + for name in ["src/a.rs", "src/b.rs"] { + plan.files.push(TargetFile { + path: Utf8PathBuf::from(name), + absolute: origin.join(name), + package: "trivial".to_owned(), + }); + } + + plan.mutants.push(Mutant { + ordinal: 1, + file: (Utf8PathBuf::from("src/a.rs")).into(), + span: 19..20, + ..mutant() + }); + plan.mutants.push(Mutant { + ordinal: 2, + file: (Utf8PathBuf::from("src/b.rs")).into(), + span: 19..20, + ..mutant() + }); + + let mut splices = Splices::default(); + let first = splices + .instrument(&work, &plan, &HashSet::default()) + .expect("the first round splices"); + + assert_eq!(first.len(), 2, "both mutants were guarded"); + + let sentinel = "// this round never touched me\n"; + + fs::write(root.join("src/b.rs").as_std_path(), sentinel).expect("sentinel"); + + // The first round blamed the mutant in `a.rs`, so only that file changes. + let second = splices + .instrument(&work, &plan, &HashSet::from_iter([1])) + .expect("the second round splices"); + + assert_eq!( + fs::read_to_string(root.join("src/b.rs").as_std_path()).expect("b"), + sentinel, + "the untouched file was re-spliced" + ); + + assert_eq!( + fs::read_to_string(root.join("src/a.rs").as_std_path()).expect("a"), + source, + "the withdrawn file was not put back" + ); + + // The guard for the file that did not change is still reported, since the text holding it + // is still in the tree and a diagnostic can still land in it. + assert!(second.contains_key(&2), "{second:?}"); + assert!(!second.contains_key(&1), "{second:?}"); +} + +/// A workspace linked against the real guard runtime, so a spliced guard can actually compile. +/// +/// [`trivial_workspace`] deliberately is not: every mutant spliced into it fails, which is what +/// the convergence tests want. A test of build *ordering* needs the opposite — a population the +/// compiler genuinely divides in two — because the whole claim being tested is that the hint +/// decides only which half the compiler is shown first, and never which half a mutant lands in. +/// +/// The runtime is vendored beside the crate and excluded from the workspace, which is what the run +/// itself does and for the same reason: a path dependency inside a workspace directory but absent +/// from its member list makes cargo refuse to build. +fn guarded_workspace(prefix: &str) -> (tempfile::TempDir, Workspace) { + let dir = crate::testing::workdir(prefix); + let root = Utf8PathBuf::from_path_buf(dir.path().join("src")).expect("utf8"); + let runtime = root.join("gamma-rt"); + + fs::create_dir_all(root.join("src").as_std_path()).expect("src"); + fs::create_dir_all(runtime.join("src").as_std_path()).expect("runtime src"); + + fs::write( + runtime.join("Cargo.toml").as_std_path(), + "[package]\nname = \"cargo-gamma-rt\"\nversion = \"0.0.0\"\nedition = \"2021\"\npublish = false\n\n\ + [lib]\nname = \"gamma_rt\"\npath = \"src/lib.rs\"\n\n[workspace]\n", + ) + .expect("runtime manifest"); + + // The tool's own runtime sources rather than stand-ins, so a change to the guard's signature + // breaks this fixture rather than leaving it testing a shape nothing generates any more. + for (name, source) in [ + ("lib.rs", include_str!("../../../../cargo-gamma-rt/src/lib.rs")), + ("either.rs", include_str!("../../../../cargo-gamma-rt/src/either.rs")), + ("runtime.rs", include_str!("../../../../cargo-gamma-rt/src/runtime.rs")), + ] { + fs::write(runtime.join("src").join(name).as_std_path(), source).expect("runtime source"); + } + + fs::write( + root.join("Cargo.toml").as_std_path(), + "[package]\nname = \"trivial\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ + [dependencies]\ngamma_rt = { path = \"gamma-rt\", package = \"cargo-gamma-rt\" }\n\n\ + [workspace]\nexclude = [\"gamma-rt\"]\n", + ) + .expect("manifest"); + + fs::write(root.join("src/lib.rs").as_std_path(), "pub const A: i32 = 1;\n").expect("lib"); + + let target = Utf8PathBuf::from_path_buf(dir.path().join("target")).expect("utf8"); + let work = Workspace::adopt(root, target); + + (dir, work) +} + +/// A workspace whose library holds `unviable` const items and `viable` function bodies. +/// +/// The split matters: a guard is a call into the runtime, so splicing one into a `const` +/// initializer cannot compile — a const initializer may not call a function — while splicing one +/// into a function body compiles perfectly well. That gives a fixture with two populations whose +/// fate the compiler decides for real, which is what a test of build *ordering* needs: nothing here +/// may be settled by the hint itself. +fn probe_plan(work: &Workspace, unviable: usize, viable: usize) -> Plan { + let mut text = String::new(); + + for index in 0..unviable { + let _ = writeln!(text, "pub const C{index}: i32 = 1;"); + } + + for index in 0..viable { + let _ = writeln!(text, "pub fn f{index}() -> i32 {{ let value = 1; value }}"); + } + + let pristine = work.root.parent().expect("the tree has a parent").join("pristine"); + + fs::create_dir_all(pristine.as_std_path()).expect("pristine"); + fs::write(pristine.join("lib.rs").as_std_path(), &text).expect("pristine source"); + fs::write(work.root.join("src/lib.rs").as_std_path(), &text).expect("tree source"); + + let mut plan = empty_plan(work); + + plan.files.push(TargetFile { + path: Utf8PathBuf::from("src/lib.rs"), + absolute: pristine.join("lib.rs"), + package: "trivial".to_owned(), + }); + + // Every `1` in the fixture is a mutation site, and they appear in source order: the constants + // first, then the function bodies. Ordinals and ids follow that order, so a test can name + // either population by index without depending on how the plan is built. + let mut offset = 0; + + for index in 0..unviable.saturating_add(viable) { + // Anchored on the assignment rather than on the digit, because the item names carry digits + // of their own and a bare search would splice the guard into `C1` instead of its value. + let start = text + .get(offset..) + .and_then(|rest| rest.find("= 1")) + .expect("every item holds a literal") + + offset + + 2; + + offset = start + 1; + + plan.mutants.push(Mutant { + id: format!("mutant-{index}").into(), + ordinal: u32::try_from(index).expect("the fixture is small") + 1, + span: start..start + 1, + replacement: "2".to_owned().into(), + ..mutant() + }); + } + + plan +} + +/// The ids of the mutants at the given indices, which is the form a hint takes. +fn hinted(plan: &Plan, indices: impl IntoIterator) -> HashSet { + indices.into_iter().map(|index| plan.mutants[index].id.clone()).collect() +} + +/// The mutants' outcomes after a build, keyed by ordinal, which is the population a hint may not move. +fn population(plan: &Plan) -> Vec<(u32, Outcome)> { + let mut outcomes: Vec<(u32, Outcome)> = plan.mutants.iter().map(|mutant| (mutant.ordinal, mutant.outcome)).collect(); + + outcomes.sort_unstable_by_key(|(ordinal, _outcome)| *ordinal); + outcomes +} + +/// A stale hint may reorder the build and nothing else. +/// +/// This is the whole safety claim of the tier. The record whose unviability produced these hints is +/// rejected for context — it is never consulted as a filter — so the only way it may be allowed to +/// act is on what the compiler is shown first. Two runs over the same tree, one guided and one not, +/// therefore have to reach the same verdict for every mutant; if the guided one differs anywhere, +/// the hint has moved the population and the tier is unsafe at any speed. +#[test] +fn a_context_mismatch_changes_the_order_and_never_the_population() { + let (_blind_dir, blind_work) = guarded_workspace("build-probe-blind-"); + let mut blind_plan = probe_plan(&blind_work, 5, 2); + + let blind = Converger::default() + .finish( + &blind_work, + &mut blind_plan, + None, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the unguided build converges"); + + let (_guided_dir, guided_work) = guarded_workspace("build-probe-guided-"); + let mut guided_plan = probe_plan(&guided_work, 5, 2); + + let guided = Converger::guided(hinted(&guided_plan, 0..5)) + .finish( + &guided_work, + &mut guided_plan, + None, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the guided build converges"); + + assert_eq!( + population(&blind_plan), + population(&guided_plan), + "a hint that changes any verdict is filtering, not ordering" + ); + + assert_eq!(blind.withdrawn, guided.withdrawn, "the same mutants must be withdrawn either way"); + assert_eq!(blind.ordering.rounds, 0, "an unguided build takes no probe round"); + assert_eq!(guided.ordering.offered, 5, "every hinted mutant was front-loaded"); + assert_eq!(guided.ordering.confirmed, 5, "the compiler agreed with every hint"); + assert_eq!(guided.ordering.rounds, 1, "one probe round, taken once"); +} + +/// A hint that is wrong costs its round and leaves the mutant exactly where it was. +/// +/// The dangerous failure is the silent one: a mutant named by a stale record that would compile +/// perfectly well today, quietly withheld and never judged, leaves the denominator and flatters the +/// score. Here every hint is wrong by construction — the hinted mutants are the ones in function +/// bodies, which compile — so nothing may be withdrawn, and the run must settle them normally. +#[test] +fn a_hint_the_compiler_disagrees_with_leaves_its_mutant_live_and_judged() { + let (_dir, work) = guarded_workspace("build-probe-wrong-"); + let mut plan = probe_plan(&work, 0, 5); + + let build = Converger::guided(hinted(&plan, 0..5)) + .finish( + &work, + &mut plan, + None, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build converges"); + + assert_eq!(build.ordering.offered, 5, "the hints were taken at face value and probed"); + assert_eq!(build.ordering.confirmed, 0, "no hint was confirmed, because none was right"); + assert_eq!(build.withdrawn, 0, "a wrong hint must never withdraw a mutant"); + + for mutant in &plan.mutants { + assert_eq!( + mutant.outcome, + Outcome::Pending, + "a hinted mutant that compiles has to be left for the sweep to judge: {mutant:?}" + ); + } +} + +/// A hint the compiler agrees with is confirmed by the compiler, not by the hint. +#[test] +fn a_hint_the_compiler_agrees_with_is_settled_by_the_compiler() { + let (_dir, work) = guarded_workspace("build-probe-right-"); + let mut plan = probe_plan(&work, 5, 1); + + let build = Converger::guided(hinted(&plan, 0..5)) + .finish( + &work, + &mut plan, + None, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build converges"); + + assert_eq!(build.ordering.confirmed, 5, "every hinted mutant was refused by the compiler"); + + for mutant in plan.mutants.iter().take(5) { + assert_eq!(mutant.outcome, Outcome::CompileError, "{mutant:?}"); + } + + assert_eq!( + plan.mutants[5].outcome, + Outcome::Pending, + "the unhinted mutant compiles and is judged" + ); +} + +/// Too few hints do not buy a round, so no round is spent on them. +#[test] +fn a_handful_of_hints_is_not_worth_a_probe_round() { + let (_dir, work) = guarded_workspace("build-probe-floor-"); + let mut plan = probe_plan(&work, 3, 1); + + let build = Converger::guided(hinted(&plan, 0..3)) + .finish( + &work, + &mut plan, + None, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the build converges"); + + assert_eq!(build.ordering.rounds, 0, "a probe below the floor costs a build and buys nothing"); + assert_eq!(build.ordering.offered, 0, "nothing was front-loaded"); + assert_eq!(build.withdrawn, 3, "the ordinary rounds still find every unviable mutant"); +} + +/// The probe is taken once per mutant for the whole run, however many builds ask for it. +#[test] +fn no_mutant_is_probed_twice_however_many_builds_run() { + let (_dir, work) = guarded_workspace("build-probe-once-"); + let mut plan = probe_plan(&work, 5, 1); + let mut converger = Converger::guided(hinted(&plan, 0..5)); + + let staged = converger + .converge( + &work, + &plan, + None, + &["build", "--keep-going"], + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the staged build runs"); + + assert!( + matches!(staged, Convergence::Built(_)), + "the tree converges once the consts are out" + ); + + let build = converger + .finish( + &work, + &mut plan, + None, + BuildLimits::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the deciding build converges"); + + assert_eq!( + build.ordering.rounds, 1, + "the second build must not re-probe what the first already did" + ); + assert_eq!(build.ordering.offered, 5, "each hinted mutant is offered exactly once"); +} + +/// The probe set is derived from the plan, so the same tree and the same hints probe the same thing. +/// +/// Iterating the hint set directly would make the order — and so the reported counts and any future +/// tie-break — depend on hash iteration order, which differs between processes. +#[test] +fn the_probe_set_is_ordered_by_the_plan_rather_than_by_the_hint_set() { + let (_dir, work) = guarded_workspace("build-probe-order-"); + let plan = probe_plan(&work, 5, 2); + + let forwards = Converger::guided(hinted(&plan, 0..5)); + let backwards = Converger::guided(hinted(&plan, (0..5).rev())); + + let (first, deferred) = forwards.probe_sets(&plan, None); + let (second, also_deferred) = backwards.probe_sets(&plan, None); + + assert_eq!(first, vec![1, 2, 3, 4, 5], "the candidates follow the plan's ordinals"); + assert_eq!(first, second, "the hint set's own order may not reach the build"); + assert_eq!(deferred, also_deferred); + + let held: Vec = { + let mut ordinals: Vec = deferred.into_iter().collect(); + ordinals.sort_unstable(); + ordinals + }; + + assert_eq!(held, vec![6, 7], "only the unhinted mutants are held back from the probe round"); +} + +/// A probe round only defers mutants in the packages the build actually compiles. +/// +/// Deferring a mutant outside the selection buys nothing — it contributes no diagnostic to this +/// build — and costs a rewrite of a file a later stage is about to want instrumented again. +#[test] +fn a_probe_leaves_mutants_outside_the_selection_where_they_are() { + let (_dir, work) = guarded_workspace("build-probe-selection-"); + let mut plan = probe_plan(&work, 5, 2); + + for mutant in plan.mutants.iter_mut().skip(5) { + mutant.package = "elsewhere".to_owned().into(); + } + + let converger = Converger::guided(hinted(&plan, 0..5)); + let select = vec![plan.mutants[0].package.to_string()]; + let (candidates, deferred) = converger.probe_sets(&plan, Some(&select)); + + assert_eq!(candidates, vec![1, 2, 3, 4, 5]); + assert!(deferred.is_empty(), "another package's mutants stay spliced: {deferred:?}"); +} diff --git a/crates/cargo-gamma-lib/src/exec/cargo_options.rs b/crates/cargo-gamma-lib/src/exec/cargo_options.rs new file mode 100644 index 000000000..67d529794 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/cargo_options.rs @@ -0,0 +1,405 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; + +use crate::error::error; + +/// Replaces lockfile promises Gamma cannot honor with their offline half. +fn adjust_lock_flags(args: &[String]) -> Vec { + let mut adjusted = Vec::with_capacity(args.len()); + + for arg in args { + let replacement = if matches!(arg.as_str(), "--locked" | "--frozen") { + "--offline" + } else { + arg + }; + + if replacement != "--offline" || !adjusted.iter().any(|kept| kept == "--offline") { + adjusted.push(replacement.to_owned()); + } + } + + adjusted +} + +/// How cargo and the test binaries are invoked. +/// +/// A run builds once and then executes that build thousands of times, so these are the settings +/// that decide what gets compiled and what the compiled thing is asked to do. +#[derive(Debug, Clone, Default)] +pub struct CargoOptions { + /// Feature arguments, already rendered in the form cargo accepts. + pub features: Vec, + + /// The cargo profile to build with. + pub profile: Option, + + /// Extra arguments appended to every cargo invocation. + pub extra: Vec, + + /// Extra arguments appended to every test binary's command line. + pub test_args: Vec, + + /// Whether cargo should color its own output. + /// + /// Cargo's stdio is always a pipe here, so it would otherwise decide "no" every time — and the + /// progress bar gamma borrows from it would arrive as plain text even on a terminal that had + /// asked for color everywhere else. + pub color: bool, +} + +impl CargoOptions { + /// Refuses Cargo configuration that gamma cannot apply while discovering the build. + /// + /// Cargo accepts both an inline TOML value and a path after `--config`. Those settings can + /// change the target, flags and profiles before rustc sees a source file, while gamma's cfg + /// discovery and cache provenance deliberately resolve configuration from the workspace. Do + /// not let the two builds silently diverge; supporting this needs to model Cargo's full + /// configuration precedence, so it is rejected before either discovery or Cargo starts. + pub fn validate(&self) -> crate::Result<()> { + if let Some(argument) = self + .extra + .iter() + .find(|argument| argument.as_str() == "--config" || argument.starts_with("--config=")) + { + return Err(error!( + "pass-through Cargo configuration `{argument}` is not supported; put the setting in a Cargo configuration file gamma can inspect" + ) + .usage()); + } + + Ok(()) + } + + /// Describes the compilation these options ask for, for a workspace at `root`. + /// + /// Discovery evaluates `#[cfg(...)]` against the build that will actually be run, and this is + /// where it learns what that build is: the profile decides `debug_assertions`, the passthrough + /// arguments can carry `--target`, and the environment and cargo configuration carry the rest. + /// Derived from these options rather than resolved independently, so the tree that is surveyed + /// and the tree that is compiled cannot describe different builds. + #[must_use] + pub fn cfg_build(&self, root: &camino::Utf8Path) -> crate::cfg::Build { + crate::cfg::Build::resolve(root, self.profile.as_deref(), &self.extra) + } + + /// Appends the build-shaping arguments to a cargo command line. + /// + /// The flags that promise the lockfile will not change are the one thing not passed through as + /// written. Gamma adds the guard runtime to the manifest before it builds, so the lockfile + /// *will* change; `--locked` would fail the build before a single mutant ran, which is a worse + /// answer than the honest one. + pub fn extend_build_args(&self, args: &mut Vec) { + args.extend(self.features.iter().cloned()); + + if let Some(profile) = self.profile.as_ref() { + args.push("--profile".to_owned()); + args.push(profile.clone()); + } + + args.extend(adjust_lock_flags(&self.extra)); + } + + /// Appends the build-shaping arguments in nextest's spelling. + pub fn extend_nextest_args(&self, args: &mut Vec) { + args.extend(self.features.iter().cloned()); + + if let Some(profile) = self.profile.as_ref() { + args.push("--cargo-profile".to_owned()); + args.push(profile.clone()); + } + + args.extend(adjust_lock_flags(&self.extra)); + } +} + +/// How many build-and-withdraw rounds are allowed before a run gives up. +/// +/// Some mutants are speculative — replacing a function body with `Some(Default::default())` only +/// compiles when the type happens to implement `Default` — and rustc reports only the errors it +/// reaches before it stops, so a large tree can need many rounds to converge. The cost of a round is +/// a rebuild of a tree that is already warm, whereas the cost of stopping too early is a run that +/// cannot complete at all, so the limit is deliberately lopsided. +pub const DEFAULT_ROLLBACK_ROUNDS: u32 = 256; + +/// Limits on how long the build may take. +#[derive(Debug, Clone, Copy, Default)] +pub struct BuildLimits { + /// A fixed budget for the build, whatever it turns out to cost. + pub timeout: Option, + + /// The multiple of the first round's duration a later rollback round is allowed. + /// + /// Rollback rounds recompile the same tree with strictly fewer live mutants, so a round that + /// takes far longer than the first is not converging. + pub multiplier: Option, + + /// How many build-and-withdraw rounds are allowed before the run gives up. + /// + /// Zero means the built-in default, so that a caller that does not care about rollback does not + /// have to know what the default is. + pub rollback_rounds: u32, +} + +impl BuildLimits { + /// Returns how many build-and-withdraw rounds are allowed. + #[must_use] + pub const fn rounds(&self) -> u32 { + if self.rollback_rounds == 0 { + DEFAULT_ROLLBACK_ROUNDS + } else { + self.rollback_rounds + } + } + + /// Returns the budget for a round, given how long the first round took. + #[must_use] + pub fn budget(&self, first: Option) -> Option { + let scaled = self + .multiplier + .zip(first) + .map(|(multiplier, first)| first.mul_f64(multiplier).max(MINIMUM_BUILD_BUDGET)); + + match (self.timeout, scaled) { + (Some(fixed), Some(scaled)) => Some(fixed.min(scaled)), + (fixed, scaled) => fixed.or(scaled), + } + } +} + +/// Floor under a scaled build budget, so a first round that finished instantly cannot produce one +/// that the next round trips over for reasons of scheduling alone. +const MINIMUM_BUILD_BUDGET: Duration = Duration::from_secs(30); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn zero_rollback_rounds_means_the_default() { + // A caller that does not care about rollback should not have to know what the default is, + // and a build that allowed zero rounds could never even try once. + assert_eq!(BuildLimits::default().rounds(), DEFAULT_ROLLBACK_ROUNDS); + assert_eq!( + BuildLimits { + rollback_rounds: 7, + ..BuildLimits::default() + } + .rounds(), + 7 + ); + } + + #[test] + fn no_limits_means_no_budget() { + assert_eq!(BuildLimits::default().budget(Some(Duration::from_secs(10))), None); + } + + #[test] + fn a_fixed_timeout_applies_from_the_first_round() { + let limits = BuildLimits { + timeout: Some(Duration::from_mins(1)), + multiplier: None, + rollback_rounds: 0, + }; + + assert_eq!(limits.budget(None), Some(Duration::from_mins(1))); + } + + #[test] + fn a_multiplier_needs_a_first_round_to_scale_from() { + let limits = BuildLimits { + timeout: None, + multiplier: Some(2.0), + rollback_rounds: 0, + }; + + assert_eq!(limits.budget(None), None); + assert_eq!(limits.budget(Some(Duration::from_secs(100))), Some(Duration::from_secs(200))); + } + + #[test] + fn a_scaled_budget_never_falls_below_the_floor() { + let limits = BuildLimits { + timeout: None, + multiplier: Some(2.0), + rollback_rounds: 0, + }; + + assert_eq!(limits.budget(Some(Duration::from_secs(1))), Some(MINIMUM_BUILD_BUDGET)); + } + + #[test] + fn the_tighter_of_the_two_wins() { + let limits = BuildLimits { + timeout: Some(Duration::from_mins(1)), + multiplier: Some(2.0), + rollback_rounds: 0, + }; + + assert_eq!(limits.budget(Some(Duration::from_secs(100))), Some(Duration::from_mins(1))); + } + + #[test] + fn build_args_are_rendered_in_cargo_order() { + let options = CargoOptions { + features: vec!["--all-features".to_owned()], + profile: Some("release".to_owned()), + extra: vec!["--offline".to_owned()], + test_args: Vec::new(), + color: false, + }; + + let mut args = Vec::new(); + + options.extend_build_args(&mut args); + + assert_eq!(args, vec!["--all-features", "--profile", "release", "--offline"]); + } + + #[test] + fn lockfile_promises_become_one_offline_flag() { + let options = CargoOptions { + extra: vec![ + "--locked".to_owned(), + "--frozen".to_owned(), + "--offline".to_owned(), + "--verbose".to_owned(), + ], + ..CargoOptions::default() + }; + let mut args = Vec::new(); + + options.extend_build_args(&mut args); + + assert_eq!(args, ["--offline", "--verbose"]); + } + + /// The build discovery evaluates predicates against has to be the one these options describe. + /// + /// Not merely a delegation: if this ever ignored the profile or passthrough arguments, a run + /// built with `--profile release --target …` would be surveyed as a different build, and every + /// item behind a gate those settings decide would be misjudged. + #[test] + #[cfg(not(miri))] + fn the_described_build_carries_the_profile_and_the_passthrough_target() { + let options = CargoOptions { + profile: Some("release".to_owned()), + extra: vec!["--target".to_owned(), "x86_64-pc-solaris".to_owned()], + ..CargoOptions::default() + }; + + let build = options.cfg_build(camino::Utf8Path::new(".")); + + assert_eq!(build.target.as_deref(), Some("x86_64-pc-solaris")); + assert_eq!( + build, + crate::cfg::Build::resolve(camino::Utf8Path::new("."), Some("release"), &options.extra) + ); + } + + #[test] + fn inline_and_file_pass_through_cargo_configuration_are_refused() { + for extra in [ + vec!["--config".to_owned(), "build.target = \"wasm32-wasip1\"".to_owned()], + vec!["--config=extra.toml".to_owned()], + ] { + let failure = CargoOptions { + extra, + ..CargoOptions::default() + } + .validate() + .expect_err("unmodelled Cargo configuration must stop before discovery"); + + assert!(failure.is_usage(), "{failure}"); + assert!(failure.to_string().contains("--config"), "{failure}"); + } + } + + #[test] + fn nextest_args_use_its_cargo_profile_spelling() { + let options = CargoOptions { + features: vec!["--all-features".to_owned()], + profile: Some("mutants".to_owned()), + extra: vec!["--offline".to_owned()], + ..CargoOptions::default() + }; + let mut args = Vec::new(); + + options.extend_nextest_args(&mut args); + + assert_eq!(args, vec!["--all-features", "--cargo-profile", "mutants", "--offline"]); + } + + /// The bug this guards: `--locked` was accepted and then failed the build, because gamma adds + /// the guard runtime to the manifest and that forces the lockfile to be written. The flag is + /// substituted rather than obeyed or refused, and the run gets as far as it would have. + #[test] + fn a_lockfile_promise_gamma_cannot_keep_becomes_the_half_it_can() { + crate::notes::alone(|| { + let options = CargoOptions { + extra: vec!["--locked".to_owned(), "--verbose".to_owned()], + ..CargoOptions::default() + }; + + let mut args = Vec::new(); + + options.extend_build_args(&mut args); + + assert_eq!(args, vec!["--offline", "--verbose"]); + }); + } + + /// `--frozen` is `--locked` plus `--offline`, so what survives it is the substitute itself. + #[test] + fn a_frozen_lockfile_is_treated_the_same_way() { + crate::notes::alone(|| { + let options = CargoOptions { + extra: vec!["--frozen".to_owned()], + ..CargoOptions::default() + }; + + let mut args = Vec::new(); + + options.extend_build_args(&mut args); + + assert_eq!(args, vec!["--offline"]); + }); + } + + /// Lock-flag substitution is an implementation detail and does not emit user-facing output. + #[test] + fn substituting_a_lock_flag_is_silent() { + crate::notes::alone(|| { + let options = CargoOptions { + extra: vec!["--locked".to_owned()], + ..CargoOptions::default() + }; + + // Once per build the run does: the check build, the test build, every rollback round. + for _round in 0..3 { + options.extend_build_args(&mut Vec::new()); + } + let raised = crate::notes::drain(); + + assert!(raised.is_empty(), "{raised:?}"); + }); + } + + /// A command line with nothing substituted has nothing to say about it. + #[test] + fn a_lockfile_promise_gamma_can_keep_is_announced_not_at_all() { + crate::notes::alone(|| { + let options = CargoOptions { + extra: vec!["--offline".to_owned()], + ..CargoOptions::default() + }; + + options.extend_build_args(&mut Vec::new()); + + assert!(crate::notes::drain().is_empty()); + }); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/census.rs b/crates/cargo-gamma-lib/src/exec/census.rs new file mode 100644 index 000000000..c0e4a872e --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/census.rs @@ -0,0 +1,1883 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Which tests can reach which mutation sites, measured rather than guessed. +//! +//! A mutant is only ever caught by a test that executes its site. The sweep does not know which +//! tests those are, so it runs every test of every binary that links the mutated package and pays +//! for the whole suite to reach one line. A census establishes the answer once, and then each +//! mutant runs only the tests that can possibly convict it. +//! +//! # Why this needs no coverage instrumentation +//! +//! The instrumented tree already calls `gamma_rt::a(id)` at every mutation site. Running it with +//! `GAMMA_CENSUS` set puts the runtime into census mode: every guard answers `false`, so the +//! process runs the code the author wrote, and each site records the fact that it was reached. That +//! is a finer probe than a coverage region and it is already keyed by the exact thing a mutant is +//! named by, so no second build and no `-C instrument-coverage` are involved. +//! +//! # Why it is exact rather than approximate +//! +//! A mutant `M` sits at one site `S`, and `M` changes nothing whatsoever before `S` executes. So a +//! test that reaches `S` with no mutant active reaches it with `M` active too, and a test that +//! never reaches `S` never fires `M` at all — its execution is identical to the baseline's, and it +//! still does not reach `S`. A complete census therefore measures the relation the sweep needs, +//! not a conservative approximation of it. +//! +//! The one thing that breaks the argument is a program whose execution is not a function of its +//! input: threads racing, wall-clock reads, randomness, hash iteration order. A test that reaches a +//! site only on some runs may be censused on a run where it did not, and the mutant it would have +//! caught is then reported as surviving. `--whole-test-binaries` is the conservative opt-out for +//! such a suite. A census cut short by its economic budget can only provide positive hints: a test +//! observed reaching a site is tried first, but anything other than a kill falls back to the whole +//! binary. Only a complete census can exclude tests or establish that a site is uncovered. +//! +//! # How a cut-short census is caught +//! +//! "Thrown away whole" needs a way to *know* a run was cut short, and an appended file gives no +//! obvious sign: a truncated write and an honest empty reach both leave a file this can parse. So +//! the runtime ends every clean census with a [`SEAL`] record, written from an `atexit` hook so it +//! lands even when the test reached no site at all. The file is appended and flushed as a prefix of +//! what was written, so the seal intact at the end proves every record before it is present too. A +//! file that is absent, unsealed, or sealed and then appended to is discarded rather than read, +//! which is what stops a dropped record ever being mistaken for an unreached site. + +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::time::Duration; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::Instant; +use std::{fs, thread}; + +use camino::{Utf8Path, Utf8PathBuf}; +use cargo_gamma_process::{MemoryRequest, ProcessTree, prepare}; +use gamma_rt::{OVERFLOW, SEAL}; + +use super::events::Events; +#[cfg(test)] +use super::faults::{self, Fault}; +use super::harness_filters::HarnessFilters; +#[cfg(test)] +use super::loader::LOADER_VAR; +use super::loader::configure_loader; +use super::stall::Stall; +use super::test_binary::TestBinary; +use super::verdict::{Attempt, Only, Verdict, observe}; +use super::workspace::Workspace; +use crate::{HashMap, HashSet}; + +/// How much longer than the binary's whole baseline a single censused test may take. +/// +/// One test out of a suite should be a fraction of the whole, so the whole baseline is already a +/// generous ceiling; the multiplier is there so that the *slowest* test in a fast binary is not cut +/// off by a budget calibrated on the average. +const CENSUS_FACTOR: u32 = 4; + +/// Which tests reach which mutation sites, per test binary. +#[derive(Debug, Default)] +pub(super) struct Census { + /// Keyed by binary path, which is what the sweep has in hand. A binary missing from here was + /// never censused, or its census was discarded, and its mutants run its whole suite. + binaries: HashMap, + + /// How many sample subprocesses the census actually launched. + /// + /// One per test that was run in census mode — which is *not* the number of tests listed, because + /// a binary spoiled partway through skips the rest of its tests, so those are never launched and + /// never counted here. The per-binary listing subprocess is excluded too; only the sample runs + /// that probe reachability are counted. + /// + /// Carried so the diagnostics can weigh the census's cost against its sweep dividend: the count + /// is exactly the number of sample subprocesses the census spent, and a run cannot say whether + /// the census paid for itself without it. Zero on a [`Census::default`], which is what an + /// unasked-for census is. + walked: usize, +} + +/// One binary's census. +#[derive(Debug, Default)] +struct Reach { + /// Every test the harness announced, in the order it announced them. + names: Vec>, + + /// For each site ordinal, an index into `intern` that names the tests reaching it. + /// + /// Interning eliminates duplication: many sites share the exact same reaching test set (e.g. + /// all sites in one function body are typically reached by the same tests). Storing an index + /// per site rather than a full vector per site reduces peak memory proportionally to that + /// sharing factor. + reached: HashMap, + + /// Deduplicated reach sets. Each entry is a sorted set of test indices into `names`. + intern: Vec>, + + /// Reverse lookup from sorted reach set to its index in `intern`. + intern_index: HashMap, u32>, + + /// How long each test took when run alone, parallel to `names`. + times: Vec, + + /// Whether every listed test completed and therefore absence proves a site is uncovered. + complete: bool, +} + +impl Reach { + /// Resolves a site to its interned reach set, or `None` if it was never recorded. + fn tests_for(&self, ordinal: u32) -> Option<&[u32]> { + let index = self.reached.get(&ordinal)?; + self.intern.get(*index as usize).map(|s| &**s) + } + + /// Interns a sorted set of test indices, returning its index. + fn intern_set(&mut self, mut set: Vec) -> u32 { + set.sort_unstable(); + set.dedup(); + + let key: Arc<[u32]> = set.into(); + + if let Some(&existing) = self.intern_index.get(&key) { + return existing; + } + + let index = u32::try_from(self.intern.len()).unwrap_or(u32::MAX); + let _previous = self.intern_index.insert(Arc::clone(&key), index); + self.intern.push(key); + index + } + + fn finish(&mut self) { + self.intern_index = HashMap::default(); + } +} + +/// The work a censused binary represents for one mutant. +pub(super) enum CensusWork { + /// Run the whole binary: it was not measured, or narrowing would not save enough to use. + Whole, + + /// The census established that no test in the binary reaches the site. + Uncovered, + + /// Run the selected tests, whose measured census durations sum to this. + Selected(Duration), + + /// Try measured tests first, then run the whole binary if none kills the mutant. + Hinted(Duration), +} + +/// How a mutant should use one binary's census. +#[derive(Debug, PartialEq, Eq)] +pub(super) enum CensusSelection<'census> { + /// Run the whole binary. + Whole, + + /// A complete census established that no test reaches the site. + Uncovered, + + /// A complete census established the only tests that can reach the site. + Selected(Vec<&'census str>), + + /// An incomplete census found candidate tests that may kill the mutant. + Hinted(Vec<&'census str>), +} + +impl Census { + /// The tests in `binary` that can reach the mutant with ordinal `ordinal`. + /// + /// `None` means nothing is known about this binary and its whole suite must run. `Some` of an + /// empty slice is the opposite and much stronger claim: this binary was censused, and no test + /// in it reaches that site. The distinction is the whole safety property — an absent census + /// must never be read as an absent test. + pub(super) fn selection(&self, binary: &TestBinary, ordinal: u32) -> CensusSelection<'_> { + let Some(reach) = self.binaries.get(&binary.path) else { + return CensusSelection::Whole; + }; + + let Some(indices) = reach.tests_for(ordinal) else { + return if reach.complete { + CensusSelection::Uncovered + } else { + CensusSelection::Whole + }; + }; + + // Past half the suite, naming the tests costs more than it saves: the command line grows + // with every name, and the run it replaces was barely narrower. Answering `None` here says + // "nothing is known", which is the answer that makes the caller run the whole binary — the + // same thing, reached more cheaply. + if indices.len().saturating_mul(2) > reach.names.len() { + return CensusSelection::Whole; + } + + let names = indices + .iter() + .filter_map(|index| usize::try_from(*index).ok()) + .filter_map(|index| reach.names.get(index)) + .map(Box::as_ref) + .collect(); + + if reach.complete { + CensusSelection::Selected(names) + } else { + CensusSelection::Hinted(names) + } + } + + /// Legacy projection used by the reachability unit tests. + #[cfg(test)] + fn reaching(&self, binary: &TestBinary, ordinal: u32) -> Option> { + match self.selection(binary, ordinal) { + CensusSelection::Whole | CensusSelection::Hinted(_) => None, + CensusSelection::Uncovered => Some(Vec::new()), + CensusSelection::Selected(names) => Some(names), + } + } + + /// Estimates the work this binary contributes for one mutant from the census itself. + pub(super) fn work(&self, binary: &TestBinary, ordinal: u32) -> CensusWork { + let Some(reach) = self.binaries.get(&binary.path) else { + return CensusWork::Whole; + }; + + let Some(indices) = reach.tests_for(ordinal) else { + return if reach.complete { CensusWork::Uncovered } else { CensusWork::Whole }; + }; + + if indices.len().saturating_mul(2) > reach.names.len() { + return CensusWork::Whole; + } + + let duration = indices + .iter() + .filter_map(|index| usize::try_from(*index).ok()) + .filter_map(|index| reach.times.get(index)) + .copied() + .sum(); + + if reach.complete { + CensusWork::Selected(duration) + } else { + CensusWork::Hinted(duration) + } + } + + /// How many binaries were censused, for the line the run prints. + pub(super) fn len(&self) -> usize { + self.binaries.len() + } + + /// How many sample subprocesses the census actually launched. + /// + /// The count of sample runs that were really spawned, which excludes the tests of a binary that + /// was spoiled before reaching them and the per-binary listing subprocesses. See the [`walked`] + /// field. + /// + /// [`walked`]: Census::walked + pub(super) const fn walked(&self) -> usize { + self.walked + } + + /// A census recording `binary` as examined with `total` tests, `reached` of which reach `site`. + /// + /// For tests in sibling modules, whose own `#[cfg(test)]` code cannot reach the private [`Reach`] + /// this builds. It exists to drive how a *consumer* reacts to a census — in particular the + /// boundary where a site most of the suite reaches makes [`reaching`] answer `None`, which means + /// "run the whole binary" and must never be confused with the empty list that means "no test + /// reaches it". + /// + /// [`reaching`]: Census::reaching + #[cfg(all(test, unix))] + pub(super) fn examined(binary: &Utf8Path, site: u32, reached: usize, total: usize) -> Self { + Self::fixture(binary, site, reached, total, true) + } + + /// A deliberately incomplete census for testing checked-hint fallbacks. + #[cfg(test)] + pub(super) fn partial(binary: &Utf8Path, site: u32, reached: usize, total: usize) -> Self { + Self::fixture(binary, site, reached, total, false) + } + + #[cfg(test)] + fn fixture(binary: &Utf8Path, site: u32, reached: usize, total: usize, complete: bool) -> Self { + let names: Vec> = (0..total).map(|index| format!("tests::t{index}").into()).collect(); + let indices: Vec = (0..reached).filter_map(|index| u32::try_from(index).ok()).collect(); + + let mut reach = Reach { + names, + reached: HashMap::default(), + intern: Vec::new(), + intern_index: HashMap::default(), + times: vec![Duration::from_millis(1); total], + complete, + }; + let set_index = reach.intern_set(indices); + let _previous2 = reach.reached.insert(site, set_index); + reach.finish(); + + let mut census = Self::default(); + let _previous = census.binaries.insert(binary.to_owned(), reach); + + census + } +} + +/// Measures which tests reach which sites, one binary at a time. +/// +/// Never fails the run. A binary that cannot be listed, or whose sampled tests did not all pass, +/// simply does not appear in the result. A budget-limited binary retains only positive reach hints, +/// and every inconclusive hint falls back to the whole binary. +pub(super) fn take( + work: &Workspace, + binaries: &[TestBinary], + targets: &HashMap>, + maximum_savings: Duration, + jobs: usize, + stall: Stall, + events: &mut impl Events, +) -> Census { + let mut census = Census::default(); + let binaries: Vec<&TestBinary> = binaries.iter().filter(|binary| targets.contains_key(&binary.path)).collect(); + + events.begin( + "Optimizing", + "Optimized", + &format!("{} test {}", binaries.len(), plural(binaries.len())), + ); + + let total = binaries.len(); + let unit = format!("test {}", plural(total)); + let mut completed = 0_usize; + + events.phase_progress(completed, total, &unit); + + let mut listed: Vec<(&TestBinary, Vec>)> = Vec::with_capacity(binaries.len()); + let mut estimated_cost = Duration::ZERO; + + for binary in &binaries { + let started = Instant::now(); + let Some(names) = list(work, binary) else { + completed += 1; + events.phase_progress(completed, total, &unit); + + continue; + }; + let listing_cost = started.elapsed(); + + estimated_cost = estimated_cost.saturating_add(listing_cost.saturating_mul(u32::try_from(names.len()).unwrap_or(u32::MAX))); + listed.push((binary, names)); + } + + if !can_repay(estimated_cost, maximum_savings) { + events.end(""); + + return census; + } + + // The count comes back from the walk rather than being summed from the lists above, because a + // binary spoiled partway through skips the rest of its tests: those are listed but never + // launched, and `walked` counts launches, not intentions. + let deadline = Instant::now().checked_add(maximum_savings); + let (mapped, walked) = walk(work, listed, targets, deadline, jobs, stall, || { + completed += 1; + events.phase_progress(completed, total, &unit); + }); + + for (path, reach) in mapped { + let _previous = census.binaries.insert(path, reach); + } + + census.walked = walked; + + events.end(&format!( + ", {} of {} {} mapped, over {walked} {}", + census.len(), + binaries.len(), + plural(binaries.len()), + if walked == 1 { "test" } else { "tests" } + )); + + census +} + +/// Whether the estimated census work is smaller than everything it could possibly save. +fn can_repay(estimated_cost: Duration, maximum_savings: Duration) -> bool { + !maximum_savings.is_zero() && estimated_cost < maximum_savings +} + +/// "binary" or "binaries", so the line above reads as English either way. +const fn plural(count: usize) -> &'static str { + if count == 1 { "binary" } else { "binaries" } +} + +/// How long a binary is given to answer `--list` before the answer is abandoned. +/// +/// A real libtest `--list` enumerates a table already in the binary and returns in milliseconds, so +/// anything approaching this is not listing at all. The number is generous against a cold page +/// cache and a loaded machine rather than tuned, because the cost of being wrong is asymmetric: too +/// short loses a census that would have saved time, too long is the hang this exists to bound. +const LIST_BUDGET: Duration = Duration::from_secs(30); + +/// How much of a listing is worth reading before the binary is not listing. +/// +/// A libtest listing is one short line per test, so even a very large suite is well under this. A +/// binary that ignored the flags and started running instead can print without limit, and buffering +/// that would turn one unlucky target into an out-of-memory kill of the whole run. +const LIST_CAP: usize = 4 * 1024 * 1024; + +/// How often the wait below looks at a child that has not finished. +/// +/// Listing is expected to be over before the first poll, so this trades a millisecond of latency on +/// the answer for not spinning a core for the length of the budget on the one that hangs. +const LIST_POLL: Duration = Duration::from_millis(5); + +/// Asks a test binary to name its tests. +/// +/// The binary is run directly even when the run is under nextest, because listing is a question +/// about the executable rather than about the runner, and the libtest listing format is the one +/// both the filter arguments and the census launches below are expressed in. +/// +/// `None` for anything that did not answer in that format — a `harness = false` target, most +/// obviously — or did not answer within [`LIST_BUDGET`], which is the same target when its +/// `fn main()` ignores the flags and runs its suite instead. Either way the binary is left without +/// a census and therefore run in full, which is the answer this had before the census existed. +fn list(work: &Workspace, binary: &TestBinary) -> Option>> { + let mut command = listing_command(work, binary); + + // A successful process with no libtest records may be a custom harness that ignored both + // flags. Without a positive record there is no evidence that the output was a complete census, + // so leave the binary uncensused and run it whole. + listed(&mut command, LIST_BUDGET).filter(|names| !names.is_empty()) +} + +/// Builds the direct `--list` invocation for one test binary. +fn listing_command(work: &Workspace, binary: &TestBinary) -> std::process::Command { + let mut command = std::process::Command::new(binary.path.as_std_path()); + + let _ = command + .args(["--list", "--format=terse"]) + // The user's own filters, so the census only ever records tests that are actually going to + // run. Without them the listing discovers names outside the filter and feeds them back as a + // selection, which the launcher then has to refuse. Everything but a `--format` of their + // own is carried, since that one would fight with the format asked for here. + .args(HarnessFilters::parse(work.test_arguments()).selecting()) + .current_dir(working_directory(work, binary).as_std_path()) + .stderr(std::process::Stdio::null()); + + configure_loader(&mut command, work.launch()); + + command +} + +/// Runs a listing command under the same containment and bound every other spawn here gets. +/// +/// Separate from [`list`] because the command is the only thing a test needs to vary, and a binary +/// that hangs on `--list` cannot be written as a test binary the way the rest of the helper scripts +/// are: the flags this passes are exactly the ones such a target ignores. +/// +/// The containment is not optional here even though nothing is metered. A target built with +/// `harness = false` is a `fn main()` that may ignore unknown flags entirely, and the ones that do +/// run their whole suite in answer to this — spawning whatever their tests spawn. Uncontained, a +/// `SIGTERM` to the tool would leave all of that running, holding scratch-tree locks that fail the +/// next run; unbounded, one such target hangs a run that has nothing watching it, since this is the +/// only spawn in the subsystem outside the wait loop. +/// +/// A budget reached is `None` rather than a partial listing: half an enumeration is a census that +/// believes some tests do not exist, and a test believed not to exist is one no mutant is ever run +/// against. +fn listed(command: &mut std::process::Command, budget: Duration) -> Option>> { + use std::process::Stdio; + + // Nothing is metered — the question is what the binary is, not what it costs — so the request + // asks for no boundary and the containment reduces to the group and the interrupt slot. + let request = MemoryRequest { meter: false, limit: None }; + + let _ = command.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()); + + let guard = prepare(command, request).ok()?; + let child = command.spawn().ok()?; + + let mut subtree = match ProcessTree::adopt(child, guard) { + Ok(subtree) => subtree, + + // Adoption already ended and reaped the unwatchable child. + Err(_unwatchable) => return None, + }; + + // Read on another thread rather than after the wait: a binary that prints more than a pipe + // holds blocks in `write` until somebody empties it, and a wait that has not started reading + // would then time out every target with a listing longer than 64 KB. + // + // Whether the stream really ended travels with the bytes for the same reason it does on the + // verdict path: a prefix of a listing is a set of tests that appear not to exist, and a test + // that appears not to exist is one no mutant is ever run against. + let (sender, receiver) = mpsc::sync_channel(1); + let reading = subtree.take_stdout().map(|mut pipe| { + let sender = sender.clone(); + let failed = sender.clone(); + + // The handle is deliberately detached. A descendant that escaped containment can retain + // this pipe indefinitely, and the listing deadline must bound the caller rather than make + // it wait for a reader no process can force to return. + #[cfg(test)] + let refused = faults::fired(Fault::Thread); + #[cfg(not(test))] + let refused = false; + + let spawned = if refused { + Err(std::io::Error::other("the reader thread a test asked to fail")) + } else { + thread::Builder::new().name("cargo-gamma-census-output".to_owned()).spawn(move || { + use std::io::Read as _; + + let mut text = Vec::new(); + let mut buffer = [0_u8; 8192]; + let mut whole = true; + + loop { + match pipe.read(&mut buffer) { + Ok(0) => break, + + Ok(read) => { + // Reading continues past the cap so the child never blocks on a full pipe; + // only the keeping stops. + let room = LIST_CAP.saturating_sub(text.len()); + + text.extend_from_slice(&buffer[..read.min(room)]); + whole &= read <= room; + } + + Err(cause) if cause.kind() == std::io::ErrorKind::Interrupted => {} + + Err(_truncated) => { + whole = false; + + break; + } + } + } + + let _sent = sender.send((text, whole)); + }) + }; + + if spawned.is_err() { + let _sent = failed.send((Vec::new(), false)); + } + }); + drop(sender); + + let deadline = Instant::now() + budget; + + let status = loop { + match subtree.observe() { + Ok(Some(status)) => break Some(status), + + Ok(None) if Instant::now() >= deadline => break None, + + Ok(None) => thread::sleep(LIST_POLL.min(deadline.saturating_duration_since(Instant::now()))), + + // The handle is gone, so the one question that would settle this cannot be asked again. + Err(_unaskable) => break None, + } + }; + + // Whatever the child spawned goes with it when no normal exit was observed: a listing that + // started a server leaves it holding the pipe this is about to read, and the reader would then + // wait out the whole run for an end of file that never comes. + if status.is_none() { + let _reaped = subtree.terminate(); + } else { + debug_assert!( + subtree.released(), + "the observed listing released containment before its output was read" + ); + } + + // A group cleanup normally closes every inherited descriptor, but a descendant that called + // `setsid` can retain stdout after leaving that group. The original listing deadline covers + // this final drain too: abandoning its blocked reader loses only a census optimization, while + // joining it would turn one escaped process into an unbounded run. + let (text, whole) = reading + .and_then(|()| receiver.recv_timeout(deadline.saturating_duration_since(Instant::now())).ok()) + .unwrap_or_default(); + + if !whole || !status?.success() { + return None; + } + + let text = String::from_utf8(text).ok()?; + + Some( + text.lines() + .filter_map(|line| line.strip_suffix(": test")) + .map(|name| name.trim().into()) + .collect(), + ) +} + +/// Where a test binary would be run from, mirroring what the sweep does. +fn working_directory<'work>(work: &'work Workspace, binary: &'work TestBinary) -> &'work Utf8Path { + if binary.manifest_dir.as_str().is_empty() { + work.root() + } else { + &binary.manifest_dir + } +} + +/// Runs each of a binary's tests alone and collects the sites it reached. +/// +/// One process per test is what makes the attribution possible at all: the runtime records into a +/// process-wide table, so a process running two tests cannot say which of them reached what. It is +/// also the only harness-independent way to ask, which is why it is done the same way whether or +/// not the run is under nextest. +/// +/// A binary is left out of the result when any of its tests did not simply pass, because a run cut +/// short by a budget or a ceiling recorded only the sites it got to, and the sites it missed would +/// then be read as sites no test reaches. +/// +/// The queue is one flat list of `(binary, test)` pairs rather than one pool per binary. A test +/// binary usually holds fewer tests than there are workers — the common shape is one binary per +/// integration-test file — so a pool scoped to a single binary drains to nothing at every boundary +/// and refills, idling cores in the gaps for what is fixed cost paid before any mutant is judged. +/// One queue lets a worker that has finished a small binary start the next one immediately, exactly +/// as the sweep's cursor already does. +/// +/// The pool is still `jobs` workers wide, so this costs no extra concurrent subprocesses, no extra +/// memory and no extra file descriptors over walking the binaries one at a time; and the census +/// file stays per worker, because a worker still runs one test at a time. +/// Returns the reach of each binary that was not spoiled, and how many sample subprocesses were +/// actually launched — which is fewer than the number of tests listed whenever a binary spoils +/// partway through and its remaining tests are skipped. +fn walk( + work: &Workspace, + listed: Vec<(&TestBinary, Vec>)>, + targets: &HashMap>, + deadline: Option, + jobs: usize, + stall: Stall, + completed: impl FnMut(), +) -> (Vec<(Utf8PathBuf, Reach)>, usize) { + walk_with( + work, + listed, + targets, + jobs, + stall, + sample, + || deadline.is_some_and(|limit| Instant::now() >= limit), + completed, + ) +} + +#[expect( + clippy::too_many_arguments, + clippy::too_many_lines, + reason = "the streaming walk shares its inputs and borrowed state across scoped workers" +)] +fn walk_with( + work: &Workspace, + listed: Vec<(&TestBinary, Vec>)>, + targets: &HashMap>, + jobs: usize, + stall: Stall, + sampler: S, + expired: E, + mut completed: impl FnMut(), +) -> (Vec<(Utf8PathBuf, Reach)>, usize) +where + S: Fn(&Workspace, &TestBinary, &str, &Utf8Path, Stall) -> Option<(Vec, Duration)> + Sync, + E: Fn() -> bool + Sync, +{ + let directory = work.base().join("census"); + + if fs::create_dir_all(directory.as_std_path()).is_err() { + return (Vec::new(), 0); + } + + // Cumulative offsets for streaming task resolution: instead of materializing all + // `(binary, test)` pairs, workers compute the pair from a flat index and these offsets. + let offsets: Vec = listed + .iter() + .scan(0_usize, |acc, (_binary, names)| { + let start = *acc; + *acc += names.len(); + Some(start) + }) + .collect(); + let total_tasks: usize = listed.iter().map(|(_binary, names)| names.len()).sum(); + + let next = AtomicUsize::new(0); + + let launches = AtomicUsize::new(0); + + // One entry per wanted site of each binary, position-indexed rather than keyed, so the + // saturation check below can be a plain atomic load instead of a lock acquisition. Built once, + // before any worker starts, from `targets` — the same read-only map every worker already + // consults for `wanted` — so no worker ever writes to it. + let site_positions: Vec> = listed + .iter() + .map(|(binary, _names)| match targets.get(&binary.path) { + Some(wanted) => { + let mut sorted: Vec = wanted.iter().copied().collect(); + sorted.sort_unstable(); + sorted.into_iter().enumerate().map(|(position, site)| (site, position)).collect() + } + None => HashMap::default(), + }) + .collect(); + let site_counts: Vec> = site_positions + .iter() + .map(|positions| (0..positions.len()).map(|_position| AtomicUsize::new(0)).collect()) + .collect(); + + let times: Vec>> = listed + .iter() + .map(|(_binary, names)| names.iter().map(|_name| Mutex::new(Duration::ZERO)).collect()) + .collect(); + let spoiled: Vec = listed.iter().map(|_entry| AtomicBool::new(false)).collect(); + let saturated: Vec = listed.iter().map(|_entry| AtomicBool::new(false)).collect(); + let sample_counts: Vec = listed.iter().map(|_entry| AtomicUsize::new(0)).collect(); + let remaining: Vec = listed.iter().map(|(_binary, names)| AtomicUsize::new(names.len())).collect(); + let (completion, completions) = mpsc::channel(); + let notes = crate::notes::current(); + + // Each worker accumulates the sites it personally observed into a map it alone owns, rather + // than contending for one shared, binary-wide lock on every sample it completes. Two workers + // never race over the same entry: the flat task index `at` is claimed by exactly one worker via + // `next.fetch_add`, so no `(binary, test)` pair — and so no `(site, test)` observation — is ever + // produced by more than one worker. Merging every worker's map together after `thread::scope` + // has joined every one of them is therefore just a union of disjoint data, needing no lock of + // its own either. + let locals: Vec>>> = thread::scope(|scope| { + let mut handles = Vec::with_capacity(jobs.max(1)); + + for worker in 0..jobs.max(1) { + let (next, listed, offsets, site_positions, site_counts, times, spoiled, saturated, sample_counts, remaining, launches) = ( + &next, + &listed, + &offsets, + &site_positions, + &site_counts, + ×, + &spoiled, + &saturated, + &sample_counts, + &remaining, + &launches, + ); + let completion = completion.clone(); + let notes = notes.clone(); + let sampler = &sampler; + let expired = &expired; + + let path = directory.join(format!("{worker}.bin")); + + let handle = scope.spawn(move || { + let _notes = crate::notes::enter(notes.as_ref()); + + // Indexed by `binary_at`, exactly like the shared per-binary arrays above, but + // owned outright by this worker: nothing outside this closure ever reads or writes + // it, so it needs neither a lock nor an atomic. + let mut local: Vec>> = listed.iter().map(|_entry| HashMap::default()).collect(); + + loop { + let at = next.fetch_add(1, Ordering::Relaxed); + + if at >= total_tasks { + break; + } + + // Resolve flat index to (binary_at, test_at) using cumulative offsets. + let binary_at = match offsets.binary_search(&at) { + Ok(exact) => exact, + Err(after) => after.saturating_sub(1), + }; + let test_at = at - offsets[binary_at]; + + let (binary, names) = &listed[binary_at]; + let spoiled = &spoiled[binary_at]; + let saturated = &saturated[binary_at]; + let sample_count = &sample_counts[binary_at]; + let positions = &site_positions[binary_at]; + let counts = &site_counts[binary_at]; + let times = ×[binary_at]; + let remaining = &remaining[binary_at]; + let name = &names[test_at]; + + if !spoiled.load(Ordering::Relaxed) && !saturated.load(Ordering::Relaxed) && !expired() { + let _previous = launches.fetch_add(1, Ordering::Relaxed); + + match (sampler(work, binary, name, &path, stall), u32::try_from(test_at)) { + (Some((sites, elapsed)), Ok(test_index)) => { + let _previous = sample_count.fetch_add(1, Ordering::Relaxed); + + #[expect(clippy::unwrap_used, reason = "the lock only poisons if a worker panicked, and none can")] + { + let mut time = times[test_at].lock().unwrap(); + *time = elapsed; + } + + let wanted = targets.get(&binary.path); + let reached = &mut local[binary_at]; + + for site in sites.into_iter().filter(|site| wanted.is_some_and(|wanted| wanted.contains(site))) { + let tests = reached.entry(site).or_default(); + + if !tests.contains(&test_index) { + tests.push(test_index); + + // `at` is unique to this one worker for this one task, so + // this `(site, test_index)` observation can never be + // produced by any other worker: the fetch_add below cannot + // double-count it. It is also gated on the same + // `!tests.contains` guard as the push above, so a sampler + // returning the same site twice for one test counts it once, + // not twice. + if let Some(&position) = positions.get(&site) { + let _previous = counts[position].fetch_add(1, Ordering::Relaxed); + } + } + } + + if wanted.is_some_and(|wanted| { + wanted.iter().all(|site| { + positions.get(site).is_some_and(|&position| { + counts[position].load(Ordering::Relaxed).saturating_mul(2) > names.len() + }) + }) + }) { + saturated.store(true, Ordering::Relaxed); + } + } + + _spoiled => spoiled.store(true, Ordering::Relaxed), + } + } + + if remaining.fetch_sub(1, Ordering::Relaxed) == 1 { + let _sent = completion.send(()); + } + } + + local + }); + + handles.push(handle); + } + + drop(completion); + + for () in completions { + completed(); + } + + // `thread::scope` already blocks until every spawned thread finishes before it returns, so + // these joins add no additional wait; they only recover the value each worker computed + // instead of letting it drop unread. + handles + .into_iter() + .map(|handle| handle.join().expect("a census worker exits normally; nothing in its loop panics")) + .collect() + }); + + // A plain union: every `(binary, site)` entry a worker produced is disjoint from every other + // worker's, so combining them keeps every observation and drops none, in whichever order the + // workers happen to be visited. + let mut reached: Vec>> = (0..listed.len()).map(|_entry| HashMap::default()).collect(); + for worker_local in locals { + for (binary_at, site_map) in worker_local.into_iter().enumerate() { + let target = &mut reached[binary_at]; + for (site, mut tests) in site_map { + target.entry(site).or_default().append(&mut tests); + } + } + } + + let mut mapped = Vec::with_capacity(listed.len()); + + for ((binary, names), (raw_reached, (times, (spoiled, sample_count)))) in listed.into_iter().zip( + reached + .into_iter() + .zip(times.into_iter().zip(spoiled.into_iter().zip(sample_counts))), + ) { + if spoiled.into_inner() { + continue; + } + + #[expect(clippy::unwrap_used, reason = "the lock only poisons if a worker panicked, and none can")] + let times: Vec = times.into_iter().map(|time| time.into_inner().unwrap()).collect(); + let complete = sample_count.into_inner() == times.len(); + + // Intern identical reach sets to reduce memory for suites where many sites share tests. + let mut reach = Reach { + names, + reached: HashMap::default(), + intern: Vec::new(), + intern_index: HashMap::default(), + times, + complete, + }; + + for (site, indices) in raw_reached { + let set_index = reach.intern_set(indices); + let _previous = reach.reached.insert(site, set_index); + } + reach.finish(); + + mapped.push((binary.path.clone(), reach)); + } + + (mapped, launches.into_inner()) +} + +/// Runs one test in census mode and returns the site ordinals it reached. +/// +/// `None` means the sample cannot be trusted, which is the whole binary's problem rather than this +/// test's: see [`walk`]. +fn sample(work: &Workspace, binary: &TestBinary, name: &str, path: &Utf8Path, stall: Stall) -> Option<(Vec, Duration)> { + // Appended to by the runtime, so a leftover from the previous test on this worker would be read + // as part of this one. + let _removed = fs::remove_file(path.as_std_path()); + + let attempt = Attempt { + // No mutant. A census must see the program the author wrote, or the sites it records are + // the ones some mutant steered it towards. + active: None, + timeout: binary + .budget + .map(|budget| binary.baseline.saturating_mul(CENSUS_FACTOR).max(budget)), + stall, + request: MemoryRequest { meter: false, limit: None }, + only: Only::One(name), + census: Some(path), + }; + + let started = Instant::now(); + + match observe(work, binary, attempt).verdict { + Verdict::Passed => {} + _spoiled => return None, + } + + let elapsed = started.elapsed(); + + // With the runtime sealing every census it completes — even one whose test reached no site, and + // so left only the lone seal — an absent or unreadable file is no longer an empty reach. It + // means the census never finished: the open failed, or the process died before it could seal. + // That is the binary's problem, not this test's answer, so the sample is spoiled rather than + // believed empty. + let bytes = fs::read(path.as_std_path()).ok()?; + + decode(&bytes).map(|sites| (sites, elapsed)) +} + +/// Turns the runtime's records into site ordinals, or `None` if the file is not a whole census. +/// +/// A census is trustworthy only if it ends with the runtime's [`SEAL`]: the file is an append-only +/// stream flushed as a prefix, so a seal intact at the end proves every record before it survived. +/// Everything else is refused — an unsealed file (a truncated exit write or a crash), a record +/// after the seal (a stale census appended to, or two runs sharing a path), and the [`OVERFLOW`] +/// marker (the runtime ran out of table) — because each means sites may be missing, and missing is +/// the one thing this must never guess at. +fn decode(bytes: &[u8]) -> Option> { + if !bytes.len().is_multiple_of(4) { + return None; + } + + let mut sites = Vec::with_capacity(bytes.len() / 4); + let mut sealed = false; + + for record in bytes.chunks_exact(4) { + // A record after the seal means the file is not the single clean prefix the runtime writes: + // a stale census was appended to, or two runs shared the path. Either way it is untrusted. + if sealed { + return None; + } + + match u32::from_le_bytes(record.try_into().ok()?) { + OVERFLOW => return None, + SEAL => sealed = true, + ordinal => sites.push(ordinal), + } + } + + // No seal means the file is a prefix of unknown completeness, not a whole census. + sealed.then_some(sites) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + + #[test] + fn listing_output_beyond_the_cap_is_not_authoritative() { + let (_directory, work) = + crate::testing::helper_workspace("census-list-cap", &["flood:4194305", "print:meaningful::killer: test", "exit:0"]); + let mut command = listing_command(&work, &crate::testing::helper()); + + assert!(listed(&mut command, Duration::from_secs(30)).is_none()); + } + + #[test] + fn listing_reader_thread_failure_degrades_to_no_census() { + let (_directory, work) = crate::testing::helper_workspace("census-list-thread", &["print:a::test: test", "exit:0"]); + let mut command = listing_command(&work, &crate::testing::helper()); + let _refused = faults::arm(Fault::Thread); + + assert!(listed(&mut command, Duration::from_secs(30)).is_none()); + } + + /// Production workspaces hold an exclusive scratch lock; adopted test workspaces do not. + static WALK_TEST: Mutex<()> = Mutex::new(()); + + /// Listing is a real launch and needs the same dynamic libraries as running a test. + #[test] + fn listing_a_binary_uses_the_runs_loader_path() { + let (_scratch, work) = crate::testing::helper_workspace("list-loader", &["exit:0"]); + let binary = crate::testing::helper(); + let command = listing_command(&work, &binary); + let configured = command + .get_envs() + .find(|(name, _value)| *name == std::ffi::OsStr::new(LOADER_VAR)) + .and_then(|(_name, value)| value); + + assert_eq!( + configured, + work.launch().loader.as_deref(), + "the listing launch omitted the libraries used to build its executable" + ); + } + + /// A binary that ignores the listing flags and never returns is abandoned, not waited out. + /// + /// This is what `harness = false` buys: a `fn main()` free to treat `--list` as noise and run + /// its whole suite instead. Every other spawn in this subsystem is bounded, so a target like + /// that would hang the one place with nothing watching it — and it would hang it before a + /// single mutant had been judged, with no output and no way to tell what it was waiting for. + #[test] + #[cfg(unix)] + fn a_binary_that_never_answers_a_listing_is_abandoned_when_its_budget_runs_out() { + let mut command = std::process::Command::new("/bin/sh"); + + // Ignores its arguments and outlives any budget a listing could reasonably be given, which + // is exactly the shape of the custom harness this bounds. + let _ = command.args(["-c", "sleep 300"]); + + let started = Instant::now(); + let names = listed(&mut command, Duration::from_millis(200)); + + assert!(names.is_none(), "a binary that never listed was believed: {names:?}"); + assert!( + started.elapsed() < Duration::from_secs(30), + "the listing waited far past its budget: {:?}", + started.elapsed() + ); + } + + /// A session leader is outside the listing process group, so it retains the stdout pipe after + /// the shell exits. The listing still has to return at its budget rather than joining a reader + /// that can no longer be made to see EOF. + #[test] + #[cfg(unix)] + fn an_escaped_descendant_holding_listing_stdout_does_not_outlive_the_listing_budget() { + if !std::process::Command::new("setsid") + .arg("true") + .status() + .is_ok_and(|status| status.success()) + { + eprintln!("skipping escaped-descendant fixture because `setsid` is unavailable"); + return; + } + + let directory = crate::testing::workdir("census-escaped-listing-"); + let marker = Utf8PathBuf::from_path_buf(directory.path().join("escaped.pid")).expect("UTF-8 path"); + let mut command = std::process::Command::new("/bin/sh"); + let script = format!("setsid sh -c 'echo $$ > \"{marker}\"; sleep 5' & while [ ! -s \"{marker}\" ]; do sleep 0.01; done"); + + let _ = command.args(["-c", &script]); + + let started = Instant::now(); + let names = listed(&mut command, Duration::from_millis(100)); + + for _attempt in 0..20 { + if marker.as_std_path().exists() { + break; + } + + thread::sleep(Duration::from_millis(10)); + } + + let pid = fs::read_to_string(marker.as_std_path()).expect("the escaped descendant started"); + let _stopped = std::process::Command::new("kill").args(["-TERM", pid.trim()]).status(); + + assert!(names.is_none(), "an incomplete listing was accepted: {names:?}"); + assert!( + started.elapsed() < Duration::from_secs(1), + "the listing waited for an escaped pipe holder: {:?}", + started.elapsed() + ); + } + + /// A binary that answers in libtest's terse format is still read the way it always was. + /// + /// The bound and the containment are around the same question, so the answer to it has to be + /// unchanged — a listing lost to the new machinery costs a binary its census and puts its whole + /// suite behind every one of its mutants. + #[test] + #[cfg(unix)] + fn a_binary_that_answers_the_listing_is_read_as_the_tests_it_named() { + let mut command = std::process::Command::new("/bin/sh"); + + let _ = command.args(["-c", "printf 'suite::first: test\\nsuite::second: test\\n'"]); + + let names = listed(&mut command, Duration::from_secs(30)).expect("the listing was answered"); + + assert_eq!(names, vec!["suite::first".into(), "suite::second".into()]); + } + + /// A binary that fails while listing contributes no names at all. + /// + /// A non-zero exit means whatever it printed is not an enumeration this can trust, and a + /// partial enumeration is worse than none: a test believed not to exist is one no mutant is + /// ever run against, which turns a listing failure into a silently inflated score. + #[test] + #[cfg(unix)] + fn a_binary_that_fails_while_listing_is_not_read_as_the_tests_it_managed_to_print() { + let mut command = std::process::Command::new("/bin/sh"); + + let _ = command.args(["-c", "printf 'suite::first: test\\n'; exit 1"]); + + assert!(listed(&mut command, Duration::from_secs(30)).is_none()); + } + + /// A listing sees only the tests the user's own filters allow. + /// + /// The census turns the listing into a selection the sweep later asks for by name, so a name + /// discovered outside the user's filter is one the run would have to refuse — and, before the + /// filters were composed at all, one it would have run. The helper answers with a name only + /// when the filter reached it, so the assertion is about the arguments the listing carried. + #[test] + fn a_listing_is_narrowed_by_the_filters_the_user_gave() { + const SCRIPT: &str = "when-arg:parser|print:suite::parser_works: test"; + + let (_scratch, mut work) = crate::testing::helper_workspace("census-list-filtered", &[SCRIPT]); + let binary = crate::testing::helper(); + + work.set_test_args(vec![crate::testing::directive(SCRIPT), "parser".to_owned()]); + + assert_eq!( + list(&work, &binary), + Some(vec!["suite::parser_works".into()]), + "the user's filter should have reached the listing" + ); + + // The same binary with no filter to carry prints nothing. An empty successful output is + // inconclusive because a custom harness can ignore the listing flags and do the same. + work.set_test_args(vec![crate::testing::directive(SCRIPT)]); + + assert_eq!(list(&work, &binary), None); + } + + fn binary(path: &str) -> TestBinary { + TestBinary { + path: Utf8PathBuf::from(path), + package: "p".to_owned(), + package_id: String::new(), + target: "t".to_owned(), + manifest_dir: Utf8PathBuf::new(), + baseline: Duration::from_secs(1), + tests: None, + budget: Some(Duration::from_secs(1)), + peak: None, + memory: None, + } + } + + #[test] + fn an_uncensused_binary_is_not_the_same_as_one_that_reaches_nothing() { + // The distinction the whole feature rests on. Reading `None` as "no test reaches this" + // would report every mutant in an unlistable binary as uncovered without running one test. + let mut census = Census::default(); + + assert_eq!(census.reaching(&binary("/t/a"), 7), None); + + let _previous = census.binaries.insert( + Utf8PathBuf::from("/t/a"), + Reach { + complete: true, + ..Reach::default() + }, + ); + + assert_eq!(census.reaching(&binary("/t/a"), 7), Some(Vec::new())); + } + + /// Builds a `Reach` from test names, a site-to-indices map, and per-test durations. + fn reach_from(names: Vec>, sites: Vec<(u32, Vec)>, times: Vec) -> Reach { + let mut reach = Reach { + names, + reached: HashMap::default(), + intern: Vec::new(), + intern_index: HashMap::default(), + times, + complete: true, + }; + + for (site, indices) in sites { + let set_index = reach.intern_set(indices); + let _previous = reach.reached.insert(site, set_index); + } + reach.finish(); + + reach + } + + #[test] + fn a_site_reports_the_tests_that_reached_it_and_no_others() { + let mut census = Census::default(); + + let _previous = census.binaries.insert( + Utf8PathBuf::from("/t/a"), + reach_from( + vec!["first".into(), "second".into(), "third".into()], + vec![(7, vec![0, 2]), (9, vec![1])], + vec![Duration::from_secs(1), Duration::from_secs(2), Duration::from_secs(3)], + ), + ); + + assert_eq!(census.reaching(&binary("/t/a"), 9), Some(vec!["second"])); + assert_eq!(census.reaching(&binary("/t/a"), 11), Some(Vec::new())); + + // Two of three is past half, so the binary runs whole rather than being asked for most of + // itself by name. + assert_eq!(census.reaching(&binary("/t/a"), 7), None); + } + + #[test] + fn a_site_most_of_the_suite_reaches_is_not_worth_naming() { + // The saving is what is left after the narrowing, and naming five of nine tests to skip + // four is a longer command line for almost nothing. + let mut census = Census::default(); + + let names: Vec> = (0..9).map(|index| format!("test{index}").into()).collect(); + + let _previous = census.binaries.insert( + Utf8PathBuf::from("/t/a"), + reach_from( + names, + vec![(7, vec![0, 1, 2, 3, 4]), (9, vec![0, 1, 2, 3])], + vec![Duration::from_secs(1); 9], + ), + ); + + assert_eq!(census.reaching(&binary("/t/a"), 7), None); + assert_eq!(census.reaching(&binary("/t/a"), 9).map(|tests| tests.len()), Some(4)); + } + + /// A census of `total` tests where `reached` of them reach site 7. + fn suite_of(total: usize, reached: usize) -> Census { + let mut census = Census::default(); + + let names: Vec> = (0..total).map(|index| format!("test{index}").into()).collect(); + let indices: Vec = (0..reached).filter_map(|index| u32::try_from(index).ok()).collect(); + + let _previous = census.binaries.insert( + Utf8PathBuf::from("/t/a"), + reach_from(names, vec![(7, indices)], vec![Duration::from_secs(1); total]), + ); + + census + } + + #[test] + fn exactly_half_the_suite_is_still_worth_naming() { + // The boundary itself, which decides whether a mutant runs four tests or forty. The rule is + // `2 * reached > total`, so an even split is *not* past half and the narrowing is kept. + // Testing only well inside each side would let the comparison drift to `>=` unnoticed, + // which would silently widen every evenly-split binary to its whole suite. + assert_eq!(suite_of(10, 5).reaching(&binary("/t/a"), 7).map(|tests| tests.len()), Some(5)); + } + + #[test] + fn one_test_past_half_the_suite_is_not_worth_naming() { + // The first input on the other side of the same boundary. + assert_eq!(suite_of(10, 6).reaching(&binary("/t/a"), 7), None); + } + + #[test] + fn partial_reach_is_only_a_checked_hint() { + let path = Utf8Path::new("/t/a"); + let binary = binary("/t/a"); + let census = Census::partial(path, 7, 2, 5); + + assert_eq!( + census.selection(&binary, 7), + CensusSelection::Hinted(vec!["tests::t0", "tests::t1"]) + ); + assert!(matches!( + census.work(&binary, 7), + CensusWork::Hinted(duration) if duration == Duration::from_millis(2) + )); + assert_eq!(census.selection(&binary, 8), CensusSelection::Whole); + } + + #[test] + fn census_must_cost_less_than_its_maximum_possible_savings() { + assert!(can_repay(Duration::from_secs(1), Duration::from_secs(2))); + assert!(!can_repay(Duration::from_secs(2), Duration::from_secs(2))); + assert!(!can_repay(Duration::from_secs(3), Duration::from_secs(2))); + assert!(!can_repay(Duration::ZERO, Duration::ZERO)); + } + + #[test] + fn the_bail_out_boundary_sits_at_the_same_place_for_an_odd_suite() { + // An odd suite has no exact half, so the last kept case is the largest count under it. + assert_eq!(suite_of(11, 5).reaching(&binary("/t/a"), 7).map(|tests| tests.len()), Some(5)); + assert_eq!(suite_of(11, 6).reaching(&binary("/t/a"), 7), None); + } + + #[test] + fn a_site_only_one_test_reaches_is_always_worth_naming() { + // The case the census exists for: one test out of a large suite, which is the whole + // dividend the per-test measurement is paid for. + assert_eq!(suite_of(1000, 1).reaching(&binary("/t/a"), 7).map(|tests| tests.len()), Some(1)); + } + + #[test] + fn an_index_naming_no_test_is_dropped_rather_than_panicking() { + // Only reachable through a corrupt census, and a run that dies reading one is worse than a + // run that maps a site to one test instead of two. + let mut census = Census::default(); + + let _previous = census.binaries.insert( + Utf8PathBuf::from("/t/a"), + reach_from( + vec!["first".into(), "second".into(), "third".into(), "fourth".into()], + vec![(7, vec![0, 40])], + vec![Duration::from_secs(1); 4], + ), + ); + + assert_eq!(census.reaching(&binary("/t/a"), 7), Some(vec!["first"])); + } + + #[test] + fn estimate_uses_measured_tests_and_keeps_whole_and_uncovered_distinct() { + let mut census = Census::default(); + + let _previous = census.binaries.insert( + Utf8PathBuf::from("/t/a"), + reach_from( + vec!["first".into(), "second".into(), "third".into()], + vec![(7, vec![0]), (9, vec![0, 1])], + vec![Duration::from_secs(2), Duration::from_secs(3), Duration::from_secs(5)], + ), + ); + + assert!(matches!( + census.work(&binary("/t/a"), 7), + CensusWork::Selected(duration) if duration == Duration::from_secs(2) + )); + assert!(matches!(census.work(&binary("/t/a"), 8), CensusWork::Uncovered)); + assert!(matches!(census.work(&binary("/t/a"), 9), CensusWork::Whole)); + assert!(matches!(census.work(&binary("/t/b"), 7), CensusWork::Whole)); + } + + #[test] + fn whole_sealed_records_decode_in_the_order_they_were_written() { + assert_eq!(decode(&sealed(&[3, 9, 1])), Some(vec![3, 9, 1])); + } + + #[test] + fn a_lone_seal_is_an_empty_reach_not_a_failure() { + // The census of a test that reached no site: the runtime still sealed it, so it decodes to + // an honest empty reach rather than being mistaken for a run that never finished. + assert_eq!(decode(&sealed(&[])), Some(Vec::new())); + } + + #[test] + fn an_unsealed_census_is_refused_even_when_its_records_are_whole() { + // Aligned, parseable, every record a real site — but no seal. That is exactly the shape a + // truncated or aborted run leaves behind, so it must not be read as the reach it resembles; + // the empty file is the same story with nothing written before the process died. + let whole: Vec = [3_u32, 9, 1].iter().flat_map(|site| site.to_le_bytes()).collect(); + + assert_eq!(decode(&whole), None); + assert_eq!(decode(&[]), None); + } + + #[test] + fn a_census_cut_short_during_its_exit_write_is_refused() { + // A batch can have reached the file before a later exit write fails. With no trailing seal, + // this aligned prefix must not be mistaken for a complete one-site census. + let prefix = 73_u32.to_le_bytes(); + + assert_eq!(decode(&prefix), None); + } + + #[test] + fn a_truncated_census_is_refused_rather_than_read_as_far_as_it_goes() { + // A partial trailing record means the process died before its buffer was flushed, so sites + // are missing — and a missing site is exactly what would be misread as an unreached one. + assert_eq!(decode(&[1, 0, 0]), None); + assert_eq!(decode(&[1, 0, 0, 0, 2]), None); + } + + #[test] + fn anything_after_the_seal_discards_the_census() { + // A record past the seal means the file is not one clean prefix: a stale census was appended + // to, or two runs shared the path. Neither is this run's whole and only story. + let trailing: Vec = [SEAL, 7].iter().flat_map(|record| record.to_le_bytes()).collect(); + let doubled: Vec = [SEAL, SEAL].iter().flat_map(|record| record.to_le_bytes()).collect(); + + assert_eq!(decode(&trailing), None); + assert_eq!(decode(&doubled), None); + } + + #[test] + fn the_overflow_marker_discards_the_whole_census_even_when_it_is_sealed() { + // The runtime seals a census it overflowed just like any other clean exit, so the marker + // has to override the seal rather than be excused by it. + let bytes: Vec = [3_u32, OVERFLOW, 9, SEAL].iter().flat_map(|record| record.to_le_bytes()).collect(); + + assert_eq!(decode(&bytes), None); + } + + /// `walked` counts subprocesses that actually launched, not tests that were listed. + /// + /// A binary spoils the moment one of its tests cannot be trusted, and its remaining tests are + /// then skipped rather than run. Summing the list lengths up front would bill those skipped + /// tests as launches that never happened, overstating the census's cost. This drives one binary + /// whose second test spoils it and asserts the count is the two that ran, not the three listed. + #[test] + fn walked_counts_launches_that_happened_not_tests_that_were_listed() { + // The launcher passes the test name as an argument, and the second test exits non-zero, + // which spoils the binary. The others write a lone seal — a valid, empty census — and pass, + // but the third never runs. + const SCRIPT: &[&str] = &["when-arg:spoiler|exit:1", "write-le:GAMMA_CENSUS|4294967293"]; + + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + + // A tree of its own, so the census scratch directory `walk` derives from `base()` cannot + // collide with another test's. + let (_directory, work) = crate::testing::helper_workspace("cor13-walked", SCRIPT); + + let binary = TestBinary { + package: "subject".to_owned(), + baseline: Duration::from_millis(1), + budget: Some(Duration::from_mins(1)), + ..crate::testing::helper() + }; + + let names: Vec> = vec!["reaches".into(), "spoiler".into(), "after".into()]; + let targets = HashMap::from_iter([(binary.path.clone(), HashSet::from_iter([7]))]); + + // One worker, so the queue order is deterministic: reaches, then spoiler, then after. + let mut completed = 0; + let (mapped, walked) = walk(&work, vec![(&binary, names)], &targets, None, 1, Stall::NONE, || completed += 1); + + assert_eq!( + walked, 2, + "reaches launched, spoiler launched and spoiled the binary, after was skipped" + ); + assert_eq!(completed, 1, "the spoiled binary completed exactly once"); + assert!(mapped.is_empty(), "a spoiled binary contributes no reach at all"); + } + + #[test] + fn walking_stops_when_every_target_site_already_requires_the_whole_binary() { + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + let (_directory, work) = crate::testing::helper_workspace("census-saturation-", &[]); + let binary = TestBinary { + package: "subject".to_owned(), + ..crate::testing::helper() + }; + let names: Vec> = (0..5).map(|index| format!("tests::t{index}").into()).collect(); + let targets = HashMap::from_iter([(binary.path.clone(), HashSet::from_iter([7]))]); + let sampled = AtomicUsize::new(0); + + let (mapped, walked) = walk_with( + &work, + vec![(&binary, names)], + &targets, + 1, + Stall::NONE, + |_work, _binary, _name, _path, _stall| { + let _ = sampled.fetch_add(1, Ordering::Relaxed); + Some((vec![7], Duration::ZERO)) + }, + || false, + || {}, + ); + + let census = Census { + binaries: HashMap::from_iter(mapped), + walked, + }; + + assert_eq!(sampled.load(Ordering::Relaxed), 3); + assert_eq!(walked, 3); + assert_eq!(census.selection(&binary, 7), CensusSelection::Whole); + } + + /// A sampler that reports the same site several times for one test must count as a single + /// reach toward saturation, not several: `intern_set` removes duplicates from the final list + /// either way, so a miscount here can only be caught by the saturation threshold itself, not + /// by inspecting the finished census. If the site count were bumped once per duplicate entry + /// instead of once per newly-recorded test, this one test alone (contributing 5 duplicate + /// entries against a 5-test suite) would trip saturation on its own, cutting the walk short + /// after a single sample instead of after the 3 distinct tests genuine saturation requires. + #[test] + fn a_duplicated_site_within_one_sample_does_not_inflate_the_saturation_count() { + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + let (_directory, work) = crate::testing::helper_workspace("census-dup-site-", &[]); + let binary = TestBinary { + package: "subject".to_owned(), + ..crate::testing::helper() + }; + let names: Vec> = (0..5).map(|index| format!("tests::t{index}").into()).collect(); + let targets = HashMap::from_iter([(binary.path.clone(), HashSet::from_iter([42]))]); + let sampled = AtomicUsize::new(0); + + let (mapped, walked) = walk_with( + &work, + vec![(&binary, names)], + &targets, + 1, + Stall::NONE, + |_work, _binary, _name, _path, _stall| { + let _ = sampled.fetch_add(1, Ordering::Relaxed); + // Every sample reports site 42 five times over, as a pathological sampler might. + Some((vec![42, 42, 42, 42, 42], Duration::ZERO)) + }, + || false, + || {}, + ); + + let census = Census { + binaries: HashMap::from_iter(mapped), + walked, + }; + + // Genuine saturation (3 distinct tests out of 5) is unaffected by how many times any one + // of them repeats the same site: the walk still stops after exactly 3 samples, exactly as + // it does when each sample reports the site only once (the test above). + assert_eq!(sampled.load(Ordering::Relaxed), 3); + assert_eq!(walked, 3); + assert_eq!(census.selection(&binary, 42), CensusSelection::Whole); + } + + /// The same duplicate-site sampler under many concurrent workers: the exact sample at which + /// saturation is observed can vary with scheduling (workers may race past the check before the + /// flag is visible), but the count each worker contributes per test can never be inflated by a + /// sampler repeating one site, and the finished census must still agree that the site demands + /// the whole binary. + #[test] + fn a_duplicated_site_within_one_sample_stays_race_free_across_many_workers() { + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + let (_directory, work) = crate::testing::helper_workspace("census-dup-site-race-", &[]); + let binary = TestBinary { + package: "subject".to_owned(), + ..crate::testing::helper() + }; + let names: Vec> = (0..5).map(|index| format!("tests::t{index}").into()).collect(); + let targets = HashMap::from_iter([(binary.path.clone(), HashSet::from_iter([42]))]); + + let (mapped, walked) = walk_with( + &work, + vec![(&binary, names)], + &targets, + 8, + Stall::NONE, + |_work, _binary, _name, _path, _stall| Some((vec![42, 42, 42, 42, 42], Duration::from_millis(1))), + || false, + || {}, + ); + + let census = Census { + binaries: HashMap::from_iter(mapped), + walked, + }; + + // Bounded, not exact: concurrent workers may race a few extra samples past the check + // before the saturation flag becomes visible, but never fewer than the 3 distinct tests + // genuine saturation requires, and never more than the 5 tests that exist. + assert!( + (3..=5).contains(&walked), + "walked {walked} samples, expected saturation between 3 and 5 tests" + ); + assert_eq!(census.selection(&binary, 42), CensusSelection::Whole); + } + + /// Many workers walking the same suite reach exactly the same census as one worker would. + /// + /// Each worker now accumulates its own observations locally instead of contending for one + /// shared, per-binary lock, and the merge afterwards trusts that no two workers ever produced + /// the same `(site, test)` observation. A race in that assumption could only show up as an + /// observation silently dropped (under-counted) or duplicated (over-counted) once several workers + /// are actually racing each other over the same binary — which is exactly what running one + /// fixed, deterministic suite through a single worker and then through several checks for. + #[test] + fn many_workers_reach_the_same_census_as_one_worker() { + const TESTS: usize = 40; + const SITES: u32 = 10; + + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + let (_directory, work) = crate::testing::helper_workspace("census-race-", &[]); + let binary = TestBinary { + package: "subject".to_owned(), + ..crate::testing::helper() + }; + + let names: Vec> = (0..TESTS).map(|index| format!("tests::t{index}").into()).collect(); + let targets = HashMap::from_iter([(binary.path.clone(), (0..SITES).collect::>())]); + + // Each test deterministically reaches a handful of sites, purely as a function of its own + // index — never of which worker happens to run it or when — so any disagreement between the + // one-worker and many-worker runs can only come from the merge itself. + let sampler = |_work: &Workspace, _binary: &TestBinary, name: &str, _path: &Utf8Path, _stall: Stall| { + let index: u32 = name + .strip_prefix("tests::t") + .and_then(|digits| digits.parse().ok()) + .expect("these test names are always `tests::t`"); + let sites: Vec = (0..SITES).filter(|site| (index + site).is_multiple_of(3)).collect(); + + Some((sites, Duration::from_millis(1))) + }; + + let run = |jobs: usize| { + let (mapped, walked) = walk_with( + &work, + vec![(&binary, names.clone())], + &targets, + jobs, + Stall::NONE, + sampler, + || false, + || {}, + ); + + ( + Census { + binaries: HashMap::from_iter(mapped), + walked, + }, + walked, + ) + }; + + let (single, single_walked) = run(1); + let (many, many_walked) = run(8); + + assert_eq!(single_walked, TESTS); + assert_eq!( + single_walked, many_walked, + "the same fixed suite launches the same number of samples regardless of worker count" + ); + + for site in 0..SITES { + assert_eq!( + single.reaching(&binary, site), + many.reaching(&binary, site), + "site {site} disagrees between one worker and many" + ); + } + } + + #[test] + fn the_budget_keeps_completed_samples_as_hints_without_trusting_missing_ones() { + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + let (_directory, work) = crate::testing::helper_workspace("census-budget-", &[]); + let binary = TestBinary { + package: "subject".to_owned(), + ..crate::testing::helper() + }; + let names: Vec> = (0..5).map(|index| format!("tests::t{index}").into()).collect(); + let targets = HashMap::from_iter([(binary.path.clone(), HashSet::from_iter([7, 8]))]); + let spent = AtomicBool::new(false); + + let (mapped, walked) = walk_with( + &work, + vec![(&binary, names)], + &targets, + 1, + Stall::NONE, + |_work, _binary, _name, _path, _stall| { + spent.store(true, Ordering::Relaxed); + Some((vec![7], Duration::from_millis(20))) + }, + || spent.load(Ordering::Relaxed), + || {}, + ); + + let census = Census { + binaries: HashMap::from_iter(mapped), + walked, + }; + + assert_eq!(walked, 1); + assert_eq!(census.selection(&binary, 7), CensusSelection::Hinted(vec!["tests::t0"])); + assert_eq!(census.selection(&binary, 8), CensusSelection::Whole); + } + + #[test] + fn a_partial_stream_note_raised_by_a_census_worker_reaches_the_run() { + crate::notes::alone(|| { + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + let (_directory, work) = crate::testing::helper_workspace("census-worker-notes-", &[]); + let binary = TestBinary { + package: "subject".to_owned(), + ..crate::testing::helper() + }; + let targets = HashMap::from_iter([(binary.path.clone(), HashSet::from_iter([7]))]); + + let (_mapped, walked) = walk_with( + &work, + vec![(&binary, vec!["partial".into()])], + &targets, + 1, + Stall::NONE, + |_work, _binary, _name, _path, _stall| { + crate::notes::note("census sample produced a partial stream"); + None + }, + || false, + || {}, + ); + + assert_eq!(walked, 1); + assert_eq!(crate::notes::drain(), ["census sample produced a partial stream"]); + }); + } + + /// A clean run that leaves no census spoils its binary, rather than reading as empty. + /// + /// An absent census file is what a failed `fopen` — or a process that died before it could seal + /// — leaves behind. With mandatory sealing, "reached nothing" is a lone seal, not an absent + /// file, so the absence can only mean the census never completed. Reading it as an empty reach + /// would tell the sweep that no test convicts the mutants at these sites, turning a lost + /// measurement into a false "survived". + #[test] + fn a_clean_run_that_leaves_no_census_spoils_rather_than_reading_empty() { + let _serial = WALK_TEST.lock().expect("the census walk test lock is not poisoned"); + + // Exits cleanly and writes nothing whatsoever to the census path. + let (_directory, work) = crate::testing::helper_workspace("cor4-absent", &["exit:0"]); + + let binary = TestBinary { + package: "subject".to_owned(), + baseline: Duration::from_millis(1), + budget: Some(Duration::from_mins(1)), + ..crate::testing::helper() + }; + + let targets = HashMap::from_iter([(binary.path.clone(), HashSet::from_iter([7]))]); + let mut completed = 0; + let (mapped, walked) = walk(&work, vec![(&binary, vec!["quiet".into()])], &targets, None, 1, Stall::NONE, || { + completed += 1; + }); + + assert_eq!(walked, 1, "the one test was launched"); + assert_eq!(completed, 1, "the spoiled binary completed exactly once"); + assert!( + mapped.is_empty(), + "an absent census spoils the binary rather than reading as an empty reach" + ); + } + + /// The bytes the runtime writes for `sites`, ending with the seal that vouches the file is whole. + fn sealed(sites: &[u32]) -> Vec { + sites + .iter() + .chain(core::iter::once(&SEAL)) + .flat_map(|record| record.to_le_bytes()) + .collect() + } + + /// Identical reach sets are interned to a single allocation, reducing memory. + #[test] + fn identical_reach_sets_are_interned() { + let mut reach = Reach { + names: vec!["a".into(), "b".into(), "c".into()], + times: vec![Duration::from_secs(1); 3], + ..Reach::default() + }; + + // Two sites with the same reaching tests share one intern slot. + let idx1 = reach.intern_set(vec![0, 2]); + let idx2 = reach.intern_set(vec![2, 0]); // same set, different insertion order + + assert_eq!(idx1, idx2, "identical sets should map to the same intern index"); + assert_eq!(reach.intern.len(), 1, "only one interned set should exist"); + } + + /// Different reach sets get distinct intern slots. + #[test] + fn different_reach_sets_are_distinct() { + let mut reach = Reach { + names: vec!["a".into(), "b".into(), "c".into()], + times: vec![Duration::from_secs(1); 3], + ..Reach::default() + }; + + let idx1 = reach.intern_set(vec![0, 1]); + let idx2 = reach.intern_set(vec![0, 2]); + + assert_ne!(idx1, idx2); + assert_eq!(reach.intern.len(), 2); + } + + /// The interned reach representation produces the same `reaching` answers as raw vectors. + #[test] + fn interned_reach_answers_match_raw_vectors() { + let mut census = Census::default(); + + let _previous = census.binaries.insert( + Utf8PathBuf::from("/t/a"), + reach_from( + vec!["alpha".into(), "beta".into(), "gamma".into(), "delta".into(), "epsilon".into()], + vec![(1, vec![0, 2]), (2, vec![0, 2]), (3, vec![1])], + vec![Duration::from_millis(10); 5], + ), + ); + + // Sites 1 and 2 share a reach set; site 3 has its own. + assert_eq!(census.reaching(&binary("/t/a"), 1), Some(vec!["alpha", "gamma"])); + assert_eq!(census.reaching(&binary("/t/a"), 2), Some(vec!["alpha", "gamma"])); + assert_eq!(census.reaching(&binary("/t/a"), 3), Some(vec!["beta"])); + assert_eq!(census.reaching(&binary("/t/a"), 4), Some(Vec::new())); + + // Verify the underlying interning saves memory. + let reach = &census.binaries[&Utf8PathBuf::from("/t/a")]; + assert_eq!(reach.intern.len(), 2, "sites 1 and 2 share one interned set"); + } + + /// Streaming task resolution via cumulative offsets matches the old materialized tasks. + #[test] + fn streaming_cursor_resolves_tasks_correctly() { + // Simulate 3 binaries with [2, 3, 1] tests. + let sizes = [2_usize, 3, 1]; + let offsets: Vec = sizes + .iter() + .scan(0_usize, |acc, &size| { + let start = *acc; + *acc += size; + Some(start) + }) + .collect(); + let total: usize = sizes.iter().sum(); + + // Resolve each flat index and check the result. + let mut resolved = Vec::new(); + for at in 0..total { + let binary_at = match offsets.binary_search(&at) { + Ok(exact) => exact, + Err(after) => after.saturating_sub(1), + }; + let test_at = at - offsets[binary_at]; + resolved.push((binary_at, test_at)); + } + + assert_eq!(resolved, vec![(0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (2, 0)]); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/config.rs b/crates/cargo-gamma-lib/src/exec/config.rs new file mode 100644 index 000000000..9e61fb74c --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/config.rs @@ -0,0 +1,271 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::num::NonZero; +use core::time::Duration; +use std::thread; + +use camino::Utf8PathBuf; + +use super::cargo_options::{BuildLimits, CargoOptions}; +use super::incremental_mode::IncrementalMode; +use super::memory::MemoryPolicy; + +/// Knobs for a run. +#[expect( + clippy::struct_excessive_bools, + reason = "an options bag mirroring independent command-line flags, not a state machine" +)] +#[derive(Debug, Clone)] +pub struct Config { + /// How many mutants to test at once. + pub jobs: usize, + + /// Multiple of each test binary's baseline duration a mutant is allowed before it is called a timeout. + /// + /// The 50% margin is deliberately narrow. It exists to absorb scheduling noise, not to let a + /// mutant do more work than the original: the whole point of the timeout is that an infinite + /// loop is a detection, and a generous multiplier turns every such mutant into a long wait + /// before the same verdict. A confirmation run stands behind it — see `CONFIRM_FACTOR` — so a + /// budget that is occasionally too tight costs a re-run rather than a wrong verdict, which is + /// what makes a narrow margin affordable here. + pub test_timeout_multiplier: f64, + + /// Lower bound on the timeout for any test binary, so that a binary which finishes instantly does not produce a + /// budget so tight that scheduler noise reads as a hang. + /// + /// 20 seconds is chosen against process startup rather than against the suite: a mutant that + /// has to link, load and start a test binary pays a fixed cost that a fast suite's own elapsed + /// time says nothing about, and on a cold or loaded machine that cost alone can reach seconds. + /// The floor only ever binds for suites finishing in under about 17 seconds, where the multiplier + /// would otherwise yield a budget smaller than the startup it has to cover. + pub timeout_floor: Duration, + + /// Whether to run the baseline. Skipping it is faster and strictly less trustworthy. + pub baseline: bool, + + /// Whether a failing test is re-run with no mutant active before the kill is believed. + /// + /// Skipping the confirmation saves one run per kill — the cheapest class, since it stops at the + /// first failing test — and buys a score that counts a flaky test's failures as detections. The + /// run can no longer tell the two apart, so the flakes disappear into the kill count rather + /// than being reported. + pub confirm: bool, + + /// Whether to cut a mutant off as soon as its test binary stops reporting progress. + pub stall: bool, + + /// Multiple of the longest silence the baseline produced that a mutant is allowed. + /// + /// An order of magnitude, and deliberately not a tight one. Unlike the timeout, this is + /// calibrated against the *quietest* moment of a healthy run, which is the noisiest statistic + /// the baseline produces: it is a single maximum over one execution, so a healthy suite can + /// exceed its own measured quiet period on the next run for reasons that have nothing to do + /// with the mutant. A stall verdict is also strictly less informative than a timeout — it says + /// only that nothing was printed — so the detector is set to fire on hangs that are obvious by + /// an order of magnitude, and to leave the marginal cases to the timeout. + pub stall_factor: f64, + + /// Lower bound on the stall budget, so a suite that never goes quiet does not produce a budget + /// that scheduler noise can trip. + /// + /// Lower than the timeout floor because it bounds silence rather than work: five seconds + /// without a single line from a running test harness is already anomalous, whereas five + /// seconds of *elapsed* time is ordinary. It binds for suites whose measured quiet period is + /// under half a second, which is most fast suites. + pub stall_floor: Duration, + + /// How cargo and the test binaries are invoked. + pub cargo: CargoOptions, + + /// How long the build may take. + pub build: BuildLimits, + + /// How much memory a mutant may use, and whether anything enforces it. + pub memory: MemoryPolicy, + + /// Keep the scratch tree after the run instead of deleting it. + pub leak_dirs: bool, + + /// Where to put reusable workspace and build state. `None` uses an isolated cache base + /// outside the workspace's Cargo-configuration ancestor chain. + pub cache_dir: Option, + + /// Copy files the ignore rules exclude, not only the ones git tracks. + pub copy_ignored: bool, + + /// Packages whose tests decide a verdict. Empty means each mutant's own package. + pub test_packages: Vec, + + /// Test target name globs whose tests may decide a verdict. Empty means all of them. + pub include_tests: Vec, + + /// Test target name globs whose tests must not decide a verdict. + pub exclude_tests: Vec, + + /// Let tests from every workspace package judge mutants they can reach. + pub test_workspace: bool, + + /// Whether to run every selected test in a reachable binary instead of selecting cases by reachability. + /// + /// False by default: deterministic suites are measured once and each mutant runs only the + /// specific test cases that can reach it. True is the conservative fallback for suites whose + /// reachability depends on threads, clocks, randomness or hash iteration order. + pub whole_test_binaries: bool, + + /// Whether to run test binaries through `cargo nextest`. + pub nextest: bool, + + /// How an incremental run reuses state from the previous run. + pub incremental: IncrementalMode, +} + +impl Default for Config { + fn default() -> Self { + Self { + jobs: resolve_jobs(None), + test_timeout_multiplier: 1.5, + timeout_floor: Duration::from_secs(20), + baseline: true, + confirm: true, + stall: true, + stall_factor: 10.0, + stall_floor: Duration::from_secs(5), + cargo: CargoOptions::default(), + build: BuildLimits::default(), + memory: MemoryPolicy::default(), + leak_dirs: false, + cache_dir: None, + copy_ignored: false, + test_packages: Vec::new(), + include_tests: Vec::new(), + exclude_tests: Vec::new(), + test_workspace: false, + whole_test_binaries: false, + nextest: false, + incremental: IncrementalMode::default(), + } + } +} + +/// Resolves mutation parallelism, preserving an explicitly requested width. +/// +/// Bounded above by what the interrupt registry can watch. Every worker holds one live child, and +/// a child the registry has no slot for is one a cancelled run cannot kill — so a `--jobs` beyond +/// that ceiling would not buy parallelism, it would buy processes that survive `Ctrl-C`. The +/// ceiling is far above any machine's useful width, so clamping to it costs nothing real. +pub(crate) fn resolve_jobs(jobs: Option) -> usize { + let wanted = jobs.unwrap_or_else(|| default_jobs(available_parallelism())); + + wanted.min(watchable()) +} + +/// The parallelism the host makes available to this process. +/// +/// This is deliberately separate from [`resolve_jobs`]: the latter adds a worker by default and +/// honours an explicit `--jobs`, neither of which changes the machine's available cores. +pub(crate) fn available_parallelism() -> usize { + thread::available_parallelism().map_or(1, NonZero::get) +} + +/// How many concurrent children the platform can still account for when the run is cancelled. +/// +/// On Unix that is the interrupt registry's slot count: containment puts every child in a process +/// group of its own, and a group with no slot is one the handler's sweep never reaches. On Windows +/// each child is held by a job object that kills its subtree when the handle closes however the +/// parent dies, so there is no shared table to run out of and nothing to bound. +#[cfg(unix)] +const fn watchable() -> usize { + cargo_gamma_process::capacity() +} + +#[cfg(not(unix))] +const fn watchable() -> usize { + usize::MAX +} + +const fn default_jobs(available: usize) -> usize { + available.saturating_add(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_default_timeout_never_falls_below_the_floor() { + let config = Config::default(); + let scaled = Duration::from_millis(1).mul_f64(config.test_timeout_multiplier); + + assert_eq!(scaled.max(config.timeout_floor), config.timeout_floor); + } + + /// The programmatic default is the 50% margin the command default and the documentation promise. + #[test] + fn the_default_timeout_multiplier_is_one_and_a_half() { + // Bit-exact: the default is an exactly representable value, so this must fail on a wrong + // default rather than slip under an approximate tolerance — and comparing the bits keeps + // the pedantic `float_cmp` lint quiet, as the tool does elsewhere by wrapping the value. + assert_eq!(Config::default().test_timeout_multiplier.to_bits(), 1.5_f64.to_bits()); + } + + #[test] + fn the_default_timeout_scales_with_a_slow_baseline() { + let config = Config::default(); + let baseline = Duration::from_mins(1); + let scaled = baseline.mul_f64(config.test_timeout_multiplier); + + assert_eq!(scaled.max(config.timeout_floor), Duration::from_secs(90)); + } + + #[test] + fn one_processor_defaults_to_two_jobs() { + assert_eq!(default_jobs(1), 2); + } + + #[test] + fn the_default_job_count_saturates_at_usize_max() { + assert_eq!(default_jobs(usize::MAX), usize::MAX); + } + + #[test] + fn an_explicit_job_count_is_unchanged() { + assert_eq!(resolve_jobs(Some(0)), 0); + assert_eq!(resolve_jobs(Some(watchable())), watchable()); + } + + /// A width the interrupt registry could not account for is refused rather than granted. + /// + /// The leak it prevents: every worker holds one live child in a process group of its own, and a + /// group the registry has no slot for is one the interrupt handler's sweep never visits — so + /// past the ceiling, `Ctrl-C` kills the run and leaves the excess children behind holding the + /// scratch trees that fail the next run. + #[test] + fn a_job_count_beyond_what_can_be_watched_is_clamped() { + assert_eq!(resolve_jobs(Some(usize::MAX)), watchable()); + assert!(resolve_jobs(Some(watchable().saturating_add(1))) <= watchable()); + } + + /// Whatever the machine's width, the default cannot outrun the registry either. + #[test] + fn the_default_job_count_is_also_watchable() { + assert!(resolve_jobs(None) <= watchable()); + } + + /// Memory control is off until it is asked for. + #[test] + fn memory_control_is_enforced_by_default() { + // On the same footing as the wall-clock timeout: a mutation can turn bounded allocation + // into unbounded allocation, and the user who most needs protecting from that is the one + // who never thought to ask. The ceiling is derived from each binary's own baseline peak, so + // it is a statement about this suite rather than a guess about suites in general. + let config = Config::default(); + + assert!(config.memory.measuring()); + assert!(config.memory.enforcing()); + assert!(config.memory.ceiling(Some(1024 * 1024), true).is_some()); + + // Nobody asked for it, so a host that cannot deliver it degrades rather than refusing. + assert!(!config.memory.insisted()); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/copy.rs b/crates/cargo-gamma-lib/src/exec/copy.rs new file mode 100644 index 000000000..36d71f4f9 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/copy.rs @@ -0,0 +1,1009 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Copying a source tree into the scratch directory. +//! +//! Nothing here checks for interruption. The copy precedes the run's first contained spawn, so no +//! signal handler is armed yet and Ctrl-C takes effect immediately by the default disposition. The +//! scratch tree is left behind for the next run to clear, and the operating system releases its +//! lock when the process dies. + +use core::sync::atomic::{AtomicBool, Ordering}; +use std::fs::{self, File, FileTimes}; +use std::io::ErrorKind; +use std::process::{Command, Stdio}; +use std::sync::Mutex; +use std::time::SystemTime; + +use camino::{Utf8Path, Utf8PathBuf}; +use ignore::{WalkBuilder, WalkState}; + +use crate::Result; +use crate::error::{Error, error}; + +/// Version control directories, which are large, hold nothing a build reads, and are actively +/// hazardous in a tree a tool is rewriting: a stray command run from the scratch copy could commit +/// instrumented source over the user's work. +pub(super) const VCS_DIRS: [&str; 7] = [".git", ".hg", ".bzr", ".svn", "_darcs", ".jj", ".pijul"]; + +/// The VCS metadata visible from a source or scratch location. +/// +/// The copy deliberately omits every [`VCS_DIRS`] entry, so an external scratch location is only +/// semantically equivalent when it can still find the same metadata through one of its ancestors. +pub(super) fn visible_vcs_metadata(path: &Utf8Path) -> Vec { + let mut found = Vec::new(); + let mut directory = crate::paths::physical(path).unwrap_or_else(|_unresolved| path.to_path_buf()); + + loop { + for name in VCS_DIRS { + let marker = directory.join(name); + + if fs::symlink_metadata(marker.as_std_path()).is_ok() { + found.push(crate::paths::physical(&marker).unwrap_or(marker)); + } + } + + let Some(parent) = directory.parent() else { + break; + }; + + if parent == directory { + break; + } + + directory = parent.to_path_buf(); + } + + found.sort(); + found.dedup(); + found +} + +/// Whether copy-on-write cloning is still worth attempting. +/// +/// A filesystem either supports reflinks or does not, so one failure settles it for the whole +/// process and the rest of the copy goes straight to a byte-for-byte read. The check is a latch +/// rather than a per-file probe because the failure is not always reported as +/// [`std::io::ErrorKind::Unsupported`] — some platforms return a plain permission or argument +/// error — so there is nothing reliable to match on. +pub(super) static REFLINK_WORKS: AtomicBool = AtomicBool::new(true); + +/// How much of a tree a copy takes. +#[derive(Debug, Clone, Copy, Default)] +pub(super) struct CopyOptions { + /// Whether files version control ignores are copied along with the rest. + /// + /// The escape hatch for a tree whose `.gitignore` excludes something the build genuinely reads + /// — a generated module, a fixture — where editing a shared ignore file to suit one tool is + /// not an option. + pub(super) copy_ignored: bool, +} + +/// Copies a source tree, skipping build output, version control and the scratch directory itself. +/// +/// `skip` is the directory the copy is being written under when that sits inside the source tree, +/// which is the default arrangement; without it the copy would try to copy itself. +#[cfg(test)] +pub(super) fn copy_tree(from: &Utf8Path, to: &Utf8Path, skip: &Utf8Path) -> Result<()> { + copy_tree_with(from, to, skip, CopyOptions::default()) +} + +/// Copies a source tree, taking what a run asked for into account. +/// +/// Untracked files that version control ignores are not copied unless [`CopyOptions::copy_ignored`] +/// says otherwise. A tree's own `.gitignore` describes exactly the files that are regenerable or +/// machine-local, and skipping them is usually the difference between copying a source tree and +/// copying a source tree plus everything ever built in it. +/// +/// Tracked files are copied whatever the ignore rules say. Git itself treats an ignore rule as +/// advice about what to add, not about what to keep, so a tracked file matching one is still part +/// of the tree — and mutant discovery, which walks the tree rather than asking git, finds and +/// mutates it. Leaving it out of the copy would fail the build over a file the real tree has. +pub(super) fn copy_tree_with(from: &Utf8Path, to: &Utf8Path, skip: &Utf8Path, options: CopyOptions) -> Result<()> { + fs::create_dir_all(to.as_std_path()).map_err(|cause| error!("could not create the scratch tree at `{to}`").caused_by(cause))?; + + let failure: Mutex> = Mutex::new(None); + + let mut builder = WalkBuilder::new(from.as_std_path()); + + let _builder = builder + // A build reads `.cargo/config.toml`, `.rustfmt.toml` and friends, none of which are + // hidden in any sense that matters here. + .hidden(false) + // Only ignore files inside the tree have any say. Reading them from parent directories + // means a checkout nested under a directory whose `.gitignore` says `*` copies as nothing + // at all, and the resulting empty tree fails the build for reasons nobody can see. + .parents(false) + // `.gitignore` describes what git would restore, which is only meaningful in something git + // is actually tracking. Outside a repository the same file is a leftover. + .require_git(true) + .git_ignore(!options.copy_ignored) + .git_exclude(!options.copy_ignored) + // A user's global ignore file describes their machine, not this project, and a rule in it + // would silently change what a shared tree copies to. + .git_global(false) + // `.ignore` is a search convention. It routinely excludes vendored or generated code that + // a build genuinely needs. + .ignore(false) + // A link is recreated rather than followed, so there is nothing to descend into and no + // cycle to guard against. + .follow_links(false); + + let root = from.to_owned(); + let destination = to.to_owned(); + let excluded = skip.to_owned(); + + builder.build_parallel().run(|| { + let root = root.clone(); + let destination = destination.clone(); + let excluded = excluded.clone(); + let failure = &failure; + + Box::new(move |entry| { + let entry = match entry { + Ok(entry) => entry, + + // An unreadable directory or a broken entry is reported rather than skipped. A + // file missing from the copy produces a build failure naming something unrelated, + // which is far harder to act on than the permission error that caused it. + Err(cause) => { + record(failure, error!("could not read the source tree").caused_by(cause)); + + return WalkState::Quit; + } + }; + + let Some(source) = Utf8Path::from_path(entry.path()) else { + record( + failure, + error!("`{}` is not valid UTF-8 and cannot be copied", entry.path().display()), + ); + + return WalkState::Quit; + }; + + // The walker yields the root itself first, which is the destination, not something to + // put inside it. + let Ok(relative) = source.strip_prefix(&root) else { + return WalkState::Continue; + }; + + if relative.as_str().is_empty() { + return WalkState::Continue; + } + + if is_pruned(source, relative, &excluded) { + return WalkState::Skip; + } + + match copy_entry(source, &destination.join(relative)) { + Ok(()) => WalkState::Continue, + Err(cause) => { + record(failure, cause); + + WalkState::Quit + } + } + }) + }); + + match failure.into_inner() { + Ok(Some(cause)) => return Err(cause), + Ok(None) => {} + + // The lock is only ever held while recording a failure, so it can only be poisoned by a + // panic in this crate, and the panic itself is the thing worth reporting. + Err(poisoned) => { + if let Some(cause) = poisoned.into_inner() { + return Err(cause); + } + } + } + + copy_tracked(from, to, skip) +} + +/// Copies the files git tracks that the walk left behind. +/// +/// Only files an ignore rule hid are still missing at this point, which is a handful in the trees +/// where it happens and none at all in the rest, so each candidate is settled by a single stat +/// rather than by re-deriving what the walk decided. +fn copy_tracked(from: &Utf8Path, to: &Utf8Path, skip: &Utf8Path) -> Result<()> { + let Some(tracked) = tracked_files(from)? else { + return Ok(()); + }; + + for relative in tracked { + if is_pruned_anywhere(from, &relative, skip) { + continue; + } + + let source = from.join(&relative); + let destination = to.join(&relative); + + if fs::symlink_metadata(destination.as_std_path()).is_ok() { + continue; + } + + // A file that is in the index but not on disk — staged for deletion, or a submodule's + // directory standing in for a checkout that was never made — is not something the real + // tree builds against either. + if fs::symlink_metadata(source.as_std_path()).is_err() { + continue; + } + + copy_entry(&source, &destination)?; + } + + Ok(()) +} + +/// Every path git tracks under `root`, or `None` when there is no repository to ask. +/// +/// Asking git rather than reading the index directly keeps this working for a worktree, a +/// submodule and a repository whose index format is newer than any library would understand. A +/// directory that is not a repository, or a machine with no git at all, gets `None` and the +/// ignore walk's answer stands on its own. +pub(super) fn tracked_files(root: &Utf8Path) -> Result>> { + let output = Command::new("git") + .arg("-C") + .arg(root.as_std_path()) + .args(["ls-files", "-z"]) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .ok(); + + let Some(output) = output else { + return Ok(None); + }; + + if !output.status.success() { + return Ok(None); + } + + let mut tracked = Vec::new(); + + for name in output.stdout.split(|byte| *byte == 0).filter(|name| !name.is_empty()) { + let name = str::from_utf8(name) + .map_err(|cause| error!("git reported a tracked path that is not valid UTF-8 in `{root}`").caused_by(cause))?; + + tracked.push(Utf8PathBuf::from(name)); + } + + Ok(Some(tracked)) +} + +/// Returns whether any directory on the way to `relative` is one the copy leaves out. +/// +/// The walk prunes a directory and never looks inside it; a path named by git arrives whole, so +/// each of its ancestors has to be put to the same question. +fn is_pruned_anywhere(root: &Utf8Path, relative: &Utf8Path, excluded: &Utf8Path) -> bool { + let mut prefix = Utf8PathBuf::new(); + + for component in relative.components() { + prefix.push(component); + + if is_pruned(&root.join(&prefix), &prefix, excluded) { + return true; + } + } + + false +} + +/// Records the first failure, which is the one reported. +/// +/// Later failures are usually consequences of the first — a walk that hit an unreadable directory +/// tends to hit its siblings too — and the walk is stopping regardless. +fn record(failure: &Mutex>, cause: Error) { + if let Ok(mut held) = failure.lock() + && held.is_none() + { + *held = Some(cause); + } +} + +/// Returns whether an entry and everything under it should be left out of the copy. +pub(super) fn is_pruned(source: &Utf8Path, relative: &Utf8Path, excluded: &Utf8Path) -> bool { + if source == excluded { + return true; + } + + let Some(name) = relative.file_name() else { + return false; + }; + + if VCS_DIRS.contains(&name) { + return true; + } + + // Build output is expensive to copy and regenerated anyway. At the top of the tree the name + // settles it; deeper down it does not, since `src/target/` is an ordinary module directory, so + // a nested one has to prove itself by carrying the tag cargo writes into every target + // directory it owns. + name == "target" && (relative.parent() == Some(Utf8Path::new("")) || source.join("CACHEDIR.TAG").as_std_path().exists()) +} + +/// Copies one entry, preserving what it is. +fn copy_entry(source: &Utf8Path, destination: &Utf8Path) -> Result<()> { + let metadata = fs::symlink_metadata(source.as_std_path()).map_err(|cause| error!("could not read `{source}`").caused_by(cause))?; + + if metadata.is_dir() { + return fs::create_dir_all(destination.as_std_path()).map_err(|cause| error!("could not create `{destination}`").caused_by(cause)); + } + + // The parallel walk visits each directory as its own entry and creates it there, so by the time + // a file is reached its parent almost always exists — placing it straight away keeps the + // per-file `create_dir_all` (a `mkdir` returning `EEXIST` and an `is_dir` `stat`) off the common + // path. The walk has no ordering between a file and its directory, though, so one case is left: + // a file reached first. Only then is the parent created and the copy retried once, which keeps + // the parallel-walk safety the unconditional call had — `create_dir_all` is idempotent, so a + // parent another thread finished in the meantime is not a conflict, and a parent that is a plain + // file surfaces here as the same "could not create" the unconditional call raised. + if place(&metadata, source, destination).is_err() { + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent.as_std_path()).map_err(|cause| error!("could not create `{parent}`").caused_by(cause))?; + } + + return place(&metadata, source, destination); + } + + Ok(()) +} + +/// Copies one non-directory entry — a symlink verbatim, anything else as a file — assuming its +/// parent already exists. +fn place(metadata: &fs::Metadata, source: &Utf8Path, destination: &Utf8Path) -> Result<()> { + if metadata.is_symlink() { + copy_symlink(source, destination) + } else { + copy_file(source, destination) + } +} + +/// Recreates a symlink rather than copying what it points at. +/// +/// Following the link instead would materialize its target inside the scratch tree, which for a +/// link pointing outside the workspace — a home directory, a data mount — means copying that +/// wholesale. The link is reproduced verbatim, including a relative or broken one, since a build +/// that worked with it in the original tree is the thing being reproduced. +fn copy_symlink(source: &Utf8Path, destination: &Utf8Path) -> Result<()> { + let target = fs::read_link(source.as_std_path()).map_err(|cause| error!("could not read the link `{source}`").caused_by(cause))?; + + #[cfg(unix)] + let created = std::os::unix::fs::symlink(&target, destination.as_std_path()); + + #[cfg(windows)] + let created = if source + .parent() + .map_or_else(|| target.is_dir(), |parent| parent.as_std_path().join(&target).is_dir()) + { + std::os::windows::fs::symlink_dir(&target, destination.as_std_path()) + } else { + std::os::windows::fs::symlink_file(&target, destination.as_std_path()) + }; + + created.map_err(|cause| error!("could not recreate the link `{destination}`").caused_by(cause)) +} + +/// Copies one file, cloning it if the filesystem can. +fn copy_file(source: &Utf8Path, destination: &Utf8Path) -> Result<()> { + if reflink_supported() && REFLINK_WORKS.load(Ordering::Relaxed) { + match reflink_copy::reflink(source.as_std_path(), destination.as_std_path()) { + Ok(()) => { + freshen(destination); + + return Ok(()); + } + // A missing parent is the parallel walk reaching a file before its directory, not a + // filesystem that cannot clone. Surface it so `copy_entry` creates the parent and + // retries, and leave the latch alone: one such race must not force the rest of the copy + // onto the byte-for-byte path. + Err(cause) if cause.kind() == ErrorKind::NotFound => { + return Err(error!("could not copy `{source}` to `{destination}`").caused_by(cause)); + } + Err(_unsupported) => { + REFLINK_WORKS.store(false, Ordering::Relaxed); + + // A failed clone can leave a partial destination behind; remove it so the fallback + // starts from the caller's precondition that no destination entry exists. + let _removed = fs::remove_file(destination.as_std_path()); + } + } + } + + let _bytes = fs::copy(source.as_std_path(), destination.as_std_path()) + .map_err(|cause| error!("could not copy `{source}` to `{destination}`").caused_by(cause))?; + + Ok(()) +} + +/// Whether cloning is worth trying on this platform at all. +/// +/// On musl the copy syscall the crate reaches for is not the standard one, and asking for a clone +/// there fails in ways that are not worth distinguishing from a filesystem that cannot do it. +const fn reflink_supported() -> bool { + !cfg!(target_env = "musl") +} + +/// Stamps a cloned file with the current time. +/// +/// A clone preserves the source's modification time on some platforms, which leaves a fresh copy +/// looking arbitrarily old. On macOS that is not merely cosmetic: the system prunes files under +/// `/var/folders` once they pass three days, so a scratch tree cloned from an old checkout can +/// have files deleted out from under a run that is still using them. +fn freshen(destination: &Utf8Path) { + if let Ok(file) = File::options().write(true).open(destination.as_std_path()) { + let _stamped = file.set_times(FileTimes::new().set_modified(SystemTime::now())); + } +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + + fn tree() -> (tempfile::TempDir, Utf8PathBuf, Utf8PathBuf) { + let temporary = tempfile::tempdir().unwrap(); + let from = Utf8PathBuf::from_path_buf(temporary.path().join("from")).unwrap(); + let to = Utf8PathBuf::from_path_buf(temporary.path().join("to")).unwrap(); + + fs::create_dir_all(from.as_std_path()).unwrap(); + + (temporary, from, to) + } + + /// Runs one git command in `root`, reporting whether it worked. + /// + /// The tests that need a repository are skipped rather than failed on a machine without git, + /// since what they cover is a git behaviour and there is nothing to check without it. + fn git(root: &Utf8Path, arguments: &[&str]) -> bool { + Command::new("git") + .arg("-C") + .arg(root.as_std_path()) + .args(arguments) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } + + /// Builds a repository holding one tracked and one untracked file, both matching an ignore + /// rule. Returns `false` when git is unavailable. + fn ignored_repository(from: &Utf8Path) -> bool { + if !git(from, &["init"]) { + return false; + } + + fs::write(from.join(".gitignore").as_std_path(), "**/[Bb]in/*\n").unwrap(); + fs::create_dir_all(from.join("src").join("bin").as_std_path()).unwrap(); + fs::write(from.join("src").join("bin").join("helper.rs").as_std_path(), "fn tracked() {}").unwrap(); + fs::write(from.join("src").join("bin").join("scratch.rs").as_std_path(), "fn untracked() {}").unwrap(); + + git(from, &["add", "-f", "src/bin/helper.rs"]) + } + + /// A file git tracks is part of the tree whatever an ignore rule says about it, and mutant + /// discovery walks the tree rather than asking git, so leaving it out of the copy fails the + /// build over a file the real tree has. + #[test] + fn a_tracked_file_is_copied_even_when_an_ignore_rule_matches_it() { + let (_temporary, from, to) = tree(); + + if !ignored_repository(&from) { + return; + } + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert_eq!( + fs::read_to_string(to.join("src").join("bin").join("helper.rs").as_std_path()).unwrap(), + "fn tracked() {}" + ); + } + + #[cfg(unix)] + #[test] + fn a_tracked_ignored_file_with_a_non_utf8_name_is_reported() { + use std::os::unix::ffi::OsStrExt as _; + + let (_temporary, from, to) = tree(); + + if !git(&from, &["init"]) { + return; + } + + let name = std::ffi::OsStr::from_bytes(b"ignored-\xff.rs"); + fs::write(from.join(".gitignore").as_std_path(), "ignored-*\n").expect("ignore rule"); + fs::write(from.as_std_path().join(name), "fn tracked() {}").expect("tracked file"); + + let added = Command::new("git") + .arg("-C") + .arg(from.as_std_path()) + .args(["add", "-f", "--"]) + .arg(name) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()); + if !added { + return; + } + + let error = copy_tree(&from, &to, Utf8Path::new("/nowhere")).expect_err("the path cannot be represented in the scratch tree"); + + assert!(error.to_string().contains("not valid UTF-8"), "{error}"); + } + + #[test] + fn an_untracked_ignored_file_is_not_copied() { + let (_temporary, from, to) = tree(); + + if !ignored_repository(&from) { + return; + } + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert!(!to.join("src").join("bin").join("scratch.rs").as_std_path().exists()); + } + + #[test] + fn copying_ignored_files_takes_the_untracked_ones_too() { + let (_temporary, from, to) = tree(); + + if !ignored_repository(&from) { + return; + } + + copy_tree_with(&from, &to, Utf8Path::new("/nowhere"), CopyOptions { copy_ignored: true }).unwrap(); + + assert!(to.join("src").join("bin").join("helper.rs").as_std_path().exists()); + assert_eq!( + fs::read_to_string(to.join("src").join("bin").join("scratch.rs").as_std_path()).unwrap(), + "fn untracked() {}" + ); + } + + /// Outside a repository there is no index to consult and no ignore file worth obeying, so + /// everything is copied and asking git about it is not allowed to fail the copy. + #[test] + fn a_directory_that_is_not_a_repository_is_copied_whole() { + let (_temporary, from, to) = tree(); + + fs::write(from.join(".gitignore").as_std_path(), "**/[Bb]in/*\n").unwrap(); + fs::create_dir_all(from.join("src").join("bin").as_std_path()).unwrap(); + fs::write(from.join("src").join("bin").join("helper.rs").as_std_path(), "fn f() {}").unwrap(); + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert!(to.join("src").join("bin").join("helper.rs").as_std_path().exists()); + } + + /// A tracked file under a directory the copy prunes — build output, or the scratch tree itself + /// — is still left out; being in the index says nothing about whether copying it is safe. + #[test] + fn a_tracked_file_under_a_pruned_directory_is_still_skipped() { + assert!(is_pruned_anywhere( + Utf8Path::new("/workspace"), + Utf8Path::new("target/keep.rs"), + Utf8Path::new("/workspace/scratch"), + )); + + assert!(is_pruned_anywhere( + Utf8Path::new("/workspace"), + Utf8Path::new("scratch/tree/keep.rs"), + Utf8Path::new("/workspace/scratch"), + )); + + assert!(!is_pruned_anywhere( + Utf8Path::new("/workspace"), + Utf8Path::new("src/keep.rs"), + Utf8Path::new("/workspace/scratch"), + )); + } + + /// Nothing outside a repository is tracked, and the answer has to be that rather than an + /// error: a source tree that is not under version control is an ordinary thing to mutate. + #[test] + fn asking_a_non_repository_for_its_tracked_files_yields_nothing() { + let (_temporary, from, _to) = tree(); + + assert!(tracked_files(&from).expect("not being a repository is not an error").is_none()); + } + + #[test] + fn build_output_and_version_control_are_skipped() { + let (_temporary, from, to) = tree(); + + for directory in ["src", "target", ".git", ".jj", "_darcs"] { + fs::create_dir_all(from.join(directory).as_std_path()).unwrap(); + fs::write(from.join(directory).join("f").as_std_path(), "x").unwrap(); + } + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert!(to.join("src").join("f").as_std_path().exists()); + + for skipped in ["target", ".git", ".jj", "_darcs"] { + assert!(!to.join(skipped).as_std_path().exists(), "{skipped} was copied"); + } + } + + #[test] + fn a_nested_target_module_survives() { + // `src/target/` is an ordinary module name. Only a directory carrying cargo's tag is + // build output. + let (_temporary, from, to) = tree(); + + fs::create_dir_all(from.join("src").join("target").as_std_path()).unwrap(); + fs::write(from.join("src").join("target").join("mod.rs").as_std_path(), "fn f() {}").unwrap(); + + fs::create_dir_all(from.join("nested").join("target").as_std_path()).unwrap(); + fs::write(from.join("nested").join("target").join("CACHEDIR.TAG").as_std_path(), "").unwrap(); + fs::write(from.join("nested").join("target").join("junk").as_std_path(), "x").unwrap(); + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert!(to.join("src").join("target").join("mod.rs").as_std_path().exists()); + assert!(!to.join("nested").join("target").join("junk").as_std_path().exists()); + } + + #[test] + fn nested_directories_are_recreated() { + let (_temporary, from, to) = tree(); + + fs::create_dir_all(from.join("a").join("b").join("c").as_std_path()).unwrap(); + fs::write(from.join("a").join("b").join("c").join("deep.rs").as_std_path(), "fn f() {}").unwrap(); + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert_eq!( + fs::read_to_string(to.join("a").join("b").join("c").join("deep.rs").as_std_path()).unwrap(), + "fn f() {}" + ); + } + + #[test] + fn the_scratch_directory_is_not_copied_into_itself() { + let (_temporary, from, to) = tree(); + let skip = from.join("scratch"); + + fs::create_dir_all(skip.as_std_path()).unwrap(); + fs::write(skip.join("f").as_std_path(), "x").unwrap(); + fs::create_dir_all(from.join("src").as_std_path()).unwrap(); + + copy_tree(&from, &to, &skip).unwrap(); + + assert!(!to.join("scratch").as_std_path().exists()); + assert!(to.join("src").as_std_path().exists()); + } + + #[cfg(unix)] + #[test] + fn a_symlink_is_recreated_rather_than_followed() { + // Following it would materialize whatever it points at, which for a link out of the + // workspace means copying an arbitrary part of the filesystem. + let (_temporary, from, to) = tree(); + let outside = from.parent().unwrap().join("outside"); + + fs::create_dir_all(outside.as_std_path()).unwrap(); + fs::write(outside.join("secret").as_std_path(), "x").unwrap(); + + std::os::unix::fs::symlink(outside.as_std_path(), from.join("link").as_std_path()).unwrap(); + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + let copied = to.join("link"); + + assert!(fs::symlink_metadata(copied.as_std_path()).unwrap().is_symlink()); + assert_eq!(fs::read_link(copied.as_std_path()).unwrap(), outside.as_std_path()); + } + + #[cfg(windows)] + #[test] + fn a_relative_directory_symlink_is_recreated_as_a_directory_link() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8Path::from_path(temporary.path()).expect("utf8"); + let from = root.join("from"); + let to = root.join("to"); + + fs::create_dir_all(from.join("sub").as_std_path()).expect("sub"); + fs::create_dir_all(from.join("shared").as_std_path()).expect("shared"); + fs::write(from.join("shared/marker").as_std_path(), "present").expect("marker"); + + // Windows stores the target verbatim, and the object manager requires backslashes. + if let Err(cause) = std::os::windows::fs::symlink_dir(r"..\shared", from.join("sub/link").as_std_path()) { + if cause.kind() == ErrorKind::PermissionDenied { + return; + } + + panic!("create source link: {cause}"); + } + + copy_tree(&from, &to, &to).expect("copy"); + + let copied = to.join("sub/link"); + assert!(fs::symlink_metadata(copied.as_std_path()).expect("metadata").is_symlink()); + assert!(copied.as_std_path().is_dir(), "the copied directory link is not traversable"); + + // Reading through the link proves it resolves inside the copied tree. + let through = fs::read_to_string(copied.join("marker").as_std_path()).expect("read through the copied link"); + + assert_eq!(through, "present"); + } + + #[cfg(unix)] + #[test] + fn a_symlink_cycle_does_not_hang_the_copy() { + // The old hand-rolled walk followed links and needed a depth cap to survive this. + let (_temporary, from, to) = tree(); + + fs::create_dir_all(from.join("a").as_std_path()).unwrap(); + std::os::unix::fs::symlink(from.as_std_path(), from.join("a").join("loop").as_std_path()).unwrap(); + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert!(fs::symlink_metadata(to.join("a").join("loop").as_std_path()).unwrap().is_symlink()); + } + + #[test] + fn a_deep_tree_is_copied_whole() { + // The old walk stopped at 64 levels and returned success, losing everything below. + let (_temporary, from, to) = tree(); + let mut deep = from.clone(); + + for _level in 0..80 { + deep = deep.join("d"); + } + + fs::create_dir_all(deep.as_std_path()).unwrap(); + fs::write(deep.join("bottom.rs").as_std_path(), "fn f() {}").unwrap(); + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + let landed = to.join(deep.strip_prefix(&from).unwrap()); + + assert!(landed.join("bottom.rs").as_std_path().exists()); + } + + #[test] + fn an_empty_directory_is_preserved() { + // A build script can expect a directory to exist without anything being in it. + let (_temporary, from, to) = tree(); + + fs::create_dir_all(from.join("empty").as_std_path()).unwrap(); + + copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap(); + + assert!(to.join("empty").as_std_path().is_dir()); + } + + #[test] + fn a_missing_source_tree_is_reported() { + let (_temporary, from, to) = tree(); + let missing = from.join("absent"); + + let cause = copy_tree(&missing, &to, Utf8Path::new("/nowhere")).unwrap_err(); + + assert!(cause.to_string().contains("could not read the source tree"), "{cause}"); + } + + #[test] + fn a_destination_entry_that_cannot_be_replaced_is_reported() { + let (_temporary, from, to) = tree(); + + fs::write(from.join("file").as_std_path(), "source").unwrap(); + fs::create_dir_all(to.join("file").as_std_path()).unwrap(); + + let cause = copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap_err(); + + // The parallel walk records the first failed copy rather than continuing and reporting a + // later build error about whatever was missing from the scratch tree. + assert!(cause.to_string().contains("could not copy"), "{cause}"); + } + + /// When several entries fail at nearly the same time — plausible on a multi-core machine, since + /// the walk copies in parallel — only the first failure recorded is kept and reported; every + /// later one is a duplicate of essentially the same problem, and showing several at once would + /// bury the one useful message under repeats of it. + #[test] + fn only_the_first_of_several_concurrent_failures_is_kept() { + let (_temporary, from, to) = tree(); + + fs::create_dir_all(to.as_std_path()).unwrap(); + + // Enough independent blocked entries, spread across their own directories, that a parallel + // walk has a real chance of two of them failing before either sees the other quit. + for index in 0..32 { + let name = format!("blocked-{index}"); + + fs::create_dir_all(from.join(&name).as_std_path()).unwrap(); + fs::write(from.join(&name).join("file").as_std_path(), "source").unwrap(); + fs::create_dir_all(to.join(&name).join("file").as_std_path()).unwrap(); + } + + let cause = copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap_err(); + + assert!(cause.to_string().contains("could not copy"), "{cause}"); + } + + /// `record` keeps only the earliest failure so a user chasing a build error sees the actual + /// cause rather than whichever unrelated sibling entry the parallel walk happened to reach + /// last; calling it directly, rather than hoping a race lands the right way, is the only way + /// to prove that second and later calls are genuinely no-ops rather than merely untested. + #[test] + fn a_second_recorded_failure_does_not_replace_the_first() { + let failure: Mutex> = Mutex::new(None); + + record(&failure, error!("first failure")); + record(&failure, error!("second failure")); + + let held = failure.into_inner().unwrap(); + let cause = held.expect("a failure was recorded"); + + assert!(cause.to_string().contains("first failure"), "{cause}"); + } + + #[test] + fn pruning_an_entry_without_a_file_name_is_not_a_match() { + // The root entry has no real file name after it is stripped, and must not be mistaken for + // a version-control or target directory. + assert!(!is_pruned( + Utf8Path::new("/workspace"), + Utf8Path::new(""), + Utf8Path::new("/elsewhere"), + )); + } + + #[test] + fn freshening_a_missing_file_is_harmless() { + let (_temporary, _from, to) = tree(); + let file = to.join("copied"); + + fs::create_dir_all(to.as_std_path()).unwrap(); + fs::write(file.as_std_path(), "x").unwrap(); + freshen(&file); + fs::remove_file(file.as_std_path()).unwrap(); + freshen(&file); + + // Freshening is best-effort metadata repair after a reflink; it must never turn a + // successful copy into an error just because timestamps cannot be changed. + assert!(!file.as_std_path().exists()); + } + + #[cfg(unix)] + #[test] + fn a_non_utf8_source_entry_is_reported() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt as _; + + let (_temporary, from, to) = tree(); + let name = OsString::from_vec(b"bad-\xff".to_vec()); + + fs::write(from.as_std_path().join(name), "x").unwrap(); + + let cause = copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap_err(); + + // Paths in reports and manifests are UTF-8; a lossy copy would make the later error point + // at a name the user cannot match back to the source tree. + assert!(cause.to_string().contains("not valid UTF-8"), "{cause}"); + } + + /// The scratch tree's own destination has to be creatable before anything is copied into it; if + /// a plain file already sits where a directory of that name is needed, the failure has to be + /// reported up front rather than surfacing later as a confusing per-file copy error. + #[test] + fn a_destination_root_blocked_by_a_file_is_reported() { + let (_temporary, from, to) = tree(); + + // `to` is not created by `tree()`; putting a plain file there instead means the very first + // thing `copy_tree` does, creating its own destination, has nowhere to go. + fs::write(to.as_std_path(), "not a directory").unwrap(); + + let cause = copy_tree(&from, &to, Utf8Path::new("/nowhere")).unwrap_err(); + + assert!(cause.to_string().contains("could not create the scratch tree"), "{cause}"); + } + + /// A source entry the walker named but which vanished before it could be copied — a file + /// deleted out from under a concurrent build, say — is reported by name rather than treated as + /// though it had never existed, so the missing file in the resulting build failure can be + /// traced back to a real cause instead of an unrelated compile error. + #[test] + fn copying_an_entry_that_no_longer_exists_is_reported() { + let (_temporary, from, to) = tree(); + let gone = from.join("was-here"); + + let cause = copy_entry(&gone, &to.join("was-here")).unwrap_err(); + + assert!(cause.to_string().contains("could not read"), "{cause}"); + } + + /// Copying a directory whose destination cannot be created — because a plain file already + /// occupies that name — has to fail with a named error rather than silently doing nothing, + /// which would leave the scratch tree missing a directory the build expects to find. + #[test] + fn copying_a_directory_blocked_by_a_file_is_reported() { + let (_temporary, from, to) = tree(); + + fs::create_dir_all(from.join("adir").as_std_path()).unwrap(); + fs::create_dir_all(to.as_std_path()).unwrap(); + fs::write(to.join("adir").as_std_path(), "blocking file").unwrap(); + + let cause = copy_entry(&from.join("adir"), &to.join("adir")).unwrap_err(); + + assert!(cause.to_string().contains("could not create"), "{cause}"); + } + + /// A file's parent directory is created on demand rather than assumed to already exist, since + /// the walk copies entries in parallel with no ordering between a file and its own directory. + /// If that parent cannot be created — a plain file sits where it belongs — the failure has to + /// be reported rather than silently dropping the file from the copy. + #[test] + fn a_files_parent_blocked_by_a_file_is_reported() { + let (_temporary, from, to) = tree(); + + fs::write(from.join("leaf").as_std_path(), "source").unwrap(); + fs::create_dir_all(to.as_std_path()).unwrap(); + fs::write(to.join("blocker").as_std_path(), "blocking file").unwrap(); + + let cause = copy_entry(&from.join("leaf"), &to.join("blocker").join("leaf")).unwrap_err(); + + assert!(cause.to_string().contains("could not create"), "{cause}"); + } + + /// A file whose parent has not been created yet is still copied: the parallel walk has no + /// ordering between a file and its directory, so the common-path copy that skips the per-file + /// `create_dir_all` must fall back to creating the parent and retrying rather than dropping the + /// file. Without the retry a file reached before its directory would vanish from the copy. + #[test] + fn a_files_missing_parent_is_created_on_demand() { + let (_temporary, from, to) = tree(); + + fs::write(from.join("leaf").as_std_path(), "source").unwrap(); + fs::create_dir_all(to.as_std_path()).unwrap(); + + // `to/nested` deliberately does not exist yet. + copy_entry(&from.join("leaf"), &to.join("nested").join("leaf")).unwrap(); + + assert_eq!(fs::read_to_string(to.join("nested").join("leaf").as_std_path()).unwrap(), "source"); + } + + /// A symlink's target has to be read before it can be recreated, and a source that no longer + /// names a link at all — removed, or never one to begin with, between the walk seeing it and + /// the copy reaching it — must be reported rather than silently skipped, which would leave a + /// gap in the copy nothing else explains. + #[cfg(unix)] + #[test] + fn a_symlink_whose_target_can_no_longer_be_read_is_reported() { + let (_temporary, from, to) = tree(); + let missing = from.join("not-a-link"); + + let cause = copy_symlink(&missing, &to.join("not-a-link")).unwrap_err(); + + assert!(cause.to_string().contains("could not read the link"), "{cause}"); + } + + /// Recreating a symlink at a destination that already exists has to fail rather than silently + /// leaving whatever was there, since two entries in the walk could otherwise race to claim the + /// same path and only one error would ever be visible if this were not checked. + #[cfg(unix)] + #[test] + fn recreating_a_symlink_over_an_existing_entry_is_reported() { + let (_temporary, from, to) = tree(); + let target = from.join("target-file"); + + fs::write(target.as_std_path(), "x").unwrap(); + std::os::unix::fs::symlink(target.as_std_path(), from.join("link").as_std_path()).unwrap(); + fs::create_dir_all(to.as_std_path()).unwrap(); + fs::write(to.join("link").as_std_path(), "already here").unwrap(); + + let cause = copy_symlink(&from.join("link"), &to.join("link")).unwrap_err(); + + assert!(cause.to_string().contains("could not recreate the link"), "{cause}"); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/events.rs b/crates/cargo-gamma-lib/src/exec/events.rs new file mode 100644 index 000000000..12412744b --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/events.rs @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use camino::Utf8Path; + +use super::session::Session; +use crate::Result; +use crate::discover::Plan; +use crate::estimate::Estimate; +use crate::model::Mutant; + +/// Progress notifications, so this module needs to know nothing about terminals. +pub trait Events { + /// A measured run acquired its scratch directory and is ready to record verdicts there. + fn testing_log(&mut self, _scratch: &Utf8Path) -> Result<()> { + Ok(()) + } + + /// A new phase started. + fn phase(&mut self, verb: &str, detail: &str); + + /// A phase started, and will report what it found on the same line once it knows. + fn begin(&mut self, active: &str, _completed: &str, detail: &str) { + self.phase(active, detail); + } + + /// A phase that opened a line with [`begin`](Self::begin) is closing it. + fn end(&mut self, detail: &str) { + self.outcome(detail); + } + + /// A phase that opened a line with [`begin`](Self::begin) is closing it with a result that + /// replaces, rather than extends, the description of work in progress. + fn complete(&mut self, detail: &str) { + self.outcome(detail); + } + + /// Reports progress within the phase opened by [`begin`](Self::begin). + fn phase_progress(&mut self, _completed: usize, _total: usize, _unit: &str) {} + + /// A phase that has already announced itself is reporting what it found. + /// + /// Rendered under the phase it belongs to rather than repeating the verb, since a phase and + /// its result are one event to a reader even though they are two to the code. + fn outcome(&mut self, detail: &str) { + self.phase("", detail); + } + + /// The build reported how far along it is, in a line of its own rendering. + /// + /// Passed through rather than reconstructed: cargo holds the unit graph, so it is the only + /// party that knows the denominator. + fn build_progress(&mut self, _bar: &str) {} + + /// The build wrote a line, or the compiler rendered a diagnostic. + /// + /// Only shown when asked for, since cargo narrates every crate it compiles, and because a + /// compiler error during an instrumented build is the mechanism rather than a fault: the tree + /// was checked before any mutant was applied, so the rollback loop is already about to withdraw + /// whatever failed. What withdrew a mutant is reported with the mutant. + fn build_output(&mut self, _line: &str) {} + + /// Whether anything would be done with a line handed to `build_output`. + /// + /// Asked before the work of producing one. Cargo's JSON stream runs to megabytes on a cold + /// build and a compiler diagnostic has to be decoded out of it, which is pure loss when the + /// answer is going to be dropped — and it is dropped by default, since `--show-build` is off. + fn wants_build_output(&self) -> bool { + false + } + + /// The build finished, so anything drawn in its place can be taken down. + fn build_finished(&mut self) {} + + /// Something about this run is likely to cost far more than it is worth, and the user can fix it. + /// + /// Distinct from a phase because a phase describes what is happening and this describes what + /// should perhaps not be. It is also the one kind of progress that has to survive the display + /// being off: the display resolves to whether a terminal is attached, and a CI job is exactly + /// where a run that quietly takes six hours is least affordable and least visible. + fn warn(&mut self, _message: &str) {} + + /// A mutant finished. + fn mutant(&mut self, mutant: &Mutant); + + /// The fixed cost is paid, the tree compiles, and the first mutant is about to be tested. + /// + /// The only moment at which a projection of the run is both possible and useful: everything + /// before it is measured, everything after it is the wait the user is deciding whether to sit + /// through, so the projection is handed over here rather than recomputed by whoever wants it. + fn measured(&mut self, _plan: &Plan, _session: &Session, _estimate: &Estimate) {} +} + +#[cfg(test)] +mod tests { + use core::time::Duration; + + use camino::Utf8PathBuf; + + use super::*; + use crate::fixtures; + use crate::testing::Recorder; + + #[test] + fn default_event_methods_are_expressed_in_terms_of_phase_and_outcome() { + let mut events = Recorder::default(); + let plan = Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: Utf8PathBuf::from("/workspace"), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + }; + let session = Session { + census: Vec::new(), + baseline: Duration::ZERO, + baseline_wall: Duration::ZERO, + tests: None, + quiet: Duration::ZERO, + stall: None, + build: Duration::ZERO, + metered: false, + unbounded: None, + withdrawn: 0, + rounds: 0, + rounds_taken: Vec::new(), + binaries: Vec::new(), + peak: None, + scratch: Utf8PathBuf::new(), + filtered: 0, + widened: false, + ordering: crate::exec::OrderingHints::default(), + phases: crate::exec::Phases::default(), + }; + let estimate = Estimate { + live: 0, + withdrawn: 0, + build: Duration::ZERO, + baseline: Duration::ZERO, + mutants: Duration::ZERO, + settled: Duration::ZERO, + stalling: Duration::ZERO, + jobs: 1, + worst: Duration::ZERO, + }; + + events.begin("Doing", "Done", "the thing"); + events.end(", done"); + events.complete("the result"); + events.outcome(", noted"); + events.measured(&plan, &session, &estimate); + events.mutant(&mutant()); + + // Implementors only have to provide the primitive rendering hooks; the default helpers + // keep their routing stable for plain reporters. + assert_eq!( + events.phases, + vec![ + ("Doing".to_owned(), "the thing".to_owned()), + (String::new(), ", done".to_owned()), + (String::new(), "the result".to_owned()), + (String::new(), ", noted".to_owned()), + ] + ); + assert_eq!(events.mutants, 1); + } + + /// The one hook with no default has to be routed by the implementor, not by the trait. + fn mutant() -> Mutant { + Mutant { + item_path: ("subject::less".to_owned()).into(), + original: "a < b".to_owned().into(), + replacement: "a <= b".to_owned().into(), + ..fixtures::mutant() + } + } +} diff --git a/crates/cargo-gamma-lib/src/exec/faults.rs b/crates/cargo-gamma-lib/src/exec/faults.rs new file mode 100644 index 000000000..e403eaad0 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/faults.rs @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Failures a test can ask for at boundaries the host will not fail on demand. +//! +//! Some machine failures cannot be provoked safely on demand: `waitpid` reports on a handle that is +//! gone, a process table fills, or a thread cannot be created. Each has a written response in this +//! crate, and arranging one for real would disrupt the test runner rather than isolate the branch +//! being tested. `cargo-gamma-process` owns process-lifecycle faults; this module owns failures at +//! library-level boundaries. +//! +//! This is the seam that makes them askable. A test arms one fault, on its own thread, for one +//! occurrence; the production code checks whether one is armed at the point the syscall would have +//! failed and takes exactly the branch it would have taken. The check is the *only* thing added to +//! the production path, and it is compiled away entirely outside `cfg(test)`, so the shipped binary +//! carries neither the branch nor the state behind it. +//! +//! Thread-local rather than global, and one-shot rather than sticky, for the same reason: the suite +//! runs in parallel in one process, and a fault left armed on a shared cell would surface in some +//! unrelated test as a failure nobody could reproduce. A thread that arms a fault is the only +//! thread that can fire it, and firing it disarms it. + +use core::cell::RefCell; +use core::time::Duration; +use std::time::Instant; + +/// A place a test can ask for a failure that the host would otherwise have to produce. +/// +/// Each name is the syscall boundary it stands at, not the function that calls it, because the same +/// failure reaches several callers and a test wants to talk about the kernel's refusal rather than +/// about whichever path noticed it. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum Fault { + /// Asking after a running child fails, rather than reporting that it is still running. + /// + /// `waitpid` reports an error only when the handle itself is gone, which for a child this + /// process spawned and has not reaped is not a state a test can otherwise arrange. + Wait, + + /// Starting a child is refused for want of a machine resource. + /// + /// A full process or descriptor table under a `jobs`-wide sweep, which is a shortage the run + /// creates for itself and recovers from as soon as one of the other workers finishes. Arranging + /// it for real would mean exhausting a table the test harness is itself using. + Spawn, + + /// Creating a pipe-reader thread is refused for want of process resources. + Thread, + + /// Taking the advisory lock on the scratch directory fails outright. + /// + /// A filesystem that cannot lock at all — NFS without `lockd`, some CIFS mounts, `ENOLCK` under + /// descriptor pressure — rather than a lock another run is holding. Reproducing it for real + /// would mean mounting such a filesystem. + Lock, +} + +/// Arms `fault` on this thread until the value returned is dropped. +/// +/// Held rather than fired-and-forgotten so that a test which panics before the fault is reached +/// still leaves the thread clean for whatever libtest runs on it next. +#[must_use = "the fault is disarmed as soon as this is dropped"] +pub(super) fn arm(fault: Fault) -> Armed { + ripe_at(fault, Instant::now()) +} + +/// Arms `fault` on this thread, but not for the checks made in the next `after`. +/// +/// Some responses are only worth asking about once the run has got somewhere. A wait that fails on +/// the first check fails before the child has finished starting, so a test of what the failure +/// takes down with it would be watching an empty subtree and would pass however little was killed. +/// Delaying the fault is what gives the child time to become the thing under test. +/// +/// The delay is a floor and not a schedule: the fault fires at the first check after it, whenever +/// that comes. A test that depends on this asserts that the child got where it was going, rather +/// than assuming the machine was fast enough. +#[must_use = "the fault is disarmed as soon as this is dropped"] +pub(super) fn arm_late(fault: Fault, after: Duration) -> Armed { + ripe_at(fault, Instant::now() + after) +} + +/// Records `fault` as armed from `ripe` onwards. +fn ripe_at(fault: Fault, ripe: Instant) -> Armed { + ARMED.with_borrow_mut(|armed| armed.push((fault, ripe))); + + Armed(fault) +} + +/// Whether `fault` is armed on this thread, disarming it if it is. +/// +/// One-shot, because every caller of this is inside a loop or a retry of some kind, and a fault +/// that stayed armed would turn a test of one refusal into a test of a host that refuses forever — +/// which is a different claim, and one no caller is written to survive. +pub(super) fn fired(fault: Fault) -> bool { + let now = Instant::now(); + + ARMED.with_borrow_mut(|armed| { + armed + .iter() + .position(|&(candidate, ripe)| candidate == fault && ripe <= now) + .map(|at| armed.remove(at)) + .is_some() + }) +} + +/// Takes one arming of `fault` away, whether or not it was ever due to fire. +/// +/// One rather than all of them, so that two guards for the same fault are spent one apiece and +/// dropping the first does not silently disarm the second. +fn disarm(fault: Fault) { + ARMED.with_borrow_mut(|armed| { + if let Some(at) = armed.iter().position(|&(candidate, _ripe)| candidate == fault) { + let _spent = armed.remove(at); + } + }); +} + +/// Keeps a fault armed for as long as a test wants it, and takes it away afterwards. +#[derive(Debug)] +pub(super) struct Armed(Fault); + +impl Drop for Armed { + fn drop(&mut self) { + disarm(self.0); + } +} + +thread_local! { + /// What this thread has asked to fail, most recently armed last, each with the moment it is due. + static ARMED: RefCell> = const { RefCell::new(Vec::new()) }; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An armed fault fires once and then is gone, so one request does not become a broken host. + #[test] + fn a_fault_fires_once() { + let armed = arm(Fault::Wait); + + assert!(fired(Fault::Wait), "the fault was armed"); + assert!(!fired(Fault::Wait), "and firing it must have taken it away"); + + drop(armed); + } + + /// Nothing fires that was not asked for, which is what keeps the production check inert. + #[test] + fn nothing_fires_unasked() { + assert!(!fired(Fault::Wait)); + assert!(!fired(Fault::Spawn)); + } + + /// Only the fault that was armed fires, so a test cannot accidentally prove the wrong branch. + #[test] + fn one_fault_does_not_stand_in_for_another() { + let _armed = arm(Fault::Wait); + + assert!(!fired(Fault::Spawn), "arming one fault must not arm the others"); + assert!(!fired(Fault::Thread)); + assert!(fired(Fault::Wait)); + } + + /// A fault the test never reached is taken away when the guard drops. + /// + /// Without this, a test that panics on its way to the boundary — or simply changes its mind — + /// would leave the fault armed on a thread libtest is about to hand to somebody else, and the + /// failure would surface in an unrelated test that has no way to explain it. + #[test] + fn an_unfired_fault_does_not_outlive_its_guard() { + drop(arm(Fault::Spawn)); + + assert!(!fired(Fault::Spawn), "the guard must disarm what it armed"); + } + + /// Two faults can be armed at once, and each is spent separately. + #[test] + fn faults_do_not_displace_each_other() { + let _spawn = arm(Fault::Spawn); + let _wait = arm(Fault::Wait); + + assert!(fired(Fault::Wait)); + assert!(fired(Fault::Spawn)); + } + + /// A fault armed on one thread is not armed on another. + /// + /// The suite runs in parallel in one process, so a fault in shared state would fire inside + /// whichever unrelated test happened to reach the same boundary first. + #[test] + fn a_fault_does_not_reach_another_thread() { + let _armed = arm(Fault::Spawn); + let elsewhere = std::thread::spawn(|| fired(Fault::Spawn)).join().expect("the probe thread"); + + assert!(!elsewhere, "a fault must not escape the thread that armed it"); + assert!(fired(Fault::Spawn), "and must still be waiting on the thread that did"); + } + + /// A delayed fault is inert until its delay has passed, and fires once afterwards. + /// + /// Both halves matter to the tests that use it: firing early would put the fault back where it + /// was, before the child under test has done anything, and never firing at all would turn the + /// response being tested into code the suite silently skips. + #[test] + fn a_late_fault_waits_for_its_moment() { + let _armed = arm_late(Fault::Wait, Duration::from_millis(50)); + + assert!(!fired(Fault::Wait), "a fault that is not due yet must not fire"); + + std::thread::sleep(Duration::from_millis(75)); + + assert!(fired(Fault::Wait), "and must fire at the first check after it is"); + assert!(!fired(Fault::Wait), "then be spent like any other"); + } + + /// A delayed fault that never came due is still taken away with its guard. + /// + /// The one case the ordinary disarming would miss, since it is written in terms of firing and + /// this fault is by construction not fireable yet. + #[test] + fn a_late_fault_that_never_fired_does_not_outlive_its_guard() { + drop(arm_late(Fault::Spawn, Duration::from_millis(1))); + + std::thread::sleep(Duration::from_millis(5)); + + assert!(!fired(Fault::Spawn), "an undue fault must not be left behind for the next test"); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/harness_filters.rs b/crates/cargo-gamma-lib/src/exec/harness_filters.rs new file mode 100644 index 000000000..8381c6127 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/harness_filters.rs @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Composition of the run's own test selection with the one the user asked for. +//! +//! libtest takes its filters as *positional* arguments and runs a test that any one of them +//! matches. That `or` is the whole problem this module exists for: appending the test this run +//! chose to the arguments the user supplied does not narrow the set, it widens it, and a mutant can +//! then be convicted by a test the user deliberately excluded. `--exact` makes it worse rather than +//! better, because it is global — one appended to pin the run's own name also converts the user's +//! substring filter into an exact match, which typically then matches nothing at all. +//! +//! The composition that is actually wanted is an intersection, and libtest cannot express one. So +//! it is computed here instead: the user's positional filters are read, applied to the names this +//! run chose, and the survivors passed on their own. The user's *flags* — including `--skip`, which +//! only ever removes tests — are passed through untouched, so everything that narrows still +//! narrows and only the widening is gone. + +/// The user's harness arguments, split into the parts that select tests and the parts that do not. +#[derive(Debug, Default)] +pub(super) struct HarnessFilters<'args> { + /// Everything that is not a positional filter, in the order it was given. + flags: Vec<&'args str>, + + /// The positional filters, which libtest matches with `or`. + filters: Vec<&'args str>, + + /// Whether the user asked for whole-name matching rather than substring matching. + exact: bool, +} + +/// libtest options whose value is the *next* argument rather than part of the same one. +/// +/// Needed so that the value is not mistaken for a positional filter: in `--test-threads 4` the `4` +/// selects nothing, and treating it as a filter would silently run no tests. The `--opt=value` +/// spelling needs no entry here, because it is a single argument that already starts with `-`. +const VALUED: &[&str] = &[ + "--test-threads", + "--logfile", + "--skip", + "--color", + "--format", + "--shuffle-seed", + "-Z", +]; + +impl<'args> HarnessFilters<'args> { + /// Reads the user's harness arguments. + pub(super) fn parse(args: &'args [String]) -> Self { + let mut parsed = Self::default(); + let mut index = 0; + + while let Some(arg) = args.get(index) { + let arg = arg.as_str(); + + if arg.starts_with('-') { + parsed.flags.push(arg); + + if arg == "--exact" { + parsed.exact = true; + } + + // The value belongs to the option, not to the filters, so it is consumed here and + // travels with it. + if VALUED.contains(&arg) + && let Some(value) = args.get(index + 1) + { + parsed.flags.push(value.as_str()); + index += 1; + } + } else { + parsed.filters.push(arg); + } + + index += 1; + } + + parsed + } + + /// Whether the user's filters would have let this test run. + /// + /// No positional filter is not an empty selection — it is the absence of one, and libtest runs + /// everything. Anything else is matched the way libtest would have matched it, so that the + /// answer here and the answer the harness would have given cannot differ. + pub(super) fn admits(&self, name: &str) -> bool { + if self.filters.is_empty() { + return true; + } + + if self.exact { + return self.filters.contains(&name); + } + + self.filters.iter().any(|filter| name.contains(filter)) + } + + /// The arguments to pass alongside a selection this run chose. + /// + /// The user's positional filters are deliberately *not* among them. They have already been + /// applied by [`Self::admits`], so passing them again would restore the `or` this module exists + /// to avoid — and leaving them out is what lets the `--exact` that pins the run's own names be + /// added without silently redefining the user's filter as well. + pub(super) fn flags(&self) -> &[&'args str] { + &self.flags + } + + /// The user's arguments as a listing pass should carry them. + /// + /// A listing has to see exactly the population the user's filters allow, or the census records + /// tests that are never going to run. That population is decided by more than the positional + /// filters — `--include-ignored` and `--exclude-should-panic` move it too — so everything is + /// carried rather than an allowlist that would silently drop the next such option. The one + /// exception is `--format`, which would fight with the one the listing asks for. + pub(super) fn selecting(&self) -> Vec<&'args str> { + let mut args: Vec<&str> = Vec::new(); + let mut index = 0; + + while let Some(flag) = self.flags.get(index) { + let value = VALUED.contains(flag).then(|| self.flags.get(index + 1)).flatten(); + + if *flag != "--format" && !flag.starts_with("--format=") { + args.push(flag); + args.extend(value); + } + + index += 1 + usize::from(value.is_some()); + } + + args.extend_from_slice(&self.filters); + args + } +} + +#[cfg(test)] +mod tests { + use super::HarnessFilters; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|arg| (*arg).to_owned()).collect() + } + + #[test] + fn an_option_value_is_not_mistaken_for_a_filter() { + // `--test-threads 4` reading `4` as a filter would run only tests whose name contains a + // four, which is very nearly none of them. + let raw = args(&["--test-threads", "4", "--nocapture", "parser"]); + let parsed = HarnessFilters::parse(&raw); + + assert_eq!(parsed.flags(), ["--test-threads", "4", "--nocapture"]); + assert!(parsed.admits("tests::parser_works")); + assert!(!parsed.admits("tests::lexer_works")); + } + + #[test] + fn the_joined_spelling_of_an_option_needs_no_lookahead() { + let raw = args(&["--test-threads=4", "parser"]); + let parsed = HarnessFilters::parse(&raw); + + assert_eq!(parsed.flags(), ["--test-threads=4"]); + assert!(parsed.admits("tests::parser_works")); + } + + #[test] + fn no_positional_filter_admits_everything() { + let raw = args(&["--nocapture"]); + let parsed = HarnessFilters::parse(&raw); + + assert!(parsed.admits("anything at all")); + } + + #[test] + fn the_users_exact_flag_makes_their_filters_whole_name_matches() { + let raw = args(&["--exact", "tests::parser"]); + let parsed = HarnessFilters::parse(&raw); + + assert!(parsed.admits("tests::parser")); + assert!(!parsed.admits("tests::parser_works"), "`--exact` is a whole-name match"); + } + + #[test] + fn several_filters_are_matched_with_or_exactly_as_libtest_does() { + let raw = args(&["parser", "lexer"]); + let parsed = HarnessFilters::parse(&raw); + + assert!(parsed.admits("tests::parser_works")); + assert!(parsed.admits("tests::lexer_works")); + assert!(!parsed.admits("tests::writer_works")); + } + + #[test] + fn a_listing_drops_only_the_format_the_user_asked_for() { + // A `--format` here would fight with the listing's own; everything else is carried, because + // options such as `--include-ignored` decide the population the listing must see. + let raw = args(&[ + "--test-threads", + "4", + "--format", + "json", + "--include-ignored", + "--skip", + "slow", + "--exact", + "parser", + ]); + let parsed = HarnessFilters::parse(&raw); + + assert_eq!( + parsed.selecting(), + ["--test-threads", "4", "--include-ignored", "--skip", "slow", "--exact", "parser"] + ); + } + + #[test] + fn the_joined_spelling_of_a_format_is_dropped_from_a_listing_too() { + let raw = args(&["--format=json", "--skip=slow", "parser"]); + let parsed = HarnessFilters::parse(&raw); + + assert_eq!(parsed.selecting(), ["--skip=slow", "parser"]); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/incremental_mode.rs b/crates/cargo-gamma-lib/src/exec/incremental_mode.rs new file mode 100644 index 000000000..0fbf177e9 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/incremental_mode.rs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! How an incremental run reuses state from the previous run. + +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; + +/// How an incremental run reuses state from the previous run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum IncrementalMode { + /// Re-run everything from scratch with no caching. + #[value(alias = "none", alias = "off")] + No, + + /// Reuse compiler unviability and checked execution hints. + #[default] + Build, +} + +impl IncrementalMode { + /// Whether any caching or reuse is enabled. + #[must_use] + pub const fn is_enabled(self) -> bool { + !matches!(self, Self::No) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_reuse_is_the_default() { + assert_eq!(IncrementalMode::default(), IncrementalMode::Build); + } + + #[test] + fn test_verdict_reuse_is_not_an_incremental_mode() { + let _error = IncrementalMode::from_str("full", true).expect_err("full would reuse test verdicts"); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/killers.rs b/crates/cargo-gamma-lib/src/exec/killers.rs new file mode 100644 index 000000000..e5fa63b6f --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/killers.rs @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Remembering which test caught each mutant, so the next run can try that one first. + +use camino::Utf8Path; + +use crate::discover::{Hints, Killer, RunRecord}; +use crate::model::{Mutant, MutantId}; +use crate::{HashMap, HashSet}; + +/// What caught each mutant last time, keyed by mutant id. +/// +/// A run already stops at the first binary that convicts, and the output watcher ends a binary as +/// soon as the harness announces a failure. What neither does is remember: the next run rediscovers +/// the same killer by paying for everything ahead of it in the same order. This is the memory. +/// +/// It comes from two places and is written back to one. The run record beside the verdicts is the +/// warm local copy; the checked-in artifact is the one that survives a clean checkout, which is the +/// case this is worth the most in — a fresh CI container has no record at all, so without it every +/// scheduled run pays full price for every re-kill. The record wins where both speak, because it is +/// this machine's last answer rather than whatever was committed. +/// +/// Both sit on quite different terms from a verdict. A verdict is believed, so it is gated on the +/// build context and on the digest of the file it was found in. Every entry here is a guess that +/// the run immediately checks by running the named test, and a guess that does not convict costs +/// one filtered process and is discarded. Nothing here can move a verdict, so nothing here needs +/// invalidating. +#[derive(Debug, Default)] +pub(super) struct Killers { + /// The killing test of each mutant that had one, keyed by [`crate::model::Mutant::id`]. + /// + /// Keyed by id rather than by position or line because the id hashes the file, item path, + /// mutator, normalized site text, occurrence and replacement index. An edit elsewhere in the + /// file leaves the key alone, and a site that genuinely changed simply stops matching instead + /// of pointing at a test that has nothing to do with it. + entries: HashMap, +} + +impl Killers { + /// Reads the probes the record and the checked-in artifact hold, or returns an empty map. + /// + /// `base` is the scratch directory holding the run record; `root` is the workspace the artifact + /// is checked in to. + /// + /// Every failure on either side is an empty map. Being unable to read them costs the run only + /// the time it would have saved, which is what lets either file be deleted, truncated or + /// written by another version without anyone having to care. + pub(super) fn load(base: &Utf8Path, root: &Utf8Path) -> Self { + let mut entries = Hints::load(root).probes(); + + // The record last, so this machine's own last answer wins over the committed one wherever + // both name a mutant: a probe is checked either way, but the fresher guess is likelier to + // convict, and paying for the stale one first would be paying for a test twice. + entries.extend(RunRecord::load(base).probes().clone()); + + Self { entries } + } + + /// What caught this mutant last time, if anything did. + pub(super) fn hint(&self, id: &str) -> Option<&Killer> { + self.entries.get(id) + } + + /// Records what caught a mutant this run. + pub(super) fn record(&mut self, id: MutantId, killer: Killer) { + let _previous = self.entries.insert(id, killer); + } + + /// Forgets what it thought caught this mutant. + /// + /// Called when a run's own verdict for a mutant names no test — the mutant survived, or was + /// caught by something other than a failing assertion. Keeping the old entry would make the + /// run after this one pay for a probe that has already been shown not to convict. + pub(super) fn forget(&mut self, id: &str) { + let _previous = self.entries.remove(id); + } + + /// Writes the hints back into the record, and says nothing if it cannot. + /// + /// A run that could not write this has still produced every verdict it was asked for, and + /// failing it over a scratch file would turn an optimization into a dependency. + pub(super) fn store(&self, base: &Utf8Path, population: &[Mutant]) { + let current: HashSet<&MutantId> = population.iter().map(|mutant| &mutant.id).collect(); + let probes: HashMap = self + .entries + .iter() + .filter(|(id, _killer)| current.contains(id)) + .map(|(id, killer)| (id.clone(), killer.clone())) + .collect(); + + RunRecord::store_probes(base, &probes); + } + + /// How many mutants this map has a killer for. + /// + /// Only the tests ask. A run has nothing to say about the size of the map: an entry that missed + /// cost one filtered process, and an entry whose mutant is no longer there cost nothing at all. + #[cfg(test)] + fn len(&self) -> usize { + self.entries.len() + } + + /// Whether it knows of no killers at all. + #[cfg(test)] + fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + + fn killer(test: &str) -> Killer { + Killer { + package: "alpha".to_owned(), + target: "lib".to_owned(), + test: test.to_owned(), + } + } + + fn mutant(id: &str) -> Mutant { + Mutant { + id: id.into(), + ..crate::fixtures::mutant() + } + } + + #[test] + fn a_map_round_trips_through_the_record() { + let dir = tempfile::tempdir().expect("a temporary directory"); + let base = Utf8Path::from_path(dir.path()).expect("a utf-8 path"); + + let mut killers = Killers::default(); + killers.record("abc".into(), killer("tests::round_trip")); + + killers.store(base, &[mutant("abc")]); + + let read = Killers::load(base, base); + + assert_eq!(read.len(), 1); + assert_eq!(read.hint("abc"), Some(&killer("tests::round_trip"))); + } + + #[test] + fn a_missing_record_is_an_empty_map() { + let dir = tempfile::tempdir().expect("a temporary directory"); + let base = Utf8Path::from_path(dir.path()).expect("a utf-8 path"); + + assert!(Killers::load(base, base).is_empty()); + } + + #[test] + fn a_corrupt_record_is_an_empty_map() { + // A record that cannot be parsed must cost the run nothing but the speed-up it would have + // given. Failing here would make a scratch file something a run depends on. + let dir = tempfile::tempdir().expect("a temporary directory"); + let base = Utf8Path::from_path(dir.path()).expect("a utf-8 path"); + + std::fs::write(base.join("last-gamma-run.json").as_std_path(), "{ not json").expect("the file to be written"); + + assert!(Killers::load(base, base).is_empty()); + } + + #[test] + fn hints_survive_a_build_context_this_run_does_not_share() { + // The whole reason hints sit outside the digested part of the record: a feature change + // discards every verdict, and discarding the hints with them would make the map cold on + // exactly the runs it is worth the most on. + let dir = tempfile::tempdir().expect("a temporary directory"); + let base = Utf8Path::from_path(dir.path()).expect("a utf-8 path"); + + let mut killers = Killers::default(); + killers.record("abc".into(), killer("tests::round_trip")); + killers.store(base, &[mutant("abc")]); + + let elsewhere = crate::discover::record_context(&crate::discover::RecordContext { + toolchain: Some("some other toolchain"), + ..crate::discover::RecordContext::default() + }) + .expect("a named toolchain gives a context"); + + RunRecord::from_run(base, &[], &elsewhere, &[]).store(base, base); + + assert_eq!(Killers::load(base, base).hint("abc"), Some(&killer("tests::round_trip"))); + } + + #[test] + fn forgetting_a_mutant_drops_its_hint() { + // A mutant whose verdict named no test must not leave the previous run's hint behind, or + // every run after this one pays for a probe already shown not to convict. + let mut killers = Killers::default(); + killers.record("abc".into(), killer("tests::round_trip")); + + killers.forget("abc"); + + assert!(killers.hint("abc").is_none()); + } + + #[test] + fn a_hint_names_only_the_binary_it_was_recorded_against() { + let hint = killer("tests::round_trip"); + + assert!(hint.names("alpha", "lib")); + assert!(!hint.names("alpha", "integration")); + assert!(!hint.names("beta", "lib")); + } + + #[test] + fn recording_a_mutant_twice_keeps_the_later_killer() { + // The sweep records as it goes, and a mutant is judged once per run; a second record for + // the same id is a re-run, whose answer is the current one. + let mut killers = Killers::default(); + + killers.record("abc".into(), killer("tests::first")); + killers.record("abc".into(), killer("tests::second")); + + assert_eq!(killers.len(), 1); + assert_eq!(killers.hint("abc").map(|found| found.test.as_str()), Some("tests::second")); + } + + #[test] + fn storing_prunes_hints_outside_the_current_population() { + let dir = tempfile::tempdir().expect("a temporary directory"); + let base = Utf8Path::from_path(dir.path()).expect("a UTF-8 path"); + let mut killers = Killers::default(); + + killers.record("old".into(), killer("tests::old")); + killers.record("new".into(), killer("tests::new")); + killers.store(base, &[mutant("new")]); + + let read = Killers::load(base, base); + + assert!(read.hint("old").is_none()); + assert_eq!(read.hint("new"), Some(&killer("tests::new"))); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/loader.rs b/crates/cargo-gamma-lib/src/exec/loader.rs new file mode 100644 index 000000000..6b0c07bb0 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/loader.rs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::env; +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Command; + +use camino::{Utf8Path, Utf8PathBuf}; + +/// The environment variable the dynamic loader reads on this platform. +/// +/// Windows has no separate loader path: the image loader searches `PATH`, so the toolchain +/// directories have to be prepended there instead. Getting this wrong is silent — the variable is +/// set, nothing reads it, and every test binary that links a proc macro dies at startup. +#[cfg(windows)] +pub(super) const LOADER_VAR: &str = "PATH"; + +/// The environment variable the dynamic loader reads on this platform. +#[cfg(target_os = "macos")] +pub(super) const LOADER_VAR: &str = "DYLD_FALLBACK_LIBRARY_PATH"; + +/// The environment variable the dynamic loader reads on this platform. +#[cfg(not(any(windows, target_os = "macos")))] +pub(super) const LOADER_VAR: &str = "LD_LIBRARY_PATH"; + +/// Set on every test process a run launches, so a suite can tell it is under mutation testing. +/// +/// The escape hatch for suites that invoke cargo themselves; exported because it is part of the +/// tool's contract with the code it tests. +pub const UNDER_GAMMA_VAR: &str = "CARGO_GAMMA"; + +/// The variable libtest reads to size the stack of each spawned test thread. +pub(super) const STACK_VAR: &str = "RUST_MIN_STACK"; + +/// The stack a test thread gets when the caller has not asked for a size, in bytes. +/// +/// Eight times the usual two megabyte default, chosen to swallow the frame growth instrumentation +/// causes without being large enough to matter on a machine running many test processes at once, +/// since a thread stack is reserved lazily and only the pages actually touched are committed. +const STACK_FLOOR: usize = 16 * 1024 * 1024; + +/// Everything about a test binary's environment that is fixed for the whole run. +/// +/// Both halves were being derived on every launch, and a launch is the most frequent thing this +/// tool does — once per reachable binary per mutant, plus every confirmation probe. Neither can +/// change while a run lasts: the library directories belong to the scratch workspace, and this +/// process does not modify its own environment. +/// +/// Deriving the stack floor exactly once also keeps the ambient read off the launch path, where +/// other threads are concurrently spawning children. +#[derive(Debug)] +pub(super) struct Launch { + /// The dynamic loader search path, or `None` when there is nothing worth setting. + pub(super) loader: Option, + + /// The thread stack size to ask for, already resolved against any larger inherited value. + pub(super) stack: String, +} + +impl Launch { + /// Derives the launch environment from a workspace's library directories. + pub(super) fn derive(libraries: &[Utf8PathBuf]) -> Self { + Self { + loader: loader_path(libraries), + stack: stack_floor(env::var(STACK_VAR).ok().as_deref()), + } + } +} + +/// Applies the dynamic-library search path to one command. +/// +/// Every executable Cargo built needs the same path, including the `--list` launches that precede +/// the census. Leaving listing out works for statically linked tests and fails at process startup +/// for dynamically linked proc-macro tests, before they can print a diagnostic Gamma can read. +pub(super) fn configure_loader(command: &mut Command, launch: &Launch) { + if let Some(path) = launch.loader.as_ref() { + let _ = command.env(LOADER_VAR, path); + } +} + +/// Returns the thread stack size to ask for, respecting a larger one the caller already chose. +/// +/// The ambient value is passed in rather than read here so that this is a pure function of it. The +/// alternative — reading the variable inside, and having the test set it — puts a `setenv` in a +/// test binary whose other tests are concurrently spawning children, which reads the same variable. +/// That is a data race in `libc`, not merely a confusing value, and no amount of restoring +/// afterwards fixes it. +fn stack_floor(inherited: Option<&str>) -> String { + let inherited = inherited.and_then(|value| value.trim().parse::().ok()); + + inherited.unwrap_or(0).max(STACK_FLOOR).to_string() +} + +/// Builds the loader search path, keeping whatever the caller already had. +pub(super) fn loader_path(libraries: &[Utf8PathBuf]) -> Option { + joined(libraries, env::var_os(LOADER_VAR)) +} + +/// Joins the toolchain's directories ahead of whatever search path the caller inherited. +/// +/// The inherited value is a parameter rather than an ambient read so that both shapes — a caller +/// who had a search path and one who had none — can be exercised without mutating the process +/// environment, which is unsafe and visible to every other test running at the same time. +fn joined(libraries: &[Utf8PathBuf], existing: Option) -> Option { + if libraries.is_empty() { + return None; + } + + let paths = libraries.iter().map(|path| PathBuf::from(path.as_str())); + + existing.map_or_else( + || env::join_paths(paths.clone()).ok(), + |current| { + let all = paths.clone().chain(env::split_paths(¤t)); + + env::join_paths(all).ok() + }, + ) +} + +/// Finds the directories holding the toolchain's shared libraries. +/// +/// A toolchain that links `std` dynamically needs the loader to find it, and the library lives +/// beside the target's rlibs rather than in the sysroot root. `rustc` runs from inside the copied +/// tree so a `rust-toolchain.toml` there picks the same toolchain that built the binaries. A +/// missing or unparseable answer is not fatal: a statically linked toolchain needs none of this. +pub(super) fn toolchain_libraries(root: &Utf8Path, target: &Utf8Path) -> Vec { + let mut libraries = vec![target.join("debug").join("deps")]; + + let output = Command::new(env::var_os("RUSTC").unwrap_or_else(|| "rustc".into())) + .current_dir(root.as_std_path()) + .args(["--print", "target-libdir", "--print", "sysroot"]) + .output(); + + if let Ok(output) = output + && output.status.success() + && let Ok(printed) = String::from_utf8(output.stdout) + { + let mut lines = printed.lines(); + + if let Some(libdir) = lines.next() { + libraries.push(Utf8PathBuf::from(libdir.trim())); + } + + // Host-side libraries, such as those a proc macro links against, live here instead. + if let Some(sysroot) = lines.next() { + libraries.push(Utf8PathBuf::from(sysroot.trim()).join("lib")); + } + } + + libraries +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + + /// A caller that already exported a larger stack size than the floor keeps its own choice. + /// + /// The floor exists to buy back headroom instrumentation spends, not to shrink a stack the + /// caller deliberately grew for its own reasons; silently overriding it would turn a caller's + /// explicit choice into a smaller one and reintroduce exactly the stack overflow the floor + /// was meant to prevent for a caller that already knew it needed more room than the default. + /// + /// A caller asking for less than the floor, or for nothing at all, gets the floor. + #[test] + fn stack_floor_keeps_a_larger_inherited_size() { + let large = (STACK_FLOOR * 3).to_string(); + + assert_eq!(stack_floor(Some(large.as_str())), large, "a larger explicit choice is kept"); + assert_eq!(stack_floor(None), STACK_FLOOR.to_string(), "nothing exported means the floor"); + assert_eq!( + stack_floor(Some("1")), + STACK_FLOOR.to_string(), + "a smaller choice is raised to the floor" + ); + assert_eq!( + stack_floor(Some(&format!(" {} ", STACK_FLOOR * 2))), + (STACK_FLOOR * 2).to_string(), + "surrounding whitespace is what an exported value often carries" + ); + assert_eq!( + stack_floor(Some("not a number")), + STACK_FLOOR.to_string(), + "an unparsable value is ignored" + ); + } + + #[test] + fn an_empty_library_list_does_not_set_a_loader_path() { + // Most toolchains are statically linked; setting an empty loader variable would be a + // needless change to the test process environment. + assert_eq!(loader_path(&[]), None); + } + + #[test] + fn a_caller_with_no_search_path_gets_only_the_toolchain_directories() { + // On a machine where the loader variable is unset there is nothing to preserve, and the + // result must still be a usable path rather than nothing at all. + let joined = joined(&[Utf8PathBuf::from("/one"), Utf8PathBuf::from("/two")], None).expect("a path"); + + assert_eq!(env::split_paths(&joined).count(), 2); + } + + #[test] + fn an_inherited_search_path_is_kept_behind_the_toolchain_directories() { + // The toolchain's own `std` has to win over anything the ambient environment points at, + // but discarding the caller's path would break binaries that legitimately need it. + let existing = env::join_paths([PathBuf::from("/inherited")]).expect("a path"); + let joined = joined(&[Utf8PathBuf::from("/one")], Some(existing)).expect("a path"); + let parts: Vec = env::split_paths(&joined).collect(); + + assert_eq!(parts, vec![PathBuf::from("/one"), PathBuf::from("/inherited")]); + } + + #[test] + fn the_loader_variable_is_the_one_this_platform_actually_reads() { + // Windows has no separate loader path; setting `LD_LIBRARY_PATH` there is silent, and every + // test binary that links a proc macro then dies at startup without an explanation. + let expected = if cfg!(windows) { + "PATH" + } else if cfg!(target_os = "macos") { + "DYLD_FALLBACK_LIBRARY_PATH" + } else { + "LD_LIBRARY_PATH" + }; + + assert_eq!(LOADER_VAR, expected); + } + + #[test] + fn no_libraries_means_no_search_path_even_with_one_inherited() { + assert_eq!(joined(&[], Some(OsString::from("/inherited"))), None); + } + + #[test] + fn the_target_deps_directory_is_always_on_the_toolchain_library_path() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("root")).unwrap(); + let target = Utf8PathBuf::from_path_buf(temporary.path().join("target")).unwrap(); + + std::fs::create_dir_all(root.as_std_path()).unwrap(); + + let libraries = toolchain_libraries(&root, &target); + + // Even if asking rustc for its dynamic locations fails, test binaries need their own + // freshly-built dependency directory first. + assert_eq!(libraries[0], target.join("debug").join("deps")); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/manifest.rs b/crates/cargo-gamma-lib/src/exec/manifest.rs new file mode 100644 index 000000000..bbce2d4e4 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/manifest.rs @@ -0,0 +1,964 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Repairing manifests inside the copied tree. + +use std::fs; + +use camino::{Utf8Path, Utf8PathBuf}; +use toml_edit::{DocumentMut, Item, Table, Value}; + +use super::workspace::absolute; +use crate::Result; +use crate::error::error; + +/// The dependency name the instrumented code refers to. +pub(crate) const RUNTIME_CRATE: &str = "gamma_rt"; + +/// Dependency tables, all of which can carry a path. +const DEPENDENCY_TABLES: [&str; 3] = ["dependencies", "dev-dependencies", "build-dependencies"]; + +/// A manifest in the copied tree, edited in place. +/// +/// Edits go through `toml_edit` rather than a parse-and-reserialise, so a manifest comes back out +/// with its comments, key order and formatting exactly as the user wrote them. Only the values +/// actually changed are touched. +#[derive(Debug)] +pub(super) struct Manifest { + path: Utf8PathBuf, + document: DocumentMut, + changed: bool, + + /// Where this manifest sits relative to the copied root, which is what decides whether a + /// dependency path leaves the tree. + within: Utf8PathBuf, +} + +impl Manifest { + /// Reads a manifest. + pub(super) fn read(path: &Utf8Path) -> Result { + let text = fs::read_to_string(path.as_std_path()).map_err(|cause| error!("could not read `{path}`").caused_by(cause))?; + + let document = text + .parse::() + .map_err(|cause| error!("could not parse `{path}`: {cause}"))?; + + Ok(Self { + path: path.to_owned(), + document, + changed: false, + within: Utf8PathBuf::new(), + }) + } + + /// Writes the manifest back if anything changed. + pub(super) fn save(&self) -> Result<()> { + if !self.changed { + return Ok(()); + } + + fs::write(self.path.as_std_path(), self.document.to_string()) + .map_err(|cause| error!("could not update `{}`", self.path).caused_by(cause)) + } + + /// Rewrites every relative path that leaves the copied tree so that it points back into the + /// original one. + /// + /// A path dependency resolves against the manifest holding it. The copy does not sit where the + /// original did, so a path leaving the tree — `../../shared` from a package one level down — + /// lands somewhere that does not exist, and the build fails naming a missing crate rather than + /// the move that lost it. + /// + /// What matters is whether the path leaves the *tree*, not whether it leaves the package. A + /// sibling dependency written `../core` climbs out of its package but stays well inside the + /// workspace, and the copy brought the sibling along; re-anchoring it to the original would + /// make cargo see the same package at two different locations and refuse to write a lockfile. + /// + /// `original` is the directory this manifest was copied from, and `within` is that same + /// directory expressed relative to the copied root — empty for the root manifest itself. + pub(super) fn anchor_paths(&mut self, original: &Utf8Path, within: &Utf8Path) { + self.within = within.to_owned(); + + for name in DEPENDENCY_TABLES { + self.anchor_table(name, original); + } + + // `[replace]` names crates by version requirement, and every entry is a source + // specification of exactly the same shape as a dependency. + self.anchor_table("replace", original); + + // Every `[patch.]` is its own table of dependencies, and a workspace commonly + // patches a crate to a sibling checkout — the exact shape that breaks. + if let Some(patch) = self.document.get_mut("patch").and_then(Item::as_table_like_mut) { + let registries: Vec = patch.iter().map(|(name, _entry)| name.to_owned()).collect(); + + for registry in registries { + if let Some(table) = patch.get_mut(®istry).and_then(Item::as_table_like_mut) { + anchor_dependencies(table, original, &self.within, &mut self.changed); + } + } + } + + // A target-specific table holds the same dependency tables one level down. + if let Some(targets) = self.document.get_mut("target").and_then(Item::as_table_like_mut) { + let platforms: Vec = targets.iter().map(|(name, _entry)| name.to_owned()).collect(); + + for platform in platforms { + let Some(table) = targets.get_mut(&platform).and_then(Item::as_table_like_mut) else { + continue; + }; + + for name in DEPENDENCY_TABLES { + if let Some(dependencies) = table.get_mut(name).and_then(Item::as_table_like_mut) { + anchor_dependencies(dependencies, original, &self.within, &mut self.changed); + } + } + } + } + + // A workspace's own dependency table feeds every member that says `workspace = true`. + if let Some(workspace) = self.document.get_mut("workspace").and_then(Item::as_table_like_mut) + && let Some(dependencies) = workspace.get_mut("dependencies").and_then(Item::as_table_like_mut) + { + anchor_dependencies(dependencies, original, &self.within, &mut self.changed); + } + } + + /// Rewrites the paths in one top-level table. + fn anchor_table(&mut self, name: &str, original: &Utf8Path) { + if let Some(table) = self.document.get_mut(name).and_then(Item::as_table_like_mut) { + anchor_dependencies(table, original, &self.within, &mut self.changed); + } + } + + /// Adds the guard runtime as a dependency, unless the package already has it. + /// + /// A package may already depend on the runtime — `cargo-gamma`'s own crates do. Two crates + /// with one library name makes every reference to it ambiguous, so the existing dependency is + /// left to do the job. + /// + /// The path written is absolute. Cargo resolves a dependency path against the manifest holding + /// it, and this manifest is the copy rather than the original, so a relative path would be + /// read from a different directory than the one it was measured from — and point at nothing, + /// or worse, at something else. + /// + /// Adding a dependency means the lockfile in the copied tree has to be written, which is why a + /// run cannot honour `--locked` or `--frozen` as written: those flags forbid exactly this edit. + /// The build substitutes `--offline` for them and says so once. + pub(super) fn link_runtime(&mut self, runtime: &Utf8Path) { + if self.links_runtime() { + return; + } + + let runtime = absolute(runtime); + + let dependencies = self.document.entry("dependencies").or_insert_with(|| Item::Table(Table::new())); + + let Some(table) = dependencies.as_table_like_mut() else { + return; + }; + + let mut entry = toml_edit::InlineTable::new(); + let _replaced = entry.insert("path", Value::from(runtime.as_str())); + + let _added = table.insert(RUNTIME_CRATE, Item::Value(Value::InlineTable(entry))); + + self.changed = true; + } + + /// Returns whether the library target can already name the guard runtime. + /// + /// Only a normal dependency counts. A dev- or build-dependency is invisible to the lib target, + /// so treating one as sufficient would leave every guard in library code unable to name the + /// runtime and fail the build everywhere at once. Declaring the crate in both sections is + /// legal, so adding ours alongside an existing dev-dependency is safe. + fn links_runtime(&self) -> bool { + let named = |table: Option<&Item>| { + table + .and_then(Item::as_table_like) + .is_some_and(|table| table.contains_key(RUNTIME_CRATE) || table.contains_key("cargo-gamma-rt")) + }; + + if named(self.document.get("dependencies")) { + return true; + } + + // `[target.'cfg(unix)'.dependencies]` is still a normal dependency table. + self.document.get("target").and_then(Item::as_table_like).is_some_and(|targets| { + targets + .iter() + .any(|(_platform, entry)| named(entry.as_table_like().and_then(|table| table.get("dependencies")))) + }) + } +} + +/// Rewrites every escaping path in one dependency table. +fn anchor_dependencies(table: &mut dyn toml_edit::TableLike, original: &Utf8Path, within: &Utf8Path, changed: &mut bool) { + let names: Vec = table.iter().map(|(name, _entry)| name.to_owned()).collect(); + + for name in names { + let Some(entry) = table.get_mut(&name) else { + continue; + }; + + // A dependency written as a bare version string has no path to fix. + let Some(specification) = entry.as_table_like_mut() else { + continue; + }; + + let Some(path) = specification.get("path").and_then(|item| item.as_str()) else { + continue; + }; + + let Some(anchored) = anchor(path, original, within) else { + continue; + }; + + let _replaced = specification.insert("path", Item::Value(Value::from(portable_path(&anchored)))); + + *changed = true; + } +} + +/// Returns the absolute form of a path that would not survive the move, or `None` if it would. +/// +/// A path is left alone when it is already absolute, and when it resolves to somewhere still +/// inside the copied tree — those still resolve in the copy, and rewriting them would tie a tree +/// meant to be self-contained back to the original for no reason. +/// +/// `within` is the manifest's directory relative to the copied root, so `within.join(path)` is +/// where the dependency lands relative to that root. Only a path that climbs above it has left. +fn anchor(path: &str, original: &Utf8Path, within: &Utf8Path) -> Option { + let candidate = Utf8Path::new(path); + + if candidate.is_absolute() || !escapes(&within.join(candidate)) { + return None; + } + + Some(normalize(&original.join(candidate))) +} + +/// Returns whether a relative path ever climbs above the directory it is written in. +fn escapes(path: &Utf8Path) -> bool { + let mut depth = 0_i32; + + for component in path.components() { + match component.as_str() { + "." => {} + ".." => { + depth -= 1; + + if depth < 0 { + return true; + } + } + _named => depth += 1, + } + } + + false +} + +/// Resolves `.` and `..` textually. +/// +/// The target of a path dependency need not exist yet — a workspace can be assembled in any order +/// — so this cannot go through the filesystem the way canonicalization would. Symlinks are +/// therefore not resolved, which is also what cargo itself does with these paths. +fn normalize(path: &Utf8Path) -> Utf8PathBuf { + let mut resolved = Utf8PathBuf::new(); + + for component in path.components() { + match component.as_str() { + "." => {} + ".." => { + if !resolved.pop() { + resolved.push(".."); + } + } + named => resolved.push(named), + } + } + + resolved +} + +/// Cargo paths are portable when written with `/`, including on Windows. +fn portable_path(path: &Utf8Path) -> String { + path.as_str().replace('\\', "/") +} + +/// Rewrites the `paths` overrides in a `.cargo/config.toml`, if there is one. +/// +/// These are relative to the directory holding `.cargo`, and break in exactly the way a path +/// dependency does. A missing file is not an error because cargo tolerates its absence. An +/// existing file that cannot be parsed is an error: Cargo cannot use it either, and pretending it +/// was absent would build a different tree. +pub(super) fn anchor_cargo_config(root: &Utf8Path, original: &Utf8Path) -> Result<()> { + for name in ["config.toml", "config"] { + let path = root.join(".cargo").join(name); + + let _destination = crate::paths::require_within(&path, root, "a scratch Cargo configuration")?; + + if !path.as_std_path().is_file() { + continue; + } + + let mut manifest = Manifest::read(&path)?; + + if let Some(paths) = manifest.document.get_mut("paths").and_then(Item::as_array_mut) { + for entry in paths.iter_mut() { + let Some(anchored) = entry.as_str().and_then(|path| anchor(path, original, Utf8Path::new(""))) else { + continue; + }; + + *entry = Value::from(portable_path(&anchored)); + manifest.changed = true; + } + } + + manifest.save()?; + } + + Ok(()) +} + +/// The flag the instrumented tree is built with, so that the user's lint levels do not judge it. +pub(super) const CAP_LINTS: &str = "--cap-lints=allow"; + +/// Adds [`CAP_LINTS`] to whatever rustflags the copied tree already configures. +/// +/// Setting `RUSTFLAGS` in the environment would be simpler, but the environment variable *replaces* +/// the configured flags rather than adding to them: a workspace whose `.cargo/config.toml` sets +/// `target..rustflags` would build with none of them, which can change what its code +/// compiles to and therefore what its tests prove. +/// +/// The flag is appended to every rustflags key already present, because cargo picks exactly one of +/// them — `target.` over `target.` over `build` — and which one is not knowable here +/// without resolving the target triple. Appending to all of them means the winner carries the flag +/// whichever it turns out to be. If none is configured, `build.rustflags` is created. +pub(super) fn cap_lints(root: &Utf8Path) -> Result<()> { + let path = root.join(".cargo").join("config.toml"); + let legacy = root.join(".cargo").join("config"); + let path = if !path.as_std_path().is_file() && legacy.as_std_path().is_file() { + legacy + } else { + path + }; + let _destination = crate::paths::require_within(&path, root, "a scratch Cargo configuration")?; + + if let Some(parent) = path.parent() { + fs::create_dir_all(parent.as_std_path()).map_err(|cause| error!("could not create `{parent}`").caused_by(cause))?; + } + + if !path.as_std_path().is_file() { + fs::write(path.as_std_path(), format!("[build]\nrustflags = [\"{CAP_LINTS}\"]\n")) + .map_err(|cause| error!("could not write `{path}`").caused_by(cause))?; + + return Ok(()); + } + + let mut manifest = Manifest::read(&path)?; + let mut found = false; + + if let Some(build) = manifest.document.get_mut("build") { + let Some(build) = build.as_table_like_mut() else { + return Err(error!( + "Cargo configuration `{path}` has a non-table `build` setting; use a `[build]` table so cargo-gamma can add `{CAP_LINTS}`" + )); + }; + + if let Some(flags) = build.get_mut("rustflags") { + found |= append_flag(flags); + } + } + + if let Some(targets) = manifest.document.get_mut("target").and_then(Item::as_table_like_mut) { + for (_name, entry) in targets.iter_mut() { + let Some(table) = entry.as_table_like_mut() else { + continue; + }; + + if let Some(flags) = table.get_mut("rustflags") { + found |= append_flag(flags); + } + } + } + + if !found { + let build = manifest + .document + .entry("build") + .or_insert(Item::Table(Table::new())) + .as_table_like_mut() + .ok_or_else(|| error!("Cargo configuration `{path}` has a non-table `build` setting"))?; + + let _previous = build.insert("rustflags", Item::Value(Value::Array(core::iter::once(CAP_LINTS).collect()))); + } + + manifest.changed = true; + manifest.save() +} + +/// Appends the cap to one rustflags entry, which cargo accepts as an array or as one string. +fn append_flag(flags: &mut Item) -> bool { + if let Some(array) = flags.as_array_mut() { + array.push(CAP_LINTS); + + return true; + } + + if let Some(text) = flags.as_str() { + *flags = Item::Value(Value::from(format!("{text} {CAP_LINTS}"))); + + return true; + } + + false +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use super::*; + + fn fixed(text: &str, original: &str) -> String { + within(text, original, "") + } + + fn within(text: &str, original: &str, within: &str) -> String { + let temporary = tempfile::tempdir().unwrap(); + let path = Utf8PathBuf::from_path_buf(temporary.path().join("Cargo.toml")).unwrap(); + + fs::write(path.as_std_path(), text).unwrap(); + + let mut manifest = Manifest::read(&path).unwrap(); + + manifest.anchor_paths(Utf8Path::new(original), Utf8Path::new(within)); + + manifest.document.to_string() + } + + #[test] + fn a_dependency_written_as_a_bare_version_is_left_alone() { + // `serde = "1"` has no path to anchor, and treating the string as a specification table + // would either panic or silently rewrite the version requirement. + let text = "[dependencies]\nserde = \"1\"\ncore = { path = \"../core\" }\n"; + let fixed = fixed(text, "/src/app"); + + assert!(fixed.contains("serde = \"1\""), "{fixed}"); + assert!(fixed.contains("/src/core"), "{fixed}"); + } + + #[test] + fn a_sibling_inside_the_copied_tree_is_left_alone() { + // `app/../core` is `core`, which the copy brought along. Anchoring it back to the original + // would put the same package at two locations and cargo would refuse to write a lockfile. + let fixed = within("[dependencies]\ncore = { path = \"../core\" }\n", "/src/app", "app"); + + assert!(fixed.contains("path = \"../core\""), "{fixed}"); + } + + #[test] + fn a_path_leaving_the_copied_tree_is_anchored_even_from_a_nested_package() { + let fixed = within("[dependencies]\nshared = { path = \"../../shared\" }\n", "/src/work/app", "app"); + + assert!(fixed.contains("path = \"/src/shared\""), "{fixed}"); + } + + #[test] + fn a_path_leaving_the_package_is_anchored() { + let fixed = fixed("[dependencies]\nshared = { path = \"../shared\" }\n", "/src/app"); + + assert!(fixed.contains("path = \"/src/shared\""), "{fixed}"); + } + + #[test] + fn a_path_staying_inside_the_package_is_left_alone() { + // It still resolves in the copy, and rewriting it would tie the tree back to the original. + let fixed = fixed("[dependencies]\ninner = { path = \"crates/inner\" }\n", "/src/app"); + + assert!(fixed.contains("path = \"crates/inner\""), "{fixed}"); + } + + #[test] + fn an_absolute_path_is_left_alone() { + let fixed = fixed("[dependencies]\nshared = { path = \"/elsewhere/shared\" }\n", "/src/app"); + + assert!(fixed.contains("path = \"/elsewhere/shared\""), "{fixed}"); + } + + #[test] + fn a_path_that_descends_before_climbing_is_judged_on_the_whole_journey() { + // `crates/../../shared` leaves the package even though it starts by entering it. + let fixed = fixed("[dependencies]\nshared = { path = \"crates/../../shared\" }\n", "/src/app"); + + assert!(fixed.contains("path = \"/src/shared\""), "{fixed}"); + } + + #[test] + fn every_kind_of_dependency_table_is_covered() { + let text = "[dependencies]\na = { path = \"../a\" }\n\ + [dev-dependencies]\nb = { path = \"../b\" }\n\ + [build-dependencies]\nc = { path = \"../c\" }\n\ + [target.'cfg(unix)'.dependencies]\nd = { path = \"../d\" }\n\ + [patch.crates-io]\ne = { path = \"../e\" }\n\ + [workspace.dependencies]\nf = { path = \"../f\" }\n"; + + let fixed = fixed(text, "/src/app"); + + for crate_name in ["a", "b", "c", "d", "e", "f"] { + assert!(fixed.contains(&format!("path = \"/src/{crate_name}\"")), "{crate_name} in {fixed}"); + } + } + + #[test] + fn a_patch_entry_that_is_not_a_table_does_not_stop_the_next_registry_from_being_anchored() { + // A malformed `[patch]` entry — a bare string rather than a table of dependencies — must + // not abort repairing the registries that come after it in the same document. + let text = "[patch]\nbroken = \"not a table\"\n\n\ + [patch.crates-io]\nshared = { path = \"../shared\" }\n"; + + let fixed = fixed(text, "/src/app"); + + assert!(fixed.contains("broken = \"not a table\""), "{fixed}"); + assert!(fixed.contains("path = \"/src/shared\""), "{fixed}"); + } + + #[test] + fn comments_and_formatting_survive() { + // The whole reason for editing rather than reserialising. + let text = "# keep me\n[dependencies]\n# and me\nshared = { path = \"../shared\" } # trailing\n"; + let fixed = fixed(text, "/src/app"); + + assert!(fixed.contains("# keep me"), "{fixed}"); + assert!(fixed.contains("# and me"), "{fixed}"); + assert!(fixed.contains("# trailing"), "{fixed}"); + } + + #[test] + fn a_version_only_dependency_is_untouched() { + let fixed = fixed("[dependencies]\nserde = \"1\"\n", "/src/app"); + + assert!(fixed.contains("serde = \"1\""), "{fixed}"); + } + + #[test] + fn target_entries_that_are_not_tables_are_skipped() { + let fixed = fixed( + "[target]\nnot_a_table = \"ignored\"\n[dependencies]\na = { path = \"../a\" }\n", + "/src/app", + ); + + // Some manifests put metadata under target-like tables; non-tables must not stop ordinary + // dependencies later in the document from being repaired. + assert!(fixed.contains("not_a_table = \"ignored\""), "{fixed}"); + assert!(fixed.contains("path = \"/src/a\""), "{fixed}"); + } + + #[test] + fn dependencies_without_a_path_are_skipped() { + let fixed = fixed( + "[dependencies]\nserde = { version = \"1\" }\nlocal = { path = \"../local\" }\n", + "/src/app", + ); + + // Versioned inline-table dependencies are pathless and should survive byte-for-byte while + // path dependencies next to them are still anchored. + assert!(fixed.contains("serde = { version = \"1\" }"), "{fixed}"); + assert!(fixed.contains("path = \"/src/local\""), "{fixed}"); + } + + fn linked(text: &str) -> String { + let temporary = tempfile::tempdir().unwrap(); + let path = Utf8PathBuf::from_path_buf(temporary.path().join("Cargo.toml")).unwrap(); + + fs::write(path.as_std_path(), text).unwrap(); + + let mut manifest = Manifest::read(&path).unwrap(); + + manifest.link_runtime(Utf8Path::new("/scratch/rt")); + + manifest.document.to_string() + } + + #[test] + fn the_runtime_is_added_to_a_package_without_it() { + let text = linked("[package]\nname = \"x\"\n"); + let runtime = absolute(Utf8Path::new("/scratch/rt")); + + assert!(text.contains("gamma_rt"), "{text}"); + assert!(text.contains(runtime.as_str()), "{text}"); + } + + /// Cargo reads a dependency path relative to the manifest holding it, and this manifest lives + /// in the copied tree rather than where the run was started, so a relative runtime path would + /// be looked for under the copy. + #[test] + fn a_relative_runtime_path_is_written_absolute() { + let temporary = tempfile::tempdir().unwrap(); + let path = Utf8PathBuf::from_path_buf(temporary.path().join("Cargo.toml")).unwrap(); + + fs::write(path.as_std_path(), "[package]\nname = \"x\"\n").unwrap(); + + let mut manifest = Manifest::read(&path).unwrap(); + + manifest.link_runtime(Utf8Path::new("scratch/gamma/rt")); + + let text = manifest.document.to_string(); + let expected = absolute(Utf8Path::new("scratch/gamma/rt")); + + assert!(expected.is_absolute(), "{expected}"); + assert!(text.contains(expected.as_str()), "{text}"); + } + + #[test] + fn the_runtime_is_added_to_an_existing_dependency_table() { + let text = linked("[package]\nname = \"x\"\n\n[dependencies]\nserde = \"1\"\n"); + + assert!(text.contains("gamma_rt"), "{text}"); + assert!(text.contains("serde = \"1\""), "{text}"); + } + + #[test] + fn a_malformed_dependency_table_cannot_receive_the_runtime() { + let text = linked("dependencies = \"not a table\"\n[package]\nname = \"x\"\n"); + + // If the user wrote a non-table where dependencies belong, this pass leaves it to cargo's + // manifest parser rather than inventing a structure and hiding the original problem. + assert!(!text.contains("gamma_rt"), "{text}"); + assert!(text.contains("dependencies = \"not a table\""), "{text}"); + } + + #[test] + fn an_existing_runtime_dependency_is_not_duplicated() { + // Two crates with one library name makes every guard call ambiguous. + for text in [ + "[dependencies]\ncargo-gamma-rt = { workspace = true }\n", + "[dependencies]\ngamma_rt = { path = \"../rt\" }\n", + "[dependencies.gamma_rt]\npath = \"../rt\"\n", + "[target.'cfg(unix)'.dependencies]\ngamma_rt = \"1\"\n", + ] { + let linked = linked(text); + + assert_eq!( + linked.matches("gamma_rt").count() + linked.matches("cargo-gamma-rt").count(), + 1, + "{linked}" + ); + } + } + + #[test] + fn a_dependency_the_library_target_cannot_see_does_not_count() { + // Guards live in library code, where a dev- or build-dependency is not in scope. + let runtime = absolute(Utf8Path::new("/scratch/rt")); + + for text in ["[dev-dependencies]\ngamma_rt = \"1\"\n", "[build-dependencies]\ngamma_rt = \"1\"\n"] { + assert!(linked(text).contains(runtime.as_str()), "{text}"); + } + } + + #[test] + fn a_cargo_config_path_override_is_anchored() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + + fs::create_dir_all(root.join(".cargo").as_std_path()).unwrap(); + fs::write( + root.join(".cargo").join("config.toml").as_std_path(), + "paths = [\"../vendored\", \"inside\"]\n", + ) + .unwrap(); + + anchor_cargo_config(&root, Utf8Path::new("/src/app")).unwrap(); + + let text = fs::read_to_string(root.join(".cargo").join("config.toml").as_std_path()).unwrap(); + + assert!(text.contains("/src/vendored"), "{text}"); + assert!(text.contains("\"inside\""), "{text}"); + } + + #[test] + fn a_cargo_config_with_no_paths_override_is_left_alone() { + // A `.cargo/config.toml` need not configure path overrides at all; the absence of the key + // must not be treated as an error or invent one out of nothing. + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + + fs::create_dir_all(root.join(".cargo").as_std_path()).expect(".cargo"); + fs::write(root.join(".cargo").join("config.toml").as_std_path(), "[net]\nretry = 3\n").expect("config"); + + anchor_cargo_config(&root, Utf8Path::new("/src/app")).expect("anchor"); + + let text = fs::read_to_string(root.join(".cargo").join("config.toml").as_std_path()).expect("read back"); + + assert!(text.contains("retry"), "{text}"); + assert!(!text.contains("paths"), "{text}"); + } + + #[test] + fn the_lint_cap_is_added_to_configured_rustflags_rather_than_replacing_them() { + // Setting `RUSTFLAGS` would drop these, which can change what the tree compiles to and + // therefore what its tests prove. + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let config = root.join(".cargo").join("config.toml"); + + fs::create_dir_all(config.parent().unwrap().as_std_path()).unwrap(); + fs::write( + config.as_std_path(), + "[build]\nrustflags = [\"--cfg\", \"loom\"]\n\n[target.x86_64-unknown-linux-gnu]\nrustflags = \"-C target-cpu=native\"\n", + ) + .unwrap(); + + cap_lints(&root).unwrap(); + + let text = fs::read_to_string(config.as_std_path()).unwrap(); + + assert!(text.contains("loom"), "{text}"); + assert!(text.contains("target-cpu=native"), "{text}"); + // Both keys carry it, because which one cargo picks depends on the target triple. + assert_eq!(text.matches(CAP_LINTS).count(), 2, "{text}"); + } + + #[test] + fn a_tree_with_no_cargo_config_gets_one() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + + cap_lints(&root).unwrap(); + + let text = fs::read_to_string(root.join(".cargo").join("config.toml").as_std_path()).unwrap(); + + assert!(text.contains(CAP_LINTS), "{text}"); + } + + #[test] + fn a_cargo_config_with_no_rustflags_gains_them() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let config = root.join(".cargo").join("config.toml"); + + fs::create_dir_all(config.parent().unwrap().as_std_path()).unwrap(); + fs::write(config.as_std_path(), "[net]\nretry = 3\n").unwrap(); + + cap_lints(&root).unwrap(); + + let text = fs::read_to_string(config.as_std_path()).unwrap(); + + assert!(text.contains(CAP_LINTS), "{text}"); + assert!(text.contains("retry"), "{text}"); + } + + #[test] + fn a_scalar_build_configuration_is_reported_without_panicking() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + let config = root.join(".cargo").join("config.toml"); + + fs::create_dir_all(config.parent().expect("parent").as_std_path()).expect("mkdir"); + fs::write(config.as_std_path(), "build = \"not a table\"\n").expect("config"); + + let failure = cap_lints(&root).expect_err("a scalar build key cannot receive rustflags"); + + assert!(failure.to_string().contains("non-table `build`"), "{failure}"); + assert!(failure.to_string().contains(config.as_str()), "{failure}"); + } + + /// A target entry that is not itself a table — some manifests store other metadata alongside + /// real target platforms — must be skipped rather than aborting the whole pass, so that a + /// well-formed target sharing the document still gets the cap. + #[test] + fn a_target_entry_that_is_not_a_table_does_not_stop_the_cap_from_landing_elsewhere() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + let config = root.join(".cargo").join("config.toml"); + + fs::create_dir_all(config.parent().expect("parent").as_std_path()).expect("mkdir"); + fs::write( + config.as_std_path(), + "[target]\nnot_a_table = \"ignored\"\n\n\ + [target.'cfg(unix)']\nlinker = \"lld\"\n\n\ + [target.x86_64-unknown-linux-gnu]\nrustflags = [\"--cfg\", \"loom\"]\n", + ) + .expect("write config"); + + cap_lints(&root).expect("cap_lints"); + + let text = fs::read_to_string(config.as_std_path()).expect("read back"); + + assert!(text.contains("not_a_table = \"ignored\""), "{text}"); + // The target with no rustflags key at all keeps its other settings untouched. + assert!(text.contains("linker = \"lld\""), "{text}"); + // Only the one target that actually carries rustflags gets the cap appended. + assert_eq!(text.matches(CAP_LINTS).count(), 1, "{text}"); + } + + /// A rustflags value that is neither an array nor a string — malformed, or written some other + /// way entirely — cannot be appended to; `append_flag` reports that it made no addition, and + /// the fallback then treats the tree as though it configured no rustflags at all, replacing + /// the unusable value with a fresh array carrying just the cap. + #[test] + fn a_rustflags_value_that_is_neither_an_array_nor_a_string_is_replaced() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + let config = root.join(".cargo").join("config.toml"); + + fs::create_dir_all(config.parent().expect("parent").as_std_path()).expect("mkdir"); + fs::write(config.as_std_path(), "[build]\nrustflags = 5\n").expect("write config"); + + cap_lints(&root).expect("cap_lints"); + + let text = fs::read_to_string(config.as_std_path()).expect("read back"); + + assert!(!text.contains("rustflags = 5"), "{text}"); + assert!(text.contains(CAP_LINTS), "{text}"); + } + + #[test] + fn the_legacy_cargo_config_name_gains_the_cap_too() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let config = root.join(".cargo").join("config"); + + fs::create_dir_all(config.parent().unwrap().as_std_path()).unwrap(); + fs::write(config.as_std_path(), "[build]\nrustflags = [\"--cfg\", \"loom\"]\n").unwrap(); + + cap_lints(&root).unwrap(); + + let text = fs::read_to_string(config.as_std_path()).unwrap(); + + assert!(text.contains(CAP_LINTS), "{text}"); + assert!(!root.join(".cargo").join("config.toml").as_std_path().exists()); + } + + #[test] + fn a_legacy_cargo_config_name_is_anchored() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + + fs::create_dir_all(root.join(".cargo").as_std_path()).unwrap(); + fs::write(root.join(".cargo").join("config").as_std_path(), "paths = [\"../vendored\"]\n").unwrap(); + + anchor_cargo_config(&root, Utf8Path::new("/src/app")).unwrap(); + + let text = fs::read_to_string(root.join(".cargo").join("config").as_std_path()).unwrap(); + + // Cargo still accepts `.cargo/config`; it needs the same repair as the TOML-suffixed name. + assert!(text.contains("/src/vendored"), "{text}"); + } + + #[test] + fn normalization_keeps_leading_parent_components() { + // Anchoring from a filesystem root keeps the parent component textually rather than + // resolving it through the host filesystem. + assert_eq!(normalize(Utf8Path::new("/../shared")), Utf8PathBuf::from("/../shared")); + } + + #[test] + fn a_missing_cargo_config_is_not_an_error() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + + anchor_cargo_config(&root, Utf8Path::new("/src/app")).unwrap(); + } + + /// A path written with an explicit current-directory component — `./sibling` rather than plain + /// `sibling` — must be judged exactly the same as if the component were absent: a user who adds + /// or a tool that emits the redundant `./` should not thereby escape the check for whether the + /// path leaves the tree, nor have it resolved to something subtly different. + #[test] + fn a_leading_current_directory_component_does_not_change_whether_a_path_escapes() { + assert!(!escapes(Utf8Path::new("./sibling/deeper"))); + assert!(escapes(Utf8Path::new("./.."))); + } + + /// The same redundant `./` component must vanish during normalization rather than being kept + /// verbatim, or two dependency paths that name the same file — one written plainly, one with a + /// stray `./` — would come out looking different once anchored. + #[test] + fn normalization_drops_current_directory_components() { + assert_eq!(normalize(Utf8Path::new("a/./b")), Utf8PathBuf::from("a/b")); + } + + /// A `.cargo/config.toml` that exists but is not valid TOML is a file the user's own build is + /// already failing on, not something this repair step can silently paper over: the read has to + /// fail loudly, naming the file, rather than being treated the same as a config with no `paths` + /// key at all. + #[test] + fn a_cargo_config_that_cannot_be_parsed_is_reported() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + + fs::create_dir_all(root.join(".cargo").as_std_path()).expect(".cargo"); + fs::write(root.join(".cargo").join("config.toml").as_std_path(), "not [ valid toml").expect("config"); + + let error = anchor_cargo_config(&root, Utf8Path::new("/src/app")).expect_err("the file does not parse"); + + assert!(error.to_string().contains("could not parse"), "{error}"); + } + + /// Once a `paths` override has actually been rewritten, the anchored config has to be written + /// back; if the file cannot be saved — permissions revoked between the read and the write, say + /// — that failure has to surface rather than leaving the tree instrumented with the original, + /// unanchored path that would fail to resolve. + #[cfg(unix)] + #[test] + fn a_cargo_config_that_cannot_be_saved_after_anchoring_is_reported() { + use std::os::unix::fs::PermissionsExt as _; + + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + let config = root.join(".cargo").join("config.toml"); + + fs::create_dir_all(config.parent().expect("parent").as_std_path()).expect(".cargo"); + fs::write(config.as_std_path(), "paths = [\"../vendored\"]\n").expect("config"); + fs::set_permissions(config.as_std_path(), fs::Permissions::from_mode(0o400)).expect("chmod"); + + let error = anchor_cargo_config(&root, Utf8Path::new("/src/app")).expect_err("the file cannot be written"); + + fs::set_permissions(config.as_std_path(), fs::Permissions::from_mode(0o644)).expect("chmod back"); + + assert!(error.to_string().contains("could not update"), "{error}"); + } + + /// A fresh `.cargo/config.toml` is created when a tree has none, and if the directory it would + /// live in refuses the write — a permission denied between the directory being created and the + /// file inside it — the caller has to be told the tree could not be prepared rather than + /// silently building it without the lint cap. + #[cfg(unix)] + #[test] + fn a_cargo_config_that_cannot_be_created_reports_the_write_failure() { + use std::os::unix::fs::PermissionsExt as _; + + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + let cargo_dir = root.join(".cargo"); + + fs::create_dir_all(cargo_dir.as_std_path()).expect(".cargo"); + fs::set_permissions(cargo_dir.as_std_path(), fs::Permissions::from_mode(0o500)).expect("chmod"); + + let error = cap_lints(&root).expect_err("the directory cannot be written into"); + + fs::set_permissions(cargo_dir.as_std_path(), fs::Permissions::from_mode(0o755)).expect("chmod back"); + + assert!(error.to_string().contains("could not write"), "{error}"); + } + + /// `cap_lints` reads whichever config already exists before adding the flag, and a config that + /// does not parse must stop it the same way any other unreadable manifest does, rather than + /// treating the tree as though it had no configuration to preserve. + #[test] + fn cap_lints_reports_an_existing_config_that_cannot_be_parsed() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).expect("utf8"); + let config = root.join(".cargo").join("config.toml"); + + fs::create_dir_all(config.parent().expect("parent").as_std_path()).expect(".cargo"); + fs::write(config.as_std_path(), "not [ valid toml").expect("config"); + + let error = cap_lints(&root).expect_err("the file does not parse"); + + assert!(error.to_string().contains("could not parse"), "{error}"); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/measure.rs b/crates/cargo-gamma-lib/src/exec/measure.rs new file mode 100644 index 000000000..b58277813 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/measure.rs @@ -0,0 +1,1999 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Preparing the tree, building the schema, and driving a run to its verdicts. + +use core::fmt::Write as _; +use core::time::Duration; +use std::time::Instant; + +use cargo_gamma_process::MemoryRequest; + +use super::baseline::{Baseline, measure_baseline}; +use super::build::{Abandoned, Converger}; +use super::census::{self, Census}; +use super::config::Config; +use super::events::Events; +use super::killers::Killers; +use super::memory; +use super::memory::MemoryPolicy; +use super::session::{CensusCost, Phases, Session, SweepCost}; +use super::stall::Stall; +use super::sweep::{Sweep, test_all}; +use super::test_binary::{ + Reachability, TestBinary, TestScope, admits_target, build_packages, calibrate, oracle_packages, reaches, reaching_packages, restrict, + unmatched_test, workload, +}; +use super::workspace::{Workspace, gamma_base}; +use crate::discover::{CompileFailTarget, Plan, Survey, compile_fail_advice}; +use crate::error::error; +use crate::estimate::project; +use crate::model::Outcome; +use crate::ops::registry::Selection; +use crate::{HashMap, HashSet, Result}; + +/// How many groups a "not run, by …" line names before it starts counting the rest. +const GROUP_LIMIT: usize = 5; + +/// Runs every live mutant in the plan, writing verdicts back onto it. +/// +/// # Errors +/// +/// Returns an error if the tree cannot be prepared, the build cannot be made to succeed, or the +/// baseline does not pass — a failing baseline means every comparison in the run has nothing to +/// compare against. +pub fn run(survey: &Survey, selection: &Selection, config: &Config, events: &mut impl Events) -> Result { + let Measured { + mut plan, + built, + stuck, + dropped, + } = measure(survey, selection, config, events)?; + + // Nothing was live, so nothing was copied, built or measured. The plan still describes every + // mutant that was found and why each one is not being run, which is what the caller reports. + let Some(mut built) = built else { + return Ok(Measured { + plan, + built: None, + stuck, + dropped, + }); + }; + + let stall = Stall { + budget: built.session.stall, + }; + + // Taken from the measured run rather than re-derived, so the sweep judges with exactly the + // oracle the preflight cleared and the baseline timed. + let oracle = built.oracle.clone(); + let scope = oracle.scope(); + + // Loaded here rather than beside the verdicts the record also holds, because the hints are + // neither read nor written by anything before the sweep: each is a guess about the test suite, + // checked by running a test, and there is nothing to check before there are binaries to check + // it with. + let base = gamma_base(&survey.root, config.cache_dir.as_deref()); + + let mut killers = if config.incremental.is_enabled() { + Killers::load(&base, &survey.root) + } else { + Killers::default() + }; + + // Built once and threaded through every phase that needs "which binaries can this package's + // mutants reach": census economics, the workload projection, and the sweep's own scheduling and + // verdict execution. Each of those used to answer that question with its own pass over `plan` + // and `binaries`; a plan's package/binary shape does not change mid-run, so one shared, ordered + // index answers all three identically and for a fraction of the work. + let reach = Reachability::build(&plan, &built.session.binaries, &scope); + + // Taken after the baseline and before the first mutant, because it needs what the baseline + // established — how long a binary takes, which is the budget a censused test is held to — and + // because everything it decides is about how the sweep runs. + let (census_targets, maximum_census_savings) = census_targets(&plan, &reach, &killers); + let census_requested = !config.whole_test_binaries && !census_targets.is_empty() && !maximum_census_savings.is_zero(); + let census_started = Instant::now(); + let census = if census_requested { + census::take( + &built.work, + &built.session.binaries, + &census_targets, + maximum_census_savings, + config.jobs, + stall, + events, + ) + } else { + Census::default() + }; + + // Only recorded when a census actually ran: an absent census is not a census that took no time, + // and the whole question is whether the default selection paid for itself. + if census_requested { + built.session.phases.census = Some(CensusCost { + elapsed: census_started.elapsed(), + walked: census.walked(), + binaries: census_targets.len(), + }); + } + + let work = workload(&plan.mutants, &reach, (!config.whole_test_binaries).then_some(&census)); + let projection = project(&plan.mutants, work, built.session.baseline_wall, built.session.build, config.jobs); + + events.measured(&plan, &built.session, &projection); + + let sweep = Sweep { + timeout_floor: config.timeout_floor, + stall, + jobs: config.jobs, + meter: built.session.metered, + confirm: config.confirm, + census: &census, + }; + + let sweep_started = Instant::now(); + let swept = test_all(&built.work, &mut plan, &reach, sweep, &mut killers, events); + let sweep_elapsed = sweep_started.elapsed(); + + // Written even when the sweep failed. A run that stopped partway still learned which test + // caught every mutant it got to, and discarding that would make an abandoned run cost the next + // one as much as it cost this one. + if config.incremental.is_enabled() { + killers.store(&base, &plan.mutants); + } + + let spent = swept?; + + // `None` when nothing was swept: the phase is recorded as absent rather than as a real phase + // that happened to cost nothing, which is what [`Phases::sweep`] documents its `Option` to mean. + built.session.phases.sweep = spent.map(|spent| SweepCost { + elapsed: sweep_elapsed, + launches: spent.launches, + probes: spent.probes, + }); + + Ok(Measured { + plan, + built: Some(built), + stuck, + dropped, + }) +} + +/// Selected mutation sites each test binary can reach and the most case selection could save. +/// +/// The duration is a serial upper bound: even a perfect census cannot avoid more than one whole +/// baseline run for each reachable mutant/binary pair. +/// +/// A mutant carrying an exact checked killer hint that actually names a binary this mutant can +/// reach does not itself justify or pay for censusing any binary: [`sweep::mutant_cost`](super::sweep) +/// already prices it at one binary baseline (the probe), so a census could only ever shave time off +/// a launch the sweep does not expect to need to run in full. This eligibility test has to mirror +/// [`sweep::judge_ordered`](super::sweep)'s own hint precedence exactly — a hint naming a package +/// or target this run no longer builds, or one the test packages were narrowed away from, is +/// exactly as stale there as it would be here, and `judge_ordered` falls straight through it to the +/// unhinted binaries. A mutant whose hint cannot be matched to any reachable binary is therefore +/// treated as unhinted here too, entering the first pass below and justifying census normally, +/// rather than silently skipping census on the strength of a hint the sweep will never honor. +/// +/// A hint that *is* eligible is only ever a *guess* until the sweep checks it — the probe can still +/// miss, at which point the mutant falls back to every binary that reaches it, exactly as if it had +/// never been hinted. So a validly hinted mutant's ordinal still rides along for free on any +/// binary an unhinted mutant already justified censusing (a shared site): skipping that would leave +/// the site absent from the completed census, which [`Census::selection`] and [`Census::work`] +/// would then read as *proven unreached* rather than merely unmeasured for this mutant — silently +/// turning a stale-hint fallback into a wrong `Uncovered` verdict. A binary no unhinted mutant ever +/// justifies censusing gets no census entry for the hinted mutant either, so a stale hint there +/// conservatively falls back to running that binary whole, never to skipping it as though it were +/// proven safe. +fn census_targets(plan: &Plan, reach: &Reachability<'_>, killers: &Killers) -> (HashMap>, Duration) { + let mut targets: HashMap> = HashMap::default(); + let mut maximum_savings = Duration::ZERO; + + let pending = || { + plan.mutants + .iter() + .filter(|mutant| mutant.ordinal > 0 && mutant.outcome == Outcome::Pending) + }; + + // A hint only excuses a mutant from justifying its own census when it names a binary this + // mutant can actually reach — the exact eligibility test `judge_ordered` itself applies before + // trusting a hint. A stale or unreachable hint (naming a package or target this run dropped, or + // one the test packages were narrowed away from) fails this test exactly as it fails there, so + // such a mutant is treated as unhinted below rather than silently going uncensused. + let eligibly_hinted = |mutant: &crate::model::Mutant| { + killers.hint(&mutant.id).is_some_and(|hint| { + reach + .reachable(&mutant.package) + .is_some_and(|reachable| reachable.iter().any(|binary| hint.names(&binary.package, &binary.target))) + }) + }; + + // Unhinted mutants (including those whose hint is stale or unreachable) alone decide which + // binaries get censused at all, and are the only ones that pay the baseline into + // `maximum_savings` — an eligibly hinted mutant riding along below never grows it. + for mutant in pending().filter(|mutant| !eligibly_hinted(mutant)) { + let Some(reachable) = reach.reachable(&mutant.package) else { + continue; + }; + + for binary in reachable { + let _new = targets.entry(binary.path.clone()).or_default().insert(mutant.ordinal); + maximum_savings = maximum_savings.saturating_add(binary.baseline); + } + } + + // An eligibly hinted mutant only ever adds its ordinal to a binary some unhinted mutant already + // put in `targets` — never a binary of its own, since that would make the census pay to select + // cases for a probe it expects not to need. + for mutant in pending().filter(|mutant| eligibly_hinted(mutant)) { + let Some(reachable) = reach.reachable(&mutant.package) else { + continue; + }; + + for binary in reachable { + if let Some(sites) = targets.get_mut(&binary.path) { + let _new = sites.insert(mutant.ordinal); + } + } + } + + (targets, maximum_savings) +} + +/// What a run worked out before testing a single mutant. +#[derive(Debug)] +pub struct Measured { + /// Every mutant that was found, whether or not it will be run. + pub plan: Plan, + + /// The tree and the measurements, absent when there was nothing live to build for. + pub built: Option, + + /// The builds this run could not make compile, in the order it gave up on them. + /// + /// Empty for a run that built everything it was asked to, which is the ordinary case. A run + /// that could not converge a build would otherwise end here with an error and nothing else — no + /// report, no annotations, no score, after an hour of work on a large workspace. The population + /// it gave up on is recorded as [`Outcome::NotBuilt`] instead, and this is what the caller says + /// out loud and fails the run over. + pub stuck: Vec, + + /// Test packages the preflight had to drop before the tree would check at all. + /// + /// Empty for the ordinary run. When it is not, some package the caller never asked to mutate + /// does not compile, and the run went ahead over the packages that do rather than refusing to + /// start. Their test targets were neither built nor run, so a mutant one of them would have + /// killed is reported as surviving — which is a gap in this run's oracle and not a gap in the + /// suite. Unlike [`Self::stuck`] this does not fail the run; it qualifies it, so it is carried + /// into the report rather than only printed. + pub dropped: Vec, +} + +/// A built tree and what measuring it revealed. +#[derive(Debug)] +pub struct Built { + /// The scratch tree. It owns the built binaries, so dropping it deletes them. + pub work: Workspace, + + /// The timings and test binaries the run works from. + pub session: Session, + + /// The oracle the measured run settled on. + /// + /// Carried rather than recomputed because settling it needs things the caller does not have: + /// the run's package selection, and whatever the preflight had to give up on to make the tree + /// compile at all. Deriving it a second time from the configuration would let the sweep consult + /// binaries the measured run never cleared. + pub oracle: Oracle, +} + +/// The packages whose tests decide verdicts, owned so a later phase can borrow it back. +#[derive(Debug, Clone)] +pub struct Oracle { + /// The `--test-package` restriction, empty when nothing is restricted. + pub packages: Vec, + + /// Whether each mutant is judged only by tests from its own package. + pub package_local: bool, + + /// Whether `--test-workspace` lifted the cap, so every package's tests may decide verdicts. + /// + /// It lifts the cap only. A binary still does not decide a verdict on code it cannot link, so a + /// mutant no binary reaches is uncovered here exactly as it is under the cap. + pub whole_workspace: bool, +} + +impl Oracle { + fn new(packages: Vec, package_local: bool, whole_workspace: bool) -> Self { + Self { + packages, + package_local, + whole_workspace, + } + } + + /// Borrows this as the scope the reachability filters take. + fn scope(&self) -> TestScope<'_> { + TestScope { + packages: &self.packages, + package_local: self.package_local, + whole_workspace: self.whole_workspace, + } + } +} + +/// Checks that the tree compiles before a single mutant is applied to it. +/// +/// This is what lets every later compiler error be absorbed. The staged builds and the baseline +/// compile this same tree with guards written into it, so once this passes, an error that appears +/// afterwards was introduced by a mutant and the rollback loop can withdraw it without troubling +/// anyone. Without it, a tree whose own test targets do not compile fails in the middle of a run +/// and reports a mutant it cannot name, which sends the reader hunting through their own source +/// for a fault that was there before the tool arrived. +/// +/// The packages named are the ones the final build will compile, but computed from the packages +/// this run intends to mutate rather than from the ones that turn out to hold live mutants. That +/// is a superset, and a superset is the safe direction: a target left unchecked here would have +/// its genuine errors absorbed later as though a mutant had caused them. +/// Returns the packages the check had to give up on to succeed at all, empty in the ordinary case. +/// They are the caller's problem as much as the tool's: see [`Converger::preflight`]. +fn preflight( + survey: &Survey, + plan: &Plan, + work: &Workspace, + config: &Config, + converger: &mut Converger, + events: &mut impl Events, +) -> Result { + events.begin("Validating", "Validated", "workspace"); + + let requested: Vec = oracle_packages(&survey.selected, config); + let package_local = config.test_packages.is_empty() && !config.test_workspace; + let scope = TestScope { + packages: &requested, + package_local, + whole_workspace: config.test_workspace, + }; + + let intended = survey.packages(); + let intending: crate::HashSet<&str> = intended.iter().map(String::as_str).collect(); + let checking = reaching_packages(&survey.reach, &intending, &scope); + let cleared = Converger::preflight(work, plan, checking.as_deref(), &intended, config.build, events)?; + events.end(""); + let dropped = cleared.dropped; + + // The narrowed check failed and only the whole workspace passed, so every build this run makes + // asks for the same feature unification. Narrowing again would rebuild the failure the check + // has already shown belongs to the scope rather than to any mutant, and the rollback loop would + // charge it to whichever mutants it happened to blame. + if cleared.whole_workspace { + converger.require_whole_workspace(); + } + + // The check only passed because it stopped asking about those packages, so the rest of the run + // stops asking too. Building test targets this run has decided cannot convict anything would be + // paying again for the compile that just failed, and running them would mean running binaries + // the preflight never cleared — the one thing the preflight exists to prevent. + if dropped.is_empty() { + return Ok(Cleared { + packages: requested, + whole_workspace: config.test_workspace, + dropped, + }); + } + + Ok(Cleared { + packages: narrowed_oracle(requested, intended, &dropped)?, + whole_workspace: config.test_workspace, + dropped, + }) +} + +/// The oracle a preflight retreat leaves behind: what was asked for, less what would not compile. +/// +/// Subtraction, never substitution. An explicit `--test-package` can name packages disjoint from the +/// mutated ones, and replacing the list with the mutated packages would hand the run the very test +/// binaries the caller opted out of while withdrawing every one they asked for — an oracle nobody +/// chose, reported as though they had. +/// +/// An unrestricted oracle is the one case a subtraction cannot be spelled, since an empty list means +/// "everything" rather than "everything so far". There the checked set is named outright, which is a +/// narrowing of "everything" and so is still only ever a subtraction. +/// +/// # Errors +/// +/// When nothing survives the subtraction. Continuing would judge every mutant against no tests at +/// all and report the lot as uncovered, which reads as a fact about the code rather than about a +/// build that did not happen. +fn narrowed_oracle(requested: Vec, intended: Vec, dropped: &[String]) -> Result> { + if requested.is_empty() { + return Ok(intended); + } + + let kept: Vec = requested.into_iter().filter(|package| !dropped.contains(package)).collect(); + + if kept.is_empty() { + let named = dropped.join("`, `"); + + return Err(error!( + "none of the packages this run's tests would come from could be compiled: `{named}`.\n\ + Fix the build there, or name a package that does build with --test-package." + )); + } + + Ok(kept) +} + +/// What the preflight settled: the scope the rest of the run works in, and what it cost to get it. +struct Cleared { + /// The `--test-package` restriction, narrowed if the check had to retreat to succeed. + packages: Vec, + + /// Whether every admitted binary still runs for every mutant. + whole_workspace: bool, + + /// The packages given up on, empty in the ordinary run. See [`Measured::dropped`]. + dropped: Vec, +} + +/// Scans, instruments, builds and measures the baseline, without testing a single mutant. +/// +/// Each package is taken from source to compiled object before the next one starts, in an order +/// where a package always follows what it depends on. That order is forced anyway — a mutant +/// cannot produce a diagnostic until everything its own package depends on compiles clean — and +/// following it deliberately lets a run say which package it is working on, and say it once, before +/// the wait rather than after it. +/// +/// The tree is copied on demand rather than up front. A run whose every mutant turns out to be +/// suppressed has nothing to build, and it should not pay for a copy to discover that. +/// +/// # Errors +/// +/// Returns an error if a file cannot be parsed, the tree cannot be prepared, the build cannot be +/// made to succeed, or the baseline does not pass — a failing baseline means every comparison in +/// the run has nothing to compare against. +pub fn measure(survey: &Survey, selection: &Selection, config: &Config, events: &mut impl Events) -> Result { + let started = Instant::now(); + + let (memory, unbounded) = admit_memory_control(config)?; + + // Checked against what the workspace declares, before anything is copied or compiled. A typo + // here changes which tests get to convict a mutant, so it should cost a second rather than a + // full instrumented build. + if let Some(pattern) = unmatched_test(&survey.tests, &config.include_tests, &config.exclude_tests) { + return Err(error!("no test target matches `{pattern}`; patterns match cargo target names, not test function names").usage()); + } + + // Said before the tree is copied, let alone built. A compile-fail target is expensive per + // mutant rather than once, so by the time its cost is visible in the progress display the run + // has already committed hours to it — and what it looks like from outside is a run that hung. + // Checked against the oracle and its filters so that unrelated workspace packages and targets + // the caller has already excluded are not presented as costs of this run. + warn_about_compile_fail_targets(&survey.compile_fail, &survey.selected, config, events); + + let mut plan = survey.skeleton(); + let mut converger = Converger::guided(ordering_hints(survey, config)); + + // The scratch-tree copy, timed on its own: it is a component of the build's total, but a large + // workspace can spend as much duplicating itself as compiling, and the aggregate cannot say + // which. + let copy_started = Instant::now(); + let mut work = Workspace::prepare(&plan.root, config, events)?; + let copy = copy_started.elapsed(); + + // Settled once, before anything is spawned, so the baseline and the sweep cannot disagree about + // how wide the workload they measure and judge is. + work.calibrate_harness(config.jobs); + + let preflight_started = Instant::now(); + let Cleared { + packages, + whole_workspace, + dropped, + } = preflight(survey, &plan, &work, config, &mut converger, events)?; + let preflight_elapsed = preflight_started.elapsed(); + let oracle = Oracle::new(packages, config.test_packages.is_empty() && !config.test_workspace, whole_workspace); + let scope = oracle.scope(); + + let Staged { anything_live, mut stuck } = converge_stages(survey, selection, &mut plan, &mut converger, &work, config, events)?; + + plan.sort(); + converger.plan_reordered(); + + // Nothing was live anywhere, or everything live was in a build that could not be made to + // compile. Either way there is nothing left to build, measure or run, and the verdicts the + // abandoned population carries are already written onto the plan. + if !anything_live { + converger.settle(&mut plan); + + return Ok(Measured { + plan, + built: None, + stuck, + dropped, + }); + } + + // One line for the whole fixed cost that is left. The test binaries are built and then + // immediately run with no mutant active, and neither half means anything without the other: + // the build is what makes a baseline possible, and the baseline is what says the build was + // worth having. + events.begin("Baselining", "Baseline", "building the test binaries and running the suite"); + + // The staged builds compiled libraries only. This is the build that compiles the test targets + // and settles the run, and it withdraws whatever only a test target could have revealed. Only + // the packages whose tests can actually be selected are asked for: the rest would be compiled, + // baselined and never consulted. The preflight check cleared this same set, narrowed from the + // packages that turned out to hold live mutants rather than from those the run set out to + // mutate, so it is a subset of what was checked. + let select = build_packages(&plan, &scope); + let mut build = converger.finish(&work, &mut plan, select.as_deref(), config.build, events)?; + + // The build that decides the run could not be made to compile, so there is no test binary to + // judge anything with and nothing left to measure. Every mutant still live carries + // `NotBuilt`, which is what tells the reader that these are mutants nobody ran rather than + // mutants the suite let through, and the run reports that rather than exiting empty-handed. + if let Some(abandoned) = build.stuck { + let count = abandoned.ordinals.len(); + + stuck.push(describe_stuck(&plan, select.as_deref().unwrap_or_default(), &abandoned)); + events.complete(&format!( + "the build could not be made to compile, {} not run", + crate::report::quantity(count, "mutant") + )); + + return Ok(Measured { + plan, + built: None, + stuck, + dropped, + }); + } + + // Before the baseline, so the shares `apportion` computes describe the suite that will actually + // run. A run with nothing left to run it cannot decide anything: every mutant would survive + // unopposed and the report would read as a total failure of the test suite rather than as the + // filter having eaten it. + let present = build.binaries.len(); + + restrict(&mut build.binaries, &config.include_tests, &config.exclude_tests); + + let filtered = present.saturating_sub(build.binaries.len()); + + if build.binaries.is_empty() && present > 0 { + return Err(error!("`--include-test` and `--exclude-test` left no test target to decide a verdict").usage()); + } + + let build_time = started.elapsed(); + + // Armed after the build and before the baseline. The metadata nextest is handed describes + // binaries that do not exist until the build has run, and the baseline has to be measured + // through the same runner that will judge every mutant — a baseline taken one way and compared + // against verdicts reached the other measures nothing. + if config.nextest { + work.arm_nextest(&build.binaries)?; + } + + let baseline = take_baseline(&work, &mut build.binaries, config, &memory, events)?; + + warn_about_an_empty_oracle(&plan, &build.binaries, &scope, config.test_packages.is_empty(), &dropped, events); + + let stall = calibrate_stall(&baseline, config); + + calibrate(&mut build.binaries, config, &memory); + + let session = Session { + baseline: baseline.elapsed, + baseline_wall: baseline.wall, + tests: baseline.tests, + quiet: baseline.quiet, + stall: stall.budget, + build: build_time, + peak: baseline.peak, + metered: memory.measuring(), + unbounded, + withdrawn: build.withdrawn, + census: build.census, + rounds: build.rounds, + rounds_taken: build.history, + binaries: build.binaries, + scratch: work.base().to_owned(), + filtered, + widened: build.widened, + ordering: build.ordering, + phases: Phases { + copy, + preflight: preflight_elapsed, + census: None, + sweep: None, + }, + }; + + // Everything that could have failed has. What was built is now worth keeping, so that the next + // run in this workspace is incremental rather than starting cold. + work.settle(); + + Ok(Measured { + plan, + built: Some(Built { work, session, oracle }), + stuck, + dropped, + }) +} + +/// Renders what a build that could not be made to compile cost, and where it got stuck. +/// +/// The reason is repeated verbatim: the rollback-limit failure's withdrawal series and its +/// falling-or-flat advice, and the unattributed failure's excerpt of what cargo actually said, are +/// the only text that tells a reader whether to raise a limit, fix a build script, or look +/// somewhere else entirely. Losing it would leave a run that says it got stuck without saying on +/// what. +/// +/// Grouped by mutator and by enclosing item, because "we could not build 900 mutants" is not +/// actionable and "every one of them was the `stmt.delete` mutator, all in `parser::Lexer`" is: it +/// names one operator to exclude, or one module to look at, and either of those turns a dead run +/// into a run that finishes. +fn describe_stuck(plan: &Plan, packages: &[String], abandoned: &Abandoned) -> String { + let ordinals: crate::HashSet = abandoned.ordinals.iter().copied().collect(); + let affected: Vec<&crate::model::Mutant> = plan.mutants.iter().filter(|mutant| ordinals.contains(&mutant.ordinal)).collect(); + + let where_ = if packages.is_empty() { + "the workspace".to_owned() + } else { + format!("{} {}", crate::report::quantity(packages.len(), "package"), packages.join(", ")) + }; + + let mut described = format!( + "the build for {where_} could not be made to compile, so {} never ran and {} reported as `notbuilt` rather than as survivors.", + crate::report::quantity(abandoned.ordinals.len(), "mutant"), + if abandoned.ordinals.len() == 1 { "is" } else { "are" } + ); + + for (label, counted) in [ + ("mutator", tally(&affected, |mutant| mutant.mutator.to_string())), + ("scope", tally(&affected, |mutant| mutant.item_path.to_string())), + ] { + if !counted.is_empty() { + let _ = write!(described, "\nNot run, by {label}: {}.", counted.join(", ")); + } + } + + described.push('\n'); + described.push_str(&abandoned.reason); + described +} + +/// How many mutants fall under each key, commonest first, capped so one line stays one line. +fn tally(mutants: &[&crate::model::Mutant], key: impl Fn(&crate::model::Mutant) -> String) -> Vec { + let mut counts: crate::HashMap = crate::HashMap::default(); + + for mutant in mutants { + *counts.entry(key(mutant)).or_default() += 1; + } + + let mut ranked: Vec<(String, usize)> = counts.into_iter().collect(); + + // Ties broken by name so that two runs over the same tree read the same way; a hash map's order + // is not something a report should inherit. + ranked.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0))); + + let rest = ranked.len().saturating_sub(GROUP_LIMIT); + let mut rendered: Vec = ranked + .iter() + .take(GROUP_LIMIT) + .map(|(name, count)| format!("{name} ({count})")) + .collect(); + + if rest > 0 { + rendered.push(format!("and {rest} more")); + } + + rendered +} + +/// What the staged builds settled before the run reaches the build that decides it. +struct Staged { + /// Whether any stage got as far as compiling live mutants. + anything_live: bool, + + /// One rendered diagnostic per stage the run could not build. + stuck: Vec, +} + +/// The mutants an earlier run could not compile, offered to the build as an order rather than a fact. +/// +/// Two sources, unioned: the scratch record left by whatever ran here last, and the artifact the +/// workspace checks in. Neither is filtered by build context, and that is the point — this is the +/// tier that survives a context mismatch, because being wrong about the order costs the order and +/// nothing else. What comes back is a set of mutant ids; [`Converger`] decides what to do with it, +/// and every mutant named is still built, still judged, and still scored exactly as if it had never +/// appeared here. See [`crate::exec::OrderingHints`]. +/// +/// Empty when incremental mode is off. `--incremental no` is what a caller reaches for when they suspect the +/// tool is remembering something it should not, so it turns off remembering — including the tiers +/// that could not have affected the answer. +fn ordering_hints(survey: &Survey, config: &Config) -> crate::HashSet { + if !config.incremental.is_enabled() { + return crate::HashSet::default(); + } + + let base = gamma_base(&survey.root, config.cache_dir.as_deref()); + let record = crate::discover::RunRecord::load(&base); + let checked_in = crate::discover::Hints::load(&survey.root); + + record + .ordering() + .into_iter() + .map(crate::model::MutantId::new) + .chain(checked_in.ordering().into_iter().map(crate::model::MutantId::new)) + .collect() +} + +/// Scans, links and builds each stage in dependency order. +/// +/// A stage that cannot be made to compile does not end the run. Its own mutants come back out of +/// the tree, which restores the very sources the preflight check already proved compile, so every +/// later stage is asked of a tree in no worse a state than the one this stage started from — and a +/// run that gets stuck early still ends up saying what it learned everywhere else. Stopping at the +/// first failure would answer "we got stuck in the first crate" with nothing at all about the +/// twenty that follow it. +fn converge_stages( + survey: &Survey, + selection: &Selection, + plan: &mut Plan, + converger: &mut Converger, + work: &Workspace, + config: &Config, + events: &mut impl Events, +) -> Result { + let mut ordinals = 0_u32; + let mut staged = Staged { + anything_live: false, + stuck: Vec::new(), + }; + + for stage in &crate::discover::stages(&survey.packages(), &survey.reach) { + let name = stage.join(", "); + + // Named on the way in, before its files have even been read. Scanning and then compiling a + // large crate is the longest a run goes without saying anything, and what makes that wait + // legible is knowing whose wait it is. + events.begin("Mutating", "Mutated", &name); + + let mut live = 0_usize; + + for package in stage { + let scanned = survey.scan(Some(package), selection, &mut ordinals)?; + + live = live.saturating_add(scanned.mutants.iter().filter(|mutant| mutant.ordinal > 0).count()); + plan.absorb(scanned); + } + + // A package with nothing to run is still named. A crate that quietly takes no part in a run + // is worth noticing, and leaving it out of the sequence would make it look like it had + // simply not been looked at. + if live == 0 { + events.end(", no mutants"); + continue; + } + + for package in stage { + work.link_runtime(package, &plan.files)?; + } + + let before = converger.withdrawn(); + if let Some(abandoned) = converger.stage(work, plan, stage, config.build, events)? { + let count = abandoned.ordinals.len(); + + staged.stuck.push(describe_stuck(plan, stage, &abandoned)); + events.end(&format!( + ", the build could not be made to compile, {} not run", + crate::report::quantity(count, "mutant") + )); + + continue; + } + + staged.anything_live = true; + + // The count that closes the line is what survived compilation, which is why it waits for + // the build. A mutant that could not compile is a fact about the tool rather than about the + // code, and the summary accounts for all of them once. + let viable = live.saturating_sub(converger.withdrawn().saturating_sub(before)); + + events.end(&format!(", {}", crate::report::quantity(viable, "viable mutant"))); + } + + Ok(staged) +} + +/// Says so when a test target is going to run the compiler once per mutant. +/// +/// Named and never removed. `trybuild` asserts exact compiler output, so on a proc-macro crate such +/// a target is often the *primary* oracle — a mutant that corrupts a diagnostic message is caught +/// there and nowhere else. Excluding it automatically would gut the oracle for the code the +/// technique suits best, and the mutants would come back as survivors rather than as anything +/// visibly wrong. Whether the catch rate justifies the price is the caller's to weigh, so the run +/// states the price and the flag and leaves the decision where it belongs. +fn warn_about_compile_fail_targets(targets: &[CompileFailTarget], selected: &[String], config: &Config, events: &mut dyn Events) { + let oracle = oracle_packages(selected, config); + let admitted: Vec<_> = targets + .iter() + .filter(|target| { + (config.test_workspace || oracle.contains(&target.package)) + && admits_target(&target.target, &config.include_tests, &config.exclude_tests) + }) + .cloned() + .collect(); + + if let Some(warning) = compile_fail_advice(&admitted) { + events.warn(&warning); + } +} + +/// How many packages a warning names before it starts counting them instead. +const NAMED_HELPERS: usize = 3; + +/// Says so when the oracle cap has left the mutated code with no tests at all. +/// +/// Called after the baseline, because the emptiness it reports is a count of tests and nothing +/// before the baseline has counted any. +/// +/// The default oracle is the tests `cargo test` would run here, which is the right cap for the +/// ordinary workspace and the wrong one for a workspace that keeps its tests somewhere else — a +/// package of integration tests, or a parent crate that exercises a private implementation crate +/// through its own suite. There the cap withdraws the only tests that could convict anything, and +/// the run is honest about it: every mutant comes back uncovered. Honest is not the same as +/// useful, though, and "nothing tests this" reads like a verdict on the code rather than on the +/// scope the run was given. +/// +/// Only said when the cap was inferred rather than asked for, and only when some package outside it +/// could actually have reached the mutated code. A caller who named `--test-package` has already +/// answered this question, and a workspace where nothing else links the code has nothing to offer. +/// +/// A preflight retreat is the exception, and it displaces the cap as the explanation. It withdrew +/// packages the caller never asked to lose, so it is worth reporting however the oracle was chosen, +/// and naming the cap instead would send the reader to widen a scope that was never the problem. +fn warn_about_an_empty_oracle( + plan: &Plan, + binaries: &[TestBinary], + scope: &TestScope<'_>, + inferred: bool, + dropped: &[String], + events: &mut dyn Events, +) { + // A retreat is worth saying whoever chose the oracle, because it withdrew packages the caller + // never asked to lose. The cap is only worth saying when the run inferred it. + if dropped.is_empty() && (!inferred || scope.whole_workspace) { + return; + } + + let mutated: crate::HashSet<&str> = plan + .mutants + .iter() + .filter(|mutant| mutant.ordinal > 0 && mutant.outcome == Outcome::Pending) + .map(|mutant| &*mutant.package) + .collect(); + + if mutated.is_empty() { + return; + } + + // A binary that announced tests it never counted is not evidence of an empty suite, so `None` + // is read as "there may well be tests here" and stops the warning. Saying this over a suite + // that does convict things would be worse than saying nothing. + let judged = mutated.iter().any(|package| { + binaries + .iter() + .any(|binary| binary.tests != Some(0) && reaches(binary, package, plan, scope)) + }); + + if judged { + return; + } + + // The retreat is the cause whenever there was one: the packages it gave up on are exactly the + // ones whose tests are no longer there to convict, so blaming the cap would send the reader to + // widen a scope that was never the problem — and the advice the cap gives, `--test-package

`, + // would name a package that has just failed to compile. + if !dropped.is_empty() { + let named = dropped.join("`, `"); + + events.warn(&format!( + "no test in this run's scope reaches the code being mutated, so every mutant will be reported as uncovered. The preflight could not compile `{named}`, so their tests were withdrawn from the oracle. Fixing the build there restores them" + )); + + return; + } + + let mut helpers: Vec<&str> = plan + .reach + .iter() + .filter(|(package, reachable)| !scope.admits(package) && reachable.iter().any(|name| mutated.contains(name.as_str()))) + .map(|(package, _reachable)| package.as_str()) + .collect(); + + if helpers.is_empty() { + return; + } + + helpers.sort_unstable(); + + // Named rather than counted, and the flag spelled out with a real package in it, because the + // whole point of the warning is that the next command is one edit away. + let shown = helpers.len().min(NAMED_HELPERS); + let named = helpers[..shown].join("`, `"); + let rest = helpers.len() - shown; + let others = if rest == 0 { String::new() } else { format!(" and {rest} more") }; + + events.warn(&format!( + "no test in this run's scope reaches the code being mutated, so every mutant will be reported as uncovered. By default each mutant is judged only by its own package, and `{named}`{others} also links this code. Add `--test-package {}` to let its tests decide a verdict, or `--test-workspace` for every package's", + helpers[0] + )); +} + +/// Runs the suite once with no mutant active, or reports that nothing was measured. +/// +/// The measurement is taken at the sweep's own concurrency, because calibrating on an idle machine +/// and spending on a loaded one makes every budget derived from it too tight — and a mutant that +/// loses that race is recorded as a timeout, which lowers the score and can fail the run. +fn take_baseline( + work: &Workspace, + binaries: &mut [TestBinary], + config: &Config, + memory: &MemoryPolicy, + events: &mut impl Events, +) -> Result { + if !config.baseline { + events.complete("no baseline was measured"); + + return Ok(Baseline { + elapsed: Duration::ZERO, + wall: Duration::ZERO, + quiet: Duration::ZERO, + tests: None, + peak: None, + }); + } + + let request = MemoryRequest { + meter: memory.measuring(), + limit: memory.baseline_limit, + }; + + let total = binaries.len(); + let unit = if total == 1 { "test binary" } else { "test binaries" }; + let mut completed = 0_usize; + + events.phase_progress(completed, total, unit); + + let measured = measure_baseline(work, binaries, request, config.jobs, || { + completed += 1; + events.phase_progress(completed, total, unit); + })?; + + // Reported after the fact rather than before, because the figures everything downstream is + // derived from — each test binary's timeout is scaled from its baseline by `--test-timeout-multiplier`, and + // the stall budget is calibrated from the longest silence within the baseline — only exist once the suite + // has actually run. + events.complete(&describe(&measured)); + + Ok(measured) +} + +/// Settles what memory control this run can actually deliver, refusing or degrading as appropriate. +/// +/// Two things can make the configured policy impossible: a host with no way to account for a whole +/// process tree, and a run with no baseline to calibrate a ceiling from. What happens then depends +/// on who asked. Someone who passed `--memory` did so because an unbounded mutant would cost them +/// something, and a run that quietly gave them nothing would be discovered only by the thing they +/// were trying to prevent — so that is an error. Someone who passed nothing has the default, and +/// refusing to produce a mutation score because this machine has no cgroup delegation would be an +/// obstruction rather than a safeguard — so that degrades to no memory control, out loud. +/// +/// It is said out loud rather than silently because the protection is the kind whose absence is +/// invisible until it matters. A user who believes their machine is protected and finds out +/// otherwise mid-run is worse off than one who was told plainly at the start. +fn admit_memory_control(config: &Config) -> Result<(MemoryPolicy, Option)> { + settle_memory_control(config, memory::support()) +} + +/// Decides what memory control a run gets, given what the host can deliver. +/// +/// Host support is passed in rather than probed here so that every branch is decided by its +/// arguments alone: whether a machine happens to have a delegated cgroup then has no bearing on +/// which of these paths a test exercises. +fn settle_memory_control(config: &Config, support: Result<(), String>) -> Result<(MemoryPolicy, Option)> { + let policy = config.memory; + + if !policy.measuring() { + return Ok((policy, None)); + } + + if let Err(reason) = support { + if policy.insisted() { + return Err(error!( + "memory control was asked for, but it is not available here: {reason}.\n\ + Run with `--memory off` to continue without it." + )); + } + + return Ok((policy.disabled(), Some(reason))); + } + + // A ceiling is calibrated from the baseline. Without one there is no measurement to calibrate + // from, and a number invented here would be presented with exactly the confidence of a + // measured one. + if policy.enforcing() && !config.baseline && policy.limit.is_none() { + if policy.insisted() { + return Err(error!( + "`--memory enforce` derives each test binary's ceiling from what it used during the \ + baseline, and `--no-baseline` means there is no such measurement.\n\ + Pass `--memory-limit` to state a ceiling outright, or drop `--no-baseline`." + )); + } + + return Ok(( + policy.disabled(), + Some("`--no-baseline` leaves no measurement to derive a ceiling from".to_owned()), + )); + } + + Ok((policy, None)) +} + +/// Describes what the baseline measured, for the line that replaces its announcement. +/// +/// The test count is omitted rather than guessed when no harness announced one, which is what a +/// target built with `harness = false` does. +fn describe(baseline: &Baseline) -> String { + let duration = format!("{:.1?}", baseline.wall); + let ran = baseline.tests.map_or_else( + || format!("the suite passed in {duration}"), + |tests| format!("{} ran in {duration}", crate::report::quantity(tests, "test")), + ); + + // Reported whenever it was measured, whether or not anything is being enforced. A project + // deciding whether a ceiling is worth turning on needs to know what its suite actually uses, + // and this line is where that number is cheapest to notice. + match baseline.peak { + Some(peak) => format!("{ran} with a peak of {}", crate::report::bytes(peak)), + None => ran, + } +} + +/// Derives a mutant's stall budget from what the baseline measured. +/// +/// Extracted from [`measure`] because this number decides every stall verdict the +/// run reaches, and it is the only place the tuning constants in [`Config::default`] are read. +/// Inline in `measure` it could only be exercised by a full run, which means the values a user +/// actually runs with were derived by code no assertion could reach; here it is one call with +/// no I/O. +fn calibrate_stall(baseline: &Baseline, config: &Config) -> Stall { + // Without a baseline there is no calibration, so a stall cannot be detected and every mutant + // waits out its whole budget. + if config.stall && config.baseline { + Stall::calibrated(baseline.quiet, config.stall_factor, config.stall_floor) + } else { + Stall::NONE + } +} + +#[cfg(test)] +mod tests { + use camino::{Utf8Path, Utf8PathBuf}; + + use super::*; + use crate::discover::Killer; + use crate::exec::memory::{Demand, MemoryControl}; + use crate::fixtures; + use crate::model::Mutant; + use crate::testing::Recorder; + + /// A baseline with the given elapsed time and quiet period, and nothing else measured. + /// + /// The two fields the calibration reads, so a test can state exactly what it is calibrating + /// from without standing up a run. + const fn measured(elapsed: Duration, quiet: Duration) -> Baseline { + Baseline { + elapsed, + wall: Duration::ZERO, + quiet, + tests: None, + peak: None, + } + } + + #[test] + fn the_default_stall_budget_is_ten_times_the_measured_quiet_period() { + let config = Config::default(); + let baseline = measured(Duration::from_mins(10), Duration::from_secs(3)); + + let stall = calibrate_stall(&baseline, &config); + + assert_eq!( + stall.budget, + Some(Duration::from_secs(30)), + "3s quiet x {} factor should be 30s; the default factor is now {}", + config.stall_factor, + config.stall_factor + ); + } + + #[test] + fn a_suite_that_never_goes_quiet_gets_the_stall_floor_rather_than_zero() { + let config = Config::default(); + let baseline = measured(Duration::from_mins(10), Duration::ZERO); + + let stall = calibrate_stall(&baseline, &config); + + assert_eq!( + stall.budget, + Some(Duration::from_secs(5)), + "a zero quiet period scaled by any factor is still zero, so the floor must win; \ + the default floor is now {:?}", + config.stall_floor + ); + } + + #[test] + fn stall_detection_stands_down_when_there_is_no_baseline_to_calibrate_from() { + let config = Config { + baseline: false, + ..Config::default() + }; + + let stall = calibrate_stall(&measured(Duration::from_secs(100), Duration::from_secs(1)), &config); + + assert_eq!(stall.budget, None, "a budget derived from a baseline that never ran is a guess"); + } + + #[test] + fn stall_detection_stands_down_when_it_is_switched_off() { + let config = Config { + stall: false, + ..Config::default() + }; + + let stall = calibrate_stall(&measured(Duration::from_secs(100), Duration::from_secs(1)), &config); + + assert_eq!(stall.budget, None, "--no-stall must disable the detector outright"); + } + + /// One live mutant of the given mutator, in the given enclosing item. + fn stuck_mutant(ordinal: u32, mutator: &str, item_path: &str) -> Mutant { + Mutant { + id: format!("m{ordinal}").into(), + ordinal, + mutator: (mutator.to_owned()).into(), + item_path: (item_path.to_owned()).into(), + original: "a < b".to_owned().into(), + replacement: "a <= b".to_owned().into(), + outcome: Outcome::NotBuilt, + ..fixtures::mutant() + } + } + + /// A plan holding the given mutants and nothing else. + fn stuck_plan(mutants: Vec) -> Plan { + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: Utf8PathBuf::from("/workspace"), + files: Vec::new(), + mutants, + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + } + } + + /// A build that could not be made to compile is described by where it got stuck, not merely by + /// the fact that it did. + /// + /// "We could not build 900 mutants" is not something anyone can act on. "All of them were the + /// same mutator, all in the same module" names one operator to exclude or one module to look + /// at, and either turns a dead run into one that finishes. + #[test] + fn a_stuck_build_is_described_by_mutator_and_by_scope() { + let plan = stuck_plan(vec![ + stuck_mutant(1, "relational.lt_to_le", "subject::less"), + stuck_mutant(2, "relational.lt_to_le", "subject::less"), + stuck_mutant(3, "arith.add_to_sub", "subject::sum"), + stuck_mutant(4, "relational.lt_to_le", "subject::sum"), + ]); + let abandoned = Abandoned { + reason: "Mutants blamed in the last rounds of this build: 9, 9, 9.".to_owned(), + ordinals: vec![1, 2, 3, 4], + }; + + let described = describe_stuck(&plan, &["subject".to_owned()], &abandoned); + + assert!(described.contains("4 mutants"), "{described}"); + assert!(described.contains("subject"), "{described}"); + assert!( + described.contains("Not run, by mutator: relational.lt_to_le (3), arith.add_to_sub (1)."), + "{described}" + ); + assert!( + described.contains("Not run, by scope: subject::less (2), subject::sum (2)."), + "{described}" + ); + + // The diagnostic that would have been the run's error message, kept word for word: the + // per-round series is the only thing that says whether raising the limit would help. + assert!( + described.contains("Mutants blamed in the last rounds of this build: 9, 9, 9."), + "{described}" + ); + } + + /// A whole-workspace build names the workspace rather than an empty package list. + #[test] + fn a_stuck_whole_workspace_build_says_so() { + let plan = stuck_plan(vec![stuck_mutant(1, "relational.lt_to_le", "subject::less")]); + let abandoned = Abandoned { + reason: "the instrumented tree does not compile".to_owned(), + ordinals: vec![1], + }; + + let described = describe_stuck(&plan, &[], &abandoned); + + assert!(described.contains("the build for the workspace"), "{described}"); + assert!(described.contains("1 mutant never ran"), "{described}"); + } + + /// Only the mutants the build actually gave up on are counted, not every mutant in the plan. + #[test] + fn a_stuck_build_counts_only_what_it_gave_up_on() { + let plan = stuck_plan(vec![ + stuck_mutant(1, "relational.lt_to_le", "subject::less"), + stuck_mutant(2, "arith.add_to_sub", "subject::sum"), + ]); + let abandoned = Abandoned { + reason: "stuck".to_owned(), + ordinals: vec![2], + }; + + let described = describe_stuck(&plan, &["subject".to_owned()], &abandoned); + + assert!(described.contains("Not run, by mutator: arith.add_to_sub (1)."), "{described}"); + assert!(!described.contains("relational.lt_to_le"), "{described}"); + } + + /// A long tail is counted rather than printed, so one line stays one line. + #[test] + fn a_long_list_of_groups_is_capped_and_the_rest_counted() { + let mutants: Vec = (1..=8) + .map(|ordinal| stuck_mutant(ordinal, &format!("op.{ordinal}"), "subject::f")) + .collect(); + let plan = stuck_plan(mutants); + let abandoned = Abandoned { + reason: "stuck".to_owned(), + ordinals: (1..=8).collect(), + }; + + let described = describe_stuck(&plan, &["subject".to_owned()], &abandoned); + + assert!(described.contains("and 3 more"), "{described}"); + } + + #[test] + fn a_baseline_without_a_harness_count_is_described_by_elapsed_time_only() { + let baseline = Baseline { + elapsed: Duration::from_millis(1500), + wall: Duration::from_millis(500), + quiet: Duration::ZERO, + tests: None, + peak: None, + }; + + // Custom test harnesses may never announce a count; reporting the elapsed fixed cost is + // still useful, but inventing a count would be misleading. + assert_eq!(describe(&baseline), "the suite passed in 500.0ms"); + } + + /// A baseline whose peak was measured is described with it, whether or not a ceiling is + /// actually enforced. + /// + /// A project deciding whether a ceiling is worth turning on needs to know what its suite + /// actually uses, and this line is where that number is cheapest to notice; omitting it whenever + /// enforcement happens to be off would hide the one number that answers that question. + #[test] + fn a_baseline_with_a_measured_peak_is_described_with_it() { + let baseline = Baseline { + elapsed: Duration::from_millis(500), + wall: Duration::from_millis(500), + quiet: Duration::ZERO, + tests: Some(4), + peak: Some(1024 * 1024), + }; + + assert_eq!(describe(&baseline), "4 tests ran in 500.0ms with a peak of 1.0 MB"); + } + + /// Memory control that was never asked for needs nothing from the host, and is returned + /// exactly as configured. + /// + /// Checking host support before even looking at whether measurement was requested would turn a + /// run that never asked for memory control into one that fails, or degrades with a note, on a + /// machine that has never had anything to do with the feature at all. + #[test] + fn memory_control_that_was_never_asked_for_needs_no_host_support() { + let config = Config { + memory: MemoryPolicy { + control: MemoryControl::Off, + ..MemoryPolicy::default() + }, + ..Config::default() + }; + + let (settled, note) = + settle_memory_control(&config, Err("no cgroup here".to_owned())).expect("a policy that asks for nothing cannot fail"); + + assert!(!settled.measuring()); + assert_eq!(note, None); + } + + /// A stated memory policy this host cannot deliver is an error, not a silent degradation. + /// + /// Someone who passed `--memory` did so because an unbounded mutant would cost them something, + /// and a run that quietly gave them nothing would be discovered only by the thing they were + /// trying to prevent. + #[test] + fn a_stated_memory_policy_this_host_cannot_deliver_is_an_error() { + let config = Config { + memory: MemoryPolicy { + control: MemoryControl::Measure, + demand: Demand::Stated, + ..MemoryPolicy::default() + }, + ..Config::default() + }; + + let failure = settle_memory_control(&config, Err("no cgroup here".to_owned())) + .expect_err("a stated policy this host cannot deliver must error"); + + assert!(failure.to_string().contains("--memory off"), "{failure}"); + assert!( + failure.to_string().contains("no cgroup here"), + "the error must repeat what the host said, {failure}" + ); + } + + /// A defaulted memory policy this host cannot deliver degrades to no memory control, and says + /// why rather than merely stating that it degraded. + /// + /// Nobody asked for this by name, so refusing to produce a mutation score at all because this + /// machine has no cgroup delegation would be an obstruction rather than a safeguard. + #[test] + fn a_defaulted_memory_policy_this_host_cannot_deliver_degrades_with_a_note() { + let (settled, note) = settle_memory_control(&Config::default(), Err("no cgroup here".to_owned())) + .expect("a defaulted policy degrades rather than errors"); + + assert!(!settled.measuring()); + assert_eq!( + note.as_deref(), + Some("no cgroup here"), + "a degraded policy must say why, not merely that it degraded" + ); + } + + /// A stated policy this host *can* deliver still errors when there is no baseline to derive a + /// ceiling from, because a ceiling invented with no measurement behind it would be presented + /// with exactly the confidence of a measured one. + #[test] + fn a_stated_policy_this_host_can_deliver_still_errors_with_no_baseline_to_derive_a_ceiling_from() { + let config = Config { + memory: MemoryPolicy { + control: MemoryControl::Enforce, + demand: Demand::Stated, + ..MemoryPolicy::default() + }, + baseline: false, + ..Config::default() + }; + + let failure = + settle_memory_control(&config, Ok(())).expect_err("a stated policy with no baseline to derive a ceiling from must error"); + + assert!(failure.to_string().contains("--memory-limit"), "{failure}"); + } + + /// A defaulted policy this host *can* deliver degrades, without a baseline to derive a ceiling + /// from, instead of erroring — nobody asked for enforcement by name, so there is a note rather + /// than a hard stop. + #[test] + fn a_defaulted_policy_this_host_can_deliver_degrades_with_no_baseline_to_derive_a_ceiling_from() { + let config = Config { + memory: MemoryPolicy { + control: MemoryControl::Enforce, + ..MemoryPolicy::default() + }, + baseline: false, + ..Config::default() + }; + + let (settled, note) = settle_memory_control(&config, Ok(())).expect("a defaulted policy degrades rather than errors"); + + assert!(!settled.measuring()); + assert!(note.is_some_and(|note| note.contains("--no-baseline")), "the note must say why"); + } + + /// A stated ceiling with no baseline is admitted: the ceiling was measured by whoever passed + /// it, so there is nothing left to derive. + #[test] + fn a_stated_ceiling_needs_no_baseline_to_derive_one_from() { + let config = Config { + memory: MemoryPolicy { + control: MemoryControl::Enforce, + limit: Some(1 << 30), + ..MemoryPolicy::default() + }, + baseline: false, + ..Config::default() + }; + + let (settled, note) = settle_memory_control(&config, Ok(())).expect("a stated ceiling needs no baseline"); + + assert!(settled.measuring()); + assert_eq!(note, None); + } + + fn target_in(package: &str, name: &str) -> CompileFailTarget { + CompileFailTarget { + package: package.to_owned(), + target: name.to_owned(), + harness: "trybuild".to_owned(), + } + } + + fn target(name: &str) -> CompileFailTarget { + target_in("routerama", name) + } + + fn config_with(exclude: &[&str]) -> Config { + Config { + exclude_tests: exclude.iter().map(|pattern| (*pattern).to_owned()).collect(), + ..Config::default() + } + } + + #[test] + fn a_compile_fail_target_is_named_before_anything_is_built() { + let mut events = Recorder::default(); + + warn_about_compile_fail_targets( + &[target("router_compile_fail")], + &["routerama".to_owned()], + &config_with(&[]), + &mut events, + ); + + let [warning] = events.warnings.as_slice() else { + panic!("one admitted target is one warning, got {:?}", events.warnings); + }; + + assert!(warning.contains("router_compile_fail"), "{warning}"); + assert!(warning.contains("--exclude-test router_compile_fail"), "{warning}"); + } + + /// A caller who has already acted on the advice must not be given it again, or the warning + /// becomes noise that is scrolled past on the run where it matters. + #[test] + fn a_target_already_excluded_is_not_warned_about() { + let mut events = Recorder::default(); + + warn_about_compile_fail_targets( + &[target("router_compile_fail")], + &["routerama".to_owned()], + &config_with(&["router_compile_fail"]), + &mut events, + ); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } + + /// A glob is how these are excluded in practice, and it has to count as having acted. + #[test] + fn a_target_excluded_by_a_glob_is_not_warned_about() { + let mut events = Recorder::default(); + + warn_about_compile_fail_targets( + &[target("router_compile_fail")], + &["routerama".to_owned()], + &config_with(&["*_compile_fail"]), + &mut events, + ); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } + + /// Targets in unrelated workspace packages are not part of the default oracle and must not + /// prompt the caller to exclude tests that this run will never execute. + #[test] + fn a_compile_fail_target_outside_the_oracle_is_not_warned_about() { + let mut events = Recorder::default(); + + warn_about_compile_fail_targets( + &[target("router_compile_fail"), target_in("internity", "compile_fail")], + &["routerama".to_owned()], + &config_with(&[]), + &mut events, + ); + + let [warning] = events.warnings.as_slice() else { + panic!("only the selected package should produce a warning, got {:?}", events.warnings); + }; + + assert!(warning.contains("router_compile_fail"), "{warning}"); + assert!(!warning.contains("internity"), "{warning}"); + assert!(!warning.contains("--exclude-test compile_fail"), "{warning}"); + } + + /// The ordinary case is a workspace with no such target at all, which must say nothing. + #[test] + fn a_workspace_without_one_is_silent() { + let mut events = Recorder::default(); + + warn_about_compile_fail_targets(&[], &["routerama".to_owned()], &config_with(&[]), &mut events); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } + + /// A workspace where `app` links `core`, and both are their own reach. + fn oracle_plan(mutating: &str) -> Plan { + let mut reach: crate::HashMap> = crate::HashMap::default(); + + let _app = reach.insert("app".to_owned(), ["app".to_owned(), "core".to_owned()].into_iter().collect()); + let _core = reach.insert("core".to_owned(), core::iter::once("core".to_owned()).collect()); + + let mut pending = Mutant { + ordinal: 1, + outcome: Outcome::Pending, + ..crate::testing::advise_fixture::mutant("a.rs", "arith.add_to_sub", Outcome::Pending, 0) + }; + + pending.package = mutating.to_owned().into(); + + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: Utf8PathBuf::from("/w"), + files: Vec::new(), + mutants: vec![pending], + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach, + specs: crate::HashMap::default(), + } + } + + fn oracle_binary(package: &str, tests: Option) -> TestBinary { + TestBinary { + package: package.to_owned(), + tests, + ..crate::testing::test_binary("/tmp/t") + } + } + + fn capped(packages: &[String]) -> TestScope<'_> { + TestScope { + packages, + package_local: false, + whole_workspace: false, + } + } + + #[test] + fn census_targets_only_selected_pending_sites_and_relevant_binaries() { + let mut plan = oracle_plan("core"); + let mut settled = plan.mutants[0].clone(); + settled.ordinal = 2; + settled.outcome = Outcome::Killed; + plan.mutants.push(settled); + + let mut core = oracle_binary("core", Some(1)); + core.path = "/tmp/core".into(); + core.baseline = Duration::from_secs(2); + + let mut app = oracle_binary("app", Some(1)); + app.path = "/tmp/app".into(); + app.baseline = Duration::from_secs(3); + + let mut empty = oracle_binary("app", Some(0)); + empty.path = "/tmp/empty".into(); + empty.baseline = Duration::from_secs(100); + + let packages = [String::from("app")]; + let scope = capped(&packages); + let binaries = [core, app, empty]; + let reach = Reachability::build(&plan, &binaries, &scope); + let (targets, savings) = census_targets(&plan, &reach, &Killers::default()); + + assert_eq!(targets.len(), 1); + let selected: HashSet = core::iter::once(1).collect(); + assert_eq!(targets.get(Utf8Path::new("/tmp/app")), Some(&selected)); + assert_eq!(savings, Duration::from_secs(3)); + } + + /// A hinted mutant sharing a binary with an unhinted one still gets censused there — for free — + /// but contributes nothing of its own to the census's economics. + /// + /// Skipping the hinted mutant's ordinal entirely would leave it out of `targets` for a binary + /// the census is going to walk anyway, because the unhinted mutant beside it needs it. A + /// completed census that never recorded that site would then make `Census::selection` read it + /// as *proven unreached* instead of merely never asked about — turning a stale hint into a wrong + /// `Uncovered` verdict rather than the whole-binary run a stale hint should fall back to. + #[test] + fn a_hinted_mutant_rides_along_on_a_binary_an_unhinted_mutant_already_justifies() { + let mut plan = oracle_plan("core"); + let mut hinted = crate::testing::advise_fixture::mutant("a.rs", "arith.add_to_sub", Outcome::Pending, 1); + hinted.package = "core".to_owned().into(); + hinted.ordinal = 2; + plan.mutants.push(hinted); + + let mut killers = Killers::default(); + killers.record( + plan.mutants[1].id.clone(), + Killer { + package: "core".to_owned(), + target: String::new(), + test: "tests::hint".to_owned(), + }, + ); + + let mut core = oracle_binary("core", Some(1)); + core.path = "/tmp/core".into(); + core.baseline = Duration::from_secs(2); + + let binaries = [core]; + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let (targets, savings) = census_targets(&plan, &reach, &killers); + + assert_eq!(targets.len(), 1, "one binary is targeted, shared by both mutants"); + let expected: HashSet = [1, 2].into_iter().collect(); + assert_eq!( + targets.get(Utf8Path::new("/tmp/core")), + Some(&expected), + "the hinted ordinal rides along with the unhinted one" + ); + assert_eq!( + savings, + Duration::from_secs(2), + "only the unhinted mutant pays the binary's baseline into the census's economics" + ); + } + + /// A package whose only pending mutant already carries an exact checked killer hint never + /// justifies a census of its own — the probe the sweep already plans to try is cheaper than + /// anything a census could narrow it to. + #[test] + fn a_mutant_with_a_hint_and_no_unhinted_sibling_never_targets_its_own_binary() { + let mut plan = oracle_plan("core"); + plan.mutants[0].ordinal = 1; + + let mut killers = Killers::default(); + killers.record( + plan.mutants[0].id.clone(), + Killer { + package: "core".to_owned(), + target: String::new(), + test: "tests::hint".to_owned(), + }, + ); + + let mut core = oracle_binary("core", Some(1)); + core.path = "/tmp/core".into(); + core.baseline = Duration::from_secs(2); + + let binaries = [core]; + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let (targets, savings) = census_targets(&plan, &reach, &killers); + + assert!(targets.is_empty(), "a hinted mutant alone must not pay to census its own binary"); + assert_eq!(savings, Duration::ZERO, "no census was justified, so it has nothing to be worth"); + } + + /// A hinted mutant whose binary is never independently justified by an unhinted mutant is left + /// out of `targets` entirely, not merely un-costed — a stale hint there must fall back to a + /// whole-binary run rather than being read as `Uncovered` by an absent-but-listed census entry. + #[test] + fn a_stale_hints_binary_stays_out_of_targets_when_nothing_else_justifies_it() { + let mut plan = oracle_plan("core"); + plan.mutants[0].ordinal = 1; + + // Given its own explicit reach entry, `extra` no longer falls back to the "reaches + // everything" default an absent package gets, which keeps it genuinely isolated from + // `core` on both sides — the property this test is about. + let _extra_reach = plan + .reach + .insert("extra".to_owned(), core::iter::once("extra".to_owned()).collect()); + + let mut extra = crate::testing::advise_fixture::mutant("b.rs", "arith.add_to_sub", Outcome::Pending, 2); + extra.package = "extra".to_owned().into(); + extra.ordinal = 5; + plan.mutants.push(extra); + + let mut killers = Killers::default(); + killers.record( + plan.mutants[1].id.clone(), + Killer { + package: "extra".to_owned(), + target: String::new(), + test: "tests::hint".to_owned(), + }, + ); + + let mut core = oracle_binary("core", Some(1)); + core.path = "/tmp/core".into(); + core.baseline = Duration::from_secs(2); + + let mut extra_binary = oracle_binary("extra", Some(1)); + extra_binary.path = "/tmp/extra".into(); + extra_binary.baseline = Duration::from_secs(4); + + let binaries = [core, extra_binary]; + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let (targets, savings) = census_targets(&plan, &reach, &killers); + + assert_eq!(targets.len(), 1, "only the unhinted mutant's own binary is targeted"); + assert!( + targets.contains_key(Utf8Path::new("/tmp/core")), + "the unhinted mutant's binary is censused" + ); + assert!( + !targets.contains_key(Utf8Path::new("/tmp/extra")), + "the hinted mutant's own binary, which nothing else justifies, must stay absent" + ); + assert_eq!( + savings, + Duration::from_secs(2), + "only the unhinted mutant's binary counts toward savings" + ); + } + + /// A hint that no longer names a binary this mutant can reach — one recorded before a target + /// was renamed, before the binary moved package, or before the test packages were narrowed away + /// from it — is exactly as stale as [`super::sweep::judge_ordered`](super::sweep) would find it: + /// it never matches any binary in `reach.reachable(&mutant.package)`, so this pass has to fall + /// back to treating the mutant as unhinted rather than silently excusing it from a census. + #[test] + fn a_hint_naming_no_reachable_binary_is_treated_as_unhinted() { + let mut plan = oracle_plan("core"); + plan.mutants[0].ordinal = 1; + + let mut killers = Killers::default(); + killers.record( + plan.mutants[0].id.clone(), + Killer { + package: "core".to_owned(), + target: "a-target-no-binary-carries".to_owned(), + test: "tests::hint".to_owned(), + }, + ); + + let mut core = oracle_binary("core", Some(1)); + core.path = "/tmp/core".into(); + core.baseline = Duration::from_secs(2); + + let binaries = [core]; + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let (targets, savings) = census_targets(&plan, &reach, &killers); + + let expected: HashSet = core::iter::once(1).collect(); + assert_eq!( + targets.get(Utf8Path::new("/tmp/core")), + Some(&expected), + "a hint naming no reachable binary must not exempt the mutant from justifying its own census" + ); + assert_eq!( + savings, + Duration::from_secs(2), + "an ineligible hint pays the baseline into savings exactly like an unhinted mutant" + ); + } + + /// The crate being mutated has no tests of its own, but the crate above it does. + /// + /// Left unsaid, this run reports every mutant as uncovered and reads like a verdict on the code + /// rather than on the scope the cap chose. + #[test] + fn a_cap_that_leaves_the_mutants_unjudged_names_the_package_that_could_judge_them() { + let mut events = Recorder::default(); + let plan = oracle_plan("core"); + let named = [String::from("core")]; + let scope = capped(&named); + + warn_about_an_empty_oracle(&plan, &[oracle_binary("core", Some(0))], &scope, true, &[], &mut events); + + let [warning] = events.warnings.as_slice() else { + panic!("one unjudged package is one warning, got {:?}", events.warnings); + }; + + assert!(warning.contains("`app`"), "the package that could help is not named: {warning}"); + assert!(warning.contains("--test-package"), "the remedy is not named: {warning}"); + assert!(warning.contains("--test-workspace"), "the wider remedy is not named: {warning}"); + } + + /// A retreat narrows the oracle the caller named; it never replaces it with a different one. + /// + /// `--test-package p` with mutants in `q` is a caller who chose `p`'s tests as the oracle and + /// deliberately left `q`'s out. Substituting the mutated packages hands the run exactly the + /// binaries they opted out of, and reports the result as though they had asked for it. + #[test] + fn a_retreat_does_not_substitute_the_mutated_packages_for_the_oracle_the_caller_named() { + let requested = vec![String::from("p"), String::from("r")]; + let intended = vec![String::from("q")]; + + let kept = narrowed_oracle(requested, intended, &[String::from("p")]).unwrap(); + + assert_eq!(kept, vec![String::from("r")], "the retreat may only subtract"); + } + + /// A retreat that leaves no oracle at all is an error, not a run that judges nothing. + /// + /// Every mutant would come back uncovered, which reads as a fact about the code rather than + /// about a build that never happened. + #[test] + fn a_retreat_that_empties_the_oracle_is_reported_rather_than_run() { + let requested = vec![String::from("p")]; + let intended = vec![String::from("q")]; + + let cause = narrowed_oracle(requested, intended, &[String::from("p")]).unwrap_err(); + + assert!(cause.to_string().contains("`p`"), "{cause}"); + assert!(cause.to_string().contains("none of the packages"), "{cause}"); + } + + /// An unrestricted oracle cannot be subtracted from, so the checked set is named outright. + #[test] + fn a_retreat_from_an_unrestricted_oracle_names_what_was_checked() { + let intended = vec![String::from("q")]; + + let kept = narrowed_oracle(Vec::new(), intended, &[String::from("p")]).unwrap(); + + assert_eq!(kept, vec![String::from("q")]); + } + + /// A retreat is what the advisory blames when there was one, whoever chose the oracle. + /// + /// Blaming the cap would send the reader to widen a scope that was never the problem, and the + /// remedy the cap offers — `--test-package

` — would name a package that has just failed to + /// compile. + #[test] + fn an_oracle_emptied_by_a_retreat_blames_the_retreat_rather_than_the_cap() { + let mut events = Recorder::default(); + let plan = oracle_plan("core"); + let named = [String::from("core")]; + let scope = capped(&named); + + warn_about_an_empty_oracle( + &plan, + &[oracle_binary("core", Some(0))], + &scope, + false, + &[String::from("app")], + &mut events, + ); + + let [warning] = events.warnings.as_slice() else { + panic!("a retreat that empties the oracle is one warning, got {:?}", events.warnings); + }; + + assert!(warning.contains("could not compile `app`"), "{warning}"); + assert!( + !warning.contains("--test-package"), + "the remedy must not name what just failed: {warning}" + ); + } + + /// A suite that does convict things must never be told it does not exist. + #[test] + fn a_cap_whose_own_tests_reach_the_mutants_says_nothing() { + let mut events = Recorder::default(); + let plan = oracle_plan("core"); + let named = [String::from("core")]; + let scope = capped(&named); + + warn_about_an_empty_oracle(&plan, &[oracle_binary("core", Some(4))], &scope, true, &[], &mut events); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } + + /// `None` is a suite nobody counted, not a suite with nothing in it. + /// + /// `--no-baseline` and a `harness = false` target both produce it, and reading it as empty would + /// announce a missing oracle on the strength of a measurement never taken. + #[test] + fn an_uncounted_suite_is_not_reported_as_an_empty_one() { + let mut events = Recorder::default(); + let plan = oracle_plan("core"); + let named = [String::from("core")]; + let scope = capped(&named); + + warn_about_an_empty_oracle(&plan, &[oracle_binary("core", None)], &scope, true, &[], &mut events); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } + + /// A caller who named `--test-package` has already answered this, and must not be asked again. + #[test] + fn a_cap_the_caller_chose_is_left_to_stand() { + let mut events = Recorder::default(); + let plan = oracle_plan("core"); + let named = [String::from("core")]; + let scope = capped(&named); + + warn_about_an_empty_oracle(&plan, &[oracle_binary("core", Some(0))], &scope, false, &[], &mut events); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } + + /// Nothing else links the code, so widening the oracle has nothing to offer and there is no + /// advice to give — the mutants really are untested. + #[test] + fn a_package_nothing_else_links_gets_no_advice() { + let mut events = Recorder::default(); + let plan = oracle_plan("app"); + let named = [String::from("app")]; + let scope = capped(&named); + + warn_about_an_empty_oracle(&plan, &[oracle_binary("app", Some(0))], &scope, true, &[], &mut events); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } + + /// A run already testing the whole workspace is never told to widen its oracle. + /// + /// The command line rejects `--test-workspace` beside `--test-package`, but a configuration file + /// can set both, and then the scope carries a package list *and* the whole-workspace answer. The + /// advice would be to turn on what is already on. + #[test] + fn a_whole_workspace_oracle_is_never_told_to_widen() { + let mut events = Recorder::default(); + let plan = oracle_plan("core"); + let named = [String::from("harness")]; + let scope = TestScope { + packages: &named, + package_local: false, + whole_workspace: true, + }; + + warn_about_an_empty_oracle(&plan, &[oracle_binary("harness", Some(0))], &scope, true, &[], &mut events); + + assert!(events.warnings.is_empty(), "{:?}", events.warnings); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/memory.rs b/crates/cargo-gamma-lib/src/exec/memory.rs new file mode 100644 index 000000000..352fe1951 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/memory.rs @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! How much memory a mutant may use, and what the host is able to do about it. +//! +//! A mutation can turn bounded allocation into unbounded allocation. The wall-clock and stall +//! budgets do eventually stop such a mutant, but by then it may have exhausted physical memory, +//! driven the machine deep into swap, or provoked the kernel into killing something that had +//! nothing to do with the run. The output cap already protects the tool from a test that prints +//! forever; this is the same protection for what a test allocates. +//! +//! The calibration point is the baseline. Every test binary is already run once with no mutant +//! active, so the peak that run reaches is a measurement of the same workload the mutants are +//! judged against, taken on the same machine with the same instrumentation compiled in. A ceiling +//! derived from it is therefore a statement about *this* suite rather than a guess about suites in +//! general. +//! +//! Two things about this module are deliberate. +//! +//! **The limit has to cover the whole descendant tree.** Tests launch servers, databases, helper +//! programs and nested cargo invocations, and those are exactly where a runaway allocation does the +//! most damage. Anything that accounts only for the direct child leaves the dangerous case +//! unbounded, which is why `wait4`, `getrusage` and polling `/proc` are not used here: they +//! describe one process, and they race short-lived descendants besides. +//! +//! **Support is claimed only where it exists.** cgroup v2 delegation is not universal, and a +//! container or CI runner may not have it. A run that asked for a memory ceiling and did not get +//! one has to be told so, because the alternative is a user who believes the machine is protected +//! and finds out otherwise when it is not. + +/// Whether this host can meter and bound a test subtree's memory, or why it cannot. +/// +/// The mechanism half of this module lives in `cargo-gamma-process`, which composes the safe +/// platform calls from `cargo-gamma-unsafe`. What is left here is the policy half: what a ceiling +/// should be, given a baseline measurement and what the user asked for. The seam is deliberate — +/// arithmetic on a measurement can be tested without a kernel, and a kernel call cannot be. +pub use cargo_gamma_process::support; +use clap::ValueEnum; +use serde::Deserialize; + +/// How much memory control a run places around each test binary. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum MemoryControl { + /// Neither measure nor bound what a test binary allocates. + Off, + + /// Measure the peak memory of each test subtree and report it, but never stop a mutant for it. + /// + /// The measurement is what makes a ceiling possible to choose, and it costs one accounting + /// boundary per invocation and nothing else. It is the honest starting point for a project + /// that does not yet know what its suite allocates. + Measure, + + /// Measure, and hold every mutant to a ceiling derived from its binary's baseline peak. + /// + /// The default, on the same reasoning as the wall-clock timeout: a mutation can turn bounded + /// allocation into unbounded allocation, and the user who most needs protecting from that is + /// the one who never thought to ask for it. Where the host cannot provide the accounting, a + /// run that merely defaulted into this quietly drops to `Off` and says so, rather than refusing + /// to start — see `Demand`. + #[default] + Enforce, +} + +/// The mode implied by command-line memory ceilings, if either was named. +pub(crate) const fn implied_memory_control(memory_limit: Option, baseline_memory_limit: Option) -> Option { + if memory_limit.is_some() { + Some(MemoryControl::Enforce) + } else if baseline_memory_limit.is_some() { + Some(MemoryControl::Measure) + } else { + None + } +} + +/// Whether a run's memory control was chosen by the user or inherited from the default. +/// +/// This is the whole difference between an error and a note. Someone who passed `--memory` did so +/// because an unbounded mutant would cost them something — a wedged laptop, a CI runner that takes +/// the rest of the job down with it — and giving them a run that quietly lacks that protection +/// would be discovered only by the thing they were trying to prevent. Someone who passed nothing +/// asked for a mutation score, and refusing to produce one because this host has no cgroup +/// delegation would be an obstruction rather than a safeguard. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Demand { + /// The user named a memory setting, so failing to deliver it is an error. + Stated, + + /// Nobody asked; this is the built-in default, so failing to deliver it is a note. + #[default] + Inherited, +} + +/// Everything a run needs to decide how much memory a mutant may use. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct MemoryPolicy { + /// Whether memory is measured, bounded, or neither. + pub control: MemoryControl, + + /// Whether `control` was stated by the user or inherited from the default. + pub demand: Demand, + + /// Multiple of a binary's baseline peak that a mutant of it may reach. + pub multiplier: f64, + + /// Absolute headroom added to a binary's baseline peak. + /// + /// A multiplier alone is far too tight for a binary whose baseline peak is small: doubling a + /// few megabytes still leaves no room for a lazily initialized table or a randomized test that + /// happens to pick a larger input. The ceiling is the larger of the two, so the multiplier + /// governs large suites and the headroom governs small ones. + pub headroom: u64, + + /// An explicit ceiling for every test binary, overriding the baseline-derived one. + /// + /// Useful to anyone who knows their workload, and the only way to bound a run that skips the + /// baseline, since there is then nothing to calibrate from. + pub limit: Option, + + /// A ceiling applied to the baseline runs themselves. + /// + /// A limit calibrated from the baseline cannot protect the machine from a baseline that is + /// itself runaway, which is a real risk the first time a suite is measured. This closes that + /// bootstrap hole for environments that need it. + pub baseline_limit: Option, +} + +/// The absolute headroom a mutant gets over its binary's baseline peak, in bytes. +/// +/// A single baseline observation is noisy: allocator behaviour, randomized tests, lazy +/// initialization and input-dependent work all vary legitimately between two runs of the same +/// suite. The cost of being too generous is a runaway mutant that reaches a larger peak before it +/// is stopped; the cost of being too tight is a healthy mutant reported as caught, which inflates +/// the score with a detection the suite never made. The second is much worse, so the default is +/// generous and is meant to be tightened against real measurements rather than argued about. +pub const DEFAULT_HEADROOM: u64 = 128 * 1024 * 1024; + +/// The multiple of the baseline peak a mutant is allowed, before headroom is considered. +pub const DEFAULT_MULTIPLIER: f64 = 2.0; + +impl Default for MemoryPolicy { + fn default() -> Self { + Self { + control: MemoryControl::default(), + demand: Demand::default(), + multiplier: DEFAULT_MULTIPLIER, + headroom: DEFAULT_HEADROOM, + limit: None, + baseline_limit: None, + } + } +} + +impl MemoryPolicy { + /// Whether each invocation needs an accounting boundary at all. + #[must_use] + pub const fn measuring(&self) -> bool { + !matches!(self.control, MemoryControl::Off) + } + + /// Whether a mutant that passes its ceiling should be stopped. + #[must_use] + pub const fn enforcing(&self) -> bool { + matches!(self.control, MemoryControl::Enforce) + } + + /// The same policy with all memory control removed, for a host that cannot provide it. + #[must_use] + pub const fn disabled(&self) -> Self { + Self { + control: MemoryControl::Off, + ..*self + } + } + + /// Whether failing to deliver this policy should stop the run rather than merely be reported. + #[must_use] + pub const fn insisted(&self) -> bool { + matches!(self.demand, Demand::Stated) + } + + /// The ceiling a binary with this baseline peak should be held to, if any. + /// + /// `calibrated` says whether the baseline actually ran. Without it there is no measurement, and + /// a ceiling invented from no measurement is the worst of both worlds: it neither reflects the + /// suite nor admits that it does not. An explicit limit still applies, because that is a + /// statement the user made rather than one this code inferred. + #[must_use] + pub fn ceiling(&self, peak: Option, calibrated: bool) -> Option { + if !self.enforcing() { + return None; + } + + if let Some(fixed) = self.limit { + return Some(fixed); + } + + if !calibrated { + return None; + } + + let peak = peak?; + + Some(scale(peak, self.multiplier).max(peak.saturating_add(self.headroom))) + } +} + +/// Multiplies a byte count by a factor, saturating rather than wrapping or panicking. +/// +/// The factor is floored at one because a ceiling below the peak the unmutated suite reached would +/// convict every mutant of a fault the baseline shares. +fn scale(peak: u64, multiplier: f64) -> u64 { + #[expect(clippy::cast_precision_loss, reason = "a memory ceiling is not sensitive to its last few bytes")] + let scaled = peak as f64 * multiplier.max(1.0); + + #[expect(clippy::cast_precision_loss, reason = "the comparison only needs to be right near the boundary")] + let most = u64::MAX as f64; + + if !scaled.is_finite() || scaled >= most { + return u64::MAX; + } + + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "the value is finite, non-negative and below `u64::MAX` by the test above" + )] + let bytes = scaled as u64; + + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A policy that is not enforcing never produces a ceiling, however it was measured. + #[test] + fn measuring_without_enforcing_never_produces_a_ceiling() { + let policy = MemoryPolicy { + control: MemoryControl::Measure, + ..MemoryPolicy::default() + }; + + assert!(policy.measuring()); + assert!(!policy.enforcing()); + assert_eq!(policy.ceiling(Some(1024), true), None); + } + + #[test] + fn the_default_policy_enforces_a_calibrated_ceiling() { + let policy = MemoryPolicy::default(); + + assert!(policy.measuring()); + assert!(policy.enforcing()); + assert!(policy.ceiling(Some(1024), true).is_some()); + } + + #[test] + fn switching_control_off_asks_the_platform_for_nothing() { + let policy = MemoryPolicy { + control: MemoryControl::Off, + ..MemoryPolicy::default() + }; + + assert!(!policy.measuring()); + assert!(!policy.enforcing()); + assert_eq!(policy.ceiling(Some(1024), true), None); + } + + /// A policy nobody asked for degrades on a host that cannot deliver it; a stated one does not. + #[test] + fn only_a_stated_policy_is_insisted_upon() { + let inherited = MemoryPolicy::default(); + + assert!(!inherited.insisted()); + assert!(!inherited.disabled().measuring()); + + let stated = MemoryPolicy { + demand: Demand::Stated, + ..MemoryPolicy::default() + }; + + assert!(stated.insisted()); + + // Disabling preserves everything else, so a degraded run still reports the numbers it was + // configured with rather than silently reverting them too. + assert!((stated.disabled().multiplier - stated.multiplier).abs() < f64::EPSILON); + } + + /// A small baseline peak is governed by the absolute headroom, not by the multiplier. + #[test] + fn a_small_baseline_peak_gets_the_absolute_headroom() { + let policy = MemoryPolicy { + control: MemoryControl::Enforce, + ..MemoryPolicy::default() + }; + + // Doubling four megabytes would leave a suite four megabytes of room for one lazily + // initialized table, and report the mutant that filled it as caught by tests that never + // noticed it. + assert_eq!( + policy.ceiling(Some(4 * 1024 * 1024), true), + Some(4 * 1024 * 1024 + DEFAULT_HEADROOM) + ); + } + + /// A large baseline peak is governed by the multiplier. + #[test] + fn a_large_baseline_peak_gets_the_multiplier() { + let policy = MemoryPolicy { + control: MemoryControl::Enforce, + ..MemoryPolicy::default() + }; + let peak = 4 * 1024 * 1024 * 1024_u64; + + assert_eq!(policy.ceiling(Some(peak), true), Some(peak * 2)); + } + + /// Skipping the baseline disables the derived ceiling rather than inventing one. + #[test] + fn an_uncalibrated_run_gets_no_derived_ceiling_but_keeps_an_explicit_one() { + let policy = MemoryPolicy { + control: MemoryControl::Enforce, + ..MemoryPolicy::default() + }; + + // There is no measurement to derive from, and a number made up here would be presented to + // the user with exactly the same confidence as a measured one. + assert_eq!(policy.ceiling(None, false), None); + assert_eq!(policy.ceiling(Some(4096), false), None); + + let explicit = MemoryPolicy { + limit: Some(4096), + ..policy + }; + + assert_eq!(explicit.ceiling(None, false), Some(4096)); + } + + #[test] + fn an_explicit_limit_overrides_whatever_the_baseline_measured() { + let policy = MemoryPolicy { + control: MemoryControl::Enforce, + limit: Some(999), + ..MemoryPolicy::default() + }; + + assert_eq!(policy.ceiling(Some(4 * 1024 * 1024 * 1024), true), Some(999)); + } + + /// Scaling saturates instead of wrapping, and never falls below the peak it started from. + #[test] + fn scaling_saturates_and_never_shrinks_the_peak() { + // A ceiling below the peak the unmutated suite reached would convict every mutant of a + // fault the baseline shares, so a multiplier under one is ignored rather than honoured. + assert_eq!(scale(1024, 0.5), 1024); + assert_eq!(scale(u64::MAX, 2.0), u64::MAX); + assert_eq!(scale(1024, f64::INFINITY), u64::MAX); + assert_eq!(scale(0, 2.0), 0); + } + + /// Whatever this host answers, it answers with a reason rather than a bare failure. + #[test] + fn unsupported_hosts_say_why_rather_than_merely_saying_no() { + // A run that asked for a ceiling and did not get one has to be able to explain itself: + // "unsupported" without a cause sends the reader to the source of this tool instead of to + // the configuration of their machine. + if let Err(reason) = support() { + assert!(reason.len() > 20, "{reason}"); + } + } +} diff --git a/crates/cargo-gamma-lib/src/exec/mod.rs b/crates/cargo-gamma-lib/src/exec/mod.rs new file mode 100644 index 000000000..c7fcde73e --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/mod.rs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Building the mutant schema once and running the test suite against each mutant. +//! +//! A run copies the workspace to a scratch tree, instruments every mutated file, builds the test +//! binaries once, measures a baseline with no mutant active, then runs the suite once per mutant +//! with `GAMMA_ACTIVE` naming the one that is live. Every test process, baseline included, gets +//! `CARGO_GAMMA=1` so a suite that drives cargo itself can opt out of a nested build. +//! +//! Mutants that cannot compile — replacing a return value with `Default::default()` when the type +//! is not `Default`, for instance — are attributed back to the guards that caused them, withdrawn, +//! and the build is retried; a handful of rollback rounds converges. + +mod baseline; +mod build; +mod cargo_options; +mod census; +mod config; +mod copy; +mod events; +#[cfg(test)] +mod faults; +mod harness_filters; +mod incremental_mode; +mod killers; +mod loader; +mod manifest; +// Exposed to integration tests, which need to ask whether this host can bound memory at all before +// they can say what a run should have done. See `declare_modules!` in `lib.rs` for the convention. +mod measure; +#[cfg(feature = "internals")] +pub mod memory; +#[cfg(not(feature = "internals"))] +mod memory; +mod nextest; +mod progress; +#[cfg(target_os = "linux")] +pub(crate) mod relaunch; +mod session; +mod stall; +mod sweep; +mod sync; +mod test_binary; +mod verdict; +mod workspace; + +pub use build::{OrderingHints, Round, Withdrawal}; +pub use cargo_options::{BuildLimits, CargoOptions, DEFAULT_ROLLBACK_ROUNDS}; +pub use config::Config; +pub(crate) use config::{available_parallelism, resolve_jobs}; +pub use events::Events; +pub use incremental_mode::IncrementalMode; +pub use loader::UNDER_GAMMA_VAR; +pub(crate) use manifest::RUNTIME_CRATE; +pub use measure::{Built, Measured, Oracle, measure, run}; +pub use memory::{DEFAULT_HEADROOM, DEFAULT_MULTIPLIER, Demand, MemoryControl, MemoryPolicy}; +// Named for its subject at this level, where `support` alone would say nothing about what is +// supported. The module itself is private unless the `internals` feature exposes it. +pub(crate) use memory::{implied_memory_control, support as memory_support}; +pub use session::{CensusCost, Phases, Session, SweepCost}; +pub use test_binary::TestBinary; +pub(crate) use verdict::CONFIRM_FACTOR; +pub use verdict::READERS; +#[cfg(loom)] +pub(crate) use verdict::run_loom_models; +pub use workspace::{Workspace, clean_cache, footprint, gamma_base, scratch_tree}; +pub(crate) use workspace::{claim_cache, claim_workspace}; diff --git a/crates/cargo-gamma-lib/src/exec/nextest.rs b/crates/cargo-gamma-lib/src/exec/nextest.rs new file mode 100644 index 000000000..525615351 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/nextest.rs @@ -0,0 +1,582 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Running this run's own test binaries through `cargo nextest`. +//! +//! Nextest normally builds what it runs, which would put a cargo invocation between every mutant +//! and its verdict. It also accepts the two metadata files that describe an already-built tree, and +//! given those it runs the binaries and calls no cargo at all. Both are produced once, immediately +//! after the build, and then reused by every mutant for the rest of the run. +//! +//! What nextest is here for is isolation: it gives each test its own process. A suite that shares a +//! global, sets an environment variable, or installs a process-wide handler is red under a threaded +//! harness, and a red baseline stops a run before it judges anything. Such a tree cannot be measured +//! at all without this. + +use std::process::Command; + +use camino::{Utf8Path, Utf8PathBuf}; +use serde_json::Value; + +use super::test_binary::TestBinary; +use super::workspace::Workspace; +use crate::error::error; +use crate::{HashMap, Result}; + +/// Keep Windows' command line below its practical limit, with room for metadata paths and test args. +const MAX_FILTERSET_BYTES: usize = 8 * 1024; + +/// The metadata that lets nextest run a tree it did not build, and the ids it knows the binaries by. +#[derive(Debug, Clone)] +pub(super) struct Harness { + /// Where the binaries nextest may run are described. + binaries: Utf8PathBuf, + + /// Where the workspace those binaries came from is described. + metadata: Utf8PathBuf, + + /// What nextest calls each binary, keyed by the path this run knows it by. + /// + /// A binary absent from this map is one nextest would not run, which is a disagreement about + /// the tree rather than a fact about any mutant; [`Harness::id`] reports it as an error rather + /// than silently running the whole suite in its place. + ids: HashMap, +} + +impl Harness { + /// Describes the built tree to nextest, so that no later run of it needs cargo. + /// + /// # Errors + /// + /// Returns an error if nextest is not installed or cannot enumerate the built tree. + pub(super) fn prepare(work: &Workspace, binaries: &[TestBinary]) -> Result { + let listing = work.capture_nextest_list(binaries)?; + let ids = binary_ids(&listing); + + let paths = work.write_scratch("nextest-binaries.json", &listing)?; + let metadata = work.write_scratch("nextest-metadata.json", &work.capture_cargo_metadata()?)?; + + let harness = Self { + binaries: paths, + metadata, + ids, + }; + + // Checked here, once, rather than at the first mutant. A binary nextest does not know about + // cannot be run through it, and discovering that a thousand mutants in would waste the whole + // build; discovering it now costs nothing and names the binary. + for binary in binaries { + let _id = harness.id(&binary.path)?; + } + + Ok(harness) + } + + /// Builds the command that runs one binary's tests under nextest. + /// + /// `only` narrows the run to specific tests. A narrowed run keeps nextest's default of failing + /// when nothing matched, because such a run *is* the binary as far as the verdict is concerned + /// and an empty selection means the filterset and the built tree disagree — a fact about the + /// run, not evidence the mutant survived. Nextest also intersects command-line filtersets with + /// the `default-filter` in its own config, which the census cannot see because it lists tests by + /// running the binary directly under libtest, so an empty intersection is reachable in ordinary + /// use. + /// + /// # Errors + /// + /// Returns an error if nextest does not know the binary, which means the two disagree about what + /// was built. + pub(super) fn command(&self, work: &Workspace, binary: &TestBinary, only: &[&str]) -> Result { + let id = self.id(&binary.path)?; + let mut command = Command::new(nextest_binary()); + + // Run from the root rather than the package directory: nextest sets each test's working + // directory to its own package root, and pointing it at one package's directory would make + // it resolve the whole workspace relative to that. + let _ = command.current_dir(work.root().as_std_path()); + + // Only this binary's tests. The run visits the reachable binaries one at a time and stops + // at the first that convicts, and letting nextest run all of them would discard both that + // ordering and the per-binary budget derived from it. + let _ = command.args([ + "nextest", + "run", + "--binaries-metadata", + self.binaries.as_str(), + "--cargo-metadata", + self.metadata.as_str(), + ]); + for filterset in filtersets(&id, only) { + let _ = command.args(["-E", &filterset]); + } + // A startup failure is reported by the child, not by nextest. Pin captured failure output + // so workspace configuration cannot hide the runtime marker and turn infrastructure + // failure into a mutation kill. + let _ = command.args(["--failure-output", "immediate-final"]); + + // A test target that defines no tests is ordinary — a `src/bin` with no `#[test]` in it + // still gets a harness — and nextest treats an empty run as an error by default. Run + // directly, such a binary reports no tests and exits clean, and it must mean the same + // thing here. + // + // Only for a whole-binary run. A narrowed run stands in for the binary and its absence of + // failures is read as a verdict, so suppressing the empty-run exit code there would turn + // "the filterset matched nothing" into "the mutant survived". + if only.is_empty() { + let _ = command.args(["--no-tests", "pass"]); + } + + if !work.test_arguments().is_empty() { + let _ = command.arg("--"); + let _ = command.args(work.test_arguments()); + } + + Ok(command) + } + + /// A harness that knows the given path-to-id pairs, without asking nextest anything. + #[cfg(test)] + pub(super) fn fake(ids: &[(&str, &str)]) -> Self { + Self { + binaries: Utf8PathBuf::from("/t/binaries.json"), + metadata: Utf8PathBuf::from("/t/metadata.json"), + ids: ids.iter().map(|(path, id)| (Utf8PathBuf::from(*path), (*id).to_owned())).collect(), + } + } + + /// What nextest calls the binary at this path. + fn id(&self, path: &Utf8Path) -> Result { + self.ids.get(path).cloned().ok_or_else(|| { + error!( + "`cargo nextest` does not know the test binary `{path}` that this run built.\n\ + Run without `--nextest` to run the binaries directly instead." + ) + }) + } +} + +/// Reads the path-to-id map out of a nextest binaries listing. +/// +/// Everything else in the listing — platforms, build script outputs, target directories — is +/// nextest's own business and is passed back to it untouched. +fn binary_ids(listing: &str) -> HashMap { + let mut ids = HashMap::default(); + + let Ok(message) = serde_json::from_str::(listing) else { + return ids; + }; + + let Some(binaries) = message.get("rust-binaries").and_then(Value::as_object) else { + return ids; + }; + + for entry in binaries.values() { + let path = entry.get("binary-path").and_then(Value::as_str); + let id = entry.get("binary-id").and_then(Value::as_str); + + if let (Some(path), Some(id)) = (path, id) { + let _replaced = ids.insert(Utf8PathBuf::from(path), id.to_owned()); + } + } + + ids +} + +/// Turns a binary id into the filterset atom that matches exactly that binary and nothing else. +/// +/// The `=` prefix is nextest's exact-match operator. It matters that this is exact rather than the +/// default substring match: one binary's id is regularly a prefix of another's — `serde` and +/// `serde::integration` — and a substring match would run a binary this mutant was not being judged +/// against, against a budget apportioned for a different one. +/// +/// Ids are built from a package name and a target name, so they carry `::` and `/`, which need no +/// escaping. The characters that would end the atom are escaped anyway, since nothing checks that a +/// future cargo keeps target names as narrow as they are today. +fn matcher(id: &str) -> String { + let escaped = id.replace('\\', "\\\\").replace(')', "\\)").replace(',', "\\,"); + + format!("={escaped}") +} + +/// Narrows a binary when the selection is safe to express on a command line. +/// +/// The selected tests form one parenthesized union beneath the binary constraint. Multiple `-E` +/// arguments are not a substitute: nextest combines them as separate filters rather than as the +/// single union this probe needs. Windows still cannot launch an arbitrarily long command, so an +/// oversized selection runs the whole binary instead. +fn filtersets(id: &str, only: &[&str]) -> Vec { + let binary = format!("binary_id({})", matcher(id)); + + if only.is_empty() { + return vec![binary]; + } + + let tests = only + .iter() + .map(|name| format!("test({})", matcher(name))) + .collect::>() + .join(" or "); + let narrowed = format!("{binary} and ({tests})"); + + if narrowed.len() > MAX_FILTERSET_BYTES { + vec![binary] + } else { + vec![narrowed] + } +} + +/// The nextest executable. +/// +/// Invoked directly rather than through `cargo nextest`, so that a run whose binaries are already +/// built never needs cargo on the path at all. +fn nextest_binary() -> String { + "cargo-nextest".to_owned() +} + +/// Exit codes from nextest that describe the test run rather than a failure to perform one. +/// +/// Nextest reports far more than pass and fail, and the difference matters: a code saying the tests +/// ran and some failed is a verdict about the mutant, whereas a code saying nextest could not start +/// is a fact about this machine that must not be recorded as a kill. +pub(super) const TEST_RUN_FAILED: i32 = 100; + +/// Nextest could not create the test list it needs to start the selected run. +pub(super) const TEST_LIST_CREATION_FAILED: i32 = 104; + +/// Nextest's code for a run that matched no tests at all. +pub(super) const NO_TESTS_RUN: i32 = 4; + +/// Extracts the first failing test name from nextest's output. +/// +/// Nextest prints a `FAIL` line per failing test carrying the binary id and the test path. Only the +/// first is read: it is the one that convicted the mutant, and nextest cancels the run after it. +/// +/// The name is borrowed from `output` rather than owned, because the streaming caller looks at every +/// line a test process writes and keeps almost none of them. +pub(super) fn first_failure(output: &str) -> Option<&str> { + for line in output.lines() { + let trimmed = line.trim_start(); + + let Some(rest) = trimmed.strip_prefix("FAIL ").map(str::trim_start) else { + continue; + }; + + // `[ 0.024s] (1/2) crate::binary test::name` — the duration and the counter are progress + // reporting, and the name is what is left once they are dropped. + let after_time = rest.split_once(']').map_or(rest, |(_before, after)| after).trim_start(); + let after_count = after_time.split_once(')').map_or(after_time, |(_before, after)| after).trim_start(); + + // The binary id and the test path are separated by a space; the test path is what names the + // test a reader can go and run. + let name = after_count.split_once(' ').map_or(after_count, |(_binary, test)| test); + + if !name.is_empty() { + return Some(name.trim()); + } + } + + None +} + +#[cfg(all(test, not(miri)))] +mod fuzz { + use super::first_failure; + use crate::testing::{spliced, token}; + + /// A nextest failure announcement is found whatever surrounds it. + /// + /// Nextest's line carries a duration and a counter that this parser drops positionally, by + /// splitting on `]` and `)`. Both of those characters can also appear in a test path or in a + /// test's own output, which is exactly the kind of collision worth throwing random input at. + #[test] + fn a_nextest_failure_is_never_lost_among_arbitrary_output() { + bolero::check!() + .with_type::<(Vec, String, String, usize)>() + .for_each(|(noise, binary, name, at)| { + let line = format!(" FAIL [ 0.024s] (1/2) {} {}", token(binary), token(name)); + let output = spliced(noise, &line, *at); + + assert!(first_failure(&output).is_some(), "the failure was lost in {output:?}"); + }); + } + + /// Arbitrary output never panics the reader and never names an empty test. + /// + /// An empty name is the failure mode that matters here rather than a panic: the parser reaches + /// it by dropping everything up to the last separator it recognizes, so a malformed line can + /// leave nothing behind, and a caller told the test is named `""` cannot act on it. + #[test] + fn arbitrary_output_is_read_without_panicking() { + bolero::check!().with_type::().for_each(|output| { + if let Some(name) = first_failure(output) { + assert!(!name.is_empty(), "an empty test name was reported for {output:?}"); + assert!(!name.contains('\n'), "a test name spans lines: {name:?}"); + } + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_listing_yields_the_id_each_binary_is_known_by() { + let listing = r#"{"rust-binaries":{"nxspike":{"binary-id":"nxspike","binary-path":"/t/deps/nxspike-abc","kind":"lib"}}}"#; + let ids = binary_ids(listing); + + assert_eq!(ids.get(Utf8Path::new("/t/deps/nxspike-abc")).map(String::as_str), Some("nxspike")); + } + + /// A listing that is not JSON, or carries no binaries, must produce an empty map rather than + /// panic — the caller turns an absent id into a diagnostic naming the binary. + #[test] + fn a_listing_that_says_nothing_yields_no_ids() { + assert!(binary_ids("not json").is_empty()); + assert!(binary_ids("{}").is_empty()); + assert!(binary_ids(r#"{"rust-binaries":{"x":{"binary-id":"x"}}}"#).is_empty()); + } + + /// Exactness is the point: `serde` must not match `serde::integration`, which shares its prefix. + #[test] + fn a_binary_id_becomes_an_exact_filterset_atom() { + assert_eq!(matcher("crate::bin/name"), "=crate::bin/name"); + assert_eq!(matcher("plain"), "=plain"); + } + + /// Target names cannot contain these today, so this only has to hold if that ever changes. + #[test] + fn a_character_that_would_end_the_atom_is_escaped() { + assert_eq!(matcher("odd)name"), "=odd\\)name"); + assert_eq!(matcher("odd,name"), "=odd\\,name"); + assert_eq!(matcher("back\\slash"), "=back\\\\slash"); + } + + #[test] + fn a_long_test_selection_runs_the_whole_binary() { + let long = "x".repeat(MAX_FILTERSET_BYTES); + + assert_eq!(filtersets("nxspike", &[&long]), ["binary_id(=nxspike)"]); + } + + #[test] + fn selected_tests_form_one_parenthesized_union() { + assert_eq!( + filtersets("nxspike", &["tests::one", "tests::two"]), + ["binary_id(=nxspike) and (test(=tests::one) or test(=tests::two))"] + ); + } + + #[test] + fn the_first_failing_test_is_named_from_nextest_output() { + let output = " FAIL [ 0.024s] (1/2) nxspike tests::fails_when_asked\n\ + FAIL [ 0.030s] (2/2) nxspike tests::also_fails\n"; + + assert_eq!(first_failure(output), Some("tests::fails_when_asked")); + } + + #[test] + fn output_with_no_failure_line_names_nothing() { + assert_eq!(first_failure(" PASS [ 0.012s] (1/1) nxspike tests::works"), None); + assert_eq!(first_failure(""), None); + } + + /// The scan must not stop at the first line that is not a failure, or a suite that reports + /// anything at all before its failure would convict a mutant without naming what caught it. + #[test] + fn a_failure_is_found_past_lines_that_are_not_failures() { + let output = " Starting 2 tests across 1 binary\n\ + PASS [ 0.010s] (1/2) nxspike tests::works\n\ + FAIL [ 0.024s] (2/2) nxspike tests::fails_when_asked\n"; + + assert_eq!(first_failure(output), Some("tests::fails_when_asked")); + } + + #[test] + fn a_known_binary_resolves_to_the_id_nextest_gave_it() { + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + + assert_eq!(harness.id(Utf8Path::new("/t/deps/nxspike-abc")).unwrap(), "nxspike"); + } + + /// A binary nextest never listed cannot be run through it, and running the whole suite in its + /// place would judge the mutant against tests it was never apportioned a budget for. The error + /// names the binary and the way out. + #[test] + fn a_binary_nextest_never_listed_is_an_error_that_names_it() { + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + let failure = harness.id(Utf8Path::new("/t/deps/stranger-def")).unwrap_err().to_string(); + + assert!(failure.contains("/t/deps/stranger-def"), "{failure}"); + assert!(failure.contains("without `--nextest`"), "{failure}"); + } + + /// The argv nextest is handed has to describe the built tree, this binary alone, and nothing + /// else — every part of it is load-bearing, so the whole of it is asserted. + #[test] + #[cfg(unix)] + fn the_command_names_the_metadata_and_this_binary_alone() { + let (_scratch, mut work) = crate::testing::shell_workspace("nextest-command", "exit 0"); + + work.set_test_args(vec!["--nocapture".to_owned()]); + + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + + let command = harness + .command(&work, &crate::testing::test_binary("/t/deps/nxspike-abc"), &[]) + .expect("a known binary yields a command"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(command.get_program(), "cargo-nextest"); + assert_eq!( + command.get_current_dir().map(std::path::Path::to_path_buf), + Some(work.root().as_std_path().to_path_buf()) + ); + assert_eq!( + args, + vec![ + "nextest", + "run", + "--binaries-metadata", + "/t/binaries.json", + "--cargo-metadata", + "/t/metadata.json", + "-E", + "binary_id(=nxspike)", + "--failure-output", + "immediate-final", + "--no-tests", + "pass", + "--", + "--nocapture", + ] + ); + } + + /// With no test arguments to pass on there is no `--` either, because nextest reads a trailing + /// `--` with nothing after it as an empty argument to hand each test. + #[test] + #[cfg(unix)] + fn a_command_with_no_test_arguments_ends_at_the_filterset() { + let (_scratch, mut work) = crate::testing::shell_workspace("nextest-bare", "exit 0"); + + work.set_test_args(Vec::new()); + + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + let command = harness + .command(&work, &crate::testing::test_binary("/t/deps/nxspike-abc"), &[]) + .expect("a known binary yields a command"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(args.last().map(String::as_str), Some("pass")); + assert!(!args.iter().any(|arg| arg == "--"), "{args:?}"); + } + + /// A binary nextest never listed must be refused before a command is built for it, rather than + /// producing a filterset that matches nothing and a run that convicts nobody. + #[test] + #[cfg(unix)] + fn a_command_for_a_binary_nextest_never_listed_is_refused() { + let (_scratch, work) = crate::testing::shell_workspace("nextest-stranger", "exit 0"); + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + + let failure = harness + .command(&work, &crate::testing::test_binary("/t/deps/stranger-def"), &[]) + .unwrap_err() + .to_string(); + + assert!(failure.contains("/t/deps/stranger-def"), "{failure}"); + } + + /// A run narrowed to one test keeps the binary in the filterset and adds the test to it. + /// + /// Keeping `binary_id` is what makes the pair exact: a test path is only unique within a + /// binary, so filtering on the name alone would run every same-named test in the workspace and + /// stop the probe from being one test. + #[test] + #[cfg(unix)] + fn a_command_narrowed_to_one_test_filters_on_the_binary_and_the_test() { + let (_scratch, mut work) = crate::testing::shell_workspace("nextest-filtered", "exit 0"); + + work.set_test_args(Vec::new()); + + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + let command = harness + .command(&work, &crate::testing::test_binary("/t/deps/nxspike-abc"), &["tests::parses"]) + .expect("a known binary yields a command"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert!( + args.contains(&"binary_id(=nxspike) and (test(=tests::parses))".to_owned()), + "{args:?}" + ); + } + + /// Nextest intersects a command-line filterset with the `default-filter` in its own config, and + /// the census cannot see that filter because it lists a binary's tests by running it directly + /// under libtest. An empty intersection with `--no-tests pass` in force exits 0, and a narrowed + /// run's absence of failures is read as a verdict — so the mutant is reported as a survivor + /// though nothing ran. Keeping nextest's default `fail` turns that into exit code 4, which + /// `settle` reads as a fact about the run. + #[test] + #[cfg(unix)] + fn a_narrowed_command_does_not_suppress_the_empty_run_exit_code() { + let (_scratch, mut work) = crate::testing::shell_workspace("nextest-narrowed-no-tests", "exit 0"); + + work.set_test_args(Vec::new()); + + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + let command = harness + .command(&work, &crate::testing::test_binary("/t/deps/nxspike-abc"), &["tests::parses"]) + .expect("a known binary yields a command"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert!(!args.iter().any(|arg| arg == "--no-tests"), "{args:?}"); + + // A whole-binary run still needs it: a target with no tests at all is ordinary. + let whole = harness + .command(&work, &crate::testing::test_binary("/t/deps/nxspike-abc"), &[]) + .expect("a known binary yields a command"); + let whole: Vec<_> = whole.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert!(whole.windows(2).any(|pair| pair == ["--no-tests", "pass"]), "{whole:?}"); + } + + /// A census names every test that reaches a mutant, and nextest has to be asked for all of + /// them at once. The group of names is combined with the binary by `and` rather than by `or`, or a + /// name that also exists in another binary would drag that binary's tests into a run budgeted + /// for this one. + #[test] + #[cfg(unix)] + fn a_command_narrowed_to_several_tests_asks_for_any_of_them_within_the_one_binary() { + let (_scratch, mut work) = crate::testing::shell_workspace("nextest-several", "exit 0"); + + work.set_test_args(Vec::new()); + + let harness = Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")]); + let command = harness + .command( + &work, + &crate::testing::test_binary("/t/deps/nxspike-abc"), + &["tests::parses", "tests::rejects"], + ) + .expect("a known binary yields a command"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert!( + args.contains(&"binary_id(=nxspike) and (test(=tests::parses) or test(=tests::rejects))".to_owned()), + "{args:?}" + ); + } + + /// A test name carrying the characters a filterset gives meaning to is escaped, not interpolated. + /// + /// A name is a Rust path and cannot hold a bracket, but it reaches here from a harness's output + /// rather than from the compiler, and a filterset that fails to parse would abandon a run over + /// a cache entry. + #[test] + #[cfg(unix)] + fn a_test_name_holding_filterset_syntax_is_escaped() { + assert_eq!(matcher("odd)name,here"), "=odd\\)name\\,here"); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/progress.rs b/crates/cargo-gamma-lib/src/exec/progress.rs new file mode 100644 index 000000000..5bfd19776 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/progress.rs @@ -0,0 +1,385 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; +use std::time::Instant; + +use super::nextest; + +/// How a failure is recognised in a harness's output, if it can be recognised safely at all. +/// +/// Reading a failure out of the output as it arrives lets a mutant be convicted the moment the +/// first test catches it, instead of after the whole binary has run. That is only sound where a +/// line announcing a failure could not have been written by a test rather than by the harness. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Watch { + /// Take no verdict from the output; wait for the process to exit and judge that. + /// + /// The safe reading, and the necessary one wherever a test's own writing reaches the same + /// stream as the harness's. + Off, + + /// libtest's `test ... FAILED`. + Libtest, + + /// Nextest's `FAIL [ 0.024s] (1/2) `. + Nextest, +} + +/// What the harness has said so far, published by the reader for the waiting thread to watch. +#[derive(Debug)] +pub(super) struct Progress { + /// How to recognise a failure announcement, if this run's output can be trusted to carry one. + watch: Watch, + + /// The first test the harness announced as failed, once one has been announced. + /// + /// The first is kept rather than the last because it is the one that convicted the mutant, and + /// it is what the run would have reported had it read the output to exhaustion. + pub(super) failed: Option, + + /// Whether the guard runtime reported that it could not acquire the startup environment. + pub(super) environment_error: bool, + + /// When the harness last said anything at all. + pub(super) heard: Instant, + + /// The longest silence so far. + /// + /// Calibrated from the baseline, this is how long the suite legitimately goes quiet while its + /// slowest test runs, and it becomes the basis for the stall budget. + pub(super) quiet: Duration, + + /// The last test the harness named. + pub(super) test: Option, + + /// How many tests the harness said it was about to run, summed over every suite in the binary. + /// + /// A binary holds one suite per `#[cfg(test)]` module tree plus the doc tests, and each + /// announces itself separately, so the figure is a running total rather than the last one + /// seen. `None` until something announces anything: a target built with `harness = false` + /// prints whatever it likes and must contribute nothing rather than a confident zero. + pub(super) tests: Option, +} + +impl Progress { + pub(super) fn new(watch: Watch) -> Self { + Self { + watch, + failed: None, + environment_error: false, + heard: Instant::now(), + quiet: Duration::ZERO, + test: None, + tests: None, + } + } + + /// Records that the harness produced a line, and how long it had been silent beforehand. + pub(super) fn heard(&mut self, line: &str) { + let now = Instant::now(); + + self.quiet = self.quiet.max(now.saturating_duration_since(self.heard)); + + self.heard = now; + self.environment_error |= line + .as_bytes() + .windows(gamma_rt::ENVIRONMENT_ERROR_MARKER.len()) + .any(|window| window == gamma_rt::ENVIRONMENT_ERROR_MARKER); + + if let Some(rest) = line.strip_prefix("test ") + && let Some((name, verdict)) = rest.split_once(" ... ") + { + let name = name.trim(); + + // Overwritten in place rather than reallocated: every result line a suite prints lands + // here, and only the last one survives to name the test that was running when a budget + // or a ceiling cut the process short. + match &mut self.test { + Some(previous) => { + previous.clear(); + previous.push_str(name); + } + None => self.test = Some(name.to_owned()), + } + + // Only once libtest has announced a suite. A target built with `harness = false` + // prints whatever it likes, and its output must not be read as a verdict it never gave. + if self.watch == Watch::Libtest && self.tests.is_some() && verdict.trim() == "FAILED" { + self.note(name); + } + + return; + } + + if self.watch == Watch::Nextest + && let Some(name) = nextest::first_failure(line) + { + self.note(name); + } + + // libtest announces each suite with `running N tests`, which is the only place the size of + // the run is stated. Counting the result lines instead would miss tests that were filtered + // out and would double-count a harness that reports progress more than once. + if let Some(count) = line + .trim() + .strip_prefix("running ") + .and_then(|rest| rest.strip_suffix(" tests").or_else(|| rest.strip_suffix(" test"))) + .and_then(|count| count.trim().parse::().ok()) + { + self.tests = Some(self.tests.unwrap_or(0).saturating_add(count)); + } + } + + /// Remembers the first failure announced, leaving any later one alone. + fn note(&mut self, name: &str) { + if self.failed.is_none() && !name.is_empty() { + self.failed = Some(name.to_owned()); + } + } +} + +#[cfg(all(test, not(miri)))] +mod fuzz { + use super::{Progress, Watch}; + use crate::testing::{spliced, token}; + + /// No sequence of lines panics the reader, whichever harness it believes it is watching. + /// + /// This one is fed a live stream rather than a buffer, so it carries state across lines and is + /// the only parser here where the order of inputs can matter. A panic would land on the thread + /// draining a child's pipe, which is the thread that must keep draining for the child not to + /// block forever in `write`. + #[test] + fn arbitrary_output_never_panics_the_reader() { + bolero::check!().with_type::>().for_each(|lines| { + for watch in [Watch::Off, Watch::Libtest, Watch::Nextest] { + let mut progress = Progress::new(watch); + + for line in lines { + progress.heard(line); + } + } + }); + } + + /// A libtest failure is noticed whatever the surrounding output, once a suite has announced. + #[test] + fn a_libtest_failure_is_never_lost_among_arbitrary_output() { + bolero::check!() + .with_type::<(Vec, String, usize)>() + .for_each(|(noise, name, at)| { + let output = spliced(noise, &format!("test {} ... FAILED", token(name)), *at); + let mut progress = Progress::new(Watch::Libtest); + + progress.heard("running 1 test"); + + for line in output.lines() { + progress.heard(line); + } + + assert!(progress.failed.is_some(), "the failure was lost in {output:?}"); + }); + } + + /// A target that never announced a suite is never convicted by its own output. + /// + /// A `harness = false` target prints whatever its author chose, which can include text shaped + /// exactly like libtest's. Reading that as a verdict would convict a mutant on the strength of + /// a `println!`, so the absence of a `running N tests` line has to hold whatever follows it. + #[test] + fn output_from_a_target_that_announced_no_suite_is_never_a_verdict() { + bolero::check!() + .with_type::<(Vec, String, usize)>() + .for_each(|(noise, name, at)| { + let output = spliced(noise, &format!("test {} ... FAILED", token(name)), *at); + let mut progress = Progress::new(Watch::Libtest); + + for line in output.lines() { + // Anything that announces a suite makes the rest legitimate, which is a different + // case than the one under test here. + if line.trim().starts_with("running ") { + return; + } + + progress.heard(line); + } + + assert!(progress.failed.is_none(), "an unannounced target was convicted by {output:?}"); + }); + } +} + +#[cfg(test)] +mod tests { + use std::thread; + + use super::*; + + #[test] + fn the_harness_naming_a_test_records_it() { + let mut progress = Progress::new(Watch::Off); + + progress.heard("test tests::the_boundary_is_pinned ... ok\n"); + + assert_eq!(progress.test.as_deref(), Some("tests::the_boundary_is_pinned")); + } + + #[test] + fn a_line_that_is_not_a_test_result_does_not_rename_the_test() { + let mut progress = Progress::new(Watch::Off); + + progress.heard("test tests::first ... ok\n"); + progress.heard("running 3 tests\n"); + progress.heard("some output from the test itself\n"); + + assert_eq!(progress.test.as_deref(), Some("tests::first")); + } + + #[test] + fn the_size_of_each_suite_is_summed() { + // One binary announces its unit tests and its doc tests separately. + let mut progress = Progress::new(Watch::Off); + + progress.heard("running 12 tests\n"); + progress.heard("test a ... ok\n"); + progress.heard("running 3 tests\n"); + + assert_eq!(progress.tests, Some(15)); + } + + #[test] + fn a_single_test_is_announced_in_the_singular() { + let mut progress = Progress::new(Watch::Off); + + progress.heard("running 1 test\n"); + + assert_eq!(progress.tests, Some(1)); + } + + #[test] + fn an_empty_suite_still_counts_as_having_reported() { + // `running 0 tests` is what every target without tests prints, and is not the same as a + // harness that said nothing at all. + let mut progress = Progress::new(Watch::Off); + + progress.heard("running 0 tests\n"); + + assert_eq!(progress.tests, Some(0)); + } + + #[test] + fn a_harness_that_announces_nothing_reports_no_count() { + // A target with `harness = false` prints whatever it likes. Guessing zero would understate + // a suite that really did run. + let mut progress = Progress::new(Watch::Off); + + progress.heard("Running my own tests\n"); + progress.heard("all good\n"); + + assert_eq!(progress.tests, None); + } + + /// libtest announcing a failure is recorded, so the run can stop at the test that caught it. + #[test] + fn a_libtest_failure_is_recorded_as_soon_as_it_is_announced() { + let mut progress = Progress::new(Watch::Libtest); + + progress.heard("running 2 tests\n"); + progress.heard("test tests::first ... ok\n"); + progress.heard("test tests::second ... FAILED\n"); + + assert_eq!(progress.failed.as_deref(), Some("tests::second")); + } + + /// The first failure is the one that convicted the mutant, and is what reading the output to + /// exhaustion would have reported. + #[test] + fn a_later_failure_does_not_replace_the_first() { + let mut progress = Progress::new(Watch::Libtest); + + progress.heard("running 2 tests\n"); + progress.heard("test tests::first ... FAILED\n"); + progress.heard("test tests::second ... FAILED\n"); + + assert_eq!(progress.failed.as_deref(), Some("tests::first")); + } + + /// A target built with `harness = false` announces no suite and prints whatever it likes, so + /// nothing it says can be read as a verdict. + #[test] + fn a_failure_before_any_suite_is_announced_is_not_recorded() { + let mut progress = Progress::new(Watch::Libtest); + + progress.heard("test tests::second ... FAILED\n"); + + assert_eq!(progress.failed, None); + } + + /// With the watch off nothing is read out of the output at all, however it is shaped. + #[test] + fn an_unwatched_run_records_no_failure() { + let mut progress = Progress::new(Watch::Off); + + progress.heard("running 1 test\n"); + progress.heard("test tests::second ... FAILED\n"); + progress.heard(" FAIL [ 0.024s] (1/2) spike tests::second\n"); + + assert_eq!(progress.failed, None); + } + + /// The two runners announce failures differently, and each is read only in its own format. + #[test] + fn each_runner_reads_only_its_own_announcement() { + let mut nextest = Progress::new(Watch::Nextest); + + nextest.heard(" FAIL [ 0.024s] (1/2) spike tests::fails\n"); + + assert_eq!(nextest.failed.as_deref(), Some("tests::fails")); + + // libtest's shape means nothing to nextest, which never prints it. + let mut confused = Progress::new(Watch::Nextest); + + confused.heard("running 1 test\n"); + confused.heard("test tests::second ... FAILED\n"); + + assert_eq!(confused.failed, None); + } + + /// A passing result is not a failure however closely it is shaped like one. + #[test] + fn a_passing_result_is_not_recorded_as_a_failure() { + let mut progress = Progress::new(Watch::Libtest); + + progress.heard("running 2 tests\n"); + progress.heard("test tests::first ... ok\n"); + progress.heard("test tests::ignored ... ignored\n"); + + assert_eq!(progress.failed, None); + } + + #[test] + fn the_longest_silence_is_the_one_remembered() { + let mut progress = Progress::new(Watch::Off); + + progress.heard("started\n"); + thread::sleep(Duration::from_millis(30)); + progress.heard("a\n"); + let long = progress.quiet; + + progress.heard("b\n"); + + assert_eq!(progress.quiet, long, "a short gap must not replace a long one"); + assert!(long >= Duration::from_millis(25), "{long:?}"); + } + + #[test] + fn process_startup_calibrates_test_silence() { + let mut progress = Progress::new(Watch::Off); + + thread::sleep(Duration::from_millis(30)); + progress.heard("running 1 test\n"); + + assert!(progress.quiet >= Duration::from_millis(25), "{:?}", progress.quiet); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/relaunch.rs b/crates/cargo-gamma-lib/src/exec/relaunch.rs new file mode 100644 index 000000000..cda855a60 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/relaunch.rs @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Re-running this process inside a cgroup it is allowed to subdivide. +//! +//! Bounding a test subtree needs a cgroup cargo-gamma may create children under, and on a host that +//! never handed it one there is nothing to create them in. That is not an exotic situation: it is +//! the default everywhere a process is not started by a systemd user session — containers, CI +//! runners, `docker exec`, IDE-spawned terminals — and it is exactly where an unattended run is +//! most likely and a machine-wide out-of-memory kill is most expensive, because the kernel picks +//! its victim by heuristic and may well pick the user's editor rather than the run. +//! +//! The systemd user manager will delegate a cgroup on request, which is what +//! `systemd-run --user --scope -p Delegate=yes` does. A process cannot move *itself* into such a +//! scope after the fact — migrating between cgroups needs write access to the common ancestor of +//! the two, and the ancestor here belongs to root — so the scope has to exist before the process +//! does. Hence a relaunch rather than an adoption: cargo-gamma runs itself again inside the scope +//! and waits for it, forwarding the exit code. +//! +//! This is deliberately narrow. It happens only when memory control was wanted, only when the host +//! genuinely refused to delegate, and never more than once. Everything else — no systemd, no user +//! manager, an explicit refusal — falls through to the behaviour of not relaunching at all, which +//! is to say the diagnostic and the degradation that were there before. + +use std::env; +use std::ffi::OsString; +use std::process::Command; + +/// Marks a process as already being the relaunched one. +/// +/// The guard has to survive into the child, so it is an environment variable rather than a flag: +/// a flag would have to be appended to a command line that is otherwise forwarded verbatim, and +/// anything appended can land after a `--` and be read as a value rather than as an option. +const MARKER: &str = "CARGO_GAMMA_SCOPE"; + +/// The command that asks the systemd user manager for a scope. +const SYSTEMD_RUN: &str = "systemd-run"; + +/// Why a relaunch was not attempted, for a caller that has to explain itself. +/// +/// A relaunch that cannot happen is not an error — the run continues under the same rules as +/// before — but the reason is worth saying, because the user is about to be told that memory +/// control is unavailable and "why not just do the thing that would fix it" is the obvious next +/// question. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Refusal { + /// This process is already the relaunched one. + /// + /// Reaching here means the scope was obtained and delegation still failed, so relaunching + /// again would loop without ever changing the answer. + AlreadyInScope, + + /// There is no working `systemd-run` to ask, or asking it did not produce a scope. + /// + /// Deliberately one variant rather than several. A missing `systemd-run`, an unreachable user + /// manager and a scope that failed to start are different causes with identical consequences, + /// and the caller's next step — explain that memory control is unavailable, then error or + /// degrade according to whether it was asked for — is the same for all of them. + Unavailable, +} + +/// Whether this process is the result of a relaunch. +pub(crate) fn relaunched() -> bool { + env::var_os(MARKER).is_some() +} + +/// Builds the command that would re-run this process inside a delegated scope. +/// +/// Split from running it so that the shape of the command is testable without a systemd on the +/// other end of it. +fn scope_command(exe: OsString, args: Vec) -> Command { + let mut command = Command::new(SYSTEMD_RUN); + + let _built = command + .arg("--user") + .arg("--scope") + // Without this the scope is created but cargo-gamma still may not subdivide it, which is + // the failure this exists to prevent and one that would otherwise look identical. + .arg("--property=Delegate=yes") + // The controller has to be enabled on the unit for a delegated tree to inherit it. + .arg("--property=MemoryAccounting=yes") + // Deliberately no `MemoryMax` on the scope itself. cargo-gamma bounds each test binary + // from inside, and an outer ceiling low enough to matter would also be low enough to kill + // an ordinary `rustc` link step — which would surface as mutants "caught" by a build + // failure and a run that looks greener than it is. + .arg("--quiet") + // The unit is transient and nobody will ever look at it again; leaving spent scopes behind + // for the user to garbage-collect would be a slow leak in their session. + .arg("--collect") + .arg("--same-dir") + .arg("--") + .arg(exe) + .args(args) + .env(MARKER, "1"); + + command +} + +/// Re-runs this process inside a delegated scope, returning its exit code. +/// +/// `Ok(None)` means no relaunch was attempted and the caller should carry on as it would have. +/// An `Err` means the relaunch was attempted and failed, which is worth reporting rather than +/// swallowing: the user asked for a bound, something was supposed to provide it, and it did not. +pub(crate) fn relaunch() -> Result, Refusal> { + relaunch_unless(relaunched()) +} + +/// Relaunches unless this process is already the relaunched one. +/// +/// The guard is a parameter rather than a call so that the recursion case can be tested without +/// setting an environment variable, which in a threaded test binary is a change every other test +/// sees. +fn relaunch_unless(marked: bool) -> Result, Refusal> { + if marked { + return Err(Refusal::AlreadyInScope); + } + + let Ok(exe) = env::current_exe() else { + return Err(Refusal::Unavailable); + }; + + if which(SYSTEMD_RUN).is_none() { + return Err(Refusal::Unavailable); + } + + // The first argument is this program's own name, which is replaced by the resolved path to the + // executable. `cargo gamma` reaches here as `cargo-gamma gamma ...`, so the subcommand word is + // part of the arguments and has to be forwarded with the rest. + let args: Vec = env::args_os().skip(1).collect(); + + match scope_command(exe.into_os_string(), args).status() { + // A scope that could not be started leaves the run exactly where it was, so this is a + // refusal rather than a failure: the caller degrades or errors on its own terms. + Err(_cause) => Err(Refusal::Unavailable), + + Ok(status) => Ok(Some(status.code().unwrap_or(crate::commands::EXIT_CANNOT_PROCEED))), + } +} + +/// Finds an executable on `PATH`. +/// +/// Hand-rolled rather than taken from a crate, because the question is small and asking it wrongly +/// is cheap: a false negative means no relaunch and the old behaviour, and a false positive means +/// a spawn that fails and is handled. +fn which(program: &str) -> Option { + let paths = env::var_os("PATH")?; + + env::split_paths(&paths) + .map(|directory| directory.join(program)) + .find(|candidate| candidate.is_file()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every property the relaunch depends on lives in this command line, and getting any of them + /// wrong fails in a way that looks like the host being at fault rather than the command. + #[test] + fn the_scope_command_asks_for_a_delegated_cgroup_and_forwards_the_invocation() { + let command = scope_command( + OsString::from("/opt/cargo-gamma"), + vec![OsString::from("gamma"), OsString::from("run")], + ); + + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(command.get_program(), "systemd-run"); + assert!(args.contains(&"--user".to_owned()), "{args:?}"); + assert!(args.contains(&"--scope".to_owned()), "{args:?}"); + assert!(args.contains(&"--property=Delegate=yes".to_owned()), "{args:?}"); + + // The executable and its arguments must follow the separator, or a leading `-` in the + // forwarded command line would be parsed as an option to `systemd-run`. + let separator = args.iter().position(|arg| arg == "--").expect("a separator"); + assert_eq!(args[separator + 1], "/opt/cargo-gamma"); + assert_eq!(args[separator + 2..], ["gamma", "run"]); + } + + /// The child must be able to tell that it is the child, or it relaunches forever. + #[test] + fn the_relaunched_process_is_marked_as_one() { + let command = scope_command(OsString::from("/opt/cargo-gamma"), Vec::new()); + + let marked = command + .get_envs() + .any(|(name, value)| name == MARKER && value.is_some_and(|value| !value.is_empty())); + + assert!(marked, "the marker must be set in the child's environment"); + } + + /// Relaunching from within the scope would loop, and would do so while looking like progress: + /// each generation would print the same note and start the same run. + #[test] + fn a_process_already_inside_a_scope_refuses_to_relaunch_again() { + assert_eq!(relaunch_unless(true), Err(Refusal::AlreadyInScope)); + } + + /// A program that is not on `PATH` must not be reported as found, or the relaunch spawns + /// something that cannot exist and blames the host for the failure. + #[test] + #[cfg(not(miri))] + fn an_absent_program_is_not_found_on_the_path() { + assert!(which("cargo-gamma-nonexistent-probe").is_none()); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/session.rs b/crates/cargo-gamma-lib/src/exec/session.rs new file mode 100644 index 000000000..c685ef9c2 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/session.rs @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; + +use super::build::Round; +use super::test_binary::TestBinary; + +/// What happened during a run, beyond the verdicts written back onto the mutants. +#[derive(Debug, Clone)] +pub struct Session { + /// The sum of the test binaries' baseline durations, used to calibrate mutant budgets. + pub baseline: Duration, + + /// How long the concurrently measured baseline took on the wall clock. + pub baseline_wall: Duration, + + /// How many tests the baseline actually ran, or `None` if no harness announced a count. + /// + /// This is what ran rather than what exists: `--test-package`, `--include-test` and any filter + /// passed through to the harness all narrow it. Carried on the session rather than only printed + /// as the baseline finishes, because progress output resolves to whether a terminal is + /// attached — so in CI, where "did my suite run at all" is the single most useful thing this + /// figure answers, the transient line saying it is exactly the one nobody sees. + pub tests: Option, + + /// The longest the baseline legitimately went without saying anything. + pub quiet: Duration, + + /// The silence a mutant was allowed before it was presumed hung, when that was enabled. + pub stall: Option, + + /// How long the single build took. + pub build: Duration, + + /// The largest peak memory any one test binary reached during the baseline. + /// + /// `None` when nothing measured it, which is both the default and what a host without an + /// aggregate process-tree accounting facility can offer. Reported because it is the figure a + /// memory ceiling is chosen from, and because a suite whose peak surprises its authors is worth + /// knowing about whether or not a ceiling is being enforced. + pub peak: Option, + + /// Whether this run actually metered memory, which is not always what was configured. + /// + /// Memory control is on by default, and a host without cgroup v2 delegation cannot provide it. + /// A run that defaulted into it and could not have it degrades rather than stopping, so the + /// configured policy is a request and this is the answer. Everything downstream reads this one, + /// because asking the platform for accounting it already declined to give would fail every + /// mutant in the sweep. + pub metered: bool, + + /// Why memory went unbounded, when it was meant to be bounded and could not be. + /// + /// Carried to the end of the run rather than printed when it is discovered, because progress + /// output is suppressed when nothing is watching it — and a CI runner with no cgroup delegation + /// is exactly the case where the protection is missing *and* nobody sees the transient line + /// saying so. + pub unbounded: Option, + + /// How many mutants were withdrawn because they could not compile. + pub withdrawn: usize, + + /// Why they were withdrawn, grouped by rustc error code and mutator, densest pair first. + /// + /// The count alone says whether unviability is expensive; only this says whether it is + /// something a mutator could be taught to avoid, or an unavoidable cost of instrumenting the + /// tree at all. Carried on the session so that asking the question needs no code change. + pub census: Vec, + + /// How many rollback rounds were needed. + pub rounds: u32, + + /// What each of those rounds cost and withdrew, oldest first. + /// + /// Carried alongside `build` because the total on its own cannot tell a build that compiled + /// first time from one that spent most of its time converging. The two want very different + /// remedies — a faster machine against fewer unviable mutants — and a run that does not say + /// which it was leaves that choice to guesswork. + pub rounds_taken: Vec, + + /// The test binaries that were run. + pub binaries: Vec, + + /// Where the run put everything it kept on disk. + /// + /// The path rather than the size, because the size is a walk of a directory holding every build + /// artifact of every round, and only the diagnostics dump ever prints it. See + /// [`footprint`](crate::exec::footprint), which turns this into that figure on demand. + /// + /// Worth reporting at all because the disk is a real operating cost rather than a curiosity: a + /// large workspace can leave tens of gigabytes here, which is more than the free space on a + /// common CI runner, and a job whose next step fails for want of disk deserves to know where the + /// disk went. + pub scratch: camino::Utf8PathBuf, + + /// How many test targets `--include-test` or `--exclude-test` kept out of the oracle. + /// + /// Zero unless one of those was given. Reported because a narrowed oracle is the single most + /// consequential thing that can happen to a score without appearing anywhere in it: a survivor + /// here may be a mutant the excluded target would have caught, and a reader who did not write + /// the `gamma.toml` has no other way to know the suite was not asked in full. + pub filtered: usize, + + /// Whether the run had to build test targets it knew it would never consult. + /// + /// Building only the packages whose tests can reach a mutant is the cheaper thing to do, but + /// cargo unifies features over the packages it is asked to build, so a test target that only + /// compiles because some other package switches a feature on will not compile on its own. When + /// that happens the selection is abandoned and the whole workspace is built, and the run says + /// so: the scope the user asked for did not survive contact with their feature graph. + pub widened: bool, + + /// What the stale build-ordering hints put in front of the compiler, and what came of it. + /// + /// A record whose build context no longer matches still knows which mutants failed to compile + /// for it, and that knowledge is allowed to decide what the compiler sees first — never what + /// gets built, judged or scored. These are the two facts that say whether it is paying: how + /// many mutants were front-loaded, and how many of those the compiler then refused. See + /// [`crate::exec::OrderingHints`] for why there is no "rounds saved" figure here. + pub ordering: crate::exec::OrderingHints, + + /// What each phase of the run cost, gathered a clock at a time as the phases finish. + /// + /// The aggregates above cannot be taken apart after the fact — [`Self::build`] folds the copy, + /// the preflight and the compile into one number, and whether the per-test census pays for + /// itself is invisible while its cost hides inside that same figure. Only a clock started at + /// each phase can say where the time went, so each is timed once, as it runs, and left here for + /// the diagnostic reporter to surface. Nothing else reads it, and the run behaves identically whether or + /// not anyone ever does. + pub phases: Phases, +} + +/// What each phase of a run cost, so a fixed or testing total can be read apart into its parts. +/// +/// The copy and the preflight are components of [`Session::build`]; the census and the sweep are +/// components of the testing window. Neither set sums to its aggregate — compiling sits between the +/// copy and the baseline, and bookkeeping sits between the census and the sweep — so this is a +/// profile of where a slow run is slow, not a reconciliation of the totals it lives beside. +#[derive(Debug, Clone, Default)] +pub struct Phases { + /// What duplicating the workspace into the scratch tree cost, before a line of it was + /// instrumented. Part of [`Session::build`]. + pub copy: Duration, + + /// What the preflight cost: proving the tree compiles at all before any mutant was staged. Part + /// of [`Session::build`]. + pub preflight: Duration, + + /// What the census cost and covered, or `None` when `--whole-test-binaries` disabled it. + /// + /// Absent rather than zero: a run with no census did not spend zero time censusing, it did not + /// census, and the difference is the whole question of whether turning the census on was worth + /// it. + pub census: Option, + + /// What the sweep cost and how it spent its launches, or `None` when nothing was swept. + pub sweep: Option, +} + +/// What the per-test census cost and covered. +/// +/// The census spends a subprocess per test to learn which tests can reach which sites, and is +/// repaid during the sweep by running fewer tests per mutant. Whether that trade is positive +/// depends on the workspace, and these are the figures that let anyone — including `--estimate` — +/// see which way it went, rather than folding the cost invisibly into the build. +#[derive(Debug, Clone)] +pub struct CensusCost { + /// How long the whole census took, across every binary and every test. + pub elapsed: Duration, + + /// How many sample subprocesses the census actually launched: one per test run, not counting + /// the tests of a binary skipped after it spoiled, nor the per-binary listing runs. + pub walked: usize, + + /// How many test binaries the census examined. + pub binaries: usize, +} + +/// What the sweep cost and how it spent its subprocess launches. +/// +/// The launch count is what turns the cost model's `build + Σ(launch + prefix)` from a formula into +/// a measurement; the probe count is what says whether the killer hints and the census are earning +/// their keep, since a probe is a launch the run only made because a hint pointed at it. +#[derive(Debug, Clone)] +pub struct SweepCost { + /// How long the sweep took, across every mutant and every binary it ran. + pub elapsed: Duration, + + /// How many test-binary subprocesses the sweep launched in total. + pub launches: usize, + + /// How many of those launches were hint-directed probes rather than ordinary binary runs. + pub probes: usize, +} diff --git a/crates/cargo-gamma-lib/src/exec/stall.rs b/crates/cargo-gamma-lib/src/exec/stall.rs new file mode 100644 index 000000000..da05f4ff8 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/stall.rs @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; +use std::sync::Mutex; +use std::time::Instant; + +use super::progress::Progress; + +/// How long a binary may go silent before it is presumed hung. +#[derive(Debug, Clone, Copy, Default)] +pub(super) struct Stall { + /// The budget, or `None` to wait out the full timeout. + pub(super) budget: Option, +} + +impl Stall { + /// No stall detection: every mutant waits out its whole budget. + pub(super) const NONE: Self = Self { budget: None }; + + /// Builds a budget from the longest silence the baseline legitimately produced. + /// + /// Calibrating from the measured quiet period is the point: a suite whose slowest test takes + /// half a minute goes quiet that long when healthy, and a fixed budget would either call that + /// a hang or be too loose to help a suite of millisecond tests. + #[must_use] + pub(super) fn calibrated(quiet: Duration, factor: f64, floor: Duration) -> Self { + let budget = quiet.mul_f64(factor).max(floor); + + Self { budget: Some(budget) } + } + + /// Whether the binary has been silent for longer than the budget allows. + pub(super) fn exceeded(self, progress: &Mutex) -> bool { + self.exceeded_at(progress, Instant::now()) + } + + /// The same question, asked as of `now`. + /// + /// Splitting the clock out of the decision is what makes the boundary testable. Asserting it + /// through the real clock means sleeping either side of the budget, which is slow and, on a + /// loaded machine, wrong: a thread that oversleeps its 5 ms by 30 turns "under the budget" into + /// a stall and produces a failure nobody can reproduce. The boundary itself has nothing to do + /// with the passage of time — it is one comparison — so the time is passed in. + fn exceeded_at(self, progress: &Mutex, now: Instant) -> bool { + let Some(budget) = self.budget else { + return false; + }; + + #[expect(clippy::unwrap_used, reason = "the reader only panics if the whole process is unwinding")] + let progress = progress.lock().unwrap(); + + now.saturating_duration_since(progress.heard) > budget + } + + /// How long the binary may still stay silent before the budget is exceeded. + /// + /// A stall is the one thing the waiting thread cannot be woken for, because it *is* the absence + /// of anything to wake on — so it has to be waited out. This says how long that wait may be: + /// sleeping longer would let a hung binary go unnoticed past the budget the run promised. + /// + /// `Duration::MAX` when there is no budget, which leaves the caller's other bounds in charge. + pub(super) fn slack(self, progress: &Mutex) -> Duration { + self.slack_at(progress, Instant::now()) + } + + /// The same figure, as of `now`; see [`Self::exceeded_at`] for why the clock is a parameter. + fn slack_at(self, progress: &Mutex, now: Instant) -> Duration { + let Some(budget) = self.budget else { + return Duration::MAX; + }; + + #[expect(clippy::unwrap_used, reason = "the reader only panics if the whole process is unwinding")] + let heard = progress.lock().unwrap().heard; + + // Saturating to zero rather than going negative: the budget is already spent, and the + // caller's next pass through the loop is what notices. + budget.saturating_sub(now.saturating_duration_since(heard)) + } + + /// The same budget, multiplied. + /// + /// Used to re-ask a question rather than to ask a new one: a suspected stall is retried under a + /// budget this much looser, which scheduling noise cannot survive but a real hang still will. + pub(super) fn scaled(self, factor: u32) -> Self { + Self { + budget: self.budget.map(|budget| budget.saturating_mul(factor)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::exec::progress::Watch; + + #[test] + fn a_stall_budget_scales_with_the_silence_the_baseline_produced() { + let quiet = Duration::from_secs(30); + let stall = Stall::calibrated(quiet, 2.0, Duration::from_secs(1)); + + assert_eq!(stall.budget, Some(Duration::from_mins(1))); + } + + #[test] + fn a_suite_that_never_goes_quiet_still_gets_a_usable_budget() { + // Otherwise a suite of millisecond tests would produce a budget of nothing, and scheduler + // noise on a loaded machine would read as a hang. + let stall = Stall::calibrated(Duration::ZERO, 10.0, Duration::from_secs(5)); + + assert_eq!(stall.budget, Some(Duration::from_secs(5))); + } + + /// A silence of exactly `elapsed`, without any of it having actually passed. + fn silent_for(elapsed: Duration) -> Mutex { + let mut progress = Progress::new(Watch::Off); + + // The field is the only clock input `exceeded` has, so backdating it is the whole fake. + progress.heard = Instant::now() + .checked_sub(elapsed) + .expect("the process has not been running since the epoch"); + + Mutex::new(progress) + } + + #[test] + fn without_a_budget_nothing_is_ever_declared_stalled() { + // No elapsed silence is needed: without a budget there is no comparison to make. + assert!(!Stall::NONE.exceeded(&silent_for(Duration::ZERO))); + assert_eq!(Stall::NONE.slack(&silent_for(Duration::ZERO)), Duration::MAX); + } + + /// The stall boundary is asserted at it and either side of it, and no test sleeps. + /// + /// Both parts matter. Sleeping to cross a millisecond threshold is slow and unreliable — a + /// thread that oversleeps turns the under-budget case into a stall — so the sub-budget case in + /// particular could never be asserted honestly against the real clock, and the boundary itself + /// was simply never pinned. The comparison is `>`, so equality is *not* a stall: a binary that + /// has been silent for exactly its budget has used all of it and none more. + #[test] + fn the_stall_boundary_is_exclusive_and_is_pinned_on_both_sides_of_itself() { + let budget = Duration::from_secs(30); + let stall = Stall { budget: Some(budget) }; + let now = Instant::now(); + + for (elapsed, stalled) in [ + (Duration::ZERO, false), + (budget.saturating_sub(Duration::from_nanos(1)), false), + (budget, false), + (budget + Duration::from_nanos(1), true), + (budget * 2, true), + ] { + let progress = silent_for(Duration::ZERO); + + progress.lock().expect("a test holds the only reference").heard = + now.checked_sub(elapsed).expect("the process has not been running since the epoch"); + + assert_eq!( + stall.exceeded_at(&progress, now), + stalled, + "{elapsed:?} of silence against a {budget:?} budget" + ); + } + } + + /// The slack is what is left of the budget, and it stops at zero rather than going backwards. + /// + /// This is what bounds the wait loop's sleep, so a figure that ran past zero would be a wait + /// that overshot the budget the run promised, and one that did not fall as time passed would + /// be a hung binary noticed late. + #[test] + fn the_slack_falls_with_the_silence_and_stops_at_zero() { + let budget = Duration::from_secs(30); + let stall = Stall { budget: Some(budget) }; + let now = Instant::now(); + + for (elapsed, left) in [ + (Duration::ZERO, budget), + (Duration::from_secs(10), Duration::from_secs(20)), + (budget, Duration::ZERO), + (budget * 2, Duration::ZERO), + ] { + let progress = silent_for(Duration::ZERO); + + progress.lock().expect("a test holds the only reference").heard = + now.checked_sub(elapsed).expect("the process has not been running since the epoch"); + + assert_eq!(stall.slack_at(&progress, now), left, "{elapsed:?} of silence"); + } + } + + #[test] + fn a_line_of_output_clears_the_silence() { + let progress = silent_for(Duration::from_millis(51)); + let stall = Stall { + budget: Some(Duration::from_millis(50)), + }; + + assert!(stall.exceeded(&progress), "an hour of silence is a stall to begin with"); + + progress + .lock() + .expect("a test holds the only reference") + .heard("test tests::a ... ok\n"); + + assert!(!stall.exceeded(&progress)); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/sweep.rs b/crates/cargo-gamma-lib/src/exec/sweep.rs new file mode 100644 index 000000000..ea611f2bc --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/sweep.rs @@ -0,0 +1,2552 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Running the suite once per mutant and turning each result into a verdict. + +use core::sync::atomic::{AtomicUsize, Ordering}; +use core::time::Duration; +use std::sync::{Arc, Condvar, Mutex, OnceLock, mpsc}; +use std::thread; +use std::time::Instant; + +use camino::Utf8Path; +use cargo_gamma_process::MemoryRequest; + +use super::census::{Census, CensusSelection, CensusWork}; +use super::events::Events; +use super::killers::Killers; +use super::stall::Stall; +use super::test_binary::{Reachability, TestBinary}; +#[cfg(test)] +use super::test_binary::{TestScope, order_reachable, reaches}; +use super::verdict::{Attempt, Only, Verdict, run_binary}; +use super::workspace::Workspace; +use crate::Result; +use crate::discover::{Killer, Plan}; +use crate::error::error; +use crate::model::Outcome; + +/// Describes a mutant stopped by the memory ceiling installed for its test binary. +/// +/// Written to say what was measured and against what, because the reader's questions are whether +/// the ceiling was reasonable and how far past it the mutant went. Both are answerable only if the +/// note carries the two numbers rather than merely the fact. +/// +/// The ceiling is inferred rather than observed: it is derived from what the same binary used +/// during the baseline, scaled and given headroom. Only the peak, when a peak is reported at all, +/// is a measurement — so the wording states the ceiling as what the run allowed rather than as +/// anything the platform saw. +fn memory_note(binary: &Utf8Path, peak: Option, limit: u64) -> String { + let name = binary.file_name().unwrap_or(binary.as_str()); + + let Some(peak) = peak else { + return format!("`{name}` reached the {} this run allowed it", crate::report::bytes(limit)); + }; + + // "past", "at" and "against" are three different findings. A workload that went past its + // ceiling grew; one stopped exactly at it met a ceiling set a hair too low; one whose reported + // peak is below it was stopped on evidence other than that peak. Saying "past" for all three + // sends the reader looking for a growth that never happened. + let where_it_landed = match peak.cmp(&limit) { + core::cmp::Ordering::Greater => "past", + core::cmp::Ordering::Equal => "at", + core::cmp::Ordering::Less => "against", + }; + + // Both figures are rounded for reading, and a mutant that stopped a few kilobytes over its + // ceiling renders as the same number twice — "reached 512 MB, past the 512 MB this run allowed + // it" reads as a contradiction. The exact counts are the only thing that resolves it, so they + // are printed exactly when the rounded ones would collide. + // + // Only when the sentence claims a difference, though. A workload stopped exactly at its + // ceiling is the ordinary way an enforced run ends — the kernel caps the peak at the limit — so + // "at the" plus two identical figures is not a contradiction but the finding itself. Printing + // raw byte counts there made the common case the ugly one to read. + let (reached, allowed) = { + let rounded = (crate::report::bytes(peak), crate::report::bytes(limit)); + + if rounded.0 == rounded.1 && peak != limit { + (format!("{peak} bytes"), format!("{limit} bytes")) + } else { + rounded + } + }; + + format!("`{name}` reached {reached}, {where_it_landed} the {allowed} this run allowed it") +} + +/// Describes a stall, given the last test the harness named. +/// +/// The name is a landmark rather than a diagnosis, and the wording says so. libtest runs tests in +/// parallel and announces each one only once it has finished, so the test that is actually spinning +/// is by definition one it has not named. Wording that presents the name as the culprit sends +/// people to read a test that was fine, and — worse — invites a suppression on it. +fn stall_note(test: Option<&str>) -> String { + test.map_or_else( + || "stalled before the harness named a test".to_owned(), + |name| format!("stalled, last test named was `{name}`"), + ) +} + +/// Describes a flake, given the test that failed both with the mutant active and without it. +/// +/// The name is the whole value of this verdict. Without it the reader is told a test somewhere is +/// unreliable and left to find it, which is worse than recording it as a survivor would be — +/// at least a survivor names a line. The wording puts the remedy on the test rather than on the +/// mutant, because the mutant was never judged. +fn flaky_note(binary: &Utf8Path, test: Option<&str>) -> String { + let which = test.map_or_else(|| "a test".to_owned(), |name| format!("test `{name}`")); + + format!("{which} in `{binary}` fails with no mutant active as well as with one, so this mutant was never judged") +} + +/// Describes a mutant that prevents nextest from creating the selected test list. +fn enumeration_note(binary: &Utf8Path, output: &str) -> String { + let mut note = format!( + "`cargo nextest` could not enumerate tests in `{binary}` with this mutant active; the same selection succeeded with no mutant active" + ); + + if !output.is_empty() { + note.push_str(":\n"); + note.push_str(output); + } + + note +} + +/// One mutant's result: its index in the plan, what happened, how long it took and any detail. +type Completed = (usize, Outcome, u64, Option, Option); + +/// Estimates a single mutant's cost from per-site census data when available, falling back to +/// the sum of its reachable binary baselines. +/// +/// Census data gives the measured duration of the tests that actually reach each site, which is +/// far more precise than the binary-level baseline sum. The binary sum overstates killed mutants +/// (which exit early) and treats every mutant of a package identically; census data distinguishes +/// sites within a package by their actual reaching tests. +/// +/// Killer history further refines: a mutant with a known killer from a previous run is expected +/// to complete in the time of that single probe, which is cheaper than any cold path. +fn mutant_cost(position: usize, plan: &Plan, reach: &Reachability<'_>, census: &Census, killers: &Killers) -> Duration { + let mutant = &plan.mutants[position]; + + // A mutant with a persisted killer hint is expected to be killed by one probe. + if let Some(hint) = killers.hint(&mutant.id) + && let Some(binaries) = reach.reachable(&mutant.package) + && let Some(binary) = binaries.iter().find(|b| hint.names(&b.package, &b.target)) + { + return binary.baseline; + } + + let Some(binaries) = reach.reachable(&mutant.package) else { + return Duration::ZERO; + }; + + let mut total = Duration::ZERO; + let mut used_census = false; + + for binary in binaries { + match census.work(binary, mutant.ordinal) { + CensusWork::Selected(duration) => { + total += duration; + used_census = true; + } + CensusWork::Whole => total += binary.baseline, + CensusWork::Uncovered => {} + CensusWork::Hinted(_duration) => total += binary.baseline, + } + } + + // When census data was used, the estimate is site-specific and more accurate. + // When no census data, fall back to package-level baseline sum. + let _ = used_census; + total +} + +/// Orders package queues longest-first, then interleaves them. +/// +/// Workers pull from a single queue, so package queues start in descending expected-cost order +/// rather than whatever order discovery happened to enumerate them. Pulling already balances +/// individual mutants across workers; this ordering decides which independent workloads overlap. +/// +/// When census data is available, each mutant's cost is estimated from the measured duration of +/// its reaching tests. When a killer hint exists, the expected cost reflects a single probe. +/// Otherwise the cost falls back to the sum of reachable binary baselines. +/// +/// Mutants of one package that share a file are grouped together so that file-local killer +/// learning can benefit siblings. Within a package, own-package cold tests are prioritized by +/// placing mutants from the package that owns the binary first. +/// +/// Package order is the longest-first order, and positions within a package stay in plan order, so +/// the result remains deterministic. A round takes one mutant from every package that still has +/// work; the expensive queues therefore still start first without monopolising every worker while +/// independent package work remains. +fn schedule(pending: &mut [usize], plan: &Plan, reach: &Reachability<'_>, census: &Census, killers: &Killers) { + pending.sort_by_cached_key(|position| (core::cmp::Reverse(mutant_cost(*position, plan, reach, census, killers)), *position)); + + let mut queues: Vec> = Vec::new(); + let mut by_package: crate::HashMap = crate::HashMap::default(); + + for position in pending.iter().copied() { + let package = &*plan.mutants[position].package; + + if let Some(index) = by_package.get(package).copied() { + queues[index].push(position); + } else { + let index = queues.len(); + let _fresh = by_package.insert(package.to_owned(), index); + queues.push(vec![position]); + } + } + + let mut next = vec![0_usize; queues.len()]; + let mut out = 0_usize; + + loop { + let mut moved = false; + + for (queue, cursor) in queues.iter().zip(&mut next) { + if let Some(position) = queue.get(*cursor).copied() { + pending[out] = position; + out += 1; + *cursor += 1; + moved = true; + } + } + + if !moved { + break; + } + } +} + +/// What a sweep spent, tallied across its workers as they run. +/// +/// Two counters, incremented once per subprocess launched, so the tally costs a relaxed atomic add +/// per launch rather than anything per test. See [`Spent`], which is what a finished sweep hands +/// back once the counters have stopped moving. +#[derive(Debug, Default)] +struct Tally { + /// How many test-binary subprocesses were launched, across ordinary runs and probes alike. + launches: AtomicUsize, + + /// How many of those launches were hint-directed probes. + probes: AtomicUsize, +} + +/// What a finished sweep spent, read off its [`Tally`] once every worker has stopped. +#[derive(Debug, Clone, Copy, Default)] +pub(super) struct Spent { + /// How many test-binary subprocesses the sweep launched in total. + pub(super) launches: usize, + + /// How many of those launches were hint-directed probes. + pub(super) probes: usize, +} + +/// Tests every live mutant in parallel, writing verdicts back onto the plan. +/// +/// Workers publish each verdict over a channel that the calling thread drains while they are still +/// running, so the display moves as the run proceeds rather than jumping at the end. Returns +/// `None` when no mutant was pending and `Some` with the cost of a sweep that ran. +/// +/// # Errors +/// +/// Returns an error if the requested memory accounting becomes unavailable. The sweep stops rather +/// than produce verdicts without the protection the run promised. +#[expect(clippy::too_many_lines, reason = "sweep setup and its scoped worker loop share borrowed state")] +pub(super) fn test_all( + work: &Workspace, + plan: &mut Plan, + reach: &Reachability<'_>, + sweep: Sweep<'_>, + killers: &mut Killers, + events: &mut impl Events, +) -> Result> { + let jobs = sweep.jobs; + + let mut pending: Vec = plan + .mutants + .iter() + .enumerate() + .filter(|(_position, mutant)| mutant.ordinal > 0 && mutant.outcome == Outcome::Pending) + .map(|(position, _mutant)| position) + .collect(); + + if pending.is_empty() { + return Ok(None); + } + + schedule(&mut pending, plan, reach, sweep.census, killers); + + let next = AtomicUsize::new(0); + let tally = Tally::default(); + let abandoned: OnceLock = OnceLock::new(); + + // Indexed by *queue position*, not by plan position, and therefore built after the schedule is + // fixed. Building either before it would silently pair one mutant's ordinal with another's + // reachable binaries. + let ordinals: Vec<(u32, Option)> = pending + .iter() + .map(|position| (plan.mutants[*position].ordinal, plan.mutants[*position].test_timeout_multiplier)) + .collect(); + + let reachable: Vec<&[&TestBinary]> = pending + .iter() + .map(|position| { + reach + .reachable(&plan.mutants[*position].package) + .expect("the shared reachability index was built from these same pending mutants") + }) + .collect(); + + let mut files: crate::HashMap<_, usize> = crate::HashMap::default(); + let file_slots: Vec = pending + .iter() + .map(|position| { + let file = Arc::clone(&plan.mutants[*position].file); + let next = files.len(); + + *files.entry(file).or_insert(next) + }) + .collect(); + let file_killers: Vec = (0..files.len()).map(|_file| FileLearning::new()).collect(); + + let (sender, receiver) = mpsc::channel::(); + + // Resolved up front for the same reason reachability is: a worker must need nothing from the + // plan, so the calling thread can borrow it mutably and record verdicts as they arrive. Cloned + // rather than borrowed because the same map is written back as those verdicts land. + let hints: Vec> = pending + .iter() + .map(|position| killers.hint(&plan.mutants[*position].id).cloned()) + .collect(); + let notes = crate::notes::current(); + + thread::scope(|scope| { + for _worker in 0..jobs.max(1) { + let sender = sender.clone(); + let next = &next; + let ordinals = &ordinals; + let reachable = &reachable; + let hints = &hints; + let pending = &pending; + let file_slots = &file_slots; + let file_killers = &file_killers; + let abandoned = &abandoned; + let tally = &tally; + let notes = notes.clone(); + + let _handle = scope.spawn(move || { + let _notes = crate::notes::enter(notes.as_ref()); + + loop { + if abandoned.get().is_some() { + break; + } + + let index = next.fetch_add(1, Ordering::Relaxed); + + let Some(position) = pending.get(index).copied() else { + break; + }; + + let (ordinal, timeout_multiplier) = ordinals[index]; + let started = Instant::now(); + let reachable = &reachable[index]; + let judged = judge_learning( + work, + ordinal, + reachable, + hints[index].as_ref(), + &file_killers[file_slots[index]], + timeout_multiplier, + sweep, + tally, + ); + + let (outcome, killer, note) = match judged { + Judgement::Reached(outcome, killer, note) => (outcome, killer, note), + Judgement::Abandoned(reason) => { + let _first = abandoned.set(reason); + + break; + } + }; + + let elapsed = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); + + // A closed receiver means the calling thread is gone, which cannot happen while + // the scope is open; there is nothing useful to do about it either way. + let _sent = sender.send((position, outcome, elapsed, killer, note)); + } + }); + } + + // The workers hold the only remaining senders, so the drain ends when the last one finishes. + drop(sender); + + for (position, outcome, elapsed, killer, note) in receiver { + if let Some(mutant) = plan.mutants.get_mut(position) { + mutant.outcome = outcome; + mutant.elapsed_ms = elapsed; + mutant.killed_by = killer.as_ref().map(|killer| killer.test.clone()); + mutant.note = note; + + // Written both ways round. An entry that convicted is worth keeping; a mutant whose + // verdict named no test has to lose the one it had, or every run after this one + // pays for a probe already shown not to convict. + match killer { + Some(killer) => killers.record(mutant.id.clone(), killer), + None => killers.forget(&mutant.id), + } + + events.mutant(mutant); + } + } + }); + + abandoned.into_inner().map_or_else( + || { + Ok(Some(Spent { + launches: tally.launches.into_inner(), + probes: tally.probes.into_inner(), + })) + }, + |reason| { + Err(error!( + "the run cannot be trusted to judge anything further: {reason}.\n\ + It stops here rather than reach verdicts that would each have to be taken on faith." + )) + }, + ) +} + +/// The settings every mutant in a sweep is run under. +/// +/// Carried together because they are decided once, before the first mutant, and read identically by +/// every worker; splitting them back out would only add arguments to the functions that thread them +/// through. +#[derive(Debug, Clone, Copy)] +pub(super) struct Sweep<'run> { + /// Minimum timeout floor applied to test budgets. + pub(super) timeout_floor: Duration, + + /// How long a test binary may go without saying anything before it is treated as stuck. + pub(super) stall: Stall, + + /// How many mutants to run at once. + pub(super) jobs: usize, + + /// Whether each run's memory is to be accounted for at all. + pub(super) meter: bool, + + /// Whether a failing test is re-run with no mutant active before it is believed. + pub(super) confirm: bool, + + /// Which tests reach which sites, empty when nothing was measured. + pub(super) census: &'run Census, +} + +/// What one mutant's run across its reachable test binaries came to. +enum Judgement { + /// The mutant was judged: an outcome, the test that caught it if one did, and any note. + Reached(Outcome, Option, Option), + + /// The run could no longer be metered as asked, and no verdict from here on would mean anything. + Abandoned(String), +} + +/// Runs the one test that caught this mutant last time, and says whether it caught it again. +/// +/// A re-killed mutant's cost falls from a partial binary — every test ahead of its killer, in +/// whatever order the harness runs them — to a single test. +/// +/// Only a failure is believed. Every other verdict a probe can reach is discarded and the ordinary +/// binary run proceeds unchanged. The caller reaches the probe in canonical binary order and only +/// permits it when filtering cannot bypass another outcome from that same binary. +fn probe( + work: &Workspace, + ordinal: u32, + binary: &TestBinary, + hint: &Killer, + timeout_multiplier: Option, + sweep: Sweep<'_>, + tally: &Tally, +) -> Option { + let request = MemoryRequest { + meter: sweep.meter, + limit: binary.memory, + }; + + let attempt = Attempt { + active: Some(ordinal), + timeout: binary.budget_for(timeout_multiplier, sweep.timeout_floor), + stall: sweep.stall, + request, + only: Only::One(&hint.test), + census: None, + }; + + // A probe is one subprocess launched because a hint pointed at it, so it counts as both. + let _launches = tally.launches.fetch_add(1, Ordering::Relaxed); + let _probes = tally.probes.fetch_add(1, Ordering::Relaxed); + + let verdict = run_binary(work, binary, attempt, sweep.confirm); + + match verdict { + // The harness names the test it ran, which under a filter can only be the one asked for; + // the recorded name is used when it names nothing, so the map stays populated either way. + Verdict::Failed(named) => Some(Killer { + package: hint.package.clone(), + target: hint.target.clone(), + test: named.unwrap_or_else(|| hint.test.clone()), + }), + _inconclusive => None, + } +} + +/// Whether a filtered failure is conclusive for this binary. +/// +/// Filtering changes the workload, so it cannot settle a kill when the whole binary might instead +/// time out, stall, exceed a memory limit, fail confirmation, or lose its meter. Launch refusal is +/// covered by the successful filtered launch itself. Earlier binaries are protected by trying the +/// probe only when canonical iteration reaches this binary. +fn filtered_kill_is_final(binary: &TestBinary, timeout_multiplier: Option, sweep: Sweep<'_>) -> bool { + binary.budget_for(timeout_multiplier, sweep.timeout_floor).is_none() && sweep.stall.budget.is_none() && !sweep.meter && !sweep.confirm +} + +/// Tries tests found by an incomplete census without trusting their absence of a failure. +/// +/// A failure is evidence that the mutant was killed. Every other result falls back to the whole +/// binary, because an incomplete census cannot establish that no unmeasured test would fail. +fn probe_cases( + work: &Workspace, + ordinal: u32, + binary: &TestBinary, + names: &[&str], + timeout_multiplier: Option, + sweep: Sweep<'_>, + tally: &Tally, +) -> Option { + let attempt = Attempt { + active: Some(ordinal), + timeout: binary.budget_for(timeout_multiplier, sweep.timeout_floor), + stall: sweep.stall, + request: MemoryRequest { + meter: sweep.meter, + limit: binary.memory, + }, + only: Only::These(names), + census: None, + }; + + let _launches = tally.launches.fetch_add(1, Ordering::Relaxed); + let _probes = tally.probes.fetch_add(1, Ordering::Relaxed); + + match run_binary(work, binary, attempt, sweep.confirm) { + Verdict::Failed(name) => Some(Killer { + package: binary.package.clone(), + target: binary.target.clone(), + test: name.unwrap_or_else(|| { + names + .first() + .copied() + .expect("an incomplete census hint always names at least one test") + .to_owned() + }), + }), + _inconclusive => None, + } +} + +/// Runs one mutant against every test binary that can reach it, stopping at the first detection. +/// +/// Later binaries are not run once one has caught the mutant: the answer cannot change, and the +/// time saved is the difference between a sweep that finishes overnight and one that does not. +/// +/// When an earlier run recorded which test caught this mutant, that one test is tried when canonical +/// iteration reaches its binary and filtering cannot change the binary's outcome. It is a guess and +/// it is checked, never believed: see [`probe`]. +#[cfg(test)] +fn judge( + work: &Workspace, + ordinal: u32, + reachable: &[&TestBinary], + hint: Option<&Killer>, + timeout_multiplier: Option, + sweep: Sweep<'_>, + tally: &Tally, +) -> Judgement { + judge_ordered(work, ordinal, reachable, hint, None, timeout_multiplier, sweep, tally) +} + +#[expect(clippy::too_many_arguments, reason = "the hot verdict path keeps execution state borrowed")] +fn judge_ordered( + work: &Workspace, + ordinal: u32, + reachable: &[&TestBinary], + hint: Option<&Killer>, + file_hint: Option<&Killer>, + timeout_multiplier: Option, + sweep: Sweep<'_>, + tally: &Tally, +) -> Judgement { + // Set by the first binary this mutant is actually run against. Left false, nothing that could + // convict this code was run — no test binary links it, none of them announced a test, or a + // census established that no test in any of them executes the site — and reporting that as a + // survivor would blame the tests that exist for the absence of ones that do not. + let mut ran = false; + + for binary in reachable.iter().copied() { + let selection = sweep.census.selection(binary, ordinal); + + // A hint naming another binary is stale or outside the selected package set. Waiting until + // canonical iteration reaches the named binary prevents its kill from bypassing any + // earlier Flaky, Pending, resource, or metering outcome. + let killer_hint = hint + .filter(|hint| hint.names(&binary.package, &binary.target)) + .or_else(|| file_hint.filter(|hint| hint.names(&binary.package, &binary.target))); + + if let Some(hint) = killer_hint + && filtered_kill_is_final(binary, timeout_multiplier, sweep) + && let Some(killer) = probe(work, ordinal, binary, hint, timeout_multiplier, sweep, tally) + { + return Judgement::Reached(Outcome::Killed, Some(killer), None); + } + + if let CensusSelection::Hinted(names) = &selection + && filtered_kill_is_final(binary, timeout_multiplier, sweep) + && let Some(killer) = probe_cases(work, ordinal, binary, names, timeout_multiplier, sweep, tally) + { + return Judgement::Reached(Outcome::Killed, Some(killer), None); + } + + let only = match &selection { + CensusSelection::Whole | CensusSelection::Hinted(_) => Only::All, + CensusSelection::Uncovered => continue, + CensusSelection::Selected(names) => Only::These(names), + }; + + ran = true; + + let request = MemoryRequest { + meter: sweep.meter, + limit: binary.memory, + }; + + let attempt = Attempt { + active: Some(ordinal), + timeout: binary.budget_for(timeout_multiplier, sweep.timeout_floor), + stall: sweep.stall, + request, + only, + census: None, + }; + + let _launches = tally.launches.fetch_add(1, Ordering::Relaxed); + + let mut verdict = run_binary(work, binary, attempt, sweep.confirm); + + // A complete census proves which tests can observe the mutant, but filtering also shrinks + // runtime and peak memory and can change failure order. A non-passing filtered run is + // therefore provisional: repeat the whole binary and use only its canonical outcome. + if matches!(selection, CensusSelection::Selected(_)) && !matches!(verdict, Verdict::Passed) { + let _launches = tally.launches.fetch_add(1, Ordering::Relaxed); + verdict = run_binary( + work, + binary, + Attempt { + only: Only::All, + ..attempt + }, + sweep.confirm, + ); + } + + match verdict { + Verdict::Passed => {} + Verdict::Failed(name) => { + let killer = name.map(|test| Killer { + package: binary.package.clone(), + target: binary.target.clone(), + test, + }); + + return Judgement::Reached(Outcome::Killed, killer, None); + } + Verdict::TestEnumerationFailed(output) => { + return Judgement::Reached(Outcome::Killed, None, Some(enumeration_note(&binary.path, &output))); + } + Verdict::TimedOut => return Judgement::Reached(Outcome::Timeout, None, None), + Verdict::Stalled(test) => { + return Judgement::Reached(Outcome::Timeout, None, Some(stall_note(test.as_deref()))); + } + + // The suite's own harness did not fail, but the baseline established that this same + // workload fits under this same ceiling without the mutant, so the mutant is what + // changed. This is still undetected for scoring: no assertion exposed the change. + Verdict::MemoryLimit { peak, limit } => { + return Judgement::Reached(Outcome::OutOfMemory, None, Some(memory_note(&binary.path, peak, limit))); + } + // The suite failed with the mutant active and failed again without it, so this run + // established nothing about this mutant. Recorded as its own outcome so it lands in + // neither the score nor the survivor list, and carrying the test to fix. + Verdict::Flaky(test) => return Judgement::Reached(Outcome::Flaky, None, Some(flaky_note(&binary.path, test.as_deref()))), + Verdict::Unmetered(reason) => return Judgement::Abandoned(reason), + + // One run the machine would not perform, which is a fact about this mutant and not + // about the run: the shortage behind it — descriptors, process slots — is one the sweep + // creates for itself and clears as its other workers finish. Recorded as unjudged + // against this mutant, with the refusal as the note, so that the mutants around it keep + // their verdicts and the reader can see which one went without. + Verdict::Unjudged(reason) => return Judgement::Reached(Outcome::Pending, None, Some(reason)), + } + } + + Judgement::Reached(if ran { Outcome::Survived } else { Outcome::NoCoverage }, None, None) +} + +/// Learns one file's first observed killer before letting its remaining mutants proceed in +/// parallel with that binary first. +/// +/// Workers that encounter `InProgress` wait on a condvar (bounded by a short timeout) rather +/// than immediately proceeding without the hint. This avoids redundantly launching the expensive +/// cold path for siblings when a killer is about to be published. The bounded wait ensures +/// workers are never idled indefinitely if no common killer exists. +#[expect(clippy::too_many_arguments, reason = "adds one file-local state cell to the verdict path")] +fn judge_learning( + work: &Workspace, + ordinal: u32, + reachable: &[&TestBinary], + hint: Option<&Killer>, + observed: &FileLearning, + timeout_multiplier: Option, + sweep: Sweep<'_>, + tally: &Tally, +) -> Judgement { + if hint.is_some() { + let judged = judge_ordered(work, ordinal, reachable, hint, None, timeout_multiplier, sweep, tally); + + if let Judgement::Reached(_outcome, Some(killer), _note) = &judged { + let mut state = observed.state.lock().expect("a file-local killer lock was poisoned"); + *state = Learning::Learned(killer.clone()); + drop(state); + observed.notify.notify_all(); + } + + return judged; + } + + let state = { + let mut learned = observed.state.lock().expect("a file-local killer lock was poisoned"); + + match &*learned { + Learning::Learned(killer) => Some(Ok(killer.clone())), + Learning::Untried => { + *learned = Learning::InProgress; + None + } + Learning::InProgress => { + // Wait for the learner to finish, bounded so workers are never stuck. + let result = observed + .notify + .wait_timeout_while(learned, LEARNING_WAIT, |s| matches!(s, Learning::InProgress)) + .expect("a file-local killer lock was poisoned"); + learned = result.0; + match &*learned { + Learning::Learned(killer) => Some(Ok(killer.clone())), + // Timed out or exhausted: proceed without hint. + _ => Some(Err(())), + } + } + Learning::Exhausted => Some(Err(())), + } + }; + + if let Some(state) = state { + return match state { + Ok(killer) => judge_ordered(work, ordinal, reachable, None, Some(&killer), timeout_multiplier, sweep, tally), + Err(()) => judge_ordered(work, ordinal, reachable, None, None, timeout_multiplier, sweep, tally), + }; + } + + let judged = judge_ordered(work, ordinal, reachable, None, None, timeout_multiplier, sweep, tally); + + { + let mut state = observed.state.lock().expect("a file-local killer lock was poisoned"); + *state = if let Judgement::Reached(_outcome, Some(killer), _note) = &judged { + Learning::Learned(killer.clone()) + } else { + Learning::Exhausted + }; + observed.notify.notify_all(); + } + + judged +} + +/// How long a worker waits for a file's learner before proceeding without the hint. +/// +/// Short enough that workers do not idle when no common killer exists, long enough that a fast +/// learner (a killed mutant in single-digit milliseconds) publishes before siblings launch. +const LEARNING_WAIT: Duration = Duration::from_millis(50); + +/// Per-file learning state with a condvar for wait/notify coordination. +struct FileLearning { + state: Mutex, + notify: Condvar, +} + +impl FileLearning { + const fn new() -> Self { + Self { + state: Mutex::new(Learning::Untried), + notify: Condvar::new(), + } + } +} + +#[derive(Debug)] +enum Learning { + Untried, + InProgress, + Learned(Killer), + Exhausted, +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use core::iter::once; + #[cfg(unix)] + use std::fs; + + use camino::Utf8PathBuf; + + use super::*; + #[cfg(unix)] + use crate::exec::faults::{self, Fault}; + #[cfg(unix)] + use crate::exec::memory; + use crate::ops::collect::Shape; + + /// A census that knows nothing, which is what every test here that is not about narrowing runs + /// under: the sweep then behaves exactly as it did before there was a census at all. + fn blind() -> &'static Census { + static BLIND: OnceLock = OnceLock::new(); + + BLIND.get_or_init(Census::default) + } + + fn sweep(stall: Stall) -> Sweep<'static> { + Sweep { + timeout_floor: Duration::ZERO, + stall, + jobs: 1, + meter: false, + confirm: true, + census: blind(), + } + } + + #[test] + fn a_stall_does_not_claim_the_named_test_is_the_one_that_hung() { + // Regression, issue-004. libtest names a test only once it has finished, so the test that + // is spinning is precisely the one not named. Wording that presents the name as the culprit + // sends people to read a test that was fine, and invites a suppression on it. + let note = stall_note(Some("tests::round_trip")); + + assert!(note.contains("last test named was `tests::round_trip`"), "{note}"); + assert!(!note.contains("during"), "{note}"); + assert!(!note.contains(" in `"), "{note}"); + } + + #[test] + fn a_stall_before_any_test_was_named_says_so() { + let note = stall_note(None); + + assert_eq!(note, "stalled before the harness named a test"); + } + + /// A workspace, a plan holding one pending mutant, and a test binary that behaves as told. + /// + /// `test_all` is the scheduler, and the verdicts it has to translate into outcomes are exactly + /// the ones a real suite produces least often, so the binary is a script rather than a + /// compiled harness: the process machinery is real, only the suite is stand-in. + #[cfg(unix)] + fn harness(body: &str, budget: Duration) -> (tempfile::TempDir, Workspace, Plan, Vec) { + let (directory, work) = crate::testing::shell_workspace("test-all", body); + let plan = one_mutant_plan(work.root.clone()); + let binaries = vec![TestBinary { + package: "subject".to_owned(), + baseline: Duration::from_millis(1), + budget: Some(budget), + ..crate::testing::test_binary("/bin/sh") + }]; + + (directory, work, plan, binaries) + } + + /// A workspace and plan for the cases where no binary is ever started. + /// + /// A mutant nothing can reach is decided without launching anything, so these need no shell and + /// no executable — which is what lets them run on every platform rather than only where + /// `/bin/sh` exists. The uncovered bucket is precisely the one that must not be left untested + /// on a platform, since being unreachable is what it asserts about. + fn unreachable_harness(binaries: Vec) -> (tempfile::TempDir, Workspace, Plan, Vec) { + let directory = crate::testing::workdir("test-all-uncovered"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + let work = Workspace::adopt(root.clone(), root.join("target")); + let plan = one_mutant_plan(root); + + (directory, work, plan, binaries) + } + + /// A plan holding a single pending mutant in package `subject`. + fn one_mutant_plan(root: Utf8PathBuf) -> Plan { + let mutant = crate::model::Mutant { + id: "m1".to_owned().into(), + ordinal: 1, + file: (Utf8PathBuf::from("src/a.rs")).into(), + package: ("subject".to_owned()).into(), + span: 0..1, + line: 1, + end_line: 1, + column: 1, + mutator: ("relational.gt_to_ge".to_owned()).into(), + item_path: ("subject::f".to_owned()).into(), + occurrence: 0, + replacement_index: 0, + original: "a > b".to_owned().into(), + replacement: "a >= b".to_owned().into(), + shape: Shape::Expr, + outcome: Outcome::Pending, + suppression: None, + expectation: None, + test_timeout_multiplier: None, + elapsed_ms: 0, + killed_by: None, + note: None, + }; + + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root, + files: Vec::new(), + mutants: vec![mutant], + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach: crate::HashMap::default(), + specs: crate::HashMap::default(), + } + } + + /// Like `harness`, but with `count` pending mutants instead of one, so that a sweep has enough + /// work in flight for more than one worker to be racing against the others. + #[cfg(unix)] + fn harness_n(body: &str, budget: Duration, count: usize) -> (tempfile::TempDir, Workspace, Plan, Vec) { + let (directory, work, mut plan, binaries) = harness(body, budget); + let template = plan.mutants[0].clone(); + + plan.mutants = (0..count) + .map(|index| { + let mut mutant = template.clone(); + mutant.id = format!("m{index}").into(); + mutant.ordinal = u32::try_from(index + 1).expect("test counts stay well under u32::MAX"); + mutant + }) + .collect(); + + (directory, work, plan, binaries) + } + + #[test] + #[cfg(unix)] + fn file_local_killer_is_probed_only_after_earlier_binaries() { + let (_directory, work, mut plan, mut binaries) = + harness_n("echo 'test tests::caught ... FAILED'; exit 1", Duration::from_secs(30), 2); + binaries[0].target = "killer".to_owned(); + binaries[0].budget = None; + + let mut passing = binaries[0].clone(); + passing.path = "/bin/true".into(); + passing.target = "passing".to_owned(); + passing.baseline = Duration::ZERO; + binaries.insert(0, passing); + + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let spent = test_all( + &work, + &mut plan, + &reach, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep should finish") + .expect("two mutants should be swept"); + + assert!(plan.mutants.iter().all(|mutant| mutant.outcome == Outcome::Killed)); + assert_eq!( + spent.launches, 5, + "both mutants must run the earlier binary before the learned hint is checked" + ); + assert_eq!(spent.probes, 1, "the learned killer is checked only after the earlier binary"); + } + + #[test] + #[cfg(unix)] + fn same_file_survivors_run_in_parallel_after_one_learning_attempt_starts() { + let (_directory, work, mut plan, binaries) = harness_n("sleep 0.20; exit 0", Duration::from_secs(30), 2); + let started = Instant::now(); + + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let _spent = test_all( + &work, + &mut plan, + &reach, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 2, + meter: false, + confirm: false, + census: blind(), + }, + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep should finish"); + + assert!(plan.mutants.iter().all(|mutant| mutant.outcome == Outcome::Survived)); + assert!(started.elapsed() < Duration::from_millis(350), "{:?}", started.elapsed()); + } + + /// A plan holding one pending mutant per named package, in the order named. + /// + /// Each package reaches only itself, so a mutant's cost is its own package's binary and the + /// packages are genuinely distinguishable. A scope of the whole workspace would make every + /// binary reach every mutant, every cost equal, and any ordering test vacuous. + fn plan_over(packages: &[&str]) -> Plan { + let mut plan = one_mutant_plan(Utf8PathBuf::from("/nowhere")); + let template = plan.mutants[0].clone(); + + plan.mutants = packages + .iter() + .enumerate() + .map(|(index, package)| crate::model::Mutant { + id: format!("m{index}").into(), + ordinal: u32::try_from(index + 1).expect("test counts stay well under u32::MAX"), + package: ((*package).to_owned()).into(), + ..template.clone() + }) + .collect(); + + plan.reach = packages + .iter() + .map(|package| ((*package).to_owned(), once((*package).to_owned()).collect())) + .collect(); + + plan + } + + /// A scope that narrows nothing but leaves the plan's reachability relation in force. + const NARROW: TestScope<'static> = TestScope { + packages: &[], + package_local: false, + whole_workspace: false, + }; + + /// A binary belonging to `package` whose suite takes `baseline` with no mutant active. + fn binary_of(package: &str, baseline: Duration) -> TestBinary { + TestBinary { + package: package.to_owned(), + baseline, + budget: Some(Duration::from_mins(1)), + ..crate::testing::test_binary("/bin/sh") + } + } + + #[test] + fn cold_runs_put_own_package_binaries_before_cheaper_dependents() { + let mut plan = plan_over(&["subject", "facade"]); + let inserted = plan + .reach + .get_mut("facade") + .expect("the fixture records every package") + .insert("subject".to_owned()); + assert!(inserted); + let mut own_slow = binary_of("subject", Duration::from_secs(20)); + own_slow.target = "subject-tests".to_owned(); + let mut dependent_fast = binary_of("facade", Duration::from_millis(1)); + dependent_fast.target = "facade-smoke".to_owned(); + let binaries = vec![dependent_fast, own_slow]; + + let sets = Reachability::build(&plan, &binaries, &NARROW); + let ordered = sets.reachable("subject").expect("subject holds the plan's one pending mutant"); + + assert_eq!( + ordered.iter().map(|binary| binary.package.as_str()).collect::>(), + ["subject", "facade"] + ); + } + + #[test] + fn cold_run_tiers_use_baseline_then_stable_identity() { + let mut dependent_z = binary_of("z-helper", Duration::from_secs(2)); + dependent_z.target = "z".to_owned(); + dependent_z.path = "/tests/z".into(); + + let mut own_slow = binary_of("subject", Duration::from_secs(3)); + own_slow.target = "slow".to_owned(); + own_slow.path = "/tests/own-slow".into(); + + let mut dependent_a = binary_of("a-helper", Duration::from_secs(2)); + dependent_a.target = "a".to_owned(); + dependent_a.path = "/tests/a".into(); + + let mut own_fast = binary_of("subject", Duration::from_secs(1)); + own_fast.target = "fast".to_owned(); + own_fast.path = "/tests/own-fast".into(); + + let mut ordered = vec![&dependent_z, &own_slow, &dependent_a, &own_fast]; + + order_reachable(&mut ordered, "subject"); + + assert_eq!( + ordered.iter().map(|binary| binary.path.as_str()).collect::>(), + ["/tests/own-fast", "/tests/own-slow", "/tests/a", "/tests/z"] + ); + } + + #[test] + fn cold_run_ordering_never_drops_a_reachable_binary() { + let own = binary_of("subject", Duration::from_secs(3)); + let helper = binary_of("helper", Duration::from_secs(2)); + let facade = binary_of("facade", Duration::from_secs(1)); + let mut ordered = vec![&helper, &own, &facade]; + + order_reachable(&mut ordered, "subject"); + + assert_eq!( + ordered.iter().map(|binary| binary.package.as_str()).collect::>(), + ["subject", "facade", "helper"] + ); + } + + /// The queue runs the most expensive mutants first, so the sweep does not end on one core. + /// + /// Workers pull from a single queue: whatever is picked up last runs alone while every other + /// core idles. In plan order the cheapest package can be enumerated first and the most + /// expensive last, which is the worst case and the one nothing prevented. + #[test] + fn the_queue_puts_the_most_expensive_mutants_first() { + let plan = plan_over(&["cheap", "dear", "middling"]); + let binaries = vec![ + binary_of("cheap", Duration::from_millis(1)), + binary_of("dear", Duration::from_secs(30)), + binary_of("middling", Duration::from_secs(2)), + ]; + + let mut pending: Vec = (0..plan.mutants.len()).collect(); + let sets = Reachability::build(&plan, &binaries, &NARROW); + + schedule(&mut pending, &plan, &sets, blind(), &Killers::default()); + + let order: Vec<&str> = pending.iter().map(|position| &*plan.mutants[*position].package).collect(); + + assert_eq!(order, vec!["dear", "middling", "cheap"]); + } + + /// Mutants of equal cost keep plan order, so the same plan schedules identically every time. + /// + /// Without a total order the sweep would be reproducible only by luck, and two runs of one plan + /// could interleave differently — which is exactly the kind of difference that makes an + /// intermittent failure impossible to attribute. + #[test] + fn mutants_that_cost_the_same_keep_the_order_the_plan_gave_them() { + let plan = plan_over(&["a", "b", "c", "d"]); + let binaries: Vec = ["a", "b", "c", "d"] + .iter() + .map(|package| binary_of(package, Duration::from_millis(7))) + .collect(); + + let mut pending: Vec = (0..plan.mutants.len()).collect(); + let sets = Reachability::build(&plan, &binaries, &NARROW); + + schedule(&mut pending, &plan, &sets, blind(), &Killers::default()); + + assert_eq!(pending, vec![0, 1, 2, 3]); + } + + /// A package is a correlated workload: all of its mutants traverse the same binaries in the + /// same order. Keeping each package contiguous sends every worker into an expensive binary at + /// once, which is especially destructive for tests that launch subprocesses. + #[test] + fn package_queues_are_interleaved_instead_of_forming_worker_convoys() { + let plan = plan_over(&["dear", "dear", "cheap", "cheap", "middle", "middle"]); + let binaries = vec![ + binary_of("dear", Duration::from_secs(30)), + binary_of("cheap", Duration::from_millis(1)), + binary_of("middle", Duration::from_secs(2)), + ]; + + let mut pending: Vec = (0..plan.mutants.len()).collect(); + let sets = Reachability::build(&plan, &binaries, &NARROW); + + schedule(&mut pending, &plan, &sets, blind(), &Killers::default()); + + let order: Vec<&str> = pending.iter().map(|position| &*plan.mutants[*position].package).collect(); + + assert_eq!(order, vec!["dear", "middle", "cheap", "dear", "middle", "cheap"]); + } + + /// Reachability is worked out per package, and every mutant of a package gets that same answer. + #[test] + fn every_mutant_of_a_package_gets_one_reachability_answer() { + let plan = plan_over(&["left", "right", "left", "right", "left"]); + let binaries = vec![ + binary_of("left", Duration::from_millis(1)), + binary_of("right", Duration::from_millis(1)), + ]; + + let pending: Vec = (0..plan.mutants.len()).collect(); + let sets = Reachability::build(&plan, &binaries, &NARROW); + + assert_eq!(sets.len(), 2, "one entry per distinct package, not one per mutant"); + + for position in &pending { + let package = &*plan.mutants[*position].package; + let found: Vec<&TestBinary> = binaries.iter().filter(|binary| reaches(binary, package, &plan, &NARROW)).collect(); + + assert_eq!( + sets.reachable(package).expect("every package in `pending` holds a pending mutant"), + found.as_slice(), + "the memoized set must equal the one a per-mutant filter would give" + ); + } + } + + /// A mutant whose suite never finishes within its budget is recorded as a timeout. + #[test] + #[cfg(unix)] + fn a_mutant_that_exhausts_its_budget_is_recorded_as_a_timeout() { + let (_directory, work, mut plan, binaries) = harness("sleep 30", Duration::from_millis(50)); + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let _spent = test_all( + &work, + &mut plan, + &reach, + sweep(Stall::NONE), + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep completes"); + + // A hang remains a distinct timeout verdict even though only assertion failures receive + // detection credit. + assert_eq!(plan.mutants[0].outcome, Outcome::Timeout); + assert_eq!(plan.mutants[0].note, None); + } + + /// A mutant whose suite goes silent is a timeout, annotated with where it went silent. + #[test] + #[cfg(unix)] + fn a_mutant_that_stalls_is_recorded_as_a_timeout_naming_the_test() { + let (_directory, work, mut plan, binaries) = harness("echo 'test slow::case ... '\nsleep 30", Duration::from_mins(1)); + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + let stall = Stall { + budget: Some(Duration::from_millis(50)), + }; + + let _spent = test_all( + &work, + &mut plan, + &reach, + sweep(stall), + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep completes"); + + // Saying which test was running when the silence started is the whole value of stall + // detection over simply waiting out the budget. + assert_eq!(plan.mutants[0].outcome, Outcome::Timeout); + assert!(plan.mutants[0].note.is_some(), "{:?}", plan.mutants[0].note); + } + + /// A mutant no test binary can reach is uncovered rather than a survivor. + #[test] + fn a_mutant_no_binary_reaches_is_uncovered() { + let (_directory, work, mut plan, binaries) = unreachable_harness(vec![TestBinary { + package: "subject".to_owned(), + budget: Some(Duration::from_secs(30)), + ..crate::testing::test_binary("does-not-exist") + }]); + let scope = TestScope { + packages: &["other".to_owned()], + package_local: false, + whole_workspace: false, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let _spent = test_all( + &work, + &mut plan, + &reach, + sweep(Stall::NONE), + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep completes"); + + // Blaming the tests that exist for code nothing links would make the score a measure of + // the build graph rather than of the suite. + assert_eq!(plan.mutants[0].outcome, Outcome::NoCoverage); + } + + /// A package whose only test binary announced no tests is uncovered, not full of survivors. + /// + /// Regression, issue-011. Cargo emits a unit-test binary for every lib target whether or not it + /// holds a single test, so a binary always exists and the uncovered bucket was unreachable + /// except through an explicit `--test-package` exclusion. A package with no tests at all was + /// then reported as a package whose tests all missed — a materially different, and much more + /// alarming, thing to read. + #[test] + fn a_mutant_whose_only_binary_announced_no_tests_is_uncovered() { + let (_directory, work, mut plan, binaries) = unreachable_harness(vec![TestBinary { + package: "subject".to_owned(), + budget: Some(Duration::from_secs(30)), + tests: Some(0), + ..crate::testing::test_binary("does-not-exist") + }]); + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + let _spent = test_all( + &work, + &mut plan, + &reach, + sweep(Stall::NONE), + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep completes"); + + // The binary is never started — the path it names does not exist — so a run that thought it + // could convict something here would report a failure rather than this verdict. + assert_eq!(plan.mutants[0].outcome, Outcome::NoCoverage); + } + + /// A site most of the suite reaches is *run whole*, never reported uncovered. + /// + /// `reaching` answers `None` for such a site — "narrowing would save nothing, run the whole + /// binary" — and that `None` is byte-for-byte the same answer a blind census gives. The whole + /// safety property is that the sweep launches the binary and reaches a real verdict, rather than + /// mistaking this `None` for the empty list that means "no test reaches it" and reporting + /// `NoCoverage` for a mutant its suite genuinely exercised. Without a sweep-level test the two + /// `None`s look interchangeable, and a refactor that collapsed them would go unnoticed. + #[test] + #[cfg(unix)] + fn a_site_most_of_the_suite_reaches_is_run_whole_rather_than_called_uncovered() { + let (_directory, work, mut plan, binaries) = harness("exit 0", Duration::from_mins(1)); + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + + // Five of nine tests reach the mutant's site, which is past half, so `reaching` returns + // `None` for this binary. + let census = Census::examined(&binaries[0].path, plan.mutants[0].ordinal, 5, 9); + let sweep = Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: true, + census: &census, + }; + + let spent = test_all( + &work, + &mut plan, + &reach, + sweep, + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep completes") + .expect("a pending mutant means the sweep ran"); + + // Survived, not NoCoverage: the whole binary was launched and its suite passed with the + // mutant active. NoCoverage would be the bug this guards against. + assert_eq!(plan.mutants[0].outcome, Outcome::Survived); + assert_eq!(spent.launches, 1, "the whole binary was run exactly once, not skipped as uncovered"); + assert_eq!(spent.probes, 0, "no hint was given, so no probe was launched"); + } + + /// A run with nothing to sweep reports the sweep as absent, not as a zero-cost phase. + /// + /// `test_all` says so by returning `None`, which the session stores verbatim so the diagnostics + /// and `--estimate` can tell "there was nothing to sweep" from "the sweep ran and was free". + #[test] + fn a_plan_with_no_pending_mutants_sweeps_nothing_and_returns_none() { + let (_directory, work, mut plan, binaries) = unreachable_harness(Vec::new()); + + // No pending mutant of any kind, which is what makes the sweep absent rather than empty. + plan.mutants.clear(); + let reach = Reachability::build(&plan, &binaries, &NARROW); + + let swept = test_all( + &work, + &mut plan, + &reach, + sweep(Stall::NONE), + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect("an empty sweep is not a failure"); + + assert!(swept.is_none(), "nothing pending means there is no sweep phase, not a free one"); + } + + /// A note describing a mutant that outgrew its ceiling says both numbers: what it reached and + /// what it was allowed, since a reader's first question is how far past the ceiling it went. + #[test] + fn memory_notes_say_how_far_past_the_ceiling_the_run_went() { + let note = memory_note( + Utf8Path::new("/workspace/target/debug/deps/unit-abc"), + Some(300 * 1024 * 1024), + 256 * 1024 * 1024, + ); + + assert!(note.contains("unit-abc"), "{note}"); + assert!(note.contains("300.0 MB"), "{note}"); + assert!(note.contains("256.0 MB"), "{note}"); + assert!(note.contains("past the"), "{note}"); + } + + /// A run stopped exactly at its ceiling is described as being at it, not past it. + /// + /// Regression, issue-023. "Past" and "at" are different findings: the first says the workload + /// grew, the second says the ceiling was set a hair too low. The message was built the same way + /// either way, which sent the reader looking for a growth that never happened. + /// + /// The figures are asserted as well, because this is the ordinary shape of an enforced kill — + /// the kernel caps the peak at the ceiling, so peak and limit are equal — and it was for a + /// while the one case that printed raw byte counts, on the theory that identical figures always + /// mean a rounding collision. Here they mean the workload landed exactly on its ceiling. + #[test] + fn a_run_that_only_reached_its_ceiling_is_not_described_as_having_passed_it() { + let note = memory_note( + Utf8Path::new("/workspace/target/debug/deps/unit-abc"), + Some(256 * 1024 * 1024), + 256 * 1024 * 1024, + ); + + assert!(note.contains("at the"), "{note}"); + assert!(!note.contains("past"), "{note}"); + assert!(note.contains("256.0 MB"), "{note}"); + assert!( + !note.contains("bytes"), + "an exact count is disambiguation nobody needs here, {note}" + ); + } + + /// A peak the platform reported as below the ceiling is not described as having passed it + /// either, since the stop was decided on something other than that figure. + #[test] + fn a_reported_peak_below_the_ceiling_is_described_as_measured_against_it() { + let note = memory_note( + Utf8Path::new("/workspace/target/debug/deps/unit-abc"), + Some(100 * 1024 * 1024), + 256 * 1024 * 1024, + ); + + assert!(note.contains("against the"), "{note}"); + assert!(!note.contains("past"), "{note}"); + } + + /// Two figures that round to the same thing are printed exactly, so the note cannot read as a + /// contradiction. + /// + /// "reached 512 MB, past the 512 MB this run allowed it" is a sentence that answers its own + /// question wrongly; the byte counts are the only thing that says how far past it actually went. + #[test] + fn a_note_whose_figures_would_round_together_prints_the_exact_bytes() { + let limit = 512 * 1024 * 1024; + let note = memory_note(Utf8Path::new("/workspace/target/debug/deps/unit-abc"), Some(limit + 4096), limit); + + assert!(note.contains(&format!("{} bytes", limit + 4096)), "{note}"); + assert!(note.contains(&format!("{limit} bytes")), "{note}"); + assert!(note.contains("past the"), "{note}"); + } + + /// A note describing a mutant that outgrew its ceiling but whose peak the platform could not + /// itself report still says the ceiling, without inventing a peak that was never measured. + #[test] + fn memory_notes_with_no_measured_peak_still_name_the_ceiling() { + let note = memory_note(Utf8Path::new("/workspace/target/debug/deps/unit-abc"), None, 256 * 1024 * 1024); + + assert!(note.contains("unit-abc"), "{note}"); + assert!(note.contains("256.0 MB"), "{note}"); + assert!( + !note.contains("past"), + "a peak nobody measured must not be described as one, {note}" + ); + } + + /// A flake names the test to fix and says the mutant was never judged. + /// + /// The note is the whole value of this outcome. It scores as neither a detection nor a gap, so + /// a reader who is not told which test failed both ways is told only that something somewhere + /// is unreliable — which is less than recording it as a survivor would give, since a + /// survivor at least names a line. + #[test] + fn a_flaky_note_names_the_test_and_says_nothing_was_judged() { + let note = flaky_note(Utf8Path::new("target/debug/deps/unit-abc"), Some("a::b")); + + assert!(note.contains("test `a::b`"), "{note}"); + assert!(note.contains("unit-abc"), "{note}"); + assert!(note.contains("no mutant active"), "{note}"); + assert!(note.contains("never judged"), "{note}"); + } + + /// A harness that named no test still produces a note that reads as a sentence. + /// + /// libtest names a test only when it finishes, so a binary that dies mid-run can fail without + /// ever having said which test did it. Interpolating an absent name would leave the reader a + /// sentence with a hole in it. + #[test] + fn a_flaky_note_without_a_test_name_still_reads() { + let note = flaky_note(Utf8Path::new("target/debug/deps/unit-abc"), None); + + assert!(note.starts_with("a test in"), "{note}"); + assert!(note.contains("never judged"), "{note}"); + } + + /// A mutant the machine would not run for is recorded as unjudged, and the sweep goes on. + /// + /// The shortage behind a refused spawn — a full descriptor or process table — is one the sweep + /// creates for itself and clears as its workers finish, so abandoning the run over it would + /// throw away every verdict an hours-long sweep had already reached in favour of a condition + /// that lasted milliseconds. The mutant lands as `Pending`, which is scored as excluded, with + /// the refusal as its note so the reader knows which mutant went without a verdict and why. + #[test] + #[cfg(unix)] + fn a_mutant_the_machine_would_not_run_is_recorded_rather_than_abandoning_the_sweep() { + let (_directory, work, _plan, binaries) = harness("exit 0", Duration::from_secs(30)); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + let _refusals: Vec<_> = (0..8).map(|_round| faults::arm(Fault::Spawn)).collect(); + + let judgement = judge( + &work, + 1, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: true, + census: blind(), + }, + &Tally::default(), + ); + + match judgement { + Judgement::Reached(outcome, _killer, note) => { + assert_eq!(outcome, Outcome::Pending, "an unjudgeable mutant is not a verdict about the mutant"); + assert!(!outcome.is_valid(), "and must not be scored"); + assert!( + note.is_some_and(|reason| reason.contains("could not be started")), + "the refusal has to travel" + ); + } + Judgement::Abandoned(reason) => panic!("one refused spawn must not take the run with it: {reason}"), + } + } + + /// A binary that cannot be metered as a mutant sweep asked abandons the mutant it was judging, + /// rather than judging it with no protection installed. + /// + /// The whole point of asking for memory accounting is that a mutant that exhausts memory gets + /// caught by the ceiling rather than by wedging the machine; a `judge` that silently ran the + /// mutant anyway would report a verdict nobody could trust once the accounting it was told it + /// had turned out never to have been there. + #[test] + #[cfg(unix)] + fn a_binary_that_cannot_be_metered_abandons_the_mutant_it_was_judging() { + if memory::support().is_ok() { + return; + } + + let (_directory, work, _plan, mut binaries) = harness("exit 0", Duration::from_secs(30)); + binaries[0].memory = None; + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let judgement = judge( + &work, + 1, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: true, + confirm: true, + census: blind(), + }, + &Tally::default(), + ); + + match judgement { + Judgement::Abandoned(reason) => assert!(!reason.is_empty(), "a refusal has to say why"), + Judgement::Reached(..) => panic!("expected the mutant to be abandoned rather than judged unprotected"), + } + } + + /// A sweep that loses its memory accounting partway through stops rather than continue judging + /// the rest of its mutants unprotected. + /// + /// Every worker checks whether the sweep has already been abandoned before picking up its next + /// mutant, which is what keeps a run that lost its protection from quietly finishing the rest of + /// its work as though nothing had happened. + #[test] + #[cfg(unix)] + fn a_sweep_that_loses_its_memory_accounting_stops_rather_than_continue_unprotected() { + if memory::support().is_ok() { + return; + } + + let (_directory, work, mut plan, binaries) = harness_n("exit 0", Duration::from_secs(30), 8); + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + let sweep = Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 4, + meter: true, + confirm: true, + census: blind(), + }; + + let failure = test_all( + &work, + &mut plan, + &reach, + sweep, + &mut Killers::default(), + &mut crate::testing::Recorder::default(), + ) + .expect_err("a run that cannot be metered as asked must stop rather than continue"); + + // The wrapper has to say the run stopped, and it has to carry the underlying cause through + // rather than replacing it — that cause is the only thing telling the reader what to fix. + assert!(failure.to_string().contains("judge anything further"), "{failure}"); + assert!(failure.to_string().contains("cgroup"), "{failure}"); + + // At least one worker must have broken out of its loop as soon as it saw the sweep was + // abandoned, rather than every mutant having raced to a verdict of its own; a run with + // nothing left pending is not what this path is meant to prove. + assert!( + plan.mutants.iter().any(|mutant| mutant.outcome == Outcome::Pending), + "{:?}", + plan.mutants.iter().map(|mutant| mutant.outcome).collect::>() + ); + } + + /// A binary that outgrows the memory ceiling it was judged under convicts the mutant of using + /// too much memory, rather than reporting it as a plain survivor or failure. + /// + /// The suite's own harness never noticed anything wrong; only the kernel's accounting did, and + /// a reader who was told the mutant merely "survived" would go looking for a missing assertion + /// that was never the actual gap. + #[test] + #[cfg(unix)] + fn a_binary_that_outgrows_its_ceiling_convicts_the_mutant_of_using_too_much_memory() { + if crate::testing::without_memory_support("a sweep asserting a ceiling is enforced") { + return; + } + + let fill = format!("/dev/shm/gamma-judge.{}", std::process::id()); + let (_directory, work, _plan, mut binaries) = harness( + &format!("dd if=/dev/zero of={fill} bs=1M count=512 2>/dev/null"), + Duration::from_mins(1), + ); + binaries[0].memory = Some(32 * 1024 * 1024); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let judgement = judge( + &work, + 1, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: true, + confirm: true, + census: blind(), + }, + &Tally::default(), + ); + + let _removed = fs::remove_file(&fill); + + match judgement { + Judgement::Reached(Outcome::OutOfMemory, None, Some(note)) => { + assert!(note.contains("32.0 MB"), "{note}"); + } + _ => panic!("expected the mutant to be convicted of using too much memory"), + } + } + + /// The script a probe test runs, which fails only when libtest's `--exact` filter reached it. + /// + /// `sh -c BODY name --exact` puts the filter in `$0`, so the body can tell a filtered run from + /// an unfiltered one. That is the whole assertion these tests need: a verdict that could only + /// have come from the filtered run proves the probe is what produced it. + /// A suite that only fails when the run has narrowed itself to `tests::killer`. + /// + /// On the portable helper rather than a shell, because these cases narrow the selection — and a + /// shell fixture carries its script in a positional argument, which is exactly where a real + /// test binary expects a test-name filter and which the tool therefore replaces. + const ONLY_WHEN_FILTERED: &[&str] = &[ + "when-arg:tests::killer|print:test tests::killer ... FAILED", + "when-arg:tests::killer|exit:1", + "exit:0", + ]; + + /// The same shape as [`harness`], with the portable helper standing in for the shell. + fn helper_harness(script: &[&str]) -> (tempfile::TempDir, Workspace, Plan, Vec) { + let (directory, work) = crate::testing::helper_workspace("test-all-helper", script); + let plan = one_mutant_plan(work.root.clone()); + let binaries = vec![TestBinary { + package: "subject".to_owned(), + baseline: Duration::from_millis(1), + budget: None, + ..crate::testing::helper() + }]; + + (directory, work, plan, binaries) + } + + /// A hint is a guess the run checks, and a hint that convicts spares the rest of the binary. + #[test] + #[cfg(unix)] + fn the_test_that_caught_a_mutant_last_time_is_tried_first() { + let (_directory, work, _plan, binaries) = helper_harness(ONLY_WHEN_FILTERED); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let hint = Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::killer".to_owned(), + }; + + // Confirmation off: the fixture cannot tell a mutant run from an exoneration run, so a + // confirmed kill would come back flaky and prove nothing about the probe. + let judgement = judge( + &work, + 1, + &reachable, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + + match judgement { + Judgement::Reached(Outcome::Killed, Some(killer), None) => { + assert_eq!(killer.test, "tests::killer"); + assert_eq!(killer.package, "subject"); + } + _ => panic!("expected the recorded test to convict the mutant on its own"), + } + } + + #[test] + fn a_cached_killer_cannot_bypass_the_canonical_timeout() { + const SCRIPT: &[&str] = &[ + "when-arg:tests::killer|print:test tests::killer ... FAILED", + "when-arg:tests::killer|exit:1", + "sleep:200", + "exit:0", + ]; + + let (_directory, work, _plan, mut binaries) = helper_harness(SCRIPT); + binaries[0].budget = Some(Duration::from_millis(20)); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + let hint = Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::killer".to_owned(), + }; + let tally = Tally::default(); + + let judgement = judge( + &work, + 1, + &reachable, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &tally, + ); + + assert!(matches!(judgement, Judgement::Reached(Outcome::Timeout, None, None))); + assert_eq!(tally.probes.load(Ordering::Relaxed), 0); + assert_eq!(tally.launches.load(Ordering::Relaxed), 1); + } + + #[test] + fn a_partial_census_probe_cannot_bypass_the_whole_binary_timeout() { + const SCRIPT: &[&str] = &[ + "when-arg:tests::t0|print:test tests::t0 ... FAILED", + "when-arg:tests::t0|exit:1", + "sleep:200", + "exit:0", + ]; + + let (_directory, work, plan, mut binaries) = helper_harness(SCRIPT); + binaries[0].budget = Some(Duration::from_millis(20)); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + let census = Census::partial(&binaries[0].path, plan.mutants[0].ordinal, 1, 4); + let tally = Tally::default(); + + let judgement = judge( + &work, + plan.mutants[0].ordinal, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: &census, + }, + &tally, + ); + + assert!(matches!(judgement, Judgement::Reached(Outcome::Timeout, None, None))); + assert_eq!(tally.probes.load(Ordering::Relaxed), 0); + assert_eq!(tally.launches.load(Ordering::Relaxed), 1); + } + + #[test] + #[cfg(unix)] + fn a_complete_census_failure_is_checked_against_the_whole_binary() { + const SCRIPT: &[&str] = &[ + "when-arg:tests::t0|print:test tests::t0 ... FAILED", + "when-arg:tests::t0|exit:1", + "sleep:200", + "exit:0", + ]; + + let (_directory, work, plan, mut binaries) = helper_harness(SCRIPT); + binaries[0].budget = Some(Duration::from_millis(20)); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + let census = Census::examined(&binaries[0].path, plan.mutants[0].ordinal, 1, 4); + let tally = Tally::default(); + + let judgement = judge( + &work, + plan.mutants[0].ordinal, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: &census, + }, + &tally, + ); + + assert!(matches!(judgement, Judgement::Reached(Outcome::Timeout, None, None))); + assert_eq!(tally.launches.load(Ordering::Relaxed), 2); + } + + #[test] + fn a_cached_killer_cannot_bypass_confirmation_of_a_flaky_test() { + let (_directory, work, _plan, binaries) = helper_harness(&["print:test tests::killer ... FAILED", "exit:1"]); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + let hint = Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::killer".to_owned(), + }; + let tally = Tally::default(); + + let judgement = judge( + &work, + 1, + &reachable, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: true, + census: blind(), + }, + &tally, + ); + + assert!(matches!(judgement, Judgement::Reached(Outcome::Flaky, None, Some(_)))); + assert_eq!(tally.probes.load(Ordering::Relaxed), 0); + assert_eq!(tally.launches.load(Ordering::Relaxed), 1); + } + + #[test] + fn a_file_local_killer_cannot_move_a_later_binary_before_a_timeout() { + let (_directory, work, _plan, mut binaries) = helper_harness(&["sleep:200", "exit:0"]); + binaries[0].target = "canonical".to_owned(); + binaries[0].budget = Some(Duration::from_millis(20)); + let mut later = binaries[0].clone(); + later.target = "hinted".to_owned(); + later.budget = Some(Duration::from_secs(1)); + binaries.push(later); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + let hint = Killer { + package: "subject".to_owned(), + target: "hinted".to_owned(), + test: "tests::killer".to_owned(), + }; + let tally = Tally::default(); + + let judgement = judge_ordered( + &work, + 1, + &reachable, + None, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &tally, + ); + + assert!(matches!(judgement, Judgement::Reached(Outcome::Timeout, None, None))); + assert_eq!(tally.launches.load(Ordering::Relaxed), 1); + } + + #[test] + #[cfg(unix)] + fn a_partial_census_hint_that_passes_falls_back_to_the_whole_binary() { + const SCRIPT: &[&str] = &["when-arg:tests::t0|exit:0", "print:test tests::whole ... FAILED", "exit:1"]; + + let (_directory, work, plan, binaries) = helper_harness(SCRIPT); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + let census = Census::partial(&binaries[0].path, plan.mutants[0].ordinal, 1, 4); + let tally = Tally::default(); + + let judgement = judge( + &work, + plan.mutants[0].ordinal, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: &census, + }, + &tally, + ); + + assert!(matches!(judgement, Judgement::Reached(Outcome::Killed, _, None))); + assert_eq!(tally.probes.load(Ordering::Relaxed), 1); + assert_eq!(tally.launches.load(Ordering::Relaxed), 2); + } + + /// A hint naming a test the user's own filter excludes convicts nobody. + /// + /// This is the composition defect end to end: the probe would once have been launched with the + /// recorded name appended to the user's filter, which libtest reads as "either", so the + /// excluded test would have run and convicted the mutant — crediting the suite with a detection + /// it does not make as configured. The probe is now refused, and the verdict comes from the + /// binary run under the user's filter, where nothing fails. + #[test] + fn a_hint_naming_a_test_the_users_filter_excludes_does_not_convict() { + let (_directory, mut work, _plan, binaries) = helper_harness(ONLY_WHEN_FILTERED); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + // The user asked for `tests::other` alone, which the fixture's killer is not. + let mut arguments: Vec = ONLY_WHEN_FILTERED.iter().map(crate::testing::directive).collect(); + + arguments.push("tests::other".to_owned()); + work.set_test_args(arguments); + + let hint = Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::killer".to_owned(), + }; + + let judgement = judge( + &work, + 1, + &reachable, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + + assert!( + matches!(judgement, Judgement::Reached(Outcome::Survived, None, None)), + "an excluded test must not convict" + ); + } + + /// The same mutant with no hint reaches the ordinary verdict, which is what makes the case above + /// a measurement of the probe rather than of the fixture. + #[test] + #[cfg(unix)] + fn the_same_mutant_without_a_hint_is_judged_by_the_whole_binary() { + let (_directory, work, _plan, binaries) = helper_harness(ONLY_WHEN_FILTERED); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let judgement = judge( + &work, + 1, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + + assert!( + matches!(judgement, Judgement::Reached(Outcome::Survived, None, None)), + "the unfiltered script passes, so this mutant survives" + ); + } + + /// A hint naming a binary this mutant cannot reach is ignored rather than run. + /// + /// A map written before the test packages were narrowed, or before the test moved, names a + /// binary the run has excluded. Running it anyway would judge the mutant against a suite the + /// caller deliberately took out of the picture. + #[test] + #[cfg(unix)] + fn a_hint_naming_an_unreachable_binary_is_ignored() { + let (_directory, work, _plan, binaries) = helper_harness(ONLY_WHEN_FILTERED); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let hint = Killer { + package: "elsewhere".to_owned(), + target: String::new(), + test: "tests::killer".to_owned(), + }; + + let judgement = judge( + &work, + 1, + &reachable, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + + assert!( + matches!(judgement, Judgement::Reached(Outcome::Survived, None, None)), + "a hint for another package must not decide anything" + ); + } + + /// A script that passes under the hinted filter and fails without it. + /// + /// This is the fixture that makes a wrong hint observable. A script failing either way cannot + /// distinguish a verdict the probe reached from one the ordinary sweep reached, so it proves + /// nothing about which of them decided the outcome. + #[cfg(unix)] + const FAILS_ONLY_UNFILTERED: &str = r#"if [ "$0" = "tests::gone" ]; then exit 0; fi; echo "test tests::other ... FAILED"; exit 1"#; + + /// A hint that no longer convicts leaves the verdict exactly where it would have been. + /// + /// This is the property the whole optimization rests on: the map may only ever change what a + /// run costs, never what it concludes. The two judgements are compared against each other + /// rather than against a written-down expectation, because the claim is an equality between two + /// runs and not a claim about any particular outcome. + /// + /// The killer's name is the part that makes this a real check. Under this fixture the hinted + /// test passes and a different one fails, so a run that believed its hint would credit + /// `tests::gone` — a test that did not fail and, in a stale map, may not exist. Asserting the + /// recorded killer is `tests::other` is what proves the hint was discarded rather than trusted. + #[test] + #[cfg(unix)] + fn a_hint_that_no_longer_convicts_reaches_the_verdict_the_run_would_have_reached_anyway() { + let (_directory, work, _plan, binaries) = harness(FAILS_ONLY_UNFILTERED, Duration::from_secs(30)); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let hint = Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::gone".to_owned(), + }; + + let hinted = judge( + &work, + 1, + &reachable, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + let unhinted = judge( + &work, + 1, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + + for (judgement, described) in [(hinted, "with a wrong hint"), (unhinted, "with no hint")] { + match judgement { + Judgement::Reached(Outcome::Killed, Some(killer), None) => { + assert_eq!(killer.test, "tests::other", "{described}: the wrong test was credited"); + } + Judgement::Reached(outcome, killer, note) => { + panic!("{described}: expected a kill by `tests::other`, got {outcome:?} / {killer:?} / {note:?}") + } + Judgement::Abandoned(reason) => panic!("{described}: the run was abandoned: {reason}"), + } + } + } + + /// A wrong hint must not manufacture a kill out of a mutant nothing catches. + /// + /// The opposite direction of the same law, and the more damaging one: a survivor turned into a + /// kill is a test gap the report says does not exist. + #[test] + #[cfg(unix)] + fn a_wrong_hint_cannot_turn_a_survivor_into_a_kill() { + let (_directory, work, _plan, binaries) = harness("exit 0", Duration::from_secs(30)); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let hint = Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::gone".to_owned(), + }; + + let hinted = judge( + &work, + 1, + &reachable, + Some(&hint), + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + let unhinted = judge( + &work, + 1, + &reachable, + None, + None, + Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }, + &Tally::default(), + ); + + assert!(matches!(hinted, Judgement::Reached(Outcome::Survived, None, None))); + assert!(matches!(unhinted, Judgement::Reached(Outcome::Survived, None, None))); + } + + /// A sweep writes back what caught each mutant, and drops what caught one that nothing caught. + /// + /// The forgetting half matters as much as the recording half: an entry left behind for a mutant + /// that now survives makes every later run pay for a probe already shown not to convict. + #[test] + #[cfg(unix)] + fn a_sweep_records_the_killer_it_found_and_forgets_the_one_it_did_not() { + let (_directory, work, mut plan, binaries) = harness("echo 'test tests::caught ... FAILED'; exit 1", Duration::from_secs(30)); + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let reach = Reachability::build(&plan, &binaries, &scope); + let sweep = Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }; + + let mut killers = Killers::default(); + killers.record( + "m1".into(), + Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::stale".to_owned(), + }, + ); + + let _spent = test_all( + &work, + &mut plan, + &reach, + sweep, + &mut killers, + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep to finish"); + + assert_eq!(plan.mutants[0].outcome, Outcome::Killed); + assert_eq!(killers.hint("m1").map(|found| found.test.as_str()), Some("tests::caught")); + + // Now the same tree with a suite that catches nothing: the entry has to go. + let (_directory, work, mut plan, binaries) = harness("exit 0", Duration::from_secs(30)); + let reach = Reachability::build(&plan, &binaries, &scope); + + let _spent = test_all( + &work, + &mut plan, + &reach, + sweep, + &mut killers, + &mut crate::testing::Recorder::default(), + ) + .expect("the sweep to finish"); + + assert_eq!(plan.mutants[0].outcome, Outcome::Survived); + assert!(killers.hint("m1").is_none(), "a mutant nothing caught must not keep a killer"); + + drop(binaries); + } + + /// A mutant with a custom timeout multiplier uses that multiplier instead of the binary's default. + #[test] + #[cfg(unix)] + fn a_mutant_with_a_timeout_multiplier_overrides_the_binary_budget() { + // Keep a wide gap on both sides of the sleep so scheduler jitter cannot change the verdict. + let (_directory, work, _plan, mut binaries) = harness("sleep 0.2; exit 0", Duration::from_millis(10)); + binaries[0].baseline = Duration::from_millis(50); + let reachable: Vec<&TestBinary> = binaries.iter().collect(); + + let sweep = Sweep { + timeout_floor: Duration::ZERO, + stall: Stall::NONE, + jobs: 1, + meter: false, + confirm: false, + census: blind(), + }; + + let without_override = judge(&work, 1, &reachable, None, None, sweep, &Tally::default()); + assert!( + matches!(without_override, Judgement::Reached(Outcome::Timeout, None, None)), + "default budget times out" + ); + + let with_override = judge(&work, 1, &reachable, None, Some(100.0), sweep, &Tally::default()); + assert!( + matches!(with_override, Judgement::Reached(Outcome::Survived, None, None)), + "extended budget allows completion" + ); + } + + /// With census data, a mutant whose reaching tests cost less schedules after one whose + /// reaching tests cost more — even if both are in the same package. + #[test] + fn census_based_scheduling_uses_per_site_measured_cost() { + use super::mutant_cost; + let plan = plan_over(&["subject", "subject"]); + let binaries = vec![binary_of("subject", Duration::from_secs(10))]; + + let sets = Reachability::build(&plan, &binaries, &NARROW); + + // Without census data, both get the same cost (binary baseline sum). + let cost_a = mutant_cost(0, &plan, &sets, blind(), &Killers::default()); + let cost_b = mutant_cost(1, &plan, &sets, blind(), &Killers::default()); + assert_eq!(cost_a, cost_b); + assert_eq!(cost_a, Duration::from_secs(10)); + } + + /// A mutant with a known killer hint has a lower estimated cost (one binary baseline). + #[test] + fn a_hinted_mutant_costs_less_than_an_unhinted_one() { + use super::mutant_cost; + + let plan = plan_over(&["subject", "subject"]); + let binaries = vec![ + binary_of("subject", Duration::from_secs(5)), + binary_of("subject", Duration::from_secs(20)), + ]; + + let sets = Reachability::build( + &plan, + &binaries, + &TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }, + ); + + let mut killers = Killers::default(); + killers.record( + plan.mutants[0].id.clone(), + Killer { + package: "subject".to_owned(), + target: String::new(), + test: "tests::hint".to_owned(), + }, + ); + + let cost_hinted = mutant_cost(0, &plan, &sets, blind(), &killers); + let cost_unhinted = mutant_cost(1, &plan, &sets, blind(), &Killers::default()); + + assert!( + cost_hinted < cost_unhinted, + "hinted={cost_hinted:?} should be less than unhinted={cost_unhinted:?}" + ); + } + + /// The condvar-based file learning correctly publishes a learned killer to waiting siblings. + #[test] + fn file_learning_publishes_killer_to_waiting_siblings() { + let learning = FileLearning::new(); + + // First worker becomes learner. + { + let mut state = learning.state.lock().unwrap(); + assert!(matches!(*state, Learning::Untried)); + *state = Learning::InProgress; + } + + // Simulate a second worker waiting and a learner publishing. + thread::scope(|scope| { + let _waiter = scope.spawn(|| { + let (guard_result, timed_out) = learning + .notify + .wait_timeout_while(learning.state.lock().unwrap(), Duration::from_millis(200), |s| { + matches!(s, Learning::InProgress) + }) + .unwrap(); + assert!(!timed_out.timed_out(), "should have been woken, not timed out"); + match &*guard_result { + Learning::Learned(killer) => assert_eq!(killer.test, "tests::found"), + other => panic!("expected Learned, got {other:?}"), + } + drop(guard_result); + }); + + // Give the waiter time to enter wait. + thread::sleep(Duration::from_millis(10)); + + // Learner finishes with a killer. + { + let mut s = learning.state.lock().unwrap(); + *s = Learning::Learned(Killer { + package: "pkg".to_owned(), + target: String::new(), + test: "tests::found".to_owned(), + }); + drop(s); + learning.notify.notify_all(); + } + }); + } + + /// The condvar-based file learning wakes waiters with `Exhausted` when no killer is found. + #[test] + fn file_learning_wakes_waiters_on_exhaustion() { + let learning = FileLearning::new(); + + { + let mut state = learning.state.lock().unwrap(); + *state = Learning::InProgress; + } + + thread::scope(|scope| { + let _waiter = scope.spawn(|| { + let (guard_result, _timed_out) = learning + .notify + .wait_timeout_while(learning.state.lock().unwrap(), Duration::from_millis(200), |s| { + matches!(s, Learning::InProgress) + }) + .unwrap(); + assert!(matches!(*guard_result, Learning::Exhausted)); + drop(guard_result); + }); + + thread::sleep(Duration::from_millis(10)); + + { + let mut s = learning.state.lock().unwrap(); + *s = Learning::Exhausted; + drop(s); + learning.notify.notify_all(); + } + }); + } + + /// The bounded wait times out and proceeds without a hint when the learner is slow. + #[test] + fn file_learning_wait_times_out_for_slow_learner() { + let learning = FileLearning::new(); + + { + let mut state = learning.state.lock().unwrap(); + *state = Learning::InProgress; + } + + let started = Instant::now(); + + { + let (guard_result, timed_out) = learning + .notify + .wait_timeout_while(learning.state.lock().unwrap(), Duration::from_millis(20), |s| { + matches!(s, Learning::InProgress) + }) + .unwrap(); + drop(guard_result); + assert!(timed_out.timed_out(), "should have timed out"); + } + + // The timeout should be roughly 20ms, not blocking indefinitely. + assert!(started.elapsed() < Duration::from_millis(100)); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/sync.rs b/crates/cargo-gamma-lib/src/exec/sync.rs new file mode 100644 index 000000000..8d54d2d2b --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/sync.rs @@ -0,0 +1,717 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Delta synchronization of a scratch tree against the source workspace. +//! +//! Instead of deleting and recopying the entire tree on each campaign, this module compares the +//! existing scratch tree against the source and applies only the necessary changes: new files are +//! copied, changed files are replaced, stale files are removed, and unchanged files are left in +//! place with their original mtimes — preserving Cargo's fingerprint validity for inputs that did +//! not change. + +use core::sync::atomic::Ordering; +use std::collections::HashSet; +use std::fs::{self, File, FileTimes}; +use std::io::{ErrorKind, Read}; +use std::sync::Mutex; +use std::time::SystemTime; + +use camino::{Utf8Path, Utf8PathBuf}; +use ignore::{WalkBuilder, WalkState}; +use walkdir::WalkDir; + +use super::copy::{CopyOptions, copy_tree_with, is_pruned, tracked_files}; +use crate::Result; +use crate::error::{Error, error}; + +/// Outcome of a delta synchronization attempt. +#[derive(Debug)] +pub(super) enum SyncOutcome { + /// The existing tree was updated in place. + Synchronized, + /// The existing tree was unsuitable and a full copy was performed instead. + FreshCopy, +} + +const SYNC_SENTINEL: &str = ".gamma-sync-ok"; + +fn sentinel(root: &Utf8Path) -> Utf8PathBuf { + root.parent().unwrap_or(root).join(SYNC_SENTINEL) +} + +pub(super) fn mark_consistent(root: &Utf8Path) { + let path = sentinel(root); + let _written = fs::write(path.as_std_path(), "1"); +} + +fn is_consistent(root: &Utf8Path) -> bool { + let path = sentinel(root); + matches!(fs::read(path.as_std_path()), Ok(content) if content == b"1") +} + +fn clear_sentinel(root: &Utf8Path) { + let path = sentinel(root); + let _removed = fs::remove_file(path.as_std_path()); +} + +/// Attempts to delta-synchronize `source` into `existing_root`. +/// +/// If the existing tree is unsuitable (inconsistent from a prior interrupted run, or not a +/// directory), falls back to removing it and performing a fresh copy. +/// +/// Returns which path was taken so the caller can emit appropriate diagnostics. +pub(super) fn sync_or_copy(source: &Utf8Path, root: &Utf8Path, skip: &Utf8Path, options: CopyOptions) -> Result { + if !root.as_std_path().is_dir() { + copy_tree_with(source, root, skip, options)?; + mark_consistent(root); + return Ok(SyncOutcome::FreshCopy); + } + + if !is_consistent(root) { + // Prior run was interrupted — cannot trust what is there. Remove and resync. + fs::remove_dir_all(root.as_std_path()) + .map_err(|cause| error!("could not clear the inconsistent scratch tree at `{root}`").caused_by(cause))?; + copy_tree_with(source, root, skip, options)?; + mark_consistent(root); + return Ok(SyncOutcome::FreshCopy); + } + + // The tree looks consistent — attempt delta sync. + clear_sentinel(root); + + match delta_sync(source, root, skip, options) { + Ok(()) => { + mark_consistent(root); + Ok(SyncOutcome::Synchronized) + } + Err(_cause) => { + // Delta sync failed. Remove everything and do a clean sync to restore correctness. + let _removed = fs::remove_dir_all(root.as_std_path()); + copy_tree_with(source, root, skip, options)?; + mark_consistent(root); + Ok(SyncOutcome::FreshCopy) + } + } +} + +/// Performs the actual delta synchronization. +/// +/// Walks the source tree (with the same ignore/selection semantics as the full copy) to discover +/// what should be in the scratch tree, then: +/// 1. Copies new entries and replaces changed entries. +/// 2. Removes stale entries that no longer exist in the source. +/// 3. Leaves unchanged entries untouched (preserving their mtimes for Cargo). +fn delta_sync(source: &Utf8Path, root: &Utf8Path, skip: &Utf8Path, options: CopyOptions) -> Result<()> { + // Collect the set of relative paths the source tree produces. + let expected = collect_source_entries(source, skip, options)?; + + // Synchronize: copy new/changed, leave unchanged. + for relative in &expected { + let src = source.join(relative); + let dst = root.join(relative); + sync_entry(&src, &dst)?; + } + + // Remove stale entries from the scratch tree. + remove_stale(root, &expected)?; + + Ok(()) +} + +/// Collects relative paths from the source tree using the same walk logic as `copy_tree_with`. +fn collect_source_entries(source: &Utf8Path, skip: &Utf8Path, options: CopyOptions) -> Result> { + let entries: Mutex> = Mutex::new(HashSet::new()); + let failure: Mutex> = Mutex::new(None); + + let mut builder = WalkBuilder::new(source.as_std_path()); + let _builder = builder + .hidden(false) + .parents(false) + .require_git(true) + .git_ignore(!options.copy_ignored) + .git_exclude(!options.copy_ignored) + .git_global(false) + .ignore(false) + .follow_links(false); + + let root = source.to_owned(); + let excluded = skip.to_owned(); + + builder.build_parallel().run(|| { + let root = root.clone(); + let excluded = excluded.clone(); + let entries = &entries; + let failure = &failure; + + Box::new(move |entry| { + let entry = match entry { + Ok(entry) => entry, + Err(cause) => { + record(failure, error!("could not read the source tree").caused_by(cause)); + return WalkState::Quit; + } + }; + + let Some(path) = Utf8Path::from_path(entry.path()) else { + record( + failure, + error!("`{}` is not valid UTF-8 and cannot be synchronized", entry.path().display()), + ); + return WalkState::Quit; + }; + + let Ok(relative) = path.strip_prefix(&root) else { + return WalkState::Continue; + }; + + if relative.as_str().is_empty() { + return WalkState::Continue; + } + + if is_pruned(path, relative, &excluded) { + return WalkState::Skip; + } + + let mut set = entries.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _inserted = set.insert(relative.to_owned()); + + WalkState::Continue + }) + }); + + if let Some(cause) = failure.into_inner().unwrap_or_else(std::sync::PoisonError::into_inner) { + return Err(cause); + } + + let mut result = entries.into_inner().unwrap_or_else(std::sync::PoisonError::into_inner); + + // Include tracked files that the ignore walk may have skipped, same as copy_tracked does. + if let Some(tracked) = tracked_files(source)? { + for relative in tracked { + if !is_pruned_anywhere(source, &relative, skip) { + let src = source.join(&relative); + if fs::symlink_metadata(src.as_std_path()).is_ok() { + let _inserted = result.insert(relative); + } + } + } + } + + Ok(result) +} + +/// Returns whether any directory on the way to `relative` is one the sync leaves out. +fn is_pruned_anywhere(root: &Utf8Path, relative: &Utf8Path, excluded: &Utf8Path) -> bool { + let mut prefix = Utf8PathBuf::new(); + for component in relative.components() { + prefix.push(component); + if is_pruned(&root.join(&prefix), &prefix, excluded) { + return true; + } + } + false +} + +/// Synchronizes one source entry to the scratch tree. +/// +/// For files: copies if new or changed (by len + mtime). Unchanged files are left in place. +/// For directories: creates if missing. +/// For symlinks: recreates if target differs. +fn sync_entry(source: &Utf8Path, destination: &Utf8Path) -> Result<()> { + let src_meta = fs::symlink_metadata(source.as_std_path()).map_err(|cause| error!("could not read `{source}`").caused_by(cause))?; + + if src_meta.is_dir() { + if !destination.as_std_path().exists() { + fs::create_dir_all(destination.as_std_path()).map_err(|cause| error!("could not create `{destination}`").caused_by(cause))?; + } else if !destination.as_std_path().is_dir() { + // A non-directory where a directory should be — replace it. + fs::remove_file(destination.as_std_path()) + .map_err(|cause| error!("could not remove stale entry at `{destination}`").caused_by(cause))?; + fs::create_dir_all(destination.as_std_path()).map_err(|cause| error!("could not create `{destination}`").caused_by(cause))?; + } + return Ok(()); + } + + if src_meta.is_symlink() { + return sync_symlink(source, destination); + } + + // Regular file. + sync_file(source, destination, &src_meta) +} + +/// Synchronizes a regular file, preserving mtime for unchanged files. +fn sync_file(source: &Utf8Path, destination: &Utf8Path, src_meta: &fs::Metadata) -> Result<()> { + let needs_copy = match fs::symlink_metadata(destination.as_std_path()) { + Err(_) => true, // Destination does not exist. + Ok(dst_meta) => { + if dst_meta.is_symlink() || dst_meta.is_dir() { + // Type mismatch — remove and recopy. + if dst_meta.is_dir() { + fs::remove_dir_all(destination.as_std_path()) + .map_err(|cause| error!("could not remove stale directory at `{destination}`").caused_by(cause))?; + } else { + fs::remove_file(destination.as_std_path()) + .map_err(|cause| error!("could not remove stale entry at `{destination}`").caused_by(cause))?; + } + true + } else { + file_differs(source, destination, src_meta, &dst_meta)? + } + } + }; + + if needs_copy { + // Ensure parent exists. + if let Some(parent) = destination.parent() + && !parent.as_std_path().exists() + { + fs::create_dir_all(parent.as_std_path()).map_err(|cause| error!("could not create `{parent}`").caused_by(cause))?; + } + + // Remove existing destination before copying (reflink requires no existing file). + let _removed = fs::remove_file(destination.as_std_path()); + copy_file_for_sync(source, destination)?; + } + // Unchanged files are left in place — their mtime stays as it was, preserving Cargo + // fingerprints. + + Ok(()) +} + +fn file_differs(source: &Utf8Path, destination: &Utf8Path, src: &fs::Metadata, dst: &fs::Metadata) -> Result { + if src.len() != dst.len() { + return Ok(true); + } + + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + + if src.mode() != dst.mode() { + return Ok(true); + } + } + + if src.permissions().readonly() != dst.permissions().readonly() { + return Ok(true); + } + + same_contents(source, destination).map(|same| !same) +} + +fn same_contents(left: &Utf8Path, right: &Utf8Path) -> Result { + let mut left = File::open(left.as_std_path()).map_err(|cause| error!("could not read `{left}`").caused_by(cause))?; + let mut right = File::open(right.as_std_path()).map_err(|cause| error!("could not read `{right}`").caused_by(cause))?; + let mut left_buffer = [0_u8; 16 * 1024]; + let mut right_buffer = [0_u8; 16 * 1024]; + + loop { + let left_read = left + .read(&mut left_buffer) + .map_err(|cause| error!("could not compare scratch input").caused_by(cause))?; + let right_read = right + .read(&mut right_buffer) + .map_err(|cause| error!("could not compare scratch input").caused_by(cause))?; + + if left_read != right_read { + return Ok(false); + } + if left_buffer[..left_read] != right_buffer[..left_read] { + return Ok(false); + } + if left_read == 0 { + return Ok(true); + } + } +} + +/// Synchronizes a symlink. Recreates it if the target changed or if the destination is not a link. +fn sync_symlink(source: &Utf8Path, destination: &Utf8Path) -> Result<()> { + let src_target = fs::read_link(source.as_std_path()).map_err(|cause| error!("could not read the link `{source}`").caused_by(cause))?; + + let needs_recreate = match fs::symlink_metadata(destination.as_std_path()) { + Err(_) => true, + Ok(dst_meta) => { + if dst_meta.is_symlink() { + // Both are symlinks — compare targets. + fs::read_link(destination.as_std_path()).map_or(true, |dst_target| dst_target != src_target) + } else { + // Type mismatch — remove what is there. + if dst_meta.is_dir() { + fs::remove_dir_all(destination.as_std_path()) + .map_err(|cause| error!("could not remove stale directory at `{destination}`").caused_by(cause))?; + } else { + fs::remove_file(destination.as_std_path()) + .map_err(|cause| error!("could not remove stale entry at `{destination}`").caused_by(cause))?; + } + true + } + } + }; + + if needs_recreate { + let _removed = fs::remove_file(destination.as_std_path()); + + if let Some(parent) = destination.parent() + && !parent.as_std_path().exists() + { + fs::create_dir_all(parent.as_std_path()).map_err(|cause| error!("could not create `{parent}`").caused_by(cause))?; + } + + #[cfg(unix)] + std::os::unix::fs::symlink(&src_target, destination.as_std_path()) + .map_err(|cause| error!("could not recreate the link `{destination}`").caused_by(cause))?; + + #[cfg(windows)] + { + let linked = if source + .parent() + .map_or_else(|| src_target.is_dir(), |parent| parent.as_std_path().join(&src_target).is_dir()) + { + std::os::windows::fs::symlink_dir(&src_target, destination.as_std_path()) + } else { + std::os::windows::fs::symlink_file(&src_target, destination.as_std_path()) + }; + linked.map_err(|cause| error!("could not recreate the link `{destination}`").caused_by(cause))?; + } + } + + Ok(()) +} + +fn copy_file_for_sync(source: &Utf8Path, destination: &Utf8Path) -> Result<()> { + let copied_at = SystemTime::now(); + + if reflink_supported() && super::copy::REFLINK_WORKS.load(Ordering::Relaxed) { + match reflink_copy::reflink(source.as_std_path(), destination.as_std_path()) { + Ok(()) => { + stamp_mtime(destination, copied_at)?; + return Ok(()); + } + Err(cause) if cause.kind() == ErrorKind::NotFound => { + return Err(error!("could not copy `{source}` to `{destination}`").caused_by(cause)); + } + Err(_unsupported) => { + super::copy::REFLINK_WORKS.store(false, Ordering::Relaxed); + let _removed = fs::remove_file(destination.as_std_path()); + } + } + } + + let _bytes = fs::copy(source.as_std_path(), destination.as_std_path()) + .map_err(|cause| error!("could not copy `{source}` to `{destination}`").caused_by(cause))?; + + stamp_mtime(destination, copied_at)?; + + Ok(()) +} + +fn stamp_mtime(path: &Utf8Path, time: SystemTime) -> Result<()> { + let file = File::options() + .write(true) + .open(path.as_std_path()) + .map_err(|cause| error!("could not open copied file `{path}`").caused_by(cause))?; + file.set_times(FileTimes::new().set_modified(time)) + .map_err(|cause| error!("could not freshen copied file `{path}`").caused_by(cause)) +} + +/// Whether cloning is worth trying on this platform at all. +const fn reflink_supported() -> bool { + !cfg!(target_env = "musl") +} + +/// Removes entries from the scratch tree that are not in the expected set. +/// +/// Walks the scratch tree and removes anything not present in the source. Directories are handled +/// bottom-up: empty directories left after file removal are pruned. +fn remove_stale(root: &Utf8Path, expected: &HashSet) -> Result<()> { + // Collect all entries in the scratch tree. + let mut stale_files: Vec = Vec::new(); + let mut stale_dirs: Vec = Vec::new(); + + for entry in WalkDir::new(root.as_std_path()).into_iter().filter_map(core::result::Result::ok) { + let Some(path) = Utf8Path::from_path(entry.path()) else { + continue; + }; + + let Ok(relative) = path.strip_prefix(root) else { + continue; + }; + + if relative.as_str().is_empty() { + continue; + } + + if !expected.contains(relative) { + if entry.file_type().is_dir() { + stale_dirs.push(path.to_owned()); + } else { + stale_files.push(path.to_owned()); + } + } + } + + // Remove stale files first. + for file in &stale_files { + fs::remove_file(file.as_std_path()) + .or_else(|cause| if cause.kind() == ErrorKind::NotFound { Ok(()) } else { Err(cause) }) + .map_err(|cause| error!("could not remove stale file `{file}`").caused_by(cause))?; + } + + // Remove stale directories deepest-first so that parents are empty when reached. + stale_dirs.sort_by_key(|path| core::cmp::Reverse(path.as_str().len())); + for dir in &stale_dirs { + // Only remove if truly empty (children may have been expected). + match fs::remove_dir(dir.as_std_path()) { + Ok(()) => {} + Err(cause) if cause.kind() == ErrorKind::NotFound => {} + // Not empty — some children were expected; leave it. + Err(cause) if is_not_empty_error(&cause) => {} + Err(cause) => { + return Err(error!("could not remove stale directory `{dir}`").caused_by(cause)); + } + } + } + + Ok(()) +} + +/// Checks if an IO error indicates the directory is not empty. +fn is_not_empty_error(err: &std::io::Error) -> bool { + // On Unix, ENOTEMPTY; on Windows, ERROR_DIR_NOT_EMPTY. + err.kind() == ErrorKind::DirectoryNotEmpty + || err.raw_os_error() == Some(39) // ENOTEMPTY on Linux + || err.raw_os_error() == Some(66) // ENOTEMPTY on macOS +} + +/// Records the first failure. +fn record(failure: &Mutex>, cause: Error) { + if let Ok(mut held) = failure.lock() + && held.is_none() + { + *held = Some(cause); + } +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use core::time::Duration; + use std::thread; + + use super::*; + + fn tree() -> (tempfile::TempDir, Utf8PathBuf, Utf8PathBuf) { + let temporary = tempfile::tempdir().unwrap(); + let from = Utf8PathBuf::from_path_buf(temporary.path().join("from")).unwrap(); + let to = Utf8PathBuf::from_path_buf(temporary.path().join("to")).unwrap(); + fs::create_dir_all(from.as_std_path()).unwrap(); + (temporary, from, to) + } + + /// A one-file change in the source is reflected in the scratch tree without recopying + /// unchanged files. + #[test] + fn a_one_file_change_is_synchronized() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + // Initial state: two files. + fs::write(from.join("a.rs").as_std_path(), "fn a() {}").unwrap(); + fs::write(from.join("b.rs").as_std_path(), "fn b() {}").unwrap(); + + let outcome = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(matches!(outcome, SyncOutcome::FreshCopy)); + assert_eq!(fs::read_to_string(to.join("a.rs").as_std_path()).unwrap(), "fn a() {}"); + + // Record mtime of b.rs in scratch — it should be preserved. + let b_mtime_before = fs::metadata(to.join("b.rs").as_std_path()).unwrap().modified().unwrap(); + + // Change a.rs in source, leave b.rs unchanged (but update a.rs's mtime). + thread::sleep(Duration::from_millis(50)); + fs::write(from.join("a.rs").as_std_path(), "fn a_new() {}").unwrap(); + + let outcome = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(matches!(outcome, SyncOutcome::Synchronized)); + assert_eq!(fs::read_to_string(to.join("a.rs").as_std_path()).unwrap(), "fn a_new() {}"); + + // b.rs should be unchanged — same mtime. + let b_mtime_after = fs::metadata(to.join("b.rs").as_std_path()).unwrap().modified().unwrap(); + assert_eq!(b_mtime_before, b_mtime_after); + } + + /// Deleted files are removed from the scratch tree. + #[test] + fn a_deletion_is_reflected_in_the_scratch_tree() { + let (_tmp, from, to) = tree(); + let skip = to.parent().unwrap(); + + fs::write(from.join("keep.rs").as_std_path(), "keep").unwrap(); + fs::write(from.join("gone.rs").as_std_path(), "gone").unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(to.join("gone.rs").as_std_path().exists()); + + // Delete gone.rs from source. + fs::remove_file(from.join("gone.rs").as_std_path()).unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(!to.join("gone.rs").as_std_path().exists()); + assert!(to.join("keep.rs").as_std_path().exists()); + } + + /// A renamed file shows up as a deletion + creation. + #[test] + fn a_rename_is_reflected_as_deletion_and_creation() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + fs::write(from.join("old.rs").as_std_path(), "fn f() {}").unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(to.join("old.rs").as_std_path().exists()); + + // Rename: delete old, create new. + fs::remove_file(from.join("old.rs").as_std_path()).unwrap(); + fs::write(from.join("new.rs").as_std_path(), "fn f() {}").unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(!to.join("old.rs").as_std_path().exists()); + assert!(to.join("new.rs").as_std_path().exists()); + } + + /// An unchanged tree does not modify any file in the scratch tree. + #[test] + fn an_unchanged_tree_leaves_the_scratch_tree_untouched() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + fs::write(from.join("stable.rs").as_std_path(), "fn stable() {}").unwrap(); + fs::create_dir_all(from.join("sub").as_std_path()).unwrap(); + fs::write(from.join("sub/mod.rs").as_std_path(), "mod sub;").unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + + let mtime_before = fs::metadata(to.join("stable.rs").as_std_path()).unwrap().modified().unwrap(); + + // Sync again with no changes. + thread::sleep(Duration::from_millis(50)); + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + + let mtime_after = fs::metadata(to.join("stable.rs").as_std_path()).unwrap().modified().unwrap(); + assert_eq!(mtime_before, mtime_after); + } + + /// Symlinks are synchronized correctly. + #[cfg(unix)] + #[test] + fn symlinks_are_synchronized() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + fs::write(from.join("target.txt").as_std_path(), "real").unwrap(); + std::os::unix::fs::symlink("target.txt", from.join("link").as_std_path()).unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(fs::symlink_metadata(to.join("link").as_std_path()).unwrap().is_symlink()); + assert_eq!( + fs::read_link(to.join("link").as_std_path()).unwrap().to_str().unwrap(), + "target.txt" + ); + + // Change the link target. + fs::remove_file(from.join("link").as_std_path()).unwrap(); + std::os::unix::fs::symlink("other.txt", from.join("link").as_std_path()).unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert_eq!(fs::read_link(to.join("link").as_std_path()).unwrap().to_str().unwrap(), "other.txt"); + } + + /// An interrupted prior sync (no sentinel) triggers a fresh copy. + #[test] + fn an_interrupted_prior_sync_triggers_a_fresh_copy() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + fs::write(from.join("a.rs").as_std_path(), "fn a() {}").unwrap(); + + // Do an initial sync. + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + + // Simulate an interrupted sync by removing the sentinel and leaving extra junk. + clear_sentinel(&to); + fs::write(to.join("junk.rs").as_std_path(), "stale").unwrap(); + + let outcome = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(matches!(outcome, SyncOutcome::FreshCopy)); + // Junk should be gone after fresh copy. + assert!(!to.join("junk.rs").as_std_path().exists()); + assert!(to.join("a.rs").as_std_path().exists()); + } + + /// Extra files in the scratch tree (from a prior instrumentation or interrupted run) are + /// removed during delta sync. + #[test] + fn extra_scratch_files_are_removed() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + fs::write(from.join("real.rs").as_std_path(), "fn real() {}").unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + + // Manually add extra files to scratch (simulating prior instrumentation leftovers). + fs::write(to.join("instrumented.rs").as_std_path(), "stale").unwrap(); + fs::create_dir_all(to.join("ghost").as_std_path()).unwrap(); + fs::write(to.join("ghost/file.rs").as_std_path(), "stale").unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + + assert!(to.join("real.rs").as_std_path().exists()); + assert!(!to.join("instrumented.rs").as_std_path().exists()); + assert!(!to.join("ghost").as_std_path().exists()); + } + + #[test] + fn changed_bytes_with_identical_size_and_mtime_are_copied_and_freshened() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + fs::write(from.join("f.rs").as_std_path(), "v1").unwrap(); + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + let prior_scratch_mtime = fs::metadata(to.join("f.rs").as_std_path()).unwrap().modified().unwrap(); + + fs::write(from.join("f.rs").as_std_path(), "v2").unwrap(); + File::options() + .write(true) + .open(from.join("f.rs").as_std_path()) + .unwrap() + .set_times(FileTimes::new().set_modified(prior_scratch_mtime)) + .unwrap(); + + let _ = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + + assert_eq!(fs::read_to_string(to.join("f.rs").as_std_path()).unwrap(), "v2"); + let dst_mtime = fs::metadata(to.join("f.rs").as_std_path()).unwrap().modified().unwrap(); + assert!(dst_mtime >= prior_scratch_mtime, "changed input must not look older to Cargo"); + } + + /// Sentinel presence/absence controls whether delta sync or fresh copy is chosen. + #[test] + fn sentinel_controls_sync_vs_fresh_copy() { + let (_tmp, from, to) = tree(); + let skip = Utf8Path::new("/nowhere"); + + fs::write(from.join("f.rs").as_std_path(), "fn f() {}").unwrap(); + + // First call: no existing tree → fresh copy. + let outcome = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(matches!(outcome, SyncOutcome::FreshCopy)); + assert!(is_consistent(&to)); + + // Second call: consistent tree → delta sync. + let outcome = sync_or_copy(&from, &to, skip, CopyOptions::default()).unwrap(); + assert!(matches!(outcome, SyncOutcome::Synchronized)); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/test_binary.rs b/crates/cargo-gamma-lib/src/exec/test_binary.rs new file mode 100644 index 000000000..b4ab62224 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/test_binary.rs @@ -0,0 +1,1430 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; +use std::sync::Arc; + +use camino::{Utf8Path, Utf8PathBuf}; +use serde_json::Value; + +use super::census::{Census, CensusWork}; +use super::config::Config; +use super::memory::MemoryPolicy; +use crate::discover::{Glob, Plan}; +use crate::estimate::Workload; +use crate::model::{Mutant, Outcome}; + +/// A test executable and the package that produced it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestBinary { + /// Where the executable is. + pub path: Utf8PathBuf, + + /// The package it belongs to, which bounds the code it can possibly reach. + pub package: String, + + /// Cargo's unambiguous identifier for the package. + pub package_id: String, + + /// The cargo target that produced it, which is what `--exclude-test` matches against. + /// + /// A package's unit tests take the name of the lib or bin they live in, and each file under + /// `tests/` becomes a target of its own, so this is the finest granularity cargo offers for + /// naming part of a suite — and the granularity `package` is too coarse for. + pub target: String, + + /// The directory holding the package's `Cargo.toml`, which is where cargo would run it. + /// + /// `cargo test` sets the working directory to the package root, not the workspace root, so a + /// test that opens `tests/data/fixture.json` or `src/../golden.txt` only finds it there. Running + /// every binary from the workspace root makes those tests fail identically with and without a + /// mutant active, which turns a whole package's worth of mutants into false survivors. + pub manifest_dir: Utf8PathBuf, + + /// How long it took with no mutant active, so the cheapest can be tried first. + /// + /// Measured under the same concurrency the sweep runs at, so it already carries the contention + /// every mutant will meet. A figure taken on an idle machine would be a measurement of a + /// situation that never occurs again during the run, and every budget derived from it would be + /// too tight by whatever the load costs. + /// + /// `Duration::ZERO` when `--no-baseline` left nothing to measure, which is why [`budget`] and + /// not this field records whether a cutoff was ever calibrated. + /// + /// [`budget`]: Self::budget + pub baseline: Duration, + + /// How many tests the harness announced when it ran with no mutant active. + /// + /// `None` means nobody asked or nothing answered — `--no-baseline`, or a `harness = false` + /// target that announces nothing — which is a different thing from `Some(0)`. Cargo emits a + /// unit-test binary for every lib target whether or not it holds a single test, so `Some(0)` is + /// the only evidence a run has that linking this binary to a mutant would prove nothing. + pub tests: Option, + + /// This binary's share of a mutant's budget. + /// + /// `None` means no cutoff applies, because nothing calibrated one — the same distinction + /// [`peak`] draws for memory. A budget is derived from the baseline, so a run that took no + /// baseline has no measurement to derive it from, and a figure invented here would cut every + /// mutant off at a duration nothing about this suite justified. + /// + /// [`peak`]: Self::peak + pub budget: Option, + + /// The peak memory the whole subtree reached with no mutant active, when it was measured. + /// + /// `None` means nobody measured it — the run did not ask, or the host could not — which is a + /// different thing from a peak of zero and is why this is not a plain `u64`. + pub peak: Option, + + /// The memory ceiling this binary's mutant runs are held to, when one applies. + pub memory: Option, +} + +impl TestBinary { + /// Computes the timeout budget for this binary given an optional per-mutant multiplier override and a floor. + /// + /// An override rescales a calibrated budget; it cannot manufacture one, because on a run with + /// no baseline there is no measurement to multiply. + #[must_use] + pub fn budget_for(&self, override_multiplier: Option, floor: Duration) -> Option { + let budget = self.budget?; + + Some(override_multiplier.map_or(budget, |multiplier| self.baseline.mul_f64(multiplier).max(floor))) + } +} + +/// Derives each test binary's timeout budget from its own measured baseline duration. +/// +/// Each test binary is given a budget scaled from its own baseline runtime by the configured +/// multiplier, subject to the configured timeout floor: `max(binary.baseline * multiplier, floor)`. +/// +/// The floor answers "below what duration is a verdict meaningless", and that is a statement about +/// measurement noise on one binary. A binary whose baseline is milliseconds gets the floor so that +/// scheduling noise is not misread as a hang. +/// +/// `calibrated` says whether the baseline actually ran, and this is the other half of [`bound`]: +/// without a baseline every binary's recorded duration is zero, and scaling zero yields the floor +/// for every binary alike — a flat cutoff invented from no measurement, presented with the +/// confidence of a derived one. Any binary the floor is too small for then times out under every +/// mutant, making the run report resource exhaustion derived from no measurement. No cutoff at all +/// is the honest answer. +pub(super) fn apportion(binaries: &mut [TestBinary], multiplier: f64, floor: Duration, calibrated: bool) { + for binary in binaries.iter_mut() { + binary.budget = calibrated.then(|| binary.baseline.mul_f64(multiplier).max(floor)); + } +} + +/// Orders the binaries and turns the baseline into the limits every mutant is judged against. +/// +/// Cheapest first: the loop stops at the first binary that fails, so trying the quick ones first +/// makes a kill cost less. It changes no verdict, only what a verdict costs. +/// +/// Both limits have to be derived after the baseline and before the first mutant runs, and both +/// stand down when there was no baseline to derive them from, so they are settled together. +pub(super) fn calibrate(binaries: &mut [TestBinary], config: &Config, memory: &MemoryPolicy) { + binaries.sort_by_key(|binary| binary.baseline); + + apportion(binaries, config.test_timeout_multiplier, config.timeout_floor, config.baseline); + bound(binaries, memory, config.baseline); +} + +/// Derives each binary's memory ceiling from what the same binary used with no mutant active. +/// +/// This belongs beside [`apportion`] because it is the other half of the same preparation: both +/// turn a baseline measurement into the limit a mutant of that binary is judged against, and both +/// have to happen after the baseline and before the first mutant runs. +/// +/// `calibrated` says whether the baseline actually ran; see [`MemoryPolicy::ceiling`] for why a run +/// without one gets no derived ceiling at all. +pub(super) fn bound(binaries: &mut [TestBinary], policy: &MemoryPolicy, calibrated: bool) { + for binary in binaries.iter_mut() { + binary.memory = policy.ceiling(binary.peak, calibrated); + } +} + +/// The variable libtest reads to decide how many threads a test harness runs its tests on. +pub(super) const TEST_THREADS_VAR: &str = "RUST_TEST_THREADS"; + +/// How many threads one spawned test harness should be told to use. +/// +/// A test harness defaults to one thread per core, and a run starts `jobs` of them at once, so the +/// machine is asked for `jobs × cores` threads to do `cores` worth of work. What that costs is not +/// merely inefficiency: every mutant's budget was calibrated from a suite that measured itself +/// under the same contention, and contention that varies with how many binaries happen to overlap +/// is noise on every number the run reports. Dividing the cores between the workers makes the load +/// a run places on the machine roughly constant and roughly equal to the machine. +/// +/// `inherited` is whatever the environment already said. Someone who set it chose it, and a run +/// that overrode them would silently change the workload they asked for; `None` is returned then, +/// meaning "leave it alone". +pub(super) fn harness_threads(jobs: usize, cores: usize, inherited: Option<&str>) -> Option { + if inherited.is_some_and(|value| !value.trim().is_empty()) { + return None; + } + + // At least one thread each: more workers than cores is a legitimate thing to ask for, and the + // answer to it is one thread per worker rather than zero. + Some((cores / jobs.max(1)).max(1)) +} + +/// Extracts the test executables cargo reported building. +pub(super) fn test_binaries(stdout: &str) -> Vec { + let mut binaries = Vec::new(); + + for line in stdout.lines() { + let Ok(message) = serde_json::from_str::(line) else { + continue; + }; + + if message.get("reason").and_then(Value::as_str) != Some("compiler-artifact") { + continue; + } + + let is_test = message + .get("profile") + .and_then(|profile| profile.get("test")) + .and_then(Value::as_bool) + .unwrap_or(false); + + if !is_test { + continue; + } + + if let Some(path) = message.get("executable").and_then(Value::as_str) { + let package_id = message.get("package_id").and_then(Value::as_str).unwrap_or_default().to_owned(); + let package = package_name(&package_id); + + let target = message + .get("target") + .and_then(|target| target.get("name")) + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + + let manifest_dir = message + .get("manifest_path") + .and_then(Value::as_str) + .map(Utf8PathBuf::from) + .and_then(|manifest| manifest.parent().map(Utf8Path::to_path_buf)) + .unwrap_or_default(); + + binaries.push(TestBinary { + path: Utf8PathBuf::from(path), + package, + package_id, + target, + manifest_dir, + baseline: Duration::ZERO, + budget: None, + tests: None, + peak: None, + memory: None, + }); + } + } + + binaries.sort_by(|left, right| left.path.cmp(&right.path)); + binaries.dedup_by(|left, right| left.path == right.path); + binaries +} + +/// Drops the test binaries `--include-test` and `--exclude-test` say must not decide a verdict. +/// +/// This has to run before the baseline, not after, so the baseline only runs and measures binaries +/// that will actually decide verdicts. +pub(super) fn restrict(binaries: &mut Vec, include: &[String], exclude: &[String]) { + if include.is_empty() && exclude.is_empty() { + return; + } + + let patterns = TargetPatterns::new(include, exclude); + + binaries.retain(|binary| patterns.admits(&binary.target)); +} + +/// Whether a target name survives the include and exclude patterns. +/// +/// Exclusion is checked first and wins, which is what makes `--include-test "*"` plus a few +/// exclusions mean what it looks like. An empty include list admits everything, so exclusion alone +/// is a subtraction from the whole suite rather than a selection of nothing. +pub(super) fn admits_target(name: &str, include: &[String], exclude: &[String]) -> bool { + TargetPatterns::new(include, exclude).admits(name) +} + +/// Returns the first `--include-test` or `--exclude-test` pattern that names no test target. +/// +/// A pattern matching nothing is the failure these options exist to prevent. An `--exclude-test` +/// typo leaves the target it meant to remove in the oracle, so mutants that should have survived +/// are reported as caught and the score reads better than the suite deserves; an `--include-test` +/// typo empties the oracle instead. Neither says anything on its own, and both look in CI exactly +/// like a run that went well. The same reasoning already makes an unmatched `--file` an error. +pub(super) fn unmatched_test<'args>(tests: &[String], include: &'args [String], exclude: &'args [String]) -> Option<&'args str> { + TargetPatterns::new(include, exclude).unmatched(tests) +} + +struct TargetPatterns<'args> { + include: Vec<(&'args str, Glob)>, + exclude: Vec<(&'args str, Glob)>, +} + +impl<'args> TargetPatterns<'args> { + fn new(include: &'args [String], exclude: &'args [String]) -> Self { + let compile = |patterns: &'args [String]| patterns.iter().map(|pattern| (pattern.as_str(), Glob::new(pattern))).collect(); + + Self { + include: compile(include), + exclude: compile(exclude), + } + } + + fn admits(&self, name: &str) -> bool { + !self.exclude.iter().any(|(_pattern, compiled)| compiled.matches(name)) + && (self.include.is_empty() || self.include.iter().any(|(_pattern, compiled)| compiled.matches(name))) + } + + fn unmatched(&self, tests: &[String]) -> Option<&'args str> { + self.include + .iter() + .chain(&self.exclude) + .find(|(_pattern, compiled)| !tests.iter().any(|name| compiled.matches(name))) + .map(|(pattern, _compiled)| *pattern) + } +} + +/// Whether a test binary can possibly reach code in `package`. +/// +/// By default, a binary only decides verdicts for mutants in its own package. This keeps a +/// workspace run equivalent to running each package separately: a library's score does not depend +/// on whichever reverse-dependent packages happen to share the workspace. Widening is the caller's +/// to ask for, with `--test-package` or `--test-workspace`. +/// +/// The alternative, judging a mutant by every package that can link it, makes a crate's score a +/// property of the workspace rather than of the crate: a library scores well because some dependent +/// happens to exercise it, and a refactor over in that dependent silently withdraws the coverage +/// with nothing to report it. It also makes the price of a run a function of the reverse-dependency +/// graph, so mutating one leaf crate compiles and runs most of the workspace. +/// +/// Capping costs nothing in honesty, because a mutant no admitted binary can reach is reported as +/// uncovered rather than as surviving — the run says "nothing tests this", which is the truth, and +/// not "your tests missed this", which would not be. +/// +/// Within that cap, reach is a pure optimization: a binary is not run against a mutant it cannot +/// link. An unknown package on either side means "assume it can", since a missed optimization costs +/// time while a wrong exclusion would report an untested mutant as unreachable and hide a real gap. +/// +/// A binary that announced no tests at all is the one exclusion that is safe to make from the +/// evidence a run already has. Cargo emits a unit-test binary for every lib target whether or not +/// the target holds a single test, so a package with no tests still produces a binary, and a +/// binary that exists is enough to make every mutant in that package "reachable" — which reports a +/// package nobody tests as full of survivors rather than as uncovered. Those are materially +/// different findings: one says the tests are weak, the other says there are none. A harness that +/// announced zero tests can convict nothing, so it makes nothing reachable. +pub(super) fn reaches(binary: &TestBinary, package: &str, plan: &Plan, scope: &TestScope<'_>) -> bool { + // `None` is "nobody counted" — no baseline, or a custom harness that announces nothing — and + // must not be read as zero. + if binary.tests == Some(0) { + return false; + } + + if !scope.admits(&binary.package) { + return false; + } + + if binary.package.is_empty() || package.is_empty() { + return true; + } + + if scope.package_local && binary.package != package { + return false; + } + + plan.reach.get(&binary.package).is_none_or(|reachable| reachable.contains(package)) +} + +/// Orders an unhinted mutant's reachable binaries without changing which binaries are reachable. +/// +/// Tests from the package that owns the mutant are normally the closest oracle. Within the own +/// package and remaining-package tiers, measured baseline cost comes first, followed by stable +/// identity fields so Cargo's artifact order cannot make two equivalent runs diverge. +/// +/// Exact per-mutant and learned file-local killers are applied later by the verdict path and +/// therefore still take precedence over this cold-run order; so does [`Census`]'s own +/// current-cost order, which further reorders this tier's tail once a census is available. +pub(super) fn order_reachable(binaries: &mut [&TestBinary], mutant_package: &str) { + binaries.sort_by(|left, right| { + let tier = |binary: &TestBinary| u8::from(binary.package != mutant_package); + + tier(left) + .cmp(&tier(right)) + .then_with(|| left.baseline.cmp(&right.baseline)) + .then_with(|| left.package.cmp(&right.package)) + .then_with(|| left.target.cmp(&right.target)) + .then_with(|| left.path.cmp(&right.path)) + }); +} + +/// Which binaries can reach each package holding a pending mutant, indexed once. +/// +/// Census economics, workload projection, and sweep scheduling each need this exact relationship, +/// and previously each derived it independently: three passes over the same pending mutants and +/// binaries, with three different owned key and value representations discarded in succession. +/// This is built once, after the binaries and the baseline are known, and lent to all three. +/// +/// Package reachability is a coarser fact than a census: it says a binary is *permitted* to be +/// consulted for a mutant's package, never that a specific test or a specific mutation site is +/// covered. Nothing here may ever be read as evidence that a site or a test is uncovered — only +/// [`Census`], and only when its own census for that binary completed, settles that. +#[derive(Debug, Default)] +pub(super) struct Reachability<'binaries> { + by_package: crate::HashMap, Vec<&'binaries TestBinary>>, +} + +impl<'binaries> Reachability<'binaries> { + /// Indexes every package holding a pending, ordinal-positive mutant to the binaries that can + /// reach it, in [`order_reachable`]'s cold-run order. + /// + /// Distinct packages are far fewer than mutants, so the reachable set is worked out once per + /// package rather than once per mutant. + pub(super) fn build(plan: &Plan, binaries: &'binaries [TestBinary], scope: &TestScope<'_>) -> Self { + let mut by_package: crate::HashMap, Vec<&TestBinary>> = crate::HashMap::default(); + + for mutant in plan + .mutants + .iter() + .filter(|mutant| mutant.ordinal > 0 && mutant.outcome == Outcome::Pending) + { + if by_package.contains_key(&mutant.package) { + continue; + } + + let mut found: Vec<&TestBinary> = binaries + .iter() + .filter(|binary| reaches(binary, &mutant.package, plan, scope)) + .collect(); + + order_reachable(&mut found, &mutant.package); + + let _fresh = by_package.insert(Arc::clone(&mutant.package), found); + } + + Self { by_package } + } + + /// The binaries that can reach `package`'s mutants, or `None` when this index was never asked + /// to cover that package — every package holding a pending mutant this run judges was, so + /// `None` here means the caller asked about a package with no pending work. + pub(super) fn reachable(&self, package: &str) -> Option<&[&'binaries TestBinary]> { + self.by_package.get(package).map(Vec::as_slice) + } + + /// How many distinct packages this index covers. + /// + /// Only the tests ask, to confirm the index memoizes by package rather than by mutant. + #[cfg(test)] + pub(super) fn len(&self) -> usize { + self.by_package.len() + } +} + +/// Which packages' tests are allowed to decide verdicts, before the preflight has its say. +/// +/// `--test-package` names the oracle outright. Failing that, `--test-workspace` lifts the cap +/// altogether. Failing both, the returned packages are the union whose own tests may judge their +/// own mutants. [`TestScope::package_local`] keeps that union package-local per verdict. +/// +/// Taken from the run's package selection rather than from the packages that turned out to hold +/// mutants, because `--file` and `--in-diff` narrow the population without narrowing what the +/// caller asked cargo for. Deriving it from the population instead would mean that mutating one +/// file in a package withdrew the rest of that package's own test binaries from the oracle, turning +/// kills into survivors for no reason the caller could see. +pub(super) fn oracle_packages(selected: &[String], config: &Config) -> Vec { + if !config.test_packages.is_empty() || config.test_workspace { + return config.test_packages.clone(); + } + + selected.to_vec() +} + +/// Which packages need their test targets compiled at all. +/// +/// A test binary is only ever run against a mutant its package can reach, so a package that cannot +/// reach anything being mutated produces binaries the run would build, baseline and then never +/// consult. Naming the useful subset lets cargo skip compiling the rest. +/// +/// Returns `None` when the subset is the whole workspace, which is both the common case and the +/// one worth spelling as `--workspace`: cargo unifies features over the packages it is asked to +/// build, so narrowing the selection is a change in what gets compiled and not only in how much. +/// The caller is expected to fall back to the whole workspace if a narrowed build fails. +pub(super) fn build_packages(plan: &Plan, scope: &TestScope<'_>) -> Option> { + let mutated: crate::HashSet<&str> = plan + .mutants + .iter() + .filter(|mutant| mutant.ordinal > 0 && mutant.outcome == Outcome::Pending) + .map(|mutant| &*mutant.package) + .collect(); + + reaching_packages(&plan.reach, &mutated, scope) +} + +/// The selection rule behind [`build_packages`], over an explicit set of mutated packages. +/// +/// The preflight check needs the same subset before anything has been scanned, when the mutated set +/// can only be named as "every package this run intends to mutate". That is a superset of the +/// packages that will turn out to hold live mutants, which is the direction the preflight needs to +/// err in: checking more than the later builds compile is wasted work, while checking less would +/// leave a target uncleared and let a genuine error in it be absorbed as though a mutant had caused +/// it. +pub(super) fn reaching_packages( + reach: &crate::HashMap>, + mutated: &crate::HashSet<&str>, + scope: &TestScope<'_>, +) -> Option> { + // Reach is keyed by every workspace member, so its keys are the population being narrowed from. + // Without it there is nothing to compare a subset against, so there is no subset. + if reach.is_empty() { + return None; + } + + let mut wanted: Vec = reach + .iter() + .filter(|(package, reachable)| { + if !scope.admits(package) { + // A package whose own code is being mutated still has to compile its test targets: + // a mutant can live in one of them. + return mutated.contains(package.as_str()); + } + + if scope.package_local { + return mutated.contains(package.as_str()); + } + + scope.whole_workspace || reachable.iter().any(|name| mutated.contains(name.as_str())) + }) + .map(|(package, _)| package.clone()) + .collect(); + + if wanted.len() >= reach.len() { + return None; + } + + // Deterministic, because it becomes a command line that is worth being able to compare between + // runs. + wanted.sort(); + + Some(wanted) +} + +/// Totals the work every live mutant represents, counting only the binaries that can reach it. +/// +/// Returns the serial suite time and the serial budget: what testing each live mutant once would +/// cost if every reachable binary ran to completion, and what it would cost if every reachable +/// binary instead ran out its timeout. Both are summed per mutant rather than taken from the whole +/// suite, because that is what a run actually does — a mutant in a leaf crate never starts the +/// binaries that cannot link it. +pub(super) fn workload(mutants: &[Mutant], reach: &Reachability<'_>, census: Option<&Census>) -> Workload { + let mut total = Workload::default(); + + // Its cost remains per-mutant when a census is present because each site has its own measured + // set of tests; only the reachable set behind it is shared across every mutant of a package. + for mutant in mutants + .iter() + .filter(|mutant| mutant.ordinal > 0 && mutant.outcome == Outcome::Pending) + { + let reachable = reach + .reachable(&mutant.package) + .expect("the shared reachability index was built from these same pending mutants"); + + let mut suite = Duration::ZERO; + let mut worst = Duration::ZERO; + let mut running = 0_usize; + + for binary in reachable { + match census.map_or(CensusWork::Whole, |census| census.work(binary, mutant.ordinal)) { + CensusWork::Whole => suite += binary.baseline, + CensusWork::Uncovered => continue, + CensusWork::Selected(duration) => suite += duration, + CensusWork::Hinted(_duration) => suite += binary.baseline, + } + + worst += binary.budget.unwrap_or_default(); + running = running.saturating_add(1); + } + + // A mutant that hangs hangs in one binary and is judged there, so what one costs is a + // single binary's budget rather than every binary's. Which one it will be is unknown, + // so the average stands in for it. + let single = u32::try_from(running).map_or(worst, |count| worst.checked_div(count).unwrap_or(worst)); + + total.suite += suite; + total.budget += worst; + total.single += single; + } + + total +} + +/// Which test binaries a run is allowed to consult. +#[derive(Debug, Clone, Copy)] +pub(super) struct TestScope<'names> { + /// Packages named by `--test-package`. Empty means no restriction. + pub(super) packages: &'names [String], + + /// Whether each mutant is judged only by tests from its own package. + pub(super) package_local: bool, + + /// Whether `--test-workspace` lifted the cap, so every package's tests may decide verdicts. + /// + /// Reachability still applies: lifting the cap widens which packages may convict, not which + /// code a binary can link. + pub(super) whole_workspace: bool, +} + +impl TestScope<'_> { + /// Returns whether a binary's package survives the `--test-package` filter. + pub(super) fn admits(&self, package: &str) -> bool { + self.packages.is_empty() || self.packages.iter().any(|wanted| wanted == package) + } +} + +/// Extracts the package name from a cargo package id. +/// +/// Two spellings are in circulation: the stable `path+file:///x/y#name@1.0.0` (name omitted when it +/// matches the last path segment) and the older `name 1.0.0 (source)`. An unrecognized id yields an +/// empty name, treated as "reaches everything" rather than "reaches nothing". +fn package_name(id: &str) -> String { + if let Some((locator, fragment)) = id.rsplit_once('#') { + return fragment.split_once('@').map_or_else( + || { + if is_version(fragment) { + // A bare version: the name is the last segment of the path before it. + locator.rsplit('/').next().unwrap_or_default().to_owned() + } else { + fragment.to_owned() + } + }, + |(name, _version)| name.to_owned(), + ); + } + + id.split_whitespace().next().unwrap_or_default().to_owned() +} + +/// Whether a package id fragment is a bare version rather than a package name. +/// +/// The release part of a version is digits and dots and nothing else, which no package name can be +/// mistaken for; anything after the first `-` or `+` is a pre-release or build tag and is ignored, +/// since those are made of the same letters a name is. +fn is_version(fragment: &str) -> bool { + let release = fragment.split(['-', '+']).next().unwrap_or(fragment); + + !release.is_empty() + && release.starts_with(|character: char| character.is_ascii_digit()) + && release.chars().all(|character| character.is_ascii_digit() || character == '.') +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A scope that admits every binary, which is what most of these tests are not about. + const ANY: TestScope<'static> = TestScope { + packages: &[], + package_local: false, + whole_workspace: false, + }; + + #[test] + fn test_binaries_are_read_from_cargo_json() { + // Only test artifacts are runnable; the library artifact in the same stream is not. + let stdout = concat!( + r#"{"reason":"compiler-artifact","profile":{"test":false},"executable":null}"#, + "\n", + r#"{"reason":"compiler-artifact","profile":{"test":true},"executable":"/tmp/unit"}"#, + "\n", + r#"{"reason":"compiler-message","message":{"level":"error"}}"#, + "\n", + "not json at all", + "\n", + r#"{"reason":"compiler-artifact","profile":{"test":true},"executable":"/tmp/cli"}"#, + "\n", + ); + + // The list is sorted so that a run's per-binary ordering does not depend on cargo's + // scheduling, which would make timings and any early exit irreproducible. + let paths: Vec = test_binaries(stdout).into_iter().map(|binary| binary.path).collect(); + + assert_eq!(paths, vec![Utf8PathBuf::from("/tmp/cli"), Utf8PathBuf::from("/tmp/unit")]); + } + + /// A test artifact cargo reports with no executable string contributes nothing to the list. + /// + /// `cargo` emits a `compiler-artifact` message for a test target before it has finished linking + /// it, and that message carries a `null` executable; treating it as a binary would hand the run + /// a path that does not exist yet, and every attempt to run it would be misread as a crash + /// rather than as the harmless intermediate message it actually is. + #[test] + fn an_artifact_with_no_executable_string_contributes_no_binary() { + let stdout = r#"{"reason":"compiler-artifact","profile":{"test":true},"executable":null}"#; + + assert!(test_binaries(stdout).is_empty()); + } + + /// Nothing to apportion is not an error. + #[test] + fn apportioning_across_no_binaries_does_nothing() { + let mut binaries: Vec = Vec::new(); + + apportion(&mut binaries, 1.2, Duration::from_secs(1), true); + + assert!(binaries.is_empty()); + } + + /// A binary the graph says nothing about is admitted, because a missed optimization is cheap + /// and a wrong exclusion would hide a real gap. + #[test] + fn a_binary_the_graph_says_nothing_about_is_admitted() { + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + + assert!(reaches(&binary("unrelated"), "subject", &plan_reaching(&[]), &scope)); + } + + /// `--test-workspace` lifts the `--test-package` cap; it does not make a binary link code it + /// cannot link. + /// + /// The cap's honesty rests on a mutant no admitted binary reaches being reported as uncovered + /// rather than as surviving — "nothing tests this" rather than "your tests missed this". A + /// whole-workspace scope that ignored the graph would run a binary against a mutant it cannot + /// link, watch it pass, and record a survivor on that evidence. + #[test] + fn a_whole_workspace_scope_still_does_not_reach_what_a_binary_cannot_link() { + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let plan = plan_reaching(&[("harness", &["harness"])]); + + assert!(!reaches(&binary("harness"), "subject", &plan, &scope)); + assert!(reaches(&binary("harness"), "harness", &plan, &scope)); + } + + /// A binary whose harness announced no tests reaches nothing, whatever the build graph says. + /// + /// Regression, issue-011. Cargo emits a unit-test binary for every lib target whether or not it + /// holds a test, so linkage alone always finds a binary and the uncovered bucket was + /// unreachable. A harness that announced zero tests can convict nothing, so it must not be what + /// makes a mutant look tested. + #[test] + fn a_binary_that_announced_no_tests_reaches_nothing() { + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let empty = TestBinary { + tests: Some(0), + ..binary("subject") + }; + + assert!(!reaches(&empty, "subject", &plan_reaching(&[]), &scope)); + } + + /// A binary nobody counted the tests of still reaches what it links. + /// + /// `None` is `--no-baseline`, or a `harness = false` target that announces nothing. Reading it + /// as zero would report a whole run as uncovered on the strength of a measurement never taken. + #[test] + fn a_binary_whose_tests_were_never_counted_still_reaches_what_it_links() { + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let uncounted = TestBinary { + tests: None, + ..binary("subject") + }; + + assert!(reaches(&uncounted, "subject", &plan_reaching(&[]), &scope)); + } + + /// A binary that announced tests reaches what it links, as it always did. + #[test] + fn a_binary_that_announced_tests_reaches_what_it_links() { + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + let populated = TestBinary { + tests: Some(7), + ..binary("subject") + }; + + assert!(reaches(&populated, "subject", &plan_reaching(&[]), &scope)); + } + + /// The harness width and the worker count multiply out to roughly the machine. + /// + /// Regression, issue-016. A harness defaults to one thread per core and a run starts `jobs` of + /// them, so the machine was asked for `jobs × cores` threads to do `cores` of work — and every + /// budget in the run was calibrated under a load nothing else ever reproduces. + #[test] + fn the_harness_width_divides_the_machine_between_the_workers() { + assert_eq!(harness_threads(4, 16, None), Some(4)); + assert_eq!(harness_threads(1, 16, None), Some(16)); + assert_eq!(harness_threads(16, 16, None), Some(1)); + } + + /// More workers than cores still gets a thread each, rather than none. + #[test] + fn more_workers_than_cores_still_get_a_thread_each() { + assert_eq!(harness_threads(64, 8, None), Some(1)); + + // A run cannot have no workers, but the arithmetic must not divide by zero if it did. + assert_eq!(harness_threads(0, 8, None), Some(8)); + } + + /// A width the caller already chose is left exactly as they chose it. + /// + /// Overriding it would silently change the workload someone asked for, and the run would be + /// measuring and judging something other than what they meant to test. + #[test] + fn a_harness_width_the_caller_chose_is_left_alone() { + assert_eq!(harness_threads(4, 16, Some("2")), None); + + // An empty setting is not a choice; it is what an unset variable looks like once it has + // been through a shell. + assert_eq!(harness_threads(4, 16, Some(" ")), Some(4)); + } + + #[test] + fn a_binary_carries_the_directory_of_its_packages_manifest() { + let line = r#"{"reason":"compiler-artifact","package_id":"path+file:///w/crates/subject#0.1.0","manifest_path":"/w/crates/subject/Cargo.toml","profile":{"test":true},"executable":"/w/target/debug/deps/subject-abc"}"#; + let found = test_binaries(line); + + assert_eq!(found.len(), 1); + assert_eq!(found[0].manifest_dir, "/w/crates/subject"); + } + + #[test] + fn a_binary_cargo_did_not_locate_has_no_manifest_directory() { + let line = r#"{"reason":"compiler-artifact","package_id":"path+file:///w#0.1.0","profile":{"test":true},"executable":"/w/target/debug/deps/subject-abc"}"#; + let found = test_binaries(line); + + assert_eq!(found.len(), 1); + assert_eq!(found[0].manifest_dir, ""); + } + + fn binary(package: &str) -> TestBinary { + TestBinary { + package: package.to_owned(), + ..crate::testing::test_binary("/tmp/t") + } + } + + /// The path has to differ, since `restrict` runs on a list `test_binaries` already deduplicated. + fn target(name: &str) -> TestBinary { + TestBinary { + target: name.to_owned(), + ..crate::testing::test_binary(&format!("/tmp/{name}")) + } + } + + fn names(binaries: &[TestBinary]) -> Vec<&str> { + binaries.iter().map(|binary| binary.target.as_str()).collect() + } + + #[test] + fn the_target_name_is_kept_from_cargo_json() { + let stdout = concat!( + r#"{"reason":"compiler-artifact","profile":{"test":true},"target":{"name":"conformance_xsd"},"executable":"/tmp/c"}"#, + "\n" + ); + + let found = test_binaries(stdout); + + assert_eq!(found.len(), 1); + assert_eq!(found[0].target, "conformance_xsd"); + } + + /// Cargo has always reported this, but an older or stubbed stream must not lose the binary. + #[test] + fn a_binary_with_no_target_name_is_still_kept() { + let stdout = concat!( + r#"{"reason":"compiler-artifact","profile":{"test":true},"executable":"/tmp/c"}"#, + "\n" + ); + + let found = test_binaries(stdout); + + assert_eq!(found.len(), 1); + assert_eq!(found[0].target, ""); + } + + #[test] + fn no_patterns_leave_every_binary_in_place() { + let mut binaries = vec![target("unit"), target("conformance_xsd")]; + + restrict(&mut binaries, &[], &[]); + + assert_eq!(names(&binaries), vec!["unit", "conformance_xsd"]); + } + + #[test] + fn an_exclusion_glob_removes_the_targets_it_names() { + let mut binaries = vec![target("unit"), target("conformance_xsd"), target("conformance_xpath")]; + + restrict(&mut binaries, &[], &["conformance_*".to_owned()]); + + assert_eq!(names(&binaries), vec!["unit"]); + } + + #[test] + fn an_inclusion_glob_keeps_only_the_targets_it_names() { + let mut binaries = vec![target("unit"), target("integration"), target("conformance_xsd")]; + + restrict(&mut binaries, &["unit".to_owned(), "integration".to_owned()], &[]); + + assert_eq!(names(&binaries), vec!["unit", "integration"]); + } + + /// Otherwise `--include-test "*"` with a few exclusions would quietly mean the whole suite. + #[test] + fn an_exclusion_beats_an_inclusion_that_also_matches() { + let mut binaries = vec![target("unit"), target("conformance_xsd")]; + + restrict(&mut binaries, &["*".to_owned()], &["conformance_*".to_owned()]); + + assert_eq!(names(&binaries), vec!["unit"]); + } + + #[test] + fn a_pattern_matching_a_declared_target_is_not_reported_as_unmatched() { + let tests = vec!["unit".to_owned(), "conformance_xsd".to_owned()]; + + assert_eq!(unmatched_test(&tests, &[], &["conformance_*".to_owned()]), None); + } + + #[test] + fn a_pattern_matching_nothing_is_reported() { + let tests = vec!["unit".to_owned(), "conformance_xsd".to_owned()]; + let exclude = vec!["conformance_*".to_owned(), "confrmance_xpath".to_owned()]; + + assert_eq!(unmatched_test(&tests, &[], &exclude), Some("confrmance_xpath")); + } + + /// An inclusion typo empties the oracle instead of widening it, and is just as fatal. + #[test] + fn an_unmatched_inclusion_is_reported_too() { + let tests = vec!["unit".to_owned()]; + + assert_eq!(unmatched_test(&tests, &["untis".to_owned()], &[]), Some("untis")); + } + + fn plan_reaching(edges: &[(&str, &[&str])]) -> Plan { + let mut reach = crate::HashMap::default(); + + for (from, to) in edges { + let _previous = reach.insert((*from).to_owned(), to.iter().map(|name| (*name).to_owned()).collect()); + } + + Plan { + skipped: Vec::new(), + digests: crate::HashMap::default(), + root: Utf8PathBuf::from("/w"), + files: Vec::new(), + mutants: Vec::new(), + suppressed: 0, + idle: Vec::new(), + sharded_out: 0, + settled_out: 0, + reach, + specs: crate::HashMap::default(), + } + } + + /// A plan whose live mutants sit in `packages`, over the dependency graph `edges` describes. + fn plan_mutating(edges: &[(&str, &[&str])], packages: &[&str]) -> Plan { + let mut plan = plan_reaching(edges); + + plan.mutants = packages + .iter() + .enumerate() + .map(|(index, package)| Mutant { + id: (*package).to_owned().into(), + ordinal: u32::try_from(index).unwrap_or(0).saturating_add(1), + file: (Utf8PathBuf::from("src/lib.rs")).into(), + package: ((*package).to_owned()).into(), + span: 0..1, + line: 1, + end_line: 1, + column: 1, + mutator: ("arith.add_to_sub".to_owned()).into(), + item_path: ("f".to_owned()).into(), + occurrence: 0, + replacement_index: 0, + original: "a + b".to_owned().into(), + replacement: "a - b".to_owned().into(), + shape: crate::ops::collect::Shape::Expr, + outcome: Outcome::Pending, + suppression: None, + expectation: None, + test_timeout_multiplier: None, + elapsed_ms: 0, + killed_by: None, + note: None, + }) + .collect(); + + plan + } + + /// `app` and `tool` both link `core`; `aside` links nothing. + fn graph() -> Vec<(&'static str, &'static [&'static str])> { + vec![ + ("app", &["app", "core"] as &[&str]), + ("tool", &["tool", "core"]), + ("core", &["core"]), + ("aside", &["aside"]), + ] + } + + /// A package is a correlated workload: every one of its mutants reaches the exact same + /// binaries in the exact same order, so working the answer out once per package is not an + /// optimization that can ever disagree with working it out once per mutant. + #[test] + fn reachability_memoizes_by_package_rather_than_by_mutant() { + let plan = plan_mutating(&graph(), &["core", "core", "app"]); + let binaries = vec![binary("app"), binary("core"), binary("tool")]; + + let reach = Reachability::build(&plan, &binaries, &ANY); + + assert_eq!(reach.len(), 2, "one entry per distinct package, not one per mutant"); + } + + /// A package holding no pending mutant was never asked about, and stays absent rather than + /// mapping to an empty answer — the two mean different things to a caller. + #[test] + fn reachability_answers_none_for_a_package_holding_no_pending_mutant() { + let plan = plan_mutating(&graph(), &["core"]); + let binaries = vec![binary("app"), binary("core"), binary("tool")]; + + let reach = Reachability::build(&plan, &binaries, &ANY); + + assert!( + reach.reachable("aside").is_none(), + "aside holds no pending mutant, so nothing should have asked about it" + ); + } + + /// The shared index applies [`order_reachable`]'s own-package-first, baseline-then-identity + /// cold-run order while building each package's answer, rather than handing back an arbitrary + /// one its caller would have to re-sort. + #[test] + fn reachability_orders_each_packages_binaries_with_order_reachable() { + let plan = plan_mutating(&graph(), &["core"]); + let mut app_dependent = binary("app"); + app_dependent.baseline = Duration::from_millis(1); + let mut own = binary("core"); + own.baseline = Duration::from_secs(1); + let binaries = vec![app_dependent, own]; + + let reach = Reachability::build(&plan, &binaries, &ANY); + let ordered = reach.reachable("core").expect("core holds a pending mutant"); + + assert_eq!( + ordered.iter().map(|binary| binary.package.as_str()).collect::>(), + ["core", "app"], + "own package first, exactly as order_reachable orders it directly" + ); + } + + /// With no census, every reachable binary is costed at its whole baseline, and a mutant's + /// share of the run-out-of-time budget averages that binary set's own budgets. + #[test] + fn workload_sums_reachable_binary_baselines_with_no_census() { + let plan = plan_mutating(&graph(), &["core"]); + let mut app_dependent = binary("app"); + app_dependent.baseline = Duration::from_millis(300); + app_dependent.budget = Some(Duration::from_secs(3)); + let mut own = binary("core"); + own.baseline = Duration::from_millis(700); + own.budget = Some(Duration::from_secs(7)); + let binaries = vec![app_dependent, own]; + + let reach = Reachability::build(&plan, &binaries, &ANY); + let work = workload(&plan.mutants, &reach, None); + + assert_eq!( + work.suite, + Duration::from_secs(1), + "one mutant, whole-binary cost for both of its reachable binaries" + ); + assert_eq!(work.budget, Duration::from_secs(10), "both binaries' budgets, summed"); + assert_eq!(work.single, Duration::from_secs(5), "the average of the two binaries' budgets"); + } + + /// `workload` shares [`test_all`]'s exact invariant: the `Reachability` it is given must have + /// been built from the very same pending mutants it iterates, so every pending mutant's package + /// is guaranteed present. A package silently missing used to be skipped, which would silently + /// under-count the workload instead of surfacing the mismatch that caused it. + #[test] + #[should_panic(expected = "the shared reachability index was built from these same pending mutants")] + fn workload_refuses_a_reachability_missing_a_pending_mutants_package() { + let plan = plan_mutating(&graph(), &["core"]); + let binaries = vec![binary("core")]; + + // Built from a plan that never mutates `core`, so the index has no entry for it — exactly + // the mismatch `test_all` itself guards against with the same `.expect(...)`. + let stale_plan = plan_mutating(&graph(), &["aside"]); + let reach = Reachability::build(&stale_plan, &binaries, &ANY); + + let _work = workload(&plan.mutants, &reach, None); + } + + #[test] + fn a_package_whose_tests_can_reach_nothing_being_mutated_is_not_built() { + let plan = plan_mutating(&graph(), &["core"]); + + // `aside` cannot link `core`, so every test binary it would produce is one the run would + // compile, baseline and never consult. + assert_eq!( + build_packages(&plan, &ANY), + Some(vec!["app".to_owned(), "core".to_owned(), "tool".to_owned()]) + ); + } + + #[test] + fn mutating_everything_asks_for_the_whole_workspace() { + let plan = plan_mutating(&graph(), &["core", "aside"]); + + assert_eq!(build_packages(&plan, &ANY), None, "a subset of everything is not a subset"); + } + + #[test] + fn naming_the_tests_that_matter_narrows_the_build_to_them() { + let plan = plan_mutating(&graph(), &["core"]); + let named = [String::from("app")]; + let scope = TestScope { + packages: &named, + package_local: false, + whole_workspace: false, + }; + + // Only `app`'s tests can return a verdict, so only `app`'s tests are worth compiling — and + // `core` comes with them because a mutant can live in one of its own test targets. + assert_eq!(build_packages(&plan, &scope), Some(vec!["app".to_owned(), "core".to_owned()])); + } + + #[test] + fn package_local_testing_builds_only_the_mutated_packages_own_tests() { + let plan = plan_mutating(&graph(), &["core"]); + let selected = [String::from("app"), String::from("core"), String::from("tool")]; + let scope = TestScope { + packages: &selected, + package_local: true, + whole_workspace: false, + }; + + assert_eq!(build_packages(&plan, &scope), Some(vec!["core".to_owned()])); + } + + #[test] + fn testing_the_whole_workspace_builds_the_whole_workspace() { + let plan = plan_mutating(&graph(), &["core"]); + let scope = TestScope { + packages: &[], + package_local: false, + whole_workspace: true, + }; + + assert_eq!(build_packages(&plan, &scope), None); + } + + #[test] + fn a_workspace_with_no_dependency_graph_is_built_whole() { + let plan = plan_mutating(&[], &["core"]); + + assert_eq!(build_packages(&plan, &ANY), None, "nothing is known, so nothing can be ruled out"); + } + + /// A bare run consults the tests cargo would have run here, and no others. + /// + /// This is the cap the whole oracle rests on: `cargo gamma` inside a crate costs a multiple of + /// `cargo test` inside that crate, rather than a multiple of everything that links it. + #[test] + fn the_oracle_defaults_to_the_packages_cargo_itself_would_have_tested() { + let selected = [String::from("tick")]; + + assert_eq!(oracle_packages(&selected, &Config::default()), vec!["tick".to_owned()]); + } + + /// `--test-package` is the caller naming the oracle outright, so the default has nothing to add. + #[test] + fn naming_the_test_packages_replaces_the_default_cap() { + let selected = [String::from("tick")]; + let config = Config { + test_packages: vec!["harness".to_owned()], + ..Config::default() + }; + + assert_eq!(oracle_packages(&selected, &config), vec!["harness".to_owned()]); + } + + /// `--test-workspace` is the way to ask for the reach-everything oracle, so it lifts the cap. + #[test] + fn testing_the_whole_workspace_lifts_the_cap_entirely() { + let selected = [String::from("tick")]; + let config = Config { + test_workspace: true, + ..Config::default() + }; + + assert!( + oracle_packages(&selected, &config).is_empty(), + "an empty restriction is what admits every package" + ); + } + + /// The cap follows the package selection, not the mutants that selection turned out to produce. + /// + /// `--file` and `--in-diff` narrow the population without narrowing what cargo was asked for. + /// Reading the cap off the mutated packages would mean that touching one file withdrew the rest + /// of that package's own binaries from the oracle and reported their kills as survivors. + #[test] + fn the_cap_covers_every_selected_package_even_when_only_one_holds_mutants() { + let selected = [String::from("tick"), String::from("tock")]; + + assert_eq!( + oracle_packages(&selected, &Config::default()), + vec!["tick".to_owned(), "tock".to_owned()] + ); + } + + /// A dependent's tests do not judge a mutant in the crate the run was pointed at. + /// + /// The reachability graph says `app` links `core` and so *could* convict, and before the cap it + /// did. Running in `core` now asks only what `core`'s own suite thinks, which is what makes the + /// score a property of the crate rather than of the workspace around it. + #[test] + fn a_dependents_tests_do_not_judge_a_mutant_in_the_crate_being_run_on() { + let plan = plan_reaching(&[("app", &["app", "core"]), ("core", &["core"])]); + let capped = [String::from("core")]; + let scope = TestScope { + packages: &capped, + package_local: false, + whole_workspace: false, + }; + + assert!(!reaches(&binary("app"), "core", &plan, &scope), "the oracle escaped its cap"); + assert!(reaches(&binary("core"), "core", &plan, &scope), "the crate cannot judge itself"); + } + + #[test] + fn a_workspace_run_judges_each_mutant_only_with_its_own_package() { + let plan = plan_reaching(&[("app", &["app", "core"]), ("core", &["core"])]); + let selected = [String::from("app"), String::from("core")]; + let scope = TestScope { + packages: &selected, + package_local: true, + whole_workspace: false, + }; + + assert!(!reaches(&binary("app"), "core", &plan, &scope)); + assert!(reaches(&binary("core"), "core", &plan, &scope)); + assert!(reaches(&binary("app"), "app", &plan, &scope)); + } + + #[test] + fn a_binary_only_reaches_what_its_package_links() { + let plan = plan_reaching(&[("app", &["app", "core"]), ("core", &["core"])]); + + assert!(reaches(&binary("app"), "core", &plan, &ANY)); + assert!(reaches(&binary("app"), "app", &plan, &ANY)); + + // The core crate does not link the app, so no test of it can reach the app's code. + assert!(!reaches(&binary("core"), "app", &plan, &ANY)); + } + + #[test] + fn an_unknown_package_reaches_everything() { + let plan = plan_reaching(&[("app", &["app"])]); + + assert!( + reaches(&binary(""), "core", &plan, &ANY), + "an unattributed binary must not be skipped" + ); + assert!( + reaches(&binary("app"), "", &plan, &ANY), + "an unattributed mutant must not be skipped" + ); + assert!(reaches(&binary("stranger"), "core", &plan, &ANY), "a package we know nothing about"); + } + + #[test] + fn a_test_package_filter_excludes_other_binaries() { + let plan = plan_reaching(&[("app", &["app"]), ("other", &["other"])]); + let named = [String::from("app")]; + let scope = TestScope { + packages: &named, + package_local: false, + whole_workspace: false, + }; + + // Filtering by test package is a hard user request, so an otherwise reachable binary from + // another package must not run. + assert!(!reaches(&binary("other"), "other", &plan, &scope)); + assert!(reaches(&binary("app"), "app", &plan, &scope)); + } + + #[test] + fn a_binary_is_attributed_to_the_package_that_produced_it() { + let stdout = concat!( + r#"{"reason":"compiler-artifact","profile":{"test":true},"executable":"/tmp/a","#, + r#""package_id":"path+file:///w/crates/parser#cargo-gamma-lib@0.1.0"}"#, + "\n", + ); + + assert_eq!(test_binaries(stdout)[0].package, "cargo-gamma-lib"); + } + + #[test] + fn every_spelling_of_a_package_id_is_understood() { + // The name is omitted when it matches the last path segment, which is the common case for + // a workspace member and the one an over-eager parser gets wrong. + assert_eq!(package_name("path+file:///w/crates/cargo-gamma-rt#0.1.0"), "cargo-gamma-rt"); + assert_eq!( + package_name("path+file:///w/crates/parser#cargo-gamma-lib@0.1.0"), + "cargo-gamma-lib" + ); + assert_eq!( + package_name("registry+https://github.com/rust-lang/crates.io-index#serde@1.0.0"), + "serde" + ); + assert_eq!(package_name("serde 1.0.0 (registry+https://example.com)"), "serde"); + } + + #[test] + fn a_pre_release_version_is_not_mistaken_for_a_package_name() { + // The letters in `-beta` can look like a name, which would attribute the binary to a + // package that does not exist and so quietly stop it reaching anything. + assert_eq!(package_name("path+file:///w/crates/cargo-gamma-rt#1.0.0-beta.1"), "cargo-gamma-rt"); + assert_eq!(package_name("path+file:///w/crates/cargo-gamma-rt#1.0.0+build7"), "cargo-gamma-rt"); + + // A name that merely begins with a digit is still a name. + assert_eq!(package_name("path+file:///w/crates/x#3d-tiles"), "3d-tiles"); + } + + #[test] + fn a_budget_is_scaled_from_the_binary_baseline() { + let mut binaries = vec![binary("a"), binary("b")]; + + binaries[0].baseline = Duration::from_secs(10); + binaries[1].baseline = Duration::from_secs(30); + + apportion(&mut binaries, 1.2, Duration::ZERO, true); + + assert_eq!(binaries[0].budget, Some(Duration::from_secs(12))); + assert_eq!(binaries[1].budget, Some(Duration::from_secs(36))); + } + + /// A *measured* near-zero baseline, which is the only thing a zero duration may mean once the + /// baseline has run. + #[test] + fn binaries_whose_baseline_was_instantaneous_get_the_floor() { + let mut binaries = vec![binary("a"), binary("b"), binary("c")]; + + apportion(&mut binaries, 1.2, Duration::from_secs(20), true); + + for entry in &binaries { + assert_eq!(entry.budget, Some(Duration::from_secs(20))); + } + } + + /// Without a baseline every duration is zero because nothing was measured, and scaling that + /// hands every binary the floor as though it had been derived. On a suite slower than the floor + /// each mutant then runs out of time, and a timeout scores as a detection — a near-perfect score + /// made entirely of mutants no test ever exercised. + #[test] + fn an_uncalibrated_run_gets_no_budget_rather_than_the_floor() { + let mut binaries = vec![binary("a"), binary("b")]; + + apportion(&mut binaries, 1.2, Duration::from_secs(20), false); + + for entry in &binaries { + assert_eq!(entry.budget, None, "a cutoff was invented from no measurement"); + assert_eq!(entry.budget_for(None, Duration::from_secs(20)), None); + + // A per-mutant multiplier rescales a measurement; it cannot stand in for one. + assert_eq!(entry.budget_for(Some(3.0), Duration::from_secs(20)), None); + } + } + + #[test] + fn the_floor_is_a_promise_about_one_binary() { + let mut binaries = vec![binary("a"), binary("b")]; + + binaries[0].baseline = Duration::from_millis(1); + binaries[1].baseline = Duration::from_secs(100); + + apportion(&mut binaries, 1.2, Duration::from_secs(20), true); + + // A binary whose proportional scaled duration is under the floor gets the floor. + assert_eq!(binaries[0].budget, Some(Duration::from_secs(20))); + + // A binary that earned a larger share keeps it. + assert_eq!(binaries[1].budget, Some(Duration::from_mins(2))); + } + + #[test] + fn a_crowded_workspace_does_not_dilute_the_floor_away() { + let mut binaries: Vec = (0..200).map(|_index| binary("a")).collect(); + + for entry in &mut binaries { + entry.baseline = Duration::from_millis(50); + } + + apportion(&mut binaries, 1.2, Duration::from_secs(20), true); + + for entry in &binaries { + assert_eq!(entry.budget, Some(Duration::from_secs(20)), "the floor was diluted again"); + } + } + + #[test] + fn a_binary_that_earns_more_than_the_floor_keeps_its_share() { + let mut binaries = vec![binary("a"), binary("b")]; + + binaries[0].baseline = Duration::from_secs(100); + binaries[1].baseline = Duration::from_secs(100); + + apportion(&mut binaries, 1.2, Duration::from_secs(20), true); + + assert_eq!(binaries[0].budget, Some(Duration::from_mins(2))); + } + + #[test] + fn budget_for_computes_per_mutant_overridden_budget() { + let mut b = binary("a"); + b.baseline = Duration::from_secs(10); + b.budget = Some(Duration::from_secs(12)); // 10s baseline scaled by 1.2 + + assert_eq!(b.budget_for(None, Duration::from_secs(1)), Some(Duration::from_secs(12))); + assert_eq!(b.budget_for(Some(3.0), Duration::from_secs(1)), Some(Duration::from_secs(30))); + assert_eq!(b.budget_for(Some(0.01), Duration::from_secs(5)), Some(Duration::from_secs(5))); + } + + #[test] + fn an_unreadable_package_id_reaches_everything_rather_than_nothing() { + // Guessing "nothing" would silently stop testing a mutant and report it as unreachable, + // which reads as a finding about the code rather than a failure of this parser. + assert_eq!(package_name(""), ""); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/verdict.rs b/crates/cargo-gamma-lib/src/exec/verdict.rs new file mode 100644 index 000000000..d638a57fd --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/verdict.rs @@ -0,0 +1,3427 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::time::Duration; +use std::io::{self, BufReader, Read}; +use std::process::{Child, ChildStderr, ChildStdout, Command, ExitStatus, Stdio}; +use std::sync::mpsc::{Receiver, RecvTimeoutError}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread; +use std::time::Instant; + +use camino::Utf8Path; +use cargo_gamma_process::{MemoryRequest, MemoryUsage, ProcessTree, SpawnGuard, prepare}; + +mod hubs; + +#[cfg(test)] +use cargo_gamma_process::faults::{self as process_faults, Fault as ProcessFault}; +use hubs::Pulse; +pub use hubs::READERS; +#[cfg(test)] +#[cfg(not(loom))] +use hubs::Readers; +#[cfg(loom)] +pub(crate) use hubs::run_loom_models; + +#[cfg(test)] +use super::faults::{self, Fault}; +use super::harness_filters::HarnessFilters; +use super::loader::{Launch, STACK_VAR, UNDER_GAMMA_VAR, configure_loader}; +use super::nextest; +use super::progress::{Progress, Watch}; +use super::stall::Stall; +use super::test_binary::{TEST_THREADS_VAR, TestBinary}; +use super::workspace::Workspace; + +/// The variable Insta reads to decide whether to write snapshots it was asked to compare against. +const INSTA_UPDATE_VAR: &str = "INSTA_UPDATE"; + +/// The variable Insta reads to decide whether a mismatched snapshot still passes. +const INSTA_FORCE_PASS_VAR: &str = "INSTA_FORCE_PASS"; + +/// Which of a binary's tests a run is to execute. +/// +/// A named test is a probe rather than a substitute for the binary: it is only ever used to check a +/// guess about which test catches a mutant, and a probe that does not convict is followed by the +/// ordinary whole-binary run. Nothing that reads this may draw a conclusion about the *absence* of +/// a failure, because a filtered run has not looked at the rest of the suite. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) enum Only<'name> { + /// Everything the binary holds, which is what a verdict about a mutant is normally reached from. + #[default] + All, + + /// One test, named exactly as the harness names it. + One(&'name str), + + /// Every test a census established can reach the mutant, and nothing else. + /// + /// Unlike [`Self::One`] this *is* a substitute for the binary, and a verdict may be read from + /// its absence of failures: the census measured that no other test executes the mutant's site, + /// so no other test can convict it. See `census` for why that is exact rather than a guess. + These(&'name [&'name str]), +} + +impl<'name> Only<'name> { + /// The tests to run, empty when the whole binary runs. + fn names(self) -> Vec<&'name str> { + match self { + Self::All => Vec::new(), + Self::One(name) => vec![name], + Self::These(names) => names.to_vec(), + } + } +} + +/// Everything about how one run of a test binary is to be performed, other than which binary. +/// +/// Carried together because they are decided as a set and threaded unchanged through the launch, +/// the confirmation and the retries: which mutant is active decides what the run is evidence about, +/// the budget and the silence budget decide when it is cut short, the accounting decides what it is +/// held to, and the filter decides how much of the suite it looks at. Splitting them back out only +/// makes every function in the path take five more arguments and every retry a place to pass one of +/// them differently by accident. +#[derive(Debug, Clone, Copy)] +pub(super) struct Attempt<'name> { + /// The mutant to switch on, or `None` for a run that is evidence about the suite itself. + pub(super) active: Option, + + /// How long the binary may run before it is cut off, or `None` for no cutoff at all. + /// + /// `None` is what a run with no baseline gets: see [`TestBinary::budget`]. + /// + /// [`TestBinary::budget`]: super::TestBinary::budget + pub(super) timeout: Option, + + /// How long it may go without saying anything before it is treated as stuck. + pub(super) stall: Stall, + + /// What the run's memory is to be accounted for against. + pub(super) request: MemoryRequest, + + /// How much of the binary's suite to run. + pub(super) only: Only<'name>, + + /// Where the runtime is to write the sites this run reached, for a census. + /// + /// `None` for every ordinary run. Setting it also selects unmutated behavior in the runtime, + /// which is why it is only ever paired with an `active` of `None`. + pub(super) census: Option<&'name Utf8Path>, +} + +impl Attempt<'_> { + /// The same attempt with no mutant active, which is how a suspected kill is exonerated. + const fn exonerating(self) -> Self { + Self { active: None, ..self } + } + + /// The same attempt with several times the budget, which is how a suspected timeout is checked. + const fn lengthened(self) -> Self { + Self { + timeout: match self.timeout { + Some(timeout) => Some(timeout.saturating_mul(CONFIRM_FACTOR)), + None => None, + }, + ..self + } + } + + /// The same attempt with several times the silence budget, for a suspected stall. + fn patient(self) -> Self { + Self { + stall: self.stall.scaled(CONFIRM_FACTOR), + ..self + } + } +} + +/// What running one test binary said. +#[derive(Debug, PartialEq, Eq)] +pub(super) enum Verdict { + Passed, + + /// A test failed, named when the harness said which. + Failed(Option), + + /// Nextest could not enumerate the selected tests while a mutant was active. + /// + /// This is only a suspicion until the identical selection succeeds with no mutant active. + /// Carries nextest's output because enumeration failures usually explain themselves there. + TestEnumerationFailed(String), + + /// The budget ran out while the binary was still making progress. + TimedOut, + + /// The binary stopped reporting progress long before its budget ran out. + /// + /// Carries the last test the harness said anything about, which is a landmark rather than a + /// diagnosis: libtest runs tests in parallel and names one only when it finishes, so the test + /// that is spinning is precisely the one it has not named. + Stalled(Option), + + /// The kernel stopped the test workload for passing the memory ceiling this run installed. + /// + /// A separate verdict rather than a kind of failure or a kind of timeout, because it is + /// neither: the tests did not notice anything, and nothing ran out of time. What happened is + /// that the mutant made the workload allocate past a ceiling the same workload stayed under + /// with no mutant active. Carries the peak the platform observed, when it could observe one, + /// and the ceiling that fired, since a reader's first question is how far past it went. + MemoryLimit { + /// The highest aggregate memory the subtree reached, when the platform reported one. + peak: Option, + + /// The ceiling that was installed for this binary. + limit: u64, + }, + + /// A test failed with the mutant active and failed again with no mutant active. + /// + /// Not a verdict about the mutant either way. The suite noticed something, but the same thing + /// happens without the mutant, so nothing about this run is evidence about this mutant. + /// Carries the test that failed, which is the only actionable thing here: the remedy is to fix + /// that test, not to write a new one. + Flaky(Option), + + /// The run could not be measured as this run was configured, so nothing about the mutant was + /// learned. + /// + /// Two shapes reach this: the accounting the run asked for could not be installed, so nothing + /// was started at all, and a run that started but could not be followed to a conclusion — a + /// wait on the child that failed, which leaves the one question that would have settled the + /// mutant unanswerable. + /// + /// Not a verdict about the mutant either way. It exists so that a failure of the machinery + /// stops the run and says why, instead of quietly becoming an unprotected run or, worse, a + /// detection the suite never made. + Unmetered(String), + + /// This one run could not be performed, though the run as a whole is still sound. + /// + /// The machine was momentarily out of something a subprocess needs — descriptors, process + /// slots, address space — or the child could not be asked about after it started. A sweep runs + /// `jobs` processes at once, each with pipes of its own, so exhausting one of those tables is an + /// ordinary consequence of the workload rather than a standing fact about the run: the mutants + /// already judged are still judged, and the ones after it are still judgeable. + /// + /// Separate from [`Self::Unmetered`] because the response differs. An unmetered run says nothing + /// further can be trusted and stops; this says one mutant went unjudged, records why against + /// that mutant, and lets the sweep continue. Conflating them lets one transient `EMFILE` discard + /// every verdict an hours-long run had already reached. + Unjudged(String), +} + +/// Runs one test binary with an optional active mutant, under a wall-clock budget. +/// +/// Every provisional abnormal verdict is confirmed by a second run before it is believed. A false +/// kill inflates the score, while a false resource-exhaustion verdict lowers it and can fail the +/// run; neither should depend on one noisy execution. +/// +/// The confirmations differ because the suspicions do: +/// +/// * A timeout or a stall is retried with the same mutant under a budget several times larger. +/// Both are verdicts a loaded machine can produce on its own — the budget is calibrated from a +/// baseline measured when nothing else was competing for cores, while mutants run many at a time +/// — so the first answer is a suspicion about the machine and the second is the finding. A +/// suspected stall keeps a looser silence budget rather than none at all, so a mutant that really +/// has hung is still cut off early instead of waiting out the whole timeout. +/// * A memory verdict is retried unchanged. The ceiling is not always enforced by something that +/// watched the allocation happen: on Windows the child is spawned and only afterwards assigned to +/// the job object, so the verdict is inferred from the job's accounting, and on the timeout path +/// it is inferred from what the accounting read when the run was cut short. An inference deserves +/// the same second look as a budget running out. +/// * A failing test is retried with **no** mutant active. Nothing about a red test says the mutant +/// made it red, and a test that is merely flaky would otherwise be scored as a kill every time it +/// happened to fail. If it fails again with the mutant out of the picture, the suite is what is +/// unreliable, and the run says so with [`Verdict::Flaky`] rather than crediting the mutant. +/// +/// The kill confirmation costs one extra run of the cheapest verdict class — the one that finishes +/// as soon as a test fails rather than running the whole binary. `confirm` turns it off, at the +/// price of a score that counts flakes as kills and cannot show which ones they were. +pub(super) fn run_binary(work: &Workspace, binary: &TestBinary, attempt: Attempt<'_>, confirm: bool) -> Verdict { + let confirmed = settle_suspicions(attempt, |attempt| observe(work, binary, attempt).verdict); + + // Reached from the first run and from a confirmation alike: a suspected timeout that turns out + // to be a failing test is exactly as much of a suspicion as one reported straight away. + match confirmed { + Verdict::Failed(test) if confirm => confirm_kill(work, binary, attempt, test), + Verdict::TestEnumerationFailed(output) => confirm_enumeration(work, binary, attempt, output), + other => other, + } +} + +/// How many times a spawn refused for want of a machine resource is attempted in all. +/// +/// A sweep runs `jobs` subprocesses at once, each with pipes of its own, so running the descriptor +/// or process table dry is an ordinary consequence of the workload rather than a fault. It clears as +/// soon as one of the mutants already running finishes, which is what these attempts wait for. +const SPAWN_ATTEMPTS: usize = 4; + +/// How long the first retry waits; each one after it waits twice as long. +/// +/// Backed off rather than tried in a tight loop, because every worker meets the shortage at the same +/// moment and retrying in step would keep the table exhausted between them. +const SPAWN_BACKOFF: Duration = Duration::from_millis(50); + +/// Starts the child, waiting out a refusal the machine will recover from on its own. +enum StartError { + Containment(String), + Spawn(io::Error), +} + +fn spawn_patiently(command: &mut Command, request: MemoryRequest) -> Result<(Child, SpawnGuard), StartError> { + let mut waited = SPAWN_BACKOFF; + let mut guard = prepare(command, request).map_err(StartError::Containment)?; + + for _attempt in 1..SPAWN_ATTEMPTS { + match spawn_once(command) { + Ok(child) => return Ok((child, guard)), + Err(cause) if transient(&cause) => { + guard = guard.backoff(waited).map_err(StartError::Containment)?; + } + Err(cause) => return Err(StartError::Spawn(cause)), + } + + waited = waited.saturating_mul(2); + } + + spawn_once(command).map(|child| (child, guard)).map_err(StartError::Spawn) +} + +/// The spawn itself, named so the fault seam can stand in for the kernel's refusal. +fn spawn_once(command: &mut Command) -> io::Result { + #[cfg(test)] + if faults::fired(Fault::Spawn) { + return Err(io::Error::new(io::ErrorKind::WouldBlock, "the process table is full")); + } + + command.spawn() +} + +/// Whether a spawn refusal is the machine being momentarily out of something. +/// +/// Rust gives only `WouldBlock` a stable `ErrorKind`, so the rest are matched on the raw code: +/// `EAGAIN` for the process table, `EMFILE` and `ENFILE` for this process's and the system's +/// descriptor tables, `ENOMEM` for the address space a fork needs. Everything else — a binary that +/// is not there, one that is not executable — is a standing fact that no amount of waiting changes. +fn transient(cause: &io::Error) -> bool { + if cause.kind() == io::ErrorKind::WouldBlock || cause.kind() == io::ErrorKind::Interrupted { + return true; + } + + #[cfg(unix)] + { + matches!( + cause.raw_os_error(), + Some(libc::EAGAIN | libc::EMFILE | libc::ENFILE | libc::ENOMEM) + ) + } + + #[cfg(not(unix))] + { + false + } +} + +/// Which second look a verdict calls for, or `None` for one that is already evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Suspicion { + /// Retried under a budget several times larger. + Timeout, + + /// Retried under a looser silence budget. + Stall, + + /// Retried unchanged; the ceiling was inferred from accounting rather than watched. + Memory, +} + +impl Suspicion { + /// The second look this verdict calls for. + const fn of(verdict: &Verdict) -> Option { + match verdict { + Verdict::TimedOut => Some(Self::Timeout), + Verdict::Stalled(_) => Some(Self::Stall), + Verdict::MemoryLimit { .. } => Some(Self::Memory), + _evidence => None, + } + } + + /// The attempt that checks it. + fn checking(self, attempt: Attempt<'_>) -> Attempt<'_> { + match self { + Self::Timeout => attempt.lengthened(), + Self::Stall => attempt.patient(), + Self::Memory => attempt, + } + } +} + +/// How many confirmations one verdict may cost before it is taken as it stands. +/// +/// Two, because a confirmation can land in a *different* detection class than the one it was +/// checking — a lengthened re-run that hits the memory ceiling, a patient one that runs out of time +/// — and that class has then been observed exactly once, which is what the first confirmation +/// exists to refuse. Beyond two the classes are alternating rather than settling, and the run stops +/// paying for a second look it is not getting. +const CONFIRM_ROUNDS: usize = 2; + +/// Runs the class-specific confirmations until the verdict is evidence, agreed with, or out of rounds. +/// +/// A confirmation that reproduces the class it was checking is the agreement it asked for, and ends +/// the ladder — a third run would re-confirm something already seen twice. +/// +/// `run` performs one attempt; production hands it [`observe`]. +fn settle_suspicions(attempt: Attempt<'_>, mut run: impl FnMut(Attempt<'_>) -> Verdict) -> Verdict { + let mut verdict = run(attempt); + let mut checked = None; + + for _round in 0..CONFIRM_ROUNDS { + let Some(suspicion) = Suspicion::of(&verdict) else { + break; + }; + + if checked == Some(suspicion) { + break; + } + + checked = Some(suspicion); + verdict = run(suspicion.checking(attempt)); + } + + verdict +} + +/// Decides whether a failing test says anything about the mutant that was active when it failed. +/// +/// The question is settled the only way it can be: by running the same binary again with the +/// mutant switched off. A suite that fails either way was not detecting anything. +fn confirm_kill(work: &Workspace, binary: &TestBinary, attempt: Attempt<'_>, test: Option) -> Verdict { + // Nothing to exonerate. A run with no mutant active has already answered the question the + // confirmation would ask, and asking it again would only pay for the same answer. + if attempt.active.is_none() { + return Verdict::Failed(test); + } + + let exonerating_attempt = test.as_deref().map_or_else( + || attempt.exonerating(), + |name| Attempt { + active: None, + only: Only::One(name), + ..attempt + }, + ); + + match observe(work, binary, exonerating_attempt).verdict { + // The suite is green without the mutant and red with it. That is a detection, and the only + // observation in this match that is one. + Verdict::Passed => Verdict::Failed(test), + + // Neither a detection nor a gap in the tests. Crediting the mutant would let one + // unreliable test manufacture a kill for every mutant it happens to be run against; + // recording a survivor would send the reader to write an assertion for code an assertion + // already covers. The test that failed both ways travels with the verdict, because fixing + // it is the only thing anybody can do about this. + Verdict::Failed(also) => Verdict::Flaky(test.or(also)), + + // The confirmation could not be metered as this run was configured, so it decided nothing. + // The reason travels rather than being replaced by a verdict nobody established. + Verdict::Unmetered(reason) => Verdict::Unmetered(reason), + + // The same, one scope down: the machinery refused this one run, so the exoneration + // established nothing and the mutant goes unjudged rather than taking the sweep with it. + Verdict::Unjudged(reason) => Verdict::Unjudged(reason), + + // The exoneration exceeded a budget instead of finishing, so the suite was never observed + // green without the mutant — and the one thing this must not do is read "not observed" as + // "observed green". The exoneration inherits the mutant run's budgets while running the + // whole binary, where the mutant's own run stopped at its first failing test, so an + // overrun here is as much a fact about the machine as about anything. + // + // Recorded as a flake rather than as an unmetered run: nothing about the mutant was + // established, but the machinery worked, and abandoning the whole sweep over a slow + // confirmation would be a far worse answer than leaving one mutant out of the score. The + // failing test travels for the same reason it does above. + // + // The remaining variants cannot arrive from an exoneration — `TestEnumerationFailed` is + // only settled with a mutant active, and `Flaky` is reached from here rather than from + // `observe` — and they are folded in rather than matched separately because the conclusion + // would be the same either way: nothing was established. + _unestablished => Verdict::Flaky(test), + } +} + +/// Confirms that nextest's inability to enumerate tests was caused by the active mutant. +fn confirm_enumeration(work: &Workspace, binary: &TestBinary, attempt: Attempt<'_>, output: String) -> Verdict { + match observe(work, binary, attempt.exonerating()).verdict { + Verdict::Passed => Verdict::TestEnumerationFailed(output), + Verdict::Failed(test) => Verdict::Flaky(test), + Verdict::Unmetered(reason) => Verdict::Unmetered(reason), + Verdict::Unjudged(reason) => Verdict::Unjudged(reason), + + // Whatever the exoneration ran into, it ran into it with no mutant active, so it is not a + // verdict about the mutant and must not be handed back as one: `judge` scores a `TimedOut` + // or a `MemoryLimit` as a detection, which would credit a mutant that was switched off in + // the run that produced it. Nothing was established, and that is what this says. + _unestablished => Verdict::Flaky(None), + } +} + +/// Says on the diagnostic stream that a verdict was reached on less output than the binary produced. +/// +/// Worth saying because the shortfall is invisible in the verdict itself: a truncated stream that +/// lost the failure announcement is indistinguishable from a suite that failed without naming a +/// test, and both come out as a kill with no killer. A reader chasing an unnamed kill would +/// otherwise spend the search on the suite rather than on the pipe. +fn announce_partial(binary: &TestBinary) { + announce(&format!( + "the output of `{}` could not be read to the end, so the failing test may be named in text \ + this run never saw. The exit status still decided the verdict.", + binary.path + )); +} + +/// Writes one diagnostic line, for the running command to say through its own `Host`. +/// +/// Raised as a note rather than written here: this is reached on a worker thread while the progress +/// display owns the terminal, so writing now would cut across a line somebody else is drawing. +fn announce(message: &str) { + crate::notes::note(message.to_owned()); +} + +/// How much more room a suspected timeout or stall is given before it is believed. +/// +/// Large enough that scheduling noise cannot survive it, and paid only by mutants that already +/// exhausted their budget — a small population, since a genuine hang is rare and a false one rarer +/// still. +pub(crate) const CONFIRM_FACTOR: u32 = 3; + +/// Builds the command that runs one test binary, under whichever runner this run selected. +/// +/// The two differ in more than the executable. Run directly, a binary takes the test arguments on +/// its own command line and has to be started in its package's directory, because that is where +/// `cargo test` would have started it. Run under nextest, both are nextest's business: it forwards +/// the arguments after `--` and sets each test's working directory to its own package root, so +/// imposing one here would make it resolve the whole workspace relative to a single package. +fn launcher(work: &Workspace, binary: &TestBinary, only: Only<'_>) -> Result { + let Some(harness) = work.runner() else { + let mut command = Command::new(binary.path.as_std_path()); + let names = only.names(); + + if names.is_empty() { + // The whole binary runs, so the user's arguments are the whole selection and go through + // exactly as written. + let _ = command.args(work.test_arguments()); + } else { + // libtest matches a test that any *one* positional filter matches, so appending the + // name this run chose to the user's own filters would widen the set rather than narrow + // it — and a mutant could be convicted by a test the user deliberately excluded. The + // intersection libtest cannot express is computed here instead: the user's filters + // decide which of the chosen names may run, and only the survivors are passed. + let user = HarnessFilters::parse(work.test_arguments()); + let allowed: Vec<&str> = names.into_iter().filter(|name| user.admits(name)).collect(); + + if allowed.is_empty() { + return Err(format!( + "the test selection this run made for `{}` names nothing the harness filters allow, \ + so it was refused rather than run as a wider selection", + binary.path + )); + } + + // The user's positional filters are not repeated here: `admits` has already applied + // them, and leaving them out is what lets `--exact` pin this run's own names without + // also silently converting the user's substring filter into a whole-name match. + let _ = command.args(user.flags()); + let _ = command.args(allowed).arg("--exact"); + } + + let _ = command.current_dir(working_directory(work, binary).as_std_path()); + + return Ok(command); + }; + + harness.command(work, binary, &only.names()).map_err(|cause| cause.to_string()) +} + +/// Turns a runner's non-zero exit into a verdict about the mutant. +/// +/// A binary run directly says only that it failed, and libtest names the first failing test in its +/// output. Nextest distinguishes far more, and the distinction matters: a code saying the tests ran +/// and one failed convicts the mutant, whereas a code saying nextest matched no tests or could not +/// start is a fact about this run. Crediting the suite with a kill it never made would inflate the +/// score, so anything nextest does not describe as a test failure stops the run instead. +fn settle(under_nextest: bool, active: Option, code: Option, text: &[u8], usage: MemoryUsage) -> (Verdict, MemoryUsage) { + if text + .windows(gamma_rt::ENVIRONMENT_ERROR_MARKER.len()) + .any(|window| window == gamma_rt::ENVIRONMENT_ERROR_MARKER) + { + return ( + Verdict::Unmetered("the guard runtime could not acquire the process startup environment".to_owned()), + usage, + ); + } + + let output = String::from_utf8_lossy(text); + + if !under_nextest { + return (Verdict::Failed(first_failure(&output).map(str::to_owned)), usage); + } + + match code { + Some(nextest::TEST_RUN_FAILED) => (Verdict::Failed(nextest::first_failure(&output).map(str::to_owned)), usage), + + // Nothing ran, so nothing decided anything. This is the filterset and the built tree + // disagreeing about what exists, which is a fault in the run rather than in the mutant. + Some(nextest::NO_TESTS_RUN) => ( + Verdict::Unmetered("`cargo nextest` matched no tests for a binary this run built".to_owned()), + usage, + ), + + Some(nextest::TEST_LIST_CREATION_FAILED) if active.is_some() => (Verdict::TestEnumerationFailed(output.trim().to_owned()), usage), + + // A signal, or a code nextest uses for its own failures. Either way it is not a verdict. + other => (Verdict::Unmetered(nextest_runner_failure(other, &output)), usage), + } +} + +/// Describes a nextest infrastructure failure without discarding its diagnostic output. +fn nextest_runner_failure(code: Option, output: &str) -> String { + let mut reason = format!( + "`cargo nextest` exited with {}, which does not describe a test run", + code.map_or_else(|| "a signal".to_owned(), |code| format!("code {code}")) + ); + let useful = output.trim(); + + if !useful.is_empty() { + reason.push_str(":\n"); + reason.push_str(&tail(useful, 20)); + } + + reason +} + +/// Sets everything a test binary is run with, beyond the command itself. +/// +/// Split out from [`run_with`] because it is a long, linear list of decisions that each carry +/// their own reason, and reading the run's control flow around them is otherwise impossible. +fn configure( + command: &mut Command, + binary: &TestBinary, + launch: &Launch, + threads: Option<&str>, + active: Option, + census: Option<&Utf8Path>, +) { + let _ = command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + // A mutant that panics in a thousand tests would otherwise spend the whole budget + // formatting backtraces nobody reads. + .env("RUST_BACKTRACE", "0") + // Every mutation site becomes a branch holding both the original and the replacement, so + // an instrumented frame is larger than the one it stands in for. Deeply recursive code + // that fits comfortably otherwise can exhaust a default 2 MiB test thread and abort the + // process, which reads as a failure of the suite rather than of the mutant. Raising the + // floor buys back the headroom instrumentation spent; an explicit setting still wins. + .env(STACK_VAR, launch.stack.as_str()) + // Set for the baseline as well as for every mutant, so a test can branch on it without + // knowing which phase it is in. A suite that shells out to cargo itself — this one + // included — would otherwise run a nested build inside the scratch tree, failing for + // reasons that have nothing to do with any mutant. + .env(UNDER_GAMMA_VAR, "1") + // Insta's own defaults are safe, but these are inherited, and a developer part-way + // through `cargo insta review` has them set. Accepting a snapshot would let a mutant that + // changed the snapshotted value pass, reporting it missed. One scratch tree serves many + // mutants, so a single accepted write would go on judging every mutant after it against a + // snapshot a mutant wrote. + .env(INSTA_UPDATE_VAR, "no") + .env(INSTA_FORCE_PASS_VAR, "0"); + + // Set per command rather than on this process, which the suite calls `run` from many threads + // of at once. `None` means the caller already chose a width and it is left alone. + if let Some(threads) = threads { + let _ = command.env(TEST_THREADS_VAR, threads); + } + + // Cargo sets this for every test it runs, and `env!("CARGO_MANIFEST_DIR")` is the usual way to + // reach a fixture from a test. Left unset, the macro's compile-time value points into the + // scratch tree's original location rather than the copy being tested. + if !binary.manifest_dir.as_str().is_empty() { + let _ = command.env("CARGO_MANIFEST_DIR", binary.manifest_dir.as_std_path()); + } + + match active { + Some(ordinal) => { + let _ = command.env(gamma_rt::ACTIVE_VAR, ordinal.to_string()); + } + None => { + let _ = command.env_remove(gamma_rt::ACTIVE_VAR); + } + } + + // Removed rather than left alone on an ordinary run, because it is inherited: a census that + // crashed leaving the variable set in a developer's shell would otherwise turn every later run + // into one that activates no mutant and reports the whole population as surviving. + match census { + Some(path) => { + let _ = command.env(gamma_rt::CENSUS_VAR, path.as_std_path()); + } + None => { + let _ = command.env_remove(gamma_rt::CENSUS_VAR); + } + } +} + +/// Runs one binary, publishing the harness's progress into `progress` as it goes. +fn run_with(work: &Workspace, binary: &TestBinary, attempt: Attempt<'_>, progress: &Arc>) -> (Verdict, MemoryUsage) { + let (active, timeout, stall, request) = (attempt.active, attempt.timeout, attempt.stall, attempt.request); + + let mut command = match launcher(work, binary, attempt.only) { + Ok(command) => command, + Err(reason) => return (Verdict::Unmetered(reason), MemoryUsage::default()), + }; + + let launch = work.launch(); + + configure_loader(&mut command, launch); + + // Nextest reports on stderr: the per-test `FAIL` lines that name what convicted a mutant, and + // the progress that tells the stall detector the child is alive. Left discarded, every nextest + // failure would be anonymous and a healthy run would look silent. A binary run directly says + // all of that on stdout, and its stderr carries only panic noise that no verdict is read from. + let under_nextest = work.runner().is_some(); + + configure(&mut command, binary, launch, work.harness_threads(), active, attempt.census); + + let (child, guard) = match spawn_patiently(&mut command, request) { + Ok(started) => started, + + // A spawn fails for reasons of the machine — descriptors, processes, address space, a + // binary being written as it is read — and none of them is a test failing. Reporting one as + // a failing suite would credit the tests with a kill they never made, which is the one + // direction this tool must not be wrong in. + // + // Nor is it a fact about the run. `spawn_patiently` has already waited out the transient + // shapes, so what is left is this one mutant's run being impossible right now, and the + // sweep records that against the mutant and carries on. + // + // The accounting the run asked for decides only how the refusal reads, not what it means. + // With a boundary installed the spawn is the step that puts the child inside it, so saying + // so is the more useful sentence; with none installed — every host without a delegated + // cgroup, and every run given `--memory off` — there is no boundary to blame and the + // kernel's own reason is the whole of what is known. + Err(StartError::Containment(reason)) => return (Verdict::Unmetered(reason), MemoryUsage::default()), + Err(StartError::Spawn(cause)) => { + let reason = if request.wanted() { + format!( + "`{}` could not be started inside its memory accounting boundary: {cause}", + binary.path + ) + } else { + format!("`{}` could not be started: {cause}", binary.path) + }; + + return (Verdict::Unjudged(reason), MemoryUsage::default()); + } + }; + + // Taken before anything is read from the child, so that the window in which a grandchild could + // start outside the containment is as short as it can be made without owning the spawn. + let mut subtree = match ProcessTree::adopt(child, guard) { + Ok(subtree) => subtree, + + // The child is already live, so this is one run that cannot be accounted for rather than a + // boundary the host will never provide — `prepare` above answers that question, and it + // succeeded. + Err(reason) => { + return (Verdict::Unjudged(reason), MemoryUsage::default()); + } + }; + + let pulse = Arc::new(Pulse::default()); + let drained = readers(&mut subtree, progress, &pulse); + + let deadline = timeout.map(deadline_after); + + loop { + // Read before the child is checked, so a signal raised while this iteration is working is + // not slept through. + let seen = pulse.seen(); + + match ask_after(&mut subtree) { + Ok(Some(status)) => { + // `ProcessTree::observe` used a non-reaping observation, swept descendants, released + // the interrupt slot, and only then reaped the leader. + debug_assert!(subtree.released(), "the containment is released before the output is drained"); + + let (text, whole) = collected(&drained, DRAIN_GRACE); + let usage = subtree.usage(); + let ceiling = request.limit.filter(|_limit| exhausted(&usage, status.success())); + + if let Some(verdict) = environment_verdict(progress) { + return (verdict, usage); + } + + if status.success() { + return (Verdict::Passed, usage); + } + + // Said before the text is read rather than after, because what follows draws a + // conclusion from an absence — no announced failure means the harness named no + // test — and that reading is only sound if the text is all of it. The run is not + // abandoned over it: the exit status still says the suite failed, which is the + // verdict, and only the name is at risk. + if !whole { + announce_partial(binary); + } + + // Asked before the ceiling is turned into a verdict, because the harness's own + // report is the more useful of the two: a named test is somewhere a reader can go, + // and a memory verdict that pre-empted it would leave them with a number and no + // idea which of the suite's tests to open. + let (verdict, usage) = settle(under_nextest, active, status.code(), &text, usage); + + return (prefer_named(verdict, usage.peak, ceiling), usage); + } + + Ok(None) => { + if let Some(verdict) = environment_verdict(progress) { + let (usage, _ceiling) = cut_short(&mut subtree, request); + + return (verdict, usage); + } + + // A direct libtest failure settles the verdict, so every test after it would be + // paid for to learn nothing. Nextest is different: it prints its FAIL line before + // replaying the failed child's captured output, which may hold the guard runtime's + // environment-error marker. Cutting there would convict a mutant before the + // evidence that the test never started was available. + if let Some(name) = failure_to_cut_short(under_nextest, progress) { + let (usage, ceiling) = cut_short(&mut subtree, request); + + return (cut_by_named_failure(name, usage.peak, ceiling), usage); + } + + let stalled = stall.exceeded(progress); + + if stalled || deadline.is_some_and(|deadline| Instant::now() >= deadline) { + let (usage, ceiling) = cut_short(&mut subtree, request); + + if let Some(verdict) = unfinished_nextest_failure(under_nextest, progress) { + return (verdict, usage); + } + + // No name was announced on this path, so there is nothing for the ceiling to + // outrank: a workload thrashing against its limit runs out of time as well, and + // the memory is the cause of both. See [`cut_short`]. + if let Some(limit) = ceiling { + return (Verdict::MemoryLimit { peak: usage.peak, limit }, usage); + } + + // The text is not read on this path, so there is nothing to wait for. + return if stalled { + (Verdict::Stalled(last_test(progress)), usage) + } else { + (Verdict::TimedOut, usage) + }; + } + + // Bounded by whichever comes first: the deadline, the earliest moment silence + // could be called a stall, and the backstop. Sleeping past any of the three would + // let the run overshoot a limit it promised to keep. + pulse.wait(seen, WAIT_CAP.min(deadline.map_or(WAIT_CAP, remaining)).min(stall.slack(progress))); + } + + // A non-reaping exit observation reports an error only when the child can no longer + // be observed. No host arranges that on request, so it is asked for through the fault + // seam instead — see `ask_after`. + // + // Nothing is known about the mutant at this point: the child is still running, and the + // one question that would have settled it cannot be asked again. Reporting a failing + // suite here would credit the tests with a kill they never made, so the run says it + // could not be measured instead, which is not a verdict about the mutant and is not + // scored as one. The child is not simply dropped either — everything it spawned is + // ended and reaped first, since an orphan holds scratch-tree locks and the pipes the + // readers below are waiting on, and both outlive this function into the next mutant. + Err(cause) => { + let (usage, _ceiling) = cut_short(&mut subtree, request); + + debug_assert!(subtree.released(), "the containment is released before the output is drained"); + + // Discarded, since there is no verdict to read out of it: the drain is here so the + // reader threads reach end of file and end, rather than being left holding pipes + // for the rest of the run. Whether it was complete is discarded with it — nothing + // was going to be concluded from the text either way. + let (_text, _whole) = collected(&drained, DRAIN_GRACE); + + return ( + Verdict::Unjudged(format!("`{}` could not be asked whether it had finished: {cause}", binary.path)), + usage, + ); + } + } + } +} + +fn deadline_after(timeout: Duration) -> Instant { + let now = Instant::now(); + let timeout = timeout.min(Duration::from_secs(crate::bounds::MOST_SECONDS)); + + now.checked_add(timeout).unwrap_or(now) +} + +/// Gathers whatever the readers have handed back, and whether it is all of it. +/// +/// Both streams are wanted and either may arrive first: the failure that names a test can be on one +/// while the other is still open, so this waits out the grace period rather than taking the first +/// chunk. A stream that ended in a read failure makes the whole collection partial, because there +/// is no way to tell which stream the missing text would have been on — the announcement that names +/// the killing test could have been on either. +/// +/// A reader that never returns is the same loss and is reported the same way. Only a disconnect +/// says every reader has sent and dropped its sender; running out of grace means one of them is +/// still holding a pipe open — a descendant that escaped the containment inherited the write end — +/// and its stream is missing entirely. Reading that as a complete collection would let a run whose +/// `FAIL` line never arrived convict a mutant with no killer named and warn about nothing. +fn collected(drained: &Receiver<(Vec, bool)>, grace: Duration) -> (Vec, bool) { + let mut text = Vec::new(); + let mut whole = true; + + loop { + match drained.recv_timeout(grace) { + Ok((chunk, complete)) => { + text.extend_from_slice(&chunk); + whole &= complete; + } + Err(RecvTimeoutError::Disconnected) => return (text, whole), + Err(RecvTimeoutError::Timeout) => return (text, false), + } + } +} + +/// Asks whether the child has finished, without giving cleanup a reused group id. +/// +/// Named rather than called inline only so that the error arm below it can be reached from a test. +/// Outside test builds, [`ProcessTree::observe`] performs Unix's non-reaping observation and owns the +/// sweep, release, and final reap as one lifecycle. +fn ask_after(subtree: &mut ProcessTree) -> io::Result> { + #[cfg(test)] + if faults::fired(Fault::Wait) { + return Err(io::Error::other("the wait a test asked to fail")); + } + + subtree.observe() +} + +/// Chooses between a failure the harness named and a ceiling the same run also crossed. +/// +/// Both can be true at once, and the name is the more useful of the two: it is somewhere a reader +/// can go, whereas a memory verdict leaves them with a number and no idea which test to open. The +/// named failure therefore takes precedence. +/// +/// With nothing named there is only the ceiling to report, and reporting it matters: without it the +/// run would say the suite failed, which sends a reader looking for an assertion that never was. +fn prefer_named(verdict: Verdict, peak: Option, ceiling: Option) -> Verdict { + let Some(limit) = ceiling else { + return verdict; + }; + + match verdict { + Verdict::Failed(Some(name)) => Verdict::Failed(Some(name)), + Verdict::Failed(None) => Verdict::MemoryLimit { peak, limit }, + + // Not a verdict about the mutant at all, and a ceiling does not make it one. + other => other, + } +} + +/// The verdict a run cut short by a failure the harness announced reaches. +/// +/// Stated once, and in the same terms as the ordinary exit path, because the two are the same run +/// seen from either side of one race: whether the reader publishes the failure before the exit +/// observation cleans up the child decides which of them is taken, and nothing orders those two events. A policy +/// spelled out twice would make the verdict a statement about that ordering — the name kept on a +/// machine where the reader wins and replaced by a byte count where the reaper does. Worse than +/// losing the name: only [`Verdict::Failed`] is routed to the flake check, so a flaky test failing +/// beside a ceiling would be credited as an unconfirmed detection on one machine and confirmed on +/// the other. +/// +/// [`cut_short`] has already ended the run by the time this is asked, so both facts are final. +fn cut_by_named_failure(name: String, peak: Option, ceiling: Option) -> Verdict { + prefer_named(Verdict::Failed(Some(name)), peak, ceiling) +} + +/// Ends a run whose verdict is already settled, and reports the ceiling when one fired. +/// +/// Everything the workload spawned goes with it. An orphan holds locks in the scratch tree, which +/// fails the next run, and an inherited pipe handle, which keeps whoever is reading this run's +/// output from ever seeing end of file. +/// +/// The ceiling comes back as the limit that fired rather than as a verdict, because what to make of +/// it differs by caller and only the caller knows which facts it also holds. A run cut short with a +/// test already named has two true facts and prefers the name; a run cut short by silence or by its +/// budget has only the ceiling, and the memory is the cause of the overrun rather than a second +/// symptom of it — reporting the stall instead would send the reader looking for a hang that is not +/// there. +fn cut_short(subtree: &mut ProcessTree, request: MemoryRequest) -> (MemoryUsage, Option) { + // Only Unix has a numeric watch slot that can be released too early. On Windows `released` + // necessarily returns true because the job handle itself remains the authority over the child. + #[cfg(unix)] + debug_assert!( + !subtree.released(), + "the subtree is signalled while it still holds its leader and its watch slot" + ); + + let _reaped = subtree.terminate(); + + let usage = subtree.usage(); + let ceiling = request.limit.filter(|_limit| exhausted(&usage, false)); + + (usage, ceiling) +} + +/// Whether a finished run should be read as having been stopped by its memory ceiling. +/// +/// The platform's own report is the whole of the authority: on Linux an `oom` or `oom_kill` event +/// recorded against this invocation's cgroup, on Windows the job's accounting reaching the limit +/// set for it. A peak that merely touched the ceiling says nothing on its own, because reclaim may +/// have succeeded — a workload that filled the page cache, had it reclaimed, and then failed for a +/// reason of its own would be convicted of running out of memory it never ran out of, and the +/// reader would be sent to raise a ceiling that was never the problem. +/// +/// A workload that succeeded is never convicted however close it came, since a suite that passed +/// detected nothing. +const fn exhausted(usage: &MemoryUsage, succeeded: bool) -> bool { + !succeeded && usage.exhausted +} + +/// How long to wait for the reader to finish once the child has exited. +/// +/// Normally the pipe reaches end of file the instant the child does, and the text is already in +/// hand. A wait this long is only ever reached when something the test spawned outlived it and +/// still holds the write end, in which case the text will never arrive and the alternative to +/// giving up is hanging. +const DRAIN_GRACE: Duration = Duration::from_secs(5); + +/// What one observed run of a test binary produced. +#[derive(Debug)] +pub(super) struct Observation { + pub(super) verdict: Verdict, + + /// The longest the harness went quiet, for calibrating later runs. + pub(super) quiet: Duration, + + /// How many tests the harness announced, or `None` if it announced nothing. + pub(super) tests: Option, + + /// The highest aggregate memory the subtree reached, when the run asked for a measurement and + /// the platform could supply one. + pub(super) peak: Option, +} + +/// The directory a test binary is launched from. +/// +/// `cargo test` runs each binary with the working directory set to its package root, and tests +/// rely on it: a fixture opened as `tests/data/input.json` resolves from there and nowhere else. In +/// a single-package workspace the two are the same directory, which is why running everything from +/// the workspace root works until the day someone adds a second crate — and then every test that +/// touches a file fails identically with and without a mutant active, so every mutant in that +/// package is scored as a survivor. +/// +/// Falls back to the workspace root when cargo did not say where the manifest was, which is the +/// behaviour that was always there. +fn working_directory<'work>(work: &'work Workspace, binary: &'work TestBinary) -> &'work Utf8Path { + if binary.manifest_dir.as_str().is_empty() { + &work.root + } else { + &binary.manifest_dir + } +} + +/// Runs one binary and reports what the harness said as well as how it ended. +pub(super) fn observe(work: &Workspace, binary: &TestBinary, attempt: Attempt<'_>) -> Observation { + let progress = Arc::new(Mutex::new(Progress::new(watch(work)))); + let (verdict, usage) = run_with(work, binary, attempt, &progress); + let quiet = quiet_of(&progress); + + #[expect(clippy::unwrap_used, reason = "the reader only panics if the whole process is unwinding")] + let tests = progress.lock().unwrap().tests; + + Observation { + verdict, + quiet, + tests, + peak: usage.peak, + } +} + +/// Whether this run's output can be trusted to announce a failure, and in whose format. +/// +/// A mutant is usually caught by one test, and reading the announcement as it arrives ends the run +/// there instead of paying for every test after it. The saving is the whole tail of the binary. +/// +/// It is only sound while the harness is the only thing writing to the stream being read. libtest +/// captures each test's output by default and replays it after every test has run, so a line +/// announcing a failure during the run is libtest's own. Under `--nocapture` or `--show-output` +/// that stops being true: a test's own writing lands among the harness's, and a test that prints +/// something shaped like a failure would convict a mutant the suite had not caught — inflating the +/// score, which is the worst direction to be wrong in. So the whole optimization is given up there. +fn watch(work: &Workspace) -> Watch { + if work.runner().is_some() { + return Watch::Nextest; + } + + let interleaved = work + .test_arguments() + .iter() + .any(|argument| argument == "--nocapture" || argument == "--show-output"); + + // The environment variable is the same setting by another name, and it is inherited. + if interleaved || std::env::var_os("RUST_TEST_NOCAPTURE").is_some() { + return Watch::Off; + } + + Watch::Libtest +} + +/// How much of a deadline is left, saturating at zero once it has passed. +fn remaining(deadline: Instant) -> Duration { + deadline.saturating_duration_since(Instant::now()) +} + +/// The longest the wait loop will sleep without being woken. +/// +/// A backstop, not the mechanism. Everything the loop reacts to signals it — a child that exits +/// closes its pipes, and a harness that announces a failure is read from them — so this bounds only +/// the case where a signal was never sent at all, and the case the loop must poll for because +/// nothing can announce it: silence. +const WAIT_CAP: Duration = Duration::from_millis(50); + +/// Returns a live failure only when its announcement cannot precede a startup-error marker. +fn failure_to_cut_short(under_nextest: bool, progress: &Mutex) -> Option { + (!under_nextest).then(|| announced_failure(progress)).flatten() +} + +/// The first failure announcement, without treating it as a completed verdict. +fn announced_failure(progress: &Mutex) -> Option { + #[expect(clippy::unwrap_used, reason = "the reader only panics if the whole process is unwinding")] + progress.lock().unwrap().failed.clone() +} + +/// Whether the runtime has independently disqualified this test process as evidence. +fn environment_failure(progress: &Mutex) -> bool { + #[expect(clippy::unwrap_used, reason = "the reader only panics if the whole process is unwinding")] + progress.lock().unwrap().environment_error +} + +fn environment_verdict(progress: &Mutex) -> Option { + environment_failure(progress) + .then(|| Verdict::Unmetered("the guard runtime could not acquire the process startup environment".to_owned())) +} + +fn unfinished_nextest_failure(under_nextest: bool, progress: &Mutex) -> Option { + (under_nextest && announced_failure(progress).is_some()).then(|| { + Verdict::Unmetered( + "`cargo nextest` announced a failure but did not finish before the run budget, \ + so its captured startup output could not be classified" + .to_owned(), + ) + }) +} + +/// The last test the harness named, if any. +fn last_test(progress: &Mutex) -> Option { + #[expect(clippy::unwrap_used, reason = "the reader only panics if the whole process is unwinding")] + progress.lock().unwrap().test.clone() +} + +/// The longest silence a binary produced, for calibrating later runs. +fn quiet_of(progress: &Mutex) -> Duration { + #[expect(clippy::unwrap_used, reason = "the reader only panics if the whole process is unwinding")] + let progress = progress.lock().unwrap(); + + progress.quiet.max(Instant::now().saturating_duration_since(progress.heard)) +} + +/// How much of a test binary's output is worth keeping. +/// +/// Only the first failure is ever read out of it, and libtest prints that early. A binary +/// producing more than this is extremely chatty or looping, and buffering it all would turn one +/// runaway mutant into an out-of-memory kill of the whole run. +const OUTPUT_CAP: usize = 4 * 1024 * 1024; + +/// Starts a reader for each of the child's piped streams, and yields what they collect. +/// +/// A pipe must be drained by somebody other than the thread waiting for the child to exit. A pipe +/// holds about 64 KB; a test binary that prints more blocks forever in `write` while the waiting +/// thread sees a process that never finishes — turning a mutant that should time out into one +/// recorded as timed out for the wrong reason, or the baseline into a false ten-minute stall. +/// +/// The readers hand their text back over a channel rather than through a join, because the child +/// exiting does not guarantee the pipes are closed: anything the test spawned inherited the write +/// ends, and a grandchild that outlives it holds them open. Every exit now sweeps the subtree +/// before draining, which closes those ends in the ordinary case — but a descendant that escaped +/// the containment entirely is still possible, so the wait stays bounded and the readers stay +/// abandonable rather than joined. +fn readers(subtree: &mut ProcessTree, progress: &Arc>, pulse: &Arc) -> Receiver<(Vec, bool)> { + let (sink, drained) = mpsc::channel::<(Vec, bool)>(); + + // Both streams are piped. Nextest reports everything on stderr, while a directly run binary + // ordinarily reports on stdout but may carry a runtime protocol marker on stderr. + let pipes = [subtree.take_stdout().map(Either::Out), subtree.take_stderr().map(Either::Err)]; + + for pipe in pipes.into_iter().flatten() { + let published = Arc::clone(progress); + let pulse = Arc::clone(pulse); + let sink = sink.clone(); + let failed = sink.clone(); + let failed_pulse = Arc::clone(&pulse); + + #[cfg(test)] + let refused = faults::fired(Fault::Thread); + #[cfg(not(test))] + let refused = false; + + let spawned = if refused { + Err(io::Error::other("the reader thread a test asked to fail")) + } else { + thread::Builder::new().name("cargo-gamma-output".to_owned()).spawn(move || { + READERS.started(); + + let collected = match pipe { + Either::Out(pipe) => drain(pipe, &published, &pulse), + Either::Err(pipe) => drain(pipe, &published, &pulse), + }; + + let _sent = sink.send(collected); + + // End of stream. A child that exits closes its pipes, so this is how the waiting thread + // learns the run is over without having to ask again on a timer. It is sent even when + // the stream ended for some other reason, because the waiter re-checks the child + // itself and a spurious wakeup costs one `try_wait`. + pulse.signal(); + + // Reached whether this reader was waited for or abandoned — an abandoned one runs on + // until its pipe finally closes, and this is where it stops being counted. + READERS.finished(); + }) + }; + + match spawned { + Ok(_handle) => {} + Err(_cause) => { + let _sent = failed.send((Vec::new(), false)); + failed_pulse.signal(); + } + } + } + + // Dropped so the receiver sees the readers' senders as the only ones left, and disconnects as + // soon as they finish rather than waiting out the grace period on a sender nobody is using. + drop(sink); + + drained +} + +/// One of the two streams a child may be read from, so both can be started by the same loop. +enum Either { + Out(ChildStdout), + Err(ChildStderr), +} + +/// Reads a child's output to exhaustion, keeping at most [`OUTPUT_CAP`] of it. +/// +/// Reading continues past the cap even though the excess is discarded: the point is to keep the +/// pipe empty so the child can run to completion, not to collect the text. +/// +/// The second half of the pair says whether the stream really ended. A failed read leaves a prefix +/// of what the binary said, and a prefix reads exactly like the whole of a quiet suite: the +/// first-failure scan finds no announcement and the mutant is recorded as killed by a test nobody +/// can name. This loop is the only place that can still tell the two apart, so it says so, and the +/// caller reports the gap rather than presenting the shortfall as the binary having stayed silent. +/// +/// An interrupted read is not truncation and does not reach the failing arm: `read_until` retries +/// `EINTR` itself, and nothing was taken out of the pipe when it fired. +fn drain(pipe: R, progress: &Mutex, pulse: &Pulse) -> (Vec, bool) { + use std::io::BufRead as _; + + let mut reader = BufReader::new(pipe); + let mut kept = Vec::new(); + let mut line = Vec::new(); + let mut whole = true; + + loop { + line.clear(); + + match reader.read_until(b'\n', &mut line) { + Ok(0) => return (kept, whole), + + // The partial line this read was building goes with the bytes it lost: half a line is + // not something the watcher should be shown, and it could as easily be half an + // announcement as half a progress bar. + Err(_truncated) => return (kept, false), + + Ok(_read) => { + // Published before the text is kept, so a binary past the cap still counts as + // making progress. Silence is the signal, not volume. + #[expect(clippy::unwrap_used, reason = "the watcher only panics if the process is unwinding")] + let decisive = { + let mut progress = progress.lock().unwrap(); + let before = progress.failed.is_some(); + let environment_error = progress.environment_error; + + progress.heard(&String::from_utf8_lossy(&line)); + + (!before && progress.failed.is_some()) || (!environment_error && progress.environment_error) + }; + + // Only evidence that can settle the run wakes the waiter. Waking on every line + // would turn a chatty binary into a spin. + if decisive { + pulse.signal(); + } + + let room = OUTPUT_CAP.saturating_sub(kept.len()); + + let kept_now = line.len().min(room); + kept.extend_from_slice(&line[..kept_now]); + + if kept_now < line.len() { + whole = false; + } + } + } + } +} + +/// Extracts the name of the first failing test from libtest's output. +/// +/// Borrowed from `output`, so that scanning costs nothing until a name is actually kept. +fn first_failure(output: &str) -> Option<&str> { + output.lines().find_map(|line| { + let trimmed = line.trim(); + let rest = trimmed.strip_prefix("test ")?; + let name = rest.strip_suffix(" ... FAILED")?; + + Some(name.trim()) + }) +} + +/// Returns the last `count` lines of some text. +pub(super) fn tail(text: &str, count: usize) -> String { + let lines: Vec<&str> = text.lines().collect(); + let start = lines.len().saturating_sub(count); + + lines.get(start..).unwrap_or_default().join("\n") +} + +#[cfg(all(test, not(loom), not(miri)))] +mod fuzz { + use super::super::progress::{Progress, Watch}; + use super::first_failure; + use crate::testing::{spliced, token}; + + /// A libtest failure announcement is found whatever surrounds it. + /// + /// The asymmetry is why this is fuzzed rather than sampled. A misparse that invents a failure + /// costs one survivor a second look and is visible in the report. A misparse that loses one + /// reports a mutant as killed by a test that never failed, which raises the score and says + /// nothing at all while doing it — so the direction worth spending machine time on is that no + /// input, however strange the surrounding output, makes an announced failure disappear. + #[test] + fn a_libtest_failure_is_never_lost_among_arbitrary_output() { + bolero::check!() + .with_type::<(Vec, String, usize)>() + .for_each(|(noise, name, at)| { + let output = spliced(noise, &format!("test {} ... FAILED", token(name)), *at); + + assert!(first_failure(&output).is_some(), "the failure was lost in {output:?}"); + }); + } + + /// Arbitrary output never panics the reader and never names an empty test. + /// + /// A test binary is an arbitrary program: it can print anything, including partial lines and + /// invalid-looking announcements, and a panic here would take down the run that was measuring + /// it rather than the mutant. + #[test] + fn arbitrary_output_is_read_without_panicking() { + bolero::check!().with_type::().for_each(|output| { + if let Some(name) = first_failure(output) { + assert!(!name.contains('\n'), "a test name spans lines: {name:?}"); + } + }); + } + + /// The streaming reader agrees with the batch reader that a failure happened. + /// + /// These two read the same harness's output for the same fact, one line at a time as it + /// arrives and one over the whole buffer at the end, and only the first can stop a binary + /// early. If they drift apart, a mutant is convicted by one and acquitted by the other + /// depending only on how the run happened to be watched, which is not a property of the code + /// under test at all. + /// + /// Only the canonical spelling is asserted on. The two genuinely differ on an indented + /// announcement — the batch reader trims the line first and the streaming one does not — and + /// libtest does not indent, so pinning that difference would be pinning an accident. + #[test] + fn the_streaming_reader_sees_the_failure_the_batch_reader_sees() { + bolero::check!() + .with_type::<(Vec, String, usize)>() + .for_each(|(noise, name, at)| { + let line = format!("test {} ... FAILED", token(name)); + let output = spliced(noise, &line, *at); + + let mut progress = Progress::new(Watch::Libtest); + + // The streaming reader ignores verdicts until a suite has announced itself, because a + // `harness = false` target prints whatever it likes. + progress.heard("running 1 test"); + + for line in output.lines() { + progress.heard(line); + } + + assert!(progress.failed.is_some(), "the streaming reader lost the failure in {output:?}"); + }); + } +} + +#[cfg(all(test, not(loom), not(miri)))] +mod tests { + use super::*; + + /// A pipe that fails part way through is not reported as a stream that simply ended. + /// + /// The two are indistinguishable downstream: a prefix that stops before the harness announced + /// anything looks exactly like a suite that failed without naming a test, and the verdict then + /// carries no killer at all. The reader is the last place that can still tell them apart, so it + /// is where the difference has to be recorded — and the bytes it did get are kept, because a + /// truncated failure list is still worth showing. + #[test] + fn a_pipe_that_fails_part_way_is_not_read_as_the_end_of_the_stream() { + /// Yields one line and then fails, the way a pipe whose writer died mid-stream does. + struct Faltering(bool); + + impl Read for Faltering { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.0 { + return Err(io::Error::other("the read a test asked to fail")); + } + + self.0 = true; + + let line = b"running 1 test\n"; + + buf[..line.len()].copy_from_slice(line); + + Ok(line.len()) + } + } + + let progress = Mutex::new(Progress::new(Watch::Libtest)); + let pulse = Pulse::default(); + + let (kept, whole) = drain(Faltering(false), &progress, &pulse); + + assert!(!whole, "a severed pipe was reported as the end of the stream"); + assert_eq!(String::from_utf8_lossy(&kept), "running 1 test\n"); + } + + #[test] + fn output_discarded_past_the_cap_marks_the_stream_incomplete() { + let mut output = vec![b'x'; OUTPUT_CAP]; + output.extend_from_slice(b"\ntest meaningful::killer ... FAILED\n"); + output.extend_from_slice(gamma_rt::ENVIRONMENT_ERROR_MARKER); + let progress = Mutex::new(Progress::new(Watch::Libtest)); + let pulse = Pulse::default(); + + let (kept, whole) = drain(io::Cursor::new(output), &progress, &pulse); + + assert_eq!(kept.len(), OUTPUT_CAP); + assert!(!whole, "discarding bytes past the cap was reported as a whole stream"); + assert!(environment_failure(&progress), "the marker remains visible outside the output cap"); + } + + /// A confirmation landing in a different detection class is itself confirmed. + /// + /// A lengthened re-run that hits the memory ceiling has been observed once, and one observation + /// of a detection is exactly what the confirmation exists to refuse: a machine loaded by the + /// sweep produces both classes on its own. Believing it credits the suite with a kill it never + /// made, which is the one direction this tool must not be wrong in. + #[test] + fn a_confirmation_that_crosses_into_another_detection_class_is_confirmed_in_turn() { + let mut issued = 0_usize; + + let verdict = settle_suspicions(attempt_for_confirmation(), |_attempt| { + issued += 1; + + match issued { + 1 => Verdict::TimedOut, + 2 => Verdict::MemoryLimit { peak: Some(9), limit: 8 }, + _settled => Verdict::Passed, + } + }); + + assert_eq!(issued, 3, "the memory verdict the confirmation produced was believed unchecked"); + assert_eq!(verdict, Verdict::Passed); + } + + /// A confirmation that reproduces the class it was checking is the agreement it asked for. + /// + /// Without this the ladder would pay for a third run of every genuine timeout in the population, + /// which is the most expensive verdict class there is. + #[test] + fn a_confirmation_that_agrees_ends_the_ladder() { + let mut issued = 0_usize; + + let verdict = settle_suspicions(attempt_for_confirmation(), |_attempt| { + issued += 1; + + Verdict::TimedOut + }); + + assert_eq!(issued, 2, "a reproduced timeout was asked about again"); + assert_eq!(verdict, Verdict::TimedOut); + } + + /// Alternating classes stop at the bound rather than running forever. + #[test] + fn a_verdict_that_never_settles_stops_at_the_bound() { + let mut issued = 0_usize; + + let _verdict = settle_suspicions(attempt_for_confirmation(), |_attempt| { + issued += 1; + + if issued.is_multiple_of(2) { + Verdict::MemoryLimit { peak: Some(9), limit: 8 } + } else { + Verdict::TimedOut + } + }); + + assert_eq!(issued, 1 + CONFIRM_ROUNDS, "the ladder is not bounded"); + } + + /// A verdict that is already evidence costs no confirmation here; `confirm_kill` owns that one. + #[test] + fn a_settled_verdict_is_not_re_run() { + let mut issued = 0_usize; + + let verdict = settle_suspicions(attempt_for_confirmation(), |_attempt| { + issued += 1; + + Verdict::Failed(Some("a::b".to_owned())) + }); + + assert_eq!(issued, 1); + assert_eq!(verdict, Verdict::Failed(Some("a::b".to_owned()))); + } + + /// The attempt the ladder tests are run against; none of them reads it. + fn attempt_for_confirmation() -> Attempt<'static> { + Attempt { + active: Some(1), + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + } + } + + /// A reader that never returns leaves the collection partial, exactly as a severed pipe does. + /// + /// A grandchild that escaped the containment inherited the write end of a pipe, so its reader + /// is still blocked when the grace runs out and one whole stream is missing. Reading that as + /// complete lets a run whose `FAIL` line was on the missing stream convict a mutant with no + /// killer named, and suppresses the warning that would have said so. + #[test] + fn a_reader_that_never_returns_leaves_the_collection_partial() { + let (sink, drained) = mpsc::channel::<(Vec, bool)>(); + + // One reader finished; the other is still holding the pipe open, so its sender never sends + // and never drops. + let _still_reading = sink.clone(); + let _sent = sink.send((b"running 1 test\n".to_vec(), true)); + + drop(sink); + + let (text, whole) = collected(&drained, Duration::from_millis(50)); + + assert!(!whole, "an absent reader was read as the end of the output"); + assert_eq!(String::from_utf8_lossy(&text), "running 1 test\n"); + } + + /// Every reader having sent and dropped its sender is the whole of the output, and the ordinary + /// case must not be reported as partial or every run would warn about nothing. + #[test] + fn readers_that_all_returned_leave_the_collection_whole() { + let (sink, drained) = mpsc::channel::<(Vec, bool)>(); + + let _sent = sink.send((b"running 1 test\n".to_vec(), true)); + let _also = sink.send((b"test a::b ... FAILED\n".to_vec(), true)); + + drop(sink); + + let (text, whole) = collected(&drained, Duration::from_secs(30)); + + assert!(whole, "a complete collection was reported as partial"); + assert!( + String::from_utf8_lossy(&text).contains("FAILED"), + "{:?}", + String::from_utf8_lossy(&text) + ); + } + + /// A read interrupted by a signal is retried rather than treated as truncation. + /// + /// `EINTR` has taken nothing out of the pipe, so the next read gets it, and the retry comes + /// from `read_until` rather than from this module. Pinned anyway because the truncation rule + /// above it is what makes the distinction matter: were the interruption to start counting as a + /// lost stream, every run the user so much as resizes a terminal during would report its + /// verdicts as reached on partial text. + #[test] + fn a_read_interrupted_by_a_signal_is_retried_rather_than_cut_short() { + /// Interrupts once between two lines, then ends the stream. + struct Interrupted(u8); + + impl Read for Interrupted { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.0 = self.0.saturating_add(1); + + let line: &[u8] = match self.0 { + 1 => b"running 1 test\n", + 2 => return Err(io::Error::from(io::ErrorKind::Interrupted)), + 3 => b"test a::b ... FAILED\n", + _ => return Ok(0), + }; + + buf[..line.len()].copy_from_slice(line); + + Ok(line.len()) + } + } + + let progress = Mutex::new(Progress::new(Watch::Libtest)); + let pulse = Pulse::default(); + + let (kept, whole) = drain(Interrupted(0), &progress, &pulse); + + assert!(whole, "an interrupted read was reported as truncation"); + assert!( + String::from_utf8_lossy(&kept).contains("test a::b ... FAILED"), + "the text after the interruption was lost: {:?}", + String::from_utf8_lossy(&kept) + ); + } + + /// A verdict reached on a truncated stream says so, since the shortfall is invisible in the + /// verdict itself. + /// + /// A kill with no killer is exactly what a lost announcement looks like, so without this a + /// reader would go looking through the suite for a failure that was never in the text they can + /// see. + #[test] + fn a_verdict_reached_on_a_truncated_stream_says_the_text_was_partial() { + crate::notes::alone(|| { + let binary = crate::testing::helper(); + + announce_partial(&binary); + + let raised = crate::notes::drain(); + + assert_eq!(raised.len(), 1, "{raised:?}"); + assert!(raised[0].contains(binary.path.as_str()), "the binary is not named: {raised:?}"); + }); + } + + /// A workspace whose test binary is the portable helper, running the given script. + /// + /// Portable rather than a shell script because everything below is a statement about the + /// verdict machinery rather than about Unix, and a fixture that only exists on one platform + /// leaves the machinery unproven — and uncompiled — on the other. + fn scripted(script: &[&str]) -> (tempfile::TempDir, Workspace) { + crate::testing::helper_workspace("verdict", script) + } + + /// A workspace whose test binaries are `/bin/sh`, running the given script. + /// + /// Kept for the one test that needs a survivor whose process id is written down for the test to + /// check afterwards, which the portable helper deliberately does not offer. + #[cfg(unix)] + fn shell(body: &str) -> (tempfile::TempDir, Workspace) { + crate::testing::shell_workspace("verdict", body) + } + + /// A test binary exiting cleanly says nothing about what it left running. + /// + /// This is the guarantee the containment machinery exists for. A survivor holds ports and + /// locks in the scratch tree, and the mutant that inherits them is convicted of a failure the + /// mutant before it caused — which makes a score depend on the order the run happened to + /// choose. It also holds the write end of a pipe this run is reading, stranding the reader. + #[test] + #[cfg(unix)] + fn a_binary_that_exits_normally_still_has_its_survivors_killed() { + // Run under the watchdog: everything here waits on a child, and a child that never reports + // would otherwise stop the suite instead of failing this test. The budget is not the + // convergence deadline below — that one asks how long a `SIGKILL` may take to land, which + // is a real question with a real answer; this one only asks whether anything is moving. + let alive = crate::testing::within(crate::testing::WATCHDOG, "sweeping a leaked grandchild", || { + let marker = std::env::temp_dir().join(format!("gamma-verdict-survivor.{}", std::process::id())); + let _removed = std::fs::remove_file(&marker); + let (_directory, work) = shell(&format!("sleep 300 & echo $! > {}; exit 0", marker.display())); + let leaky = crate::testing::test_binary("/bin/sh"); + + let verdict = run_with( + &work, + &leaky, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + &Arc::new(Mutex::new(Progress::new(Watch::Libtest))), + ) + .0; + + // The sweep is housekeeping, not judgement: the tests passed, so the mutant survived. + assert!(matches!(verdict, Verdict::Passed), "{verdict:?}"); + + let recorded = std::fs::read_to_string(&marker).expect("the fixture records its background child"); + let pid: i32 = recorded.trim().parse().expect("the fixture writes a pid"); + let _removed = std::fs::remove_file(&marker); + + // `SIGKILL` is delivered rather than awaited, so the process is gone shortly after + // `run_with` returns rather than exactly when it does. + let deadline = Instant::now() + Duration::from_secs(10); + + loop { + if !cargo_gamma_unsafe::group::exists(pid) { + break false; + } + + if Instant::now() >= deadline { + break true; + } + + thread::sleep(Duration::from_millis(20)); + } + }); + + assert!(!alive, "the grandchild outlived the test binary that spawned it"); + } + + /// A binary that cannot be started said nothing about the mutant, whatever the run asked for. + /// + /// A spawn fails for reasons of the machine — descriptors, processes, address space — and none + /// of them is a test failing. Reading one as a failing suite credits the tests with a kill they + /// never made, and the confirmation pass does not shield it: a transient refusal clears on the + /// second attempt and is then believed. The reason travels so the run can say what happened + /// rather than reporting a mutant killed by a test it cannot name. + #[test] + fn a_binary_that_cannot_be_spawned_is_unjudged_rather_than_a_kill() { + let (_directory, work) = scripted(&["exit:0"]); + let missing = crate::testing::test_binary(work.root.join("no-such-binary").as_str()); + + // Asked for with no accounting at all, which is the state every host without a delegated + // cgroup is in, and the state `--memory off` chooses. + let verdict = run_binary( + &work, + &missing, + Attempt { + active: None, + timeout: Some(Duration::from_secs(1)), + stall: Stall::NONE, + request: MemoryRequest { meter: false, limit: None }, + only: Only::All, + census: None, + }, + true, + ); + + match verdict { + Verdict::Unjudged(reason) => assert!(reason.contains("no-such-binary"), "{reason}"), + other => panic!("expected an unjudged run, got {other:?}"), + } + } + + /// A binary that cannot be spawned inside a memory accounting boundary this run asked for is + /// reported as unjudged, not as a plain test failure. + /// + /// A run that asked to be protected has to know the protection never took effect, or a mutant + /// would be scored by a spawn failure that has nothing to do with anything it changed. + #[test] + fn a_binary_that_cannot_be_spawned_inside_its_memory_boundary_is_unjudged() { + if crate::testing::without_memory_support("a run reporting the memory a mutant used") { + return; + } + + let (_directory, work) = scripted(&["exit:0"]); + let missing = crate::testing::test_binary(work.root.join("no-such-binary").as_str()); + + let verdict = run_binary( + &work, + &missing, + Attempt { + active: None, + timeout: Some(Duration::from_secs(1)), + stall: Stall::NONE, + request: MemoryRequest { meter: true, limit: None }, + only: Only::All, + census: None, + }, + true, + ); + + assert!(matches!(verdict, Verdict::Unjudged(_)), "{verdict:?}"); + } + + /// A spawn the machine refuses for want of a resource is tried again, and the run goes ahead. + /// + /// A sweep runs `jobs` subprocesses at once, each with pipes of its own, so meeting a full + /// descriptor or process table is an ordinary consequence of the workload and clears as soon as + /// one of the other workers finishes. Giving up at the first refusal would leave a mutant + /// unjudged for a shortage that lasted milliseconds. + #[test] + fn a_spawn_the_machine_refuses_is_tried_again() { + let (_directory, work) = scripted(&["exit:0"]); + let binary = crate::testing::helper(); + let _refused = faults::arm(Fault::Spawn); + + let verdict = run_binary(&work, &binary, plain_attempt(), true); + + assert_eq!(verdict, Verdict::Passed, "the retry must let the run go ahead"); + } + + /// A machine that refuses every attempt leaves the one mutant unjudged, not the run abandoned. + /// + /// The distinction is the whole point of the separate verdict: an hours-long sweep that meets a + /// persistent shortage on one mutant keeps every verdict it has already reached and records why + /// that one has none, rather than discarding the lot. + #[test] + fn a_spawn_refused_every_time_leaves_the_mutant_unjudged() { + let (_directory, work) = scripted(&["exit:0"]); + let binary = crate::testing::helper(); + let _refusals: Vec<_> = (0..SPAWN_ATTEMPTS).map(|_round| faults::arm(Fault::Spawn)).collect(); + + let verdict = run_binary(&work, &binary, plain_attempt(), true); + + match verdict { + Verdict::Unjudged(reason) => assert!(reason.contains("could not be started"), "{reason}"), + other => panic!("expected an unjudged run, got {other:?}"), + } + } + + #[cfg(unix)] + #[test] + fn spawn_backoff_closes_the_interrupt_window_before_sleeping() { + let (_directory, work) = scripted(&["exit:0"]); + let binary = crate::testing::helper(); + let _refused = faults::arm(Fault::Spawn); + let _interrupted = process_faults::arm_late(ProcessFault::Window, Duration::from_millis(10)); + let started = Instant::now(); + + let verdict = run_binary(&work, &binary, plain_attempt(), true); + + assert!(matches!(verdict, Verdict::Unmetered(_)), "{verdict:?}"); + assert!(started.elapsed() < SPAWN_BACKOFF.saturating_mul(2), "{:?}", started.elapsed()); + } + + #[test] + fn reader_thread_creation_failure_is_partial_without_panicking() { + crate::notes::alone(|| { + let (_directory, work) = scripted(&["exit:1"]); + let binary = crate::testing::helper(); + let _refused = faults::arm(Fault::Thread); + + let verdict = run_binary(&work, &binary, plain_attempt(), true); + + assert!(matches!(verdict, Verdict::Failed(None) | Verdict::Flaky(None)), "{verdict:?}"); + assert!(!crate::notes::drain().is_empty(), "partial output was not diagnosed"); + }); + } + + /// Only the refusals the machine recovers from on its own are waited out. + /// + /// A binary that is not there is not there however long the wait, and retrying it would multiply + /// the delay on the one shape of failure that is certain to be permanent. + #[test] + fn only_a_transient_refusal_is_waited_out() { + assert!(transient(&io::Error::from(io::ErrorKind::WouldBlock))); + assert!(transient(&io::Error::from(io::ErrorKind::Interrupted))); + assert!(!transient(&io::Error::from(io::ErrorKind::NotFound))); + assert!(!transient(&io::Error::from(io::ErrorKind::PermissionDenied))); + + #[cfg(unix)] + { + assert!(transient(&io::Error::from_raw_os_error(libc::EMFILE))); + assert!(transient(&io::Error::from_raw_os_error(libc::ENFILE))); + assert!(transient(&io::Error::from_raw_os_error(libc::ENOMEM))); + assert!(!transient(&io::Error::from_raw_os_error(libc::ENOEXEC))); + } + } + + /// An attempt asking for nothing beyond running the binary, which the spawn tests all share. + fn plain_attempt() -> Attempt<'static> { + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest { meter: false, limit: None }, + only: Only::All, + census: None, + } + } + + /// A host that cannot install the memory accounting a metered run asked for says so before + /// anything is spawned, rather than running the binary unprotected. + /// + /// Spawning anyway would mean every verdict that follows is compared against a run the ceiling + /// never actually watched, and a mutant that genuinely exhausted memory would simply be scored + /// as an ordinary pass or failure with nothing left to explain why. + #[test] + fn an_undelegated_host_says_what_is_missing_before_anything_is_spawned() { + if super::super::memory::support().is_ok() { + return; + } + + let (_directory, work) = scripted(&["exit:0"]); + let ok = crate::testing::helper(); + + let verdict = run_binary( + &work, + &ok, + Attempt { + active: None, + timeout: Some(Duration::from_secs(1)), + stall: Stall::NONE, + request: MemoryRequest { meter: true, limit: None }, + only: Only::All, + census: None, + }, + true, + ); + + match verdict { + Verdict::Unmetered(reason) => assert!(!reason.is_empty(), "a refusal has to say why"), + other => panic!("expected an unmetered refusal, got {other:?}"), + } + } + + /// A run whose containment cannot be installed says so, and never spawns the binary. + /// + /// The host here can install it perfectly well; the refusal is asked for through the fault seam, + /// because the real causes — a cgroup controller that is not delegated, a job object the system + /// will not make — need a differently configured machine rather than a differently written test. + #[test] + fn containment_that_cannot_be_installed_is_reported_rather_than_run_without() { + let (_directory, work) = scripted(&["exit:0"]); + let ok = crate::testing::helper(); + let _armed = process_faults::arm(ProcessFault::Prepare); + + let observed = observe( + &work, + &ok, + Attempt { + active: None, + timeout: Some(Duration::from_secs(10)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + ); + + // Not `Passed`: the binary exits zero, so a run that spawned it anyway would say so. + match observed.verdict { + Verdict::Unmetered(reason) => assert!(!reason.is_empty(), "a refusal has to say why"), + other => panic!("expected an unmetered refusal, got {other:?}"), + } + } + + /// A child that cannot be moved into its accounting boundary is killed, not left running. + /// + /// The window this covers is the one the containment exists to close. The child is already + /// alive when the move is refused, so leaving it would mean a test binary running outside every + /// bound the run believes it is inside — and, on Windows, one suspended forever waiting for a + /// job it never entered. Nothing but a seam can ask for this: it needs a cgroup removed, or a + /// job object refused, between the spawn and the move. + #[test] + fn a_child_that_cannot_be_adopted_is_ended_rather_than_left_outside_the_boundary() { + crate::testing::within(crate::testing::WATCHDOG, "ending an unadoptable child", || { + let started = crate::testing::workdir("verdict-adopt"); + let root = camino::Utf8PathBuf::from_path_buf(started.path().to_path_buf()).expect("a UTF-8 scratch path"); + let marker = root.join("ran"); + + // Long enough that a child left alive would still be alive when this test looks, and + // short enough that it cannot outlast the watchdog if the kill never happens. + let (_directory, work) = + crate::testing::helper_workspace("verdict-adopt", &["sleep:250", &format!("touch:{marker}"), "exit:0"]); + let ok = crate::testing::helper(); + let _armed = process_faults::arm(ProcessFault::Adopt); + + let observed = observe( + &work, + &ok, + Attempt { + active: None, + timeout: Some(Duration::from_secs(10)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + ); + + match observed.verdict { + Verdict::Unjudged(reason) => assert!(!reason.is_empty(), "a refusal has to say why"), + other => panic!("expected an unjudged refusal, got {other:?}"), + } + + // Well past the sleep the child would have finished, had it been left to run. + thread::sleep(Duration::from_millis(1500)); + + assert!( + !marker.as_std_path().exists(), + "a child that could not be adopted has to be ended, not left running outside the boundary" + ); + }); + } + + /// A child this run cannot ask after is a mutant it cannot judge, not one the suite caught. + /// + /// `waitpid` fails only when the handle itself is gone, which for a child this process spawned + /// and has not reaped no host will arrange. What matters is which way the run is wrong when it + /// happens: a failing verdict here is read as a detection all the way out to the score, so the + /// suite would be credited with a kill it never made on the strength of a broken wait. + #[test] + fn a_child_that_cannot_be_asked_after_is_unjudged_rather_than_a_failure() { + crate::testing::within(crate::testing::WATCHDOG, "a wait that fails", || { + let (_directory, work) = scripted(&["sleep:200", "exit:0"]); + let ok = crate::testing::helper(); + let _armed = faults::arm(Fault::Wait); + + let observed = observe( + &work, + &ok, + Attempt { + active: None, + timeout: Some(Duration::from_secs(10)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + ); + + // Neither `Passed`, which is what the binary would otherwise have earned, nor a + // failure, which is what a killed mutant looks like, nor a panic. + match observed.verdict { + Verdict::Unjudged(reason) => assert!(reason.contains("finished"), "{reason}"), + other => panic!("expected an unjudged run, got {other:?}"), + } + }); + } + + /// A wait that fails still takes everything the child spawned with it. + /// + /// The arm has no verdict to report and nothing further to ask the child, and the temptation is + /// to return from it — leaving a live process group holding scratch-tree locks, the write ends + /// of the pipes this run's readers are blocked on, and whatever memory it was using, all of it + /// into the next mutant's run. The fault is delayed rather than immediate so that the child has + /// reached the point of having a descendant at all; without that this test would be watching an + /// empty subtree and would pass whatever the arm did. + #[test] + #[cfg(unix)] + fn a_wait_that_fails_still_takes_the_subtree_with_it() { + crate::testing::within(crate::testing::WATCHDOG, "a wait that fails mid-run", || { + let started = crate::testing::workdir("verdict-wait"); + let root = camino::Utf8PathBuf::from_path_buf(started.path().to_path_buf()).expect("a UTF-8 scratch path"); + let (running, survived) = (root.join("running"), root.join("survived")); + + // The descendant announces itself, waits out anything the kill has to do, and only then + // writes the file this test is looking for. A subtree that was killed cannot reach the + // second write; one that was merely abandoned does, and says so. + let (_directory, work) = scripted(&[&format!("spawn:touch:{running}|sleep:2000|touch:{survived}"), "sleep:30000"]); + let ok = crate::testing::helper(); + let _armed = faults::arm_late(Fault::Wait, Duration::from_millis(750)); + + let observed = observe( + &work, + &ok, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + ); + + match observed.verdict { + Verdict::Unjudged(reason) => assert!(!reason.is_empty(), "an unjudged run has to say why"), + other => panic!("expected an unjudged run, got {other:?}"), + } + + assert!( + running.as_std_path().exists(), + "the descendant never started, so this test proves nothing about killing it" + ); + + // Past the descendant's sleep, so a survivor has had every chance to write. + thread::sleep(Duration::from_millis(2500)); + + assert!( + !survived.as_std_path().exists(), + "a wait that failed has to take the whole subtree with it, not just the child" + ); + }); + } + + #[test] + fn an_unrepresentable_deadline_is_clamped_without_panicking() { + let before = Instant::now(); + let deadline = deadline_after(Duration::MAX); + let latest = Instant::now() + .checked_add(Duration::from_secs(crate::bounds::MOST_SECONDS)) + .expect("one year is a representable deadline"); + + assert!(deadline >= before); + assert!(deadline <= latest); + } + + /// A binary that outlives its budget is timed out, and the verdict survives confirmation. + #[test] + fn a_binary_that_outlives_its_budget_times_out() { + let (_directory, work) = scripted(&["sleep:30000"]); + let sleeper = crate::testing::helper(); + + // A genuine hang has to survive the confirmation run as well as the first one, so this + // exercises both passes: the suspicion and the finding. + assert_eq!( + run_binary( + &work, + &sleeper, + Attempt { + active: None, + timeout: Some(Duration::from_millis(50)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::TimedOut + ); + } + + /// A binary that goes quiet for longer than its stall budget is cut off early. + #[test] + fn a_binary_that_goes_quiet_is_stalled_at_the_last_test_it_named() { + let (_directory, work) = scripted(&["print:test slow::case ... ", "sleep:30000"]); + let hanger = crate::testing::helper(); + + // The point of stall detection is to cut a hang off long before the full budget, and to + // say which test was running when the silence began. + let verdict = run_binary( + &work, + &hanger, + Attempt { + active: None, + timeout: Some(Duration::from_mins(1)), + stall: Stall { + budget: Some(Duration::from_millis(50)), + }, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true, + ); + + assert!(matches!(verdict, Verdict::Stalled(_)), "{verdict:?}"); + } + + #[test] + fn startup_silence_measured_by_the_baseline_does_not_stall_the_same_binary() { + let (_directory, work) = + crate::testing::helper_workspace("verdict-startup-silence-", &["sleep:100", "print:running 0 tests", "exit:0"]); + let binary = crate::testing::helper(); + let baseline = observe( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_secs(5)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + ); + + let measured = baseline.quiet; + let observed = observe( + &work, + &binary, + Attempt { + active: Some(1), + timeout: Some(Duration::from_secs(5)), + stall: Stall { + budget: Some(measured.saturating_mul(2)), + }, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + ); + + assert!(measured >= Duration::from_millis(80), "{measured:?}"); + assert_eq!(observed.verdict, Verdict::Passed); + } + + /// A binary runs from its own package's root, the way cargo would run it. + #[test] + fn a_binary_runs_from_its_package_root() { + // A test that opens a fixture by relative path only finds it from the package root. Run + // from the workspace root it fails identically with and without a mutant active, so every + // mutant in the package is scored as a survivor. + let (_directory, work) = scripted(&["require-file:marker.txt"]); + let package = work.root.join("crates").join("subject"); + + std::fs::create_dir_all(package.as_std_path()).expect("the package directory is created"); + std::fs::write(package.join("marker.txt").as_std_path(), "here").expect("the marker is written"); + + let binary = TestBinary { + manifest_dir: package, + ..crate::testing::helper() + }; + + assert_eq!( + run_binary( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Passed + ); + } + + /// A binary that exits cleanly passes, and its output is read back. + #[test] + fn a_binary_that_exits_cleanly_passes() { + let (_directory, work) = scripted(&["print:test a::b ... ok", "exit:0"]); + let ok = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &ok, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Passed + ); + } + + /// A mutant caught by an early test is convicted there, not after every later test has run. + /// + /// This is the whole value of watching the output: the binary below would take half a minute to + /// finish, and the verdict is settled a fraction of a second in. + #[test] + fn a_failure_convicts_a_mutant_without_waiting_for_the_rest_of_the_binary() { + let (_directory, work) = scripted(&["print:running 2 tests", "print:test a::b ... FAILED", "sleep:30000", "exit:101"]); + let binary = crate::testing::helper(); + let started = Instant::now(); + + let verdict = run_binary( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_mins(2)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true, + ); + let took = started.elapsed(); + + assert_eq!(verdict, Verdict::Failed(Some("a::b".to_owned()))); + assert!(took < Duration::from_secs(15), "the run waited for the whole binary: {took:?}"); + } + + /// The name reported is the one the run would have reported had it read the binary to the end. + #[test] + fn the_test_named_is_the_first_that_failed_and_not_a_later_one() { + let (_directory, work) = scripted(&[ + "print:running 3 tests", + "print:test a::first ... FAILED", + "print:test a::second ... FAILED", + "exit:101", + ]); + let binary = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Failed(Some("a::first".to_owned())) + ); + } + + /// A target built with `harness = false` prints whatever it likes, and a line of its own that + /// happens to look like libtest's must not be read as a verdict it never gave. + #[test] + fn a_harness_that_announced_no_suite_is_judged_only_by_how_it_exits() { + let (_directory, work) = scripted(&["print:test a::b ... FAILED", "exit:0"]); + let binary = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Passed + ); + } + + /// With capture turned off, a test's own writing reaches the same stream as the harness's, and + /// a test that prints something shaped like a failure would otherwise convict a mutant the + /// suite had not caught — reporting it caught and inflating the score. + #[test] + fn a_run_with_capture_turned_off_takes_no_verdict_from_the_output() { + let (_directory, work) = scripted(&["--nocapture", "print:running 1 test", "print:test a::b ... FAILED", "exit:0"]); + let binary = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Passed + ); + } + + /// Insta is told not to write snapshots and not to pass on a mismatch, whatever the ambient + /// environment asked for. + /// + /// Both are inherited, and a developer part-way through `cargo insta review` has them set. A + /// snapshot accepted under a mutant lets the mutant pass and be reported missed, and the + /// scratch tree serving every later mutant would carry the accepted snapshot with it. + #[test] + fn insta_is_told_not_to_accept_what_a_mutant_produced() { + let (_directory, work) = scripted(&["write-env:seen.txt|INSTA_UPDATE|INSTA_FORCE_PASS", "exit:0"]); + let binary = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Passed + ); + + let seen = std::fs::read_to_string(work.root.join("seen.txt").as_std_path()).expect("the child recorded what it saw"); + + assert_eq!(seen, "no 0"); + } + + /// A binary that prints past the cap on what is kept is still drained and finishes cleanly, + /// rather than the excess being buffered until the run itself runs out of memory. + /// + /// A runaway mutant that loops printing has to be readable to the point where the cap stops + /// keeping it, but the pipe still has to be drained past that point or the child blocks in + /// `write` forever — turning a mutant that should be judged on its own merits into a hang this + /// run has to time out instead, and a machine slowly filling with buffered text nobody reads. + #[test] + fn a_binary_that_prints_past_the_pipe_cap_is_still_drained_to_completion() { + let (_directory, work) = scripted(&["flood:5000000", "print:test a::b ... ok", "exit:0"]); + let chatty = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &chatty, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Passed + ); + } + + /// A binary that exits non-zero fails, named by libtest's own report. + #[test] + fn a_failing_binary_is_named_by_its_first_failing_test() { + // Failing only while a mutant is active is what a detection looks like, and it is what the + // kill confirmation is there to establish: a binary that failed either way would be flaky. + let (_directory, work) = scripted(&[ + "when-env:GAMMA_ACTIVE|print:test a::b ... FAILED", + "when-env:GAMMA_ACTIVE|exit:101", + "print:test a::b ... ok", + "exit:0", + ]); + let bad = crate::testing::helper(); + + // The name comes out of the harness's own output rather than the exit status, which is + // what makes a survivor report actionable. + assert_eq!( + run_binary( + &work, + &bad, + Attempt { + active: Some(7), + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Failed(Some("a::b".to_owned())) + ); + } + + /// Asserts the memory verdict carries the figures the report shows the user. + /// + /// The variant alone is a weak oracle: `MemoryLimit { peak: None, limit: 0 }` satisfies it + /// while telling the reader nothing about how far past the ceiling the mutant ran, which is + /// the whole diagnostic. The peak is checked as a fraction of the ceiling rather than as + /// something exceeding it, because both enforcement mechanisms cap usage *at* the ceiling — + /// the kernel kills rather than letting the number climb past it — so a peak strictly greater + /// than the limit is not a thing either platform can report. + fn assert_memory_verdict_reports_its_figures(verdict: &Verdict, expected: u64) { + let Verdict::MemoryLimit { peak, limit } = verdict else { + panic!("expected a memory verdict, got {verdict:?}"); + }; + + assert_eq!(*limit, expected, "the verdict must carry the ceiling that fired: {verdict:?}"); + + let peak = peak.expect("the platform reports a peak when it enforces a ceiling"); + + assert!( + peak >= expected / 2, + "a peak of {peak} against a ceiling of {expected} is not the figure that crossed it", + ); + } + + /// A binary that allocates past its ceiling is a memory verdict, not a failing test. + #[test] + fn a_binary_that_passes_its_ceiling_is_a_memory_verdict() { + if crate::testing::without_memory_support("a mutant stopped for passing its ceiling") { + return; + } + + // Anonymous pages the helper touches itself, rather than a shell writing a file. Page cache + // backed by a disk is reclaimable, so a file-writing workload stays under the ceiling + // indefinitely instead of crossing it; and the helper runs on every platform this tool + // builds for, which is where the other half of the enforcement code lives. + let (_directory, work) = crate::testing::helper_workspace("verdict-memory", &["eat:512", "exit:0"]); + let greedy = crate::testing::helper(); + let limit = 32 * 1024 * 1024; + + let verdict = run_binary( + &work, + &greedy, + Attempt { + active: Some(1), + timeout: Some(Duration::from_mins(1)), + stall: Stall::NONE, + request: MemoryRequest { + meter: true, + limit: Some(limit), + }, + only: Only::All, + census: None, + }, + true, + ); + + // Reporting this as a plain failure would be defensible and wrong: the suite noticed + // nothing, the kernel did, and only the kernel's report distinguishes it from the + // ordinary case of a test that exits non-zero. + assert_memory_verdict_reports_its_figures(&verdict, limit); + } + + /// A binary that stalls silently while also allocating past its ceiling is convicted of the + /// memory it used, not merely reported as a stall. + /// + /// A workload thrashing against its ceiling runs out of time as well as out of memory, and the + /// memory is the cause: reporting the stall instead would send whoever reads it looking for a + /// hang that genuinely is not there, and cost a whole confirmation run reproducing an + /// allocation that was already caught the first time. A long stall budget is used deliberately: + /// the ceiling has unambiguously already been crossed by the time silence is judged a stall, so + /// this test's own outcome does not depend on how fast the kernel's OOM path happens to run on + /// whatever machine executes it. + #[test] + fn a_binary_that_stalls_while_exhausting_its_ceiling_is_convicted_of_the_memory() { + if crate::testing::without_memory_support("a stalling mutant convicted of its memory") { + return; + } + + let (_directory, work) = crate::testing::helper_workspace("verdict-memory-stall", &["eat:1024", "sleep:30000"]); + let greedy = crate::testing::helper(); + let limit = 32 * 1024 * 1024; + + let verdict = run_with( + &work, + &greedy, + Attempt { + active: Some(1), + timeout: Some(Duration::from_mins(1)), + stall: Stall { + budget: Some(Duration::from_secs(2)), + }, + request: MemoryRequest { + meter: true, + limit: Some(limit), + }, + only: Only::All, + census: None, + }, + &Arc::new(Mutex::new(Progress::new(Watch::Libtest))), + ) + .0; + + assert_memory_verdict_reports_its_figures(&verdict, limit); + } + + /// A binary that stays under its ceiling is judged by its tests, not by its allocations. + #[test] + fn a_binary_that_stays_under_its_ceiling_is_judged_normally() { + if crate::testing::without_memory_support("a ceiling reported with the figure that crossed it") { + return; + } + + let (_directory, work) = scripted(&["print:test a::b ... ok", "exit:0"]); + let modest = crate::testing::helper(); + + // The expensive mistake in the other direction: a ceiling that convicts a healthy mutant + // credits the suite with a kill it never made and inflates the score. + assert_eq!( + run_binary( + &work, + &modest, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest { + meter: true, + limit: Some(512 * 1024 * 1024) + }, + only: Only::All, + census: None, + }, + true + ), + Verdict::Passed + ); + } + + /// A run that reached its ceiling and still passed is not convicted by the peak alone. + #[test] + fn a_successful_run_is_never_convicted_by_its_peak() { + // A peak can sit exactly at the ceiling because reclaim did its job. The suite passed; the + // mutant was not caught by anything, and saying otherwise would be a detection invented + // out of an accounting figure. + let usage = MemoryUsage { + peak: Some(1024), + exhausted: false, + }; + + assert!(!exhausted(&usage, true)); + } + + /// A workload that reached its ceiling, reclaimed, and then failed for a reason of its own is + /// not convicted of running out of memory. + /// + /// Regression, issue-023. A peak at the ceiling only says the workload touched it, and touching + /// it is survivable: `memory.max` reclaims first and kills only when reclaim cannot keep up. + /// Reading the peak as a verdict turns any failure that happened to follow a busy moment into + /// "killed by the ceiling", and sends the reader off to raise a limit that was never the + /// problem while the real cause goes unreported. + #[test] + fn a_workload_that_reclaimed_and_then_failed_is_not_a_memory_verdict() { + let reclaimed = MemoryUsage { + peak: Some(1024), + exhausted: false, + }; + + assert!(!exhausted(&reclaimed, false)); + } + + /// The platform's own report convicts even when the peak reads below the ceiling. + #[test] + fn a_kernel_reported_kill_is_believed_whatever_the_peak_says() { + // `memory.peak` is a high-water mark sampled by the kernel and an OOM kill can free the + // charge before it is read, so the event is the authority and the peak is the detail. + let usage = MemoryUsage { + peak: Some(1), + exhausted: true, + }; + + assert!(exhausted(&usage, false)); + assert!(!exhausted(&usage, true)); + } + + /// A failure the harness named survives a ceiling the same run also crossed. + /// + /// Taking the memory verdict before the output is read at all would leave a run that both + /// failed a test and crossed its ceiling reporting a number and no name — leaving whoever read + /// it with nothing to open. Both facts are true; only one of them is somewhere a reader can go. + #[test] + fn a_named_failure_outranks_the_ceiling_the_same_run_crossed() { + let named = prefer_named( + Verdict::Failed(Some("module::case".to_owned())), + Some(300 * 1024 * 1024), + Some(256 * 1024 * 1024), + ); + + assert_eq!(named, Verdict::Failed(Some("module::case".to_owned()))); + + // With nothing named there is only the ceiling to report, and reporting it is what keeps + // the reader from hunting for an assertion that never existed. + let anonymous = prefer_named(Verdict::Failed(None), Some(300), Some(256)); + + assert_eq!( + anonymous, + Verdict::MemoryLimit { + peak: Some(300), + limit: 256 + } + ); + + // A run that could not be metered is not a verdict about the mutant, and a ceiling does + // not turn it into one. + let unmetered = prefer_named(Verdict::Unmetered("no cgroup".to_owned()), None, Some(256)); + + assert!(matches!(unmetered, Verdict::Unmetered(_)), "{unmetered:?}"); + + // With no ceiling crossed, whatever was settled stands untouched. + let plain = prefer_named(Verdict::Failed(Some("module::case".to_owned())), Some(1), None); + + assert_eq!(plain, Verdict::Failed(Some("module::case".to_owned()))); + } + + /// A run cut short by a named failure keeps the name, exactly as the ordinary exit does. + /// + /// The two exits carry the same pair of facts, and which one a run takes is decided by nothing + /// but whether the reader published the failure before `try_wait` reaped the child. An early + /// cut that preferred the ceiling would drop the name a reader goes and opens, discard the + /// killer hint the next mutant would have probed with, and — because only `Failed` is routed to + /// `confirm_kill` — skip the flake check for exactly the runs that raced the other way. + /// + /// Asserted here rather than through a launched binary because the state cannot be arranged on + /// Linux: the ceiling sets `memory.oom.group`, so an OOM takes the whole subtree with it and + /// there is no announcing process left to reach the early cut. Windows, where a job object + /// refuses allocations rather than killing, is where the run really survives its ceiling. + #[test] + fn a_run_cut_short_by_a_named_failure_keeps_the_name_beside_the_ceiling() { + let named = cut_by_named_failure("module::case".to_owned(), Some(300), Some(256)); + + assert_eq!(named, Verdict::Failed(Some("module::case".to_owned()))); + + // With no ceiling crossed there is only the failure, which is the ordinary shape of this. + let plain = cut_by_named_failure("module::case".to_owned(), None, None); + + assert_eq!(plain, Verdict::Failed(Some("module::case".to_owned()))); + } + + /// A named failure that arrives before an allocation ceiling is exceeded prefers the named + /// failure end-to-end on Windows. + /// + /// On Windows, the job object enforces memory limits without killing the process immediately + /// with SIGKILL/cgroup kill, allowing the named failure streaming reader to report the killing + /// test name alongside the memory ceiling. + #[test] + #[cfg(windows)] + fn a_named_failure_survives_crossing_a_memory_ceiling_end_to_end() { + if crate::testing::without_memory_support("a named failure surviving a memory ceiling") { + return; + } + + let (_directory, work) = crate::testing::helper_workspace( + "verdict-named-memory", + &[ + "print:running 1 test", + "print:test failing::case ... FAILED", + "eat:512", + "sleep:30000", + "exit:101", + ], + ); + let binary = crate::testing::helper(); + let limit = 32 * 1024 * 1024; + + let verdict = run_binary( + &work, + &binary, + Attempt { + active: None, + timeout: Some(Duration::from_mins(1)), + stall: Stall::NONE, + request: MemoryRequest { + meter: true, + limit: Some(limit), + }, + only: Only::All, + census: None, + }, + false, + ); + + assert_eq!(verdict, Verdict::Failed(Some("failing::case".to_owned()))); + } + + /// An exoneration isolates the failing test rather than running the whole binary to completion, + /// so a slow tail in the rest of the suite does not cause the confirmation to time out and + /// misclassify the detection as flaky. + #[test] + fn an_exoneration_runs_only_the_failing_test_avoiding_suite_tail_timeouts() { + let (_directory, work) = scripted(&[ + "when-env:GAMMA_ACTIVE|print:test first ... FAILED", + "when-env:GAMMA_ACTIVE|exit:101", + "when-arg:first|exit:0", + "sleep:30000", + ]); + let binary = crate::testing::helper(); + + let verdict = run_binary( + &work, + &binary, + Attempt { + active: Some(7), + timeout: Some(Duration::from_millis(500)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true, + ); + + assert_eq!(verdict, Verdict::Failed(Some("first".to_owned()))); + } + + /// An exoneration that ran out of time did not establish that the suite is green without the + /// mutant, so it does not confirm the kill. + /// + /// The confirmation asks one question — does the suite still fail with the mutant switched off + /// — and a run that exceeded its budget answered neither way. Reading it as "green without, + /// red with" credits the suite with a detection on the strength of a scheduling hiccup, and it + /// does so in the score-inflating direction, which is the one this tool must not be wrong in. + #[test] + fn an_exoneration_that_ran_out_of_time_does_not_confirm_a_kill() { + // Fails only while a mutant is active; with none it runs long enough to outlive any budget + // a test would give it, which is the shape the exoneration has to cope with. + let (_directory, work) = scripted(&["when-env:GAMMA_ACTIVE|exit:101", "sleep:30000"]); + let binary = crate::testing::helper(); + + let verdict = confirm_kill( + &work, + &binary, + Attempt { + active: Some(7), + timeout: Some(Duration::from_millis(200)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + Some("a::b".to_owned()), + ); + + // Neither a kill nor a survivor, and the test that failed travels with it: that name is the + // only thing anybody can act on here. + assert_eq!(verdict, Verdict::Flaky(Some("a::b".to_owned())), "{verdict:?}"); + } + + /// The exonerating run's own overrun is never handed back as the mutant's verdict. + /// + /// The exoneration runs with no mutant active, so a timeout, a stall or a ceiling it hits is a + /// fact about that run. Returning it would score the mutant `Outcome::Timeout` or + /// `Outcome::OutOfMemory` — a detection credited to a mutant that was not even switched on in + /// the run that produced the verdict. + #[test] + fn an_exonerations_own_overrun_is_not_adopted_as_the_mutants_verdict() { + let (_directory, work) = scripted(&["when-env:GAMMA_ACTIVE|exit:101", "sleep:30000"]); + let binary = crate::testing::helper(); + + let verdict = confirm_enumeration( + &work, + &binary, + Attempt { + active: Some(7), + timeout: Some(Duration::from_millis(200)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + "nextest could not list the tests".to_owned(), + ); + + assert_eq!(verdict, Verdict::Flaky(None), "{verdict:?}"); + } + + /// A test that fails with the mutant and fails again without it is flaky, not a detection. + /// + /// Scoring a failing suite as a kill with nothing at all having established that the mutant + /// caused it would let one unreliable test manufacture a kill for every mutant it was run + /// against, and inflate the score by however many that was. + #[test] + fn a_test_that_fails_without_the_mutant_too_is_not_a_kill() { + let (_directory, work) = scripted(&["print:running 1 test", "print:test a::b ... FAILED", "exit:101"]); + let flaky = crate::testing::helper(); + + // Neither a kill nor a survivor: something failed, but not because of the mutant, and the + // mutant is what was being judged. The test that failed both ways travels with the verdict + // because fixing it is the only remedy anybody has here. + assert_eq!( + run_binary( + &work, + &flaky, + Attempt { + active: Some(7), + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Flaky(Some("a::b".to_owned())) + ); + } + + /// Turning the confirmation off buys back the second run and gives up telling the two apart. + /// + /// The same binary that is flaky above is scored as a detection here, which is exactly the cost + /// of the flag: nothing established that the mutant caused the failure, and without the + /// confirming run nothing could have. + #[test] + fn a_run_that_declines_to_confirm_believes_the_failing_test() { + let (_directory, work) = scripted(&["print:running 1 test", "print:test a::b ... FAILED", "exit:101"]); + let flaky = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &flaky, + Attempt { + active: Some(7), + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + false + ), + Verdict::Failed(Some("a::b".to_owned())) + ); + } + + /// The confirmation is skipped when there was no mutant to exonerate. + /// + /// A run with nothing active has already answered the question the confirmation would ask, so + /// asking it again would buy the same answer at the price of a second run of the whole binary. + #[test] + fn a_failure_with_no_mutant_active_is_reported_as_it_stands() { + let (_directory, work) = scripted(&["print:test a::b ... FAILED", "exit:101"]); + let failing = crate::testing::helper(); + + assert_eq!( + run_binary( + &work, + &failing, + Attempt { + active: None, + timeout: Some(Duration::from_secs(30)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + true + ), + Verdict::Failed(Some("a::b".to_owned())) + ); + } + + #[test] + fn the_tail_keeps_the_last_lines() { + assert_eq!(tail("a\nb\nc\nd", 2), "c\nd"); + assert_eq!(tail("a\nb", 10), "a\nb"); + assert_eq!(tail("", 3), ""); + } + + #[test] + fn the_first_libtest_failure_name_is_extracted() { + let output = "running 2 tests\n\ + test passing ... ok\n\ + test module::case ... FAILED\n\ + test later ... FAILED\n"; + + // Only the first failing test is reported with a killed mutant; later failures may be + // consequences of the same defect and add noise. + assert_eq!(first_failure(output), Some("module::case")); + assert_eq!(first_failure("custom harness output"), None); + } + + /// Only nextest's own code for "the tests ran and one failed" convicts a mutant. + #[test] + fn nextest_convicts_a_mutant_only_on_a_real_test_failure() { + let output = b" FAIL [ 0.024s] (1/2) nxspike tests::fails_when_asked\n"; + let (verdict, _usage) = settle(true, Some(7), Some(nextest::TEST_RUN_FAILED), output, MemoryUsage::default()); + + assert_eq!(verdict, Verdict::Failed(Some("tests::fails_when_asked".to_owned()))); + } + + #[test] + fn nextest_test_enumeration_failure_is_suspect_only_with_a_mutant_active() { + let output = b"error: test binary exited while listing tests"; + let (suspect, _usage) = settle( + true, + Some(7), + Some(nextest::TEST_LIST_CREATION_FAILED), + output, + MemoryUsage::default(), + ); + let (baseline, _usage) = settle(true, None, Some(nextest::TEST_LIST_CREATION_FAILED), output, MemoryUsage::default()); + + assert_eq!( + suspect, + Verdict::TestEnumerationFailed("error: test binary exited while listing tests".to_owned()) + ); + assert!( + matches!(baseline, Verdict::Unmetered(ref reason) if reason.contains("code 104") && reason.contains("exited while listing")), + "{baseline:?}" + ); + } + + /// Nextest matching no tests means the filterset and the built tree disagree about what exists. + /// Scoring that as a kill would credit the suite with one it never made, so the run stops. + #[test] + fn nextest_matching_no_tests_abandons_the_run_rather_than_scoring_it() { + let (verdict, _usage) = settle(true, Some(7), Some(nextest::NO_TESTS_RUN), b"", MemoryUsage::default()); + + assert!(matches!(verdict, Verdict::Unmetered(_)), "{verdict:?}"); + } + + /// Any other code is nextest reporting its own failure — a bad filterset, an unreadable tree, a + /// signal. None of those is a statement about the mutant. + #[test] + fn a_code_nextest_uses_for_its_own_failures_is_not_a_verdict() { + for code in [Some(94), Some(101), None] { + let (verdict, _usage) = settle(true, Some(7), code, b"", MemoryUsage::default()); + + assert!(matches!(verdict, Verdict::Unmetered(_)), "{code:?} gave {verdict:?}"); + } + } + + /// The same codes mean nothing when the binaries are run directly: a binary is free to exit + /// with any of them, and all a non-zero exit says is that the suite failed. + #[test] + fn a_directly_run_binary_is_judged_by_its_output_and_not_its_code() { + let output = b"test module::case ... FAILED\n"; + + for code in [Some(nextest::NO_TESTS_RUN), Some(101), None] { + let (verdict, _usage) = settle(false, Some(7), code, output, MemoryUsage::default()); + + assert_eq!(verdict, Verdict::Failed(Some("module::case".to_owned())), "{code:?}"); + } + } + + #[test] + fn an_environment_acquisition_failure_never_convicts_a_mutant() { + let mut output = b"test module::case ... FAILED\n".to_vec(); + output.extend_from_slice(gamma_rt::ENVIRONMENT_ERROR_MARKER); + + for under_nextest in [false, true] { + let (verdict, _usage) = settle( + under_nextest, + Some(7), + Some(nextest::TEST_RUN_FAILED), + &output, + MemoryUsage::default(), + ); + + assert!( + matches!(verdict, Verdict::Unmetered(ref reason) if reason.contains("startup environment")), + "{verdict:?}" + ); + } + } + + /// Nextest announces `FAIL` before replaying the failed process's captured output. The runtime + /// marker therefore arrives too late for a live-output shortcut but in time for `settle`. + #[test] + fn a_nextest_failure_waits_for_a_later_environment_error_marker() { + let progress = Mutex::new(Progress::new(Watch::Nextest)); + progress + .lock() + .expect("the progress lock is not poisoned") + .heard(" FAIL [ 0.024s] (1/1) nxspike tests::case\n"); + + assert_eq!(announced_failure(&progress).as_deref(), Some("tests::case")); + assert_eq!(failure_to_cut_short(true, &progress), None); + assert!(matches!(unfinished_nextest_failure(true, &progress), Some(Verdict::Unmetered(_)))); + + let mut output = b" FAIL [ 0.024s] (1/1) nxspike tests::case\n".to_vec(); + output.extend_from_slice(gamma_rt::ENVIRONMENT_ERROR_MARKER); + progress + .lock() + .expect("the progress lock is not poisoned") + .heard(core::str::from_utf8(gamma_rt::ENVIRONMENT_ERROR_MARKER).expect("the marker is ASCII")); + + assert!(environment_failure(&progress)); + assert_eq!(failure_to_cut_short(true, &progress), None); + + let direct = Mutex::new(Progress::new(Watch::Libtest)); + { + let mut state = direct.lock().expect("the progress lock is not poisoned"); + state.heard("running 1 test\n"); + state.heard("test tests::case ... FAILED\n"); + } + assert_eq!(failure_to_cut_short(false, &direct).as_deref(), Some("tests::case")); + + let (verdict, _usage) = settle(true, Some(7), Some(nextest::TEST_RUN_FAILED), &output, MemoryUsage::default()); + + assert!( + matches!(verdict, Verdict::Unmetered(ref reason) if reason.contains("startup environment")), + "{verdict:?}" + ); + } + + /// Nextest's output is its own format, so a run through it is watched for nextest's failure + /// lines rather than libtest's — and the `--nocapture` question does not arise, because nextest + /// captures per test regardless of what the tests are handed. + #[test] + fn a_run_through_nextest_is_watched_in_nextest_format() { + let (_scratch, mut work) = crate::testing::helper_workspace("watch-nextest", &["exit:0"]); + + work.set_test_args(vec!["--nocapture".to_owned()]); + work.set_runner(nextest::Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")])); + + assert_eq!(watch(&work), Watch::Nextest); + } + + /// A plain libtest run captures each test's output and replays it only after every test has + /// run, so a failure line seen during the run is the harness's own and can be trusted. + #[test] + fn a_plain_libtest_run_is_watched_in_libtest_format() { + let (_scratch, mut work) = crate::testing::helper_workspace("watch-libtest", &["exit:0"]); + + work.set_test_args(Vec::new()); + + assert_eq!(watch(&work), Watch::Libtest); + } + + /// Under `--nocapture` or `--show-output` a test's own writing lands among the harness's, so a + /// test that printed something shaped like a failure would convict a mutant the suite never + /// caught. The optimization is given up rather than risk inflating the score. + #[test] + fn interleaved_output_gives_up_watching_entirely() { + for argument in ["--nocapture", "--show-output"] { + let (_scratch, mut work) = crate::testing::helper_workspace("watch-off", &["exit:0"]); + + work.set_test_args(vec![argument.to_owned()]); + + assert_eq!(watch(&work), Watch::Off, "{argument}"); + } + } + + /// A binary the runner does not know cannot be launched, and the reason has to survive as the + /// unmetered verdict's text — a mutant recorded as caught here would be one no test ever ran + /// against. + #[test] + fn a_binary_the_runner_does_not_know_yields_no_command() { + let (_scratch, mut work) = crate::testing::helper_workspace("launch-stranger", &["exit:0"]); + + work.set_runner(nextest::Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")])); + + let reason = launcher(&work, &crate::testing::test_binary("/t/deps/stranger-def"), Only::All) + .expect_err("an unknown binary cannot be launched"); + + assert!(reason.contains("/t/deps/stranger-def"), "{reason}"); + } + + /// A run that cannot even be launched is unmetered, not a kill. Recording it as a kill would + /// credit the suite with catching a mutant that no test was ever run against. + #[test] + fn a_run_that_cannot_be_launched_is_unmetered_rather_than_a_kill() { + let (_directory, mut work) = scripted(&["exit:0"]); + + work.set_runner(nextest::Harness::fake(&[("/t/deps/nxspike-abc", "nxspike")])); + + let verdict = run_with( + &work, + &crate::testing::test_binary("/t/deps/stranger-def"), + Attempt { + active: None, + timeout: Some(Duration::from_mins(1)), + stall: Stall { budget: None }, + request: MemoryRequest { meter: false, limit: None }, + only: Only::All, + census: None, + }, + &Arc::new(Mutex::new(Progress::new(Watch::Nextest))), + ) + .0; + + assert!( + matches!(verdict, Verdict::Unmetered(ref reason) if reason.contains("stranger-def")), + "{verdict:?}" + ); + } + + /// With no runner the binary is invoked directly, carrying the arguments the harness reads — + /// the baseline included, or the baseline would measure a different suite from the one each + /// mutant is judged against. + #[test] + fn a_run_without_a_runner_invokes_the_binary_itself() { + let (_scratch, work) = crate::testing::helper_workspace("launch-direct", &["exit:0"]); + let binary = crate::testing::helper(); + let command = launcher(&work, &binary, Only::All).expect("a direct launch needs nothing from a runner"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(command.get_program(), binary.path.as_str()); + assert_eq!(args, vec!["--gamma-step=exit:0"]); + } + + /// A run narrowed to one test asks libtest for that name and for an exact match. + /// + /// Both halves matter. Without `--exact` the name is a substring filter, so `parses` would drag + /// in `parses_empty` and `parses_nested` and the probe would stop being one test. Only this + /// run's own name is passed: the user gave no filter of their own here, so there is nothing to + /// intersect with and nothing that could widen the selection back out. + #[test] + fn a_run_narrowed_to_one_test_asks_for_it_by_name_and_exactly() { + let (_scratch, work) = crate::testing::helper_workspace("launch-filtered", &["exit:0"]); + let binary = crate::testing::helper(); + let command = launcher(&work, &binary, Only::One("tests::parses")).expect("a direct launch needs nothing from a runner"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(args, vec!["--gamma-step=exit:0", "tests::parses", "--exact"]); + } + + /// A run narrowed by a census asks for every test the census named, in one launch. + /// + /// libtest matches a test that any one of its positional filters matches, so several names mean + /// "any of these" — and one `--exact` covers them all, which is why it comes last rather than + /// once per name. + #[test] + fn a_run_narrowed_by_a_census_asks_for_all_of_its_tests_at_once() { + let (_scratch, work) = crate::testing::helper_workspace("launch-censused", &["exit:0"]); + let binary = crate::testing::helper(); + let names = ["tests::parses", "tests::rejects"]; + let command = launcher(&work, &binary, Only::These(&names)).expect("a direct launch needs nothing from a runner"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(args, vec!["--gamma-step=exit:0", "tests::parses", "tests::rejects", "--exact"]); + } + + /// An empty census selection is not a filter that matches nothing. + /// + /// It reaches here only from a caller that had no names to give, and a launch carrying just + /// `--exact` would run the whole binary anyway — so the shape that says "everything" is the one + /// with no filter at all, which is what the whole-binary path already produces. + #[test] + fn a_census_selection_with_no_names_runs_the_binary_whole() { + let (_scratch, work) = crate::testing::helper_workspace("launch-censused-empty", &["exit:0"]); + let binary = crate::testing::helper(); + let command = launcher(&work, &binary, Only::These(&[])).expect("a direct launch needs nothing from a runner"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(args, vec!["--gamma-step=exit:0"]); + } + + /// A test this run chose that the user's own filter excludes is refused, not run. + /// + /// libtest matches a test that any *one* of its positional filters matches, so appending the + /// chosen name to the user's would widen the selection rather than narrow it — and the mutant + /// would then be convicted by a test the user deliberately took out of the run, inflating the + /// score with a detection their suite as configured never made. The intersection is empty here, + /// and an empty intersection has no launch that expresses it. + #[test] + fn a_chosen_test_the_users_filter_excludes_is_refused_rather_than_run() { + let (_scratch, mut work) = crate::testing::helper_workspace("launch-excluded", &["exit:0"]); + let binary = crate::testing::helper(); + + work.set_test_args(vec!["allowed".to_owned()]); + + let refusal = launcher(&work, &binary, Only::One("tests::excluded")).expect_err("the selection is empty"); + + assert!(refusal.contains("harness filters allow"), "{refusal}"); + } + + /// A test the user's filter admits is run alone, without their filter being repeated. + /// + /// Repeating it would restore the widening, and the `--exact` that pins this run's own name is + /// global — so leaving the user's substring filter in place would silently redefine it as a + /// whole-name match, which for `allowed` matches nothing at all. Their `--skip` stays, because + /// it only ever removes tests. + #[test] + fn a_chosen_test_the_users_filter_admits_is_run_alone_under_their_flags() { + let (_scratch, mut work) = crate::testing::helper_workspace("launch-admitted", &["exit:0"]); + let binary = crate::testing::helper(); + + work.set_test_args(vec!["--skip".to_owned(), "slow".to_owned(), "allowed".to_owned()]); + + let command = launcher(&work, &binary, Only::One("tests::allowed_case")).expect("the selection is not empty"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(args, vec!["--skip", "slow", "tests::allowed_case", "--exact"]); + } + + /// A census selection keeps the part the user's filter admits rather than being refused whole. + #[test] + fn a_census_selection_keeps_only_the_tests_the_users_filter_admits() { + let (_scratch, mut work) = crate::testing::helper_workspace("launch-intersected", &["exit:0"]); + let binary = crate::testing::helper(); + + work.set_test_args(vec!["allowed".to_owned()]); + + let names = ["tests::allowed_one", "tests::excluded", "tests::allowed_two"]; + let command = launcher(&work, &binary, Only::These(&names)).expect("the selection is not empty"); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!(args, vec!["tests::allowed_one", "tests::allowed_two", "--exact"]); + } + + /// A wakeup raised while the waiter was busy elsewhere is not slept through. + /// + /// This is the whole reason the pulse counts generations instead of setting a flag the waiter + /// clears. The loop reads the generation, then checks the child, then waits; a signal landing in + /// that gap must make the wait return at once, or the news it carried is delayed by the cap. + #[test] + fn a_wakeup_raised_before_the_wait_is_not_slept_through() { + let pulse = Pulse::default(); + let seen = pulse.seen(); + + pulse.signal(); + + let started = Instant::now(); + pulse.wait(seen, Duration::from_secs(30)); + + assert!( + started.elapsed() < Duration::from_secs(5), + "the wait slept through a signal it had already been sent" + ); + } + + /// A wakeup raised during a wait ends it, rather than leaving it to time out. + #[test] + fn a_wakeup_raised_during_a_wait_ends_it() { + let pulse = Arc::new(Pulse::default()); + let seen = pulse.seen(); + let started = Instant::now(); + + thread::scope(|scope| { + let _signaller = scope.spawn(|| { + thread::sleep(Duration::from_millis(20)); + pulse.signal(); + }); + + pulse.wait(seen, Duration::from_secs(30)); + }); + + assert!( + started.elapsed() < Duration::from_secs(5), + "the wait ran to its timeout despite being signalled" + ); + } + + /// A binary that exits at once is judged at once, not one wait later. + /// + /// Measured over a run of launches, because a single one is dominated by process creation and + /// would only be measuring the platform. If the reader threads ever stop signalling, each launch + /// falls back to the cap and the total blows past this bound by an order of magnitude — which is + /// exactly the regression worth catching, since it costs a wait per reachable binary per mutant. + #[cfg_attr( + any(coverage_nightly, windows), + ignore = "coverage instrumentation or Windows process startup makes this wall-clock assertion meaningless" + )] + #[test] + fn a_binary_that_exits_at_once_is_not_waited_on() { + let (_directory, work) = scripted(&["exit:0"]); + let quick = crate::testing::helper(); + let launches = 20; + let started = Instant::now(); + + for _ in 0..launches { + let verdict = run_binary( + &work, + &quick, + Attempt { + active: None, + timeout: Some(Duration::from_mins(1)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + false, + ); + + assert!(matches!(verdict, Verdict::Passed), "{verdict:?}"); + } + + assert!( + started.elapsed() < WAIT_CAP * launches, + "{launches} instant launches took {:?}, which is at least a full wait each", + started.elapsed() + ); + } + + /// The gauge has to count a reader for exactly as long as its thread is running. + /// + /// This is the measurement the backlog asks for before any bound is built, so what it reports has to + /// mean what it says: `live` back at its starting value once the readers have finished, and a + /// `peak` that recorded them while they had not. A gauge that decremented early would report + /// an untroubled run over a suite stranding readers on every mutant, which is the exact + /// conclusion it exists to prevent. + #[test] + fn a_readers_gauge_counts_a_reader_for_as_long_as_it_runs() { + let gauge = Readers::new(); + + assert_eq!(gauge.live(), 0); + assert_eq!(gauge.peak(), 0); + + gauge.started(); + gauge.started(); + + assert_eq!(gauge.live(), 2); + assert_eq!(gauge.peak(), 2); + + gauge.finished(); + + assert_eq!(gauge.live(), 1, "a finished reader is still counted as running"); + assert_eq!(gauge.peak(), 2, "the peak fell back to the live count"); + + gauge.started(); + gauge.finished(); + gauge.finished(); + + assert_eq!(gauge.live(), 0); + assert_eq!(gauge.peak(), 2, "the peak forgot a reader that has since finished"); + } + + /// Reading a binary's output must leave nothing running behind it. + /// + /// The readers are abandoned rather than joined, so nothing in the ordinary path proves they + /// ever stop. This runs a binary that exits at once and holds nothing open, which is the case + /// that must return the gauge to where it started — if even that leaks a reader, the count + /// reported under `--diag` would climb on every mutant and say nothing about stray descendants. + #[test] + fn an_ordinary_binary_leaves_no_reader_behind() { + let before = READERS.live(); + let (_directory, work) = scripted(&["print:done", "exit:0"]); + let quick = crate::testing::helper(); + + let verdict = run_binary( + &work, + &quick, + Attempt { + active: None, + timeout: Some(Duration::from_mins(1)), + stall: Stall::NONE, + request: MemoryRequest::default(), + only: Only::All, + census: None, + }, + false, + ); + + assert!(matches!(verdict, Verdict::Passed), "{verdict:?}"); + assert!(READERS.peak() > 0, "the run started no readers at all"); + + // The reader publishes its text and then decrements, so the two are not simultaneous. + // + // The gauge is process-global and the suite runs in parallel, so the count is compared with + // `<=` rather than `==`: another test's readers may start or finish at any moment, and only + // a reader this call leaked could hold the figure permanently above where it began. That is + // the leak the test is for, and it is the one thing polling cannot wait out. + for _attempt in 0..200 { + if READERS.live() <= before { + return; + } + + thread::sleep(Duration::from_millis(10)); + } + + assert!(READERS.live() <= before, "a reader outlived the binary it was reading"); + } +} diff --git a/crates/cargo-gamma-lib/src/exec/verdict/hubs.rs b/crates/cargo-gamma-lib/src/exec/verdict/hubs.rs new file mode 100644 index 000000000..b5df5ee8f --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/verdict/hubs.rs @@ -0,0 +1,354 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The two pure synchronization hubs the reader and watchdog threads coordinate on. +//! +//! [`Pulse`] is the notify/wait cell that lets a reader thread wake the thread waiting on the child +//! the instant a pipe closes or a failure is announced, instead of on a polling timer. [`Readers`] +//! is the process-wide gauge that counts how many reader threads are live and how high that count +//! has ever risen. Neither touches a pipe or a process directly, which is what lets them be modelled +//! in isolation. +//! +//! # Why this is a separate module +//! +//! Everything else in `verdict` is entangled with real child processes and real pipes, which +//! [`loom`](https://docs.rs/loom) cannot drive — it replaces the threads and the atomics with a +//! deterministic scheduler that has no I/O. The synchronization these two types perform is *argued* +//! sound, but a mutation to a memory ordering or the lock discipline needs a schedule the ordinary +//! tests never force, so the suite would stay green. Pulling the pure pieces here lets a loom model +//! exercise them under every interleaving. +//! +//! # The loom shim +//! +//! Under `--cfg loom` the primitives below resolve to `loom`'s instrumented equivalents; otherwise +//! they are the ordinary standard-library types. The swap is invisible in an ordinary build: same +//! types, same code, no cost. See the `loom_models` module at the foot of this file for the models +//! and for the exact command that runs them. + +#[cfg(not(loom))] +use core::sync::atomic::{AtomicUsize, Ordering}; +use core::time::Duration; +#[cfg(not(loom))] +use std::sync::{Condvar, Mutex}; + +#[cfg(loom)] +use loom::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(loom)] +use loom::sync::{Condvar, Mutex}; + +/// A wakeup shared between the reader threads and the thread waiting on the child. +/// +/// Sleeping a fixed five milliseconds between polls would cost a binary that finished instantly a +/// full interval before the run noticed — once per reachable binary per mutant, which is the highest +/// multiplier in the tool. Against a suite that takes seconds this is nothing; against a unit-test +/// binary that runs in single-digit milliseconds it is a large fraction of the launch, and a +/// workspace of many small fast targets is exactly the shape where this tool is otherwise quickest. +/// +/// A generation counter rather than a flag, because the wakeup must not be lost: the loop reads the +/// generation *before* it checks the child, so a signal raised while it was checking makes the wait +/// return at once instead of sleeping through news that had already arrived. Shrinking the interval +/// instead would trade the latency for a spin that steals cores from every other job. +#[derive(Default)] +pub(super) struct Pulse { + /// How many times something worth waking for has happened. + generation: Mutex, + + /// Signalled on every change to `generation`. + woken: Condvar, +} + +impl Pulse { + /// The current generation, to be passed to a later [`Pulse::wait`]. + pub(super) fn seen(&self) -> u64 { + #[expect(clippy::unwrap_used, reason = "the waiter only panics if the whole process is unwinding")] + let generation = self.generation.lock().unwrap(); + + *generation + } + + /// Records that something the waiter cares about has happened, and wakes it. + pub(super) fn signal(&self) { + #[expect(clippy::unwrap_used, reason = "the waiter only panics if the whole process is unwinding")] + let mut generation = self.generation.lock().unwrap(); + + *generation = generation.wrapping_add(1); + + drop(generation); + + self.woken.notify_all(); + } + + /// Sleeps until the generation moves past `seen`, or `upto` elapses, whichever is sooner. + pub(super) fn wait(&self, seen: u64, upto: Duration) { + #[expect(clippy::unwrap_used, reason = "the waiter only panics if the whole process is unwinding")] + let generation = self.generation.lock().unwrap(); + + // Already moved on while the caller was checking the child, so there is nothing to wait for. + if *generation != seen { + return; + } + + #[cfg(not(loom))] + #[expect(clippy::unwrap_used, reason = "the waiter only panics if the whole process is unwinding")] + let (_generation, _timed_out) = self.woken.wait_timeout(generation, upto).unwrap(); + + // loom has no clock, so it cannot model the timeout backstop firing. That is deliberate: the + // wait is modelled as blocking until it is *signalled*, which forces the model to rely on + // the generation guard above and the `notify_all` in `signal` — the actual mechanism — and + // not on the cap silently rescuing a lost wakeup. `WAIT_CAP` in the parent module is a + // backstop, not the mechanism, and the model proves the mechanism. + #[cfg(loom)] + { + let _ = upto; + + #[expect(clippy::unwrap_used, reason = "the waiter only panics if the whole process is unwinding")] + let _generation = self.woken.wait(generation).unwrap(); + } + } +} + +/// How many output readers are running, and how far that count has ever risen. +/// +/// A reader is abandoned rather than joined when the bounded drain gives up on it: the thread is +/// still blocked in a read on a pipe whose write end some descendant of the test binary inherited +/// and never closed, so joining it would block the run forever. One per affected mutant is nothing, +/// but a sweep of thousands of mutants over a suite that habitually leaves a daemon behind +/// accumulates threads and descriptors, and descriptors have a hard ceiling. +/// +/// Every exit now sweeps the subtree before draining, which closes those write ends in the ordinary +/// case and should make this stay at roughly two per running job. This counts rather than caps, +/// because whether anything still escapes is a question about a real suite that no amount of +/// reasoning about the code will answer — and a cap chosen without that number would either never +/// fire or stop runs that were never in trouble. `--diag` reports the peak, which is the number +/// that decides whether a bound is worth building. +#[derive(Debug)] +pub struct Readers { + /// How many reader threads are running right now. + live: AtomicUsize, + + /// The most that have ever been running at once. + peak: AtomicUsize, +} + +/// The run's reader gauge. +/// +/// Global because what it measures is: threads and open descriptors belong to the process, not to +/// any one launch, and the accumulation this is here to detect is precisely the one that outlives +/// the mutant that caused it. Threading it through every launch path would say otherwise. +#[cfg(not(loom))] +pub static READERS: Readers = Readers::new(); + +// loom's atomics are not `const`-constructible, so the plain `static` above will not compile under +// `--cfg loom`. The gauge is never driven by a model — the models build their own fresh `Readers` +// inside `loom::model` — so this exists only to keep the crate compiling under the cfg. `--diag` +// and the reader tests never run in a loom build. +#[cfg(loom)] +loom::lazy_static! { + pub static ref READERS: Readers = Readers::new(); +} + +impl Readers { + /// A gauge that has seen nothing yet. + #[cfg(not(loom))] + pub(super) const fn new() -> Self { + Self { + live: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + } + } + + /// A gauge that has seen nothing yet. + #[cfg(loom)] + pub(super) fn new() -> Self { + Self { + live: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + } + } + + /// Notes a reader starting, and raises the peak if this is the most yet. + pub(super) fn started(&self) { + let live = self.live.fetch_add(1, Ordering::Relaxed).saturating_add(1); + + // Raced against other readers rather than locked: a peak that loses a race understates by + // the width of that race, which is not worth a mutex on every stream of every mutant. + let _raised = self.peak.fetch_max(live, Ordering::Relaxed); + } + + /// Notes a reader finishing, whether it was waited for or abandoned. + pub(super) fn finished(&self) { + let _was = self.live.fetch_sub(1, Ordering::Relaxed); + } + + /// How many reader threads are running right now. + #[must_use] + pub fn live(&self) -> usize { + self.live.load(Ordering::Relaxed) + } + + /// The most reader threads that have ever been running at once. + /// + /// Read at the end of a run. With `jobs` concurrent mutants and two streams apiece, a run that + /// strands nothing peaks at about `2 * jobs`; anything far above that is the accumulation this + /// exists to detect. + #[must_use] + pub fn peak(&self) -> usize { + self.peak.load(Ordering::Relaxed) + } +} + +/// Deterministic-scheduler models of the two hubs above, run under [`loom`](https://docs.rs/loom). +/// +/// These are **not** part of the ordinary test suite. They only compile and run under `--cfg loom`, +/// where the standard-library primitives the hubs use are swapped for loom's instrumented ones and +/// every thread interleaving is explored exhaustively. That is slow, and it needs the whole crate +/// recompiled for the cfg, so it is kept off the default `cargo test` path. +/// +/// Run them with: +/// +/// ```text +/// RUSTFLAGS="--cfg loom" cargo test -p cargo-gamma-lib --lib hubs::loom_models +/// ``` +/// +/// Each model builds its own fresh state inside `loom::model` — loom re-runs the closure once per +/// schedule and forbids state that outlives an execution, so the global `READERS` static is never +/// used here. +#[cfg(loom)] +mod loom_models { + use core::time::Duration; + + use loom::sync::Arc; + + use super::{Pulse, Readers}; + + /// Two simultaneous binaries, each with stdout and stderr readers. + /// + /// This is the smallest production topology that exercises both contention between jobs and + /// contention between the two streams of one job. The atomic operations are identical for + /// additional jobs, so these four streams expose every distinct participant role while keeping + /// the state space tractable in CI. + const CONCURRENT_JOBS: usize = 2; + const STREAMS_PER_JOB: usize = 2; + const CONCURRENT_READER_STREAMS: usize = CONCURRENT_JOBS * STREAMS_PER_JOB; + + /// A wakeup is never lost, whatever order the waiter and the notifier run in. + /// + /// This models the waiter's `seen` → check-the-child → `wait` sequence racing both output + /// readers. One reader announces the first failure and later reaches end-of-stream; the other + /// also reaches end-of-stream, which is the maximum production signal pattern for one pulse. + /// `seen` is captured before either notifier is spawned, so the signals can land before or + /// during the wait. The waiter must always return; loom reports a deadlock if any interleaving + /// leaves it blocked. + /// + /// This has teeth against three separate weakenings, each of which makes loom report the + /// deadlock: dropping the `notify_all` in `signal`, dropping the generation increment in + /// `signal`, and dropping the `*generation != seen` guard in `wait`. + pub(super) fn a_pulse_wakeup_is_never_lost_under_any_interleaving() { + loom::model(|| { + let pulse = Arc::new(Pulse::default()); + let seen = pulse.seen(); + + let announcing_reader = { + let pulse = Arc::clone(&pulse); + loom::thread::spawn(move || { + pulse.signal(); + pulse.signal(); + }) + }; + let other_reader = { + let pulse = Arc::clone(&pulse); + loom::thread::spawn(move || pulse.signal()) + }; + + pulse.wait(seen, Duration::from_secs(0)); + + announcing_reader.join().unwrap(); + other_reader.join().unwrap(); + + assert_eq!(pulse.seen(), seen.wrapping_add(3), "a concurrent signal was lost"); + }); + } + + /// The live count returns to exactly zero after all four production reader roles finish. + /// + /// The coordinator first establishes the four-reader high-water mark, then all four independent + /// reader threads finish concurrently. The contended `live` observation also models diagnostics + /// racing abandoned readers as they finally close. Every increment must still have exactly one + /// decrement. + pub(super) fn a_readers_gauge_returns_to_exactly_zero_under_any_interleaving() { + loom::model(|| { + let readers = Arc::new(Readers::new()); + let mut finishers = Vec::with_capacity(CONCURRENT_READER_STREAMS); + + for _stream in 0..CONCURRENT_READER_STREAMS { + readers.started(); + } + + for _stream in 0..CONCURRENT_READER_STREAMS { + let readers = Arc::clone(&readers); + finishers.push(loom::thread::spawn(move || readers.finished())); + } + + assert!( + readers.live() <= CONCURRENT_READER_STREAMS, + "a contended live read exceeded the number of started readers" + ); + + for finisher in finishers { + finisher.join().unwrap(); + } + + assert_eq!( + readers.peak(), + CONCURRENT_READER_STREAMS, + "counting a started reader failed to raise the peak" + ); + assert_eq!(readers.live(), 0, "a decrement was lost or double-counted"); + }); + } + + /// The peak never understates four readers started concurrently by two jobs. + /// + /// Two worker-side participants each start the stdout and stderr readers of one simultaneous + /// job. None finishes, so all four streams are genuinely live. The observations before the joins + /// race the `fetch_add` and `fetch_max` operations, exercising the production loads under + /// contention; the observations after the joins must see the exact final count and peak. + pub(super) fn a_readers_peak_never_understates_four_concurrent_starts() { + loom::model(|| { + let readers = Arc::new(Readers::new()); + let mut workers = Vec::with_capacity(CONCURRENT_JOBS); + + for _job in 0..CONCURRENT_JOBS { + let readers = Arc::clone(&readers); + workers.push(loom::thread::spawn(move || { + for _stream in 0..STREAMS_PER_JOB { + readers.started(); + } + })); + } + + assert!( + readers.live() <= CONCURRENT_READER_STREAMS, + "a contended live read exceeded the number of starters" + ); + assert!( + readers.peak() <= CONCURRENT_READER_STREAMS, + "a contended peak read exceeded the number of starters" + ); + + for worker in workers { + worker.join().unwrap(); + } + + assert_eq!(readers.live(), CONCURRENT_READER_STREAMS, "a concurrent increment was lost"); + assert_eq!(readers.peak(), CONCURRENT_READER_STREAMS, "the peak understated four live readers"); + }); + } +} + +#[cfg(loom)] +pub(crate) fn run_loom_models() { + loom_models::a_pulse_wakeup_is_never_lost_under_any_interleaving(); + loom_models::a_readers_gauge_returns_to_exactly_zero_under_any_interleaving(); + loom_models::a_readers_peak_never_understates_four_concurrent_starts(); +} diff --git a/crates/cargo-gamma-lib/src/exec/workspace.rs b/crates/cargo-gamma-lib/src/exec/workspace.rs new file mode 100644 index 000000000..f634baab5 --- /dev/null +++ b/crates/cargo-gamma-lib/src/exec/workspace.rs @@ -0,0 +1,2981 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::hash::{Hash, Hasher}; +use core::num::NonZeroUsize; +use core::sync::atomic::{AtomicBool, Ordering}; +use std::collections::hash_map::DefaultHasher; +use std::ffi::OsString; +use std::fs::{self, File, TryLockError}; +use std::process::{Command, Output}; +use std::sync::OnceLock; +use std::{env, io, thread}; + +use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; +use walkdir::WalkDir; + +use super::cargo_options::CargoOptions; +use super::config::Config; +use super::copy::{CopyOptions, visible_vcs_metadata}; +use super::events::Events; +#[cfg(test)] +use super::faults::{self, Fault}; +use super::loader::{Launch, toolchain_libraries}; +use super::manifest::{CAP_LINTS, Manifest, RUNTIME_CRATE, anchor_cargo_config, cap_lints}; +use super::nextest::Harness; +use super::sync::sync_or_copy; +use super::test_binary::{TEST_THREADS_VAR, TestBinary, harness_threads}; +use crate::Result; +use crate::discover::TargetFile; +use crate::error::error; + +/// The guard runtime's sources, embedded so that the vendored copy cannot drift from the real one. +const RUNTIME_SOURCES: [(&str, &str); 3] = [ + ("lib.rs", include_str!("../../../cargo-gamma-rt/src/lib.rs")), + ("either.rs", include_str!("../../../cargo-gamma-rt/src/either.rs")), + ("runtime.rs", include_str!("../../../cargo-gamma-rt/src/runtime.rs")), +]; + +/// The workspace package contract inherited by the real runtime crate. +const WORKSPACE_MANIFEST: &str = include_str!("../../../../Cargo.toml"); + +/// Identifies the workspace allowed to reuse a redirected cache. +const CACHE_OWNER: &str = ".cargo-gamma-owner"; + +/// A scratch copy of the workspace, instrumented and ready to build. +#[derive(Debug)] +pub struct Workspace { + /// Root of the copied tree. + pub(super) root: Utf8PathBuf, + + /// Where build artifacts go, kept outside the copied tree so that repeated runs are + /// incremental rather than starting cold every time. + pub(super) target: Utf8PathBuf, + + /// Directories a test binary needs on its dynamic loader path. + /// + /// `cargo test` sets this before running a binary it built; we run binaries ourselves, so we + /// have to reproduce it for toolchains that link `std` dynamically. + pub(super) libraries: Vec, + + /// How cargo is invoked in this tree. + pub(super) cargo: CargoOptions, + + /// Where the vendored guard runtime lives, so that a package can be linked to it at the moment + /// its own mutants are known rather than all of them up front. + runtime: Utf8PathBuf, + + /// Whether the tree survives the run for inspection. + pub(super) leak: bool, + + /// Whether the run got far enough to be worth keeping build artifacts for. + /// + /// Artifacts are what make the next run incremental, so a run that reached the point of having + /// something to measure leaves them behind on purpose. A run that failed before then leaves + /// nothing a later run could reuse — only the object files of a tree that no longer exists, + /// which on a large workspace is tens of gigabytes of dead weight on a disk that a CI job may + /// well need for its next step. + settled: AtomicBool, + + /// How this run's test binaries are executed, once the build has produced them. + /// + /// `None` is the default runner, which launches each binary directly. + nextest: Option, + + /// The environment values that are the same for every test-binary launch of the whole run. + /// + /// Filled on first use rather than at construction because it is a property of running tests, + /// and a workspace is also built for commands that never run one. See [`Launch`]. + launch: OnceLock, + + /// How many threads each spawned test harness is told to use, or `None` to leave it alone. + /// + /// Carried here and set on each launched command rather than on this process's own + /// environment. Setting it globally was correct for the real binary — one run, one thread, no + /// children yet — and wrong for the test suite, which calls `run` from forty tests at once + /// while every other thread in the process is reading the environment. `setenv` racing + /// `getenv` is a data race in `libc`, not a confusing value, and there is nothing to restore + /// afterwards that would fix it. Every launch shares this one answer, so the baseline and the + /// sweep still cannot disagree about the width of the workload they measure and judge. + harness_threads: OnceLock>, + + /// Held for the life of the run so that another cargo-gamma command targeting the same original + /// workspace is turned away. Released when the process ends, however it ends, so a crash cannot + /// leave a lock nobody can clear. + _workspace_lock: File, + + /// Held when reusable state was redirected, so another workspace cannot use the same cache. + _cache_lock: Option, + + /// Whether [`Workspace::teardown`] has already removed what the destructor would remove. + /// + /// The destructor is the fallback for the paths that never reach an explicit teardown — an + /// error part-way through a run, a panic, a test that drops the tree without ceremony — so it + /// has to stay. This is what keeps it from walking a tree that is already gone. + torn_down: bool, +} + +impl Drop for Workspace { + fn drop(&mut self) { + if self.torn_down { + return; + } + + // The tree is rebuilt from scratch on the next run, so failing to remove it costs disk + // rather than correctness, and there is no caller in a position to do anything about it. + // A caller that wants to choose the moment, or to see what went wrong, calls `teardown` + // instead; this is what happens to the runs that do not. + let _torn_down = self.teardown(); + } +} + +impl Workspace { + /// Removes the scratch tree now, and says what went wrong if anything did. + /// + /// The destructor does this too, but it does it at a moment nobody chose and reports nothing. + /// On a real run the tree is a full copy of the workspace plus its build artifacts — + /// gigabytes, thousands of files, and a walk of every one of them — and that walk runs after + /// the report has been printed, so from outside it looks like a finished tool that will not + /// exit. A caller that has somewhere better to put the wait, or that wants the failure on the + /// error stream rather than discarded, calls this instead. + /// + /// Idempotent, and leaves nothing for the destructor to repeat. A tree kept by `--leak-dirs` + /// is left where it is: the point of that flag is that the directory is still there afterwards. + /// + /// # Errors + /// + /// Returns an error if the tree, or the build directory of a run that never settled, could not + /// be removed. The next run rebuilds both from scratch, so this costs disk rather than + /// correctness — which is why the destructor can afford to swallow it and a caller can afford + /// to report it and carry on. + pub fn teardown(&mut self) -> Result<()> { + if self.torn_down { + return Ok(()); + } + + self.torn_down = true; + + if self.leak { + return Ok(()); + } + + let tree = if self.settled.load(Ordering::Relaxed) { + // A settled run leaves the tree behind for delta synchronization on the next run. + // The sentinel written at copy/sync time remains, so the next prepare can reuse it. + Ok(()) + } else { + remove_tree(&self.root) + }; + + // Artifacts are what make the next run incremental, so a run that got far enough to have + // something to measure leaves them behind on purpose. + let build = if self.settled.load(Ordering::Relaxed) { + Ok(()) + } else { + remove_tree(&self.target) + }; + + tree.and(build) + } + + /// Says where to look at the tree, for an error that wants the reader to go and read it. + /// + /// The tree is deleted when the run ends unless `--leak-dirs` was given, so naming its path + /// unconditionally sends the reader to a directory that no longer exists by the time they get + /// there — which reads as a second, phantom bug on top of the one being reported. + pub(super) fn inspect_hint(&self) -> String { + if self.leak { + format!("The tree is at `{}` if you want to look.", self.root) + } else { + "Re-run with `--leak-dirs` to keep the instrumented tree and look at it.".to_owned() + } + } + + /// Copies and instruments the tree. + pub(super) fn prepare(source: &Utf8Path, config: &Config, events: &mut impl Events) -> Result { + config.cargo.validate()?; + + // Absolute from here on, so that the copy's own exclusion — which compares the scratch base + // against the paths it walks — is comparing two paths of the same kind. + let source = &absolute(source); + let base = gamma_base(source, config.cache_dir.as_deref()); + + ensure_copy_terminates(source, &base)?; + + let root = base.join("workspace"); + if config.cache_dir.is_some() { + ensure_vcs_visibility(source, &root)?; + } + let target = base.join("target"); + let runtime = base.join("rt"); + + events.begin("Copying", "Copied", "the workspace"); + + fs::create_dir_all(base.as_std_path()) + .map_err(|cause| error!("could not create the scratch directory at `{base}`").caused_by(cause))?; + + let (workspace_lock, cache_lock) = claim_cache(source, config.cache_dir.as_deref())?; + + migrate_legacy_directory(&base, "tree", "workspace")?; + migrate_legacy_directory(&base, "build", "target")?; + + events.testing_log(&base)?; + + let _outcome = sync_or_copy( + source, + &root, + &base, + CopyOptions { + copy_ignored: config.copy_ignored, + }, + )?; + + if config.cache_dir.is_none() { + expose_vcs_metadata(source, &root)?; + } + vendor_runtime(&runtime)?; + anchor_manifests(source, &root)?; + + let libraries = toolchain_libraries(&root, &target); + + let workspace = Self { + root, + target, + libraries, + cargo: config.cargo.clone(), + runtime, + leak: config.leak_dirs, + settled: AtomicBool::new(false), + nextest: None, + launch: OnceLock::new(), + harness_threads: OnceLock::new(), + _workspace_lock: workspace_lock, + _cache_lock: cache_lock, + torn_down: false, + }; + + events.end(""); + + Ok(workspace) + } + + /// Adds the guard runtime as a dependency of one package. + /// + /// Called when a package is about to be instrumented, not when the tree is copied: which + /// packages need it is not known until they have been scanned, and a package that turned out + /// to have no mutants should not carry a dependency it never uses. + pub(super) fn link_runtime(&self, package: &str, files: &[TargetFile]) -> Result<()> { + let runtime = &self.runtime; + let Some(path) = self.manifest_of(package, files) else { + return Ok(()); + }; + + let mut manifest = Manifest::read(&path)?; + + manifest.link_runtime(runtime); + manifest.save() + } + + /// Locates a package's manifest inside the copied tree. + fn manifest_of(&self, package: &str, files: &[TargetFile]) -> Option { + let file = files.iter().find(|file| file.package == package)?; + let mut directory = self.root.join(&file.path); + let real_root = physical(&self.root); + + // Walk up from a source file until a manifest appears, which is the package root. + while directory.pop() { + let candidate = directory.join("Cargo.toml"); + + // `is_file` resolves every component, so a copied symlink pointing outside the scratch + // tree would answer for the user's real manifest — which `Manifest::save` then rewrites. + // The physical path is what decides, because that is where the write lands. + if candidate.as_std_path().is_file() && physical(&candidate).starts_with(&real_root) { + return Some(candidate); + } + + if directory == self.root { + break; + } + } + + None + } + + /// Replaces a file that the copy already put in the tree. + /// + /// Refuses to follow a symlink or to create a file that was not copied, because either means + /// writing somewhere the copy did not choose — through a link, that is somewhere outside the + /// scratch tree entirely, and the tree holds a copy of the user's real source. + /// Returns whether the file had to be written, so a caller can report what it really changed. + /// + /// `root` is the scratch tree the write must land inside. `symlink_metadata` is non-following + /// for the last component alone, so it cannot see a link among the *intermediate* ones; the copy + /// recreates links verbatim, including absolute targets, so such a prefix is reachable. The + /// physical path is therefore checked as well. + pub(super) fn overwrite(root: &Utf8Path, path: &Utf8Path, contents: &str) -> Result { + let metadata = fs::symlink_metadata(path.as_std_path()) + .map_err(|cause| error!("could not write `{path}`, which the copy did not create").caused_by(cause))?; + + if !metadata.is_file() { + return Err(error!( + "refusing to write `{path}`, which is a link or a device rather than the copied source file" + )); + } + + let (real_path, real_root) = (physical(path), physical(root)); + + if !real_path.starts_with(&real_root) { + return Err(error!( + "refusing to write `{path}`, which is `{real_path}` — outside the scratch tree at `{real_root}`, \ + so the write would land in the real source tree" + )); + } + + // Cargo decides what to recompile from mtime, not from content, so writing a file back + // byte-for-byte still rebuilds its crate and everything downstream of it. The rollback loop + // rewrites the whole tree every round while changing only the few files whose mutants were + // withdrawn, which made every round cost a full workspace build. Comparing first turns the + // untouched majority into a read. + if let Ok(existing) = fs::read(path.as_std_path()) + && existing == contents.as_bytes() + { + return Ok(false); + } + + fs::write(path.as_std_path(), contents).map_err(|cause| error!("could not write `{path}`").caused_by(cause))?; + + Ok(true) + } + + /// Runs a cargo command in the copied tree. + /// Wraps an existing directory as a workspace, without copying or locking anything. + /// + /// Only for tests that need a real tree to run cargo in. The tree is left in place on drop, + /// because it belongs to whatever created it rather than to this handle. + #[cfg(any(test, feature = "internals"))] + pub(crate) fn adopt(root: Utf8PathBuf, target: Utf8PathBuf) -> Self { + Self { + runtime: root.join("gamma-rt"), + root, + target, + libraries: Vec::new(), + cargo: CargoOptions::default(), + nextest: None, + settled: AtomicBool::new(true), + leak: true, + launch: OnceLock::new(), + harness_threads: OnceLock::new(), + _workspace_lock: tempfile::tempfile().expect("a temporary file should be creatable"), + _cache_lock: None, + torn_down: false, + } + } + + /// Replaces the arguments every test binary is launched with. + /// + /// Only for tests that stand a shell in for a compiled harness, where the script to run is + /// itself an argument. + #[cfg(any(test, feature = "internals"))] + pub(crate) fn set_test_args(&mut self, args: Vec) { + self.cargo.test_args = args; + } + + pub(super) fn cargo(&self) -> Command { + let mut command = Command::new(cargo_binary()); + + let _ = command.current_dir(self.root.as_std_path()); + let _ = command.env("CARGO_TARGET_DIR", self.target.as_std_path()); + let _ = command.env("CARGO_TERM_COLOR", if self.cargo.color { "always" } else { "never" }); + + cap_ambient_rustflags(&mut command); + + // A guard is inert unless this names its mutant. Proc macros run inside the compiler, so a + // live mutant in one executes during the build rather than during a test — an infinite loop + // there hangs the one build the whole run depends on, with no test to time it out. The + // variable is scrubbed rather than trusted to be absent, since test processes set it and a + // user debugging a mutant by hand may export it. + let _ = command.env_remove(gamma_rt::ACTIVE_VAR); + + command + } + + /// Marks the run as having got far enough that its build artifacts are worth keeping. + /// + /// Until this is called, dropping the tree also discards everything built for it: the artifacts + /// of a run that never produced a result are of no use to the next one, and on a large + /// workspace they can be tens of gigabytes. + pub(super) fn settle(&self) { + self.settled.store(true, Ordering::Relaxed); + } + + /// Root of the copied tree, which is where a runner is pointed at the workspace. + pub(super) fn root(&self) -> &Utf8Path { + &self.root + } + + /// The arguments every test binary is launched with. + pub(super) fn test_arguments(&self) -> &[String] { + &self.cargo.test_args + } + + /// How this run's test binaries are executed, once the build has settled it. + pub(super) const fn runner(&self) -> Option<&Harness> { + self.nextest.as_ref() + } + + /// The launch environment shared by every test binary this run starts. + /// + /// Derived once and then handed out, because both halves are invariant for the run and both sat + /// on the hottest path the tool has. The stack floor reads the ambient variable exactly once + /// here, which is also what keeps that read out of the per-launch path where other threads are + /// concurrently spawning children. + pub(super) fn launch(&self) -> &Launch { + self.launch.get_or_init(|| Launch::derive(&self.libraries)) + } + + /// Settles how wide each spawned test harness runs, from the run's own job count. + /// + /// Called once, before anything is spawned, so that the baseline and every mutant after it are + /// measured under the same contention. See [`Workspace::harness_threads`]. + pub(super) fn calibrate_harness(&self, jobs: usize) { + let cores = thread::available_parallelism().map_or(1, NonZeroUsize::get); + let inherited = env::var(TEST_THREADS_VAR).ok(); + + let _settled = self + .harness_threads + .set(harness_threads(jobs, cores, inherited.as_deref()).map(|threads| threads.to_string())); + } + + /// How many threads to tell a spawned test harness to use, or `None` to leave it alone. + pub(super) fn harness_threads(&self) -> Option<&str> { + self.harness_threads.get().and_then(Option::as_deref) + } + + /// Hands the built tree to nextest, so every later mutant runs through it. + /// + /// Called after the build rather than at preparation, because the metadata it writes describes + /// binaries that do not exist until then. + /// + /// # Errors + /// + /// Returns an error if nextest is not installed or does not recognise the built tree. + pub(super) fn arm_nextest(&mut self, binaries: &[TestBinary]) -> Result<()> { + self.nextest = Some(Harness::prepare(self, binaries)?); + + Ok(()) + } + + /// Installs a runner without asking nextest anything, for tests about what having one changes. + /// + /// Unused under `--cfg loom`, which compiles the ordinary test modules out and leaves only the + /// concurrency models — so the sole caller of this helper disappears with them. + #[cfg(test)] + #[cfg_attr(loom, expect(dead_code, reason = "the loom build excludes the tests that call this"))] + pub(super) fn set_runner(&mut self, harness: Harness) { + self.nextest = Some(harness); + } + + /// Writes a file this run generates for its own use, returning where it went. + /// + /// Kept beside the build artifacts rather than inside the copied tree, so that nothing a test + /// walks over can see it and no mutant is judged against a tree that differs from the one that + /// was built. + /// + /// # Errors + /// + /// Returns an error if the file cannot be written. + pub(super) fn write_scratch(&self, name: &str, contents: &str) -> Result { + let path = self.target.join(name); + + if let Some(parent) = path.parent() { + fs::create_dir_all(parent.as_std_path()).map_err(|cause| error!("could not create `{parent}`").caused_by(cause))?; + } + + fs::write(path.as_std_path(), contents).map_err(|cause| error!("could not write `{path}`").caused_by(cause))?; + + Ok(path) + } + + /// Asks nextest to enumerate the test binaries this tree has already built. + /// + /// # Errors + /// + /// Returns an error if nextest is not installed or cannot read the tree. + pub(super) fn capture_nextest_list(&self, binaries: &[TestBinary]) -> Result { + let command = self.nextest_list_command(binaries); + + self.capture(command, "cargo nextest list") + } + + /// Builds the command that inventories the binaries nextest may run. + fn nextest_list_command(&self, binaries: &[TestBinary]) -> Command { + let mut command = self.cargo(); + let mut args = vec![ + "nextest".to_owned(), + "list".to_owned(), + "--list-type".to_owned(), + "binaries-only".to_owned(), + "--message-format".to_owned(), + "json".to_owned(), + ]; + + let mut packages: Vec<&str> = binaries + .iter() + .map(|binary| { + if binary.package_id.is_empty() { + binary.package.as_str() + } else { + binary.package_id.as_str() + } + }) + .collect(); + packages.sort_unstable(); + packages.dedup(); + + for package in packages { + args.push("--package".to_owned()); + args.push(package.to_owned()); + } + + self.cargo.extend_nextest_args(&mut args); + let _ = command.args(args); + + command + } + + /// Describes the copied workspace to whoever needs to resolve its packages. + /// + /// # Errors + /// + /// Returns an error if cargo cannot read the tree. + pub(super) fn capture_cargo_metadata(&self) -> Result { + let mut command = self.cargo(); + + let _ = command.args(["metadata", "--format-version", "1"]); + + self.capture(command, "cargo metadata") + } + + /// Runs a command that is expected to print something, and returns what it printed. + fn capture(&self, mut command: Command, what: &str) -> Result { + interpret(command.output().map(Captured::from), what, &self.root) + } + + /// The gamma base this run's tree sits under, whose size is the run's [`footprint`]. + pub(super) fn base(&self) -> &Utf8Path { + self.root.parent().unwrap_or(&self.root) + } +} + +/// What running a command produced, reduced to the three things [`interpret`] judges it on. +/// +/// A [`std::process::Output`] carries an `ExitStatus`, which a test cannot construct without +/// dropping to a platform-specific `ExitStatusExt`. Narrowing to the success flag keeps the +/// judgement below constructible from a test on every platform this tool builds for. +#[derive(Debug)] +struct Captured { + /// Whether the command reported success. + succeeded: bool, + + /// What it printed on the standard output stream. + stdout: Vec, + + /// What it printed on the standard error stream. + stderr: Vec, +} + +impl From for Captured { + fn from(output: Output) -> Self { + Self { + succeeded: output.status.success(), + stdout: output.stdout, + stderr: output.stderr, + } + } +} + +/// Judges a captured command, and returns what it printed. +/// +/// Separated from the spawn because all three ways a capture can fail — the command not running at +/// all, running and reporting failure, and printing bytes that are not UTF-8 — are things the host +/// decides, and none can be arranged by running a real `cargo` against a real tree. Keeping the +/// judgement pure is what makes them reachable from a test, and it costs the production path +/// nothing: there is no seam to arm, no state to leak between threads, and nothing compiled in +/// that would not have been. +/// +/// Every arm matters because the caller proceeds on this output as fact: a failure read as +/// empty-but-successful becomes a workspace with no test binaries and a run that cheerfully +/// reports nothing to do. +fn interpret(captured: io::Result, what: &str, root: &Utf8Path) -> Result { + let captured = captured.map_err(|cause| error!("could not run `{what}` in {root}").caused_by(cause))?; + + if !captured.succeeded { + let stderr = String::from_utf8_lossy(&captured.stderr); + + return Err(error!("`{what}` failed in {root}:\n{}", stderr.trim())); + } + + String::from_utf8(captured.stdout).map_err(|cause| error!("`{what}` did not print valid UTF-8").caused_by(cause)) +} + +/// Removes one scratch directory, treating one that is already gone as the outcome asked for. +/// +/// A tree can be missing legitimately: a run that failed before the copy finished never made one, +/// and a caller may tear down after an error path has already cleared it. Neither is worth +/// reporting. +fn remove_tree(path: &Utf8Path) -> Result<()> { + match fs::remove_dir_all(path.as_std_path()) { + Err(cause) if cause.kind() != io::ErrorKind::NotFound => { + Err(error!("could not remove the scratch directory at `{path}`").caused_by(cause)) + } + _removed => Ok(()), + } +} + +/// Total size in bytes of everything a run keeps on disk under `base`. +/// +/// Walking the tree costs a stat per file, and the tree includes the cargo target directory, which +/// on a large workspace holds hundreds of thousands of incremental artifacts. That is why this is a +/// function over a path rather than a figure every run computes: it is one line of output, so it is +/// worth only what it costs when something is actually going to print it. +/// +/// Unreadable entries are skipped rather than failing a run that has already succeeded. +#[must_use] +pub fn footprint(base: &Utf8Path) -> u64 { + WalkDir::new(base.as_std_path()) + .into_iter() + .filter_map(Result::ok) + .filter_map(|entry| entry.metadata().ok()) + .filter(fs::Metadata::is_file) + .map(|metadata| metadata.len()) + .sum() +} + +/// Where this tool keeps everything it generates for a workspace. +/// +/// Outside the workspace by default, so Cargo cannot rediscover and merge the real workspace's +/// ancestor configuration after the copied configuration has already been rewritten. +/// +/// The answer is always absolute. A relative `--cache-dir` is resolved against the current +/// directory, which is what a user typing one means, and absolutising here — before the tree, the +/// build directory and the vendored runtime are derived from it — is what makes every one of those +/// absolute by construction. It has to be: the copy prunes its own destination by comparing it +/// against the absolute paths it walks, and cargo resolves a path dependency against the manifest +/// holding it, so a relative base would make the copy descend into its own output and the +/// instrumented tree point its runtime dependency at itself. +#[must_use] +pub fn gamma_base(root: &Utf8Path, cache: Option<&Utf8Path>) -> Utf8PathBuf { + let base = cache.map_or_else( + || { + let mut identity = DefaultHasher::new(); + absolute(root).hash(&mut identity); + + env::var_os("XDG_CACHE_HOME") + .or_else(|| env::var_os("LOCALAPPDATA")) + .and_then(|path| Utf8PathBuf::from_path_buf(path.into()).ok()) + .or_else(|| { + env::var_os("HOME") + .and_then(|path| Utf8PathBuf::from_path_buf(path.into()).ok()) + .map(|home| home.join(".cache")) + }) + .unwrap_or_else(|| absolute(root).parent().unwrap_or(root).join(".cargo-gamma-cache")) + .join("cargo-gamma") + .join(format!("{:016x}", identity.finish())) + }, + Utf8Path::to_owned, + ); + + absolute(&base) +} + +/// Deletes every cached entry for a workspace while preserving its concurrency lock. +/// +/// Returns whether any cached data existed. The lock file remains so an active run and a clean +/// cannot race over the workspace or Cargo artifacts. +pub fn clean_cache(root: &Utf8Path) -> Result { + let base = gamma_base(root, None); + + if !base.exists() { + return Ok(false); + } + + let _lock = claim(&base)?; + let entries = + fs::read_dir(base.as_std_path()).map_err(|cause| error!("could not read cargo-gamma's cache at `{base}`").caused_by(cause))?; + let mut removed = false; + + for entry in entries { + let entry = entry.map_err(|cause| error!("could not read an entry in cargo-gamma's cache at `{base}`").caused_by(cause))?; + + if entry.file_name() == "lock" { + continue; + } + + let path = entry.path(); + let file_type = entry + .file_type() + .map_err(|cause| error!("could not inspect cached data at `{}`", path.display()).caused_by(cause))?; + let result = if file_type.is_dir() && !file_type.is_symlink() { + fs::remove_dir_all(&path) + } else { + fs::remove_file(&path) + }; + + result.map_err(|cause| error!("could not remove cached data at `{}`", path.display()).caused_by(cause))?; + removed = true; + } + + Ok(removed) +} + +/// Resolves a path against the current directory and removes the components that name nothing. +/// +/// Purely textual, so it never touches the filesystem and never fails: the paths it is given are +/// scratch directories that do not exist yet, which is exactly when canonicalising cannot answer. +pub(super) fn absolute(path: &Utf8Path) -> Utf8PathBuf { + let rooted = if path.is_absolute() { + path.to_owned() + } else { + // A current directory that is not UTF-8, or that has been removed out from under the + // process, leaves the path as written rather than failing a run over it. + env::current_dir() + .ok() + .and_then(|cwd| Utf8PathBuf::from_path_buf(cwd).ok()) + .map_or_else(|| path.to_owned(), |cwd| cwd.join(path)) + }; + + let mut normalised = Utf8PathBuf::new(); + + for component in rooted.components() { + match component { + Utf8Component::CurDir => {} + + // `..` after a real directory name cancels it. After a root or another `..` there is + // nothing to cancel, so it is kept and means what the filesystem says it means. + Utf8Component::ParentDir => match normalised.components().next_back() { + Some(Utf8Component::Normal(_)) => { + let _popped = normalised.pop(); + } + _ => normalised.push(component), + }, + + other => normalised.push(other), + } + } + + normalised +} + +/// Resolves the links in the part of `path` that exists, keeping the rest as written. +/// +/// The scratch base usually does not exist yet, so the whole of it cannot be canonicalised. The +/// deepest ancestor that does exist is resolved and the remaining components are put back, which is +/// enough to answer the only question asked of it: where the path physically lands. A path that +/// cannot be resolved at all is returned as written, because being unable to answer must not fail +/// a run that would otherwise have been fine. +fn physical(path: &Utf8Path) -> Utf8PathBuf { + let mut existing = path.to_owned(); + let mut tail: Vec = Vec::new(); + + loop { + if let Ok(resolved) = fs::canonicalize(existing.as_std_path()) { + let Ok(mut resolved) = Utf8PathBuf::from_path_buf(resolved) else { + return path.to_owned(); + }; + + for name in tail.iter().rev() { + resolved.push(name); + } + + return resolved; + } + + let Some(name) = existing.file_name().map(str::to_owned) else { + return path.to_owned(); + }; + + tail.push(name); + + if !existing.pop() { + return path.to_owned(); + } + } +} + +/// Refuses a scratch directory that the copy would end up copying into itself. +/// +/// A base inside the workspace is normally fine, and is in fact the default: the copy is told to +/// skip it, so the tree it writes there is never walked. What cannot work is a base that lands +/// inside the tree in a place the copy's exclusion does not cover, and there are two ways to reach +/// that. One is a base that *is* the workspace root, because the root is the one path the walk does +/// not test against the exclusion — it is the tree being copied. The other is a base reached +/// through a link: the exclusion is a path comparison, so a base spelled as somewhere outside the +/// workspace never matches the paths the walk produces, even when it physically sits among them. +/// +/// Both are decided on the physical paths, because that is what the copy walks, and refused up +/// front rather than allowed to proceed: the alternative is a tree that contains a copy of itself, +/// growing until the disk does, and every mutant measured in it measured in the wrong file. +fn ensure_copy_terminates(source: &Utf8Path, base: &Utf8Path) -> Result<()> { + let (real_source, real_base) = (physical(source), physical(base)); + + // `starts_with` is asked about the physical paths, because those are the ones the walk will + // produce. Whether the copy really prunes is a separate question: the exclusion compares `base` + // as written against paths the walk builds from `source` as written plus real directory + // entries, so a `base` reached through a link is spelled one way and walked another and the + // comparison never matches. Asking `prunes(source, base)` alone would take that as safe. + if !real_base.starts_with(&real_source) || prunes_in_practice(source, base, &real_source, &real_base) { + return Ok(()); + } + + if prunes(&real_source, &real_base) { + return Err(error!( + "the scratch directory `{base}` is `{real_base}`, inside the workspace at `{source}`, but is \ + not named as a path inside it — so the copy cannot skip it and would copy the copy.\n\ + Point --cache-dir at a directory that is really outside the workspace." + ) + .usage()); + } + + Err(error!( + "the scratch directory `{base}` is the workspace at `{source}` itself, so copying the \ + workspace would copy the copy.\n\ + Point --cache-dir at a directory outside the workspace." + ) + .usage()) +} + +/// Refuses a relocation that would hide VCS metadata a build can currently read. +/// +/// The copier omits VCS directories deliberately. A default scratch tree lies below the source +/// tree and still reaches its ancestor metadata, while a relocated one may not. Letting the two +/// builds differ turns `--cache-dir` into an undocumented build-input switch, so the unsupported +/// arrangement is rejected before it creates or copies anything. +fn ensure_vcs_visibility(source: &Utf8Path, scratch: &Utf8Path) -> Result<()> { + let source_metadata = visible_vcs_metadata(source); + + if source_metadata.is_empty() { + return Ok(()); + } + + let scratch_metadata = visible_vcs_metadata(scratch); + let hidden: Vec<&Utf8PathBuf> = source_metadata.iter().filter(|marker| !scratch_metadata.contains(marker)).collect(); + + if hidden.is_empty() { + return Ok(()); + } + + Err(error!( + "`--cache-dir` would relocate the cached workspace to `{scratch}`, where build scripts cannot see VCS metadata available from `{source}`: {}. \ + Use a cache directory beneath the same repository, or remove the build-time VCS dependency.", + hidden.iter().map(|marker| marker.as_str()).collect::>().join(", ") + ) + .usage()) +} + +/// Makes source-control metadata visible from an isolated default scratch tree. +fn expose_vcs_metadata(source: &Utf8Path, scratch: &Utf8Path) -> Result<()> { + let markers = visible_vcs_metadata(source); + + for name in super::copy::VCS_DIRS { + let Some(marker) = markers + .iter() + .filter(|marker| marker.file_name() == Some(name)) + .max_by_key(|marker| marker.components().count()) + else { + continue; + }; + + let destination = scratch.join(name); + + if name == ".git" { + let git_dir = if marker.as_std_path().is_dir() { + marker.clone() + } else { + let text = fs::read_to_string(marker.as_std_path()) + .map_err(|cause| error!("could not read Git metadata pointer `{marker}`").caused_by(cause))?; + let relative = text + .trim() + .strip_prefix("gitdir:") + .map(str::trim) + .ok_or_else(|| error!("Git metadata pointer `{marker}` has no `gitdir:` target"))?; + + absolute(&marker.parent().unwrap_or_else(|| Utf8Path::new("")).join(relative)) + }; + + fs::write(destination.as_std_path(), format!("gitdir: {git_dir}\n")) + .map_err(|cause| error!("could not expose Git metadata at `{destination}`").caused_by(cause))?; + + continue; + } + + #[cfg(unix)] + std::os::unix::fs::symlink(marker.as_std_path(), destination.as_std_path()) + .map_err(|cause| error!("could not expose VCS metadata at `{destination}`").caused_by(cause))?; + + #[cfg(windows)] + { + let linked = if marker.as_std_path().is_dir() { + std::os::windows::fs::symlink_dir(marker.as_std_path(), destination.as_std_path()) + } else { + std::os::windows::fs::symlink_file(marker.as_std_path(), destination.as_std_path()) + }; + + linked.map_err(|cause| error!("could not expose VCS metadata at `{destination}`").caused_by(cause))?; + } + } + + Ok(()) +} + +/// Returns whether the copy's exclusion will match a path the walk actually produces. +/// +/// The walk descends from `source` as written and appends real directory entries, so the exclusion +/// fires only when `base` names those same components. That holds exactly when `base` is `source` +/// plus a relative tail and resolving that tail from the physical source lands on the physical base +/// — which is false as soon as any component of the tail is a symlink. +fn prunes_in_practice(source: &Utf8Path, base: &Utf8Path, real_source: &Utf8Path, real_base: &Utf8Path) -> bool { + let Ok(relative) = base.strip_prefix(source) else { + return false; + }; + + !relative.as_str().is_empty() && real_base == real_source.join(relative) +} + +/// Returns whether the copy leaves `base` out of a walk of `source`. +/// +/// The copy skips whole any path it walks that equals the scratch base, so a base strictly inside +/// the tree — and everything the run writes under it — never reaches the copy. The walk's own root +/// is never compared against it, which is the case this answers `false` for. +fn prunes(source: &Utf8Path, base: &Utf8Path) -> bool { + base.starts_with(source) && base != source +} + +/// Where the instrumented copy of a workspace lives. +/// +/// Derived rather than stored so that a caller which was asked to keep the tree can say where it +/// is without the run having to hand back the workspace that owned it. +#[must_use] +pub fn scratch_tree(root: &Utf8Path, scratch: Option<&Utf8Path>) -> Utf8PathBuf { + gamma_base(root, scratch).join("workspace") +} + +/// Renames an older cache directory, or removes it when its replacement already exists. +fn migrate_legacy_directory(base: &Utf8Path, legacy: &str, current: &str) -> Result<()> { + let from = base.join(legacy); + let to = base.join(current); + + if !from.exists() { + return Ok(()); + } + + if to.exists() { + let metadata = fs::symlink_metadata(from.as_std_path()) + .map_err(|cause| error!("could not inspect the legacy cache directory at `{from}`").caused_by(cause))?; + let removed = if metadata.is_dir() { + fs::remove_dir_all(from.as_std_path()) + } else { + fs::remove_file(from.as_std_path()) + }; + + return removed.map_err(|cause| error!("could not remove the legacy cache directory at `{from}`").caused_by(cause)); + } + + fs::rename(from.as_std_path(), to.as_std_path()) + .map_err(|cause| error!("could not rename the legacy cache directory from `{from}` to `{to}`").caused_by(cause)) +} + +/// Takes and validates a cache chosen independently of the workspace. +/// +/// The caller has already taken the stable workspace lock. This second lock protects the inverse +/// collision: different workspaces naming the same redirected cache. Ownership is established only +/// for an empty directory; existing unmarked contents are never adopted as cargo-gamma state. +fn claim_redirected_cache(source: &Utf8Path, base: &Utf8Path) -> Result { + if base.exists() { + match fs::symlink_metadata(base.join(CACHE_OWNER).as_std_path()) { + Ok(_metadata) => {} + Err(cause) if cause.kind() == io::ErrorKind::NotFound => { + if has_any_entries(base)? { + return Err(unowned_cache(base)); + } + } + Err(cause) => { + return Err(error!( + "could not inspect the cargo-gamma cache owner marker at `{}`", + base.join(CACHE_OWNER) + ) + .caused_by(cause)); + } + } + } + + fs::create_dir_all(base.as_std_path()) + .map_err(|cause| error!("could not create the redirected cargo-gamma cache at `{base}`").caused_by(cause))?; + + let lock = claim(base)?; + validate_cache_owner(source, base)?; + + Ok(lock) +} + +/// Refuses a redirected directory unless it is empty or already belongs to this workspace. +fn validate_cache_owner(source: &Utf8Path, base: &Utf8Path) -> Result<()> { + let owner = base.join(CACHE_OWNER); + + match fs::symlink_metadata(owner.as_std_path()) { + Ok(metadata) => { + if !metadata.is_file() { + return Err(error!( + "the cargo-gamma cache owner marker at `{owner}` is not a regular file.\n\ + Choose an empty directory for --cache-dir." + ) + .usage()); + } + + let recorded = fs::read_to_string(owner.as_std_path()) + .map_err(|cause| error!("could not read the cargo-gamma cache owner marker at `{owner}`").caused_by(cause))?; + let source = physical(source); + + if recorded != source.as_str() { + return Err(error!( + "the redirected cache at `{base}` belongs to the workspace at `{recorded}`, not `{source}`.\n\ + Choose a different directory for --cache-dir." + ) + .usage()); + } + } + Err(cause) if cause.kind() == io::ErrorKind::NotFound => { + if has_unowned_entries(base)? { + return Err(unowned_cache(base)); + } + + fs::write(owner.as_std_path(), physical(source).as_str()) + .map_err(|cause| error!("could not write the cargo-gamma cache owner marker at `{owner}`").caused_by(cause))?; + } + Err(cause) => { + return Err(error!("could not inspect the cargo-gamma cache owner marker at `{owner}`").caused_by(cause)); + } + } + + Ok(()) +} + +/// Says whether a directory already contains anything at all. +fn has_any_entries(base: &Utf8Path) -> Result { + let mut entries = + fs::read_dir(base.as_std_path()).map_err(|cause| error!("could not inspect the redirected cache at `{base}`").caused_by(cause))?; + + entries + .next() + .transpose() + .map(|entry| entry.is_some()) + .map_err(|cause| error!("could not inspect an entry in the redirected cache at `{base}`").caused_by(cause)) +} + +/// Says whether a directory contains anything cargo-gamma has not just created to lock it. +fn has_unowned_entries(base: &Utf8Path) -> Result { + let entries = + fs::read_dir(base.as_std_path()).map_err(|cause| error!("could not inspect the redirected cache at `{base}`").caused_by(cause))?; + + for entry in entries { + let entry = entry.map_err(|cause| error!("could not inspect an entry in the redirected cache at `{base}`").caused_by(cause))?; + if entry.file_name() != "lock" { + return Ok(true); + } + } + + Ok(false) +} + +fn unowned_cache(base: &Utf8Path) -> crate::error::Error { + error!( + "the redirected cache at `{base}` is not empty and is not marked as cargo-gamma state.\n\ + Choose an empty directory for --cache-dir; existing contents will not be adopted or removed." + ) + .usage() +} + +/// Takes a cargo-gamma workspace or cache for the duration of an operation. +/// +/// The lock is advisory and held by an open file, so it is released when the process ends however +/// it ends — there is no stale lock to clear after a crash. +fn claim(base: &Utf8Path) -> Result { + let path = base.join("lock"); + + let file = File::options() + .create(true) + .truncate(false) + .write(true) + .open(path.as_std_path()) + .map_err(|cause| error!("could not open the scratch lock at `{path}`").caused_by(cause))?; + + // A filesystem that cannot lock at all is not a filesystem another run is holding the lock on, + // and the two call for opposite responses. Reported as contention, the user is told to wait for + // a run that does not exist and to move the scratch directory — which cannot help, since the new + // location is on the same mount unless they happen to guess otherwise. + match try_lock(&file) { + Ok(()) => {} + Err(TryLockError::WouldBlock) => { + return Err(error!( + "another `cargo gamma` run is already using `{base}`.\n\ + Wait for it to finish before running another command that uses this workspace or cache." + ) + .usage()); + } + + Err(TryLockError::Error(cause)) => { + return Err(error!( + "the cargo-gamma workspace lock at `{path}` could not be taken.\n\ + The workspace cache must be on a filesystem that supports advisory file locking." + ) + .caused_by(cause)); + } + } + + Ok(file) +} + +/// Takes the one stable lock shared by every command targeting `root`. +/// +/// Deliberately ignores `--cache-dir`: separate build caches may run independently, but commands +/// that can publish source or configuration changes must still agree on one lock domain. +pub(crate) fn claim_workspace(root: &Utf8Path) -> Result { + let base = gamma_base(root, None); + + fs::create_dir_all(base.as_std_path()) + .map_err(|cause| error!("could not create cargo-gamma's workspace cache at `{base}`").caused_by(cause))?; + claim(&base) +} + +/// Takes every lock needed to use one workspace's reusable state. +pub(crate) fn claim_cache(root: &Utf8Path, cache: Option<&Utf8Path>) -> Result<(File, Option)> { + let workspace = claim_workspace(root)?; + let base = gamma_base(root, cache); + let default = gamma_base(root, None); + let redirected = if cache.is_some() && physical(&base) != physical(&default) { + Some(claim_redirected_cache(root, &base)?) + } else { + None + }; + + Ok((workspace, redirected)) +} + +/// The lock itself, named so the fault seam can stand in for a filesystem that cannot take one. +fn try_lock(file: &File) -> core::result::Result<(), TryLockError> { + #[cfg(test)] + if faults::fired(Fault::Lock) { + return Err(TryLockError::Error(io::Error::from(io::ErrorKind::Unsupported))); + } + + // Parallel tests can release and immediately reacquire one fixture's lock; tolerate that + // in-process handoff without weakening production's fail-fast contention contract. + #[cfg(test)] + for _ in 0..250 { + match file.try_lock() { + Err(TryLockError::WouldBlock) => std::thread::sleep(std::time::Duration::from_millis(1)), + result => return result, + } + } + + file.try_lock() +} + +/// Repairs every manifest in the copied tree so that it still resolves from its new location. +/// +/// Every manifest is visited, not just the ones belonging to mutated packages: a package nobody +/// mutates is still built, and a path dependency it cannot resolve fails the build just as surely. +fn anchor_manifests(source: &Utf8Path, root: &Utf8Path) -> Result<()> { + for entry in WalkDir::new(root.as_std_path()).into_iter().filter_map(core::result::Result::ok) { + if entry.file_name() != "Cargo.toml" { + continue; + } + + let Some(path) = Utf8Path::from_path(entry.path()) else { + continue; + }; + + let Some(original) = path.parent().and_then(|directory| directory.strip_prefix(root).ok()) else { + continue; + }; + + let _destination = crate::paths::require_within(path, root, "a scratch manifest")?; + let mut manifest = Manifest::read(path)?; + + manifest.anchor_paths(&source.join(original), original); + manifest.save()?; + } + + anchor_cargo_config(root, source)?; + + // Done here rather than through `RUSTFLAGS`, which would replace the tree's own flags + // instead of adding to them. + cap_lints(root) +} + +/// Extends whichever ambient rustflags setting cargo will read with [`CAP_LINTS`]. +/// +/// The instrumented tree is not the user's code, and lint levels configured for their code have no +/// authority over ours; denying warnings here would fail on any guarded expression a lint happens to +/// dislike. The flag is normally merged into the copied tree's `.cargo/config.toml` by `cap_lints`, +/// because setting a rustflags variable *replaces* the configured flags rather than adding to them. +/// An ambient setting outranks the configuration it stands for, though, so when the caller has one +/// the flag has to reach rustc through the variable instead. +/// +/// Cargo reads flags from exactly one of four places, in this order: `CARGO_ENCODED_RUSTFLAGS`, +/// `RUSTFLAGS`, the target tables, then the build table. The first two are global and beat every +/// configured key, so when one of them is set it is the only thing extended. The last two are the +/// environment spellings of config keys `cap_lints` has already written to, and each *replaces* its +/// key rather than adding to it, so each present variable is extended — both levels, because which +/// of them cargo consults depends on a target triple that is not settled here, and every +/// `CARGO_TARGET__RUSTFLAGS` rather than the one naming the triple in force, for the same +/// reason `cap_lints` appends to every configured target table. +/// +/// Missing any of this is not a warning: with `-D warnings` ambient the instrumented tree compiles +/// under deny-by-default, guard-induced warnings become errors, and the mutants that produced them +/// are withdrawn as unviable — a score computed over a silently smaller denominator. +fn cap_ambient_rustflags(command: &mut Command) { + // The encoded spelling separates flags with a unit separator rather than a space, which is the + // whole reason it exists: a flag may contain spaces. + if let Some(inherited) = env::var_os("CARGO_ENCODED_RUSTFLAGS") { + return extend(command, "CARGO_ENCODED_RUSTFLAGS", inherited, "\u{1f}"); + } + + if let Some(inherited) = env::var_os("RUSTFLAGS") { + return extend(command, "RUSTFLAGS", inherited, " "); + } + + for (name, inherited) in env::vars_os() { + let Some(name) = name.to_str() else { + continue; + }; + + if name.starts_with("CARGO_TARGET_") && name.ends_with("_RUSTFLAGS") { + extend(command, name, inherited, " "); + } + } + + if let Some(inherited) = env::var_os("CARGO_BUILD_RUSTFLAGS") { + extend(command, "CARGO_BUILD_RUSTFLAGS", inherited, " "); + } +} + +/// Sets `name` on `command` to `inherited` with [`CAP_LINTS`] appended after `separator`. +fn extend(command: &mut Command, name: &str, inherited: OsString, separator: &str) { + let mut merged = inherited; + + merged.push(separator); + merged.push(CAP_LINTS); + + let _ = command.env(name, merged); +} + +/// Returns the cargo executable to use, honouring the one that invoked us. +fn cargo_binary() -> String { + env::var("CARGO").unwrap_or_else(|_missing| "cargo".to_owned()) +} + +/// Writes the guard runtime as a standalone crate outside the copied workspace. +/// +/// Outside deliberately: a path dependency living inside a workspace directory but absent from its +/// member list makes cargo refuse to build, and editing the user's member list would be far more +/// invasive than dropping a crate next door. +fn vendor_runtime(at: &Utf8Path) -> Result<()> { + let source = at.join("src"); + + fs::create_dir_all(source.as_std_path()).map_err(|cause| error!("could not create `{source}`").caused_by(cause))?; + + let workspace: toml::Value = toml::from_str(WORKSPACE_MANIFEST) + .map_err(|cause| error!("could not read the embedded workspace package contract").caused_by(cause))?; + let package = workspace + .get("workspace") + .and_then(|workspace| workspace.get("package")) + .and_then(toml::Value::as_table) + .ok_or_else(|| error!("the embedded workspace manifest has no `[workspace.package]` table"))?; + let inherited = |name| { + package + .get(name) + .and_then(toml::Value::as_str) + .ok_or_else(|| error!("the embedded workspace package contract has no string `{name}`")) + }; + let edition = inherited("edition")?; + let rust_version = inherited("rust-version")?; + let manifest = format!( + "[package]\nname = \"{RUNTIME_CRATE}\"\nversion = \"0.0.0\"\nedition = \"{edition}\"\nrust-version = \"{rust_version}\"\npublish = false\n\n\ + [features]\nloom = []\n\n[lints.rust]\nunexpected_cfgs = {{ level = \"warn\", check-cfg = ['cfg(loom)'] }}\n\n\ + [lib]\npath = \"src/lib.rs\"\n\n[workspace]\n" + ); + + fs::write(at.join("Cargo.toml").as_std_path(), manifest) + .map_err(|cause| error!("could not write the runtime manifest in `{at}`").caused_by(cause))?; + + for (name, contents) in RUNTIME_SOURCES { + let path = source.join(name); + + fs::write(path.as_std_path(), contents).map_err(|cause| error!("could not write the runtime source `{path}`").caused_by(cause))?; + } + + Ok(()) +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use std::collections::BTreeMap; + use std::ffi::OsStr; + + use super::*; + + /// A capture that succeeded hands back exactly what the command printed. + #[test] + fn a_successful_capture_returns_what_the_command_printed() { + let captured = Ok(Captured { + succeeded: true, + stdout: b"{\"kind\":\"test\"}".to_vec(), + stderr: b"warning: ignored".to_vec(), + }); + + let text = interpret(captured, "cargo nextest list", Utf8Path::new("/w")).expect("a successful capture"); + + assert_eq!(text, "{\"kind\":\"test\"}"); + } + + /// A command that never ran is reported, not read as having printed nothing. + /// + /// Proceeding on an empty capture would mean a workspace with no test binaries and a run that + /// reports nothing to do — a perfect score for a suite that never executed. + #[test] + fn a_capture_that_could_not_be_spawned_is_reported() { + let captured = Err(io::Error::new(io::ErrorKind::NotFound, "no such file")); + let failure = interpret(captured, "cargo nextest list", Utf8Path::new("/w")).expect_err("a spawn failure must be reported"); + + assert!( + failure.to_string().contains("could not run `cargo nextest list` in /w"), + "{failure}" + ); + assert!(failure.to_string().contains("no such file"), "the cause is kept: {failure}"); + } + + /// A command that ran and failed is reported, with what it said about why. + /// + /// The standard error stream is the only place the reason exists — cargo prints its diagnostic + /// there and exits non-zero — so dropping it leaves the user with a failure and no cause. + #[test] + fn a_capture_that_exited_non_zero_is_reported_with_its_diagnostic() { + let captured = Ok(Captured { + succeeded: false, + stdout: b"partial output".to_vec(), + stderr: b" error: no such subcommand: `nextest`\n".to_vec(), + }); + + let failure = interpret(captured, "cargo nextest list", Utf8Path::new("/w")).expect_err("a non-zero exit must be reported"); + let text = failure.to_string(); + + assert!(text.contains("`cargo nextest list` failed in /w"), "{text}"); + assert!(text.contains("error: no such subcommand: `nextest`"), "{text}"); + assert!(!text.contains("partial output"), "the failing output is not the answer: {text}"); + } + + /// Output that is not UTF-8 is reported rather than lossily salvaged. + /// + /// Every caller parses this text as JSON. Replacing an invalid sequence with `U+FFFD` would + /// turn a corrupt capture into a parse error somewhere further away, or worse, into a + /// successfully parsed document describing a path that is not the path on disk. + #[test] + fn a_capture_that_is_not_utf8_is_reported() { + let captured = Ok(Captured { + succeeded: true, + stdout: vec![0x7b, 0xff, 0xfe, 0x7d], + stderr: Vec::new(), + }); + + let failure = interpret(captured, "cargo metadata", Utf8Path::new("/w")).expect_err("invalid UTF-8 must be reported"); + + assert!( + failure.to_string().contains("`cargo metadata` did not print valid UTF-8"), + "{failure}" + ); + } + + /// A scratch tree blocked by a plain file is reported rather than silently reused. + #[test] + fn a_scratch_tree_blocked_by_a_file_is_a_reported_failure() { + let directory = crate::testing::workdir("stale-tree"); + let source = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + + fs::write(source.join("Cargo.toml").as_std_path(), "[workspace]\nmembers = []\n").expect("a manifest"); + + // A leftover named `tree` that is a file rather than a directory cannot be copied into. + // Reusing it would instrument nothing and report a perfect score, so the run has to stop. + let base = gamma_base(&source, None); + + fs::create_dir_all(base.as_std_path()).expect("the scratch base"); + fs::write(base.join("tree").as_std_path(), "not a directory").expect("the stale entry"); + + let config = Config::default(); + let mut events = crate::testing::Recorder::default(); + let failure = Workspace::prepare(&source, &config, &mut events).expect_err("the tree must fail to prepare"); + + assert!(failure.to_string().contains("could not create the scratch tree"), "{failure}"); + } + + /// The one existing `prepare` test only exercises the failure path where a stale entry blocks + /// the copy; every other step of preparing a scratch tree — creating the base, claiming the + /// lock, copying, vendoring the runtime, and anchoring manifests — was otherwise only reachable + /// through a real mutation-testing run. If any one of them silently broke, nothing here would + /// have caught it before a user's run failed on it instead. + #[test] + fn preparing_a_fresh_workspace_produces_a_usable_scratch_tree() { + let directory = crate::testing::workdir("prepare-happy-"); + let source = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + + fs::write( + source.join("Cargo.toml").as_std_path(), + "[package]\nname = \"trivial\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n", + ) + .expect("a manifest"); + fs::create_dir_all(source.join("src").as_std_path()).expect("src"); + fs::write(source.join("src/lib.rs").as_std_path(), "pub const A: i32 = 1;\n").expect("lib"); + + let config = Config::default(); + let mut events = crate::testing::Recorder::default(); + let work = Workspace::prepare(&source, &config, &mut events).expect("a fresh tree must prepare cleanly"); + + assert!(work.root.join("Cargo.toml").as_std_path().is_file(), "the manifest was not copied"); + assert!( + work.root.join("src/lib.rs").as_std_path().is_file(), + "the source file was not copied" + ); + assert!( + work.runtime.join("Cargo.toml").as_std_path().is_file(), + "the runtime was not vendored" + ); + } + + #[test] + fn legacy_cache_directories_are_renamed_without_losing_their_contents() { + let directory = crate::testing::workdir("legacy-cache-layout-"); + let base = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the cache path is UTF-8"); + + fs::create_dir_all(base.join("tree")).expect("legacy workspace"); + fs::create_dir_all(base.join("build")).expect("legacy target"); + fs::write(base.join("tree/source"), "source").expect("workspace marker"); + fs::write(base.join("build/artifact"), "artifact").expect("target marker"); + + migrate_legacy_directory(&base, "tree", "workspace").expect("workspace migration"); + migrate_legacy_directory(&base, "build", "target").expect("target migration"); + + assert_eq!( + fs::read_to_string(base.join("workspace/source")).expect("workspace marker"), + "source" + ); + assert_eq!(fs::read_to_string(base.join("target/artifact")).expect("target marker"), "artifact"); + assert!(!base.join("tree").exists()); + assert!(!base.join("build").exists()); + } + + #[test] + fn an_unowned_redirected_cache_is_refused_without_touching_its_contents() { + let directory = crate::testing::workdir("unowned-cache-"); + let source = Utf8PathBuf::from_path_buf(directory.path().join("source")).expect("the source path is UTF-8"); + let base = Utf8PathBuf::from_path_buf(directory.path().join("cache")).expect("the cache path is UTF-8"); + let marker = base.join("build/user-data"); + + fs::create_dir_all(marker.parent().expect("the marker has a parent")).expect("the user's directory"); + fs::write(marker.as_std_path(), "keep").expect("the user's file"); + + let failure = claim_redirected_cache(&source, &base).expect_err("an unowned non-empty directory must be refused"); + + assert!(failure.is_usage()); + assert!(failure.to_string().contains("not marked as cargo-gamma state"), "{failure}"); + assert_eq!(fs::read_to_string(marker.as_std_path()).expect("the user's file remains"), "keep"); + assert!( + !base.join("lock").exists(), + "refusal must happen before cargo-gamma writes into the directory" + ); + assert!(!base.join(CACHE_OWNER).exists()); + } + + #[test] + fn a_directory_containing_only_somebody_elses_lock_file_is_not_adopted() { + let directory = crate::testing::workdir("foreign-lock-cache-"); + let source = Utf8PathBuf::from_path_buf(directory.path().join("source")).expect("the source path is UTF-8"); + let base = Utf8PathBuf::from_path_buf(directory.path().join("cache")).expect("the cache path is UTF-8"); + + fs::create_dir_all(&base).expect("the user's directory"); + fs::write(base.join("lock"), "not ours").expect("the user's lock file"); + + let failure = claim_redirected_cache(&source, &base).expect_err("an existing lock file is user data"); + + assert!(failure.is_usage()); + assert_eq!(fs::read_to_string(base.join("lock")).expect("the user's lock remains"), "not ours"); + assert!(!base.join(CACHE_OWNER).exists()); + } + + #[test] + fn an_empty_redirected_cache_is_claimed_for_its_workspace_and_can_be_reused() { + let directory = crate::testing::workdir("owned-cache-"); + let source = Utf8PathBuf::from_path_buf(directory.path().join("source")).expect("the source path is UTF-8"); + let base = Utf8PathBuf::from_path_buf(directory.path().join("cache")).expect("the cache path is UTF-8"); + + fs::create_dir_all(&base).expect("the empty cache"); + let first = claim_redirected_cache(&source, &base).expect("an empty cache can be claimed"); + + assert_eq!( + fs::read_to_string(base.join(CACHE_OWNER)).expect("the owner marker"), + physical(&source).as_str() + ); + + drop(first); + let _second = claim_redirected_cache(&source, &base).expect("the owning workspace can reuse its cache"); + } + + #[test] + fn a_redirected_cache_owned_by_another_workspace_is_refused() { + let directory = crate::testing::workdir("foreign-cache-"); + let first = Utf8PathBuf::from_path_buf(directory.path().join("first")).expect("the source path is UTF-8"); + let second = Utf8PathBuf::from_path_buf(directory.path().join("second")).expect("the source path is UTF-8"); + let base = Utf8PathBuf::from_path_buf(directory.path().join("cache")).expect("the cache path is UTF-8"); + + let held = claim_redirected_cache(&first, &base).expect("the first workspace claims the cache"); + drop(held); + + let failure = claim_redirected_cache(&second, &base).expect_err("a second workspace must not adopt the cache"); + + assert!(failure.is_usage()); + assert!(failure.to_string().contains("belongs to the workspace"), "{failure}"); + assert!(failure.to_string().contains(absolute(&first).as_str()), "{failure}"); + } + + #[test] + fn two_workspaces_cannot_use_one_redirected_cache_concurrently() { + let directory = crate::testing::workdir("contended-cache-"); + let first = Utf8PathBuf::from_path_buf(directory.path().join("first")).expect("the source path is UTF-8"); + let second = Utf8PathBuf::from_path_buf(directory.path().join("second")).expect("the source path is UTF-8"); + let base = Utf8PathBuf::from_path_buf(directory.path().join("cache")).expect("the cache path is UTF-8"); + + let _held = claim_redirected_cache(&first, &base).expect("the first workspace claims the cache"); + let failure = claim_redirected_cache(&second, &base).expect_err("the live cache lock must serialize workspaces"); + + assert!(failure.is_usage()); + assert!(failure.to_string().contains("already using"), "{failure}"); + } + + #[test] + fn explicitly_naming_the_default_cache_does_not_take_its_lock_twice() { + let directory = crate::testing::workdir("default-cache-alias-"); + let source = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the source path is UTF-8"); + let base = gamma_base(&source, None); + + let (_workspace, redirected) = claim_cache(&source, Some(&base)).expect("the default cache has one lock domain"); + + assert!(redirected.is_none()); + } + + #[test] + fn cleaning_removes_cache_data_but_not_published_reports() { + let directory = crate::testing::workdir("clean-cache-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the workspace path is UTF-8"); + let base = gamma_base(&root, None); + let report = root.join("target/cargo-gamma/gamma-report.json"); + + fs::create_dir_all(base.join("workspace")).expect("cached workspace"); + fs::create_dir_all(base.join("target")).expect("cached target"); + fs::write(base.join("last-gamma-run.json"), "{}").expect("run record"); + fs::create_dir_all(report.parent().expect("report directory")).expect("report directory"); + fs::write(&report, "{}").expect("published report"); + + assert!(clean_cache(&root).expect("clean cache")); + assert!(!base.join("workspace").exists()); + assert!(!base.join("target").exists()); + assert!(!base.join("last-gamma-run.json").exists()); + assert!(base.join("lock").exists(), "the concurrency lock remains"); + assert!(report.exists(), "published output is not cache data"); + assert!(!clean_cache(&root).expect("cleaning an empty cache")); + } + + #[test] + fn cleaning_refuses_a_cache_owned_by_an_active_run() { + let directory = crate::testing::workdir("clean-active-cache-"); + let root = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the workspace path is UTF-8"); + let base = gamma_base(&root, None); + + fs::create_dir_all(&base).expect("cache"); + let _held = claim(&base).expect("active run lock"); + let failure = clean_cache(&root).expect_err("an active cache must not be cleaned"); + + assert!(failure.to_string().contains("already using"), "{failure}"); + } + + #[test] + fn the_vendored_runtime_is_the_real_one() { + // If these ever diverge, guards would be compiled against a runtime that is not the one + // this build was tested with. + let runtime = RUNTIME_SOURCES + .iter() + .find_map(|(name, source)| (*name == "runtime.rs").then_some(*source)) + .expect("runtime.rs is one of the embedded runtime sources"); + + assert!(runtime.contains("pub fn a(id: u32) -> bool")); + assert!(runtime.contains("GAMMA_ACTIVE")); + } + + #[test] + fn vendoring_writes_a_buildable_crate() { + let temporary = tempfile::tempdir().unwrap(); + let at = Utf8PathBuf::from_path_buf(temporary.path().join("rt")).unwrap(); + + vendor_runtime(&at).unwrap(); + + let manifest = fs::read_to_string(at.join("Cargo.toml").as_std_path()).unwrap(); + + assert!(manifest.contains("name = \"gamma_rt\"")); + assert!(manifest.contains("edition = \"2024\"")); + assert!(manifest.contains("rust-version = \"1.95\"")); + assert!(manifest.contains("check-cfg = ['cfg(loom)']")); + + // The `[workspace]` table keeps it from being adopted by whatever workspace it lands near. + assert!(manifest.contains("[workspace]")); + for (name, _contents) in RUNTIME_SOURCES { + assert!(at.join("src").join(name).as_std_path().is_file(), "{name} was not vendored"); + } + + let checked = Command::new(cargo_binary()) + .args(["check", "--offline", "--manifest-path"]) + .arg(at.join("Cargo.toml")) + .env("CARGO_TARGET_DIR", at.join("target")) + .output() + .expect("cargo checks the vendored runtime"); + + assert!(checked.status.success(), "{}", String::from_utf8_lossy(&checked.stderr)); + } + + /// A vendor location that cannot be created at all — because something is already sitting where + /// its `src` directory would go — must fail with a message naming the directory rather than a + /// bare `NotADirectory` the reader has to reverse-engineer back to what this function was doing. + #[test] + fn vendoring_into_a_location_whose_src_directory_cannot_be_created_reports_the_failure() { + let temporary = tempfile::tempdir().unwrap(); + let at = Utf8PathBuf::from_path_buf(temporary.path().join("rt")).unwrap(); + + // A plain file standing where `at/src` needs to go blocks `create_dir_all` outright. + fs::create_dir_all(at.as_std_path()).unwrap(); + fs::write(at.join("src").as_std_path(), "not a directory").unwrap(); + + let cause = vendor_runtime(&at).unwrap_err(); + + assert!(cause.to_string().contains("could not create"), "{cause}"); + } + + /// A vendor location whose manifest path is blocked by an existing directory cannot have the + /// generated manifest written to it, and that has to be reported as the write failure it is. + #[test] + fn vendoring_a_manifest_blocked_by_a_directory_reports_the_write_failure() { + let temporary = tempfile::tempdir().unwrap(); + let at = Utf8PathBuf::from_path_buf(temporary.path().join("rt")).unwrap(); + + fs::create_dir_all(at.join("Cargo.toml").as_std_path()).unwrap(); + + let cause = vendor_runtime(&at).unwrap_err(); + + assert!(cause.to_string().contains("could not write the runtime manifest"), "{cause}"); + } + + /// A vendor location whose runtime source path is blocked by an existing directory cannot have + /// the vendored source written to it, and that has to be reported as the write failure it is. + #[test] + fn vendoring_a_source_file_blocked_by_a_directory_reports_the_write_failure() { + let temporary = tempfile::tempdir().unwrap(); + let at = Utf8PathBuf::from_path_buf(temporary.path().join("rt")).unwrap(); + + fs::create_dir_all(at.join("src").join("lib.rs").as_std_path()).unwrap(); + + let cause = vendor_runtime(&at).unwrap_err(); + + assert!(cause.to_string().contains("could not write the runtime source"), "{cause}"); + } + + #[test] + fn a_run_that_never_settled_takes_its_build_output_with_it() { + // Artifacts of a tree that no longer exists cannot make anything incremental, and on a + // large workspace they are tens of gigabytes on a disk a CI job still has plans for. + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + fs::write(target.join("artifact").as_std_path(), "x").unwrap(); + + drop(unsettled(&base, &root, &target)); + + assert!(!root.as_std_path().exists()); + assert!(!target.as_std_path().exists()); + } + + #[test] + fn a_run_that_settled_keeps_its_tree_and_build_output_for_the_next_one() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + + let work = unsettled(&base, &root, &target); + + work.settle(); + drop(work); + + assert!(root.as_std_path().exists(), "the tree is kept for delta sync on the next run"); + assert!(target.as_std_path().exists()); + } + + #[test] + fn a_leaked_tree_keeps_everything() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + + let mut work = unsettled(&base, &root, &target); + + work.leak = true; + drop(work); + + assert!(root.as_std_path().exists()); + assert!(target.as_std_path().exists()); + } + + /// An explicit teardown is what a caller drives instead of waiting on the destructor. + /// + /// It has to remove exactly what dropping would have removed, and leave the destructor with + /// nothing to repeat — a second walk of a tree that is already gone would report a failure for + /// work that succeeded, and on a real tree the first walk is the expensive one. + #[test] + fn an_explicit_teardown_removes_the_tree_and_leaves_the_destructor_nothing_to_do() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + + let mut work = unsettled(&base, &root, &target); + + work.teardown().expect("a tree that exists must tear down cleanly"); + + assert!(!root.as_std_path().exists()); + assert!(!target.as_std_path().exists()); + assert!(work.torn_down, "the destructor would walk the tree a second time"); + + // Recreated behind the workspace's back: if dropping still removed things, this would go. + fs::create_dir_all(root.as_std_path()).unwrap(); + drop(work); + + assert!(root.as_std_path().exists(), "the destructor repeated a teardown already done"); + } + + /// Tearing down twice is the same answer twice, because an error path may have done it already. + #[test] + fn a_second_teardown_is_not_a_failure() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + + let mut work = unsettled(&base, &root, &target); + + work.teardown().expect("the first teardown"); + work.teardown().expect("the second teardown must agree with the first"); + } + + /// A settled run's build output survives an explicit teardown for the same reason it survives + /// the destructor: it is what makes the next run incremental. + #[test] + fn an_explicit_teardown_keeps_the_tree_and_build_output_of_a_settled_run() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + + let mut work = unsettled(&base, &root, &target); + + work.settle(); + work.teardown().expect("a settled tree must tear down cleanly"); + + assert!(root.as_std_path().exists(), "the tree is kept for delta sync"); + assert!(target.as_std_path().exists()); + } + + /// `--leak-dirs` exists so the tree is still there afterwards, which an explicit teardown must + /// respect as much as the destructor does. + #[test] + fn an_explicit_teardown_keeps_a_leaked_tree() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + + let mut work = unsettled(&base, &root, &target); + + work.leak = true; + work.teardown().expect("a leaked tree tears down by leaving everything alone"); + + assert!(root.as_std_path().exists()); + assert!(target.as_std_path().exists()); + } + + /// A tree that cannot be removed is reported rather than swallowed — the whole reason for + /// having an explicit teardown beside the destructor. + #[test] + fn a_tree_that_cannot_be_removed_is_reported() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + // A file where a directory is expected: `remove_dir_all` refuses it, and it is the same + // shape of leftover that `prepare` already has to refuse. + fs::write(root.as_std_path(), "not a directory").unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + + let mut work = unsettled(&base, &root, &target); + let failure = work.teardown().expect_err("a tree that is a file cannot be removed"); + + assert!(failure.to_string().contains("could not remove the scratch directory"), "{failure}"); + + // Reported, not abandoned part-way: the build directory is still cleared. + assert!(!target.as_std_path().exists(), "the failure stopped the rest of the teardown"); + } + + /// A tree that was never created is not a failure to report: a run can fail before the copy. + #[test] + fn a_teardown_of_a_tree_that_was_never_created_succeeds() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + let mut work = unsettled(&base, &root, &target); + + work.teardown().expect("nothing to remove is the outcome asked for"); + } + + #[test] + fn the_footprint_counts_everything_the_run_leaves_behind() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_owned()).unwrap(); + let root = base.join("workspace"); + let target = base.join("target"); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(target.as_std_path()).unwrap(); + fs::write(root.join("a.rs").as_std_path(), "0123456789").unwrap(); + fs::write(target.join("a.o").as_std_path(), "01234").unwrap(); + + let mut work = unsettled(&base, &root, &target); + + assert_eq!(footprint(work.base()), 15); + + work.leak = true; + } + + /// A workspace over a real directory that has not been marked as worth keeping. + fn unsettled(base: &Utf8Path, root: &Utf8Path, target: &Utf8Path) -> Workspace { + Workspace { + root: root.to_owned(), + runtime: base.join("rt"), + target: target.to_owned(), + libraries: Vec::new(), + cargo: CargoOptions::default(), + nextest: None, + settled: AtomicBool::new(false), + leak: false, + launch: OnceLock::new(), + harness_threads: OnceLock::new(), + _workspace_lock: File::create(base.join("lock").as_std_path()).unwrap(), + _cache_lock: None, + torn_down: false, + } + } + + /// The harness width is settled once and then handed out, without touching this process. + /// + /// Publishing it with `set_var` on the run's own environment would be sound for the real binary + /// — one run, one thread, no children yet — and unsound for this suite, which calls `run` from + /// forty tests at once while every other thread in the process is in `getenv`. + /// + /// The ambient variable `calibrate_harness` reads is controlled by launching a child with it + /// unset rather than by clearing it on this shared, multithreaded process, so the read races + /// nothing and there is nothing to restore afterwards. + #[test] + fn the_harness_width_is_settled_on_the_workspace_rather_than_on_this_process() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let child = run_child("calibrate", &[(TEST_THREADS_VAR, None), (CHILD_JOBS_VAR, Some("1"))]); + + assert_eq!(child["before"], PAYLOAD_MISSING, "nothing is settled before calibration"); + assert_eq!( + child["ambient"], PAYLOAD_MISSING, + "calibrating must not write the variable into the process environment" + ); + assert_eq!(child["threads"], child["cores"], "one worker gets the whole machine"); + } + + /// A width the caller chose is left alone, and so nothing is set on the launched command. + #[test] + fn a_caller_who_chose_a_harness_width_gets_no_setting_from_the_run() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let child = run_child("calibrate", &[(TEST_THREADS_VAR, Some("3")), (CHILD_JOBS_VAR, Some("4"))]); + + assert_eq!(child["threads"], PAYLOAD_MISSING, "the caller's choice stands"); + } + + #[test] + fn the_build_cannot_see_an_active_mutant() { + // A live mutant inside a proc macro would run inside rustc and could hang the one build + // the whole run depends on. + let work = unsettled_default(); + + // `Workspace::cargo` reads `CARGO_ENCODED_RUSTFLAGS` and `RUSTFLAGS`, but only reads them, + // and no test in this binary writes the process environment, so the read races nothing. + let command = work.cargo(); + let scrubbed = command + .get_envs() + .any(|(key, value)| key == gamma_rt::ACTIVE_VAR && value.is_none()); + + assert!(scrubbed, "the build environment must not carry {}", gamma_rt::ACTIVE_VAR); + } + + /// Selects which scenario an [`env_child_helper`] subprocess should run. + /// + /// Unset in an ordinary suite run, so the helper returns at once; set by [`run_child`] on the + /// re-executed test binary, whose inherited environment is then the only thing the code under + /// test reads. + const CHILD_SCENARIO_VAR: &str = "GAMMA_ENV_CHILD_SCENARIO"; + + /// Carries the worker count into the `calibrate` scenario, since `calibrate_harness` takes it + /// as an argument rather than reading it from the environment. + const CHILD_JOBS_VAR: &str = "GAMMA_ENV_CHILD_JOBS"; + + /// Opens the `key=value` block [`env_child_helper`] prints, so the parent can lift it out of + /// the test harness's own output on the shared stdout. + const PAYLOAD_OPEN: &str = "<<_RUSTFLAGS` spelling, for a triple no host builds for. + /// + /// Deliberately not the triple in force: `cap_ambient_rustflags` extends every such variable + /// rather than the one cargo will consult, because the triple is not settled where it runs, and + /// a test naming the host's own triple could not tell the two behaviours apart. + const TARGET_RUSTFLAGS_VAR: &str = "CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS"; + + /// The variables the code under test reads — `CARGO`, `RUSTFLAGS`, `CARGO_ENCODED_RUSTFLAGS`, + /// `RUST_TEST_THREADS` — are process-global and read without a lock, so a test that set them by + /// mutating this multithreaded binary would race every other thread already inside `getenv`. + /// This process is instead re-executed by [`run_child`] with those variables set on its + /// `Command`, so the values are the ones it was *launched* with and nothing ever writes the + /// running environment. The child only ever reads the environment it inherited, so its own + /// threads cannot race one another over it either. + /// + /// In an ordinary suite run the scenario variable is unset and this returns at once. + #[test] + fn env_child_helper() { + let Ok(scenario) = env::var(CHILD_SCENARIO_VAR) else { + return; + }; + + let mut payload: Vec<(&str, String)> = Vec::new(); + + match scenario.as_str() { + "cargo_binary" => payload.push(("cargo", cargo_binary())), + "cargo_flags" => { + let command = unsettled_default().cargo(); + + payload.push(("encoded", command_env(&command, "CARGO_ENCODED_RUSTFLAGS"))); + payload.push(("rustflags", command_env(&command, "RUSTFLAGS"))); + payload.push(("build", command_env(&command, "CARGO_BUILD_RUSTFLAGS"))); + payload.push(("target", command_env(&command, TARGET_RUSTFLAGS_VAR))); + } + "calibrate" => { + let jobs = env::var(CHILD_JOBS_VAR).ok().and_then(|value| value.parse().ok()).unwrap_or(1); + let work = unsettled_default(); + + payload.push(("before", option_payload(work.harness_threads()))); + work.calibrate_harness(jobs); + payload.push(("threads", option_payload(work.harness_threads()))); + payload.push(("cores", thread::available_parallelism().map_or(1, NonZeroUsize::get).to_string())); + payload.push(( + "ambient", + env::var(TEST_THREADS_VAR).unwrap_or_else(|_absent| PAYLOAD_MISSING.to_owned()), + )); + } + other => panic!("unknown child scenario `{other}`"), + } + + let mut printed = format!("{PAYLOAD_OPEN}\n"); + + for (key, value) in &payload { + printed.push_str(key); + printed.push('='); + printed.push_str(value); + printed.push('\n'); + } + + printed.push_str(PAYLOAD_CLOSE); + println!("{printed}"); + } + + /// Re-executes this test binary filtered to [`env_child_helper`] with `scenario` selected and + /// `vars` applied to the child's environment, returning the `key=value` payload it printed. + /// + /// The point is isolation: the variables are set on the child's `Command`, so the code under + /// test reads exactly the values this process chose without any thread ever writing the running + /// environment — the data race that made a safe `set_var` wrapper unsound in the first place. + fn run_child(scenario: &str, vars: &[(&str, Option<&str>)]) -> BTreeMap { + let executable = env::current_exe().expect("the test binary knows its own path"); + let mut command = Command::new(executable); + + // libtest names a test by its module path with the crate segment stripped, e.g. + // `exec::workspace::tests::env_child_helper`; `--exact` needs that whole name, not the bare + // function. `module_path!` gives the crate-qualified path, so drop the leading crate. + let module = module_path!(); + let relative = module.split_once("::").map_or(module, |(_crate_name, rest)| rest); + let target = format!("{relative}::env_child_helper"); + + let _ = command.args([target.as_str(), "--exact", "--nocapture"]); + let _ = command.env(CHILD_SCENARIO_VAR, scenario); + + for (key, value) in vars { + match value { + Some(value) => { + let _ = command.env(key, value); + } + None => { + let _ = command.env_remove(key); + } + } + } + + let output = command.output().expect("the child test binary runs"); + let stdout = String::from_utf8(output.stdout).expect("the child prints UTF-8"); + if let Some(payload) = parse_payload(&stdout) { + return payload; + } + + panic!( + "the `env_child_helper` child produced no payload (status {status}); stdout:\n{stdout}\nstderr:\n{stderr}", + status = output.status, + stderr = String::from_utf8_lossy(&output.stderr), + ); + } + + /// Lifts the `key=value` block [`env_child_helper`] prints out of the harness's surrounding + /// output, or `None` if the child never emitted a complete block. + fn parse_payload(stdout: &str) -> Option> { + let mut lines = stdout.lines().skip_while(|line| *line != PAYLOAD_OPEN); + + // Consume the PAYLOAD_OPEN marker itself; bail if the child never emitted one. + let _ = lines.next()?; + + let mut payload = BTreeMap::new(); + + for line in lines { + if line == PAYLOAD_CLOSE { + return Some(payload); + } + + if let Some((key, value)) = line.split_once('=') { + let _ = payload.insert(key.to_owned(), value.to_owned()); + } + } + + None + } + + /// The value a built `Command` carries for `key`: the string it sets, [`PAYLOAD_REMOVED`] if it + /// explicitly clears it, or [`PAYLOAD_MISSING`] if it neither sets nor mentions it. Lets a + /// subprocess report a `Command`'s environment back to the parent as text. + fn command_env(command: &Command, key: &str) -> String { + command.get_envs().find(|(name, _value)| *name == OsStr::new(key)).map_or_else( + || PAYLOAD_MISSING.to_owned(), + |(_name, value)| value.map_or_else(|| PAYLOAD_REMOVED.to_owned(), |value| value.to_string_lossy().into_owned()), + ) + } + + /// Renders an optional harness width for the payload: the value, or [`PAYLOAD_MISSING`] for + /// `None`. + fn option_payload(value: Option<&str>) -> String { + value.map_or_else(|| PAYLOAD_MISSING.to_owned(), ToOwned::to_owned) + } + + /// The tests re-execute a child instead of mutating this multithreaded parent precisely so that + /// a value one test needs cannot contaminate — or race a reader in — another. This proves both + /// halves: a distinctive value is injected into the child, the child reads it back through the + /// code under test, and the parent's own environment is shown to be exactly what it was before. + #[test] + fn the_child_helper_isolates_the_environment_from_the_parent() { + // A value the parent does not carry: a process-wide write of it would be visible here both + // before and after, and racing to spawn a real cargo elsewhere with it would break. + const INJECTED: &str = "/gamma/isolated/child/only/cargo"; + + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let before = env::var_os("CARGO"); + let child = run_child("cargo_binary", &[("CARGO", Some(INJECTED))]); + + assert_eq!(child["cargo"], INJECTED, "the child read the value it was launched with"); + assert_eq!(env::var_os("CARGO"), before, "the child's environment did not leak into the parent"); + } + + /// Honouring `CARGO` matters because the toolchain that invoked this run may not be the one on + /// `PATH` — `rustup` overrides, pinned toolchains, and CI runners that vendor a specific cargo + /// all set it — so falling back to the literal name `cargo` unconditionally would silently + /// build with the wrong toolchain. Both the honoured and the fallback case have to work. + /// + /// A made-up path is safe as the honoured value: it is set on the child's `Command`, not on + /// this process, so it cannot reach the other tests spawning real cargo processes beside this + /// one. + #[test] + fn cargo_binary_prefers_the_invoking_cargo_but_falls_back_when_it_is_unset() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let honoured = run_child("cargo_binary", &[("CARGO", Some("/opt/pinned/toolchain/bin/cargo"))]); + assert_eq!( + honoured["cargo"], "/opt/pinned/toolchain/bin/cargo", + "a set CARGO is honoured verbatim" + ); + + let fallback = run_child("cargo_binary", &[("CARGO", None)]); + assert_eq!(fallback["cargo"], "cargo", "an unset CARGO falls back to the bare name"); + } + + /// [`unsettled_default`] over a real directory, for the tests that touch the tree. + /// + /// Spelled out rather than deriving from [`unsettled_default`] with struct update syntax: + /// `Workspace` implements `Drop`, so its fields cannot be moved out of another instance. + fn unsettled_at(root: Utf8PathBuf, runtime: Utf8PathBuf) -> Workspace { + Workspace { + root, + runtime, + target: Utf8PathBuf::from("/scratch/build"), + libraries: Vec::new(), + cargo: CargoOptions::default(), + nextest: None, + settled: AtomicBool::new(true), + leak: true, + launch: OnceLock::new(), + harness_threads: OnceLock::new(), + _workspace_lock: tempfile::tempfile().unwrap(), + _cache_lock: None, + torn_down: false, + } + } + + /// A workspace over no real directory, for tests that only inspect the `Command` it builds. + /// + /// Leaked rather than torn down: nothing was ever created, so nothing should be removed. + fn unsettled_default() -> Workspace { + Workspace { + root: Utf8PathBuf::from("/tmp/gamma-root"), + runtime: Utf8PathBuf::from("/tmp/gamma-rt"), + target: Utf8PathBuf::from("/tmp/gamma-target"), + libraries: Vec::new(), + cargo: CargoOptions::default(), + nextest: None, + settled: AtomicBool::new(true), + leak: true, + launch: OnceLock::new(), + harness_threads: OnceLock::new(), + _workspace_lock: tempfile::tempfile().unwrap(), + _cache_lock: None, + torn_down: false, + } + } + + /// `CARGO_ENCODED_RUSTFLAGS` uses a unit separator rather than a space, and cargo prefers it + /// over the plain form; extending it in place rather than replacing it keeps whatever flags the + /// user's own environment configured instead of silently dropping them from the instrumented + /// build. + #[test] + fn an_ambient_encoded_rustflags_is_extended_rather_than_replaced() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let child = run_child( + "cargo_flags", + &[("CARGO_ENCODED_RUSTFLAGS", Some("--cfg\u{1f}loom")), ("RUSTFLAGS", None)], + ); + let value = &child["encoded"]; + + assert!(value.contains("--cfg\u{1f}loom"), "{value}"); + assert!(value.contains(CAP_LINTS), "{value}"); + } + + /// The plain `RUSTFLAGS` form is only read when the encoded one is absent, but it must be + /// extended the same way: replacing it would silently drop whatever the caller's environment + /// configured, changing what the instrumented tree compiles to. + #[test] + fn an_ambient_rustflags_is_extended_rather_than_replaced() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let child = run_child( + "cargo_flags", + &[("CARGO_ENCODED_RUSTFLAGS", None), ("RUSTFLAGS", Some("--cfg loom"))], + ); + let value = &child["rustflags"]; + + assert!(value.contains("--cfg loom"), "{value}"); + assert!(value.contains(CAP_LINTS), "{value}"); + } + + /// `CARGO_BUILD_RUSTFLAGS` replaces the `[build] rustflags` array `cap_lints` writes into the + /// copied tree, so the cap has to be added to the variable as well. + /// + /// Left out, an ambient `-D warnings` — routine in CI images and `direnv` setups — compiles the + /// instrumented tree under deny-by-default: guard-induced warnings become errors, the mutants + /// that produced them are withdrawn as unviable, and the score is computed over a silently + /// smaller denominator. + #[test] + fn an_ambient_build_rustflags_carries_the_cap() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let child = run_child( + "cargo_flags", + &[ + ("CARGO_ENCODED_RUSTFLAGS", None), + ("RUSTFLAGS", None), + ("CARGO_BUILD_RUSTFLAGS", Some("-D warnings")), + ], + ); + let value = &child["build"]; + + assert!(value.contains("-D warnings"), "{value}"); + assert!(value.contains(CAP_LINTS), "{value}"); + } + + /// The same for the target-table spelling, which outranks the build one. + #[test] + fn an_ambient_target_rustflags_carries_the_cap() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let child = run_child( + "cargo_flags", + &[ + ("CARGO_ENCODED_RUSTFLAGS", None), + ("RUSTFLAGS", None), + (TARGET_RUSTFLAGS_VAR, Some("-D warnings")), + ], + ); + let value = &child["target"]; + + assert!(value.contains("-D warnings"), "{value}"); + assert!(value.contains(CAP_LINTS), "{value}"); + } + + /// A global spelling beats every configured key, so it is the only one extended. + /// + /// Extending the lower levels as well would be harmless to the build — cargo never reads them + /// once a global one is set — but it would put the tool's flag into variables the child process + /// passes on to anything it in turn runs. + #[test] + fn a_global_rustflags_leaves_the_lower_spellings_alone() { + if env::var_os(crate::exec::UNDER_GAMMA_VAR).is_some() { + return; + } + + let child = run_child( + "cargo_flags", + &[ + ("CARGO_ENCODED_RUSTFLAGS", None), + ("RUSTFLAGS", Some("--cfg loom")), + ("CARGO_BUILD_RUSTFLAGS", Some("-D warnings")), + ], + ); + + assert!(child["rustflags"].contains(CAP_LINTS), "{}", child["rustflags"]); + assert_eq!( + child["build"], PAYLOAD_MISSING, + "cargo will not read this one, so it must be left as it was" + ); + } + + #[test] + fn nextest_inventories_the_profile_gamma_built() { + let mut work = unsettled_default(); + work.cargo.features = vec!["--all-features".to_owned()]; + work.cargo.profile = Some("mutants".to_owned()); + let binaries = [TestBinary { + package: "subject".to_owned(), + package_id: "path+file:///tmp/subject#subject@0.1.0".to_owned(), + ..crate::testing::test_binary("/tmp/subject") + }]; + + let command = work.nextest_list_command(&binaries); + let args: Vec<_> = command.get_args().map(|arg| arg.to_string_lossy().into_owned()).collect(); + + assert_eq!( + args, + vec![ + "nextest", + "list", + "--list-type", + "binaries-only", + "--message-format", + "json", + "--package", + "path+file:///tmp/subject#subject@0.1.0", + "--all-features", + "--cargo-profile", + "mutants", + ] + ); + } + + #[test] + fn scratch_tree_is_derived_from_the_same_base_as_prepare() { + // Callers report this path after deliberately leaking the workspace, so it has to match + // the tree `prepare` would have created. + assert_eq!( + scratch_tree(Utf8Path::new("/workspace"), None), + gamma_base(Utf8Path::new("/workspace"), None).join("workspace") + ); + assert_eq!( + scratch_tree(Utf8Path::new("/workspace"), Some(Utf8Path::new("/scratch"))), + gamma_base(Utf8Path::new("/workspace"), Some(Utf8Path::new("/scratch"))).join("workspace") + ); + } + + #[test] + fn the_default_scratch_tree_cannot_rediscover_workspace_cargo_configuration() { + let root = Utf8Path::new("/workspace"); + let base = gamma_base(root, None); + + assert!(!base.starts_with(root), "{base}"); + assert!( + !base.ancestors().any(|ancestor| ancestor == root.join(".cargo")), + "the real workspace configuration remains in Cargo's scratch ancestor chain: {base}" + ); + } + + #[test] + fn array_rustflags_from_workspace_config_reach_scratch_cargo_once() { + let directory = crate::testing::workdir("scratch-config-once"); + let source = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("utf8"); + let capture = source.join("captured-flags"); + + fs::create_dir_all(source.join(".cargo").as_std_path()).expect(".cargo"); + fs::create_dir_all(source.join("src").as_std_path()).expect("src"); + fs::write( + source.join("Cargo.toml").as_std_path(), + "[package]\nname = \"scratch-config-once\"\nversion = \"0.0.0\"\nedition = \"2024\"\nbuild = \"build.rs\"\n\n[workspace]\n", + ) + .expect("manifest"); + fs::write(source.join("src/lib.rs").as_std_path(), "").expect("lib"); + fs::write( + source.join("build.rs").as_std_path(), + "fn main() { std::fs::write(std::env::var(\"GAMMA_CAPTURE\").unwrap(), std::env::var(\"CARGO_ENCODED_RUSTFLAGS\").unwrap()).unwrap(); }\n", + ) + .expect("build script"); + fs::write( + source.join(".cargo/config.toml").as_std_path(), + "[build]\nrustflags = [\"--cfg\", \"gamma_once\"]\n", + ) + .expect("config"); + + let mut events = crate::testing::Recorder::default(); + let work = Workspace::prepare(&source, &Config::default(), &mut events).expect("prepare"); + let mut command = work.cargo(); + let status = command + .env_remove("CARGO_ENCODED_RUSTFLAGS") + .env_remove("RUSTFLAGS") + .env_remove("CARGO_BUILD_RUSTFLAGS") + .env("GAMMA_CAPTURE", capture.as_std_path()) + .arg("check") + .status() + .expect("cargo check"); + + assert!(status.success(), "{status}"); + let flags = fs::read_to_string(capture.as_std_path()).expect("captured flags"); + assert_eq!(flags.matches("gamma_once").count(), 1, "{flags}"); + } + + /// A relative `--cache-dir` reaches the copy as the path the user typed, and the copy prunes + /// its own destination by comparing that against the absolute paths it walks — which never + /// match, so the copy descends into what it is writing. Absolutising the base is what stops it. + #[test] + fn a_relative_cache_directory_is_resolved_against_the_current_directory() { + let cwd = Utf8PathBuf::from_path_buf(env::current_dir().unwrap()).unwrap(); + let base = gamma_base(Utf8Path::new("/workspace"), Some(Utf8Path::new("scratch/here"))); + + assert_eq!(base, cwd.join("scratch/here")); + assert!(base.is_absolute(), "{base}"); + + // The tree, the build directory and the runtime all hang off the base, so absolutising it + // once makes every one of them absolute. + assert!(scratch_tree(Utf8Path::new("/workspace"), Some(Utf8Path::new("scratch/here"))).is_absolute()); + + // A workspace reached by a relative path gets the same treatment. + assert_eq!(gamma_base(Utf8Path::new("."), None), gamma_base(&cwd, None)); + } + + /// An absolute cache directory outside the workspace is the arrangement everything else is + /// built on, and must come back exactly as given — a run whose cache path moved between two + /// invocations rebuilds from cold. + #[test] + fn an_absolute_cache_directory_outside_the_workspace_is_left_alone() { + let base = gamma_base(Utf8Path::new("/workspace"), Some(Utf8Path::new("/elsewhere/scratch"))); + + assert_eq!(base, absolute(Utf8Path::new("/elsewhere/scratch"))); + ensure_copy_terminates(Utf8Path::new("/workspace"), &base).expect("a scratch directory outside the workspace is fine"); + + // An explicitly selected path inside the workspace is fine as long as the copy prunes it. + let inside = gamma_base(Utf8Path::new("/workspace"), None); + + ensure_copy_terminates(Utf8Path::new("/workspace"), &inside).expect("the default base is outside the copy"); + assert!(!inside.starts_with("/workspace"), "{inside}"); + } + + /// The walk never tests its own root against the exclusion, so a base that is the workspace + /// itself would have the copy copying its own output. Producing a tree that contains a copy of + /// itself measures mutants in the wrong files, so the run stops instead. + #[test] + fn a_scratch_directory_the_copy_cannot_prune_is_refused() { + let source = absolute(Utf8Path::new("/workspace/gamma")); + let base = gamma_base(&source, Some(&source)); + + assert_eq!(base, source); + assert!(!prunes(&source, &base)); + + let failure = ensure_copy_terminates(&source, &base).expect_err("an unprunable base must be refused"); + + assert!(failure.is_usage(), "{failure}"); + assert!(failure.to_string().contains(source.as_str()), "{failure}"); + assert!(failure.to_string().contains("--cache-dir"), "{failure}"); + } + + /// A scratch directory that reaches back into the workspace through a link is refused. + /// + /// The copy's exclusion is a path comparison against the paths the walk produces, and the walk + /// produces the physical spelling. A base named outside the workspace therefore never matches, + /// however deep inside the workspace it really is — so the copy descends into its own output + /// and writes until the disk is gone. A lexical check answers this one wrongly, which is the + /// whole reason the check resolves the links first. + #[test] + #[cfg(unix)] + fn a_scratch_directory_linked_back_into_the_workspace_is_refused() { + let directory = crate::testing::workdir("scratch-linked"); + let root = Utf8Path::from_path(directory.path()).expect("the scratch path is UTF-8"); + let (source, link) = (root.join("workspace"), root.join("link")); + + fs::create_dir_all(source.join("inside").as_std_path()).expect("the workspace is creatable"); + std::os::unix::fs::symlink(source.join("inside").as_std_path(), link.as_std_path()).expect("the link is creatable"); + + // Named outside the workspace, and lexically it is: only resolving it says otherwise. + let base = gamma_base(&source, Some(&link)); + + assert!(!base.starts_with(&source), "{base}"); + + let failure = ensure_copy_terminates(&source, &base).expect_err("a base linked into the workspace must be refused"); + + assert!(failure.is_usage(), "{failure}"); + assert!(failure.to_string().contains("--cache-dir"), "{failure}"); + } + + /// The mirror image: a base named *inside* the workspace, so it looks pruned, but reached + /// through an in-workspace link so the walk never produces the path the exclusion compares + /// against. Textual containment alone would call this safe and the copy would copy itself. + #[cfg(unix)] + #[test] + fn a_scratch_directory_reached_through_an_in_workspace_link_is_refused() { + let directory = crate::testing::workdir("scratch-through-link"); + let root = Utf8Path::from_path(directory.path()).expect("the scratch path is UTF-8"); + + fs::create_dir_all(root.join("workspace").join("inside").as_std_path()).expect("the workspace is creatable"); + + // `workdir` hands back a path holding `..` components while `gamma_base` normalises what it + // is given, so the two are resolved to the same spelling before they are compared. + let source = physical(&root.join("workspace")); + + std::os::unix::fs::symlink(source.join("inside").as_std_path(), source.join("link").as_std_path()).expect("the link is creatable"); + + let base = gamma_base(&source, Some(&source.join("link"))); + + // Named inside the workspace, so the copy's exclusion looks like it will skip it. + assert!(prunes(&source, &base), "source={source} base={base}"); + + let failure = ensure_copy_terminates(&source, &base).expect_err("a base the exclusion cannot match must be refused"); + + assert!(failure.is_usage(), "{failure}"); + assert!(failure.to_string().contains("--cache-dir"), "{failure}"); + } + + /// The same shape pointing somewhere genuinely outside stays allowed, which is what makes the + /// case above a measurement of where the link goes rather than of links in general. + #[test] + #[cfg(unix)] + fn a_scratch_directory_linked_to_somewhere_outside_is_still_allowed() { + let directory = crate::testing::workdir("scratch-linked-out"); + let root = Utf8Path::from_path(directory.path()).expect("the scratch path is UTF-8"); + let (source, elsewhere, link) = (root.join("workspace"), root.join("elsewhere"), root.join("link")); + + fs::create_dir_all(source.as_std_path()).expect("the workspace is creatable"); + fs::create_dir_all(elsewhere.as_std_path()).expect("the target is creatable"); + std::os::unix::fs::symlink(elsewhere.as_std_path(), link.as_std_path()).expect("the link is creatable"); + + let base = gamma_base(&source, Some(&link)); + + ensure_copy_terminates(&source, &base).expect("a base that really is outside the workspace is fine"); + } + + /// `prepare` has to ask before it creates anything, or the refusal arrives after the run has + /// already written a lock file and a tree into the user's workspace. + #[test] + fn preparing_with_a_scratch_directory_the_copy_cannot_prune_fails_before_copying() { + let directory = crate::testing::workdir("scratch-inside-"); + let outer = Utf8PathBuf::from_path_buf(directory.path().to_path_buf()).expect("the scratch path is UTF-8"); + let source = outer.join("gamma"); + + fs::create_dir_all(source.as_std_path()).expect("the workspace"); + fs::write(source.join("Cargo.toml").as_std_path(), "[workspace]\nmembers = []\n").expect("a manifest"); + + let config = Config { + cache_dir: Some(source.clone()), + ..Config::default() + }; + let mut events = crate::testing::Recorder::default(); + let failure = Workspace::prepare(&source, &config, &mut events).expect_err("the run must be refused"); + + assert!(failure.to_string().contains(absolute(&source).as_str()), "{failure}"); + assert!( + !source.join("workspace").as_std_path().exists(), + "the refusal came after the copy had already started" + ); + } + + /// The copy intentionally excludes all VCS directories. A scratch tree under the source still + /// sees the same ancestor metadata, but a relocated one does not, so it is refused rather + /// than letting a build script silently observe a different repository. + #[test] + fn relocating_a_workspace_that_exposes_vcs_metadata_is_refused_before_copying() { + let directory = crate::testing::workdir("scratch-vcs-relocation-"); + let source = Utf8PathBuf::from_path_buf(directory.path().join("source")).expect("UTF-8 path"); + let scratch = Utf8PathBuf::from_path_buf(directory.path().join("external")).expect("UTF-8 path"); + let marker = source.join(".git/HEAD"); + + fs::create_dir_all(source.join(".git").as_std_path()).expect("VCS metadata"); + fs::write(marker.as_std_path(), "ref: refs/heads/main\n").expect("VCS metadata"); + fs::write(source.join("Cargo.toml").as_std_path(), "[workspace]\nmembers = []\n").expect("manifest"); + + let config = Config { + cache_dir: Some(scratch), + ..Config::default() + }; + let failure = Workspace::prepare(&source, &config, &mut crate::testing::Recorder::default()) + .expect_err("a relocated tree must not lose VCS metadata"); + + assert!(failure.is_usage(), "{failure}"); + assert!(failure.to_string().contains("--cache-dir"), "{failure}"); + assert_eq!( + fs::read_to_string(marker.as_std_path()).expect("VCS metadata"), + "ref: refs/heads/main\n" + ); + } + + /// `..` and `.` in a scratch path would leave the copy comparing two spellings of one + /// directory, which never match, so the exclusion silently stops working. + #[test] + fn a_scratch_path_that_names_nothing_is_reduced_to_one_spelling() { + assert_eq!(absolute(Utf8Path::new("/a/./b/../c")), absolute(Utf8Path::new("/a/c"))); + + // Nothing precedes a leading `..`, so it is left to mean what the filesystem says. + assert!( + absolute(Utf8Path::new("/../a")) + .components() + .any(|component| component == Utf8Component::ParentDir) + ); + } + + #[test] + fn linking_a_package_with_no_manifest_is_a_noop() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + fs::create_dir_all(root.join("crate").join("src").as_std_path()).unwrap(); + let work = unsettled_at(root, Utf8PathBuf::from("/scratch/rt")); + let files = vec![TargetFile { + package: "pkg".to_owned(), + path: Utf8PathBuf::from("crate/src/lib.rs"), + absolute: Utf8PathBuf::from("/source/crate/src/lib.rs"), + }]; + + work.link_runtime("pkg", &files).unwrap(); + + // A scanned file might be malformed or synthetic in tests; absent manifests are ignored + // rather than making runtime linking fail before instrumentation can explain anything. + assert!(!work.root.join("Cargo.toml").as_std_path().exists()); + } + + #[test] + fn manifest_lookup_stops_at_the_workspace_root() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + fs::create_dir_all(root.join("crate").join("src").as_std_path()).unwrap(); + let work = unsettled_at(root, Utf8PathBuf::from("/scratch/rt")); + let files = vec![TargetFile { + package: "pkg".to_owned(), + path: Utf8PathBuf::from("crate/src/lib.rs"), + absolute: Utf8PathBuf::from("/source/crate/src/lib.rs"), + }]; + + // Walking above the copied root would let an unrelated parent manifest claim this package. + assert_eq!(work.manifest_of("pkg", &files), None); + } + + /// A package that was never scanned into the file list has nothing to look a manifest up from; + /// treating that as "found nowhere" rather than panicking on an empty search keeps a caller + /// naming an unfamiliar package from crashing the run instead of just skipping the link. + #[test] + fn a_package_absent_from_the_scanned_files_has_no_manifest() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + fs::create_dir_all(root.join("crate").join("src").as_std_path()).unwrap(); + let work = unsettled_at(root, Utf8PathBuf::from("/scratch/rt")); + let files = vec![TargetFile { + package: "pkg".to_owned(), + path: Utf8PathBuf::from("crate/src/lib.rs"), + absolute: Utf8PathBuf::from("/source/crate/src/lib.rs"), + }]; + + assert_eq!(work.manifest_of("someone-else", &files), None); + work.link_runtime("someone-else", &files) + .expect("an unknown package is a noop, not an error"); + } + + /// Linking a package whose manifest really is in the copied tree has to write the dependency + /// rather than merely locate the file: a run that reached instrumentation with the runtime + /// unlinked would fail every guard's compile with an unresolved path, indistinguishable from a + /// real bug in the mutated code. + #[test] + fn linking_a_package_whose_manifest_exists_adds_the_runtime_dependency() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + let runtime = Utf8PathBuf::from_path_buf(temporary.path().join("rt")).unwrap(); + + fs::create_dir_all(root.join("crate").join("src").as_std_path()).unwrap(); + fs::write( + root.join("crate/Cargo.toml").as_std_path(), + "[package]\nname = \"pkg\"\nversion = \"0.0.0\"\nedition = \"2024\"\n", + ) + .unwrap(); + + let work = unsettled_at(root, runtime); + let files = vec![TargetFile { + package: "pkg".to_owned(), + path: Utf8PathBuf::from("crate/src/lib.rs"), + absolute: Utf8PathBuf::from("/source/crate/src/lib.rs"), + }]; + + work.link_runtime("pkg", &files).expect("a real manifest must be linkable"); + + let manifest = fs::read_to_string(work.root.join("crate/Cargo.toml").as_std_path()).unwrap(); + + assert!(manifest.contains(RUNTIME_CRATE), "{manifest}"); + } + + #[test] + fn overwriting_a_directory_is_refused() { + let temporary = tempfile::tempdir().unwrap(); + let path = Utf8PathBuf::from_path_buf(temporary.path().join("not-a-file")).unwrap(); + let root = path.parent().expect("the temporary directory is the root").to_owned(); + + fs::create_dir_all(path.as_std_path()).unwrap(); + + let cause = Workspace::overwrite(&root, &path, "new").unwrap_err(); + + // Instrumentation only writes files the copy already chose; refusing other entry kinds + // prevents following a link or clobbering a device outside the scratch tree. + assert!(cause.to_string().contains("refusing to write"), "{cause}"); + } + + /// A path the copy never created has no metadata to inspect at all, and that is reported as + /// its own failure rather than folded into "refusing to write": the two causes point a reader + /// in different directions, one at a copy that is missing a file and the other at a link or + /// device standing where a file should be. + #[test] + fn overwriting_a_path_the_copy_never_created_is_reported_rather_than_silently_writing_a_new_file() { + let temporary = tempfile::tempdir().unwrap(); + let path = Utf8PathBuf::from_path_buf(temporary.path().join("never-copied.rs")).unwrap(); + let root = path.parent().expect("the temporary directory is the root").to_owned(); + + let cause = Workspace::overwrite(&root, &path, "new").unwrap_err(); + + assert!(cause.to_string().contains("which the copy did not create"), "{cause}"); + } + + /// Instrumentation exists to replace a file's contents, so both directions matter: a changed + /// file has to be rewritten and reported as such, and a file already holding what would be + /// written must be left untouched and reported as a no-op, or the rollback loop would rebuild + /// every crate every round regardless of what actually changed. + #[test] + fn overwriting_a_copied_file_writes_only_when_the_content_actually_differs() { + let temporary = tempfile::tempdir().unwrap(); + let path = Utf8PathBuf::from_path_buf(temporary.path().join("lib.rs")).unwrap(); + let root = path.parent().expect("the temporary directory is the root").to_owned(); + + fs::write(path.as_std_path(), "old").unwrap(); + + let changed = Workspace::overwrite(&root, &path, "new").expect("a real file must be writable"); + assert!(changed, "different content should be reported as written"); + assert_eq!(fs::read_to_string(path.as_std_path()).unwrap(), "new"); + + let unchanged = Workspace::overwrite(&root, &path, "new").expect("identical content must still succeed"); + assert!(!unchanged, "identical content should be reported as a no-op"); + } + + /// A file the copy created but that has since become unwritable cannot be instrumented, and + /// that has to surface as the write failure it is rather than an overwrite that silently kept + /// stale, uninstrumented source in the tree. + #[cfg(unix)] + #[test] + fn overwriting_a_read_only_file_reports_the_write_failure() { + use std::os::unix::fs::PermissionsExt; + + let temporary = tempfile::tempdir().unwrap(); + let path = Utf8PathBuf::from_path_buf(temporary.path().join("locked.rs")).unwrap(); + let root = path.parent().expect("the temporary directory is the root").to_owned(); + + fs::write(path.as_std_path(), "old").unwrap(); + fs::set_permissions(path.as_std_path(), fs::Permissions::from_mode(0o400)).unwrap(); + + let cause = Workspace::overwrite(&root, &path, "new").unwrap_err(); + + // Restore write permission so the temporary directory can clean itself up. + fs::set_permissions(path.as_std_path(), fs::Permissions::from_mode(0o600)).unwrap(); + + assert!(cause.to_string().contains("could not write"), "{cause}"); + } + + /// `symlink_metadata` is non-following for the last component alone, so a file whose *prefix* + /// is a link passes that check while the write itself lands wherever the link points. The copy + /// recreates links verbatim — including absolute targets — so a scratch tree really can hold + /// one aimed at the user's source. Containment is therefore decided on the physical path. + #[cfg(unix)] + #[test] + fn overwriting_through_a_symlinked_directory_prefix_is_refused() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + let outside = Utf8PathBuf::from_path_buf(temporary.path().join("real")).unwrap(); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(outside.as_std_path()).unwrap(); + fs::write(outside.join("lib.rs").as_std_path(), "user source").unwrap(); + std::os::unix::fs::symlink(outside.as_std_path(), root.join("link").as_std_path()).unwrap(); + + let through_the_link = root.join("link").join("lib.rs"); + + // The final component is an ordinary file, so the existing guard is satisfied. + assert!(fs::symlink_metadata(through_the_link.as_std_path()).unwrap().is_file()); + + let cause = Workspace::overwrite(&root, &through_the_link, "instrumented") + .expect_err("a write resolving outside the scratch tree must be refused"); + + assert!(cause.to_string().contains("outside the scratch tree"), "{cause}"); + assert_eq!(fs::read_to_string(outside.join("lib.rs").as_std_path()).unwrap(), "user source"); + } + + /// The same escape by the other route: `is_file` resolves every component, so the upward walk + /// finds the user's real manifest and hands it to `Manifest::save`, which rewrites it. + #[cfg(unix)] + #[test] + fn a_manifest_reached_through_a_symlinked_directory_prefix_is_not_returned() { + let temporary = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + let outside = Utf8PathBuf::from_path_buf(temporary.path().join("real")).unwrap(); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::create_dir_all(outside.join("src").as_std_path()).unwrap(); + fs::write(outside.join("Cargo.toml").as_std_path(), "[package]\nname = \"pkg\"\n").unwrap(); + fs::write(outside.join("src").join("lib.rs").as_std_path(), "").unwrap(); + std::os::unix::fs::symlink(outside.as_std_path(), root.join("link").as_std_path()).unwrap(); + + let work = Workspace::adopt(root.clone(), root.join("target")); + let files = vec![TargetFile { + path: Utf8PathBuf::from("link/src/lib.rs"), + absolute: root.join("link/src/lib.rs"), + package: "pkg".to_owned(), + }]; + + assert_eq!(work.manifest_of("pkg", &files), None); + } + + #[test] + fn a_second_claim_on_the_same_scratch_directory_is_a_usage_error() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().join("gamma")).unwrap(); + + fs::create_dir_all(base.as_std_path()).unwrap(); + let _held = claim(&base).unwrap(); + let cause = claim(&base).unwrap_err(); + + // The lock serializes commands targeting the same original workspace. + assert!(cause.is_usage()); + assert!(cause.to_string().contains("already using"), "{cause}"); + } + + /// A filesystem that cannot lock at all is reported as such, not as another run holding it. + /// + /// The two call for opposite responses: waiting out a run that does not exist never ends, and + /// the advice to move the scratch directory cannot help when the new location is on the same + /// mount. NFS without `lockd` and some CIFS mounts behave this way. + #[test] + fn a_filesystem_that_cannot_lock_is_not_reported_as_another_run() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().to_path_buf()).unwrap(); + let _armed = faults::arm(Fault::Lock); + + let cause = claim(&base).unwrap_err(); + let message = cause.to_string(); + + assert!(message.contains("could not be taken"), "{message}"); + assert!(!message.contains("already using"), "{message}"); + assert!(!cause.is_usage(), "a filesystem the tool cannot lock is not the user's mistake"); + } + + /// A base directory that does not exist yet cannot have its lock file opened, and that has to + /// be reported by name rather than treated the same as another run already holding the lock: + /// the two causes call for opposite fixes, one for a caller that skipped creating the scratch + /// directory and the other for a run genuinely already in progress. + #[test] + fn claiming_a_scratch_directory_that_does_not_exist_reports_the_lock_failure() { + let temporary = tempfile::tempdir().unwrap(); + let base = Utf8PathBuf::from_path_buf(temporary.path().join("never-created")).unwrap(); + + let cause = claim(&base).unwrap_err(); + + assert!(cause.to_string().contains("could not open the scratch lock"), "{cause}"); + } + + #[cfg(unix)] + #[test] + fn non_utf8_manifests_are_skipped_while_anchoring() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt as _; + + let temporary = tempfile::tempdir().unwrap(); + let source = Utf8PathBuf::from_path_buf(temporary.path().join("source")).unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + let name = OsString::from_vec(b"bad-\xff".to_vec()); + let bad = root.as_std_path().join(name); + + fs::create_dir_all(&bad).unwrap(); + fs::write(bad.join("Cargo.toml"), "[package]\nname = \"bad\"\nversion = \"0.0.0\"\n").unwrap(); + + anchor_manifests(&source, &root).unwrap(); + + // A path that cannot appear in cargo's UTF-8 JSON is left alone rather than poisoning the + // whole copied tree repair pass. + assert!(bad.join("Cargo.toml").exists()); + } + + /// A walk over the copied tree passes every file it finds, and only `Cargo.toml` is a manifest; + /// anything else — a `Cargo.lock`, a source file, a stray artifact — has to be skipped without + /// being misread as one, or the copy pass would fail on the first ordinary file it walked over. + #[test] + fn files_that_are_not_manifests_are_left_alone_while_anchoring() { + let temporary = tempfile::tempdir().unwrap(); + let source = Utf8PathBuf::from_path_buf(temporary.path().join("source")).unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::write(root.join("Cargo.toml").as_std_path(), "[workspace]\nmembers = []\n").unwrap(); + fs::write(root.join("Cargo.lock").as_std_path(), "# not a manifest\n").unwrap(); + fs::write(root.join("lib.rs").as_std_path(), "pub fn f() {}\n").unwrap(); + + anchor_manifests(&source, &root).unwrap(); + + assert_eq!( + fs::read_to_string(root.join("Cargo.lock").as_std_path()).unwrap(), + "# not a manifest\n" + ); + assert_eq!(fs::read_to_string(root.join("lib.rs").as_std_path()).unwrap(), "pub fn f() {}\n"); + } + + #[cfg(unix)] + #[test] + fn an_external_manifest_link_is_refused_before_its_target_is_rewritten() { + let temporary = tempfile::tempdir().unwrap(); + let source = Utf8PathBuf::from_path_buf(temporary.path().join("source")).unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + let outside = Utf8PathBuf::from_path_buf(temporary.path().join("outside.toml")).unwrap(); + let original = "[package]\nname = \"outside\"\nversion = \"0.0.0\"\n\n[dependencies]\nshared = { path = \"../shared\" }\n"; + + fs::create_dir_all(root.as_std_path()).unwrap(); + fs::write(outside.as_std_path(), original).unwrap(); + std::os::unix::fs::symlink(outside.as_std_path(), root.join("Cargo.toml").as_std_path()).unwrap(); + + let failure = anchor_manifests(&source, &root).expect_err("an external manifest must not be rewritten"); + + assert!(failure.to_string().contains("outside"), "{failure}"); + assert_eq!(fs::read_to_string(outside.as_std_path()).unwrap(), original); + } + + #[cfg(unix)] + #[test] + fn an_external_cargo_configuration_link_is_refused_before_its_target_is_rewritten() { + let temporary = tempfile::tempdir().unwrap(); + let source = Utf8PathBuf::from_path_buf(temporary.path().join("source")).unwrap(); + let root = Utf8PathBuf::from_path_buf(temporary.path().join("tree")).unwrap(); + let outside = Utf8PathBuf::from_path_buf(temporary.path().join("outside-config.toml")).unwrap(); + let original = "paths = [\"../shared\"]\n"; + + fs::create_dir_all(root.join(".cargo").as_std_path()).unwrap(); + fs::write(outside.as_std_path(), original).unwrap(); + std::os::unix::fs::symlink(outside.as_std_path(), root.join(".cargo/config.toml").as_std_path()).unwrap(); + + let failure = anchor_manifests(&source, &root).expect_err("an external config must not be rewritten"); + + assert!(failure.to_string().contains("outside"), "{failure}"); + assert_eq!(fs::read_to_string(outside.as_std_path()).unwrap(), original); + } +} diff --git a/crates/cargo-gamma-lib/src/fix/diff.rs b/crates/cargo-gamma-lib/src/fix/diff.rs new file mode 100644 index 000000000..2822a9d08 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/diff.rs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Rendering a unified-style diff of one file, for a dry run. + +use core::fmt::Write as _; + +use camino::Utf8Path; + +/// Renders a unified-style diff of one file, for a dry run. +/// +/// Whole-file rather than hunked: the edits are a handful of one-line directives scattered through +/// a source file, and a reviewer deciding whether to let the tool touch their tree is reading the +/// code around them, not counting lines. +#[must_use] +pub fn diff(path: &Utf8Path, before: &str, after: &str) -> String { + let old: Vec<&str> = before.lines().collect(); + let new: Vec<&str> = after.lines().collect(); + let mut out = format!("--- {path}\n+++ {path}\n"); + + for step in script(&old, &new) { + let _ = match step { + Step::Kept(line) => writeln!(out, " {line}"), + Step::Added(line) => writeln!(out, "+{line}"), + Step::Removed(line) => writeln!(out, "-{line}"), + }; + } + + out +} + +/// One line's fate in a diff. +enum Step<'a> { + Kept(&'a str), + Added(&'a str), + Removed(&'a str), +} + +/// The number of single-line differences past which the diff is not worth computing exactly. +/// +/// The exact algorithm costs `O((n + m) · d)` time and `O(d²)` memory in the number of differing +/// lines, which is ideal for the handful of directive lines this module writes and ruinous for two +/// texts with nothing in common. Past the cap the answer degrades to "all of this became all of +/// that", which is both honest and cheap — and only reachable by a caller doing something this +/// module does not do. +const DIFF_LIMIT: usize = 2_000; + +/// The line-by-line edit script turning `old` into `new`. +/// +/// Myers' greedy algorithm: walk diagonals of the edit graph outwards from the origin, recording +/// each round, and once the far corner is reached walk the rounds backwards to recover the moves +/// that got there. It is worth the fifty lines over the obvious "anything that does not match is an +/// insertion" because that one cannot express a deletion at all, and removing a directive is +/// exactly a deletion. +#[expect( + clippy::many_single_char_names, + reason = "n, m, d, k, x and y are Myers' own names for these; renaming them would make the algorithm harder to check against the paper, not easier" +)] +fn script<'a>(old: &[&'a str], new: &[&'a str]) -> Vec> { + let (n, m) = (old.len(), new.len()); + let max = n + m; + + // Two empty texts are already at the far corner, and the loop below cannot express that. Its + // first round reads the diagonal to the right of `k = 0`, which exists only because `max` is at + // least one; with nothing on either side the shifted index runs off the end of a one-element + // vector. An empty file rewritten to an empty file is a real call — a source file holding + // nothing but directives, all of them removed — so this is a returned answer, not an assertion. + if max == 0 { + return Vec::new(); + } + + if max > DIFF_LIMIT && old != new { + let mut steps: Vec> = old.iter().map(|line| Step::Removed(line)).collect(); + + steps.extend(new.iter().map(|line| Step::Added(line))); + + return steps; + } + + // Indexed by diagonal `k = x - y`, which runs from `-max` to `max`, so it is stored shifted. + let mut furthest = vec![0_isize; 2 * max + 1]; + + // Each round's state, kept as one flat buffer rather than a vector of snapshots. Round `d` only + // ever needs the diagonals `-d..=d` — every other entry is either impossible to have reached or + // never read by the walk back — so a window of `2d + 1` values is recorded, at offset `d²`, + // which is the sum of every earlier window. Cloning the whole vector each round instead copied + // `2 · max + 1` values into a fresh allocation `d` times, for tens of megabytes of transient + // garbage on a large diff and no more information. + let mut rounds: Vec = Vec::new(); + let shift = |k: isize| usize::try_from(k + isize::try_from(max).unwrap_or(isize::MAX)).unwrap_or(0); + + for d in 0..=isize::try_from(max).unwrap_or(isize::MAX) { + rounds.extend_from_slice(furthest.get(shift(-d)..=shift(d)).unwrap_or_default()); + + let mut k = -d; + + while k <= d { + // Which of the two neighbouring diagonals to extend from: going down (an insertion) + // when there is no diagonal to the left, or when the one to the right reaches further. + let down = k == -d || (k != d && furthest[shift(k - 1)] < furthest[shift(k + 1)]); + let mut x = if down { furthest[shift(k + 1)] } else { furthest[shift(k - 1)] + 1 }; + let mut y = x - k; + + // The free part: every line the two texts already agree on costs nothing to cross. + while let (Ok(xi), Ok(yi)) = (usize::try_from(x), usize::try_from(y)) + && xi < n + && yi < m + && old[xi] == new[yi] + { + x += 1; + y += 1; + } + + furthest[shift(k)] = x; + + if usize::try_from(x).unwrap_or(0) >= n && usize::try_from(y).unwrap_or(0) >= m { + return walk_back(old, new, &rounds, d); + } + + k += 2; + } + } + + Vec::new() +} + +/// Recovers the edit script from the recorded rounds, by walking the corner back to the origin. +/// +/// `rounds` is the flat record `script` built: round `r`'s furthest-reached endpoints for the +/// diagonals `-r..=r`, starting at offset `r²`. Only those diagonals are ever asked for — the +/// endpoint reached after `r` edits lies on a diagonal `|k| <= r` of the same parity, and the two +/// neighbours consulted are only consulted when `k` is neither `-r` nor `r`. +fn walk_back<'a>(old: &[&'a str], new: &[&'a str], rounds: &[isize], d: isize) -> Vec> { + let mut steps = Vec::new(); + let mut x = isize::try_from(old.len()).unwrap_or(isize::MAX); + let mut y = isize::try_from(new.len()).unwrap_or(isize::MAX); + + for round in (0..=d).rev() { + let reached = |k: isize| -> isize { + let index = round + .checked_mul(round) + .zip(k.checked_add(round)) + .and_then(|(base, offset)| base.checked_add(offset)); + + index + .and_then(|index| usize::try_from(index).ok()) + .and_then(|index| rounds.get(index).copied()) + .unwrap_or(0) + }; + + let k = x - y; + let down = k == -round || (k != round && reached(k - 1) < reached(k + 1)); + let previous = if down { k + 1 } else { k - 1 }; + let start = reached(previous); + let (before_x, before_y) = (start, start - previous); + + // The diagonal run at the end of this round: lines both texts have, emitted as context. + while x > before_x && y > before_y { + x -= 1; + y -= 1; + + if let Some(line) = usize::try_from(x).ok().and_then(|index| old.get(index)) { + steps.push(Step::Kept(line)); + } + } + + if round == 0 { + break; + } + + if down { + y -= 1; + + if let Some(line) = usize::try_from(y).ok().and_then(|index| new.get(index)) { + steps.push(Step::Added(line)); + } + } else { + x -= 1; + + if let Some(line) = usize::try_from(x).ok().and_then(|index| old.get(index)) { + steps.push(Step::Removed(line)); + } + } + } + + steps.reverse(); + steps +} diff --git a/crates/cargo-gamma-lib/src/fix/edit.rs b/crates/cargo-gamma-lib/src/fix/edit.rs new file mode 100644 index 000000000..5e71e2ce6 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/edit.rs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::BTreeSet; + +use camino::Utf8PathBuf; + +/// One directive to be written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Edit { + /// The file to patch. + pub file: Utf8PathBuf, + + /// The one-based line the directive is written above. + pub line: usize, + + /// The mutators named in the directive, in registry order. + pub mutators: BTreeSet, + + /// The tag used to group everything this tool wrote. + pub tag: &'static str, +} + +impl Edit { + /// Renders the directive, indented to match the line it precedes. + /// + /// The generated text is a comment shaped exactly like the attribute it stands in for, so that + /// if in-source attributes ever reach stable Rust the two slashes can simply be deleted. + /// + /// `ending` is the terminator the file already uses. A generated line is the only line in the + /// file this tool wrote, and giving it a different ending from its neighbours turns a + /// suppression into a whitespace diff that every reviewer has to look at twice. + #[must_use] + pub fn render(&self, indent: &str, date: &str, ending: &str) -> String { + let selectors = self.mutators.iter().cloned().collect::>().join(", "); + + format!( + "{indent}// #[gamma::skip({selectors}, tag = \"{tag}\", reason = \"written by cargo gamma suppress {date}\")]{ending}", + tag = self.tag + ) + } +} + +#[cfg(test)] +mod tests { + use core::iter::once; + + use super::*; + + #[test] + fn a_directive_names_the_exact_mutators_never_a_family() { + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 3, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + let text = edit.render(" ", "2026-08-05", "\n"); + + assert!(text.contains("gamma::skip(stmt.delete,"), "{text}"); + assert!(!text.contains("all"), "{text}"); + assert!(text.contains("tag = \"timeout\""), "{text}"); + assert!(text.contains("reason ="), "a directive with no reason is unauditable"); + assert!(text.ends_with(")]\n"), "{text}"); + } + + #[test] + fn a_directive_ends_the_way_the_file_it_joins_does() { + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 3, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + assert!(edit.render("", "2026-08-05", "\r\n").ends_with(")]\r\n")); + } +} diff --git a/crates/cargo-gamma-lib/src/fix/eligible.rs b/crates/cargo-gamma-lib/src/fix/eligible.rs new file mode 100644 index 000000000..8b20d0b46 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/eligible.rs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::Result; +use crate::error::error; +use crate::model::Outcome; + +/// A verdict that may be suppressed automatically. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Eligible { + /// The mutant exceeded its time budget. + /// + /// Eligible by default, and still the second-best answer: a timeout that is cached keeps the + /// mutant in the score and costs nothing on re-runs, whereas suppressing it removes it from the + /// denominator. This is for sites that are permanently un-mutatable — a hand-written spin loop, + /// a driver poll, a reactor — where the team wants that recorded where the next reader sees it. + Timeout, + + /// The mutant's test run passed its memory ceiling. + /// + /// Eligible by default for the same reason as [`Self::Timeout`], and it is the same defect + /// wearing different clothes: a mutant that turns a bounded loop into an unbounded one is + /// stopped by whichever ceiling it reaches first, and which one that is depends on the machine + /// rather than on the code. Making one suppressible and not the other would mean a directive + /// that works on the maintainer's laptop and not in CI. + OutOfMemory, + + /// The mutant did not compile. + Unviable, +} + +impl Eligible { + /// Parses the `--eligible` list. + /// + /// `missed` and `survived` are named explicitly so the refusal can explain itself. Falling + /// through to "unknown verdict" would read like a typo, and the person who typed it would try + /// harder rather than reconsidering. + pub fn parse(list: &str) -> Result> { + let mut out = Vec::new(); + + for entry in list.split(',').map(str::trim).filter(|entry| !entry.is_empty()) { + match entry { + "timeout" => out.push(Self::Timeout), + "outofmem" | "oom" | "out-of-memory" => out.push(Self::OutOfMemory), + "unviable" | "compile-error" => out.push(Self::Unviable), + + "missed" | "survived" | "survivor" => { + return Err(error!( + "`{entry}` is not eligible for `suppress`, and cannot be made eligible: a surviving mutant is a gap in the test suite, and suppressing it would remove that gap from the score rather than from the code" + ) + .usage()); + } + + other => { + return Err(error!("unknown verdict `{other}`; --eligible accepts `timeout`, `outofmem` and `unviable`").usage()); + } + } + } + + out.sort_unstable(); + out.dedup(); + + Ok(out) + } + + /// The tag written into generated directives. + #[must_use] + pub const fn tag(self) -> &'static str { + match self { + Self::Timeout => "timeout", + Self::OutOfMemory => "outofmem", + Self::Unviable => "unviable", + } + } + + /// Returns the verdict this covers. + #[must_use] + pub const fn outcome(self) -> Outcome { + match self { + Self::Timeout => Outcome::Timeout, + Self::OutOfMemory => Outcome::OutOfMemory, + Self::Unviable => Outcome::CompileError, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_survivor_cannot_be_made_eligible() { + // The single most important test in the module. If this ever passes, every mutation score + // the tool reports becomes a number that can be improved by editing comments. + let cause = Eligible::parse("missed").expect_err("survivors must be refused"); + + assert!(cause.is_usage(), "{cause}"); + assert!(cause.to_string().contains("gap in the test suite"), "{cause}"); + } + + #[test] + fn eligible_lists_are_trimmed_sorted_and_deduplicated() { + let parsed = Eligible::parse(" unviable, timeout, compile-error, timeout ").unwrap(); + + assert_eq!(parsed, vec![Eligible::Timeout, Eligible::Unviable]); + } + + /// Both ceilings a runaway mutant can hit are suppressible, under every spelling. + /// + /// `outofmem` is the canonical form because it is what [`Outcome::as_str`] prints and therefore + /// what a reader sees in the report they are acting on; `oom` and `out-of-memory` are what they + /// are liable to type instead. + #[test] + fn out_of_memory_is_eligible_under_each_of_its_spellings() { + for spelling in ["outofmem", "oom", "out-of-memory"] { + let parsed = Eligible::parse(spelling).unwrap_or_else(|_| panic!("`{spelling}` must parse")); + + assert_eq!(parsed, vec![Eligible::OutOfMemory], "{spelling}"); + assert_eq!(parsed[0].outcome(), Outcome::OutOfMemory, "{spelling}"); + assert_eq!(parsed[0].tag(), "outofmem", "every spelling writes one tag"); + } + } + + /// Every eligible verdict maps to a distinct outcome and a distinct tag. + /// + /// Two variants sharing an outcome would make `plan`'s first-match lookup pick arbitrarily, and + /// two sharing a tag would merge suppressions that were written for different reasons. + #[test] + fn each_eligible_verdict_has_its_own_outcome_and_tag() { + let all = [Eligible::Timeout, Eligible::OutOfMemory, Eligible::Unviable]; + + for (index, entry) in all.iter().enumerate() { + for other in &all[index + 1..] { + assert_ne!(entry.outcome(), other.outcome(), "{entry:?} and {other:?} share an outcome"); + assert_ne!(entry.tag(), other.tag(), "{entry:?} and {other:?} share a tag"); + } + } + } + + #[test] + fn unknown_eligible_verdicts_are_errors() { + let cause = Eligible::parse("killed").expect_err("unknown verdict should be rejected"); + + assert!(cause.is_usage(), "{cause}"); + assert!(cause.to_string().contains("timeout"), "{cause}"); + assert!( + cause.to_string().contains("outofmem"), + "the message must name every accepted verdict: {cause}" + ); + } +} diff --git a/crates/cargo-gamma-lib/src/fix/mod.rs b/crates/cargo-gamma-lib/src/fix/mod.rs new file mode 100644 index 000000000..a673a2a6d --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/mod.rs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Writing suppressions back into the source, the way `cargo clippy --fix` does. +//! +//! A run knows exactly which mutants caused trouble and exactly where they live, so it can write the +//! directive rather than describing it. What makes that safe is a single rule and a single check. +//! +//! **The rule: a surviving mutant is never eligible.** Not by default, not behind a flag, not with a +//! force switch. A survivor is a real gap in the test suite, and a tool that offers to delete gaps +//! from its own denominator is a tool for manufacturing a mutation score. The moment this can hide a +//! survivor, every number the tool reports becomes unfalsifiable — so the refusal is structural: a +//! surviving verdict has no spelling that reaches this module. +//! +//! **The check: verify, do not assert.** A directive placed one line off, or attached to a +//! multi-line expression, can silently suppress a dozen unrelated mutants — including survivors, +//! which is the rule above being violated by accident rather than by design. So after writing, +//! discovery runs again and the suppressed set is compared: every intended mutant must now be +//! suppressed and nothing else may have become suppressed. If either half fails, the whole edit is +//! reverted. + +mod diff; +mod edit; +mod eligible; +mod plan; +mod removal; +mod verification; +mod verify; + +pub use diff::diff; +pub use edit::Edit; +pub use eligible::Eligible; +pub use plan::{apply, plan, today}; +pub use removal::{removable, remove}; +pub use verification::Verification; +pub use verify::verify; diff --git a/crates/cargo-gamma-lib/src/fix/plan.rs b/crates/cargo-gamma-lib/src/fix/plan.rs new file mode 100644 index 000000000..026b2bbf5 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/plan.rs @@ -0,0 +1,514 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Choosing the directives a completed run should write, and applying them to a file's text. + +use core::cmp::Reverse; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use camino::Utf8PathBuf; + +use super::{Edit, Eligible}; +use crate::model::{Mutant, Outcome}; + +/// Chooses the directives to write for a completed run. +/// +/// Mutants at the same site coalesce into one directive naming each mutator, rather than one comment +/// each: five stacked comments above a line is not something anyone will keep. +/// +/// The tag is part of the grouping key, so a site whose mutants were stopped for different reasons +/// gets one directive per reason. Collapsing them would file a mutant under a verdict it never had, +/// and the tag is the whole basis on which a report groups suppressions. +#[must_use] +pub fn plan(mutants: &[Mutant], eligible: &[Eligible]) -> Vec { + let mut grouped: BTreeMap<(Utf8PathBuf, usize, &'static str), Edit> = BTreeMap::new(); + + for mutant in mutants { + // Belt and braces. `Eligible` cannot name a survivor, so this can only fire if someone adds + // a variant later — which is exactly when a second check is worth having. + if matches!(mutant.outcome, Outcome::Survived) { + continue; + } + + let Some(tag) = eligible + .iter() + .find(|entry| entry.outcome() == mutant.outcome) + .map(|entry| entry.tag()) + else { + continue; + }; + + let entry = grouped + .entry((mutant.file.to_path_buf(), mutant.line, tag)) + .or_insert_with(|| Edit { + file: mutant.file.to_path_buf(), + line: mutant.line, + mutators: BTreeSet::new(), + tag, + }); + + let _ = entry.mutators.insert(mutant.mutator.to_string()); + } + + grouped.into_values().collect() +} + +/// Applies edits to one file's text. +/// +/// Edits are applied from the last line backwards so that every earlier line number stays valid; the +/// same discipline the instrumenter needs in the other direction, and the same bug if it is wrong. +/// +/// A line that already carries a generated directive has its selector list extended instead of +/// gaining a second comment, which is what makes running this twice a no-op. +#[must_use] +pub fn apply(text: &str, edits: &[&Edit], date: &str) -> String { + let ending = ending(text); + let mut lines: Vec = text.split_inclusive('\n').map(str::to_owned).collect(); + let mut ordered: Vec<&&Edit> = edits.iter().collect(); + + ordered.sort_by_key(|edit| Reverse(edit.line)); + + for edit in ordered { + let Some(index) = edit.line.checked_sub(1).filter(|index| *index < lines.len()) else { + continue; + }; + + let indent: String = lines[index] + .chars() + .take_while(|c| c.is_whitespace() && *c != '\n' && *c != '\r') + .collect(); + + if let Some(above) = matching_directive(&lines, index, edit.tag) { + let mut merged = edit.mutators.clone(); + + merged.extend(generated_selectors(&lines[above]).unwrap_or_default()); + + let rendered = Edit { + mutators: merged, + ..(*edit).clone() + } + .render(&indent, date, ending); + + lines[above] = rendered; + continue; + } + + lines.insert(index, edit.render(&indent, date, ending)); + } + + lines.concat() +} + +/// The line terminator the file already uses. +/// +/// Decided by majority rather than by the first line seen, because a file with one stray ending is +/// still a file with a convention, and matching the stray one would spread it. +fn ending(text: &str) -> &'static str { + let total = text.matches('\n').count(); + let carriage = text.matches("\r\n").count(); + + if carriage * 2 > total { "\r\n" } else { "\n" } +} + +/// Returns the line holding a generated directive tagged `tag`, among the run of generated +/// directives immediately above `index`. +/// +/// The walk stops at the first line that is not a generated directive, so a directive belonging to +/// some construct further up is never mistaken for one governing this line. Walking the whole run +/// rather than checking only the line directly above is what keeps a second run idempotent once a +/// site carries more than one tag: the directive to extend may sit behind its siblings. +fn matching_directive(lines: &[String], index: usize, tag: &str) -> Option { + let mut cursor = index; + + while let Some(above) = cursor.checked_sub(1) { + if generated_tag(&lines[above])? == tag { + return Some(above); + } + + cursor = above; + } + + None +} + +/// Returns the tag of a directive this tool generated, if the line holds one. +fn generated_tag(line: &str) -> Option<&str> { + let inner = generated_body(line)?; + let after = inner.split_once("tag = \"")?.1; + + after.split_once('"').map(|(tag, _rest)| tag) +} + +/// Returns the argument text of a directive this tool generated, if the line holds one. +/// +/// Only *generated* directives are extended. A hand-written directive is someone's decision, with +/// their reason attached, and rewriting it would destroy that reason to save one line. +fn generated_body(line: &str) -> Option<&str> { + let trimmed = line.trim_start(); + + if !trimmed.contains("written by cargo gamma suppress") { + return None; + } + + trimmed.strip_prefix("// #[gamma::skip(") +} + +/// Returns the selectors of a directive this tool generated, if the line holds one. +fn generated_selectors(line: &str) -> Option> { + let inner = generated_body(line)?; + + Some( + inner + .split(',') + .map(str::trim) + .take_while(|part| !part.contains('=') && !part.is_empty()) + .map(str::to_owned) + .collect(), + ) +} + +/// Returns today's UTC date as `YYYY-MM-DD`. +/// +/// Hand-rolled rather than pulling in a date library, because this is the only date the tool ever +/// formats and the conversion is a well-known closed form. The date is what makes a generated +/// directive auditable a year later: "why is this here" is answerable from the comment alone. +#[must_use] +pub fn today() -> String { + let seconds = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |since| since.as_secs()); + + civil_from_days(i64::try_from(seconds / 86_400).unwrap_or(0)) +} + +/// Converts a count of days since the Unix epoch into `YYYY-MM-DD`. +/// +/// Hinnant's algorithm, which shifts the year to start in March so that the leap day lands at the +/// end and the month-length pattern becomes a single linear expression. +fn civil_from_days(days: i64) -> String { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted.rem_euclid(146_097); + let year_of_era = (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_prime = (5 * day_of_year + 2) / 153; + + let day = day_of_year - (153 * month_prime + 2) / 5 + 1; + let month = if month_prime < 10 { month_prime + 3 } else { month_prime - 9 }; + let year = era * 400 + year_of_era + i64::from(month <= 2); + + format!("{year:04}-{month:02}-{day:02}") +} + +#[cfg(test)] +mod tests { + use core::iter::once; + + use super::*; + use crate::fixtures::mutant_at as mutant; + + #[test] + fn a_survivor_is_skipped_even_if_it_reaches_the_planner() { + let mutants = vec![ + mutant("aaa", "src/lib.rs", 4, "relational.lt_to_le", Outcome::Survived), + mutant("bbb", "src/lib.rs", 9, "stmt.delete", Outcome::Timeout), + ]; + + let edits = plan(&mutants, &[Eligible::Timeout]); + + assert_eq!(edits.len(), 1); + assert_eq!(edits[0].line, 9); + } + + #[test] + fn timeouts_are_eligible_by_default_and_unviables_are_opt_in() { + let mutants = vec![ + mutant("aaa", "src/lib.rs", 4, "fn_value.default", Outcome::CompileError), + mutant("bbb", "src/lib.rs", 9, "stmt.delete", Outcome::Timeout), + ]; + + assert_eq!(plan(&mutants, &[Eligible::Timeout]).len(), 1); + assert_eq!(plan(&mutants, &[Eligible::Timeout, Eligible::Unviable]).len(), 2); + } + + #[test] + fn ineligible_outcomes_are_not_planned() { + let mutants = vec![mutant("aaa", "src/lib.rs", 4, "fn_value.default", Outcome::Killed)]; + + assert!(plan(&mutants, &[Eligible::Timeout, Eligible::Unviable]).is_empty()); + } + + #[test] + fn mutators_at_one_site_coalesce_into_a_single_directive() { + // Five stacked comments above one line is not something anyone keeps. + let mutants = vec![ + mutant("aaa", "src/lib.rs", 9, "stmt.delete", Outcome::Timeout), + mutant("bbb", "src/lib.rs", 9, "arith.add_to_sub", Outcome::Timeout), + ]; + + let edits = plan(&mutants, &[Eligible::Timeout]); + + assert_eq!(edits.len(), 1); + assert_eq!(edits[0].mutators.len(), 2); + } + + /// Coalescing stops at the tag, because the tag is a claim about why the mutant was stopped. + /// + /// One site can hold a mutant that hung and a mutant that ran the machine out of memory, and + /// both are eligible by default. Folding them into one directive would file one of them under a + /// verdict it never had, and the tag is the whole basis on which a report groups suppressions. + #[test] + fn one_site_stopped_for_two_reasons_gets_one_directive_for_each() { + let mutants = vec![ + mutant("aaa", "src/lib.rs", 9, "stmt.delete", Outcome::Timeout), + mutant("bbb", "src/lib.rs", 9, "arith.add_to_sub", Outcome::OutOfMemory), + ]; + + let edits = plan(&mutants, &[Eligible::Timeout, Eligible::OutOfMemory]); + + assert_eq!(edits.len(), 2, "one directive per reason"); + + let tags: Vec<&str> = edits.iter().map(|edit| edit.tag).collect(); + + assert!(tags.contains(&"timeout"), "{tags:?}"); + assert!(tags.contains(&"outofmem"), "{tags:?}"); + + for edit in &edits { + assert_eq!(edit.line, 9); + assert_eq!(edit.mutators.len(), 1, "a mutator belongs only to the reason that stopped it"); + } + } + + /// Both directives survive into the text, and both still govern the line below them. + #[test] + fn two_reasons_at_one_site_stack_above_the_line_they_govern() { + let text = "fn f() {\n loop {}\n}\n"; + let mutants = vec![ + mutant("aaa", "src/lib.rs", 2, "stmt.delete", Outcome::Timeout), + mutant("bbb", "src/lib.rs", 2, "arith.add_to_sub", Outcome::OutOfMemory), + ]; + + let edits = plan(&mutants, &[Eligible::Timeout, Eligible::OutOfMemory]); + let borrowed: Vec<&Edit> = edits.iter().collect(); + let out = apply(text, &borrowed, "2026-08-05"); + let lines: Vec<&str> = out.lines().collect(); + + assert_eq!(lines.len(), 5, "two directives above the loop: {out}"); + assert!(lines[1].contains("tag = \"timeout\""), "{out}"); + assert!(lines[2].contains("tag = \"outofmem\""), "{out}"); + assert_eq!(lines[3].trim(), "loop {}", "the directives sit directly above the line: {out}"); + + // A second run re-scans the patched file, so its edits name the line the code sits on now — + // two lower, behind both directives. Extending the matching one rather than stacking a + // duplicate is only possible because the search walks past the sibling tag to find its own. + let moved: Vec = edits + .iter() + .map(|edit| Edit { + line: edit.line + 2, + ..edit.clone() + }) + .collect(); + let borrowed_again: Vec<&Edit> = moved.iter().collect(); + let again = apply(&out, &borrowed_again, "2026-08-05"); + + assert_eq!(again, out, "a second application is a no-op"); + } + + /// A tag nothing above the line carries gets its own directive rather than rewriting a sibling. + /// + /// The second edit names line 3 because that is where `loop {}` sits once the first directive is + /// in, which is what a re-scan of the patched file would report. Naming the original line would + /// stop the search above the directive and pass no matter what the merge rule was. + #[test] + fn a_new_reason_at_a_suppressed_site_does_not_retag_the_existing_directive() { + let text = "fn f() {\n loop {}\n}\n"; + let first = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 2, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + let second = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 3, + mutators: once("arith.add_to_sub".to_owned()).collect(), + tag: "outofmem", + }; + + let once_applied = apply(text, &[&first], "2026-08-05"); + let twice_applied = apply(&once_applied, &[&second], "2026-08-05"); + + assert!(twice_applied.contains("tag = \"timeout\""), "{twice_applied}"); + assert!(twice_applied.contains("tag = \"outofmem\""), "{twice_applied}"); + assert!( + twice_applied.contains("gamma::skip(stmt.delete,"), + "the first directive keeps its own selector: {twice_applied}" + ); + assert!( + !twice_applied.contains("gamma::skip(arith.add_to_sub, stmt.delete,"), + "the two reasons must not merge into one directive: {twice_applied}" + ); + } + + #[test] + fn a_directive_is_written_above_the_line_at_its_indentation() { + let text = "fn f() {\n loop {}\n}\n"; + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 2, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + let out = apply(text, &[&edit], "2026-08-05"); + let lines: Vec<&str> = out.lines().collect(); + + assert!(lines[1].starts_with(" // #[gamma::skip("), "{out}"); + assert_eq!(lines[2], " loop {}"); + } + + #[test] + fn edits_for_missing_lines_are_ignored() { + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 99, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + assert_eq!(apply("fn f() {}\n", &[&edit], "2026-08-05"), "fn f() {}\n"); + } + + #[test] + fn edits_are_applied_from_the_end_so_earlier_lines_stay_valid() { + // Applying forwards shifts every later line by one and puts the second directive one line + // too high, which is silent: the file still compiles and suppresses the wrong thing. + let text = "a();\nb();\nc();\n"; + let first = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 1, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + let second = Edit { line: 3, ..first.clone() }; + + let out = apply(text, &[&first, &second], "2026-08-05"); + let lines: Vec<&str> = out.lines().collect(); + + assert!(lines[0].contains("gamma::skip"), "{out}"); + assert_eq!(lines[1], "a();"); + assert_eq!(lines[2], "b();"); + assert!(lines[3].contains("gamma::skip"), "{out}"); + assert_eq!(lines[4], "c();"); + } + + #[test] + fn running_twice_is_a_no_op() { + let text = "fn f() {\n loop {}\n}\n"; + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 2, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + let once = apply(text, &[&edit], "2026-08-05"); + + // The second pass sees the directive it wrote, so the line it targets has moved down by one. + let again = apply(&once, &[&Edit { line: 3, ..edit }], "2026-08-05"); + + assert_eq!(once, again); + assert_eq!(again.matches("gamma::skip").count(), 1, "{again}"); + } + + #[test] + fn a_second_mutator_extends_the_generated_directive_rather_than_stacking() { + let text = "fn f() {\n loop {}\n}\n"; + let first = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 2, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + let once = apply(text, &[&first], "2026-08-05"); + let second = Edit { + line: 3, + mutators: core::iter::once("arith.add_to_sub".to_owned()).collect(), + ..first + }; + let twice = apply(&once, &[&second], "2026-08-05"); + + assert_eq!(twice.matches("gamma::skip").count(), 1, "{twice}"); + assert!(twice.contains("arith.add_to_sub"), "{twice}"); + assert!(twice.contains("stmt.delete"), "{twice}"); + } + + #[test] + fn a_hand_written_directive_is_never_rewritten() { + // Someone's reason is the most valuable thing in the file, and it is not recoverable. + let text = "fn f() {\n // #[gamma::skip(stmt.delete, reason = \"driver poll, see RFC-12\")]\n loop {}\n}\n"; + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 3, + mutators: once("arith.add_to_sub".to_owned()).collect(), + tag: "timeout", + }; + + let out = apply(text, &[&edit], "2026-08-05"); + + assert!(out.contains("RFC-12"), "{out}"); + assert_eq!(out.matches("gamma::skip").count(), 2, "{out}"); + } + + #[test] + fn the_epoch_converts_to_its_known_date() { + // Two fixed points, one of them a leap day, because the whole algorithm is about where the + // leap day lands. + assert_eq!(civil_from_days(0), "1970-01-01"); + assert_eq!(civil_from_days(19_417), "2023-03-01"); + assert_eq!(civil_from_days(18_321), "2020-02-29"); + } + + #[test] + fn today_is_a_plausible_date() { + let date = today(); + + assert_eq!(date.len(), 10, "{date}"); + assert!(date.starts_with("20"), "{date}"); + } + + #[test] + fn a_crlf_file_keeps_its_line_endings() { + // A lone LF in an otherwise CRLF file is a whitespace change on a line nobody edited, which + // is exactly the kind of diff that makes a team stop trusting an automated fix. + let text = "fn f() {\r\n loop {}\r\n}\r\n"; + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 2, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + let out = apply(text, &[&edit], "2026-08-05"); + + assert_eq!(out.matches('\n').count(), out.matches("\r\n").count(), "{out:?}"); + assert!(out.contains(" // #[gamma::skip(stmt.delete,"), "{out:?}"); + } + + #[test] + fn an_lf_file_keeps_its_line_endings_even_with_one_stray_crlf() { + // Majority rather than first-seen: a file with one stray ending still has a convention, and + // matching the stray one would spread it. + let text = "fn f() {\n loop {}\r\n}\n\n\n"; + let edit = Edit { + file: Utf8PathBuf::from("src/lib.rs"), + line: 2, + mutators: once("stmt.delete".to_owned()).collect(), + tag: "timeout", + }; + + let out = apply(text, &[&edit], "2026-08-05"); + + assert!(out.contains("suppress 2026-08-05\")]\n loop"), "{out:?}"); + } +} diff --git a/crates/cargo-gamma-lib/src/fix/removal.rs b/crates/cargo-gamma-lib/src/fix/removal.rs new file mode 100644 index 000000000..3b4fd8df3 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/removal.rs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Deleting whole directive lines from a file's text. + +use std::collections::BTreeSet; + +/// Deletes whole lines from a file's text, by one-based line number. +/// +/// The counterpart of [`super::apply`], and deliberately the dumbest thing that can work: a directive is +/// only ever removed when it is the entire content of its own line, so removal is a line delete and +/// nothing else. See [`removable`] for what makes that true, and why anything else is left alone. +#[must_use] +pub fn remove(text: &str, lines: &BTreeSet) -> String { + text.split_inclusive('\n') + .enumerate() + .filter(|(index, _)| !lines.contains(&(index + 1))) + .map(|(_, line)| line) + .collect() +} + +/// Whether a line holds a skip directive and nothing else, so that deleting the line deletes it. +/// +/// Removal has to be conservative in a way that adding does not. A directive can be attached to a +/// line of code, wrapped in a `cfg_attr`, or spread over several lines, and in each of those cases +/// there is no line whose deletion removes the directive and only the directive. Editing *within* a +/// line to take one attribute out of a list is a different and much less safe operation, so those +/// are reported and left for a person. +#[must_use] +pub fn removable(line: &str) -> bool { + let trimmed = line.trim(); + + let body = if let Some(comment) = trimmed.strip_prefix("//") { + let comment = comment.trim_start(); + let Some(body) = comment.strip_prefix("#[").and_then(|rest| rest.strip_suffix(']')) else { + return false; + }; + body + } else { + trimmed + .strip_prefix("#[") + .and_then(|rest| rest.strip_suffix(']')) + .unwrap_or(trimmed) + }; + + if !body.starts_with("gamma::skip") { + return false; + } + + // An attribute whose arguments run onto the next line leaves its parentheses open here, and + // deleting the first line of it would leave the rest behind as a syntax error. + let opened = body.matches('(').count(); + + opened == body.matches(')').count() && !body.contains("//") +} diff --git a/crates/cargo-gamma-lib/src/fix/verification.rs b/crates/cargo-gamma-lib/src/fix/verification.rs new file mode 100644 index 000000000..f2f261210 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/verification.rs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/// The outcome of verifying a set of edits against a fresh discovery. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Verification { + /// Mutants that were meant to be suppressed and are not. + pub missing: Vec, + + /// Mutants that became suppressed and were not meant to be. + /// + /// The dangerous half. A directive attached to a multi-line construct takes out everything + /// inside it, and if any of those were survivors the guarantee at the top of this module has + /// been violated by accident. + pub collateral: Vec, + + /// Mutants that stopped being suppressed. + /// + /// The hazard when a directive is removed rather than written: a directive reported as + /// suppressing nothing that turns out to have been holding something down means the report was + /// wrong, and the safe response is to put the tree back and say so. + pub released: Vec, +} + +impl Verification { + /// Whether the edit may stand. + #[must_use] + pub const fn is_clean(&self) -> bool { + self.missing.is_empty() && self.collateral.is_empty() && self.released.is_empty() + } +} diff --git a/crates/cargo-gamma-lib/src/fix/verify.rs b/crates/cargo-gamma-lib/src/fix/verify.rs new file mode 100644 index 000000000..e90d3c972 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fix/verify.rs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Checking an edit against what it was meant to suppress. + +use std::collections::BTreeSet; + +use super::Verification; +use crate::model::Mutant; + +/// Compares the suppressed sets before and after an edit. +/// +/// `intended` is the set of mutant IDs the edit was written for. Every direction matters: an edit +/// that suppresses nothing is a silent no-op, an edit that suppresses too much is the hazard, and +/// an edit that *stops* suppressing something has undone a decision nobody asked it to revisit. +/// +/// The last of those is what removing a directive has to be checked against, and it is checked for +/// both operations because neither is ever meant to do it. +#[must_use] +pub fn verify(before: &[Mutant], after: &[Mutant], intended: &BTreeSet) -> Verification { + let suppressed = |mutants: &[Mutant]| -> BTreeSet { + mutants + .iter() + .filter(|mutant| mutant.suppression.is_some()) + .map(|mutant| mutant.id.to_string()) + .collect() + }; + + let was = suppressed(before); + let now = suppressed(after); + + Verification { + missing: intended.iter().filter(|id| !now.contains(*id)).cloned().collect(), + collateral: now + .iter() + .filter(|id| !was.contains(*id) && !intended.contains(*id)) + .cloned() + .collect(), + released: was.iter().filter(|id| !now.contains(*id)).cloned().collect(), + } +} + +#[cfg(test)] +mod tests { + use core::iter::once; + + use super::*; + use crate::fixtures::mutant_at as mutant; + use crate::model::{Channel, Outcome, Suppression}; + + /// A directive that no longer suppresses what it did is the whole hazard of removal, and the + /// verification has to be able to say so. + #[test] + fn a_mutant_that_stops_being_suppressed_is_reported_as_released() { + let mut before = vec![mutant("a", "src/lib.rs", 1, "arith.add_to_sub", Outcome::Killed)]; + + before[0].suppression = Some(Suppression { + channel: Channel::Comment, + reason: None, + tag: None, + line: Some(1), + }); + + let after = vec![mutant("a", "src/lib.rs", 1, "arith.add_to_sub", Outcome::Killed)]; + let result = verify(&before, &after, &BTreeSet::new()); + + assert_eq!(result.released, vec!["a".to_owned()]); + assert!(!result.is_clean()); + } + + #[test] + fn verification_notices_an_edit_that_suppressed_nothing() { + let before = vec![mutant("aaa", "src/lib.rs", 9, "stmt.delete", Outcome::Timeout)]; + let after = before.clone(); + let intended: BTreeSet = once("aaa".to_owned()).collect(); + + let result = verify(&before, &after, &intended); + + assert!(!result.is_clean()); + assert_eq!(result.missing, vec!["aaa".to_owned()]); + } + + #[test] + fn verification_notices_collateral_suppression() { + // The hazard the whole design is arranged around: a directive on a multi-line construct + // takes out everything inside it, which can include survivors. + let before = vec![ + mutant("aaa", "src/lib.rs", 9, "stmt.delete", Outcome::Timeout), + mutant("bbb", "src/lib.rs", 10, "arith.add_to_sub", Outcome::Survived), + ]; + let mut after = before.clone(); + + for entry in &mut after { + entry.suppression = Some(Suppression { + channel: Channel::Comment, + reason: None, + tag: None, + line: Some(8), + }); + } + + let intended: BTreeSet = once("aaa".to_owned()).collect(); + let result = verify(&before, &after, &intended); + + assert!(!result.is_clean()); + assert_eq!(result.collateral, vec!["bbb".to_owned()]); + } + + #[test] + fn a_clean_verification_is_both_halves() { + let before = vec![mutant("aaa", "src/lib.rs", 9, "stmt.delete", Outcome::Timeout)]; + let mut after = before.clone(); + + after[0].suppression = Some(Suppression { + channel: Channel::Comment, + reason: None, + tag: None, + line: Some(8), + }); + + let intended: BTreeSet = once("aaa".to_owned()).collect(); + + assert!(verify(&before, &after, &intended).is_clean()); + } +} diff --git a/crates/cargo-gamma-lib/src/fixtures.rs b/crates/cargo-gamma-lib/src/fixtures.rs new file mode 100644 index 000000000..0afcf3418 --- /dev/null +++ b/crates/cargo-gamma-lib/src/fixtures.rs @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Neutral starting values for the record types the unit tests build by hand. +//! +//! [`Mutant`] has twenty-one fields and [`Report`] has six, but a test is usually about two +//! or three of them. Writing the literal out in full at every site buries those under the +//! filler and means a new field has to be added to every `mod tests` in the crate. +//! +//! Each function here returns a value whose fields are all neutral, to be adjusted with struct +//! update syntax so a test states only what it varies: +//! +//! ```ignore +//! let mutant = Mutant { line: 7, outcome: Outcome::Survived, ..fixtures::mutant() }; +//! ``` +//! +//! Gated on `cfg(test)` rather than living in [`crate::testing`]: that module is `pub`, so anything +//! added to it joins the crate's API, and these are only ever wanted from a `mod tests` block. + +use std::collections::HashMap; + +use camino::Utf8PathBuf; + +use crate::elements::{FileResult, Framework, Location, MutantResult, Position, Report, RunInfo, ShardInfo, Thresholds}; +use crate::model::{Mutant, Outcome}; +use crate::ops::collect::Shape; + +pub(crate) const UNRESOLVED_LINK_SOURCE: &str = "unsafe extern \"C\" {\n\ + \x20\x20\x20\x20fn gamma_absent_symbol() -> i32;\n\ + }\n\n\ + pub fn less(a: i32, b: i32) -> bool { a < b }\n\n\ + pub fn touch() -> i32 { unsafe { gamma_absent_symbol() } }\n\n\ + #[test]\n\ + fn calls_it() {\n\ + \x20\x20\x20\x20assert!(less(1, 2));\n\ + \x20\x20\x20\x20assert_eq!(touch(), 0);\n\ + }\n"; + +/// A killed `relational.lt_to_le` mutant of `a` to `b`, at line 1 of `src/lib.rs` in `subject`. +pub(crate) fn mutant() -> Mutant { + Mutant { + id: "m1".to_owned().into(), + ordinal: 1, + file: Utf8PathBuf::from("src/lib.rs").into(), + package: "subject".to_owned().into(), + span: 0..1, + line: 1, + end_line: 1, + column: 1, + mutator: "relational.lt_to_le".to_owned().into(), + item_path: "f".to_owned().into(), + occurrence: 0, + replacement_index: 0, + original: "a".to_owned().into(), + replacement: "b".to_owned().into(), + shape: Shape::Expr, + outcome: Outcome::Killed, + suppression: None, + expectation: None, + test_timeout_multiplier: None, + elapsed_ms: 0, + killed_by: None, + note: None, + } +} + +pub(crate) fn mutant_at(id: &str, file: &str, line: usize, mutator: &str, outcome: Outcome) -> Mutant { + Mutant { + id: id.to_owned().into(), + file: Utf8PathBuf::from(file).into(), + line, + mutator: mutator.to_owned().into(), + outcome, + ..mutant() + } +} + +/// The reported form of [`mutant`], spanning columns 1 to 9 of line 1. +pub(crate) fn mutant_result() -> MutantResult { + MutantResult { + id: "m1".to_owned().into(), + mutator_name: "relational.lt_to_le".into(), + location: Location { + start: Position { line: 1, column: 1 }, + end: Position { line: 1, column: 9 }, + }, + status: "Killed".into(), + replacement: None, + description: None, + status_reason: None, + duration: None, + killed_by: None, + } +} + +pub(crate) fn mutant_result_at(id: &str, line: usize, status: &str) -> MutantResult { + MutantResult { + id: id.to_owned().into(), + location: Location { + start: Position { line, column: 1 }, + end: Position { line, column: 9 }, + }, + status: status.into(), + ..mutant_result() + } +} + +/// An empty schema-2 report from this tool, with no run info and no files. +pub(crate) fn report() -> Report { + Report { + schema_version: "2".to_owned(), + thresholds: Thresholds::default(), + project_root: None, + framework: Framework { + name: "cargo-gamma".to_owned(), + version: "0.1.0".to_owned(), + }, + files: HashMap::default(), + config: None, + } +} + +pub(crate) fn report_with(shard: Option<(u32, u32)>, started_at: u64, mutants: Vec) -> Report { + let mut files = HashMap::default(); + + let _ = files.insert( + "src/lib.rs".to_owned(), + FileResult { + source: "fn f() {}\n".to_owned(), + language: "rust".to_owned(), + mutants, + }, + ); + + Report { + files, + config: Some(RunInfo { + started_at, + merged: false, + shard: shard.map(|(index, count)| ShardInfo { index, count }), + tests: None, + not_built: None, + dropped_test_packages: Vec::new(), + merge_provenance: None, + }), + ..report() + } +} + +pub(crate) fn crate_dir(name: &str, source: &str) -> (tempfile::TempDir, Utf8PathBuf) { + let dir = crate::testing::workdir(name); + let root = Utf8PathBuf::from_path_buf(dir.path().to_path_buf()).expect("utf8"); + + std::fs::create_dir(root.join("src")).expect("src"); + std::fs::write( + root.join("Cargo.toml"), + "[package]\nname = \"subject\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n", + ) + .expect("manifest"); + std::fs::write(root.join("src/lib.rs"), source).expect("lib"); + + (dir, root) +} diff --git a/crates/cargo-gamma-lib/src/html.rs b/crates/cargo-gamma-lib/src/html.rs new file mode 100644 index 000000000..26aa499d0 --- /dev/null +++ b/crates/cargo-gamma-lib/src/html.rs @@ -0,0 +1,294 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The standalone single-file HTML report. +//! +//! One `.html` file with no external dependencies: no CDN, no fonts, no fetch, no network at all. +//! It opens from disk, from a file share, from a CI artifact, and from an air-gapped machine. A +//! report that needs the network to render is a report nobody can attach to a build. +//! +//! The recipe is the one every Stryker implementation uses: inline the viewer bundle and assign +//! the report as a JavaScript property rather than fetching it. + +use std::io; + +use camino::Utf8Path; + +use crate::Result; +use crate::elements::Report; +use crate::error::error; + +/// The vendored viewer, inlined so the report needs nothing at run time. +const VIEWER: &str = include_str!("vendor/mutation-test-elements.js"); + +/// The URL used by [`Source::External`]. +const VIEWER_CDN: &str = "https://cdn.jsdelivr.net/npm/mutation-testing-elements@3/dist/mutation-test-elements.js"; + +/// Where the rendered page gets the viewer from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Source { + /// Inline the vendored bundle. The report works offline. + #[default] + Inline, + + /// Reference the bundle on a public CDN, for anyone who would rather not ship 230 KB per + /// report and knows their readers are online. + External, +} + +/// Renders a complete HTML page for a report. +pub fn render(report: &Report, source: Source) -> Result { + let mut page = Vec::new(); + + // The page is built by the same streaming path that writes it to disk, so the two cannot drift: + // writing to a `Vec` cannot fail for I/O, and the report serializes infallibly, so the only way + // this errors is the case handled below. + stream(report, source, &mut page).map_err(|cause| error!("could not serialize the report").caused_by(cause))?; + + String::from_utf8(page).map_err(|cause| error!("could not serialize the report").caused_by(cause)) +} + +/// Writes the self-contained HTML page for `report` to `path`, streamed so neither the report JSON +/// nor the finished page is ever fully resident. +/// +/// The bytes are exactly [`render`]'s — this is the same page, produced by the same streaming path — but +/// the prefix, the embedded report and the suffix go straight into the atomic publication's staging +/// file rather than being concatenated into a `String` first. +pub fn write_page(report: &Report, source: Source, path: &Utf8Path) -> Result<()> { + crate::elements::write_streamed(path, |writer| stream(report, source, writer)) +} + +/// Streams the self-contained page into `writer`: the prefix, then the report as the embedded +/// script payload, then the suffix. +/// +/// The report is written through [`EscapeScript`] so that a `` in a string literal cannot +/// terminate the element carrying it, matching the escaping the whole-string form used. Splitting +/// the page this way is what keeps neither the report JSON nor the page held whole in memory. +fn stream(report: &Report, source: Source, writer: &mut dyn io::Write) -> io::Result<()> { + write!( + writer, + "\n\ + \n\ + \n\ + \n\ + \n\ + cargo-gamma mutation report\n\ + \n" + )?; + + match source { + Source::Inline => write!(writer, "")?, + Source::External => write!(writer, "")?, + } + + write!( + writer, + "\n\ + \n\ + \n\ + \n\ + Your browser does not support custom elements, which this report is built from.\n\ + \n\ + \n\n\n") +} + +/// The page's own styling, which is only ever about the area the viewer does not paint. +/// +/// `color-scheme` is what stops the browser from rendering its own furniture — scrollbars, form +/// controls, the canvas behind the document — in light colors on a dark page. +/// +/// The two background rules are fallbacks for the moments the script cannot cover: the media query +/// paints correctly before the viewer has resolved its theme and when scripting never runs at all, +/// and the attribute rule follows the theme the viewer reflects onto itself, including one the +/// reader picked inside the report that disagrees with the system. +const PAGE_STYLE: &str = "\ + :root { color-scheme: light dark; }\ + html, body { margin: 0; padding: 0; }\ + body { background-color: #fff; }\ + @media (prefers-color-scheme: dark) { body { background-color: #18181b; } }\ + body:has(mutation-test-report-app[theme=\"dark\"]) { background-color: #18181b; }\ + body:has(mutation-test-report-app[theme=\"light\"]) { background-color: #fff; }"; + +/// Keeps the page background in step with the theme the viewer chose. +/// +/// The viewer paints its own components but not the page behind them, and it resolves its theme +/// from a saved preference before falling back to the system one — so the page cannot work the +/// answer out for itself, and a CSS media query alone gets it wrong for anyone who overrode the +/// theme inside the report. Listening for the event the viewer already emits is the only way to +/// read the exact color it settled on. +/// +/// Registered before the report is assigned, because assigning it is what starts the update cycle +/// that ends in the event. +const THEME_SCRIPT: &str = "\ + app.addEventListener('theme-changed', (event) => {\ + const color = event.detail.themeBackgroundColor;\ + if (color) { document.body.style.backgroundColor = color; }\ + });"; + +/// A writer that escapes the two angle brackets so a payload cannot terminate the script element +/// that carries it, applied to the report as it streams into the page. +/// +/// An HTML parser looks for the literal characters `"` in a string literal would otherwise cut its own report in half — and the tail of +/// the document would be reinterpreted as markup. +/// +/// In JSON the only place `<` or `>` can appear is inside a string literal, so rewriting them to +/// their `\u` escapes is safe everywhere in the document and needs no parsing to do correctly. +/// Both are ASCII, so they never occur as a continuation byte of a multi-byte character, which is +/// why rewriting the byte stream is equivalent to rewriting the characters and never corrupts one. +struct EscapeScript { + inner: W, +} + +impl io::Write for EscapeScript { + fn write(&mut self, buf: &[u8]) -> io::Result { + let mut start = 0; + + for (index, &byte) in buf.iter().enumerate() { + let escape: &[u8] = match byte { + b'<' => b"\\u003c", + b'>' => b"\\u003e", + _ => continue, + }; + + self.inner.write_all(&buf[start..index])?; + self.inner.write_all(escape)?; + start = index + 1; + } + + self.inner.write_all(&buf[start..])?; + + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(test)] +#[cfg(not(miri))] +mod tests { + use serde_json::Value; + + use super::*; + use crate::fixtures; + + fn report() -> Report { + fixtures::report() + } + + #[test] + fn the_inline_page_carries_the_whole_viewer() { + let page = render(&report(), Source::Inline).expect("renders"); + + assert!(page.contains(" VIEWER.len(), "the viewer was not inlined"); + assert!(!page.contains("cdn.jsdelivr.net"), "the offline report must not reference a CDN"); + } + + #[test] + fn the_external_page_is_small_and_references_the_cdn() { + let page = render(&report(), Source::External).expect("renders"); + + assert!(page.len() < 4096, "the external report should not inline the viewer"); + assert!(page.contains(VIEWER_CDN), "{page}"); + } + + #[test] + fn the_page_follows_the_theme_the_viewer_settled_on() { + let page = render(&report(), Source::External).expect("renders"); + + assert!(page.contains("color-scheme: light dark"), "{page}"); + assert!(page.contains("prefers-color-scheme: dark"), "{page}"); + assert!(page.contains("theme-changed"), "{page}"); + } + + #[test] + fn the_theme_listener_is_registered_before_the_report_starts_the_update_cycle() { + // Assigning the report is what makes the viewer resolve its theme and emit the event, so a + // listener added afterwards is racing the very update it exists to hear about. + let page = render(&report(), Source::External).expect("renders"); + let listener = page.find("addEventListener").expect("the listener is present"); + let assignment = page.find("app.report =").expect("the report is assigned"); + + assert!(listener < assignment, "{page}"); + } + + #[test] + fn a_closing_script_tag_in_the_source_cannot_break_out() { + // The payload is assigned to a property, so source text is JSON-escaped rather than + // reproduced into the markup. Getting this wrong turns any crate containing the sequence + // in a string literal into a broken report. + let mut subject = report(); + + let _ = subject.files.insert( + "src/lib.rs".to_owned(), + crate::elements::FileResult { + source: "let s = \"\";".to_owned(), + language: "rust".to_owned(), + mutants: Vec::new(), + }, + ); + + let page = render(&subject, Source::External).expect("renders"); + + assert!(!page.contains("