diff --git a/.claude/skills/test-quality/SKILL.md b/.claude/skills/test-quality/SKILL.md index 6e763cf..506b33c 100644 --- a/.claude/skills/test-quality/SKILL.md +++ b/.claude/skills/test-quality/SKILL.md @@ -68,8 +68,8 @@ better LOC efficiency — with every test traceable to a user-observable contrac ### 1. Scope & detect Identify the source package and its tests dir. Detect the **language/framework** -(this sets the `score.py --lang` profile: `python`, `js`, `go`, `kotlin`, or -`swift`) and the coverage-enabled test command. Determine the **mode**: +(this sets the `score.py --lang` profile: `python`, `js`, `go`, `kotlin`, `swift`, +or `rust`) and the coverage-enabled test command. Determine the **mode**: - tests exist for the target → **audit + improve** (refactor in place). - target is untested → **generate** a fresh suite. (A target can be mixed: improve what exists, generate for the gaps.) @@ -77,7 +77,7 @@ Identify the source package and its tests dir. Detect the **language/framework** ### 2. Capture the baseline (do this BEFORE changing anything) - Run the suite with branch coverage; record line % and branch % — this is the coverage floor. -- `python /scripts/score.py --tests [--lang python|js|go|kotlin|swift]` +- `python /scripts/score.py --tests [--lang python|js|go|kotlin|swift|rust]` for the auto axes (`--lang` auto-detects if omitted). - Preserve the starting state so you can score against it: copy the tests dir aside, or note the git ref. Later runs use `--baseline `. @@ -141,11 +141,13 @@ are no contract violations left and every source boundary has a test. - **Validation tiers.** Python/pytest is empirically validated (the scorer's numbers were checked against a controlled experiment). The `js` (Jest/Vitest/ - Mocha/node:test), `go`, `kotlin` (kotlin.test/JUnit5/Kotest), and `swift` - (XCTest/Swift Testing/Quick) profiles apply the same axes with heuristic + Mocha/node:test), `go`, `kotlin` (kotlin.test/JUnit5/Kotest), `swift` + (XCTest/Swift Testing/Quick) and `rust` (built-in `#[test]`/tokio::test/ + rstest/proptest) profiles apply the same axes with heuristic regexes — trustworthy for trends and worst-offenders, but lean harder on reading the tests, and treat the W/L/T tally as indicative, not authoritative. - The kotlin/swift regexes were calibrated against six real, well-tested suites + The kotlin/swift regexes were calibrated against six real, well-tested suites, + and the rust ones against serde_json, rust-lang/regex and RustCrypto/hashes (see `tests/test_score.py` for the named regressions). Two language-specific notes: Kotest spec leaves are only counted when written `name(...) { … }` with a body (StringSpec's `"name" { … }` and BehaviorSpec given/when/then are missed), diff --git a/.claude/skills/test-quality/scripts/score.py b/.claude/skills/test-quality/scripts/score.py index 7bd725a..38c7df6 100644 --- a/.claude/skills/test-quality/scripts/score.py +++ b/.claude/skills/test-quality/scripts/score.py @@ -11,19 +11,21 @@ one validated in the llm-testgen-bench quality experiment, where prompting a model with these axes beat human-written baselines on the auto-countable axes in 9 of 9 Python suites (8 of 9 with the model held fixed — the rubric, not the -model, drove the gain). The JavaScript/TypeScript, Go, Kotlin and Swift +model, drove the gain). The JavaScript/TypeScript, Go, Kotlin, Swift and Rust profiles apply the same axes with framework-appropriate regexes; they are heuristic and not yet empirically validated to the same degree — treat their numbers as a guide, and lean on judgement (read the tests) more heavily. The kotlin/swift regexes were calibrated against six real, well-tested suites (kotlinx.serialization, kotlinx-datetime, kotlin-result; swift-argument-parser, -swift-collections, SwiftyJSON) — see tests/test_score.py for the named regressions. +swift-collections, SwiftyJSON); the rust regexes against serde_json, rust-lang +regex and RustCrypto hashes — see tests/test_score.py for the named regressions. -Supported ``--lang``: python | js | go | kotlin | swift. Omit ``--lang`` to -auto-detect from the files. +Supported ``--lang``: python | js | go | kotlin | swift | rust. Omit ``--lang`` +to auto-detect from the files. js = Jest / Vitest / Mocha+Chai+Sinon / node:test (.js/.ts/.jsx/.tsx) kotlin = kotlin.test / JUnit5 / Kotest (.kt) swift = XCTest / Swift Testing / Quick+Nimble (.swift) + rust = built-in #[test] / tokio::test / rstest / proptest (.rs) Auto-scored axes (direction in parens): A.1 substring-match assertions on error messages (lower better) @@ -304,6 +306,105 @@ ), }, }, + # Rust (built-in #[test] / tokio::test / rstest / proptest). Heuristic. + "rust": { + "exts": [".rs"], + # Rust splits tests two ways and both must count: integration suites under + # `tests/` (and `benches/`), and unit tests inline in `src/` behind + # `#[cfg(test)] mod tests`. The trailing `\.rs$` branch covers the inline + # case (point `--tests` at `src/`), mirroring how the python profile + # accepts any `.py` under the directory you aim it at. + "test_file": r"(?:^|[/\\])(?:tests?|benches)[/\\]|_tests?\.rs$|\.rs$", + # `#[test]`, runtime-flavoured `#[tokio::test]`/`#[async_std::test]`, + # `#[bench]`, plus the rstest/test_case/proptest attributes. `#[cfg(test)]` + # is NOT matched: after `#[` the pattern needs `test` (or `ns::test`) to + # start immediately, and `cfg(` doesn't. `test_case` is added explicitly + # because `test\b` won't match across the `_`. + "test_def": ( + r"#\[(?:\w+(?:::\w+)*::)?(?:test|bench)\b|#\[rstest\b" + r"|#\[test_case\(|#\[proptest\b" + ), + # rstest case rows, value lists, test_case rows and proptest! blocks are + # Rust's table-driven parametrization. `#[case(` (a row) counts; the bare + # `#[case]` that marks *which function argument* receives the row does + # not, or every parametrized test would score one extra. + "param": r"#\[case\(|#\[values\(|#\[test_case\(|\bproptest!\s*[{(]", + "validated": False, + "axes": { + # PARTIAL matchers on error text only — an exact `assert_eq!` against a + # message literal is a fixed vector (B.1), same split as python/js. + # Scoped to error-ish subjects so ordinary `.contains(` on collections + # (a legitimate assertion in Rust) isn't counted. + "A1_substring_match": ( + r"\.to_string\(\)[^\n]{0,40}?\.contains\(" + r"|\.unwrap_err\(\)[^\n]{0,60}?\.contains\(" + r"|format!\([^\n]*\{[^\n]*\}[^\n]*\)\.contains\(" + r"|\b(?:err|error|e|msg|message)\b[^\n]{0,30}?\.contains\(" + ), + # `#[cfg(test)] mod tests` inside the module sees private items by + # design, and integration tests under `tests/` can only reach `pub`. + # So — as with go and swift — there is no countable private-access + # smell to score here. + "A2_private_symbol": None, + "A4_recomputed_crypto": ( + r"\b(?:Sha1|Sha224|Sha256|Sha384|Sha512|Sha3_\d+|Md5|Md2|Md4" + r"|Blake2[bs]\d*|Blake3|Keccak\w*|Ripemd\d*)::(?:new|digest|new_with_prefix)\(" + r"|\bHmac::<|\bHmac<[^\n>]+>::new" + r"|\bblake3::hash\(|\bmd5::compute\(|\bcrc32fast::" + r"|\blet\s+expected\s*[:=][^\n]*(?:\.digest\(|::digest\(|hex::encode\(|base64::)" + ), + "A5_or_joined": ( + r"\|\|[^\n]*(?:to_string\(\)|unwrap_err\(\)|\berr\b)[^\n]*\.contains\(" + r"|(?:to_string\(\)|unwrap_err\(\)|\berr\b)[^\n]*\.contains\([^\n]*\|\|" + ), + # mockall/faux generated doubles and hand-written Mock*/Fake*/Stub*/Spy* + # types standing in for the unit under test. + "C1_mock_real": ( + r"#\[automock\]|#\[faux::\w+\]|\bmock!\s*\{|\bmockall::" + r"|\bMock[A-Z]\w*::new\(|\bstruct\s+(?:Mock|Fake|Stub|Spy)[A-Z]\w*" + ), + # Legitimate real-I/O primitives — local HTTP servers, temp dirs, CLI + # runners (context only, never scored). + "C2_mock_framework": ( + r"\bwiremock::|\bhttpmock::|\bmockito::(?:Server|server_url|mock)\b" + r"|\btempfile::(?:tempdir|NamedTempFile|TempDir)|\bTempDir::new\(" + r"|\bassert_cmd::|\bCommand::cargo_bin\(|\bassert_fs::" + ), + # Exact-literal equality against a 12+ char string (plain, raw `r#".."#` + # or byte `b".."`), a wide hex literal, a `hex!` vector — the RustCrypto + # idiom — or an insta snapshot. Rust writes `assert_eq!(actual, expected)` + # but either order occurs, so both positions are matched. + # + # The expected-last branch scans `[^;]{0,200}?` rather than `[^,\n]+` + # because rustfmt splits long asserts across lines — and long literals + # are exactly the ones that get split — while the first argument itself + # often contains commas (`assert_eq!(format!("{:?}", v), "literal")`). + # `[^;]` can't cross a statement boundary, so the scan stays inside one + # assert. Line-anchored patterns scored serde_json's suite at 12 against + # a hand count of 43. + "B1_fixed_vector": ( + # Raw strings first: `r#"…"#` is THE idiom for embedded JSON + # fixtures, and its body legitimately contains `"` — so the plain + # `[^"\n]` class below would stop at the first inner quote and + # score serde_json's JSON vectors at zero. + r"assert_eq!\(\s*r#+\"[^\n]{12,}" + r"|assert_eq!\([^;]{0,200}?,\s*r#+\"[^\n]{12,}" + # Plain and `r"…"` (no-hash raw) literals: no inner quotes here, + # so the tighter class applies. + r"|assert_eq!\(\s*r?\"[^\"\n]{12,}" + r"|assert_eq!\([^;]{0,200}?,\s*r?\"[^\"\n]{12,}" + r"|assert_eq!\([^;]{0,200}?,\s*b\"[^\"\n]{12,}" + # Wide hex vectors anywhere in the call — the numeric crates wrap + # them in tuples and slices (`(0xA000000000000000, false)`, + # `from_u32(&[0x00140000, 0x140000])`), so anchoring to the comma + # missed them. An 8+ digit hex literal in an assert is a + # hand-written expected value by definition. + r"|assert_eq!\([^;]{0,200}?0x[0-9a-fA-F]{8,}" + r"|\bhex!\(\s*\"|\bhex_literal::hex!\(" + r"|assert_(?:debug_|json_|yaml_|ron_)?snapshot!" + ), + }, + }, } # axis -> True if lower is better, False if higher is better diff --git a/.claude/skills/test-quality/tests/test_score.py b/.claude/skills/test-quality/tests/test_score.py index b79bf41..fb3b11a 100644 --- a/.claude/skills/test-quality/tests/test_score.py +++ b/.claude/skills/test-quality/tests/test_score.py @@ -750,3 +750,137 @@ def test_python_b1_counts_human_readable_fixed_vector(tmp_path): "def test_x():\n" " assert render(x) == 'Hello, world! A long fixed vector'\n") assert score.measure(tmp_path, "python")["B1_fixed_vector"] == 1 + + +# ───────────────────────── Rust (calibrated) ──────────────────────────────── +# Counts below are hand-derived by reading each fixture, never recomputed with +# the profile's own regex. The named regressions record what real suites broke: +# serde_json (raw-string JSON vectors), rustfmt (multi-line asserts) and +# serde_json's lexical module (hex vectors nested in tuples/slices). +def test_detect_lang_rust(tmp_path): + write(tmp_path, "tests/parse.rs", "#[test]\nfn a() {}\n") + assert score.detect_lang(tmp_path) == "rust" + + +def test_rust_test_def_counts_attributes_but_not_cfg_test(tmp_path): + # 4 real tests: #[test], #[tokio::test], #[rstest], #[bench]. + # `#[cfg(test)]` is a compilation gate, not a test — it must NOT count, + # otherwise every inline unit-test module inflates test_count by one. + write(tmp_path, "src/lib.rs", + "#[cfg(test)]\n" + "mod tests {\n" + " #[test]\n fn plain() {}\n" + " #[tokio::test]\n async fn async_flavour() {}\n" + " #[rstest]\n fn parametrized() {}\n" + " #[bench]\n fn b(_: &mut Bencher) {}\n" + "}\n") + assert score.measure(tmp_path, "rust")["test_count"] == 4 + + +def test_rust_inline_unit_tests_under_src_are_measured(tmp_path): + # Rust's dominant idiom puts unit tests inline in src/ behind #[cfg(test)]; + # a tests/-only file pattern would score those suites as empty. + write(tmp_path, "src/parser.rs", "#[cfg(test)]\nmod t {\n#[test]\nfn a() {}\n}\n") + assert score.measure(tmp_path, "rust")["test_count"] == 1 + + +def test_rust_a2_is_uncountable_not_zero(tmp_path): + # As in Go: `#[cfg(test)] mod tests` sees private items by design, and + # integration tests can only reach `pub`. No countable smell → None, so the + # axis is excluded from the W/L/T tally rather than handed a free win. + write(tmp_path, "tests/a.rs", "#[test]\nfn a() { assert_eq!(1, 1); }\n") + assert score.measure(tmp_path, "rust")["A2_private_symbol"] is None + + +def test_rust_a1_counts_error_text_matching_not_collection_contains(tmp_path): + # 2 error-text matchers; the Vec::contains call is a legitimate assertion + # and must not be counted as a fragile substring match. + write(tmp_path, "tests/a.rs", + "#[test]\nfn a() {\n" + " assert!(err.to_string().contains(\"bad input\"));\n" + " assert!(res.unwrap_err().to_string().contains(\"nope\"));\n" + " assert!(names.contains(&\"alice\"));\n" + "}\n") + assert score.measure(tmp_path, "rust")["A1_substring_match"] == 2 + + +def test_rust_b1_raw_string_json_vector_SERDE_JSON_regression(tmp_path): + # `r#"{"a":1}"#` — the embedded-JSON fixture idiom. Its body contains `"`, + # so a `[^"\n]` class stops at the first inner quote and scores it 0. + write(tmp_path, "tests/a.rs", + "#[test]\nfn a() {\n" + " assert_eq!(r#\"{\"a\":1,\"b\":{\"foo\": 2},\"c\":3}\"#, out);\n" + "}\n") + assert score.measure(tmp_path, "rust")["B1_fixed_vector"] == 1 + + +def test_rust_b1_multiline_assert_RUSTFMT_regression(tmp_path): + # rustfmt splits long asserts across lines — and long literals are exactly + # the ones that get split, so a line-anchored pattern misses the vectors it + # most wants to find. + write(tmp_path, "tests/a.rs", + "#[test]\nfn a() {\n" + " assert_eq!(\n" + " to_string(&value).unwrap(),\n" + " \"a fixed vector long enough\"\n" + " );\n" + "}\n") + assert score.measure(tmp_path, "rust")["B1_fixed_vector"] == 1 + + +def test_rust_b1_hex_vector_in_tuple_or_slice_LEXICAL_regression(tmp_path): + # Numeric crates park expected values in tuples and slices; anchoring the + # hex to the comma missed both forms. 2 asserts, 2 counts. + write(tmp_path, "tests/a.rs", + "#[test]\nfn a() {\n" + " assert_eq!(big.hi64(), (0xA000000000000000, false));\n" + " assert_eq!(x.data, from_u32(&[0x00140000, 0x140000]));\n" + "}\n") + assert score.measure(tmp_path, "rust")["B1_fixed_vector"] == 2 + + +def test_rust_b1_ignores_short_literals(tmp_path): + # 11 chars is under the 12-char bar shared with js/kotlin/swift; 12 is over. + write(tmp_path, "tests/a.rs", + "#[test]\nfn a() {\n" + " assert_eq!(fmt(x), \"elevenchars\");\n" + " assert_eq!(fmt(y), \"twelvechars!\");\n" + "}\n") + assert score.measure(tmp_path, "rust")["B1_fixed_vector"] == 1 + + +def test_rust_b1_counts_hex_bang_vectors(tmp_path): + # RustCrypto's `hex!("…")` is the canonical published-test-vector form. + write(tmp_path, "tests/a.rs", + "#[test]\nfn a() {\n" + " assert_eq!(digest, hex!(\"ba7816bf8f01cfea\"));\n" + "}\n") + assert score.measure(tmp_path, "rust")["B1_fixed_vector"] >= 1 + + +def test_rust_c1_counts_mockall_and_hand_written_doubles(tmp_path): + # 3: #[automock], a mock! block, and a hand-written MockRepo struct. + write(tmp_path, "tests/a.rs", + "#[automock]\ntrait Repo {}\n" + "mock! { Other {} }\n" + "struct MockRepo;\n" + "#[test]\nfn a() {}\n") + assert score.measure(tmp_path, "rust")["C1_mock_real"] == 3 + + +def test_rust_c2_counts_real_io_primitives(tmp_path): + # Legitimate boundary fakes — reported for context, never scored. 2 here. + write(tmp_path, "tests/a.rs", + "#[test]\nfn a() {\n" + " let s = wiremock::MockServer::start().await;\n" + " let d = tempfile::tempdir().unwrap();\n" + "}\n") + assert score.measure(tmp_path, "rust")["C2_mock_framework"] == 2 + + +def test_rust_param_counts_rstest_cases_and_proptest(tmp_path): + # 2 #[case] rows + 1 proptest! block = 3 parametrization signals. + write(tmp_path, "tests/a.rs", + "#[rstest]\n#[case(1)]\n#[case(2)]\nfn a(#[case] n: u8) {}\n" + "proptest! {\n #[test]\n fn b(x in 0..10u8) {}\n}\n") + assert score.measure(tmp_path, "rust")["parametrize"] == 3 diff --git a/README.md b/README.md index 921e6b0..57be7ad 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ Or score any suite directly, without the skill harness: ```bash python .claude/skills/test-quality/scripts/score.py \ - --tests path/to/tests --baseline path/to/old_tests --lang python|js|go|kotlin|swift + --tests path/to/tests --baseline path/to/old_tests --lang python|js|go|kotlin|swift|rust ``` The Python profile is the empirically-validated one (it drove the experiment