diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 094f5c99..5dca39b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,8 +39,8 @@ jobs: continue-on-error: true # Coverage is ENFORCED, not just reported. Measured 2026-07-29 at 85.66% on every leg of this - # matrix — 3.10, 3.11 and 3.12 alike, 1584 tests — so the floor sits just below it and no matrix - # leg is closer to the line than any other. Before this the job printed a coverage report and + # matrix — 3.10, 3.11 and 3.12 alike, 1584 tests — so the floor sits just below it, and because + # no leg measured differently, enforcing it on one leg (below) loses nothing. Before this the job printed a coverage report and # failed on nothing: OCR-024 claimed a >=80% floor that was never in the workflow, so coverage # could regress to zero and CI stayed green. # Ratchet upward as the suite grows; never lower it to make a red build pass. @@ -49,12 +49,31 @@ jobs: # [model] extra (pydantic/pyyaml/sqlglot — sqlglot backs the binding-validation and # unit-resolution paths; without it ~287 tests skip). DB drivers are intentionally # omitted: those tests skip cleanly without a database. + # + # Why two steps and `-n auto` (#296): this step took ~600s on EVERY leg, one test at a time, + # under coverage tracing. `--durations` showed no handful of slow tests to fix — the slowest is + # ~4s and the 40 slowest are a quarter of the run — so the time is the long tail, and the lever + # is running it in parallel, not trimming it. The floor is enforced on 3.12 only, with the + # `sysmon` tracer that 3.12 added, because tracing is what costs most on the older versions and + # the three legs measured the same coverage when the floor was set. 3.10 and 3.11 still run the + # whole suite; they just don't trace it. Nothing is deselected on any leg. + - name: pytest + if: matrix.python-version != '3.12' + run: >- + uvx --python ${{ matrix.python-version }} + --with pytest --with pytest-xdist + --with-editable "packages/agami-core[model,server]" + pytest tests/ -q -n auto + - name: pytest --cov + if: matrix.python-version == '3.12' + env: + COVERAGE_CORE: sysmon run: >- uvx --python ${{ matrix.python-version }} - --with pytest --with pytest-cov + --with pytest --with pytest-cov --with pytest-xdist --with-editable "packages/agami-core[model,server]" - pytest tests/ -q --cov=plugins --cov=packages/agami-core/src --cov-report=term-missing + pytest tests/ -q -n auto --cov=plugins --cov=packages/agami-core/src --cov-report=term-missing --cov-fail-under=85 # The safety corpus gets its own required checks rather than riding inside `lint-and-test`. diff --git a/.gitignore b/.gitignore index f4e9b62c..d471aa92 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,8 @@ plugins/agami/samples/store/model/curation_log.jsonl # Python tooling artifacts .coverage +# per-worker data files under `-n auto`; merged and removed at the end, left behind if a run is interrupted +.coverage.* coverage.xml htmlcov/ .ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 01ad1548..f776a9af 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: hooks: - id: pytest name: pytest (full suite) - entry: uvx --with pytest-cov --with pydantic --with pyyaml --with sqlglot pytest tests/ -q + entry: uvx --with pytest-cov --with pytest-xdist --with pydantic --with pyyaml --with sqlglot pytest tests/ -q -n auto language: system always_run: true pass_filenames: false diff --git a/CLAUDE.md b/CLAUDE.md index f72b4546..0c090339 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,9 +57,10 @@ tests, docstrings, fixtures, sample data, or docs. existing self-contained style (`pytest.importorskip(...)`, `sys.path` to the scripts dir). - **Check your change is covered:** `uv run dev.py cover` reports coverage of just the lines your branch touched and fails on untested ones — independent of overall coverage. -- **Coverage is measured, not yet floored.** The suite currently sits around the mid-70s%; a - `--cov-fail-under` floor will be added in `pyproject.toml` once the baseline is locked, and it - will be the single source of that number (don't hard-code a different one here). +- **Coverage is floored in CI.** The gate is `--cov-fail-under` on the py3.12 leg of `lint + test` + in `.github/workflows/ci.yml` — the one leg that traces coverage — and that flag is the single + source of the number (don't hard-code a copy here). It is not in `pyproject.toml`, so a bare + local `pytest` stays un-floored. ## Proposing a substantial change diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e94f3c2..47f4a37e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,9 +58,12 @@ The suite imports the `agami-core` library, so install it editable with its `[mo without a database). `uvx` wires it up for the run: ```bash -uvx --with pytest-cov --with-editable "packages/agami-core[model,server]" pytest tests/ -q +uvx --with pytest-cov --with pytest-xdist --with-editable "packages/agami-core[model,server]" pytest tests/ -q -n auto ``` +`-n auto` runs one worker per core, the way CI does. A test that passes alone and fails under it is +sharing state with another test — fix the sharing rather than dropping the flag. + The privacy test (`tests/test_privacy_no_network.py`) is a contract: no shipped script may make a network call — adding a network-egress primitive fails the build. @@ -71,8 +74,8 @@ no test exercises) — the quickest way to confirm a change is tested, regardles Under the hood that's: ```bash -uvx --with pytest-cov --with-editable "packages/agami-core[model,server]" \ - pytest tests/ -q --cov=plugins --cov=packages/agami-core/src --cov-report=xml +uvx --with pytest-cov --with pytest-xdist --with-editable "packages/agami-core[model,server]" \ + pytest tests/ -q -n auto --cov=plugins --cov=packages/agami-core/src --cov-report=xml uvx diff-cover coverage.xml --compare-branch=origin/main ``` diff --git a/dev.py b/dev.py index 842f691e..f3b9d513 100644 --- a/dev.py +++ b/dev.py @@ -29,7 +29,9 @@ RUFF = ["uvx", "ruff@0.15.19"] # The suite imports the agami-core library, so install it editable with the [model] # extra (pydantic/pyyaml/sqlglot). DB drivers are omitted on purpose — those tests skip without a DB. -TEST_DEPS = ["--with", "pytest-cov", "--with-editable", "packages/agami-core[model,server]"] +TEST_DEPS = ["--with", "pytest-cov", "--with", "pytest-xdist", "--with-editable", "packages/agami-core[model,server]"] +# One worker per core, as CI runs it — the suite is a long tail of short tests, not a few slow ones. +PARALLEL = ["-n", "auto"] # `dev/` is here because it holds gate logic CI executes (changelog_gate.py), not just local # helpers — code the build depends on should be linted like the code it guards. TARGETS = ["plugins", "packages", "tests", "dev.py", "dev"] @@ -68,7 +70,7 @@ def fmt() -> int: def test() -> int: - return run(["uvx", *TEST_DEPS, "pytest", "tests/", "-q"]) + return run(["uvx", *TEST_DEPS, "pytest", "tests/", "-q", *PARALLEL]) def secrets() -> int: @@ -113,7 +115,7 @@ def cover() -> int: """Coverage of the lines THIS branch changed (fails on untested changed lines).""" # Make sure origin/main exists locally (fresh clones / worktrees may not have it). run(["git", "fetch", "--quiet", "origin", "main"], allow_fail=True) - rc = run(["uvx", *TEST_DEPS, "pytest", "tests/", "-q", + rc = run(["uvx", *TEST_DEPS, "pytest", "tests/", "-q", *PARALLEL, "--cov=plugins", "--cov=packages/agami-core/src", "--cov-report=xml"]) return rc or run(["uvx", "diff-cover", "coverage.xml", "--compare-branch=origin/main"]) diff --git a/pyproject.toml b/pyproject.toml index 3be508e7..554d2344 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ ignore = ["E402", "E702", "E731", "E741", "F841"] [tool.pytest.ini_options] testpaths = ["tests"] # No --cov in addopts on purpose: a bare `pytest` then works without pytest-cov installed -# (IDE runners, quick local runs). The CI gate and `dev.py cover` pass --cov explicitly; -# a --cov-fail-under floor will be added once the baseline is locked. +# (IDE runners, quick local runs). The CI gate and `dev.py cover` pass --cov explicitly, and the +# --cov-fail-under floor lives only on CI's py3.12 leg (.github/workflows/ci.yml). markers = [ # Applied by the safety corpus's parametrizers, and counted by tests/e2e/conftest.py — both at # collection and again at session end, against a constant per marker. Registered so an unknown-mark diff --git a/tests/e2e/test_suite_integrity.py b/tests/e2e/test_suite_integrity.py index 5e494277..4978ebfe 100644 --- a/tests/e2e/test_suite_integrity.py +++ b/tests/e2e/test_suite_integrity.py @@ -33,6 +33,7 @@ import difflib import json import os +import re import shlex import subprocess import sys @@ -310,6 +311,27 @@ def test_this_file_runs_inside_the_required_job(): assert any("pytest tests/" in command for command in _run_commands(job)), job +def test_exactly_one_leg_enforces_the_coverage_floor(): + """The floor is gated to one matrix leg (#296), which makes it one `if:` away from gating none. + + Flip the 3.12 step's condition, or rename a version in the matrix without renaming it here, and + every leg still runs the suite and goes green while no leg checks coverage at all. So: exactly + one step carries `--cov-fail-under`, and the version its condition names is one the matrix runs. + The value of the floor is deliberately not asserted — it is meant to ratchet. + """ + job = _workflow()["jobs"][LINT_AND_TEST] + floored = [step for step in _steps(job) if "--cov-fail-under" in step.get("run", "")] + assert len(floored) == 1, [step.get("name") for step in floored] + + condition = floored[0].get("if") + if condition is None: + return # an ungated step runs on every leg, which enforces the floor more, not less + match = re.fullmatch(r"\s*matrix\.python-version\s*==\s*'([^']+)'\s*", condition) + assert match, f"the floor's condition is no longer a single-version match: {condition!r}" + versions = [str(v) for v in job["strategy"]["matrix"]["python-version"]] + assert match.group(1) in versions, (match.group(1), versions) + + def test_the_stdio_child_imports_the_checkout_under_test(): """`route_stdio`'s child resolves the executor out of THIS source tree, not a pip-installed one.