Add code coverage infrastructure (#537)

## Summary

- Add `cargo llvm-cov` coverage infrastructure with two modes:
  - `just coverage` — karva crate only, ~70% line coverage, HTML output
- `just coverage-full` — all crates including `karva_test_semantic` via
instrumented worker binary, ~86% line coverage
- Add coverage CI job to the existing workflow (same trigger conditions
as `cargo test`)
- Add `KARVA_WORKER_BINARY` env var override for worker binary discovery
- Add private `__KARVA_COVERAGE` env var to gate coverage-only venv
activation in the embedded Python interpreter

The coverage-specific code (venv activation for the embedded PyO3
interpreter) is completely gated behind `__KARVA_COVERAGE` and is a
no-op during normal test runs. All 685 tests continue to pass unchanged.

## Test plan

- [x] `just test` — 685/685 tests pass (no behavior change)
- [x] `just coverage` — 457/457 tests pass, HTML report generated
- [x] `just coverage-full` — 457/457 tests pass, 86% line coverage
across all crates
- [x] `uvx prek run -a` — all pre-commit checks pass

Author: MatthewMckee4

.github/workflows/ci.yml

@@ -126,6 +126,53 @@ jobs:
         run: |
           cargo nextest run
 
+  coverage:
+    name: "coverage"
+
+    runs-on: ubuntu-latest
+
+    needs: determine_changes
+    if: ${{ (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }}
+
+    steps:
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+        with:
+          persist-credentials: false
+
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
+
+      - name: "Install Rust toolchain"
+        run: rustup component add llvm-tools-preview
+
+      - name: "Install tools"
+        uses: taiki-e/install-action@a416ddeedbd372e614cc1386e8b642692f66865e # v2.57.1
+        with:
+          tool: cargo-llvm-cov,cargo-nextest,just,maturin
+
+      - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
+        with:
+          python-version: "3.10"
+
+      - name: Setup Python
+        uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1
+        id: setup-python
+        with:
+          python-version: "3.10"
+
+      - name: "Run coverage"
+        env:
+          PYO3_PYTHON: ${{ steps.setup-python.outputs.python-path }}
+          KARVA_MAX_PARALLELISM: 2
+          VIRTUAL_ENV: ""
+          PYTHON_VERSION: "3.10"
+        run: just coverage
+
+      - name: "Upload coverage report"
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: coverage-report
+          path: target/coverage-html/
+
   build-docs:
     name: "Build docs"
 

.gitignore

@@ -12,5 +12,8 @@ uv.lock
 # Compiled Python files
 *.so
 
+# Coverage profile data
+*.profraw
+
 # prek prettier cache
 node_modules

crates/karva/tests/it/common/mod.rs

@@ -132,8 +132,30 @@ impl TestContext {
             .unwrap_or_else(|e| panic!("Failed to read file `{full_path}`: {e}"))
     }
 
+    /// Create a base `Command` for the karva binary.
+    ///
+    /// When running under `cargo test` / `cargo llvm-cov`, uses the cargo-built
+    /// binary (instrumented for coverage) and sets env vars so the worker
+    /// binary and Python packages are found correctly.
+    fn karva_command(&self) -> Command {
+        if let Some(binary) = cargo_karva_binary() {
+            let mut cmd = Command::new(binary);
+            // Tell the karva process where to find the instrumented worker binary
+            if let Some(worker) = cargo_worker_binary() {
+                cmd.env("KARVA_WORKER_BINARY", worker.as_str());
+            }
+            // Pass VIRTUAL_ENV + coverage flag so the embedded Python in the
+            // worker subprocess activates the venv (for pytest, karva package, etc.)
+            cmd.env("VIRTUAL_ENV", self.venv_path.as_str());
+            cmd.env(karva_static::EnvVars::KARVA_COVERAGE_INTERNAL, "1");
+            cmd
+        } else {
+            Command::new(self.venv_binary("karva"))
+        }
+    }
+
     pub fn command(&self) -> Command {
-        let mut command = Command::new(self.venv_binary("karva"));
+        let mut command = self.karva_command();
         command.arg("test").current_dir(self.root());
         command
     }
@@ -145,7 +167,7 @@ impl TestContext {
     }
 
     pub fn snapshot(&self, subcommand: &str) -> Command {
-        let mut command = Command::new(self.venv_binary("karva"));
+        let mut command = self.karva_command();
         command
             .arg("snapshot")
             .arg(subcommand)
@@ -154,7 +176,7 @@ impl TestContext {
     }
 
     pub fn cache(&self, subcommand: &str) -> Command {
-        let mut command = Command::new(self.venv_binary("karva"));
+        let mut command = self.karva_command();
         command
             .arg("cache")
             .arg(subcommand)
@@ -335,6 +357,47 @@ fn cleanup_old_shared_venvs(cache_dir: &Utf8Path, current_venv_name: &str) {
     }
 }
 
+/// Returns the path to the cargo-built `karva` binary when coverage mode is enabled.
+///
+/// Coverage mode is activated by setting the internal coverage env var.
+/// In this mode, the cargo-built binaries (instrumented by `cargo llvm-cov`)
+/// are used instead of the venv-installed console scripts.
+fn cargo_karva_binary() -> Option<&'static str> {
+    if std::env::var(karva_static::EnvVars::KARVA_COVERAGE_INTERNAL).is_ok() {
+        option_env!("CARGO_BIN_EXE_karva")
+    } else {
+        None
+    }
+}
+
+/// Finds the `karva-worker` binary for coverage.
+///
+/// Checks the same directory as the `karva` binary first (coverage target dir),
+/// then falls back to the standard `target/debug/` directory.
+fn cargo_worker_binary() -> Option<Utf8PathBuf> {
+    let karva = cargo_karva_binary()?;
+    let name = if cfg!(windows) {
+        "karva-worker.exe"
+    } else {
+        "karva-worker"
+    };
+
+    // Check same directory as karva binary (e.g., target/llvm-cov-target/debug/)
+    if let Some(dir) = Utf8Path::new(karva).parent() {
+        let worker = dir.join(name);
+        if worker.exists() {
+            return Some(worker);
+        }
+    }
+
+    // Fall back to standard target/debug/ directory
+    let workspace_root = Utf8Path::new(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .and_then(|p| p.parent())?;
+    let worker = workspace_root.join("target").join("debug").join(name);
+    worker.exists().then_some(worker)
+}
+
 fn create_and_populate_venv(
     venv_path: &Utf8PathBuf,
     python_version: &str,

crates/karva_runner/src/orchestration.rs

@@ -16,6 +16,7 @@ use karva_collector::{CollectedPackage, CollectionSettings};
 use karva_logging::time::format_duration;
 use karva_metadata::ProjectSettings;
 use karva_project::Project;
+use karva_static::EnvVars;
 
 use crate::collection::ParallelCollector;
 use crate::partition::{Partition, partition_collected_tests};
@@ -355,8 +356,17 @@ const MIN_TESTS_PER_WORKER: usize = 5;
 const KARVA_WORKER_BINARY_NAME: &str = "karva-worker";
 const WORKER_POLL_INTERVAL: Duration = Duration::from_millis(10);
 
-/// Find the `karva-worker` binary by checking PATH, the project venv, and the active venv.
+/// Find the `karva-worker` binary by checking the `KARVA_WORKER_BINARY` env var,
+/// then PATH, the project venv, and the active venv.
 fn find_karva_worker_binary(current_dir: &Utf8PathBuf) -> Result<Utf8PathBuf> {
+    if let Ok(path) = std::env::var(EnvVars::KARVA_WORKER_BINARY) {
+        let path = Utf8PathBuf::from(path);
+        if path.exists() {
+            tracing::debug!(path = %path, "Found binary from KARVA_WORKER_BINARY env var");
+            return Ok(path);
+        }
+    }
+
     which::which(KARVA_WORKER_BINARY_NAME)
         .ok()
         .and_then(|path| Utf8PathBuf::try_from(path).ok())

crates/karva_static/src/lib.rs

@@ -15,6 +15,18 @@ impl EnvVars {
     /// When set to "1" or "true", snapshot assertions write directly to `.snap`
     /// instead of creating `.snap.new` pending files.
     pub const KARVA_SNAPSHOT_UPDATE: &'static str = "KARVA_SNAPSHOT_UPDATE";
+
+    /// Override the path to the `karva-worker` binary.
+    /// Used for coverage instrumentation where the cargo-built binary
+    /// should be used instead of the venv-installed console script.
+    pub const KARVA_WORKER_BINARY: &'static str = "KARVA_WORKER_BINARY";
+
+    /// Private env var that activates venv support for the embedded Python
+    /// interpreter. Set by `just coverage-full` so the cargo-built worker
+    /// can find pytest, karva, etc. from the venv's site-packages.
+    /// The double-underscore prefix signals this is an internal implementation detail.
+    #[doc(hidden)]
+    pub const KARVA_COVERAGE_INTERNAL: &'static str = "__KARVA_COVERAGE";
 }
 
 pub fn max_parallelism() -> NonZeroUsize {

crates/karva_test_semantic/src/utils.rs

@@ -150,6 +150,7 @@ where
     F: for<'py> FnOnce(Python<'py>) -> R,
 {
     attach(|py| {
+        coverage::activate_venv_if_coverage(py);
         let null_file = redirect_python_output(py, show_python_output);
         let result = f(py);
         if let Ok(Some(null_file)) = null_file {
@@ -159,6 +160,93 @@ where
     })
 }
 
+/// Coverage-only venv activation, gated behind `__KARVA_COVERAGE`.
+///
+/// When the cargo-built worker runs under `cargo llvm-cov`, `PyO3`'s embedded
+/// Python doesn't know about the venv. This module fixes that by:
+/// 1. Registering the embedded `karva._karva` module (prevents type mismatches
+///    with the wheel's `.so`)
+/// 2. Adding the venv's site-packages (for pytest, etc.)
+/// 3. Fixing `sys.executable` (so `Command(sys.executable)` runs Python, not karva)
+///
+/// All of this is a no-op unless `__KARVA_COVERAGE` is set.
+mod coverage {
+    use camino::{Utf8Path, Utf8PathBuf};
+    use pyo3::prelude::*;
+    use pyo3::types::PyAnyMethods;
+
+    pub(super) fn activate_venv_if_coverage(py: Python<'_>) {
+        if std::env::var(karva_static::EnvVars::KARVA_COVERAGE_INTERNAL).is_err() {
+            return;
+        }
+        let Ok(venv_path) = std::env::var("VIRTUAL_ENV") else {
+            return;
+        };
+
+        register_embedded_module(py);
+        add_site_packages(py, &venv_path);
+        fix_sys_executable(py, &venv_path);
+    }
+
+    fn register_embedded_module(py: Python<'_>) {
+        let Ok(module) = PyModule::new(py, "_karva") else {
+            return;
+        };
+        if crate::init_module(py, &module).is_err() {
+            return;
+        }
+        let _ = module.setattr("karva_run", py.None());
+        let _ = module.setattr("karva_worker_run", py.None());
+        if let Ok(sys) = py.import("sys") {
+            if let Ok(modules) = sys.getattr("modules") {
+                let _ = modules.set_item("karva._karva", &module);
+            }
+        }
+    }
+
+    fn add_site_packages(py: Python<'_>, venv_path: &str) {
+        let Some(sp) = find_site_packages(venv_path) else {
+            return;
+        };
+        if let Ok(site) = py.import("site") {
+            let _ = site.call_method1("addsitedir", (sp.as_str(),));
+        }
+    }
+
+    fn fix_sys_executable(py: Python<'_>, venv_path: &str) {
+        let venv = Utf8Path::new(venv_path);
+        let python_bin = if cfg!(windows) {
+            venv.join("Scripts").join("python.exe")
+        } else {
+            venv.join("bin").join("python")
+        };
+        if python_bin.exists() {
+            if let Ok(sys) = py.import("sys") {
+                let _ = sys.setattr("executable", python_bin.as_str());
+            }
+        }
+    }
+
+    fn find_site_packages(venv_path: &str) -> Option<Utf8PathBuf> {
+        let venv = Utf8Path::new(venv_path);
+        if cfg!(windows) {
+            let sp = venv.join("Lib").join("site-packages");
+            return sp.exists().then_some(sp);
+        }
+        let lib_dir = venv.join("lib");
+        for entry in std::fs::read_dir(&lib_dir).ok()?.flatten() {
+            if entry.file_name().to_string_lossy().starts_with("python") {
+                if let Ok(sp) = Utf8PathBuf::from_path_buf(entry.path().join("site-packages")) {
+                    if sp.exists() {
+                        return Some(sp);
+                    }
+                }
+            }
+        }
+        None
+    }
+}
+
 /// A simple wrapper around `Python::attach` that initializes the Python interpreter first.
 pub(crate) fn attach<F, R>(f: F) -> R
 where

justfile

@@ -8,3 +8,30 @@ test *args:
     else \
         cargo test {{args}}; \
     fi
+
+coverage *args:
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    # Find llvm-cov and llvm-profdata from PATH, rustup sysroot, or Homebrew
+    find_llvm_tool() {
+        local tool="$1"
+        if command -v "$tool" > /dev/null 2>&1; then echo "$tool"; return; fi
+        local sysroot_bin="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | grep host | awk '{print $2}')/bin/$tool"
+        if [ -x "$sysroot_bin" ]; then echo "$sysroot_bin"; return; fi
+        local brew="/opt/homebrew/opt/llvm/bin/$tool"
+        if [ -x "$brew" ]; then echo "$brew"; return; fi
+        echo "error: could not find $tool" >&2; exit 1
+    }
+    LLVM_COV=$(find_llvm_tool llvm-cov)
+    LLVM_PROFDATA=$(find_llvm_tool llvm-profdata)
+    export LLVM_COV LLVM_PROFDATA
+
+    rm -rf target/wheels
+    maturin build
+    RUSTFLAGS="-C instrument-coverage -C llvm-args=--instrprof-atomic-counter-update-all" cargo build --target-dir target/llvm-cov-target -p karva_worker
+    __KARVA_COVERAGE=1 cargo llvm-cov nextest --no-report {{args}}
+    "$LLVM_PROFDATA" merge -failure-mode=warn target/llvm-cov-target/*.profraw -o target/llvm-cov-target/merged.profdata
+    "$LLVM_COV" report target/llvm-cov-target/debug/karva -object target/llvm-cov-target/debug/karva-worker -instr-profile=target/llvm-cov-target/merged.profdata -ignore-filename-regex='(\.cargo|rustc-|/rustlib/|\.claude/)'
+    "$LLVM_COV" show target/llvm-cov-target/debug/karva -object target/llvm-cov-target/debug/karva-worker -instr-profile=target/llvm-cov-target/merged.profdata -ignore-filename-regex='(\.cargo|rustc-|/rustlib/|\.claude/)' --format=html -output-dir=target/coverage-html
+    rm -f default_*.profraw