diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd082888..e19c9cda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,53 @@ jobs: run: | cargo nextest run + coverage: + name: "coverage" + + runs-on: ubuntu-latest + + needs: determine_changes + if: ${{ (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }} + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + + - name: "Install Rust toolchain" + run: rustup component add llvm-tools-preview + + - name: "Install tools" + uses: taiki-e/install-action@a416ddeedbd372e614cc1386e8b642692f66865e # v2.57.1 + with: + tool: cargo-llvm-cov,cargo-nextest,just,maturin + + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 + with: + python-version: "3.10" + + - name: Setup Python + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 + id: setup-python + with: + python-version: "3.10" + + - name: "Run coverage" + env: + PYO3_PYTHON: ${{ steps.setup-python.outputs.python-path }} + KARVA_MAX_PARALLELISM: 2 + VIRTUAL_ENV: "" + PYTHON_VERSION: "3.10" + run: just coverage + + - name: "Upload coverage report" + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-report + path: target/coverage-html/ + build-docs: name: "Build docs" diff --git a/.gitignore b/.gitignore index 159205e2..f454acdc 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,8 @@ uv.lock # Compiled Python files *.so +# Coverage profile data +*.profraw + # prek prettier cache node_modules diff --git a/crates/karva/tests/it/common/mod.rs b/crates/karva/tests/it/common/mod.rs index 9511ccbc..258e651c 100644 --- a/crates/karva/tests/it/common/mod.rs +++ b/crates/karva/tests/it/common/mod.rs @@ -132,8 +132,30 @@ impl TestContext { .unwrap_or_else(|e| panic!("Failed to read file `{full_path}`: {e}")) } + /// Create a base `Command` for the karva binary. + /// + /// When running under `cargo test` / `cargo llvm-cov`, uses the cargo-built + /// binary (instrumented for coverage) and sets env vars so the worker + /// binary and Python packages are found correctly. + fn karva_command(&self) -> Command { + if let Some(binary) = cargo_karva_binary() { + let mut cmd = Command::new(binary); + // Tell the karva process where to find the instrumented worker binary + if let Some(worker) = cargo_worker_binary() { + cmd.env("KARVA_WORKER_BINARY", worker.as_str()); + } + // Pass VIRTUAL_ENV + coverage flag so the embedded Python in the + // worker subprocess activates the venv (for pytest, karva package, etc.) + cmd.env("VIRTUAL_ENV", self.venv_path.as_str()); + cmd.env(karva_static::EnvVars::KARVA_COVERAGE_INTERNAL, "1"); + cmd + } else { + Command::new(self.venv_binary("karva")) + } + } + pub fn command(&self) -> Command { - let mut command = Command::new(self.venv_binary("karva")); + let mut command = self.karva_command(); command.arg("test").current_dir(self.root()); command } @@ -145,7 +167,7 @@ impl TestContext { } pub fn snapshot(&self, subcommand: &str) -> Command { - let mut command = Command::new(self.venv_binary("karva")); + let mut command = self.karva_command(); command .arg("snapshot") .arg(subcommand) @@ -154,7 +176,7 @@ impl TestContext { } pub fn cache(&self, subcommand: &str) -> Command { - let mut command = Command::new(self.venv_binary("karva")); + let mut command = self.karva_command(); command .arg("cache") .arg(subcommand) @@ -335,6 +357,47 @@ fn cleanup_old_shared_venvs(cache_dir: &Utf8Path, current_venv_name: &str) { } } +/// Returns the path to the cargo-built `karva` binary when coverage mode is enabled. +/// +/// Coverage mode is activated by setting the internal coverage env var. +/// In this mode, the cargo-built binaries (instrumented by `cargo llvm-cov`) +/// are used instead of the venv-installed console scripts. +fn cargo_karva_binary() -> Option<&'static str> { + if std::env::var(karva_static::EnvVars::KARVA_COVERAGE_INTERNAL).is_ok() { + option_env!("CARGO_BIN_EXE_karva") + } else { + None + } +} + +/// Finds the `karva-worker` binary for coverage. +/// +/// Checks the same directory as the `karva` binary first (coverage target dir), +/// then falls back to the standard `target/debug/` directory. +fn cargo_worker_binary() -> Option { + let karva = cargo_karva_binary()?; + let name = if cfg!(windows) { + "karva-worker.exe" + } else { + "karva-worker" + }; + + // Check same directory as karva binary (e.g., target/llvm-cov-target/debug/) + if let Some(dir) = Utf8Path::new(karva).parent() { + let worker = dir.join(name); + if worker.exists() { + return Some(worker); + } + } + + // Fall back to standard target/debug/ directory + let workspace_root = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent())?; + let worker = workspace_root.join("target").join("debug").join(name); + worker.exists().then_some(worker) +} + fn create_and_populate_venv( venv_path: &Utf8PathBuf, python_version: &str, diff --git a/crates/karva_runner/src/orchestration.rs b/crates/karva_runner/src/orchestration.rs index 862880b1..d668d125 100644 --- a/crates/karva_runner/src/orchestration.rs +++ b/crates/karva_runner/src/orchestration.rs @@ -16,6 +16,7 @@ use karva_collector::{CollectedPackage, CollectionSettings}; use karva_logging::time::format_duration; use karva_metadata::ProjectSettings; use karva_project::Project; +use karva_static::EnvVars; use crate::collection::ParallelCollector; use crate::partition::{Partition, partition_collected_tests}; @@ -355,8 +356,17 @@ const MIN_TESTS_PER_WORKER: usize = 5; const KARVA_WORKER_BINARY_NAME: &str = "karva-worker"; const WORKER_POLL_INTERVAL: Duration = Duration::from_millis(10); -/// Find the `karva-worker` binary by checking PATH, the project venv, and the active venv. +/// Find the `karva-worker` binary by checking the `KARVA_WORKER_BINARY` env var, +/// then PATH, the project venv, and the active venv. fn find_karva_worker_binary(current_dir: &Utf8PathBuf) -> Result { + if let Ok(path) = std::env::var(EnvVars::KARVA_WORKER_BINARY) { + let path = Utf8PathBuf::from(path); + if path.exists() { + tracing::debug!(path = %path, "Found binary from KARVA_WORKER_BINARY env var"); + return Ok(path); + } + } + which::which(KARVA_WORKER_BINARY_NAME) .ok() .and_then(|path| Utf8PathBuf::try_from(path).ok()) diff --git a/crates/karva_static/src/lib.rs b/crates/karva_static/src/lib.rs index 842ac2d7..a0c1dcbc 100644 --- a/crates/karva_static/src/lib.rs +++ b/crates/karva_static/src/lib.rs @@ -15,6 +15,18 @@ impl EnvVars { /// When set to "1" or "true", snapshot assertions write directly to `.snap` /// instead of creating `.snap.new` pending files. pub const KARVA_SNAPSHOT_UPDATE: &'static str = "KARVA_SNAPSHOT_UPDATE"; + + /// Override the path to the `karva-worker` binary. + /// Used for coverage instrumentation where the cargo-built binary + /// should be used instead of the venv-installed console script. + pub const KARVA_WORKER_BINARY: &'static str = "KARVA_WORKER_BINARY"; + + /// Private env var that activates venv support for the embedded Python + /// interpreter. Set by `just coverage-full` so the cargo-built worker + /// can find pytest, karva, etc. from the venv's site-packages. + /// The double-underscore prefix signals this is an internal implementation detail. + #[doc(hidden)] + pub const KARVA_COVERAGE_INTERNAL: &'static str = "__KARVA_COVERAGE"; } pub fn max_parallelism() -> NonZeroUsize { diff --git a/crates/karva_test_semantic/src/utils.rs b/crates/karva_test_semantic/src/utils.rs index fe07740c..1817a4b7 100644 --- a/crates/karva_test_semantic/src/utils.rs +++ b/crates/karva_test_semantic/src/utils.rs @@ -150,6 +150,7 @@ where F: for<'py> FnOnce(Python<'py>) -> R, { attach(|py| { + coverage::activate_venv_if_coverage(py); let null_file = redirect_python_output(py, show_python_output); let result = f(py); if let Ok(Some(null_file)) = null_file { @@ -159,6 +160,93 @@ where }) } +/// Coverage-only venv activation, gated behind `__KARVA_COVERAGE`. +/// +/// When the cargo-built worker runs under `cargo llvm-cov`, `PyO3`'s embedded +/// Python doesn't know about the venv. This module fixes that by: +/// 1. Registering the embedded `karva._karva` module (prevents type mismatches +/// with the wheel's `.so`) +/// 2. Adding the venv's site-packages (for pytest, etc.) +/// 3. Fixing `sys.executable` (so `Command(sys.executable)` runs Python, not karva) +/// +/// All of this is a no-op unless `__KARVA_COVERAGE` is set. +mod coverage { + use camino::{Utf8Path, Utf8PathBuf}; + use pyo3::prelude::*; + use pyo3::types::PyAnyMethods; + + pub(super) fn activate_venv_if_coverage(py: Python<'_>) { + if std::env::var(karva_static::EnvVars::KARVA_COVERAGE_INTERNAL).is_err() { + return; + } + let Ok(venv_path) = std::env::var("VIRTUAL_ENV") else { + return; + }; + + register_embedded_module(py); + add_site_packages(py, &venv_path); + fix_sys_executable(py, &venv_path); + } + + fn register_embedded_module(py: Python<'_>) { + let Ok(module) = PyModule::new(py, "_karva") else { + return; + }; + if crate::init_module(py, &module).is_err() { + return; + } + let _ = module.setattr("karva_run", py.None()); + let _ = module.setattr("karva_worker_run", py.None()); + if let Ok(sys) = py.import("sys") { + if let Ok(modules) = sys.getattr("modules") { + let _ = modules.set_item("karva._karva", &module); + } + } + } + + fn add_site_packages(py: Python<'_>, venv_path: &str) { + let Some(sp) = find_site_packages(venv_path) else { + return; + }; + if let Ok(site) = py.import("site") { + let _ = site.call_method1("addsitedir", (sp.as_str(),)); + } + } + + fn fix_sys_executable(py: Python<'_>, venv_path: &str) { + let venv = Utf8Path::new(venv_path); + let python_bin = if cfg!(windows) { + venv.join("Scripts").join("python.exe") + } else { + venv.join("bin").join("python") + }; + if python_bin.exists() { + if let Ok(sys) = py.import("sys") { + let _ = sys.setattr("executable", python_bin.as_str()); + } + } + } + + fn find_site_packages(venv_path: &str) -> Option { + let venv = Utf8Path::new(venv_path); + if cfg!(windows) { + let sp = venv.join("Lib").join("site-packages"); + return sp.exists().then_some(sp); + } + let lib_dir = venv.join("lib"); + for entry in std::fs::read_dir(&lib_dir).ok()?.flatten() { + if entry.file_name().to_string_lossy().starts_with("python") { + if let Ok(sp) = Utf8PathBuf::from_path_buf(entry.path().join("site-packages")) { + if sp.exists() { + return Some(sp); + } + } + } + } + None + } +} + /// A simple wrapper around `Python::attach` that initializes the Python interpreter first. pub(crate) fn attach(f: F) -> R where diff --git a/justfile b/justfile index 16fc252b..63cd6613 100644 --- a/justfile +++ b/justfile @@ -8,3 +8,30 @@ test *args: else \ cargo test {{args}}; \ fi + +coverage *args: + #!/usr/bin/env bash + set -euo pipefail + + # Find llvm-cov and llvm-profdata from PATH, rustup sysroot, or Homebrew + find_llvm_tool() { + local tool="$1" + if command -v "$tool" > /dev/null 2>&1; then echo "$tool"; return; fi + local sysroot_bin="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | grep host | awk '{print $2}')/bin/$tool" + if [ -x "$sysroot_bin" ]; then echo "$sysroot_bin"; return; fi + local brew="/opt/homebrew/opt/llvm/bin/$tool" + if [ -x "$brew" ]; then echo "$brew"; return; fi + echo "error: could not find $tool" >&2; exit 1 + } + LLVM_COV=$(find_llvm_tool llvm-cov) + LLVM_PROFDATA=$(find_llvm_tool llvm-profdata) + export LLVM_COV LLVM_PROFDATA + + rm -rf target/wheels + maturin build + RUSTFLAGS="-C instrument-coverage -C llvm-args=--instrprof-atomic-counter-update-all" cargo build --target-dir target/llvm-cov-target -p karva_worker + __KARVA_COVERAGE=1 cargo llvm-cov nextest --no-report {{args}} + "$LLVM_PROFDATA" merge -failure-mode=warn target/llvm-cov-target/*.profraw -o target/llvm-cov-target/merged.profdata + "$LLVM_COV" report target/llvm-cov-target/debug/karva -object target/llvm-cov-target/debug/karva-worker -instr-profile=target/llvm-cov-target/merged.profdata -ignore-filename-regex='(\.cargo|rustc-|/rustlib/|\.claude/)' + "$LLVM_COV" show target/llvm-cov-target/debug/karva -object target/llvm-cov-target/debug/karva-worker -instr-profile=target/llvm-cov-target/merged.profdata -ignore-filename-regex='(\.cargo|rustc-|/rustlib/|\.claude/)' --format=html -output-dir=target/coverage-html + rm -f default_*.profraw