diagnostic diff

.github/workflows/diagnostic-diff.yml

@@ -0,0 +1,111 @@
+name: Diagnostic Diff
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+# Allow only one concurrent diagnostic diff per PR
+concurrency:
+  group: diagnostic-diff-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  diagnostic-diff:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+
+      - name: Update Rust toolchain
+        run: rustup update
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.13"
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          command: build
+
+      - name: Run diagnostics on PR branch
+        run: |
+          cargo run --release -p karva_diffs --bin karva-diagnostics -- run --output pr-diagnostics.json
+        continue-on-error: true
+
+      - name: Stash PR diagnostics
+        run: |
+          mkdir -p /tmp/karva-diagnostics
+          cp pr-diagnostics.json /tmp/karva-diagnostics/
+
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          command: build
+
+      - name: Run diagnostics on main branch
+        run: |
+          cargo run --release -p karva_diffs --bin karva-diagnostics -- run --output main-diagnostics.json
+        continue-on-error: true
+
+      - name: Retrieve PR diagnostics
+        run: |
+          cp /tmp/karva-diagnostics/pr-diagnostics.json .
+
+      - name: Generate diff
+        run: |
+          cargo run --release -p karva_diffs --bin karva-diagnostics -- diff --base main-diagnostics.json --head pr-diagnostics.json --output diff.md
+
+      - name: Comment PR
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const diff = fs.readFileSync('diff.md', 'utf8');
+
+            // Find existing comment
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(comment =>
+              comment.user.type === 'Bot' &&
+              comment.body.includes('Diagnostic Diff Report')
+            );
+
+            const body = diff + '\n\n---\n*This comment is automatically generated by the diagnostic diff workflow.*';
+
+            if (botComment) {
+              // Update existing comment
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: body
+              });
+            } else {
+              // Create new comment
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: body
+              });
+            }

crates/karva_benchmark/benches/karva_walltime.rs

@@ -1,16 +1,15 @@
-use std::{path::PathBuf, sync::Once};
+use std::sync::Once;
 
 use karva_benchmark::{
+    InstalledProject, RealWorldProject, affect_project,
     criterion::{BatchSize, Criterion, criterion_group, criterion_main},
-    real_world_projects::{InstalledProject, RealWorldProject},
 };
 use karva_core::{TestRunner, testing::setup_module};
 use karva_project::{
     path::absolute,
     project::{Project, ProjectOptions},
     verbosity::VerbosityLevel,
 };
-use ruff_python_ast::PythonVersion;
 
 static SETUP_MODULE_ONCE: Once = Once::new();
 
@@ -73,15 +72,7 @@ fn bench_project(benchmark: &ProjectBenchmark, criterion: &mut Criterion) {
 }
 
 fn affect(criterion: &mut Criterion) {
-    let benchmark = ProjectBenchmark::new(RealWorldProject {
-        name: "affect",
-        repository: "https://github.com/MatthewMckee4/affect",
-        commit: "803cc916b492378a8ad8966e747cac3325e11b5f",
-        paths: vec![PathBuf::from("tests")],
-        dependencies: vec!["pydantic", "pydantic-settings", "pytest"],
-        python_version: PythonVersion::PY313,
-    });
-
+    let benchmark = ProjectBenchmark::new(affect_project());
     bench_project(&benchmark, criterion);
 }
 

crates/karva_benchmark/src/lib.rs

@@ -1,7 +1,12 @@
 use std::path::PathBuf;
 
 pub mod criterion;
-pub mod real_world_projects;
+
+// Re-export real world projects from karva_test
+pub use karva_test::{
+    InstalledProject, RealWorldProject, affect_project, get_real_world_projects,
+    real_world_projects,
+};
 
 pub static TRUE_ASSERTIONS: TestFile = TestFile::new(
     "test_true_assertions.py",
@@ -36,19 +41,6 @@ pub static PARAMETRIZE: TestFile = TestFile::new(
     include_str!("../resources/test_parametrize.py"),
 );
 
-/// Relative size of a test case. Benchmarks can use it to configure the time for how long a benchmark should run to get stable results.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
-pub enum TestCaseSpeed {
-    /// A test case that is fast to run
-    Fast,
-
-    /// A normal test case
-    Normal,
-
-    /// A slow test case
-    Slow,
-}
-
 #[derive(Debug, Clone)]
 pub struct TestCase {
     file: TestFile,

crates/karva_core/src/diagnostic/diagnostic.rs

@@ -50,7 +50,7 @@ impl Diagnostic {
     }
 
     #[must_use]
-    pub(crate) const fn severity(&self) -> &DiagnosticSeverity {
+    pub const fn severity(&self) -> &DiagnosticSeverity {
         &self.inner.severity
     }
 
@@ -175,14 +175,14 @@ impl DiagnosticInner {
 
 // Diagnostic severity
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum DiagnosticSeverity {
+pub enum DiagnosticSeverity {
     Error(DiagnosticErrorType),
     Warning(String),
 }
 
 impl DiagnosticSeverity {
     #[must_use]
-    pub(crate) const fn is_error(&self) -> bool {
+    pub const fn is_error(&self) -> bool {
         matches!(self, Self::Error(_))
     }
 
@@ -193,7 +193,7 @@ impl DiagnosticSeverity {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum DiagnosticErrorType {
+pub enum DiagnosticErrorType {
     TestCase {
         test_name: String,
         diagnostic_type: TestCaseDiagnosticType,
@@ -203,17 +203,17 @@ pub(crate) enum DiagnosticErrorType {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum TestCaseDiagnosticType {
+pub enum TestCaseDiagnosticType {
     Fail(String),
     Collection(TestCaseCollectionDiagnosticType),
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum TestCaseCollectionDiagnosticType {
+pub enum TestCaseCollectionDiagnosticType {
     FixtureNotFound,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum FixtureDiagnosticType {
+pub enum FixtureDiagnosticType {
     Invalid,
 }

crates/karva_core/src/runner/diagnostic.rs

@@ -126,6 +126,7 @@ impl TestResultStats {
         self.inner.values().sum()
     }
 
+    #[must_use]
     pub fn is_success(&self) -> bool {
         self.failed() == 0
     }
@@ -135,17 +136,17 @@ impl TestResultStats {
     }
 
     #[must_use]
-    pub(crate) fn passed(&self) -> usize {
+    pub fn passed(&self) -> usize {
         self.get(TestResultKind::Passed)
     }
 
     #[must_use]
-    pub(crate) fn failed(&self) -> usize {
+    pub fn failed(&self) -> usize {
         self.get(TestResultKind::Failed)
     }
 
     #[must_use]
-    pub(crate) fn skipped(&self) -> usize {
+    pub fn skipped(&self) -> usize {
         self.get(TestResultKind::Skipped)
     }
 

crates/karva_core/src/runner/mod.rs

@@ -9,9 +9,9 @@ use crate::{
     utils::attach,
 };
 
-pub(crate) mod diagnostic;
+pub mod diagnostic;
 
-pub(crate) use diagnostic::TestRunResult;
+pub use diagnostic::TestRunResult;
 
 pub trait TestRunner {
     fn test(&self) -> TestRunResult {

crates/karva_diffs/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "karva_diffs"
+version = "0.0.0"
+description = "Diagnostic diff tests for Karva on real-world projects"
+publish = false
+authors = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+homepage = { workspace = true }
+documentation = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+
+[[bin]]
+name = "karva-diagnostics"
+path = "src/bin/karva-diagnostics.rs"
+
+[dependencies]
+karva_core = { workspace = true }
+karva_project = { workspace = true }
+karva_test = { workspace = true }
+
+anyhow = { workspace = true }
+ruff_python_ast = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+clap = { workspace = true }
+
+[lints]
+workspace = true