Merge branch 'main' into feature/dedupe-git-test-helpers

Author: ahrav

.beads/issues.jsonl

@@ -1730,6 +1730,97 @@ fn run_git_repo_worker_treats_cursor_covered_target_as_exhausted_empty() {
     );
 }
 
+/// Multi-commit secret-bearing histories must complete successfully without
+/// triggering shard parking. History depth alone is not a parking condition;
+/// only error-class conditions (permanent failures, repeated transient errors,
+/// poisoned state) qualify.
+#[test]
+fn run_git_repo_worker_completes_multi_commit_secret_history() {
+    const COMMIT_COUNT: usize = 16;
+    let repo = create_git_repo_fixture_with_secret_history(COMMIT_COUNT);
+    let mirror_root = tempdir().expect("mirror root");
+    let mut mirrors = LocalMirrorManager::new(mirror_root.path()).expect("mirror manager");
+    let backend = TestGitBackend::default();
+    let findings_sink = InMemoryFindingsSink::new();
+    let done_ledger = InMemoryDoneLedger::new();
+    let mut coordinator =
+        setup_coordinator_with_git_shard(repo.path(), CoordCursorUpdate::initial(), 120_000);
+
+    let report = run_git_repo_worker(
+        &mut coordinator,
+        &mut mirrors,
+        git_worker_identity(repo.path()),
+        backend.clone(),
+        DistributedPersistence::new(findings_sink.clone(), done_ledger.clone()),
+        DistributedRuntimeConfig::default(),
+    )
+    .expect("multi-commit secret-bearing history should scan successfully");
+
+    assert_eq!(report.leases_seen, 1);
+    assert_eq!(report.shards_scanned, 1);
+    assert_eq!(run_progress(&coordinator).done(), 1);
+
+    let summaries = shard_summaries(&coordinator);
+    assert_eq!(summaries.len(), 1);
+    assert_ne!(
+        summaries[0].status(),
+        ShardStatus::Parked,
+        "history depth alone must not park the shard"
+    );
+    assert_eq!(
+        summaries[0].status(),
+        ShardStatus::Done,
+        "history depth alone must not strand the shard"
+    );
+
+    let expected_key = git_repo_key(repo.path());
+    assert_eq!(
+        summaries[0]
+            .last_key()
+            .expect("completed shard should have a last_key"),
+        expected_key.as_bytes(),
+        "shard cursor last_key should match the singleton repo key"
+    );
+
+    assert!(
+        backend.batch_call_count() > 0,
+        "git repo worker must durably persist repo state before advancing the shard"
+    );
+    assert!(
+        !backend.stored_keys().is_empty(),
+        "persistence backend should contain durable state after a complete scan"
+    );
+
+    let persisted = findings_sink
+        .findings_snapshot()
+        .expect("findings snapshot");
+    assert!(
+        persisted.len() >= COMMIT_COUNT,
+        "each of the {COMMIT_COUNT} secret-bearing commits should produce at least one \
+         persisted finding (got {})",
+        persisted.len()
+    );
+
+    let rows = done_ledger.snapshot().expect("done-ledger snapshot");
+    assert_eq!(
+        rows.len(),
+        1,
+        "singleton repo shard produces one done-ledger row"
+    );
+    assert_eq!(
+        rows[0].status(),
+        DoneLedgerStatus::ScannedWithFindings,
+        "secret-bearing history should produce a findings-bearing done-ledger row"
+    );
+    // Single repo-frontier shard: the done-ledger's per-item findings_count
+    // equals total persisted finding rows because there is exactly one item.
+    assert_eq!(
+        rows[0].findings_count(),
+        u32::try_from(persisted.len()).expect("findings count fits u32"),
+        "done-ledger findings_count must match persisted findings"
+    );
+}
+
 /// Git repo-frontier shards require `CursorSemantics::Completed` so the
 /// checkpoint cursor represents fully-processed and durable progress.
 /// `Dispatched` semantics are rejected before any scan work begins.

crates/gossip-scanner-runtime/src/distributed/integration_tests.rs

@@ -430,17 +430,40 @@ pub(super) fn successor_bytes(bytes: &[u8]) -> Vec<u8> {
     next
 }
 
-/// Git repo fixture seeded with a secret so scans produce at least one finding.
+/// Single-commit git repo fixture seeded with a secret so scans produce at least one finding.
 pub(super) fn create_git_repo_fixture_with_secrets() -> tempfile::TempDir {
+    create_git_repo_fixture_with_secret_history(1)
+}
+
+/// Git repo fixture where each commit introduces a secret in a distinct file.
+///
+/// Every commit adds a new `secret-{N}.txt`, so each commit's diff independently
+/// contains a detectable secret regardless of whether the scanner operates on
+/// full blobs or diffs.
+pub(super) fn create_git_repo_fixture_with_secret_history(
+    commit_count: usize,
+) -> tempfile::TempDir {
+    assert!(
+        commit_count > 0,
+        "secret-history fixture requires at least one commit"
+    );
+
     let dir = tempdir().expect("tempdir");
     init_git_repo(
         dir.path(),
         "distributed-runtime-tests@example.com",
         "Distributed Runtime Tests",
     );
-    fs::write(dir.path().join("secret.txt"), secret_fixture()).expect("write fixture");
-    run_git(dir.path(), &["add", "."]);
-    run_git(dir.path(), &["commit", "-q", "-m", "fixture"]);
+
+    for commit in 0..commit_count {
+        let filename = format!("secret-{commit}.txt");
+        let contents = format!("{}\ncommit-{commit}\n", secret_fixture());
+        fs::write(dir.path().join(&filename), contents).expect("write fixture");
+        run_git(dir.path(), &["add", "."]);
+        let message = format!("fixture-{commit}");
+        run_git(dir.path(), &["commit", "-q", "-m", message.as_str()]);
+    }
+
     dir
 }
 

crates/gossip-scanner-runtime/src/distributed/test_support.rs

@@ -59,7 +59,7 @@ Non-negotiables (project-wide):
 
 | Trait | Purpose | Key methods |
 |-------|---------|-------------|
-| `PersistenceFinding` | Unified finding-identity surface consumed by persistence translation, regardless of source family | `rule_id(&self) -> u32`, `norm_hash(&self) -> NormHash`, `span_start(&self) -> u64`, `span_end(&self) -> u64`, `span_len(&self) -> u64` |
+| `PersistenceFinding` | Unified finding-identity surface consumed by persistence translation, regardless of source family | `rule_id(&self) -> u32`, `norm_hash(&self) -> NormHash`, `blob_offset_start(&self) -> u64`, `blob_offset_end(&self) -> u64`, `blob_offset_len(&self) -> u64` |
 | `DoneLedger` | Dedupe index: "was this object-version scanned under this policy?" | `batch_get(&self, TenantId, PolicyHash, &[OvidHash]) -> Result<Vec<Option<DoneLedgerRecord>>, Self::Error>`, `list_done_hashes(&self, TenantId, PolicyHash) -> Result<Vec<OvidHash>, Self::Error>`, `batch_upsert(&self, &[DoneLedgerRecord]) -> Result<Self::CommitHandle, Self::Error>` |
 | `FindingsSink` | Triage/query plane: findings + occurrences + observations persistence | `upsert_batch(&self, FindingsUpsertBatch<'_>) -> Result<Self::CommitHandle, Self::Error>` |
 | `CommitHandle` | Durable acknowledgement handle; `wait()` consumes self and returns a receipt | `wait(self) -> Result<Self::Receipt, Self::Error>` |

docs/gossip-contracts/boundary-5-persistence.md

@@ -42,6 +42,9 @@ and validation, and Git connector mode uses the direct path.
 | `src/distributed/execution.rs` | Lease execution functions (`run_filesystem_lease`, `run_git_repo_lease`) and worker entry points (`run_worker`, `run_git_repo_worker`) |
 | `src/distributed/commit_bridge.rs` | `ReceiptCommitSink` adapter, receipt-driven checkpoint building, `drain_commit_stage` |
 | `src/distributed/lease_ops.rs` | Lease lifecycle: `advance_shard`, `ArmedLeaseDeadline`, `watch_lease_deadline`, `LeaseUncertaintySignal` |
+| `src/distributed/integration_tests.rs` | Distributed lease orchestration and coordination integration tests |
+| `src/distributed/test_support.rs` | Test helpers, builders, and mock fixtures for distributed execution tests |
+| `src/distributed/unit_tests.rs` | Unit tests for lease lifecycle, commit bridging, and distributed types |
 | `src/event_sink.rs` | JSONL, text, JSON, and SARIF event sinks |
 | `src/git_discovery.rs` | Static single-target Git repository discovery source for payload-backed repo-frontier shards |
 | `src/git_executor.rs` | Contract-level adapter that implements `GitRepoExecutor` for mirror-backed repo scans by translating `GitSelection` + `GitExecutionLimits` into `scanner-git` config, propagating repo/policy identity into persistence-aware runs, and reusing the shared runtime runner |

docs/gossip-scanner-runtime.md

@@ -34,12 +34,12 @@ Each category is a separate test binary gated behind a Cargo feature:
 
 | Binary          | Path                       | Feature Gate         | Tests |
 | --------------- | -------------------------- | -------------------- | ----: |
-| `integration`   | `tests/integration/main.rs`| `integration-tests`  |  ~162 |
+| `integration`   | `tests/integration/main.rs`| `integration-tests`  |  ~176 |
 | `property`      | `tests/property/main.rs`   | `property-tests`     |  ~130 |
-| `simulation`    | `tests/simulation/main.rs` | various (see below)  |   ~41 |
+| `simulation`    | `tests/simulation/main.rs` | various (see below)  |   ~42 |
 | `diagnostic`    | `tests/diagnostic/main.rs` | `diagnostic-tests`   |     2 |
 | `smoke`         | `tests/smoke/main.rs`      | `smoke-tests`        |     0 |
-| *(standalone)*  | `tests/chunked_file_scans.rs` | *(none)*          |     2 |
+| *(standalone)*  | `tests/chunked_file_scans.rs` | *(none)*          |     3 |
 
 ## Feature Gates
 
@@ -105,23 +105,24 @@ scanner-git boundaries.
 | -------------------------- | ----: | -------------------------------------------------- |
 | `anchor_optimization`      |    14 | Anchor derivation and optimization                 |
 | `archive_scanning`         |    49 | Archive expansion, virtual paths, budget limits    |
-| `bench_guards`             |     0 | Guards against benchmark execution without the benchmark feature gate (no active tests) |
+| `bench_guards`             |     1 | Guards against benchmark execution without the benchmark feature gate |
 | `binary_awareness`         |    10 | Binary file detection                              |
 | `finding_json`             |     4 | JSONL finding parsing helpers used by integration assertions |
-| `git_commit_walk`          |     6 | Commit graph traversal                             |
+| `git_commit_walk`          |     8 | Commit graph traversal                             |
 | `git_engine_adapter`       |     1 | Git-to-engine adapter                              |
 | `git_inmem_artifacts`      |    13 | In-memory git artifact handling                    |
 | `git_mapping_bridge`       |     3 | MIDX mapping bridge                                |
 | `git_pack_exec`            |     1 | Pack execution                                     |
 | `git_pack_inflate`         |     4 | Pack inflation/decompression                       |
 | `git_pack_inflate_corpus`  |     5 | Pathological zlib regression corpus                |
 | `git_pack_plan`            |    14 | Pack plan computation                              |
-| `git_persist`              |     3 | Git persistence                                    |
+| `git_persist`              |     4 | Git persistence                                    |
 | `git_preflight`            |     4 | Git preflight checks                               |
 | `git_repo_open`            |     4 | Repository opening                                 |
 | `git_run_format`           |     1 | Run format validation                              |
-| `git_scan_validation`      |    10 | Git scan validation                                |
-| `git_seen_unique`          |     2 | Deduplication of seen objects                      |
+| `git_scan_validation`      |    15 | Git scan validation                                |
+| `git_seen_crash_recovery`  |     3 | Seen-bitmap crash recovery                         |
+| `git_seen_unique`          |     4 | Deduplication of seen objects                      |
 | `git_snapshot`             |     1 | Snapshot testing                                   |
 | `git_tree_diff`            |    10 | Tree diff computation                              |
 | `manual_anchors`           |     3 | Manual anchor specification                        |
@@ -148,7 +149,7 @@ contains both deterministic `#[test]` assertions and `proptest!` fuzz runs.
 | `git_spill_dedupe`                   |     3 |        2 | Spill deduplication                   |
 | `git_tree_diff`                      |     2 |        2 | Tree diff properties                  |
 | `path_policy_soundness`              |     4 |        1 | Path allow/deny soundness             |
-| `proptest_support`                   |     2 |        0 | Shared proptest helpers and shrinker guards |
+| `proptest_support`                   |     2 |        1 | Shared proptest helpers and shrinker guards |
 | `regex2anchor_soundness`             |    26 |        2 | Regex-to-anchor derivation soundness  |
 | `secret_bytes_safelist_soundness`    |     3 |        1 | Safelist soundness                    |
 | `value_suppressor_soundness`         |     2 |        1 | Value suppression soundness           |

docs/scanner-engine-integration-tests.md

@@ -108,7 +108,7 @@ The device slots system implements **per-device admission control**:
 
 ```rust
 pub struct DeviceSlotPermit {
-    inner: CountPermit,
+    _inner: CountPermit,
     device: DeviceId,
 }
 ```

docs/scanner-scheduler/scheduler-device-slots.md

@@ -229,7 +229,7 @@ The spill permission uses an enum to distinguish three cases:
 enum SpillGrant {
     NotRequested,           // Job didn't request spill
     Unlimited,              // Spill enabled; no slot counting
-    Limited(CountPermit),   // Spill enabled; using 1 counted slot
+    Limited { _permit: CountPermit },   // Spill enabled; using 1 counted slot
 }
 ```
 

docs/scanner-scheduler/scheduler-global-resource-pool.md

@@ -1079,7 +1079,7 @@ struct ArchiveWork {
 struct UringArchiveSink<'a, E: ScanEngine> {
     engine: &'a E,
     scratch: &'a mut E::Scratch,
-    pending: &'a mut Vec<Finding>,
+    pending: &'a mut Vec<<E::Scratch as EngineScratch>::Finding>,
     event_sink: &'a dyn EventOutput,
     display: Vec<u8>,               // Current entry display path
     container_file_id: FileId,      // ID of the archive file itself
@@ -1099,13 +1099,14 @@ struct UringArchiveSink<'a, E: ScanEngine> {
 // Main loop for I/O worker thread
 fn io_worker_loop<E: ScanEngine>(
     _wid: usize,
-    rx: chan::Receiver<FileWork>,      // File queue
-    pool: Arc<FixedBufferPool>,         // Shared buffer pool
-    cpu: ExecutorHandle<CpuTask>,       // CPU executor handle
+    rx: chan::Receiver<FileWork>,
+    pool: Arc<FixedBufferPool>,
+    cpu: ExecutorHandle<CpuTask>,
     engine: Arc<E>,
     cfg: LocalFsUringConfig,
-    stop: Arc<AtomicBool>,              // Graceful shutdown flag
-    archive_tx: Option<chan::Sender<ArchiveWork>>, // Archive routing channel
+    stop: Arc<AtomicBool>,
+    archive_tx: Option<chan::Sender<ArchiveWork>>,
+    extract_tx: Option<chan::Sender<ExtractWork>>,
 ) -> io::Result<UringIoStats>;
 
 // Archive worker loop (blocking decompression + scan)

docs/scanner-scheduler/scheduler-local-fs-uring.md

@@ -36,9 +36,9 @@ SCHEDULER_SIM_STRESS_SEEDS=100 cargo test --features scheduler-sim scheduler_sim
 - **Corpus**: `crates/scanner-engine-integration-tests/tests/simulation/corpus/*.json` - regression tests replayed on every run
 - **Failures**: `crates/scanner-engine-integration-tests/tests/failures/*.json` - where stress failures are written
 
-## SimCase DSL Reference
+## ReproArtifact DSL Reference
 
-A `SimCase` defines a complete simulation scenario in JSON. The harness loads the case, initializes the executor, and steps through actions until completion or failure.
+A `ReproArtifact` defines a complete simulation scenario in JSON. The harness loads the case, initializes the executor, and steps through actions until completion or failure.
 
 ### Top-Level Structure