fix(scip): re-run confidence-upgrade overlay on incremental reindex, not just server startup

run_overlay (the SCIP-backed upgrade of Rust call edges to `formal`
confidence per ADR-0004) only ever ran once, right after the initial
index at server startup — watcher::run_watch_loop's incremental reindex
loop never called it. Any Rust file added or edited during a live `ci
serve` session stayed capped at inferred/resolved/textual confidence
until the next full restart.

Simply calling it again wouldn't have helped on its own: its cache key
was (rust-analyzer version, Cargo.lock hash) only, so an ordinary source
edit with an unchanged lockfile would still cache-skip and silently do
nothing. Threads a `dirty` parameter (a path@hash fingerprint of every
indexed Rust file, via the new rust_source_dirty_keys) into the cache
key so source-only changes actually invalidate it, and wires run_overlay
into the watcher loop after every non-noop incremental reindex — cheap
when nothing Rust-relevant changed, thanks to that same cache.

Also fixes a latent instance of the same bug at the startup call site
(a source-only change between two restarts, with the lockfile untouched,
previously cache-skipped too), and adds an `indexing_status.scip_overlay`
field (`{available, up_to_date}`) so an agent can observe overlay
staleness without triggering a run.

Verified live: editing a tracked .rs file while `ci serve` was running
caused .codeindex/scip.cache's key to change (confirming a real
rust-analyzer re-run, not a cache-skip), and indexing_status reflected
up_to_date flipping accordingly.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: Your Name

crates/ci-core/src/scip/cache.rs

@@ -20,4 +20,37 @@ mod tests {
         let b = overlay_cache_key("1.96.0", "hashBBB", &["src/x.rs".into()]);
         assert_ne!(a, b);
     }
+
+    /// Regression: before `dirty` was threaded into `run_overlay`, editing a
+    /// Rust file's body with an unchanged `Cargo.lock`/toolchain produced the
+    /// exact same key every time, so the overlay silently never re-ran for
+    /// that file's new content. `dirty` (per-file content fingerprints) must
+    /// change the key on its own, independent of lockfile/version.
+    #[test]
+    fn cache_key_changes_with_dirty_content_alone() {
+        let a = overlay_cache_key("1.96.0", "hashAAA", &["src/x.rs@hash1".into()]);
+        let b = overlay_cache_key("1.96.0", "hashAAA", &["src/x.rs@hash2".into()]);
+        assert_ne!(
+            a, b,
+            "same version + lockfile but different file content must not collide"
+        );
+    }
+
+    #[test]
+    fn cache_key_is_order_independent_over_dirty_set() {
+        let a = overlay_cache_key(
+            "1.96.0",
+            "hashAAA",
+            &["src/a.rs@h1".into(), "src/b.rs@h2".into()],
+        );
+        let b = overlay_cache_key(
+            "1.96.0",
+            "hashAAA",
+            &["src/b.rs@h2".into(), "src/a.rs@h1".into()],
+        );
+        assert_eq!(
+            a, b,
+            "dirty set is sorted before hashing — order must not matter"
+        );
+    }
 }

crates/ci-core/src/scip/mod.rs

@@ -22,18 +22,22 @@ use crate::config::RustConfig;
 /// silent (finding nothing is the common, expected case for a checkout that
 /// never configured this at all, not worth a log line every session).
 ///
-/// Caches on (rust-analyzer version, Cargo.lock hash): an unchanged toolchain
-/// and dependency set means a re-run would find the same call graph, so the
-/// (comparatively expensive) rust-analyzer pass is skipped and the previous
-/// upgrades — already persisted as `formal`/`ruled_out_by_scip` in the DB —
-/// stand. Per-file dirty tracking isn't wired here (would need a change-set
-/// the caller doesn't have at this point); this key alone is safe because it
-/// can only widen a "skip" into a "run" (any lockfile/toolchain difference
-/// invalidates it), never the reverse.
+/// Caches on (rust-analyzer version, Cargo.lock hash, `dirty`): an unchanged
+/// toolchain, dependency set, and Rust source state means a re-run would find
+/// the same call graph, so the (comparatively expensive) rust-analyzer pass
+/// is skipped and the previous upgrades — already persisted as
+/// `formal`/`ruled_out_by_scip` in the DB — stand. `dirty` is the caller's
+/// current Rust-source fingerprint (see `rust_source_dirty_keys`) — pass it
+/// so a source-only change (no lockfile/toolchain difference, e.g. editing a
+/// function body) still invalidates the cache instead of silently standing
+/// forever; an empty slice degrades to the old (lockfile/toolchain-only) key,
+/// which remains safe on its own because it can only widen a "skip" into a
+/// "run", never the reverse.
 pub fn run_overlay(
     conn: &Connection,
     root: &Path,
     rust: &RustConfig,
+    dirty: &[String],
 ) -> anyhow::Result<ingest::IngestStats> {
     if rust.scip.enabled == Some(false) {
         return Ok(ingest::IngestStats::default());
@@ -46,7 +50,7 @@ pub fn run_overlay(
     };
 
     let cache_path = root.join(".codeindex").join("scip.cache");
-    let key = cache::overlay_cache_key(&runner::binary_version(&bin), &lockfile_hash(root), &[]);
+    let key = cache::overlay_cache_key(&runner::binary_version(&bin), &lockfile_hash(root), dirty);
     if std::fs::read_to_string(&cache_path).is_ok_and(|prev| prev.trim() == key) {
         tracing::info!("SCIP overlay: cache key unchanged, skipping rust-analyzer run");
         return Ok(ingest::IngestStats::default());
@@ -85,6 +89,75 @@ fn lockfile_hash(root: &Path) -> String {
         .unwrap_or_default()
 }
 
+/// Fingerprint of every currently-indexed Rust file's content, for
+/// `run_overlay`'s `dirty` parameter — one `"path@hash"` entry per file
+/// (`hash` already computed by the indexer, so this is a cheap read, no
+/// re-hashing). Changes whenever any Rust file's content differs from what
+/// was indexed at the last successful overlay run, regardless of whether
+/// `Cargo.lock` or the rust-analyzer version also changed — see
+/// `run_overlay`'s doc comment for why that matters.
+pub fn rust_source_dirty_keys(conn: &Connection) -> Vec<String> {
+    let mut stmt = match conn
+        .prepare("SELECT path, hash FROM file_index WHERE language = 'rust' ORDER BY path")
+    {
+        Ok(s) => s,
+        Err(_) => return Vec::new(),
+    };
+    stmt.query_map([], |r| {
+        Ok(format!(
+            "{}@{}",
+            r.get::<_, String>(0)?,
+            r.get::<_, String>(1)?
+        ))
+    })
+    .map(|rows| rows.filter_map(|r| r.ok()).collect())
+    .unwrap_or_default()
+}
+
+/// Cheap, non-invoking snapshot of the overlay's readiness — never spawns
+/// rust-analyzer, just checks binary presence and compares the cache key that
+/// `run_overlay` would compute against what's already on disk. Backs
+/// `indexing_status`'s `scip_overlay` field so an agent can tell whether the
+/// call graph for currently-edited Rust files has actually been upgraded by
+/// SCIP yet, without waiting on or triggering a real run. `None` when
+/// `rust.scip.enabled == Some(false)` — overlay is off, nothing to report.
+pub fn overlay_status(conn: &Connection, root: &Path, rust: &RustConfig) -> Option<OverlayStatus> {
+    if rust.scip.enabled == Some(false) {
+        return None;
+    }
+    let bin = runner::resolve_binary(rust.scip.binary.as_deref());
+    let available = bin.is_some();
+    let up_to_date = match &bin {
+        Some(bin) => {
+            let dirty = rust_source_dirty_keys(conn);
+            let key = cache::overlay_cache_key(
+                &runner::binary_version(bin),
+                &lockfile_hash(root),
+                &dirty,
+            );
+            let cache_path = root.join(".codeindex").join("scip.cache");
+            std::fs::read_to_string(&cache_path).is_ok_and(|prev| prev.trim() == key)
+        }
+        None => false,
+    };
+    Some(OverlayStatus {
+        available,
+        up_to_date,
+    })
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct OverlayStatus {
+    /// `rust-analyzer` binary was found (PATH/rustup/VS Code) at last check.
+    pub available: bool,
+    /// The current Rust source fingerprint + toolchain + lockfile match the
+    /// last successful overlay run's cache key — `false` means the next
+    /// `run_overlay` call (or the next non-noop incremental reindex, if
+    /// wired to call it) would actually invoke rust-analyzer again rather
+    /// than cache-skip. Always `false` when `available` is `false`.
+    pub up_to_date: bool,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -106,11 +179,71 @@ mod tests {
             },
         };
         assert_eq!(
-            run_overlay(&conn, Path::new("."), &rust).unwrap(),
+            run_overlay(&conn, Path::new("."), &rust, &[]).unwrap(),
             ingest::IngestStats::default()
         );
     }
 
+    #[test]
+    fn rust_source_dirty_keys_reflects_path_and_hash_rust_only() {
+        let conn = Connection::open_in_memory().unwrap();
+        crate::db::schema::init_db(&conn).unwrap();
+        conn.execute(
+            "INSERT INTO file_index (path, hash, language, last_indexed) VALUES (?1, ?2, ?3, 0.0)",
+            rusqlite::params!["src/a.rs", "hashA", "rust"],
+        )
+        .unwrap();
+        conn.execute(
+            "INSERT INTO file_index (path, hash, language, last_indexed) VALUES (?1, ?2, ?3, 0.0)",
+            rusqlite::params!["src/main.py", "hashP", "python"],
+        )
+        .unwrap();
+
+        let keys = rust_source_dirty_keys(&conn);
+        assert_eq!(
+            keys,
+            vec!["src/a.rs@hashA".to_string()],
+            "must include only rust files, keyed by path+hash"
+        );
+    }
+
+    #[test]
+    fn rust_source_dirty_keys_changes_when_a_file_hash_changes() {
+        let conn = Connection::open_in_memory().unwrap();
+        crate::db::schema::init_db(&conn).unwrap();
+        conn.execute(
+            "INSERT INTO file_index (path, hash, language, last_indexed) VALUES ('src/a.rs', 'hash1', 'rust', 0.0)",
+            [],
+        )
+        .unwrap();
+        let before = rust_source_dirty_keys(&conn);
+
+        conn.execute(
+            "UPDATE file_index SET hash = 'hash2' WHERE path = 'src/a.rs'",
+            [],
+        )
+        .unwrap();
+        let after = rust_source_dirty_keys(&conn);
+
+        assert_ne!(
+            before, after,
+            "editing a rust file's content must change its dirty-key entry"
+        );
+    }
+
+    #[test]
+    fn overlay_status_none_when_explicitly_disabled() {
+        let conn = Connection::open_in_memory().unwrap();
+        crate::db::schema::init_db(&conn).unwrap();
+        let rust = RustConfig {
+            scip: crate::config::ScipConfig {
+                enabled: Some(false),
+                binary: None,
+            },
+        };
+        assert_eq!(overlay_status(&conn, Path::new("."), &rust), None);
+    }
+
     /// Live integration: real rust-analyzer against the Rust fixture workspace
     /// used throughout Phase A. Ignored by default -- requires rust-analyzer
     /// on PATH/rustup/VS Code, and a real `cargo metadata` resolve, neither of
@@ -133,7 +266,8 @@ mod tests {
                 binary: None,
             },
         };
-        let stats = run_overlay(&conn, &fixture, &rust).unwrap();
+        let dirty = rust_source_dirty_keys(&conn);
+        let stats = run_overlay(&conn, &fixture, &rust, &dirty).unwrap();
         assert!(
             stats.upgraded > 0,
             "expected at least one edge upgraded to formal"

crates/ci-server/src/lib.rs

@@ -113,7 +113,8 @@ pub async fn serve_stdio_with_preset(
                 let rust_cfg = ci_core::config::load_config(&indexer_root)
                     .map(|c| c.rust)
                     .unwrap_or_default();
-                match ci_core::scip::run_overlay(&conn, &indexer_root, &rust_cfg) {
+                let dirty = ci_core::scip::rust_source_dirty_keys(&conn);
+                match ci_core::scip::run_overlay(&conn, &indexer_root, &rust_cfg, &dirty) {
                     Ok(stats) if stats.upgraded > 0 || stats.ruled_out > 0 => {
                         // caller_count was computed by rebuild_graph before this
                         // overlay flipped edge_confidence/ruled_out_by_scip on

crates/ci-server/src/tools/recover.rs

@@ -51,6 +51,18 @@ impl CodeIntelligenceServer {
                     "Still indexing — poll again or use search/source while edges build",
                 )
             };
+
+            #[cfg(feature = "scip-overlay")]
+            let scip_overlay = {
+                let rust_cfg = ci_core::config::load_config(&self.project_root)
+                    .map(|c| c.rust)
+                    .unwrap_or_default();
+                ci_core::scip::overlay_status(&conn, &self.project_root, &rust_cfg)
+                    .map(ScipOverlayStatusOutput::from)
+            };
+            #[cfg(not(feature = "scip-overlay"))]
+            let scip_overlay: Option<ScipOverlayStatusOutput> = None;
+
             serde_json::to_string_pretty(&IndexingStatusOutput {
                 indexing_phase: phase,
                 files_indexed: files,
@@ -60,6 +72,7 @@ impl CodeIntelligenceServer {
                 embeddings_status: self.embed_status_str(),
                 edges_ready: self.edges_ready(),
                 last_updated: last_updated.map(epoch_to_iso8601),
+                scip_overlay,
                 suggested_next: self.filter_sn(sn),
             })
             .unwrap_or_default()
@@ -294,10 +307,53 @@ pub(crate) struct IndexingStatusOutput {
     pub(crate) edges_ready: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) last_updated: Option<String>,
+    /// `None` when this build wasn't compiled with the `scip-overlay` feature,
+    /// or `rust.scip.enabled` is explicitly `false` — nothing to report.
+    /// Otherwise reflects whether Rust call edges are currently up to date
+    /// with SCIP-upgraded (`formal`) confidence — see
+    /// `ci_core::scip::overlay_status`.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) scip_overlay: Option<ScipOverlayStatusOutput>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) suggested_next: Option<SuggestedNext>,
 }
 
+/// Local mirror of `ci_core::scip::OverlayStatus` — that type lives in
+/// `ci-core`, which doesn't depend on `schemars`, so it can't derive
+/// `JsonSchema` itself. Only exists when this crate is built with the
+/// `scip-overlay` feature (the same gate `ci_core::scip` itself is behind).
+#[cfg(feature = "scip-overlay")]
+#[derive(Serialize, JsonSchema)]
+pub(crate) struct ScipOverlayStatusOutput {
+    /// `rust-analyzer` binary was found (PATH/rustup/VS Code) at last check.
+    pub(crate) available: bool,
+    /// `false` means Rust source has changed since the last overlay run (or
+    /// none has ever run) — the next non-noop incremental reindex will
+    /// actually invoke rust-analyzer again rather than cache-skip.
+    pub(crate) up_to_date: bool,
+}
+
+#[cfg(feature = "scip-overlay")]
+impl From<ci_core::scip::OverlayStatus> for ScipOverlayStatusOutput {
+    fn from(s: ci_core::scip::OverlayStatus) -> Self {
+        Self {
+            available: s.available,
+            up_to_date: s.up_to_date,
+        }
+    }
+}
+
+/// Stub so `IndexingStatusOutput`'s `scip_overlay` field type-checks
+/// identically regardless of the `scip-overlay` feature — always `None` when
+/// this build lacks the feature (see the `#[cfg(not(...))]` binding at the
+/// `indexing_status` call site).
+#[cfg(not(feature = "scip-overlay"))]
+#[derive(Serialize, JsonSchema)]
+pub(crate) struct ScipOverlayStatusOutput {
+    pub(crate) available: bool,
+    pub(crate) up_to_date: bool,
+}
+
 // ---------------------------------------------------------------------------
 // Tool 14: locate
 // ---------------------------------------------------------------------------

crates/ci-server/src/watcher.rs

@@ -140,6 +140,53 @@ pub fn run_watch_loop(
                                 tracing::error!("Incremental chunk embedding failed: {e}");
                             }
                         }
+                        // Re-run the SCIP confidence-upgrade overlay so Rust call
+                        // edges touched by *this* reindex don't stay stuck below
+                        // `formal` for the rest of a long-running session — before
+                        // this, `run_overlay` only ever ran once at server startup
+                        // (see `lib.rs`), so any Rust file added/edited afterward
+                        // never got upgraded until the next restart. Cheap when
+                        // nothing Rust-relevant actually changed: `dirty` (the
+                        // current Rust source fingerprint) makes `run_overlay`'s own
+                        // cache key skip re-invoking rust-analyzer in that case —
+                        // see `run_overlay`'s doc comment.
+                        #[cfg(feature = "scip-overlay")]
+                        {
+                            let rust_cfg = ci_core::config::load_config(&project_root)
+                                .map(|c| c.rust)
+                                .unwrap_or_default();
+                            let dirty = ci_core::scip::rust_source_dirty_keys(&conn);
+                            match ci_core::scip::run_overlay(
+                                &conn,
+                                &project_root,
+                                &rust_cfg,
+                                &dirty,
+                            ) {
+                                Ok(stats) if stats.upgraded > 0 || stats.ruled_out > 0 => {
+                                    // caller_count was computed by this reindex's
+                                    // rebuild_graph before the overlay flipped
+                                    // edge_confidence/ruled_out_by_scip on some
+                                    // edges — refresh or it goes stale immediately
+                                    // relative to the columns it's filtered on.
+                                    if let Err(e) =
+                                        ci_core::indexer::pipeline::refresh_caller_counts(&conn)
+                                    {
+                                        tracing::warn!(
+                                            "caller_count refresh after incremental SCIP overlay failed: {e}"
+                                        );
+                                    }
+                                    tracing::info!(
+                                        "Incremental SCIP overlay: {} edges upgraded, {} fan-out siblings ruled out",
+                                        stats.upgraded,
+                                        stats.ruled_out
+                                    );
+                                }
+                                Ok(_) => {}
+                                Err(e) => tracing::warn!(
+                                    "Incremental SCIP overlay error (base graph intact): {e}"
+                                ),
+                            }
+                        }
                     }
                     Ok(_) => {}
                     Err(e) => tracing::error!("Incremental reindex failed: {e}"),