fix(server): layered freshness policy for index queries

src/server/compiler/compiler.cpp

@@ -801,6 +801,14 @@ kota::task<> Compiler::run_compile(std::shared_ptr<Session> session) {
             auto tu_index = index::TUIndex::from(result.value().tu_index_data.data());
             session->file_index = std::move(tu_index.main_file_index);
             session->symbols = std::move(tu_index.symbols);
+        } else {
+            // The AST and the file index settle together — that pairing is
+            // what lets navigation trust the index after ensure_compiled. A
+            // compile that produced no index data (fatal error, no AST) must
+            // therefore drop the previous buffer's index rather than leave
+            // it posing as current: an honest gap over yesterday's offsets.
+            session->file_index.reset();
+            session->symbols.reset();
         }
 
         auto version = session->version;

src/server/compiler/indexer.cpp

@@ -288,9 +288,12 @@ void Indexer::load() {
                     workspace.merged_indices[path_id] = std::move(shard);
                     expected_keys.insert(key);
                 } else {
+                    // No shard survives, so there is nothing stale to keep
+                    // serving; ContentChanged states the truth ("the index
+                    // does not describe this file") without effect.
                     LOG_INFO("Discarding unreadable shard for {}",
                              workspace.path_pool.resolve(path_id));
-                    enqueue(path_id);
+                    enqueue(path_id, ReindexReason::ContentChanged);
                 }
             }
         } else {
@@ -332,10 +335,36 @@ bool Indexer::need_update(llvm::StringRef file_path) {
     return merged_it->second.need_update();
 }
 
-void Indexer::enqueue(std::uint32_t server_path_id) {
-    // Already queued and not yet consumed — a second entry would only be
-    // skipped by need_update later; drop it here.
-    if(!pending_ids.insert(server_path_id).second)
+void Indexer::enqueue(std::uint32_t server_path_id, ReindexReason reason) {
+    // A fresh slot means any prior slot was already consumed (or none
+    // existed); a queued-and-unconsumed slot makes this call a duplicate.
+    bool fresh_slot = pending_ids.insert(server_path_id).second;
+
+    // Record (or refresh) why the file is pending. Within one queued slot
+    // ContentChanged is absorbing: a deps-only cascade cannot downgrade a
+    // file whose own content already changed. Across slots it is not: a
+    // deps-only requeue after the previous slot was consumed is new debt of
+    // its own kind — the in-flight (or finished) pass already covers the
+    // earlier content change, and keeping ContentChanged would suppress the
+    // file's rows past that pass. The fresh ticket invalidates the clear of
+    // any index task already in flight for this file.
+    ++reindex_ticket;
+    auto [it, inserted] =
+        reindex_reasons.try_emplace(server_path_id,
+                                    reason,
+                                    reindex_ticket,
+                                    reason == ReindexReason::ContentChanged ? reindex_ticket : 0);
+    if(!inserted) {
+        if(reason == ReindexReason::ContentChanged) {
+            it->second.reason = ReindexReason::ContentChanged;
+            it->second.content_ticket = reindex_ticket;
+        } else if(fresh_slot) {
+            it->second.reason = ReindexReason::DepsOnly;
+        }
+        it->second.ticket = reindex_ticket;
+    }
+
+    if(!fresh_slot)
         return;
     index_queue.push_back(server_path_id);
 }
@@ -379,15 +408,24 @@ void Indexer::schedule() {
 }
 
 kota::task<> Indexer::index_one(std::uint32_t server_path_id,
+                                std::uint64_t ticket,
                                 std::size_t index,
                                 std::size_t total) {
     auto file_path = std::string(workspace.path_pool.resolve(server_path_id));
 
     if(sessions.find(server_path_id) != nullptr)
         co_return;
 
-    if(!need_update(file_path))
+    // The engine's own observation is authoritative for content changes:
+    // it saw the event. The dep-hash check below cannot be trusted to see
+    // a file's own edit (it validates the recorded dependencies), so only
+    // deps-only slots — where it exists to deduplicate cascade storms —
+    // may take the shortcut.
+    if(auto it = reindex_reasons.find(server_path_id);
+       (it == reindex_reasons.end() || it->second.reason != ReindexReason::ContentChanged) &&
+       !need_update(file_path)) {
         co_return;
+    }
 
     // For module interface units, compile their PCM (and transitive deps)
     // first so the stateless worker has the artifacts it needs.
@@ -412,6 +450,19 @@ kota::task<> Indexer::index_one(std::uint32_t server_path_id,
     auto result = co_await pool.send_stateless(params);
     if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) {
         auto index_ms = timer.ms();
+        // Merge guard: a newer content-level invalidation during this build
+        // (or a removal clearing the entry) means this result describes text
+        // that no longer exists — e.g. a compile-command change whose
+        // erase+re-enqueue must not be undone by an in-flight merge of the
+        // old-command rows. Drop the merge; the follow-up slot redoes it.
+        // A deps-only requeue is deliberately NOT superseding: the in-flight
+        // rows are positionally right, and suppressing them would trade a
+        // tolerated semantic drift for a coverage hole.
+        if(auto it = reindex_reasons.find(server_path_id);
+           it == reindex_reasons.end() || it->second.content_ticket > ticket) {
+            LOG_INFO("Discarding superseded index result for {}", file_path);
+            co_return;
+        }
         ScopedTimer merge_timer;
         merge(result.value().tu_index_data.data(), result.value().tu_index_data.size());
         LOG_PERF("index",
@@ -435,6 +486,29 @@ kota::task<> Indexer::index_one(std::uint32_t server_path_id,
     }
 }
 
+kota::task<> Indexer::run_index_task(std::uint32_t server_path_id,
+                                     std::uint64_t ticket,
+                                     std::size_t index,
+                                     std::size_t total,
+                                     std::size_t& completed) {
+    co_await index_one(server_path_id, ticket, index, total);
+    // The pending window ends with the index attempt, success or not. On
+    // failure the last-known rows resume serving — deliberately: keeping
+    // the gate would hide a file that fails to index (broken compile,
+    // missing command) from every cross-file query with no recovery path,
+    // since only a future event re-enqueues it. Any such event re-judges
+    // staleness by content hash. A re-enqueue during the flight bumped
+    // the ticket: that newer pending state must survive this clear.
+    if(auto it = reindex_reasons.find(server_path_id);
+       it != reindex_reasons.end() && it->second.ticket == ticket) {
+        reindex_reasons.erase(it);
+    }
+    ++completed;
+    progress_data.stage = Progress::Stage::Report;
+    progress_data.completed = completed;
+    on_progress_changed.emit();
+}
+
 kota::task<> Indexer::run_background_indexing() {
     if(index_idle_timer) {
         co_await index_idle_timer->wait();
@@ -476,20 +550,30 @@ kota::task<> Indexer::run_background_indexing() {
 
         auto server_path_id = index_queue[index_queue_pos++];
         pending_ids.erase(server_path_id);
-        auto file_path = std::string(workspace.path_pool.resolve(server_path_id));
-        if(sessions.find(server_path_id) != nullptr || !need_update(file_path)) {
-            ++completed;
+        // No open-session or hash-freshness shortcut here: index_one is the
+        // single decision point for skipping (it knows the pending reason;
+        // a hash check alone cannot see a file's own edit), and the
+        // completion clear in run_index_task retires the pending state with
+        // the ticket honored. A second, reason-blind copy of these checks
+        // here is exactly what once erased ContentChanged state early and
+        // let a stale shard keep serving.
+
+        // A queued slot with no pending entry was cleared mid-batch: the
+        // file was removed from disk after being enqueued (clear_pending),
+        // so there is nothing to index — skip the slot. Every other slot
+        // has an entry, because enqueue writes it before the queue push.
+        auto pending_it = reindex_reasons.find(server_path_id);
+        if(pending_it == reindex_reasons.end()) {
             continue;
         }
 
         ++dispatched;
-        workers.spawn([&, server_path_id, n = dispatched]() -> kota::task<> {
-            co_await index_one(server_path_id, n, total);
-            ++completed;
-            progress_data.stage = Progress::Stage::Report;
-            progress_data.completed = completed;
-            on_progress_changed.emit();
-        }());
+        auto ticket = pending_it->second.ticket;
+        // A member coroutine, not an immediately-invoked capturing lambda:
+        // a lambda's captures live in the lambda object, which dies at the
+        // end of this statement — anything read after the first suspension
+        // would dangle. Coroutine parameters are copied into the frame.
+        workers.spawn(run_index_task(server_path_id, ticket, dispatched, total, completed));
     }
 
     LOG_DEBUG("Background indexing: all {} tasks spawned, waiting for completion", dispatched);
@@ -509,6 +593,7 @@ kota::task<> Indexer::run_background_indexing() {
     // the next scheduled round.
     if(index_queue_pos >= index_queue.size()) {
         assert(pending_ids.empty() && "drained queue must have no pending ids");
+        assert(reindex_reasons.empty() && "drained queue must have no pending reasons");
         index_queue.clear();
         index_queue_pos = 0;
     }
@@ -520,6 +605,14 @@ kota::task<> Indexer::run_background_indexing() {
              total,
              timer.ms());
     co_await save();
+
+    // Files enqueued while the round was joining its workers saw their
+    // schedule() no-op against indexing_active; without this kick they
+    // would wait for the next external event — and a content-changed
+    // pending file's rows stay skipped for that whole wait.
+    if(index_queue_pos < index_queue.size()) {
+        schedule();
+    }
 }
 
 }  // namespace clice

src/server/compiler/indexer.h

@@ -3,12 +3,14 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <vector>
 
 #include "server/state/workspace.h"
 #include "support/signal.h"
 
 #include "kota/async/async.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringRef.h"
 
@@ -18,6 +20,22 @@ class ContextResolver;
 class WorkerPool;
 struct SessionStore;
 
+/// Why a file awaits re-indexing. The invalidation engine knows the cause
+/// at enqueue time, so queries can decide in O(1) whether a pending file's
+/// existing index rows are still trustworthy (see IndexQuery's freshness
+/// contract).
+enum class ReindexReason : std::uint8_t {
+    /// Enqueued by a dependency cascade (or a bulk sweep of unknown
+    /// staleness): the file's own content is not known to have changed, so
+    /// its index rows are positionally intact — at worst semantically
+    /// behind — and keep serving until the reindex lands.
+    DepsOnly,
+    /// The file's own content changed: its index rows describe text that
+    /// no longer exists, so queries skip this file's contribution until
+    /// the reindex lands.
+    ContentChanged,
+};
+
 /// Background indexing scheduler.
 ///
 /// Indexer owns the indexing queue and drives disk files through
@@ -71,8 +89,33 @@ class Indexer {
         return ScopedPause{*this};
     }
 
-    /// Add a file to the background indexing queue.
-    void enqueue(std::uint32_t server_path_id);
+    /// Add a file to the background indexing queue. A file enqueued twice
+    /// keeps a single queue entry; its reason is upgraded to ContentChanged
+    /// if either enqueue says so (a file both cascaded onto and edited is
+    /// as stale as the edit makes it).
+    void enqueue(std::uint32_t server_path_id, ReindexReason reason);
+
+    /// Why the file awaits re-indexing (queued or currently being indexed),
+    /// or nullopt when its index is not pending an update. O(1), no I/O —
+    /// the query path calls this per candidate file.
+    std::optional<ReindexReason> pending_reason(std::uint32_t server_path_id) const {
+        auto it = reindex_reasons.find(server_path_id);
+        if(it == reindex_reasons.end()) {
+            return std::nullopt;
+        }
+        return it->second.reason;
+    }
+
+    /// Forget a file's pending-reindex state (reason and queue membership):
+    /// used when the file is removed from disk — nothing is left to reindex,
+    /// and a lingering ContentChanged reason would suppress its deliberately
+    /// still-serving shard forever. A queue slot already consumed stays
+    /// consumed; one not yet consumed is skipped at dispatch time (the
+    /// consume loop treats a missing pending entry as a cleared slot).
+    void clear_pending(std::uint32_t server_path_id) {
+        reindex_reasons.erase(server_path_id);
+        pending_ids.erase(server_path_id);
+    }
 
     /// Schedule background indexing (respects idle timeout and dedup).
     void schedule();
@@ -147,6 +190,46 @@ class Indexer {
     std::vector<std::uint32_t> index_queue;
     llvm::DenseSet<std::uint32_t> pending_ids;
     std::size_t index_queue_pos = 0;
+
+    /// The pending-reindex state machine, per file. This block is the
+    /// authoritative description; every rule below exists because its
+    /// absence was a concrete bug.
+    ///
+    /// States: absent → queued (slot in index_queue + entry here) →
+    /// in-flight (slot consumed, entry alive) → absent again.
+    ///
+    /// Invariants:
+    /// 1. index_one is the ONLY place that decides to skip work (open
+    ///    session, or hash-fresh shard for deps-only slots). Duplicating
+    ///    those checks elsewhere reintroduces reason-blind skips.
+    /// 2. need_update() may shortcut deps-only slots ONLY: the engine
+    ///    observed content changes itself, and the dep-hash check cannot
+    ///    see a file's own edit.
+    /// 3. The merge lands iff the entry is alive and no ContentChanged
+    ///    enqueue happened after launch (content_ticket <= launch ticket).
+    ///    Deps-only requeues do not discard an in-flight pass.
+    /// 4. Completion erases the entry iff ticket == launch ticket: a
+    ///    requeue during the flight must survive the older task's clear.
+    /// 5. Within a queued slot, ContentChanged absorbs; a fresh slot after
+    ///    consumption carries its own reason (the consumed pass owns the
+    ///    earlier debt).
+    /// 6. clear_pending (file removal) drops entry and queue membership;
+    ///    the orphaned slot is skipped at dispatch.
+    /// 7. Queries suppress a file's contributions iff its entry's reason
+    ///    is ContentChanged (see pending_reason).
+    struct PendingReindex {
+        ReindexReason reason;
+        std::uint64_t ticket;
+        /// Ticket of the newest ContentChanged enqueue. The merge guard
+        /// compares against this, not `ticket`: a deps-only requeue during a
+        /// flight bumps `ticket` (to survive the completion clear) but must
+        /// not discard an in-flight content pass — its rows are positionally
+        /// right and the follow-up slot redoes the semantic drift anyway.
+        std::uint64_t content_ticket;
+    };
+
+    llvm::DenseMap<std::uint32_t, PendingReindex> reindex_reasons;
+    std::uint64_t reindex_ticket = 0;
     bool indexing_active = false;
     bool indexing_scheduled = false;
     std::shared_ptr<kota::timer> index_idle_timer;
@@ -159,7 +242,20 @@ class Indexer {
     Progress progress_data;
 
     kota::task<> run_background_indexing();
-    kota::task<> index_one(std::uint32_t server_path_id, std::size_t index, std::size_t total);
+    kota::task<> index_one(std::uint32_t server_path_id,
+                           std::uint64_t ticket,
+                           std::size_t index,
+                           std::size_t total);
+
+    /// One dispatched unit of a background round: index the file, then end
+    /// its pending window (ticket-guarded) and report progress. `completed`
+    /// refers into run_background_indexing's frame, which outlives every
+    /// spawned task (it joins them before returning).
+    kota::task<> run_index_task(std::uint32_t server_path_id,
+                                std::uint64_t ticket,
+                                std::size_t index,
+                                std::size_t total,
+                                std::size_t& completed);
 };
 
 }  // namespace clice