feat(server): stat-polling file tracker

docs/clice.toml

@@ -22,6 +22,15 @@ logging_dir = "${workspace}/.clice/logs"
 # Compile commands files or directories to search for compile_commands.json files.
 compile_commands_paths = ["${workspace}/build"]
 
+[tracker]
+# clice polls the filesystem to notice changes the editor never reports:
+# compile_commands.json edits (regenerated by the build system) and files
+# changing on disk behind the server's back (e.g. git checkout).
+# Poll interval for compile_commands.json in seconds; 0 disables it.
+cdb_poll_seconds = 3
+# Interval of the workspace file sweep in seconds; 0 disables it.
+workspace_poll_seconds = 30
+
 # Control the behavior for specific files. Note that Clice matches rules
 # in order. If you want to add your own rules, either delete this rule
 # or insert your rule before it.

src/command/command.cpp

@@ -197,15 +197,13 @@ object_ptr<CompilationInfo> CompilationDatabase::save_compilation_info(llvm::Str
     return save_compilation_info(file, directory, arguments);
 }
 
-std::size_t CompilationDatabase::load(llvm::StringRef path) {
-    entries.clear();
-
+std::optional<std::size_t> CompilationDatabase::load(llvm::StringRef path) {
     simdjson::padded_string json_buf;
     if(auto error = simdjson::padded_string::load(std::string(path)).get(json_buf)) {
         LOG_ERROR("Failed to read compilation database from {}: {}",
                   path,
                   simdjson::error_message(error));
-        return 0;
+        return std::nullopt;
     }
 
     simdjson::ondemand::parser json_parser;
@@ -214,16 +212,24 @@ std::size_t CompilationDatabase::load(llvm::StringRef path) {
         LOG_ERROR("Failed to parse compilation database from {}: {}",
                   path,
                   simdjson::error_message(error));
-        return 0;
+        return std::nullopt;
     }
 
     simdjson::ondemand::array arr;
     if(auto error = doc.get_array().get(arr)) {
         LOG_ERROR("Invalid compilation database format in {}: root element must be an array.",
                   path);
-        return 0;
+        return std::nullopt;
     }
 
+    // Parse into a local vector and only swap it in at the end: a file that
+    // fails to read or parse at the top level leaves the loaded entries
+    // intact, so reload_and_diff() reports no change instead of dropping
+    // every file. A file truncated mid-array is NOT caught here (the
+    // entries before the cut still swap in) — the CDB poll's two-tick
+    // settle debounce is what keeps half-written files from being read.
+    std::vector<CompilationEntry> new_entries;
+
     std::size_t index = 0;
     for(auto element: arr) {
         simdjson::ondemand::object obj;
@@ -291,7 +297,7 @@ std::size_t CompilationDatabase::load(llvm::StringRef path) {
                 auto info = save_compilation_info(file_ref, dir_ref, args);
                 assert(info && "save_compilation_info must succeed with non-empty args");
                 auto path_id = paths.intern(file_ref);
-                entries.push_back({path_id, info});
+                new_entries.push_back({path_id, info});
             }
         } else {
             std::string_view cmd_sv;
@@ -311,18 +317,81 @@ std::size_t CompilationDatabase::load(llvm::StringRef path) {
                 continue;
             }
             auto path_id = paths.intern(file_ref);
-            entries.push_back({path_id, info});
+            new_entries.push_back({path_id, info});
         }
 
         ++index;
     }
 
     // Sort by file path_id for binary search.
-    ranges::sort(entries, {}, &CompilationEntry::file);
+    ranges::sort(new_entries, {}, &CompilationEntry::file);
 
+    entries = std::move(new_entries);
     return entries.size();
 }
 
+llvm::DenseMap<std::uint32_t, llvm::SmallVector<std::string, 1>>
+    CompilationDatabase::command_hash_snapshot() const {
+    llvm::DenseMap<std::uint32_t, llvm::SmallVector<std::string, 1>> snapshot;
+
+    for(auto& entry: entries) {
+        // The file-independent argv (driver + canonical flags + per-file
+        // -I/-D patch). The source file stays out: entries under one path_id
+        // share it, so it carries no signal for this comparison.
+        std::vector<std::string> args;
+        args.reserve(entry.info->canonical->arguments.size() + entry.info->patch.size());
+        for(const char* arg: entry.info->canonical->arguments) {
+            args.emplace_back(arg);
+        }
+        for(const char* arg: entry.info->patch) {
+            args.emplace_back(arg);
+        }
+        snapshot[entry.file].emplace_back(canonical_command_hash(args, entry.info->directory));
+    }
+
+    // A file's entries have no inherent order, so sort each list to make the
+    // comparison in reload_and_diff() order-independent.
+    for(auto& bucket: snapshot) {
+        ranges::sort(bucket.second);
+    }
+
+    return snapshot;
+}
+
+std::optional<CDBDiff> CompilationDatabase::reload_and_diff(llvm::StringRef path) {
+    auto before = command_hash_snapshot();
+    if(!load(path)) {
+        // Unreadable or unparsable (e.g. still locked by the generator):
+        // the old entries were kept, and the caller must not treat this as
+        // "no change" — it has to retry.
+        return std::nullopt;
+    }
+    auto after = command_hash_snapshot();
+
+    CDBDiff diff;
+
+    for(auto& bucket: after) {
+        auto it = before.find(bucket.first);
+        if(it == before.end()) {
+            diff.added.push_back(bucket.first);
+        } else if(it->second != bucket.second) {
+            diff.changed.push_back(bucket.first);
+        }
+    }
+
+    for(auto& bucket: before) {
+        if(after.find(bucket.first) == after.end()) {
+            diff.removed.push_back(bucket.first);
+        }
+    }
+
+    ranges::sort(diff.added);
+    ranges::sort(diff.removed);
+    ranges::sort(diff.changed);
+
+    return diff;
+}
+
 CompileCommand CompilationDatabase::build_command(std::uint32_t path_id,
                                                   object_ptr<CompilationInfo> info,
                                                   const CommandOptions& options) {

src/command/command.h

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -162,6 +163,23 @@ struct DenseMapInfo<clice::CompilationInfo> {
 
 namespace clice {
 
+/// Per-file delta of a compilation database reload. Path ids are this
+/// database's own pool ids (stable across reloads).
+struct CDBDiff {
+    /// Files present only after the reload (gained their first entry).
+    llvm::SmallVector<std::uint32_t> added;
+
+    /// Files present only before the reload (lost all their entries).
+    llvm::SmallVector<std::uint32_t> removed;
+
+    /// Files present on both sides whose set of command hashes differs.
+    llvm::SmallVector<std::uint32_t> changed;
+
+    bool empty() const {
+        return added.empty() && removed.empty() && changed.empty();
+    }
+};
+
 class CompilationDatabase {
 public:
     CompilationDatabase();
@@ -174,9 +192,33 @@ class CompilationDatabase {
 
 public:
     /// Load (or reload) the compilation database from the given file.
-    /// Full reload: old entries are replaced, but string pool and canonical
-    /// commands survive. Returns the number of entries loaded.
-    std::size_t load(llvm::StringRef path);
+    /// On success old entries are replaced, but the string pool and canonical
+    /// commands survive (path ids stay stable across reloads).
+    ///
+    /// Parsing is atomic at the top level: if the file cannot be read, is
+    /// not valid JSON, or has a root that is not an array, the previously
+    /// loaded entries are kept and nullopt is returned. Individual
+    /// malformed entries are still skipped as before — which means a file
+    /// truncated mid-array loads as a partial set; the poll-side settle
+    /// debounce is what guards against reading half-written files. On
+    /// success returns the number of entries loaded.
+    std::optional<std::size_t> load(llvm::StringRef path);
+
+    /// Reload the database from `path` and report the per-file delta against
+    /// the previously loaded entries. Path ids in the result are this
+    /// database's own pool ids (stable across reloads).
+    ///
+    /// Entry identity is the canonical command hash (Frontend profile), the
+    /// same identity the rest of the system uses to pin a CDB entry (e.g.
+    /// clice/switchContext). A change confined to codegen-only flags (-g,
+    /// -fPIC, -flto, ...) is therefore not reported as `changed` — this is
+    /// deliberate. (Optimization level -O* is semantic, not codegen-only: it
+    /// defines __OPTIMIZE__, so changing it does count.)
+    ///
+    /// If `path` cannot be read or does not hold a JSON array, load() keeps
+    /// the old entries and nullopt is returned — the caller must retry
+    /// rather than treat the failure as "no change".
+    std::optional<CDBDiff> reload_and_diff(llvm::StringRef path);
 
     /// Lookup the compile commands for a file. A file may have multiple
     /// compilation commands (e.g. different build configurations); all are returned.
@@ -253,6 +295,11 @@ class CompilationDatabase {
                                                       llvm::StringRef directory,
                                                       llvm::StringRef command);
 
+    /// Map each file's path_id to the sorted canonical command hashes of its
+    /// entries (a file may own several entries with different flags). Used by
+    /// reload_and_diff() to compare the database before and after a reload.
+    llvm::DenseMap<std::uint32_t, llvm::SmallVector<std::string, 1>> command_hash_snapshot() const;
+
     /// The memory pool which holds all elements of compilation database.
     /// Heap-allocated so its address is stable across moves.
     std::unique_ptr<llvm::BumpPtrAllocator> allocator = std::make_unique<llvm::BumpPtrAllocator>();

src/server/context/context_resolver.cpp

@@ -678,6 +678,18 @@ std::optional<HeaderContext> ContextResolver::resolve_header_context(std::uint32
                          std::move(deps)};
 }
 
+bool ContextResolver::entry_has_hash(llvm::StringRef entry_path, llvm::StringRef hash) const {
+    std::vector<std::string> rule_append, rule_remove;
+    workspace.config.match_rules(entry_path, rule_append, rule_remove);
+    for(auto& cmd:
+        workspace.cdb.lookup(entry_path, {.remove = rule_remove, .append = rule_append})) {
+        if(canonical_command_hash(cmd.to_string_argv(), cmd.resolved.directory) == hash) {
+            return true;
+        }
+    }
+    return false;
+}
+
 void ContextResolver::validate_saved_context(Session& session) {
     auto path_id = session.path_id;
     auto path = workspace.path_pool.resolve(path_id);
@@ -689,17 +701,6 @@ void ContextResolver::validate_saved_context(Session& session) {
     if(auto it = saved_contexts.find(path_id); it != saved_contexts.end()) {
         auto& ws = workspace;
         auto& saved = it->second;
-        auto entry_has_hash = [&ws](llvm::StringRef entry_path, llvm::StringRef hash) {
-            std::vector<std::string> rule_append, rule_remove;
-            ws.config.match_rules(entry_path, rule_append, rule_remove);
-            for(auto& cmd:
-                ws.cdb.lookup(entry_path, {.remove = rule_remove, .append = rule_append})) {
-                if(canonical_command_hash(cmd.to_string_argv(), cmd.resolved.directory) == hash) {
-                    return true;
-                }
-            }
-            return false;
-        };
 
         bool valid = false;
         if(saved.host_path_id != no_path_id) {
@@ -721,22 +722,34 @@ bool ContextResolver::drop_orphaned_choices(SessionStore& sessions) {
     bool dropped_saved = false;
     for(auto& [session_id, session]: sessions.sessions) {
         auto it = saved_contexts.find(session_id);
-        if(it == saved_contexts.end() || it->second.host_path_id == no_path_id) {
+        if(it == saved_contexts.end()) {
             continue;
         }
-        auto host_id = it->second.host_path_id;
-        auto& occurrence = it->second.occurrence;
-        bool orphaned = workspace.dep_graph.find_include_chain(host_id, session_id).empty();
-        // A pinned occurrence can vanish while other inclusions of the
-        // header survive (the chain stays non-empty) — recount it.
-        if(!orphaned && occurrence.has_value()) {
-            auto count = workspace.count_occurrences(host_id, session_id);
-            orphaned = count > 0 && *occurrence >= count;
+        auto& saved = it->second;
+        auto host_id = saved.host_path_id;
+        auto& occurrence = saved.occurrence;
+        bool orphaned = false;
+        if(host_id != no_path_id) {
+            orphaned = workspace.dep_graph.find_include_chain(host_id, session_id).empty();
+            // A pinned occurrence can vanish while other inclusions of the
+            // header survive (the chain stays non-empty) — recount it.
+            if(!orphaned && occurrence.has_value()) {
+                auto count = workspace.count_occurrences(host_id, session_id);
+                orphaned = count > 0 && *occurrence >= count;
+            }
+            // The pinned host command itself can vanish (a CDB reload
+            // changed the entry's flags): same validation didOpen applies.
+            if(!orphaned && !saved.command_hash.empty()) {
+                orphaned =
+                    !entry_has_hash(workspace.path_pool.resolve(host_id), saved.command_hash);
+            }
+        } else if(!saved.command_hash.empty()) {
+            // Own-entry pin: the pinned command must still exist in the CDB.
+            orphaned = !entry_has_hash(workspace.path_pool.resolve(session_id), saved.command_hash);
         }
         if(orphaned) {
-            LOG_INFO("Dropping orphaned context choice for {}: host {} no longer includes it",
-                     workspace.path_pool.resolve(session_id),
-                     workspace.path_pool.resolve(host_id));
+            LOG_INFO("Dropping orphaned context choice for {}: its basis no longer exists",
+                     workspace.path_pool.resolve(session_id));
             drop_header_context(session_id);
             session->pch_ref.reset();
             session->ast_dirty = true;

src/server/context/context_resolver.h

@@ -246,6 +246,11 @@ class ContextResolver {
                                                         Session* session,
                                                         bool synthesize);
 
+    /// Whether the CDB still holds an entry for `entry_path` whose canonical
+    /// command hash equals `hash` — the validity test for a pinned context
+    /// choice, shared by didOpen validation and the runtime orphan pass.
+    bool entry_has_hash(llvm::StringRef entry_path, llvm::StringRef hash) const;
+
     /// The file's context choice, or nullptr. Gated on an open session:
     /// user choices steer editor-facing compiles, never background indexing
     /// (which passes no session).

src/server/protocol/extension.h

@@ -81,4 +81,19 @@ struct InactiveRegionsParams {
     std::vector<kota::ipc::protocol::Range> regions;
 };
 
+/// clice/internal/poll — TEST-ONLY, not a stable API. Synchronously runs
+/// one file-tracker tick (stat → diff → events → dispatch → effects) and
+/// responds only once the effects are applied, so integration tests can
+/// disable the polling loops and get "change disk → poll → assert"
+/// determinism with zero sleeps. Absent from capabilities and user docs.
+struct PollParams {
+    /// Which loop to tick: "cdb" or "workspace".
+    std::string loop;
+};
+
+struct PollResult {
+    /// Number of file events the tick produced and dispatched.
+    std::uint32_t events = 0;
+};
+
 }  // namespace clice::ext

src/server/service/lsp_client.cpp

@@ -14,6 +14,7 @@
 #include "server/protocol/extension.h"
 #include "server/protocol/serialize.h"
 #include "server/service/master_server.h"
+#include "server/workspace/file_tracker.h"
 #include "support/anomaly.h"
 #include "support/filesystem.h"
 #include "support/logging.h"
@@ -488,6 +489,37 @@ void LSPClient::register_extensions() {
             this->server.dispatch(FileEvent::context_changed(path_id));
             co_return to_raw(result);
         });
+
+    // ── Test hook ───────────────────────────────────────────────────
+
+    // Runs one file-tracker tick synchronously (see ext::PollParams).
+    // Test-only and not a stable API; the CDB tick runs with force=true,
+    // so the polling loop's two-tick settling debounce does not apply here.
+    peer.on_request(
+        "clice/internal/poll",
+        [this](RequestContext& ctx, const ext::PollParams& params) -> RawResult {
+            auto& srv = this->server;
+            if(!srv.tracker) {
+                co_return kota::outcome_error(kota::ipc::Error{protocol::ErrorCode::InvalidRequest,
+                                                               "No workspace is loaded"});
+            }
+
+            llvm::SmallVector<FileEvent> events;
+            if(params.loop == "cdb") {
+                events = srv.tracker->tick_cdb(/*force=*/true);
+            } else if(params.loop == "workspace") {
+                events = co_await srv.tracker->tick_workspace();
+            } else {
+                co_return kota::outcome_error(
+                    kota::ipc::Error{protocol::ErrorCode::InvalidParams,
+                                     R"(loop must be "cdb" or "workspace")"});
+            }
+
+            if(!events.empty()) {
+                srv.dispatch(events);
+            }
+            co_return to_raw(ext::PollResult{static_cast<std::uint32_t>(events.size())});
+        });
 }
 
 /// Publish clice.toml load problems as diagnostics, each on its own file's