diff --git a/docs/clice.toml b/docs/clice.toml index 2c254400a..14150b65b 100644 --- a/docs/clice.toml +++ b/docs/clice.toml @@ -22,6 +22,15 @@ logging_dir = "${workspace}/.clice/logs" # Compile commands files or directories to search for compile_commands.json files. compile_commands_paths = ["${workspace}/build"] +[tracker] +# clice polls the filesystem to notice changes the editor never reports: +# compile_commands.json edits (regenerated by the build system) and files +# changing on disk behind the server's back (e.g. git checkout). +# Poll interval for compile_commands.json in seconds; 0 disables it. +cdb_poll_seconds = 3 +# Interval of the workspace file sweep in seconds; 0 disables it. +workspace_poll_seconds = 30 + # Control the behavior for specific files. Note that Clice matches rules # in order. If you want to add your own rules, either delete this rule # or insert your rule before it. diff --git a/src/command/command.cpp b/src/command/command.cpp index ebf07bea0..6c928f103 100644 --- a/src/command/command.cpp +++ b/src/command/command.cpp @@ -197,15 +197,13 @@ object_ptr CompilationDatabase::save_compilation_info(llvm::Str return save_compilation_info(file, directory, arguments); } -std::size_t CompilationDatabase::load(llvm::StringRef path) { - entries.clear(); - +std::optional CompilationDatabase::load(llvm::StringRef path) { simdjson::padded_string json_buf; if(auto error = simdjson::padded_string::load(std::string(path)).get(json_buf)) { LOG_ERROR("Failed to read compilation database from {}: {}", path, simdjson::error_message(error)); - return 0; + return std::nullopt; } simdjson::ondemand::parser json_parser; @@ -214,16 +212,24 @@ std::size_t CompilationDatabase::load(llvm::StringRef path) { LOG_ERROR("Failed to parse compilation database from {}: {}", path, simdjson::error_message(error)); - return 0; + return std::nullopt; } simdjson::ondemand::array arr; if(auto error = doc.get_array().get(arr)) { LOG_ERROR("Invalid compilation database format in {}: root element must be an array.", path); - return 0; + return std::nullopt; } + // Parse into a local vector and only swap it in at the end: a file that + // fails to read or parse at the top level leaves the loaded entries + // intact, so reload_and_diff() reports no change instead of dropping + // every file. A file truncated mid-array is NOT caught here (the + // entries before the cut still swap in) — the CDB poll's two-tick + // settle debounce is what keeps half-written files from being read. + std::vector new_entries; + std::size_t index = 0; for(auto element: arr) { simdjson::ondemand::object obj; @@ -291,7 +297,7 @@ std::size_t CompilationDatabase::load(llvm::StringRef path) { auto info = save_compilation_info(file_ref, dir_ref, args); assert(info && "save_compilation_info must succeed with non-empty args"); auto path_id = paths.intern(file_ref); - entries.push_back({path_id, info}); + new_entries.push_back({path_id, info}); } } else { std::string_view cmd_sv; @@ -311,18 +317,81 @@ std::size_t CompilationDatabase::load(llvm::StringRef path) { continue; } auto path_id = paths.intern(file_ref); - entries.push_back({path_id, info}); + new_entries.push_back({path_id, info}); } ++index; } // Sort by file path_id for binary search. - ranges::sort(entries, {}, &CompilationEntry::file); + ranges::sort(new_entries, {}, &CompilationEntry::file); + entries = std::move(new_entries); return entries.size(); } +llvm::DenseMap> + CompilationDatabase::command_hash_snapshot() const { + llvm::DenseMap> snapshot; + + for(auto& entry: entries) { + // The file-independent argv (driver + canonical flags + per-file + // -I/-D patch). The source file stays out: entries under one path_id + // share it, so it carries no signal for this comparison. + std::vector args; + args.reserve(entry.info->canonical->arguments.size() + entry.info->patch.size()); + for(const char* arg: entry.info->canonical->arguments) { + args.emplace_back(arg); + } + for(const char* arg: entry.info->patch) { + args.emplace_back(arg); + } + snapshot[entry.file].emplace_back(canonical_command_hash(args, entry.info->directory)); + } + + // A file's entries have no inherent order, so sort each list to make the + // comparison in reload_and_diff() order-independent. + for(auto& bucket: snapshot) { + ranges::sort(bucket.second); + } + + return snapshot; +} + +std::optional CompilationDatabase::reload_and_diff(llvm::StringRef path) { + auto before = command_hash_snapshot(); + if(!load(path)) { + // Unreadable or unparsable (e.g. still locked by the generator): + // the old entries were kept, and the caller must not treat this as + // "no change" — it has to retry. + return std::nullopt; + } + auto after = command_hash_snapshot(); + + CDBDiff diff; + + for(auto& bucket: after) { + auto it = before.find(bucket.first); + if(it == before.end()) { + diff.added.push_back(bucket.first); + } else if(it->second != bucket.second) { + diff.changed.push_back(bucket.first); + } + } + + for(auto& bucket: before) { + if(after.find(bucket.first) == after.end()) { + diff.removed.push_back(bucket.first); + } + } + + ranges::sort(diff.added); + ranges::sort(diff.removed); + ranges::sort(diff.changed); + + return diff; +} + CompileCommand CompilationDatabase::build_command(std::uint32_t path_id, object_ptr info, const CommandOptions& options) { diff --git a/src/command/command.h b/src/command/command.h index 8bfc14b89..52c1d95f9 100644 --- a/src/command/command.h +++ b/src/command/command.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -162,6 +163,23 @@ struct DenseMapInfo { namespace clice { +/// Per-file delta of a compilation database reload. Path ids are this +/// database's own pool ids (stable across reloads). +struct CDBDiff { + /// Files present only after the reload (gained their first entry). + llvm::SmallVector added; + + /// Files present only before the reload (lost all their entries). + llvm::SmallVector removed; + + /// Files present on both sides whose set of command hashes differs. + llvm::SmallVector changed; + + bool empty() const { + return added.empty() && removed.empty() && changed.empty(); + } +}; + class CompilationDatabase { public: CompilationDatabase(); @@ -174,9 +192,33 @@ class CompilationDatabase { public: /// Load (or reload) the compilation database from the given file. - /// Full reload: old entries are replaced, but string pool and canonical - /// commands survive. Returns the number of entries loaded. - std::size_t load(llvm::StringRef path); + /// On success old entries are replaced, but the string pool and canonical + /// commands survive (path ids stay stable across reloads). + /// + /// Parsing is atomic at the top level: if the file cannot be read, is + /// not valid JSON, or has a root that is not an array, the previously + /// loaded entries are kept and nullopt is returned. Individual + /// malformed entries are still skipped as before — which means a file + /// truncated mid-array loads as a partial set; the poll-side settle + /// debounce is what guards against reading half-written files. On + /// success returns the number of entries loaded. + std::optional load(llvm::StringRef path); + + /// Reload the database from `path` and report the per-file delta against + /// the previously loaded entries. Path ids in the result are this + /// database's own pool ids (stable across reloads). + /// + /// Entry identity is the canonical command hash (Frontend profile), the + /// same identity the rest of the system uses to pin a CDB entry (e.g. + /// clice/switchContext). A change confined to codegen-only flags (-g, + /// -fPIC, -flto, ...) is therefore not reported as `changed` — this is + /// deliberate. (Optimization level -O* is semantic, not codegen-only: it + /// defines __OPTIMIZE__, so changing it does count.) + /// + /// If `path` cannot be read or does not hold a JSON array, load() keeps + /// the old entries and nullopt is returned — the caller must retry + /// rather than treat the failure as "no change". + std::optional reload_and_diff(llvm::StringRef path); /// Lookup the compile commands for a file. A file may have multiple /// compilation commands (e.g. different build configurations); all are returned. @@ -253,6 +295,11 @@ class CompilationDatabase { llvm::StringRef directory, llvm::StringRef command); + /// Map each file's path_id to the sorted canonical command hashes of its + /// entries (a file may own several entries with different flags). Used by + /// reload_and_diff() to compare the database before and after a reload. + llvm::DenseMap> command_hash_snapshot() const; + /// The memory pool which holds all elements of compilation database. /// Heap-allocated so its address is stable across moves. std::unique_ptr allocator = std::make_unique(); diff --git a/src/server/context/context_resolver.cpp b/src/server/context/context_resolver.cpp index cee496125..4f1da5e5b 100644 --- a/src/server/context/context_resolver.cpp +++ b/src/server/context/context_resolver.cpp @@ -678,6 +678,18 @@ std::optional ContextResolver::resolve_header_context(std::uint32 std::move(deps)}; } +bool ContextResolver::entry_has_hash(llvm::StringRef entry_path, llvm::StringRef hash) const { + std::vector rule_append, rule_remove; + workspace.config.match_rules(entry_path, rule_append, rule_remove); + for(auto& cmd: + workspace.cdb.lookup(entry_path, {.remove = rule_remove, .append = rule_append})) { + if(canonical_command_hash(cmd.to_string_argv(), cmd.resolved.directory) == hash) { + return true; + } + } + return false; +} + void ContextResolver::validate_saved_context(Session& session) { auto path_id = session.path_id; auto path = workspace.path_pool.resolve(path_id); @@ -689,17 +701,6 @@ void ContextResolver::validate_saved_context(Session& session) { if(auto it = saved_contexts.find(path_id); it != saved_contexts.end()) { auto& ws = workspace; auto& saved = it->second; - auto entry_has_hash = [&ws](llvm::StringRef entry_path, llvm::StringRef hash) { - std::vector rule_append, rule_remove; - ws.config.match_rules(entry_path, rule_append, rule_remove); - for(auto& cmd: - ws.cdb.lookup(entry_path, {.remove = rule_remove, .append = rule_append})) { - if(canonical_command_hash(cmd.to_string_argv(), cmd.resolved.directory) == hash) { - return true; - } - } - return false; - }; bool valid = false; if(saved.host_path_id != no_path_id) { @@ -721,22 +722,34 @@ bool ContextResolver::drop_orphaned_choices(SessionStore& sessions) { bool dropped_saved = false; for(auto& [session_id, session]: sessions.sessions) { auto it = saved_contexts.find(session_id); - if(it == saved_contexts.end() || it->second.host_path_id == no_path_id) { + if(it == saved_contexts.end()) { continue; } - auto host_id = it->second.host_path_id; - auto& occurrence = it->second.occurrence; - bool orphaned = workspace.dep_graph.find_include_chain(host_id, session_id).empty(); - // A pinned occurrence can vanish while other inclusions of the - // header survive (the chain stays non-empty) — recount it. - if(!orphaned && occurrence.has_value()) { - auto count = workspace.count_occurrences(host_id, session_id); - orphaned = count > 0 && *occurrence >= count; + auto& saved = it->second; + auto host_id = saved.host_path_id; + auto& occurrence = saved.occurrence; + bool orphaned = false; + if(host_id != no_path_id) { + orphaned = workspace.dep_graph.find_include_chain(host_id, session_id).empty(); + // A pinned occurrence can vanish while other inclusions of the + // header survive (the chain stays non-empty) — recount it. + if(!orphaned && occurrence.has_value()) { + auto count = workspace.count_occurrences(host_id, session_id); + orphaned = count > 0 && *occurrence >= count; + } + // The pinned host command itself can vanish (a CDB reload + // changed the entry's flags): same validation didOpen applies. + if(!orphaned && !saved.command_hash.empty()) { + orphaned = + !entry_has_hash(workspace.path_pool.resolve(host_id), saved.command_hash); + } + } else if(!saved.command_hash.empty()) { + // Own-entry pin: the pinned command must still exist in the CDB. + orphaned = !entry_has_hash(workspace.path_pool.resolve(session_id), saved.command_hash); } if(orphaned) { - LOG_INFO("Dropping orphaned context choice for {}: host {} no longer includes it", - workspace.path_pool.resolve(session_id), - workspace.path_pool.resolve(host_id)); + LOG_INFO("Dropping orphaned context choice for {}: its basis no longer exists", + workspace.path_pool.resolve(session_id)); drop_header_context(session_id); session->pch_ref.reset(); session->ast_dirty = true; diff --git a/src/server/context/context_resolver.h b/src/server/context/context_resolver.h index b46d75ef4..582371a86 100644 --- a/src/server/context/context_resolver.h +++ b/src/server/context/context_resolver.h @@ -246,6 +246,11 @@ class ContextResolver { Session* session, bool synthesize); + /// Whether the CDB still holds an entry for `entry_path` whose canonical + /// command hash equals `hash` — the validity test for a pinned context + /// choice, shared by didOpen validation and the runtime orphan pass. + bool entry_has_hash(llvm::StringRef entry_path, llvm::StringRef hash) const; + /// The file's context choice, or nullptr. Gated on an open session: /// user choices steer editor-facing compiles, never background indexing /// (which passes no session). diff --git a/src/server/protocol/extension.h b/src/server/protocol/extension.h index 14905d3fc..199eeae74 100644 --- a/src/server/protocol/extension.h +++ b/src/server/protocol/extension.h @@ -81,4 +81,19 @@ struct InactiveRegionsParams { std::vector regions; }; +/// clice/internal/poll — TEST-ONLY, not a stable API. Synchronously runs +/// one file-tracker tick (stat → diff → events → dispatch → effects) and +/// responds only once the effects are applied, so integration tests can +/// disable the polling loops and get "change disk → poll → assert" +/// determinism with zero sleeps. Absent from capabilities and user docs. +struct PollParams { + /// Which loop to tick: "cdb" or "workspace". + std::string loop; +}; + +struct PollResult { + /// Number of file events the tick produced and dispatched. + std::uint32_t events = 0; +}; + } // namespace clice::ext diff --git a/src/server/service/lsp_client.cpp b/src/server/service/lsp_client.cpp index 4c8b9513e..6af35ac40 100644 --- a/src/server/service/lsp_client.cpp +++ b/src/server/service/lsp_client.cpp @@ -14,6 +14,7 @@ #include "server/protocol/extension.h" #include "server/protocol/serialize.h" #include "server/service/master_server.h" +#include "server/workspace/file_tracker.h" #include "support/anomaly.h" #include "support/filesystem.h" #include "support/logging.h" @@ -488,6 +489,37 @@ void LSPClient::register_extensions() { this->server.dispatch(FileEvent::context_changed(path_id)); co_return to_raw(result); }); + + // ── Test hook ─────────────────────────────────────────────────── + + // Runs one file-tracker tick synchronously (see ext::PollParams). + // Test-only and not a stable API; the CDB tick runs with force=true, + // so the polling loop's two-tick settling debounce does not apply here. + peer.on_request( + "clice/internal/poll", + [this](RequestContext& ctx, const ext::PollParams& params) -> RawResult { + auto& srv = this->server; + if(!srv.tracker) { + co_return kota::outcome_error(kota::ipc::Error{protocol::ErrorCode::InvalidRequest, + "No workspace is loaded"}); + } + + llvm::SmallVector events; + if(params.loop == "cdb") { + events = srv.tracker->tick_cdb(/*force=*/true); + } else if(params.loop == "workspace") { + events = co_await srv.tracker->tick_workspace(); + } else { + co_return kota::outcome_error( + kota::ipc::Error{protocol::ErrorCode::InvalidParams, + R"(loop must be "cdb" or "workspace")"}); + } + + if(!events.empty()) { + srv.dispatch(events); + } + co_return to_raw(ext::PollResult{static_cast(events.size())}); + }); } /// Publish clice.toml load problems as diagnostics, each on its own file's diff --git a/src/server/service/master_server.cpp b/src/server/service/master_server.cpp index c930a67d3..4555fbd0e 100644 --- a/src/server/service/master_server.cpp +++ b/src/server/service/master_server.cpp @@ -8,6 +8,7 @@ #include "server/protocol/worker.h" #include "server/service/agent_client.h" #include "server/service/lsp_client.h" +#include "server/workspace/file_tracker.h" #include "support/anomaly.h" #include "support/filesystem.h" #include "support/logging.h" @@ -92,6 +93,41 @@ void MasterServer::initialize() { wire(); load_workspace(); + + if(!workspace_root.empty()) { + // Construct after the workspace load so the tracker's baseline CDB + // stamp matches the database that was just loaded. + tracker = std::make_unique(workspace, sessions, workspace_root); + auto& tracker_cfg = workspace.config.tracker; + if(*tracker_cfg.cdb_poll_seconds > 0) { + bg_tasks.spawn(cdb_poll_task()); + } + if(*tracker_cfg.workspace_poll_seconds > 0) { + bg_tasks.spawn(workspace_poll_task()); + } + } +} + +kota::task<> MasterServer::cdb_poll_task() { + auto interval = std::chrono::seconds(*workspace.config.tracker.cdb_poll_seconds); + while(true) { + co_await kota::sleep(interval); + auto events = tracker->tick_cdb(); + if(!events.empty()) { + dispatch(events); + } + } +} + +kota::task<> MasterServer::workspace_poll_task() { + auto interval = std::chrono::seconds(*workspace.config.tracker.workspace_poll_seconds); + while(true) { + co_await kota::sleep(interval); + auto events = co_await tracker->tick_workspace(); + if(!events.empty()) { + dispatch(events); + } + } } void MasterServer::wire() { @@ -168,6 +204,9 @@ void MasterServer::dispatch(llvm::ArrayRef events) { if(auto session = sessions.find(path_id)) { session->ast_dirty = true; session->trial_done = false; + // Invalidate in-flight compiles so they cannot clobber the + // reset state when they finish (same as switchContext). + session->generation += 1; } contexts.forget_self_contained(path_id); } @@ -186,13 +225,26 @@ void MasterServer::dispatch(llvm::ArrayRef events) { if(auto session = sessions.find(path_id)) { session->ast_dirty = true; session->trial_done = false; + session->generation += 1; } } + // The header's borrowed compile command changed: its resolved context + // (and synthesized preamble) describes flags that no longer exist, so + // the next use must re-resolve. Session dirtying arrives in the same + // DirtySet via mark_ast_dirty. + for(auto path_id: dirty.drop_context) { + contexts.drop_header_context(path_id); + } + for(auto path_id: dirty.enqueue_reindex) { background_indexer.enqueue(path_id); } + if(dirty.ensure_compile_graph && !workspace.compile_graph) { + compiler.init_compile_graph(); + } + bool save = dirty.save_cache; if(dirty.recheck_contexts) { save |= contexts.drop_orphaned_choices(sessions); @@ -278,53 +330,20 @@ void MasterServer::load_workspace() { open_cache_store(); - std::string cdb_path; - for(auto& configured: cfg.compile_commands_paths) { - if(llvm::sys::fs::is_directory(configured)) { - auto candidate = path::join(configured, "compile_commands.json"); - if(llvm::sys::fs::exists(candidate)) { - cdb_path = std::move(candidate); - break; - } - } else if(llvm::sys::fs::exists(configured)) { - cdb_path = configured; - break; - } else { - LOG_WARN("Configured compile_commands_path not found: {}", configured); - } - } - - if(cdb_path.empty()) { - auto try_candidate = [&](llvm::StringRef dir) -> bool { - auto candidate = path::join(dir, "compile_commands.json"); - if(llvm::sys::fs::exists(candidate)) { - cdb_path = std::move(candidate); - return true; - } - return false; - }; - - if(!try_candidate(workspace_root)) { - std::error_code ec; - for(llvm::sys::fs::directory_iterator it(workspace_root, ec), end; it != end && !ec; - it.increment(ec)) { - if(it->type() == llvm::sys::fs::file_type::directory_file) { - if(try_candidate(it->path())) - break; - } - } - } - } - + auto cdb_path = discover_compile_commands(workspace.config, workspace_root); if(cdb_path.empty()) { LOG_GUIDANCE( "No compile_commands.json found in workspace {}. Compile commands will be " "guessed; see https://clice.io/en/guide/quick-start for setup.", workspace_root); + // Persisted index shards are CDB-independent; load them so a + // database generated later (picked up by the CDB poll) starts from + // the previous session's index. + background_indexer.load(); return; } ScopedTimer cdb_timer; - auto count = workspace.cdb.load(cdb_path); + auto count = workspace.cdb.load(cdb_path).value_or(0); LOG_INFO("Loaded CDB from {} with {} entries", cdb_path, count); LOG_PERF("startup", "phase=cdb_load entries={} elapsed_ms={}", count, cdb_timer.ms()); diff --git a/src/server/service/master_server.h b/src/server/service/master_server.h index f665cecea..fdc7f5951 100644 --- a/src/server/service/master_server.h +++ b/src/server/service/master_server.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -23,6 +24,8 @@ namespace clice { +class FileTracker; + namespace deco = kota::deco; enum class ServerMode : std::uint8_t { Pipe, Socket }; @@ -85,7 +88,6 @@ class MasterServer { void initialize(); void initialize(llvm::StringRef root); - // TODO: add periodic stat-based file watching kota::task<> shutdown_and_cleanup(); std::shared_ptr find_session(std::uint32_t path_id); @@ -122,6 +124,12 @@ class MasterServer { FeatureRouter features; Invalidator invalidator; + /// Stat-polling discovery of CDB and on-disk file changes. Created by + /// initialize() once the workspace is loaded (null before that and in + /// workspace-less sessions); its polling loops run in bg_tasks, and the + /// clice/internal/poll test hook drives ticks directly. + std::unique_ptr tracker; + /// Lifecycle state, advanced by the LSP initialize/shutdown handlers. ServerLifecycle lifecycle = ServerLifecycle::Uninitialized; @@ -154,6 +162,12 @@ class MasterServer { /// times survive crashes (the store itself is passive by design). kota::task<> cache_checkpoint_task(); + /// The file tracker's polling loops: each tick hands the tracker's + /// event batch to dispatch(). Spawned by initialize() when the + /// configured interval is non-zero. + kota::task<> cdb_poll_task(); + kota::task<> workspace_poll_task(); + kota::event shutdown_event; /// Server-owned background tasks (cache checkpoint); cancelled and diff --git a/src/server/workspace/config.cpp b/src/server/workspace/config.cpp index 447ee7b28..0b9c19822 100644 --- a/src/server/workspace/config.cpp +++ b/src/server/workspace/config.cpp @@ -74,6 +74,12 @@ void Config::apply_defaults(llvm::StringRef workspace_root) { if(p.worker_memory_limit == 0) p.worker_memory_limit = 4ULL * 1024 * 1024 * 1024; // 4GB + auto& t = tracker; + if(!t.cdb_poll_seconds) + t.cdb_poll_seconds = 3; + if(!t.workspace_poll_seconds) + t.workspace_poll_seconds = 30; + if(p.cache_dir.empty() && !workspace_root.empty()) { p.cache_dir = resolve_xdg_cache_dir(workspace_root); if(p.cache_dir.empty()) diff --git a/src/server/workspace/config.h b/src/server/workspace/config.h index c6d339692..e2b5adf95 100644 --- a/src/server/workspace/config.h +++ b/src/server/workspace/config.h @@ -42,6 +42,16 @@ struct ProjectConfig { defaulted worker_memory_limit = {}; }; +/// Corresponds to the `[tracker]` section in clice.toml: the stat-polling +/// file tracker's intervals. 0 disables the loop (integration tests drive +/// ticks through the clice/internal/poll hook instead). +struct TrackerConfig { + /// Compilation database poll interval in seconds (default 3). + std::optional cdb_poll_seconds; + /// Workspace file sweep interval in seconds (default 30). + std::optional workspace_poll_seconds; +}; + struct CompiledRule { std::vector patterns; std::vector append; @@ -72,6 +82,8 @@ struct ConfigIssue { struct Config { defaulted project; + defaulted tracker; + defaulted> rules; kota::meta::annotation, kota::meta::attrs::skip> compiled_rules; diff --git a/src/server/workspace/file_tracker.cpp b/src/server/workspace/file_tracker.cpp new file mode 100644 index 000000000..5b59eebaa --- /dev/null +++ b/src/server/workspace/file_tracker.cpp @@ -0,0 +1,259 @@ +#include "server/workspace/file_tracker.h" + +#include +#include +#include + +#include "support/logging.h" +#include "support/timer.h" + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/Chrono.h" +#include "llvm/Support/FileSystem.h" + +namespace clice { + +static std::int64_t to_nanoseconds(const llvm::sys::TimePoint<>& time) { + return std::chrono::duration_cast(time.time_since_epoch()).count(); +} + +FileTracker::FileTracker(Workspace& workspace, + const SessionStore& store, + std::string workspace_root) : + workspace(workspace), store(store), workspace_root(std::move(workspace_root)) { + cdb_path = discover_compile_commands(workspace.config, this->workspace_root); + // A change landing between the workspace load and this stat is caught + // anyway: the stamp only gates reloads, and the reload's diff is + // computed from content, so it never reports spurious changes. + applied = stat_cdb(); +} + +FileTracker::CDBStamp FileTracker::stat_cdb() const { + CDBStamp stamp; + if(cdb_path.empty()) { + return stamp; + } + llvm::sys::fs::file_status status; + if(llvm::sys::fs::status(cdb_path, status)) { + return stamp; + } + stamp.exists = true; + stamp.size = status.getSize(); + stamp.mtime_ns = to_nanoseconds(status.getLastModificationTime()); + return stamp; +} + +llvm::SmallVector FileTracker::tick_cdb(bool force) { + if(cdb_path.empty()) { + cdb_path = discover_compile_commands(workspace.config, workspace_root); + if(cdb_path.empty()) { + return {}; + } + // `applied` stays at its missing state: the fresh file is a change + // against the never-loaded database and goes through the normal + // settle-and-reload path below. + LOG_INFO("Found compilation database: {}", cdb_path); + } + + auto current = stat_cdb(); + if(!force) { + if(current == applied) { + has_pending = false; + return {}; + } + // Generators rewrite the file in place; only act once the stamp + // has been stable for two consecutive ticks (half-write guard). + if(!has_pending || !(pending == current)) { + pending = current; + has_pending = true; + return {}; + } + } + // A forced tick reloads unconditionally — the stamp gate would make a + // same-size rewrite within mtime granularity invisible to the test + // hook, and a spurious reload just yields an empty diff. + has_pending = false; + + if(!current.exists) { + // Deleted — usually mid-regeneration. Keep serving the loaded + // entries; the rewrite lands as the next observed change. Forget + // the path too: if the database reappears somewhere else among the + // configured locations, discovery must run again. + applied = current; + cdb_path.clear(); + return {}; + } + + auto diff = workspace.cdb.reload_and_diff(cdb_path); + if(!diff) { + // Stats fine but unreadable right now (e.g. still locked by the + // generator). Leave `applied` alone: the stamp stays different, so + // the reload is retried on a later tick instead of being lost. + return {}; + } + applied = current; + LOG_INFO("Reloaded CDB from {}: {} added, {} removed, {} changed", + cdb_path, + diff->added.size(), + diff->removed.size(), + diff->changed.size()); + if(diff->empty()) { + return {}; + } + + // The diff speaks in the CDB's own path ids; events speak in master + // path-pool ids. + FileEvent::CDBDelta delta; + auto convert = [&](llvm::ArrayRef cdb_ids, + llvm::SmallVectorImpl& out) { + for(auto id: cdb_ids) { + out.push_back(workspace.path_pool.intern(workspace.cdb.resolve_path(id))); + } + }; + convert(diff->added, delta.added); + convert(diff->removed, delta.removed); + convert(diff->changed, delta.changed); + + llvm::SmallVector events; + events.push_back(FileEvent::cdb_changed(std::move(delta))); + return events; +} + +kota::task> FileTracker::tick_workspace() { + constexpr std::size_t batch_size = 500; + + if(sweeping) { + // The poll hook can land while the live loop is suspended between + // batches; two interleaved sweeps would race on the baseline. The + // running sweep already covers this request. + co_return llvm::SmallVector{}; + } + sweeping = true; + auto guard = llvm::make_scope_exit([this] { sweeping = false; }); + + ScopedTimer timer; + auto epoch = workspace.context_epoch; + auto files = workspace.dep_graph.all_files(); + + // Files that left the graph (e.g. a CDB reload rebuilt it) stop being + // tracked; their baseline entries would otherwise be stat'd forever. + llvm::DenseSet known(files.begin(), files.end()); + llvm::SmallVector gone; + for(auto& [path_id, state]: baseline) { + if(!known.contains(path_id)) { + gone.push_back(path_id); + } + } + for(auto path_id: gone) { + baseline.erase(path_id); + } + + llvm::SmallVector events; + std::size_t changed = 0; + std::size_t removed = 0; + for(std::size_t begin = 0; begin < files.size(); begin += batch_size) { + if(begin != 0) { + // Yield one loop iteration between batches so a long sweep + // never starves LSP traffic. + co_await kota::sleep(std::chrono::milliseconds(0)); + } + + auto batch_end = std::min(begin + batch_size, files.size()); + for(std::size_t i = begin; i < batch_end; ++i) { + auto path_id = files[i]; + if(store.find(path_id)) { + // Open buffers are the truth and didSave owns their disk + // sync; drop the baseline so the file re-seeds silently + // once it closes (BufferClosed already reindexes it). + // TODO: a disk change landing while the file is open (git + // checkout on an open header, closed without saving) is + // forgotten by this reset — dependents are not cascaded. + // Desync hardening owns that case. + baseline.erase(path_id); + continue; + } + + auto path = workspace.path_pool.resolve(path_id); + llvm::sys::fs::file_status status; + bool exists = !llvm::sys::fs::status(path, status); + + auto it = baseline.find(path_id); + if(it == baseline.end()) { + // First sight seeds the baseline silently. + FileState state; + state.missing = !exists; + if(exists) { + state.size = status.getSize(); + state.mtime_ns = to_nanoseconds(status.getLastModificationTime()); + state.hash = hash_file(path); + if(state.hash == 0) { + // Read failure (hash_file's sentinel): don't seed a + // baseline that would later compare as a change. + // Retry next tick. + continue; + } + } + baseline.try_emplace(path_id, state); + continue; + } + + auto& state = it->second; + if(!exists) { + if(!state.missing) { + state = FileState{.missing = true}; + events.push_back(FileEvent::disk_removed(path_id)); + removed += 1; + } + continue; + } + + auto size = status.getSize(); + auto mtime_ns = to_nanoseconds(status.getLastModificationTime()); + if(!state.missing && state.size == size && state.mtime_ns == mtime_ns) { + continue; + } + + // The stamp moved: only a confirmed content change counts, so + // touches and checkouts of identical bytes stay silent. + auto hash = hash_file(path); + if(hash == 0) { + // The file stats fine but cannot be read right now (e.g. an + // antivirus scanner briefly holding a fresh file on Windows). + // No signal either way — leave the baseline untouched so the + // still-different stamp retries this check next tick, and + // emit nothing: a failed read must never count as a change. + continue; + } + bool content_changed = state.missing || hash != state.hash; + state = FileState{.size = size, .mtime_ns = mtime_ns, .hash = hash}; + if(content_changed) { + events.push_back(FileEvent::disk_changed(path_id)); + changed += 1; + } + } + } + + // A CDB reload can rebuild the include graph while this sweep is + // suspended between batches. Events for files the new graph no longer + // tracks must not dispatch — their rescan cascade would reintroduce + // edges for files that stopped being sources. Dropping them is safe: + // their baseline entries are pruned on the next sweep. + if(workspace.context_epoch != epoch && !events.empty()) { + auto current = workspace.dep_graph.all_files(); + llvm::DenseSet still_known(current.begin(), current.end()); + llvm::erase_if(events, [&](const FileEvent& event) { + return !still_known.contains(event.path_id); + }); + } + + LOG_PERF("tracker", + "phase=workspace_sweep files={} changed={} removed={} elapsed_ms={}", + files.size(), + changed, + removed, + timer.ms()); + co_return events; +} + +} // namespace clice diff --git a/src/server/workspace/file_tracker.h b/src/server/workspace/file_tracker.h new file mode 100644 index 000000000..da18a22d4 --- /dev/null +++ b/src/server/workspace/file_tracker.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include + +#include "server/session/session_store.h" +#include "server/workspace/invalidator.h" +#include "server/workspace/workspace.h" + +#include "kota/async/async.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" + +namespace clice { + +/// Stat-based discovery of changes the client never tells us about: +/// compile_commands.json edits and files changing on disk behind the +/// server's back (git checkout, code generators, save hooks). +/// +/// Core design property: polling only marks dirty and emits events — it +/// never needs to be complete. A missed change means derived state stays +/// stale for one more poll period at worst; correctness is anchored by the +/// pull side's two-layer DepsSnapshot validation (mtime, then content hash) +/// at compile and index time. That is what lets this implementation stay +/// simple and coarse, and why it polls stat instead of using inotify — for +/// clangd's reasons: portable, no fd limits, no event storms. +/// +/// The tracker only observes and returns event batches; it never +/// dispatches. MasterServer's polling loops (and the clice/internal/poll +/// test hook) hand each batch to dispatch(), which keeps the tracker +/// unit-testable against plain data structures. +class FileTracker { +public: + /// Discovers the compile_commands.json itself (it may not exist yet) + /// and records the stamp the currently loaded CDB corresponds to. + /// Construct after the workspace is loaded. + FileTracker(Workspace& workspace, const SessionStore& store, std::string workspace_root); + + /// One CDB poll tick. Stats the known compile_commands.json — or keeps + /// discovering one when none was found yet, which is how a database + /// generated after startup is picked up. Once a (size, mtime) change + /// has stayed stable for two consecutive ticks, reloads the CDB and + /// emits one CDBChanged event carrying the reload's diff. + /// + /// `force` reloads unconditionally: it skips both the (size, mtime) + /// stamp gate — which could hide a same-size rewrite landing within + /// mtime granularity — and the two-tick settling debounce (the + /// half-written-file guard). The test hook uses it so a single poll + /// request applies a change deterministically; a spurious forced + /// reload just yields an empty diff. + llvm::SmallVector tick_cdb(bool force = false); + + /// One workspace sweep. Stats every file the dependency graph knows, + /// skipping open buffers; a (mtime, size) suspect is confirmed by + /// content hash before DiskChanged is emitted, so touch-only changes + /// (mtime bump, identical bytes) stay silent. A stat failure on a + /// known file emits DiskRemoved once; a transient content-read failure + /// emits nothing and is retried on the next tick. + /// + /// Files seen for the first time only seed the baseline and emit + /// nothing — the first sweep after startup is silent by construction + /// (startup storm guard), and files entering the graph later start + /// tracking silently too. + /// + /// Stats run synchronously in batches, yielding to the event loop + /// between batches; each round's duration is perf-logged. + /// TODO: offload stats to the thread pool (and consider a directory + /// listing cache for Windows, where per-file stat is expensive) if the + /// logged sweep timing shows the need. + kota::task> tick_workspace(); + +private: + /// (existence, size, mtime) identity of the CDB file. + struct CDBStamp { + bool exists = false; + std::uint64_t size = 0; + std::int64_t mtime_ns = 0; + + friend bool operator==(const CDBStamp&, const CDBStamp&) = default; + }; + + CDBStamp stat_cdb() const; + + /// Last-known on-disk state of a tracked file. + struct FileState { + std::uint64_t size = 0; + std::int64_t mtime_ns = 0; + std::uint64_t hash = 0; + bool missing = false; + }; + + Workspace& workspace; + const SessionStore& store; + std::string workspace_root; + + /// compile_commands.json path; empty until one is discovered. + std::string cdb_path; + /// The stamp the currently loaded CDB entries correspond to. + CDBStamp applied; + /// Debounce: the stamp observed on the previous tick, not yet settled. + CDBStamp pending; + bool has_pending = false; + + /// Workspace sweep baseline. + llvm::DenseMap baseline; + + /// True while a sweep is in flight (it suspends between batches); + /// concurrent ticks are skipped instead of racing on the baseline. + bool sweeping = false; +}; + +} // namespace clice diff --git a/src/server/workspace/invalidator.cpp b/src/server/workspace/invalidator.cpp index 910645755..799049b39 100644 --- a/src/server/workspace/invalidator.cpp +++ b/src/server/workspace/invalidator.cpp @@ -30,9 +30,96 @@ static void dedup(llvm::SmallVector& ids) { ids.erase(llvm::unique(ids), ids.end()); } +void Invalidator::cascade_compile_graph(std::uint32_t path_id, DirtySet& dirty) { + if(!workspace.compile_graph || !workspace.compile_graph->has_unit(path_id)) { + return; + } + for(auto dirty_id: workspace.compile_graph->update(path_id)) { + workspace.pcm_paths.erase(dirty_id); + workspace.pcm_cache.erase(dirty_id); + if(store.find(dirty_id)) { + dirty.mark_ast_dirty.push_back(dirty_id); + } else { + dirty.enqueue_reindex.push_back(dirty_id); + } + } +} + +void Invalidator::cascade_disk_content_change(std::uint32_t path_id, DirtySet& dirty) { + // The file's own self-containment may have changed; re-evaluate on its + // next compile. + dirty.reset_header_mode.push_back(path_id); + dirty.reset_trial.push_back(path_id); + + // Root TUs transitively including the file, snapshotted before the + // rescan rewrites the include graph. A content change only rewrites + // the file's own outgoing edges, so this set normally equals the + // post-rescan one — the pre-rescan snapshot is a cheap safety net for + // a reverse map that was stale when the change landed. + auto old_dependents = workspace.dep_graph.find_host_sources(path_id); + + // Rescan disk state (include edges, module declaration, compile-graph + // cascade, PCM caches); the cascade names the module units whose build + // products went stale. + auto dirtied = workspace.rescan_after_save(path_id); + for(auto dirty_id: dirtied) { + if(store.find(dirty_id)) { + dirty.mark_ast_dirty.push_back(dirty_id); + } else { + dirty.enqueue_reindex.push_back(dirty_id); + } + } + + // The new content is a compile input of every TU that transitively + // includes it: open dependents recompile, closed ones reindex so + // cross-file references stop serving the stale state. Enqueueing is + // O(1) per TU and deliberately uncapped — the index's content-hash + // staleness check filters TUs whose dependencies did not actually + // change, and the idle/priority scheduling throttles the rest. + // TODO: observe on large projects before adding debouncing. + auto split_dependents = [&](llvm::ArrayRef roots) { + for(auto root: roots) { + if(store.find(root)) { + dirty.mark_ast_dirty.push_back(root); + } else { + dirty.enqueue_reindex.push_back(root); + } + } + }; + split_dependents(old_dependents); + split_dependents(workspace.dep_graph.find_host_sources(path_id)); + + // Headers whose resolved context embeds the file through its include + // chain must re-synthesize their preamble: it copies the chain files' + // content, so neither the dependents cascade above nor clang's own + // dependency tracking catches this. + for(auto header_id: contexts.chain_dependents(path_id)) { + dirty.force_revalidate.push_back(header_id); + // The chain change may have made the header self-contained (e.g. a + // dependency now provides the missing declarations); drop the + // persisted verdict so the trial can downgrade it. + dirty.reset_header_mode.push_back(header_id); + // Contexts outlive their sessions: a closed header's shard rows + // were indexed under the old chain and only a background reindex + // can refresh them. + if(!store.find(header_id)) { + dirty.enqueue_reindex.push_back(header_id); + } + } + + // A content change can remove the include edge a user's context choice + // depends on; the include graph was already rescanned above. + dirty.recheck_contexts = true; + dirty.reschedule_indexing = true; +} + DirtySet Invalidator::apply(llvm::ArrayRef events) { DirtySet dirty; + // DiskRemoved defers its reverse-map rebuild here so a batch of + // removals pays for one rebuild, not one per file. + bool rebuild_reverse_map = false; + for(auto& event: events) { switch(event.kind) { case FileEvent::Kind::BufferOpened: { @@ -47,76 +134,28 @@ DirtySet Invalidator::apply(llvm::ArrayRef events) { } case FileEvent::Kind::BufferSaved: { auto path_id = event.path_id; + // The disk now holds the buffer's content: the standard + // disk-content cascade covers everything a save invalidates. + cascade_disk_content_change(path_id, dirty); - // The saved file's own self-containment may have changed; - // re-evaluate on its next compile. - dirty.reset_header_mode.push_back(path_id); - dirty.reset_trial.push_back(path_id); - - // Root TUs transitively including the saved file, snapshotted - // before the rescan rewrites the include graph. A save only - // rewrites the saved file's own outgoing edges, so this set - // normally equals the post-rescan one — the pre-rescan - // snapshot is a cheap safety net for a reverse map that was - // stale at save time. - auto old_dependents = workspace.dep_graph.find_host_sources(path_id); - - // Rescan disk state (include edges, module declaration, - // compile-graph cascade, PCM caches); the cascade names the - // module units whose build products went stale. - auto dirtied = workspace.rescan_after_save(path_id); - for(auto dirty_id: dirtied) { - if(store.find(dirty_id)) { - dirty.mark_ast_dirty.push_back(dirty_id); - } else { - dirty.enqueue_reindex.push_back(dirty_id); - } - } - - // The saved content is a compile input of every TU that - // transitively includes it: open dependents recompile, closed - // ones reindex so cross-file references stop serving the - // pre-save state. Enqueueing is O(1) per TU and deliberately - // uncapped — the index's content-hash staleness check filters - // TUs whose dependencies did not actually change, and the - // idle/priority scheduling throttles the rest. - // TODO: observe on large projects before adding debouncing. - auto split_dependents = [&](llvm::ArrayRef roots) { - for(auto root: roots) { - if(store.find(root)) { - dirty.mark_ast_dirty.push_back(root); - } else { - dirty.enqueue_reindex.push_back(root); - } - } - }; - split_dependents(old_dependents); - split_dependents(workspace.dep_graph.find_host_sources(path_id)); - - // Headers whose resolved context embeds the saved file - // through its include chain must re-synthesize their - // preamble: it copies the chain files' content, so neither - // the dependents cascade above nor clang's own dependency - // tracking catches this. - for(auto header_id: contexts.chain_dependents(path_id)) { - dirty.force_revalidate.push_back(header_id); - // The chain change may have made the header - // self-contained (e.g. a dependency now provides the - // missing declarations); drop the persisted verdict so - // the trial can downgrade it. - dirty.reset_header_mode.push_back(header_id); - // Contexts outlive their sessions: a closed header's - // shard rows were indexed under the old chain and only a - // background reindex can refresh them. - if(!store.find(header_id)) { - dirty.enqueue_reindex.push_back(header_id); + // ... unless a save hook or formatter rewrote the file as it + // landed, leaving the disk ahead of the buffer. Dependents + // already read the rewritten disk through the cascade above; + // without this check the saved file itself would keep serving + // results whose deps snapshot describes a disk state that no + // longer exists ("I see my old buffer, my dependents see the + // new disk"). Recompiling does not change what the session + // compiles — an open file's own text always comes from its + // buffer — but it re-captures the deps snapshot and re-runs + // preamble/PCH validation against the rewritten disk, which + // the pull-side staleness check alone can miss when the + // rewrite lands within mtime granularity of the compile. + if(auto session = store.find(path_id)) { + auto disk = read_file(workspace.path_pool.resolve(path_id)); + if(!disk || *disk != session->text) { + dirty.mark_ast_dirty.push_back(path_id); } } - - // A save can remove the include edge a user's context choice - // depends on; the include graph was already rescanned above. - dirty.recheck_contexts = true; - dirty.reschedule_indexing = true; break; } case FileEvent::Kind::BufferClosed: { @@ -129,12 +168,171 @@ DirtySet Invalidator::apply(llvm::ArrayRef events) { break; } case FileEvent::Kind::DiskChanged: { - // TODO: no producer yet — the disk poller / cache validation - // side will emit these. + auto path_id = event.path_id; + if(store.find(path_id)) { + // Open file: the buffer is the truth, so no disk rescan — + // what the disk change means for this file is decided by + // the next compile's deps validation. Recompile so that + // validation actually runs. + dirty.mark_ast_dirty.push_back(path_id); + break; + } + // Closed file: disk is the truth. Run the same cascade a + // save does, and refresh the file's own now-stale shard. + cascade_disk_content_change(path_id, dirty); + dirty.enqueue_reindex.push_back(path_id); break; } case FileEvent::Kind::DiskRemoved: { - // TODO: no producer yet (see DiskChanged). + auto path_id = event.path_id; + // Dependents compile against a now-missing include: open + // ones recompile (the missing-file diagnostic is the truth), + // closed ones reindex — nothing else would ever queue them. + // Snapshot before the scrub below rewrites the graph. + for(auto root: workspace.dep_graph.find_host_sources(path_id)) { + if(store.find(root)) { + dirty.mark_ast_dirty.push_back(root); + } else { + dirty.enqueue_reindex.push_back(root); + } + } + // A removed module unit takes its PCM with it: importers' + // build products went stale, and it stops providing its + // module name. + cascade_compile_graph(path_id, dirty); + workspace.path_to_module.erase(path_id); + // Scrub the includer role: the file's outgoing edges vanished + // with it, so it stops being a host-source candidate. + // Incoming edges stay — includers' text still names it, and + // their own rescan owns those edges. The reverse-map rebuild + // is deferred to the end of the batch: a mass deletion (git + // checkout) would otherwise rebuild it once per file, and + // within-batch cascades tolerate a stale reverse map by + // design (they union the pre/post snapshots). + workspace.dep_graph.clear_includes(path_id); + rebuild_reverse_map = true; + workspace.context_epoch += 1; + // Contexts hosted by (or chained through) the removed file + // are cleaned by the resolver's orphan pass. + dirty.recheck_contexts = true; + dirty.reschedule_indexing = true; + // Index shards are deliberately kept: the last-known content + // still serves navigation. + // TODO: sweep orphaned shards of files that stay deleted. + break; + } + case FileEvent::Kind::CDBChanged: { + auto& delta = event.cdb; + if(delta.empty()) { + break; + } + + // The producer already reloaded the CDB; derived state must + // follow. Rebuild the include graph and module map from + // scratch against the new database: entry additions, + // removals and flag changes all funnel into one uniform + // rescan instead of per-entry graph surgery. No ScanCache is + // retained anywhere: the cache's contract requires clearing + // it on every CDB change, and CDB changes are the only + // rescan trigger, so a persistent cache would never be warm. + // TODO: this scan runs synchronously on the event loop (same + // cost as the startup scan); if it shows up on large + // projects, move it off the dispatch path. + workspace.dep_graph = DependencyGraph(); + scan_dependency_graph(workspace.cdb, + workspace.toolchain, + workspace.path_pool, + workspace.dep_graph, + /*cache=*/nullptr, + [this](llvm::StringRef path, + std::vector& append, + std::vector& remove) { + workspace.config.match_rules(path, append, remove); + }); + workspace.dep_graph.build_reverse_map(); + workspace.path_to_module.clear(); + workspace.build_module_map(); + workspace.context_epoch += 1; + + // Every delta entry needs the same treatment — the compile + // command is an input that content-based staleness cannot + // see, whether it appeared, changed or vanished. PCH/PCM + // keys embed the canonical flags, so pull-side caches miss + // naturally. + auto invalidate_entry = [&](std::uint32_t path_id, bool keep_shard) { + if(store.find(path_id)) { + // The next compile re-resolves the command (added: + // first real entry replaces the guessed one; + // changed: new flags; removed: fall back). + dirty.mark_ast_dirty.push_back(path_id); + } else if(!keep_shard) { + // The shard was indexed under the old command, and + // the indexer's freshness gate validates content + // only: evict the shard so the queued reindex is + // not filtered out as fresh. + // TODO: a background index task already in flight + // can merge its old-command result back after this + // eviction; closing that window needs an index + // generation guard in the indexer. + workspace.merged_indices.erase(path_id); + dirty.enqueue_reindex.push_back(path_id); + } + + // A module unit's command change invalidates importers' + // PCMs (no-op for files the compile graph doesn't know). + cascade_compile_graph(path_id, dirty); + + // The file's own resolved header context was built on a + // command that no longer exists in that form (a header + // gaining its first exact entry included), and so was + // every header context hosted by this file. Drop them + // so the next use re-resolves. + if(contexts.header_context(path_id)) { + dirty.drop_context.push_back(path_id); + } + for(auto& [header_id, context]: contexts.header_contexts) { + if(context.host_path_id != path_id) { + continue; + } + dirty.drop_context.push_back(header_id); + if(store.find(header_id)) { + dirty.mark_ast_dirty.push_back(header_id); + } else { + workspace.merged_indices.erase(header_id); + dirty.enqueue_reindex.push_back(header_id); + } + } + }; + + for(auto path_id: delta.added) { + invalidate_entry(path_id, /*keep_shard=*/false); + } + for(auto path_id: delta.changed) { + invalidate_entry(path_id, /*keep_shard=*/false); + } + for(auto path_id: delta.removed) { + // A removed entry keeps its shard: the last-known + // content still serves navigation, same conservative + // semantics as DiskRemoved. The graph rebuild above + // already dropped the file's source role, and the + // orphan recheck cleans choices through it. + invalidate_entry(path_id, /*keep_shard=*/true); + } + + // The first CDB of the session may have introduced C++20 + // modules; the compile graph is otherwise created at startup. + // TODO: a reload that adds a brand-new module unit to an + // already existing graph is not registered (update() only + // touches known units) — importers resolved before it + // existed keep their stale dependency lists until restart. + // Rebuilding the graph mid-session needs coordination with + // in-flight compiles. + if(!workspace.compile_graph) { + dirty.ensure_compile_graph = true; + } + + dirty.recheck_contexts = true; + dirty.reschedule_indexing = true; break; } case FileEvent::Kind::ContextChanged: { @@ -156,12 +354,17 @@ DirtySet Invalidator::apply(llvm::ArrayRef events) { } } + if(rebuild_reverse_map) { + workspace.dep_graph.build_reverse_map(); + } + dedup(dirty.mark_ast_dirty); dedup(dirty.mark_lost); dedup(dirty.reset_trial); dedup(dirty.reset_header_mode); dedup(dirty.force_revalidate); dedup(dirty.enqueue_reindex); + dedup(dirty.drop_context); return dirty; } diff --git a/src/server/workspace/invalidator.h b/src/server/workspace/invalidator.h index 0e3a3a45d..0b35ad1ca 100644 --- a/src/server/workspace/invalidator.h +++ b/src/server/workspace/invalidator.h @@ -28,21 +28,38 @@ struct FileEvent { BufferSaved, /// didClose dropped the buffer; disk is the truth again. BufferClosed, - /// The file changed on disk behind the server's back. No producer - /// yet — reserved for the disk poller / cache validation side. + /// The file's content changed on disk behind the server's back + /// (emitted by the FileTracker's workspace poll). DiskChanged, - /// The file disappeared from disk. No producer yet (see DiskChanged). + /// The file disappeared from disk (see DiskChanged). DiskRemoved, + /// The compilation database was reloaded; `cdb` lists the files + /// whose entries were added, removed or changed. + CDBChanged, /// clice/switchContext changed the file's active header context. ContextChanged, /// A stateful worker crashed; `paths` lists the documents it owned. WorkerCrashed, }; + /// CDBChanged payload: the reload's per-file delta, as master path-pool + /// ids. `changed` means the file kept an entry but its command differs. + struct CDBDelta { + llvm::SmallVector added; + llvm::SmallVector removed; + llvm::SmallVector changed; + + bool empty() const { + return added.empty() && removed.empty() && changed.empty(); + } + }; + Kind kind; std::uint32_t path_id = no_path_id; /// WorkerCrashed only: the crashed worker's lost documents. llvm::SmallVector paths; + /// CDBChanged only: the reload delta. + CDBDelta cdb; static FileEvent buffer_opened(std::uint32_t path_id) { return {Kind::BufferOpened, path_id}; @@ -60,6 +77,20 @@ struct FileEvent { return {Kind::BufferClosed, path_id}; } + static FileEvent disk_changed(std::uint32_t path_id) { + return {Kind::DiskChanged, path_id}; + } + + static FileEvent disk_removed(std::uint32_t path_id) { + return {Kind::DiskRemoved, path_id}; + } + + static FileEvent cdb_changed(CDBDelta delta) { + FileEvent event{Kind::CDBChanged}; + event.cdb = std::move(delta); + return event; + } + static FileEvent context_changed(std::uint32_t path_id) { return {Kind::ContextChanged, path_id}; } @@ -96,6 +127,12 @@ struct DirtySet { /// Closed files whose index entries went stale: enqueue for background /// reindexing. llvm::SmallVector enqueue_reindex; + /// Headers whose resolved context borrows a compile command that no + /// longer exists in that form (the host's CDB entry changed): drop the + /// context so the next use re-resolves. Content validation cannot see + /// a flag change, so neither force_revalidate nor the deps snapshot + /// covers this. Executed by the context resolver. + llvm::SmallVector drop_context; /// Include edges changed: context choices may now be orphaned; run the /// context resolver's orphan cleanup. bool recheck_contexts = false; @@ -103,11 +140,16 @@ struct DirtySet { bool save_cache = false; /// Kick the background indexer's scheduler. bool reschedule_indexing = false; + /// A CDB reload may have introduced the first C++20 modules; create the + /// compile graph if it does not exist yet. Executed by the dispatcher, + /// which owns the Compiler. + bool ensure_compile_graph = false; bool empty() const { return mark_ast_dirty.empty() && mark_lost.empty() && reset_trial.empty() && reset_header_mode.empty() && force_revalidate.empty() && enqueue_reindex.empty() && - !recheck_contexts && !save_cache && !reschedule_indexing; + drop_context.empty() && !recheck_contexts && !save_cache && !reschedule_indexing && + !ensure_compile_graph; } }; @@ -131,7 +173,8 @@ class Invalidator { public: /// Read a file's current on-disk content, or nullopt if unreadable. /// Defaults to the real filesystem; unit tests inject file content - /// through it. Reserved for the Disk* events' content validation. + /// through it. Used by BufferSaved to detect a save hook or formatter + /// rewriting the file as it lands (disk ahead of the buffer). using ReadFile = std::function(llvm::StringRef path)>; Invalidator(Workspace& workspace, @@ -143,6 +186,18 @@ class Invalidator { DirtySet apply(llvm::ArrayRef events); private: + /// The invalidation cascade for "this file's on-disk content is new": + /// rescan the file's disk state, then split every affected file into + /// open (recompile) and closed (reindex). Shared by BufferSaved (disk + /// now holds the buffer) and DiskChanged on closed files (disk changed + /// behind the server's back), and used verbatim — the two differ only + /// in what the caller adds around it. + void cascade_disk_content_change(std::uint32_t path_id, DirtySet& dirty); + + /// Cascade a module unit's compile-graph invalidation (PCM caches, + /// dependent module units), splitting dirtied units open/closed. + void cascade_compile_graph(std::uint32_t path_id, DirtySet& dirty); + Workspace& workspace; const SessionStore& store; const ContextResolver& contexts; diff --git a/src/server/workspace/workspace.cpp b/src/server/workspace/workspace.cpp index 809d97cb9..8227461eb 100644 --- a/src/server/workspace/workspace.cpp +++ b/src/server/workspace/workspace.cpp @@ -180,6 +180,48 @@ void Workspace::on_file_closed(std::uint32_t path_id) { // blob eviction is the CacheStore's job, so nothing to clean up here. } +std::string discover_compile_commands(const Config& config, llvm::StringRef workspace_root) { + for(auto& configured: config.project.compile_commands_paths) { + if(llvm::sys::fs::is_directory(configured)) { + auto candidate = path::join(configured, "compile_commands.json"); + if(llvm::sys::fs::exists(candidate)) { + return candidate; + } + } else if(llvm::sys::fs::exists(configured)) { + return configured; + } else { + LOG_DEBUG("Configured compile_commands_path not found: {}", configured); + } + } + + if(workspace_root.empty()) { + return {}; + } + + auto try_candidate = [](llvm::StringRef dir) -> std::string { + auto candidate = path::join(dir, "compile_commands.json"); + if(llvm::sys::fs::exists(candidate)) { + return candidate; + } + return {}; + }; + + if(auto found = try_candidate(workspace_root); !found.empty()) { + return found; + } + + std::error_code ec; + for(llvm::sys::fs::directory_iterator it(workspace_root, ec), end; it != end && !ec; + it.increment(ec)) { + if(it->type() == llvm::sys::fs::file_type::directory_file) { + if(auto found = try_candidate(it->path()); !found.empty()) { + return found; + } + } + } + return {}; +} + std::uint64_t hash_file(llvm::StringRef path) { auto buf = llvm::MemoryBuffer::getFile(path); if(!buf) diff --git a/src/server/workspace/workspace.h b/src/server/workspace/workspace.h index 21a20a42d..cc7134459 100644 --- a/src/server/workspace/workspace.h +++ b/src/server/workspace/workspace.h @@ -251,6 +251,12 @@ struct Workspace { void cancel_all(); }; +/// Find the workspace's compile_commands.json: the configured paths first +/// (a directory means /compile_commands.json), then the workspace root, +/// then its direct subdirectories. Returns the empty string when none +/// exists yet — the file tracker keeps looking on its CDB poll. +std::string discover_compile_commands(const Config& config, llvm::StringRef workspace_root); + /// Hash a file's content using xxh3_64bits. Returns 0 on read failure. std::uint64_t hash_file(llvm::StringRef path); diff --git a/src/syntax/dependency_graph.cpp b/src/syntax/dependency_graph.cpp index 5ace374dc..b1517c9d4 100644 --- a/src/syntax/dependency_graph.cpp +++ b/src/syntax/dependency_graph.cpp @@ -83,6 +83,20 @@ llvm::SmallVector DependencyGraph::get_all_includes(std::uint32_t return result; } +llvm::SmallVector DependencyGraph::all_files() const { + llvm::SmallVector files; + files.reserve(file_configs.size() + reverse_includes_.size()); + for(auto& [path_id, configs]: file_configs) { + files.push_back(path_id); + } + for(auto& [path_id, includers]: reverse_includes_) { + files.push_back(path_id); + } + llvm::sort(files); + files.erase(llvm::unique(files), files.end()); + return files; +} + std::size_t DependencyGraph::file_count() const { return file_configs.size(); } diff --git a/src/syntax/dependency_graph.h b/src/syntax/dependency_graph.h index 08580bbfb..848c21fe7 100644 --- a/src/syntax/dependency_graph.h +++ b/src/syntax/dependency_graph.h @@ -95,6 +95,11 @@ class DependencyGraph { std::vector find_include_chain(std::uint32_t host_path_id, std::uint32_t target_path_id) const; + /// Every file the graph knows: files with include entries plus files + /// that only appear as include targets. Sorted so callers scan in a + /// deterministic order. Requires build_reverse_map() to be current. + llvm::SmallVector all_files() const; + /// Number of files with include entries. std::size_t file_count() const; diff --git a/tests/integration/features/test_file_tracker.py b/tests/integration/features/test_file_tracker.py new file mode 100644 index 000000000..e1a82b8fe --- /dev/null +++ b/tests/integration/features/test_file_tracker.py @@ -0,0 +1,228 @@ +"""File tracker: each test drives deterministic ticks through the +clice/internal/poll hook (loops disabled). The first workspace tick only +seeds the stat baseline, so tests poll once before mutating the disk.""" + +import asyncio + +from tests.integration.utils import write_cdb +from tests.integration.utils.assertions import ( + assert_has_errors, + assert_no_errors, + get_errors, +) +from tests.integration.utils.wait import ( + MTIME_GRANULARITY, + wait_for_index, + wait_for_recompile, + wait_for_reference, +) +from tests.integration.utils.workspace import get_field + +GATED_MAIN = """\ +#ifndef FEATURE +#error missing FEATURE +#endif +int main() { return 0; } +""" + +HEADER_V1 = """\ +#define VALUE 1 +#define TARGET alpha +inline int alpha() { return 1; } +inline int beta() { return 2; } +""" + +HEADER_V2 = """\ +#define VALUE 2 +#define TARGET beta +inline int alpha() { return 1; } +inline int beta() { return 2; } +""" + + +GATED_LIB = """\ +#ifdef FEATURE +int feature_on() { return 1; } +#else +int feature_off() { return 0; } +#endif +""" + + +async def events_of(client, loop): + return get_field(await client.poll(loop), "events") + + +async def test_cdb_flag_change_recompiles(client, tmp_path): + (tmp_path / "main.cpp").write_text(GATED_MAIN, newline="\n") + write_cdb(tmp_path, ["main.cpp"]) + await client.initialize(tmp_path) + + main_uri = (tmp_path / "main.cpp").as_uri() + await client.open_and_wait(tmp_path / "main.cpp") + assert_has_errors(client, main_uri, "gate must fire without -DFEATURE") + + write_cdb(tmp_path, ["main.cpp"], extra_args=["-DFEATURE"]) + assert await events_of(client, "cdb") == 1 + + await wait_for_recompile(client, main_uri) + assert_no_errors(client, main_uri, "open file must pick up the new flags") + + +async def test_cdb_new_entry_indexed(client, tmp_path): + (tmp_path / "main.cpp").write_text("int main() { return 0; }\n", newline="\n") + write_cdb(tmp_path, ["main.cpp"]) + await client.initialize(tmp_path) + + main_uri = (tmp_path / "main.cpp").as_uri() + await client.open_and_wait(tmp_path / "main.cpp") + + (tmp_path / "lib.cpp").write_text("int lib_entry() { return 1; }\n", newline="\n") + write_cdb(tmp_path, ["main.cpp", "lib.cpp"]) + assert await events_of(client, "cdb") == 1 + + assert await wait_for_index(client, main_uri, "lib_entry"), ( + "file added to the CDB was never indexed" + ) + + +async def test_cdb_removed_entry_recheck(client, tmp_path): + (tmp_path / "header.h").write_text( + "inline int shared() { return 0; }\n", newline="\n" + ) + (tmp_path / "gone.cpp").write_text('#include "header.h"\n', newline="\n") + write_cdb(tmp_path, ["gone.cpp"]) + await client.initialize(tmp_path) + + header_uri = (tmp_path / "header.h").as_uri() + result = await client.query_context(header_uri) + assert get_field(result, "total") >= 1, "gone.cpp must host the header initially" + + write_cdb(tmp_path, []) + assert await events_of(client, "cdb") == 1 + + result = await client.query_context(header_uri) + assert get_field(result, "total") == 0, "removed entry must stop hosting the header" + + +async def test_cdb_appears_after_startup(client, tmp_path): + (tmp_path / "main.cpp").write_text(GATED_MAIN, newline="\n") + (tmp_path / "lib.cpp").write_text("int lib_entry() { return 1; }\n", newline="\n") + await client.initialize(tmp_path) + + main_uri = (tmp_path / "main.cpp").as_uri() + await client.open_and_wait(tmp_path / "main.cpp") + assert_has_errors(client, main_uri, "guessed command cannot define FEATURE") + + # The editor was opened first; cmake runs later. + write_cdb(tmp_path, ["main.cpp", "lib.cpp"], extra_args=["-DFEATURE"]) + assert await events_of(client, "cdb") == 1 + + await wait_for_recompile(client, main_uri) + assert_no_errors(client, main_uri, "open file must switch to the discovered CDB") + assert await wait_for_index(client, main_uri, "lib_entry"), ( + "closed file from the discovered CDB was never indexed" + ) + + +async def test_checkout_updates_workspace(client, tmp_path): + (tmp_path / "header.h").write_text(HEADER_V1, newline="\n") + main_v1 = '#include "header.h"\nstatic_assert(VALUE == 2, "");\nint main() { return 0; }\n' + (tmp_path / "main.cpp").write_text(main_v1, newline="\n") + closed_v1 = '#include "header.h"\nint use_target() { return TARGET(); }\n' + (tmp_path / "closed.cpp").write_text(closed_v1, newline="\n") + write_cdb(tmp_path, ["main.cpp", "closed.cpp"]) + await client.initialize(tmp_path) + + header_uri = (tmp_path / "header.h").as_uri() + main_uri = (tmp_path / "main.cpp").as_uri() + closed_uri = (tmp_path / "closed.cpp").as_uri() + await client.open_and_wait(tmp_path / "main.cpp") + assert_has_errors(client, main_uri, "static_assert must fire against header V1") + assert await wait_for_reference(client, header_uri, 2, 11, closed_uri), ( + "initial index never resolved the closed TU's alpha call" + ) + + assert await events_of(client, "workspace") == 0 # seeding sweep + + # Simulate git checkout: rewrite files on disk, no didSave. + await asyncio.sleep(MTIME_GRANULARITY) + (tmp_path / "header.h").write_text(HEADER_V2, newline="\n") + (tmp_path / "closed.cpp").write_text( + closed_v1 + "int checkout_added() { return 3; }\n", newline="\n" + ) + assert await events_of(client, "workspace") == 2 + + await wait_for_recompile(client, main_uri) + assert_no_errors(client, main_uri, "open file must compile against the new header") + assert await wait_for_reference(client, header_uri, 3, 11, closed_uri), ( + "closed TU was not reindexed against the new header" + ) + assert await wait_for_index(client, main_uri, "checkout_added"), ( + "closed TU's own disk change was not indexed" + ) + + +async def test_touch_emits_no_events(client, tmp_path): + (tmp_path / "header.h").write_text(HEADER_V1, newline="\n") + (tmp_path / "main.cpp").write_text('#include "header.h"\n', newline="\n") + write_cdb(tmp_path, ["main.cpp"]) + await client.initialize(tmp_path) + + assert await events_of(client, "workspace") == 0 # seeding sweep + + # mtime bump, identical bytes: the content-hash check must stay silent. + await asyncio.sleep(MTIME_GRANULARITY) + (tmp_path / "header.h").write_text(HEADER_V1, newline="\n") + assert await events_of(client, "workspace") == 0 + + +async def test_cdb_polling_loop_live(client, tmp_path): + (tmp_path / "main.cpp").write_text(GATED_MAIN, newline="\n") + write_cdb(tmp_path, ["main.cpp"]) + await client.initialize( + tmp_path, initialization_options={"tracker": {"cdb_poll_seconds": 1}} + ) + + main_uri = (tmp_path / "main.cpp").as_uri() + await client.open_and_wait(tmp_path / "main.cpp") + assert_has_errors(client, main_uri) + + write_cdb(tmp_path, ["main.cpp"], extra_args=["-DFEATURE"]) + # No hook: the 1s poll loop needs two stable ticks (settle debounce), + # so poll for the errors to clear instead of trusting one fixed sleep. + # Until the reload lands the hover fast-paths on a clean AST and no + # diagnostics arrive — that round just times out and retries. + for _ in range(30): + await asyncio.sleep(1) + try: + await wait_for_recompile(client, main_uri, timeout=3.0) + except TimeoutError: + continue + if not get_errors(client.diagnostics.get(main_uri, [])): + break + assert_no_errors( + client, main_uri, "the polling loop must reload the CDB on its own" + ) + + +async def test_cdb_flag_change_reindexes_closed(client, tmp_path): + (tmp_path / "main.cpp").write_text("int main() { return 0; }\n", newline="\n") + (tmp_path / "lib.cpp").write_text(GATED_LIB, newline="\n") + write_cdb(tmp_path, ["main.cpp", "lib.cpp"]) + await client.initialize(tmp_path) + + main_uri = (tmp_path / "main.cpp").as_uri() + await client.open_and_wait(tmp_path / "main.cpp") + assert await wait_for_index(client, main_uri, "feature_off"), ( + "closed file was never indexed initially" + ) + + # Only lib.cpp's flags change; its bytes do not. Content-based staleness + # cannot see this — the CDB delta must force the reindex. + write_cdb(tmp_path, ["main.cpp", "lib.cpp"], extra_args=["-DFEATURE"]) + assert await events_of(client, "cdb") == 1 + + assert await wait_for_index(client, main_uri, "feature_on"), ( + "closed file was not reindexed after its flags changed" + ) diff --git a/tests/integration/features/test_header_reindex.py b/tests/integration/features/test_header_reindex.py index 32dc15f17..389eb7f55 100644 --- a/tests/integration/features/test_header_reindex.py +++ b/tests/integration/features/test_header_reindex.py @@ -11,7 +11,11 @@ ) from tests.integration.utils import write_cdb -from tests.integration.utils.wait import MTIME_GRANULARITY +from tests.integration.utils.wait import ( + MTIME_GRANULARITY, + reference_uris, + wait_for_reference, +) HEADER_V1 = """\ #define TARGET alpha @@ -30,19 +34,6 @@ CLOSED_TU = '#include "header.h"\nint use_target() { return TARGET(); }\n' -async def reference_uris(client, uri, line, character): - refs = await client.references_at(uri, line, character, include_declaration=False) - return [ref.uri for ref in (refs or [])] - - -async def wait_for_reference(client, uri, line, character, expected_uri, timeout=30): - for _ in range(timeout): - if expected_uri in await reference_uris(client, uri, line, character): - return True - await asyncio.sleep(1) - return False - - async def test_header_save_reindexes_dependents(client, tmp_path): # newline="\n" keeps the on-disk bytes identical to the didChange text # below: after a save the buffer and the disk must agree, as they do for @@ -82,3 +73,34 @@ async def test_header_save_reindexes_dependents(client, tmp_path): "closed TU was not reindexed after the header save" ) assert closed_uri not in await reference_uris(client, header_uri, 1, 11) + + +async def test_divergent_save_follows_disk(client, tmp_path): + (tmp_path / "header.h").write_text(HEADER_V1, newline="\n") + (tmp_path / "closed.cpp").write_text(CLOSED_TU, newline="\n") + write_cdb(tmp_path, ["closed.cpp"]) + await client.initialize(tmp_path) + + header_uri = (tmp_path / "header.h").as_uri() + closed_uri = (tmp_path / "closed.cpp").as_uri() + await client.open_and_wait(tmp_path / "header.h") + + assert await wait_for_reference(client, header_uri, 1, 11, closed_uri), ( + "initial index never produced the closed TU's alpha reference" + ) + + # A save hook rewrote the file as the save landed: the disk holds V2 + # while the buffer still holds V1 and no didChange is ever sent. + # (alpha/beta keep their positions across versions, so buffer-resolved + # lookups on the open header stay valid.) + await asyncio.sleep(MTIME_GRANULARITY) + (tmp_path / "header.h").write_text(HEADER_V2, newline="\n") + client.text_document_did_save( + DidSaveTextDocumentParams(text_document=TextDocumentIdentifier(uri=header_uri)) + ) + + # Dependents must follow the disk truth, not the pre-save state. + assert await wait_for_reference(client, header_uri, 2, 11, closed_uri), ( + "closed TU was not reindexed against the hook-rewritten disk" + ) + assert closed_uri not in await reference_uris(client, header_uri, 1, 11) diff --git a/tests/integration/utils/client.py b/tests/integration/utils/client.py index d52fe06f3..573f4797e 100644 --- a/tests/integration/utils/client.py +++ b/tests/integration/utils/client.py @@ -111,6 +111,12 @@ async def initialize( project = dict(initialization_options.get("project", {})) project.setdefault("cache_dir", str(workspace / ".clice")) initialization_options["project"] = project + # Disable the stat-polling loops: tests drive ticks deterministically + # through the clice/internal/poll hook instead. + tracker = dict(initialization_options.get("tracker", {})) + tracker.setdefault("cdb_poll_seconds", 0) + tracker.setdefault("workspace_poll_seconds", 0) + initialization_options["tracker"] = tracker params = InitializeParams( capabilities=ClientCapabilities(), @@ -457,3 +463,11 @@ async def switch_context( self.protocol.send_request_async("clice/switchContext", params), timeout=timeout, ) + + async def poll(self, loop: str, *, timeout: float = 60.0): + """Send clice/internal/poll (test hook): run one tracker tick and + apply its effects synchronously. `loop` is "cdb" or "workspace".""" + return await asyncio.wait_for( + self.protocol.send_request_async("clice/internal/poll", {"loop": loop}), + timeout=timeout, + ) diff --git a/tests/integration/utils/wait.py b/tests/integration/utils/wait.py index 06a07779b..b0807eb94 100644 --- a/tests/integration/utils/wait.py +++ b/tests/integration/utils/wait.py @@ -59,3 +59,20 @@ async def wait_for_index( return True await asyncio.sleep(1) return False + + +async def reference_uris(client, uri: str, line: int, character: int) -> list[str]: + """URIs of the references at a position (declaration excluded).""" + refs = await client.references_at(uri, line, character, include_declaration=False) + return [ref.uri for ref in (refs or [])] + + +async def wait_for_reference( + client, uri: str, line: int, character: int, expected_uri: str, timeout: int = 30 +) -> bool: + """Poll references at a position until expected_uri shows up.""" + for _ in range(timeout): + if expected_uri in await reference_uris(client, uri, line, character): + return True + await asyncio.sleep(1) + return False diff --git a/tests/unit/command/cdb_diff_tests.cpp b/tests/unit/command/cdb_diff_tests.cpp new file mode 100644 index 000000000..daf87d45b --- /dev/null +++ b/tests/unit/command/cdb_diff_tests.cpp @@ -0,0 +1,290 @@ +#include +#include + +#include "test/cdb_helper.h" +#include "test/temp_dir.h" +#include "test/test.h" +#include "command/argument_parser.h" +#include "command/command.h" +#include "support/filesystem.h" + +namespace clice::testing { + +namespace { + +namespace ranges = std::ranges; + +/// path_id that `cdb` assigns to a file under the temp root. +std::uint32_t id_of(CompilationDatabase& cdb, TempDir& tmp, llvm::StringRef rel) { + return cdb.intern_path(path::join(tmp.root.str(), rel)); +} + +bool contains(llvm::ArrayRef list, std::uint32_t id) { + return ranges::find(list, id) != list.end(); +} + +/// Overwrite compile_commands.json under the temp root (without loading it). +void write_json(TempDir& tmp, llvm::ArrayRef entries) { + tmp.touch("compile_commands.json", build_cdb_json(entries)); +} + +TEST_SUITE(ReloadDiff) { + +TEST_CASE(AddedEntry) { + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {}} + }); + cdb.load(cdb_path); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {}}, + {tmp.root.str(), "b.cpp", {}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_EQ(diff->added.size(), 1U); + EXPECT_EQ(diff->added[0], id_of(cdb, tmp, "b.cpp")); + EXPECT_TRUE(diff->removed.empty()); + EXPECT_TRUE(diff->changed.empty()); +}; + +TEST_CASE(RemovedEntry) { + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {}}, + {tmp.root.str(), "b.cpp", {}} + }); + cdb.load(cdb_path); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_EQ(diff->removed.size(), 1U); + EXPECT_EQ(diff->removed[0], id_of(cdb, tmp, "b.cpp")); + EXPECT_TRUE(diff->added.empty()); + EXPECT_TRUE(diff->changed.empty()); +}; + +TEST_CASE(ChangedFlag) { + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-DFOO=1"}} + }); + cdb.load(cdb_path); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-DFOO=2"}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_EQ(diff->changed.size(), 1U); + EXPECT_EQ(diff->changed[0], id_of(cdb, tmp, "a.cpp")); + EXPECT_TRUE(diff->added.empty()); + EXPECT_TRUE(diff->removed.empty()); +}; + +TEST_CASE(IdenticalReload) { + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-DFOO=1"}}, + {tmp.root.str(), "b.cpp", {"-Wall"} } + }); + cdb.load(cdb_path); + + auto diff = cdb.reload_and_diff(cdb_path); + EXPECT_TRUE(diff->empty()); +}; + +TEST_CASE(ReorderNoDiff) { + // A file's entries are compared as a set, so shuffling the JSON must not + // register as a change — even when one file owns several entries. + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-DA=1"}}, + {tmp.root.str(), "a.cpp", {"-DB=1"}}, + {tmp.root.str(), "b.cpp", {} } + }); + cdb.load(cdb_path); + + write_json(tmp, + { + {tmp.root.str(), "b.cpp", {} }, + {tmp.root.str(), "a.cpp", {"-DB=1"}}, + {tmp.root.str(), "a.cpp", {"-DA=1"}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + EXPECT_TRUE(diff->empty()); +}; + +TEST_CASE(CodegenChangeIgnored) { + // Entry identity is the Frontend canonical hash, which drops codegen-only + // flags. Swapping one codegen flag for another therefore yields no change. + // (Note: -O* is NOT codegen-only here — it defines __OPTIMIZE__ and is + // kept, so an -O change does count; see OptLevelIsSemantic.) + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-fPIC", "-g"}} + }); + cdb.load(cdb_path); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-fno-omit-frame-pointer", "-flto"}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + EXPECT_TRUE(diff->empty()); +}; + +TEST_CASE(OptLevelIsSemantic) { + // Anchors that -O* is semantic (defines __OPTIMIZE__), not codegen-only: + // changing the optimization level must be reported as a change. + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-O2"}} + }); + cdb.load(cdb_path); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-O3"}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_EQ(diff->changed.size(), 1U); + EXPECT_EQ(diff->changed[0], id_of(cdb, tmp, "a.cpp")); + EXPECT_TRUE(diff->added.empty()); + EXPECT_TRUE(diff->removed.empty()); +}; + +TEST_CASE(MultiEntryOneChanged) { + // A file with several entries appears in `changed` once, not per entry. + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-DA=1"}}, + {tmp.root.str(), "a.cpp", {"-DB=1"}} + }); + cdb.load(cdb_path); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {"-DA=2"}}, + {tmp.root.str(), "a.cpp", {"-DB=1"}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_EQ(diff->changed.size(), 1U); + EXPECT_EQ(diff->changed[0], id_of(cdb, tmp, "a.cpp")); + EXPECT_TRUE(diff->added.empty()); + EXPECT_TRUE(diff->removed.empty()); +}; + +TEST_CASE(FirstLoadAllAdded) { + // Discovering a CDB for the first time: every file is `added`. + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {}}, + {tmp.root.str(), "b.cpp", {}} + }); + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_EQ(diff->added.size(), 2U); + EXPECT_TRUE(contains(diff->added, id_of(cdb, tmp, "a.cpp"))); + EXPECT_TRUE(contains(diff->added, id_of(cdb, tmp, "b.cpp"))); + EXPECT_TRUE(diff->removed.empty()); + EXPECT_TRUE(diff->changed.empty()); +}; + +TEST_CASE(CorruptKeepsEntries) { + // A half-written / corrupt CDB must leave the loaded entries intact and + // signal failure so the caller retries instead of seeing "no change". + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {}} + }); + cdb.load(cdb_path); + ASSERT_TRUE(cdb.has_entry(path::join(tmp.root.str(), "a.cpp"))); + + tmp.touch("compile_commands.json", "<<< corrupted compile_commands.json >>>"); + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_FALSE(diff.has_value()); + EXPECT_TRUE(cdb.has_entry(path::join(tmp.root.str(), "a.cpp"))); + + auto results = cdb.lookup(path::join(tmp.root.str(), "a.cpp"), {.inject_resource_dir = false}); + ASSERT_EQ(results.size(), 1U); + EXPECT_TRUE(llvm::StringRef(print_argv(results.front().to_argv())).contains("-std=c++20")); +}; + +TEST_CASE(MissingFileFails) { + // An unreadable file (deleted, or still locked by the generator) is a + // failure, not an empty database: entries survive and the caller retries. + TempDir tmp; + CompilationDatabase cdb; + auto cdb_path = tmp.path("compile_commands.json"); + + write_json(tmp, + { + {tmp.root.str(), "a.cpp", {}} + }); + cdb.load(cdb_path); + fs::remove_all(cdb_path); + + auto diff = cdb.reload_and_diff(cdb_path); + + ASSERT_FALSE(diff.has_value()); + EXPECT_TRUE(cdb.has_entry(path::join(tmp.root.str(), "a.cpp"))); +}; + +}; // TEST_SUITE(ReloadDiff) + +} // namespace + +} // namespace clice::testing diff --git a/tests/unit/command/command_tests.cpp b/tests/unit/command/command_tests.cpp index b55def749..9077b305b 100644 --- a/tests/unit/command/command_tests.cpp +++ b/tests/unit/command/command_tests.cpp @@ -459,7 +459,7 @@ std::size_t load_json(CompilationDatabase& database, llvm::StringRef json) { return 0; out << json; } - auto count = database.load(*path); + auto count = database.load(*path).value_or(0); llvm::sys::fs::remove(*path); return count; } diff --git a/tests/unit/server/file_tracker_tests.cpp b/tests/unit/server/file_tracker_tests.cpp new file mode 100644 index 000000000..e5aa907f4 --- /dev/null +++ b/tests/unit/server/file_tracker_tests.cpp @@ -0,0 +1,205 @@ +#include "test/cdb_helper.h" +#include "test/temp_dir.h" +#include "test/test.h" +#include "server/workspace/file_tracker.h" + +namespace clice::testing { +namespace { + +TEST_SUITE(FileTracker) { + +TEST_CASE(CDBTickDebounces) { + TempDir tmp; + tmp.touch("main.cpp", R"(int main() {})"); + tmp.touch("lib.cpp", R"(int lib() { return 1; })"); + + Workspace workspace; + SessionStore store; + write_cdb(tmp, + workspace.cdb, + build_cdb_json({ + {tmp.root, tmp.path("main.cpp"), {}} + })); + FileTracker tracker(workspace, store, tmp.root.str().str()); + + // Rewrite with one more entry: the first tick only records the pending + // stamp, the second sees it stable and reloads. + tmp.touch("compile_commands.json", + build_cdb_json({ + {tmp.root, tmp.path("main.cpp"), {}}, + {tmp.root, tmp.path("lib.cpp"), {}} + })); + ASSERT_TRUE(tracker.tick_cdb().empty()); + + auto events = tracker.tick_cdb(); + ASSERT_EQ(events.size(), 1u); + ASSERT_EQ(events[0].kind, FileEvent::Kind::CDBChanged); + auto lib_id = workspace.path_pool.intern(tmp.path("lib.cpp")); + ASSERT_EQ(events[0].cdb.added, llvm::SmallVector{lib_id}); + ASSERT_TRUE(events[0].cdb.removed.empty()); + + // Settled: further ticks are quiet. + ASSERT_TRUE(tracker.tick_cdb().empty()); +} + +TEST_CASE(CDBTickForceImmediate) { + TempDir tmp; + tmp.touch("main.cpp", R"(int main() {})"); + + Workspace workspace; + SessionStore store; + write_cdb(tmp, + workspace.cdb, + build_cdb_json({ + {tmp.root, tmp.path("main.cpp"), {}} + })); + FileTracker tracker(workspace, store, tmp.root.str().str()); + + tmp.touch("compile_commands.json", + build_cdb_json({ + {tmp.root, tmp.path("main.cpp"), {"-DFOO"}} + })); + auto events = tracker.tick_cdb(/*force=*/true); + ASSERT_EQ(events.size(), 1u); + auto main_id = workspace.path_pool.intern(tmp.path("main.cpp")); + ASSERT_EQ(events[0].cdb.changed, llvm::SmallVector{main_id}); +} + +TEST_CASE(CDBTickDiscoversLate) { + TempDir tmp; + tmp.touch("main.cpp", R"(int main() {})"); + + Workspace workspace; + SessionStore store; + // No compile_commands.json at construction time. + FileTracker tracker(workspace, store, tmp.root.str().str()); + ASSERT_TRUE(tracker.tick_cdb(/*force=*/true).empty()); + + tmp.touch("compile_commands.json", + build_cdb_json({ + {tmp.root, tmp.path("main.cpp"), {}} + })); + auto events = tracker.tick_cdb(/*force=*/true); + ASSERT_EQ(events.size(), 1u); + auto main_id = workspace.path_pool.intern(tmp.path("main.cpp")); + ASSERT_EQ(events[0].cdb.added, llvm::SmallVector{main_id}); +} + +TEST_CASE(CDBTickDeleteRecreate) { + TempDir tmp; + tmp.touch("main.cpp", R"(int main() {})"); + + Workspace workspace; + SessionStore store; + write_cdb(tmp, + workspace.cdb, + build_cdb_json({ + {tmp.root, tmp.path("main.cpp"), {}} + })); + FileTracker tracker(workspace, store, tmp.root.str().str()); + + // Deletion (mid-regeneration): keep serving the loaded entries. + fs::remove_all(tmp.path("compile_commands.json")); + ASSERT_TRUE(tracker.tick_cdb(/*force=*/true).empty()); + + // The rewrite lands as a normal change once the file is back. + tmp.touch("compile_commands.json", + build_cdb_json({ + {tmp.root, tmp.path("main.cpp"), {"-DFOO"}} + })); + auto events = tracker.tick_cdb(/*force=*/true); + ASSERT_EQ(events.size(), 1u); + auto main_id = workspace.path_pool.intern(tmp.path("main.cpp")); + ASSERT_EQ(events[0].cdb.changed, llvm::SmallVector{main_id}); +} + +TEST_CASE(WorkspaceTickStateMachine) { + TempDir tmp; + tmp.touch("header.h", R"(int x = 1;)"); + + kota::event_loop loop; + Workspace workspace; + SessionStore store; + auto tu = workspace.path_pool.intern(tmp.path("main.cpp")); + auto header = workspace.path_pool.intern(tmp.path("header.h")); + workspace.dep_graph.set_includes(tu, 0, {header}); + workspace.dep_graph.build_reverse_map(); + FileTracker tracker(workspace, store, tmp.root.str().str()); + + auto body = [&]() -> kota::task<> { + // First sweep seeds the baseline silently, even though main.cpp is + // missing on disk. + auto seeded = co_await tracker.tick_workspace(); + EXPECT_TRUE(seeded.empty()); + + // Content change is confirmed by hash and reported once. The new + // content has a different LENGTH on purpose: back-to-back writes + // can land within one mtime tick (observed on Windows CI), and only + // the size change keeps the (mtime, size) fast path deterministic. + // (ASSERT_* expands to `return` and cannot be used in coroutines.) + tmp.touch("header.h", R"(int x = 2222;)"); + auto changed = co_await tracker.tick_workspace(); + EXPECT_EQ(changed.size(), 1u); + if(changed.size() == 1) { + EXPECT_EQ(changed[0].kind, FileEvent::Kind::DiskChanged); + EXPECT_EQ(changed[0].path_id, header); + } + + // Touch: mtime may bump, identical bytes — silent either way. + tmp.touch("header.h", R"(int x = 2222;)"); + auto touched = co_await tracker.tick_workspace(); + EXPECT_TRUE(touched.empty()); + + // Removal reported once, then quiet while missing. + fs::remove_all(tmp.path("header.h")); + auto removed = co_await tracker.tick_workspace(); + EXPECT_EQ(removed.size(), 1u); + if(removed.size() == 1) { + EXPECT_EQ(removed[0].kind, FileEvent::Kind::DiskRemoved); + EXPECT_EQ(removed[0].path_id, header); + } + auto still_removed = co_await tracker.tick_workspace(); + EXPECT_TRUE(still_removed.empty()); + + // Reappearance counts as a disk change. + tmp.touch("header.h", R"(int x = 3;)"); + auto reborn = co_await tracker.tick_workspace(); + EXPECT_EQ(reborn.size(), 1u); + if(reborn.size() == 1) { + EXPECT_EQ(reborn[0].kind, FileEvent::Kind::DiskChanged); + } + }; + auto task = body(); + loop.schedule(task); + loop.run(); +} + +TEST_CASE(WorkspaceTickSkipsOpen) { + TempDir tmp; + tmp.touch("header.h", R"(int x = 1;)"); + + kota::event_loop loop; + Workspace workspace; + SessionStore store; + auto header = workspace.path_pool.intern(tmp.path("header.h")); + workspace.dep_graph.set_includes(header, 0, {}); + workspace.dep_graph.build_reverse_map(); + store.open(header); + FileTracker tracker(workspace, store, tmp.root.str().str()); + + auto body = [&]() -> kota::task<> { + EXPECT_TRUE((co_await tracker.tick_workspace()).empty()); + + // The open buffer is the truth: its disk changes are not tracked. + tmp.touch("header.h", R"(int x = 2;)"); + EXPECT_TRUE((co_await tracker.tick_workspace()).empty()); + }; + auto task = body(); + loop.schedule(task); + loop.run(); +} + +}; // TEST_SUITE(FileTracker) + +} // namespace +} // namespace clice::testing diff --git a/tests/unit/server/invalidator_tests.cpp b/tests/unit/server/invalidator_tests.cpp index 92f177ef7..e837ac754 100644 --- a/tests/unit/server/invalidator_tests.cpp +++ b/tests/unit/server/invalidator_tests.cpp @@ -1,3 +1,4 @@ +#include "test/cdb_helper.h" #include "test/temp_dir.h" #include "test/test.h" #include "server/compiler/compile_graph.h" @@ -42,10 +43,14 @@ TEST_CASE(SaveResetsTrialOnly) { Workspace workspace; SessionStore store; auto saved = workspace.path_pool.intern("/proj/a.h"); - store.open(saved); + auto session = store.open(saved); + store.apply_open(*session, "int x;", 1); ContextResolver resolver(workspace); - Invalidator invalidator(workspace, store, resolver); + // A plain save: the disk holds exactly what the buffer holds. + Invalidator invalidator(workspace, store, resolver, [](llvm::StringRef) { + return std::optional{"int x;"}; + }); auto dirty = invalidator.apply(FileEvent::buffer_saved(saved)); // The saved file itself is not stale — its buffer was already current — @@ -242,6 +247,353 @@ TEST_CASE(BatchSavesDeduplicate) { ASSERT_EQ(dirty.reset_trial, llvm::SmallVector{saved}); } +TEST_CASE(SaveDivergentDiskDirties) { + Workspace workspace; + SessionStore store; + auto saved = workspace.path_pool.intern("/proj/a.h"); + auto session = store.open(saved); + store.apply_open(*session, "int buffer;", 1); + + ContextResolver resolver(workspace); + // A save hook rewrote the file as it landed: disk != buffer. + Invalidator invalidator(workspace, store, resolver, [](llvm::StringRef) { + return std::optional{"int disk;"}; + }); + auto dirty = invalidator.apply(FileEvent::buffer_saved(saved)); + + // The session recompiles so its deps snapshot re-validates against the + // rewritten disk instead of describing a state that no longer exists. + ASSERT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{saved}); +} + +TEST_CASE(SaveUnreadableDiskDirties) { + Workspace workspace; + SessionStore store; + auto saved = workspace.path_pool.intern("/proj/a.h"); + auto session = store.open(saved); + store.apply_open(*session, "int buffer;", 1); + + ContextResolver resolver(workspace); + // The file cannot be read back after the save: the disk state is + // unknown, which is treated as divergent (conservative). + Invalidator invalidator(workspace, store, resolver, [](llvm::StringRef) { + return std::optional{}; + }); + auto dirty = invalidator.apply(FileEvent::buffer_saved(saved)); + + ASSERT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{saved}); +} + +TEST_CASE(DiskChangeOpenMarksDirty) { + Workspace workspace; + SessionStore store; + auto open_file = workspace.path_pool.intern("/proj/a.cpp"); + store.open(open_file); + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + auto dirty = invalidator.apply(FileEvent::disk_changed(open_file)); + + // The buffer is the truth for an open file: recompile so the next + // compile's deps validation judges the disk change, but no rescan and + // no cascade. + ASSERT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{open_file}); + ASSERT_TRUE(dirty.enqueue_reindex.empty()); + ASSERT_TRUE(dirty.reset_trial.empty()); + ASSERT_FALSE(dirty.recheck_contexts); +} + +TEST_CASE(DiskChangeClosedCascades) { + Workspace workspace; + SessionStore store; + auto header = workspace.path_pool.intern("/proj/h.h"); + auto open_tu = workspace.path_pool.intern("/proj/a.cpp"); + auto closed_tu = workspace.path_pool.intern("/proj/b.cpp"); + workspace.dep_graph.set_includes(open_tu, 0, {header}); + workspace.dep_graph.set_includes(closed_tu, 0, {header}); + workspace.dep_graph.build_reverse_map(); + store.open(open_tu); + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + auto dirty = invalidator.apply(FileEvent::disk_changed(header)); + + // A closed file's disk change cascades exactly like a save, plus the + // file's own stale shard is refreshed. + ASSERT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{open_tu}); + llvm::SmallVector reindexed{header, closed_tu}; + llvm::sort(reindexed); + ASSERT_EQ(dirty.enqueue_reindex, reindexed); + ASSERT_EQ(dirty.reset_trial, llvm::SmallVector{header}); + ASSERT_TRUE(dirty.recheck_contexts); + ASSERT_TRUE(dirty.reschedule_indexing); +} + +TEST_CASE(DiskRemovedScrubsSourceRole) { + Workspace workspace; + SessionStore store; + auto header = workspace.path_pool.intern("/proj/h.h"); + auto removed_tu = workspace.path_pool.intern("/proj/gone.cpp"); + auto other_tu = workspace.path_pool.intern("/proj/kept.cpp"); + workspace.dep_graph.set_includes(removed_tu, 0, {header}); + workspace.dep_graph.set_includes(other_tu, 0, {header}); + workspace.dep_graph.build_reverse_map(); + auto epoch = workspace.context_epoch; + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + auto dirty = invalidator.apply(FileEvent::disk_removed(removed_tu)); + + // The removed file stops being an includer (and thus a host-source + // candidate); surviving includers are untouched, shards are kept. + ASSERT_EQ(workspace.dep_graph.get_includers(header), llvm::ArrayRef{other_tu}); + ASSERT_TRUE(workspace.dep_graph.get_all_includes(removed_tu).empty()); + ASSERT_TRUE(dirty.recheck_contexts); + ASSERT_TRUE(dirty.enqueue_reindex.empty()); + ASSERT_TRUE(dirty.mark_ast_dirty.empty()); + ASSERT_EQ(workspace.context_epoch, epoch + 1); +} + +TEST_CASE(CDBAddedScansAndEnqueues) { + TempDir tmp; + tmp.touch("inc/header.h", R"(int x = 1;)"); + tmp.touch("src/main.cpp", R"(#include "header.h")"); + + Workspace workspace; + SessionStore store; + auto json = build_cdb_json({ + {tmp.root, tmp.path("src/main.cpp"), {"-I", tmp.path("inc")}} + }); + write_cdb(tmp, workspace.cdb, json); + auto main_id = workspace.path_pool.intern(tmp.path("src/main.cpp")); + auto header_id = workspace.path_pool.intern(tmp.path("inc/header.h")); + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + FileEvent::CDBDelta delta; + delta.added = {main_id}; + auto dirty = invalidator.apply(FileEvent::cdb_changed(std::move(delta))); + + // The rescan resolved the new entry's includes; the new file reindexes. + ASSERT_EQ(workspace.dep_graph.get_includers(header_id), llvm::ArrayRef{main_id}); + ASSERT_EQ(dirty.enqueue_reindex, llvm::SmallVector{main_id}); + ASSERT_TRUE(dirty.recheck_contexts); + ASSERT_TRUE(dirty.ensure_compile_graph); +} + +TEST_CASE(CDBChangedSplitsOpenClosed) { + TempDir tmp; + tmp.touch("a.cpp", R"(int a;)"); + tmp.touch("b.cpp", R"(int b;)"); + + Workspace workspace; + SessionStore store; + auto json = build_cdb_json({ + {tmp.root, tmp.path("a.cpp"), {}}, + {tmp.root, tmp.path("b.cpp"), {}} + }); + write_cdb(tmp, workspace.cdb, json); + auto open_id = workspace.path_pool.intern(tmp.path("a.cpp")); + auto closed_id = workspace.path_pool.intern(tmp.path("b.cpp")); + store.open(open_id); + workspace.merged_indices[open_id]; + workspace.merged_indices[closed_id]; + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + FileEvent::CDBDelta delta; + delta.changed = {open_id, closed_id}; + auto dirty = invalidator.apply(FileEvent::cdb_changed(std::move(delta))); + + // Flag changes recompile open files and reindex closed ones; the + // pull-side cache keys (canonical flags) miss on their own. + ASSERT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{open_id}); + ASSERT_EQ(dirty.enqueue_reindex, llvm::SmallVector{closed_id}); + ASSERT_TRUE(dirty.recheck_contexts); + + // The closed file's shard was built under the old command and looks + // fresh to content-only validation: it must be evicted so the queued + // reindex is not filtered out. The open file's shard stays (its next + // compile owns the refresh). + ASSERT_EQ(workspace.merged_indices.count(closed_id), 0u); + ASSERT_EQ(workspace.merged_indices.count(open_id), 1u); +} + +TEST_CASE(CDBAddedOpenMarksDirty) { + Workspace workspace; + SessionStore store; + auto file = workspace.path_pool.intern("/proj/a.cpp"); + store.open(file); + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + FileEvent::CDBDelta delta; + delta.added = {file}; + auto dirty = invalidator.apply(FileEvent::cdb_changed(std::move(delta))); + + // The open file gained its first real entry: drop the guessed command + // it was compiled with instead of queueing a background reindex. + ASSERT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{file}); + ASSERT_TRUE(dirty.enqueue_reindex.empty()); +} + +TEST_CASE(CDBChangedDropsHostedContext) { + Workspace workspace; + SessionStore store; + auto host = workspace.path_pool.intern("/proj/host.cpp"); + auto open_header = workspace.path_pool.intern("/proj/open.h"); + auto closed_header = workspace.path_pool.intern("/proj/closed.h"); + auto other_header = workspace.path_pool.intern("/proj/other.h"); + store.open(open_header); + workspace.merged_indices[closed_header]; + + ContextResolver resolver(workspace); + resolver.header_contexts[open_header].host_path_id = host; + resolver.header_contexts[closed_header].host_path_id = host; + resolver.header_contexts[other_header].host_path_id = no_path_id; + Invalidator invalidator(workspace, store, resolver); + FileEvent::CDBDelta delta; + delta.changed = {host}; + auto dirty = invalidator.apply(FileEvent::cdb_changed(std::move(delta))); + + // Headers borrowing the changed entry re-resolve their context; the + // open one recompiles, the closed one loses its stale shard and + // reindexes. Unrelated contexts are untouched. + llvm::SmallVector dropped{open_header, closed_header}; + llvm::sort(dropped); + ASSERT_EQ(dirty.drop_context, dropped); + ASSERT_TRUE(llvm::is_contained(dirty.mark_ast_dirty, open_header)); + ASSERT_TRUE(llvm::is_contained(dirty.enqueue_reindex, closed_header)); + ASSERT_EQ(workspace.merged_indices.count(closed_header), 0u); +} + +TEST_CASE(CDBChangedCascadesModule) { + kota::event_loop loop; + Workspace workspace; + SessionStore store; + auto mod = workspace.path_pool.intern("/proj/m.cppm"); + auto open_user = workspace.path_pool.intern("/proj/open_user.cppm"); + auto closed_user = workspace.path_pool.intern("/proj/closed_user.cppm"); + + llvm::DenseMap> deps; + deps[open_user] = {mod}; + deps[closed_user] = {mod}; + workspace.compile_graph = std::make_unique( + loop, + [](std::uint32_t) -> kota::task { co_return true; }, + [deps = std::move(deps)](std::uint32_t id) -> llvm::SmallVector { + auto it = deps.find(id); + return it != deps.end() ? it->second : llvm::SmallVector{}; + }); + + store.open(open_user); + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + + auto body = [&]() -> kota::task<> { + co_await workspace.compile_graph->compile(open_user); + co_await workspace.compile_graph->compile(closed_user); + + FileEvent::CDBDelta delta; + delta.changed = {mod}; + auto dirty = invalidator.apply(FileEvent::cdb_changed(std::move(delta))); + + // A module unit's flag change cascades through the compile graph + // exactly like a content change: importers' PCMs went stale. + EXPECT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{open_user}); + llvm::SmallVector reindexed{mod, closed_user}; + llvm::sort(reindexed); + EXPECT_EQ(dirty.enqueue_reindex, reindexed); + + co_await workspace.compile_graph->shutdown(); + }; + auto task = body(); + loop.schedule(task); + loop.run(); +} + +TEST_CASE(DiskRemovedReindexesIncluders) { + Workspace workspace; + SessionStore store; + auto header = workspace.path_pool.intern("/proj/h.h"); + auto open_tu = workspace.path_pool.intern("/proj/a.cpp"); + auto closed_tu = workspace.path_pool.intern("/proj/b.cpp"); + workspace.dep_graph.set_includes(open_tu, 0, {header}); + workspace.dep_graph.set_includes(closed_tu, 0, {header}); + workspace.dep_graph.build_reverse_map(); + store.open(open_tu); + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + auto dirty = invalidator.apply(FileEvent::disk_removed(header)); + + // Dependents now compile against a missing include: open ones + // recompile, closed ones reindex. + ASSERT_EQ(dirty.mark_ast_dirty, llvm::SmallVector{open_tu}); + ASSERT_EQ(dirty.enqueue_reindex, llvm::SmallVector{closed_tu}); + ASSERT_TRUE(dirty.recheck_contexts); +} + +TEST_CASE(CDBRemovedDropsSourceRole) { + TempDir tmp; + tmp.touch("inc/h.h", R"(int x;)"); + tmp.touch("kept.cpp", R"(#include "inc/h.h")"); + + Workspace workspace; + SessionStore store; + // The pre-reload graph still shows gone.cpp as an includer; the CDB has + // already been reloaded without it. + auto gone_id = workspace.path_pool.intern(tmp.path("gone.cpp")); + auto header_id = workspace.path_pool.intern(tmp.path("inc/h.h")); + workspace.dep_graph.set_includes(gone_id, 0, {header_id}); + workspace.dep_graph.build_reverse_map(); + auto json = build_cdb_json({ + {tmp.root, tmp.path("kept.cpp"), {}} + }); + write_cdb(tmp, workspace.cdb, json); + auto kept_id = workspace.path_pool.intern(tmp.path("kept.cpp")); + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + FileEvent::CDBDelta delta; + delta.removed = {gone_id}; + auto dirty = invalidator.apply(FileEvent::cdb_changed(std::move(delta))); + + // The rebuild resolves includes from the surviving entries only. + ASSERT_TRUE(workspace.dep_graph.get_all_includes(gone_id).empty()); + ASSERT_EQ(workspace.dep_graph.get_includers(header_id), llvm::ArrayRef{kept_id}); + ASSERT_TRUE(dirty.recheck_contexts); +} + +TEST_CASE(CDBEmptyDeltaNoEffects) { + Workspace workspace; + SessionStore store; + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + auto dirty = invalidator.apply(FileEvent::cdb_changed({})); + + ASSERT_TRUE(dirty.empty()); +} + +TEST_CASE(BatchDiskEventsDeduplicate) { + Workspace workspace; + SessionStore store; + auto first = workspace.path_pool.intern("/proj/a.h"); + auto second = workspace.path_pool.intern("/proj/b.h"); + + ContextResolver resolver(workspace); + Invalidator invalidator(workspace, store, resolver); + FileEvent events[] = {FileEvent::disk_changed(first), + FileEvent::disk_changed(first), + FileEvent::disk_changed(second)}; + auto dirty = invalidator.apply(events); + + llvm::SmallVector expected{first, second}; + llvm::sort(expected); + ASSERT_EQ(dirty.enqueue_reindex, expected); +} + }; // TEST_SUITE(Invalidator) TEST_SUITE(DropOrphanedChoices) {