refactor(index): unified byte-identity format, zero-copy readers

benchmarks/README.md

@@ -70,10 +70,10 @@ frontend-internal breakdown (preprocessing, parsing, Sema, PCH
 deserialization) that wall-clock stage timing cannot separate.
 
 `--log-level info` additionally surfaces the `[perf:index_detail]` lines
-from inside the index stages: semantics-table build vs projection vs
-finishing within `TUIndex::build`, and the path-rekeying copy vs the
-flatbuffers pack within `serialize`. The same lines appear in worker logs
-of a real session, so production runs decompose identically.
+from inside the index stage: semantics-table build vs projection vs
+finishing vs per-file blob encoding vs the envelope pack within
+`build_tu_index`. The same lines appear in worker logs of a real session,
+so production runs decompose identically.
 
 E2E scenarios, clice vs clangd:
 

benchmarks/index_stats_benchmark.cpp

@@ -1,14 +1,14 @@
 /// In-process measurement probe for the on-disk index redesign. Compiles
 /// every TU in a compilation database exactly like the background-index
-/// worker does (full parse without PCH + TUIndex::build), serializes the
+/// worker does (full parse without PCH + envelope build), measures the
 /// index the way production consumes it, then walks the resulting structures
 /// and accumulates the distributions the new "merged blob" format needs to
 /// pick its column tiers.
 ///
 /// Compiles run on a few worker threads, each accumulating into its own
 /// Stats; the only shared state is a per-(path, variant-hash) registry
 /// deciding which thread walks a distinct variant's rows — touched once per
-/// FileIndex, never per row — so "per distinct variant" populations are not
+/// file section, never per row — so "per distinct variant" populations are not
 /// double-counted across threads. Accumulators merge after join.
 ///
 /// This is measurement scratch code: it favours being obvious over being
@@ -189,10 +189,10 @@ struct Hist {
 /// Per source-file aggregates, keyed by path string and folded across every
 /// TU that touched the file.
 struct PathAgg {
-    /// Distinct FileIndex rows hashes this thread claimed for this path;
+    /// Distinct section blob hashes this thread claimed for this path;
     /// claims are globally unique, so the merged union is the file's M.
     std::set<std::uint64_t> variants;
-    /// Number of TU contributions (one FileIndex per TU per path) — N.
+    /// Number of TU contributions (one section per TU per path) — N.
     std::uint32_t contributions = 0;
 
     bool size_probed = false;
@@ -220,7 +220,7 @@ struct DirAgg {
 };
 
 /// Arbitrates which thread accumulates a distinct (path, variant) exactly
-/// once. Touched once per FileIndex per TU — a coarse coordination point,
+/// once. Touched once per section per TU — a coarse coordination point,
 /// not the per-row hot path.
 struct VariantRegistry {
     std::mutex mutex;
@@ -256,7 +256,14 @@ struct Stats {
     std::uint64_t indexed = 0;
     std::uint64_t had_diagnostics = 0;
 
-    void add_file_index(index::FileIndex& fi, llvm::StringRef path) {
+    /// One file's rows decoded back out of its envelope section.
+    struct DecodedRows {
+        std::vector<index::Occurrence> occurrences;
+        llvm::DenseMap<index::SymbolHash, std::vector<index::Relation>> relations{};
+    };
+
+    /// `hash` is the section's blob hash — the variant's byte identity.
+    void add_file_index(const DecodedRows& fi, llvm::StringRef path, std::uint64_t hash) {
         if(fi.occurrences.empty() && fi.relations.empty()) {
             return;
         }
@@ -272,7 +279,6 @@ struct Stats {
         }
         agg.contributions += 1;
 
-        auto hash = fi.rows_hash();
         bool inserted = registry->try_claim(path, hash);
         if(inserted) {
             agg.variants.insert(hash);
@@ -341,16 +347,16 @@ struct Stats {
         }
     }
 
-    void add_directives(index::TUIndex& tu) {
-        auto& graph = tu.graph;
-        if(graph.paths.empty()) {
+    void add_directives(const index::TUIndex& tu) {
+        if(tu.path_count() == 0) {
             return;
         }
-        auto root_path_id = static_cast<std::uint32_t>(graph.paths.size() - 1);
+        auto root_path_id = tu.path_count() - 1;
 
         // Outgoing edges keyed by parent location index (-1 = TU root).
         llvm::DenseMap<std::int64_t, std::vector<std::pair<std::uint32_t, std::uint32_t>>> outgoing;
-        for(auto& loc: graph.locations) {
+        for(std::uint32_t i = 0; i < tu.location_count(); i += 1) {
+            auto loc = tu.location(i);
             std::int64_t parent = loc.include == static_cast<std::uint32_t>(-1)
                                       ? -1
                                       : static_cast<std::int64_t>(loc.include);
@@ -360,30 +366,28 @@ struct Stats {
         llvm::StringMap<llvm::DenseSet<std::uint64_t>> local;
         for(auto& [parent, list]: outgoing) {
             std::uint32_t parent_path_id =
-                parent < 0 ? root_path_id : graph.locations[parent].path_id;
-            if(parent_path_id >= graph.paths.size()) {
+                parent < 0 ? root_path_id : tu.location(static_cast<std::uint32_t>(parent)).path_id;
+            if(parent_path_id >= tu.path_count()) {
                 continue;
             }
-            std::uint64_t parent_hash =
-                parent_path_id < graph.path_hashes.size() ? graph.path_hashes[parent_path_id] : 0;
+            std::uint64_t parent_hash = tu.path_hash(parent_path_id);
 
             std::ranges::sort(list, [&](auto& a, auto& b) {
                 if(a.first != b.first) {
                     return a.first < b.first;
                 }
-                return graph.paths[a.second] < graph.paths[b.second];
+                return tu.path(a.second) < tu.path(b.second);
             });
 
             std::string shape;
             for(auto& [line, child_path_id]: list) {
                 shape += std::format("{},{}\n",
                                      line,
-                                     child_path_id < graph.paths.size()
-                                         ? llvm::StringRef(graph.paths[child_path_id])
-                                         : llvm::StringRef());
+                                     child_path_id < tu.path_count() ? tu.path(child_path_id)
+                                                                     : llvm::StringRef());
             }
 
-            auto key = std::format("{}#{:016x}", graph.paths[parent_path_id], parent_hash);
+            auto key = std::format("{}#{:016x}", tu.path(parent_path_id), parent_hash);
             local[key].insert(llvm::xxh3_64bits(shape));
         }
 
@@ -396,28 +400,31 @@ struct Stats {
         }
     }
 
-    void add_tu(index::TUIndex& tu) {
+    void add_tu(const index::TUIndex& tu) {
         indexed += 1;
 
-        for(auto& [hash, symbol]: tu.symbols) {
-            auto scope = static_cast<std::uint8_t>(symbol.scope);
-            if(scope < scope_n.size()) {
-                scope_n[scope] += 1;
-            }
-            scope_map.try_emplace(hash, symbol.scope);
-        }
+        tu.iterate_symbols(
+            [&](index::SymbolHash hash, const index::SymbolIdentity& symbol, llvm::StringRef) {
+                auto scope = static_cast<std::uint8_t>(symbol.scope);
+                if(scope < scope_n.size()) {
+                    scope_n[scope] += 1;
+                }
+                scope_map.try_emplace(hash, symbol.scope);
+                return true;
+            });
 
-        if(!tu.graph.paths.empty()) {
-            add_file_index(tu.main_file_index, tu.graph.paths.back());
-        }
-        // Multiple FileIDs can share a path id (repeated header contexts);
-        // last-wins, like the wire sections.
-        llvm::DenseMap<std::uint32_t, index::FileIndex*> by_path;
-        for(auto& [fid, fi]: tu.file_indices) {
-            by_path[tu.graph.path_id(fid)] = &fi;
-        }
-        for(auto& [path_id, fi]: by_path) {
-            add_file_index(*fi, tu.graph.paths[path_id]);
+        for(std::uint32_t i = 0; i < tu.section_count(); i += 1) {
+            const auto& shard = tu.shard_of(tu.section_path(i));
+            DecodedRows rows;
+            shard.for_each_occurrence([&](const index::Occurrence& occurrence) {
+                rows.occurrences.push_back(occurrence);
+                return true;
+            });
+            shard.for_each_relation([&](index::SymbolHash hash, const index::Relation& relation) {
+                rows.relations[hash].push_back(relation);
+                return true;
+            });
+            add_file_index(rows, tu.path(tu.section_path(i)), tu.section_hash(i));
         }
 
         add_directives(tu);
@@ -1178,14 +1185,10 @@ int main(int argc, const char** argv) {
                 stats.had_diagnostics += 1;
             }
 
-            auto tu_index = index::TUIndex::build(unit);
-
-            llvm::SmallString<0> buffer;
-            llvm::raw_svector_ostream os(buffer);
-            tu_index.serialize(os);
-            stats.wire_sizes.add(buffer.size());
+            auto envelope = index::build_tu_index(unit);
+            stats.wire_sizes.add(envelope.size());
 
-            stats.add_tu(tu_index);
+            stats.add_tu(index::TUIndex::from_bytes(envelope));
             finish("");
         }
     };

benchmarks/pipeline_benchmark.cpp

@@ -6,12 +6,12 @@
 ///   read               source file I/O
 ///   preprocess         PreprocessOnlyAction, TokenBuffer off
 ///   preprocess_tokens  PreprocessOnlyAction, TokenBuffer on (delta = TokenBuffer cost)
-///   parse              full parse without PCH + TUIndex build + serialize
-///                      (the background-index worker shape)
-///   pch_build          preamble PCH build + preamble index/state blob incl.
-///                      disk writes (first didOpen shape)
-///   parse_pch          full parse over the PCH + interactive index build +
-///                      serialize (the didChange shape)
+///   parse              full parse without PCH + envelope build (the
+///                      background-index worker shape)
+///   pch_build          preamble PCH build + preamble envelope incl. disk
+///                      writes (first didOpen shape)
+///   parse_pch          full parse over the PCH + interactive envelope
+///                      build (the didChange shape)
 ///
 /// Usage:
 ///   pipeline_benchmark [OPTIONS] <compile_commands.json>
@@ -33,7 +33,6 @@
 #include "command/toolchain.h"
 #include "compile/compilation.h"
 #include "feature/feature.h"
-#include "index/preamble_state.h"
 #include "index/tu_index.h"
 #include "support/filesystem.h"
 #include "support/logging.h"
@@ -95,7 +94,6 @@ struct FileResult {
     double preprocess_tokens_ms = -1;
     double parse_ms = -1;
     double index_ms = -1;
-    double index_serialize_ms = -1;
     double pch_build_ms = -1;
     double parse_pch_ms = -1;
 
@@ -216,16 +214,17 @@ FileResult profile_file(llvm::StringRef file,
         };
 
         ScopedTimer index_timer;
-        auto tu_index = index::TUIndex::build(unit);
+        auto serialized = index::build_tu_index(unit);
         keep_min(result.index_ms, index_timer.ms_f());
-        result.symbols = tu_index.symbols.size();
-
-        ScopedTimer serialize_timer;
-        std::string serialized;
-        llvm::raw_string_ostream os(serialized);
-        tu_index.serialize(os);
-        keep_min(result.index_serialize_ms, serialize_timer.ms_f());
         result.index_bytes = serialized.size();
+
+        auto view = index::TUIndex::from_bytes(serialized);
+        std::uint64_t symbols = 0;
+        view.iterate_symbols([&](auto, auto&, auto) {
+            symbols += 1;
+            return true;
+        });
+        result.symbols = symbols;
         return true;
     });
     if(tracing) {
@@ -281,21 +280,13 @@ FileResult profile_file(llvm::StringRef file,
         }
 
         // The production PCH pass (stateless worker) also builds the
-        // preamble's full index, document links and inactive regions and
-        // writes the PreambleState blob next to the PCH before reporting
-        // success; the stage must carry that cost to match a didOpen.
-        auto tu_index = index::TUIndex::build(unit);
+        // preamble's envelope, document links and inactive regions and
+        // writes the blob next to the PCH before reporting success; the
+        // stage must carry that cost to match a didOpen.
         auto links = feature::document_links(unit);
         auto inactive = feature::inactive_regions(unit, {}, 0, result.preamble_bound);
         open_conditionals = std::move(inactive.open_stack);
-        std::string blob;
-        llvm::raw_string_ostream os(blob);
-        index::PreambleState::serialize(unit,
-                                        std::move(tu_index),
-                                        links,
-                                        inactive.regions,
-                                        open_conditionals,
-                                        os);
+        auto blob = index::build_preamble_index(unit, links, inactive.regions, open_conditionals);
 
         // The PCH is flushed to disk by the unit's destructor; the blob
         // write follows it, like the worker's on-disk ordering contract.
@@ -325,14 +316,10 @@ FileResult profile_file(llvm::StringRef file,
         }
 
         // The didChange pass (stateful worker) also computes inactive
-        // regions and builds and serializes the interested-only index
-        // before replying; include them so parse and parse_pch bound the
-        // same work.
+        // regions and builds the interested-only envelope before replying;
+        // include them so parse and parse_pch bound the same work.
         feature::inactive_regions(unit, open_conditionals, result.preamble_bound);
-        auto tu_index = index::TUIndex::build(unit, /*interested_only=*/true);
-        std::string serialized;
-        llvm::raw_string_ostream os(serialized);
-        tu_index.serialize(os);
+        index::build_tu_index(unit, /*interested_only=*/true);
         return true;
     });
 
@@ -373,7 +360,6 @@ void print_summary(std::vector<FileResult>& results) {
         {"preprocess_tokens"},
         {"parse"},
         {"index"},
-        {"index_serialize"},
         {"pch_build"},
         {"parse_pch"},
     };
@@ -383,9 +369,8 @@ void print_summary(std::vector<FileResult>& results) {
         stats[2].add(result.preprocess_tokens_ms);
         stats[3].add(result.parse_ms);
         stats[4].add(result.index_ms);
-        stats[5].add(result.index_serialize_ms);
-        stats[6].add(result.pch_build_ms);
-        stats[7].add(result.parse_pch_ms);
+        stats[5].add(result.pch_build_ms);
+        stats[6].add(result.parse_pch_ms);
     }
 
     std::println("");

src/driver/inspect.cc

@@ -9,6 +9,7 @@
 #include "compile/compilation.h"
 #include "driver/driver.h"
 #include "feature/feature.h"
+#include "index/shard.h"
 #include "index/tu_index.h"
 #include "support/filesystem.h"
 #include "syntax/annotation.h"
@@ -234,31 +235,32 @@ struct RawOccurrence {
 
 std::optional<kota::codec::RawValue> run_tu_index(CompilationUnitRef unit,
                                                   [[maybe_unused]] llvm::StringRef config) {
-    auto index = index::TUIndex::build(unit);
-    auto sorted = index.main_file_index.occurrences;
-    std::ranges::sort(sorted, {}, [](const index::Occurrence& occurrence) {
-        return std::tuple(occurrence.range.begin, occurrence.range.end, occurrence.target);
+    auto envelope = index::build_tu_index(unit);
+    auto index = index::TUIndex::from_bytes(envelope);
+    const index::Shard& rows = index.shard_of(index.path_count() - 1);
+
+    llvm::DenseMap<index::SymbolHash, std::vector<index::Relation>> relations;
+    rows.for_each_relation([&](index::SymbolHash hash, const index::Relation& relation) {
+        relations[hash].push_back(relation);
+        return true;
     });
 
     std::vector<RawOccurrence> out;
-    for(const auto& occurrence: sorted) {
+    rows.for_each_occurrence([&](const index::Occurrence& occurrence) {
         RawOccurrence raw;
-        raw.range = LocalSourceRange(occurrence.range.begin, occurrence.range.end);
-        auto symbol = index.symbols.find(occurrence.target);
-        raw.kind =
-            symbol != index.symbols.end() ? symbol->second.kind : SymbolKind(SymbolKind::Invalid);
-        if(auto relations = index.main_file_index.relations.find(occurrence.target);
-           relations != index.main_file_index.relations.end()) {
-            for(const auto& relation: relations->second) {
+        raw.range = occurrence.range;
+        auto symbol = index.find_symbol(occurrence.target);
+        raw.kind = symbol ? symbol->kind : SymbolKind(SymbolKind::Invalid);
+        if(auto found = relations.find(occurrence.target); found != relations.end()) {
+            for(const auto& relation: found->second) {
                 if(relation.range == occurrence.range) {
-                    raw.relations.emplace_back(
-                        kota::meta::enum_name(static_cast<RelationKind::Kind>(relation.kind),
-                                              "Invalid"));
+                    raw.relations.emplace_back(kota::meta::enum_name(relation.kind, "Invalid"));
                 }
             }
         }
         out.push_back(std::move(raw));
-    }
+        return true;
+    });
     return to_raw_json(out);
 }
 

src/feature/feature.h

@@ -275,7 +275,7 @@ struct FoldingRange {
 /// A resolved document link: the argument range of an include-like
 /// directive (byte offsets in the containing file) and the absolute path
 /// of the target file. Plain data — it serializes over the worker RPC and
-/// the PCH's PreambleState blob as-is and becomes an LSP DocumentLink only
+/// the PCH's pch.idx envelope as-is and becomes an LSP DocumentLink only
 /// at the reply edge, where the session's line map does the conversion.
 struct DocumentLink {
     LocalSourceRange range;