Skip to content

Speed up snapshot JSON generation #325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
justrach merged 1 commit into from
Apr 26, 2026
+99 −58
Merged

Speed up snapshot JSON generation #325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 91 additions & 58 deletions src/snapshot_json.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,15 @@ pub fn buildSnapshot(explorer: *Explorer, store: *Store, alloc: std.mem.Allocato
try w.writeAll("{");
try w.print("\"seq\":{d},", .{store.currentSeq()});

const tree = try explorer.getTree(alloc, false);
defer alloc.free(tree);
try w.writeAll("\"tree\":\"");
try writeJsonEscaped(alloc, &buf, tree);
try w.writeAll("\",");

try w.writeAll("\"outlines\":{");
{
explorer.mu.lockShared();
defer explorer.mu.unlockShared();

try buf.ensureTotalCapacity(alloc, roughSnapshotCapacity(explorer.outlines.count()));

var paths: std.ArrayList([]const u8) = .empty;
defer paths.deinit(alloc);
try paths.ensureTotalCapacity(alloc, explorer.outlines.count());
var iter = explorer.outlines.iterator();
while (iter.next()) |entry| {
try paths.append(alloc, entry.key_ptr.*);
Expand All @@ -34,6 +30,11 @@ pub fn buildSnapshot(explorer: *Explorer, store: *Store, alloc: std.mem.Allocato
}
}.lessThan);

try w.writeAll("\"tree\":\"");
try writeTreeJsonEscaped(alloc, &buf, explorer, paths.items);
try w.writeAll("\",");

try w.writeAll("\"outlines\":{");
for (paths.items, 0..) |path, pi| {
if (pi > 0) try w.writeAll(",");
try w.writeAll("\"");
Expand Down Expand Up @@ -67,37 +68,13 @@ pub fn buildSnapshot(explorer: *Explorer, store: *Store, alloc: std.mem.Allocato
}
try w.writeAll("]}");
}
}
try w.writeAll("},");

try w.writeAll("\"symbol_index\":{");
{
explorer.mu.lockShared();
defer explorer.mu.unlockShared();

var sym_map = std.StringHashMap(std.ArrayList(SymEntry)).init(alloc);
defer {
var si = sym_map.iterator();
while (si.next()) |e| e.value_ptr.deinit(alloc);
sym_map.deinit();
}

var oiter = explorer.outlines.iterator();
while (oiter.next()) |entry| {
for (entry.value_ptr.symbols.items) |sym| {
const gop = try sym_map.getOrPut(sym.name);
if (!gop.found_existing) gop.value_ptr.* = .empty;
try gop.value_ptr.append(alloc, .{
.path = entry.key_ptr.*,
.line = sym.line_start,
.kind = sym.kind,
});
}
}
try w.writeAll("},");

try w.writeAll("\"symbol_index\":{");
var sym_keys: std.ArrayList([]const u8) = .empty;
defer sym_keys.deinit(alloc);
var ski = sym_map.iterator();
try sym_keys.ensureTotalCapacity(alloc, explorer.symbol_index.count());
var ski = explorer.symbol_index.iterator();
while (ski.next()) |e| try sym_keys.append(alloc, e.key_ptr.*);
Comment on lines +77 to 78
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector Bot Apr 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Rebuild symbol index from outlines before snapshot emit

This change serializes symbol_index directly from explorer.symbol_index, but that map is not guaranteed complete after fast snapshot restore: insertRestoredFile (src/snapshot.zig) restores outlines/content without calling rebuildSymbolIndexFor, and findAllSymbols explicitly documents that symbol_index can be incomplete after this path (src/explore.zig, comment around lines 1325-1328). In warm-start loads, many unchanged files follow that restore path, so buildSnapshot now returns a partial or empty symbol_index even when outlines are present. Please retain an outline-based fallback (as before) when emitting snapshot symbol data.

Useful? React with 👍 / 👎.

std.mem.sort([]const u8, sym_keys.items, {}, struct {
fn lessThan(_: void, a: []const u8, b: []const u8) bool {
Expand All @@ -110,27 +87,23 @@ pub fn buildSnapshot(explorer: *Explorer, store: *Store, alloc: std.mem.Allocato
try w.writeAll("\"");
try writeJsonEscaped(alloc, &buf, name);
try w.writeAll("\":[");
const locs = sym_map.get(name) orelse continue;
const locs = explorer.symbol_index.get(name) orelse continue;
for (locs.items, 0..) |loc, li| {
if (li > 0) try w.writeAll(",");
try w.writeAll("{\"path\":\"");
try writeJsonEscaped(alloc, &buf, loc.path);
try w.print("\",\"line\":{d},\"kind\":\"{s}\"}}", .{
loc.line, @tagName(loc.kind),
loc.line_start, @tagName(loc.kind),
});
}
try w.writeAll("]");
}
}
try w.writeAll("},");

try w.writeAll("\"dep_graph\":{");
{
explorer.mu.lockShared();
defer explorer.mu.unlockShared();
try w.writeAll("},");

try w.writeAll("\"dep_graph\":{");
var dep_keys: std.ArrayList([]const u8) = .empty;
defer dep_keys.deinit(alloc);
try dep_keys.ensureTotalCapacity(alloc, explorer.dep_graph.count());
var diter = explorer.dep_graph.iterator();
while (diter.next()) |e| try dep_keys.append(alloc, e.key_ptr.*);
std.mem.sort([]const u8, dep_keys.items, {}, struct {
Expand All @@ -153,33 +126,93 @@ pub fn buildSnapshot(explorer: *Explorer, store: *Store, alloc: std.mem.Allocato
}
try w.writeAll("]");
}
try w.writeAll("}");
}
try w.writeAll("}}");
try w.writeAll("}");

return buf.toOwnedSlice(alloc);
}

const SymEntry = struct {
path: []const u8,
line: u32,
kind: @import("explore.zig").SymbolKind,
};
fn roughSnapshotCapacity(file_count: usize) usize {
const min_capacity: usize = 64 * 1024;
const max_capacity: usize = 8 * 1024 * 1024;
const per_file: usize = 32 * 1024;
if (file_count == 0) return min_capacity;
if (file_count > max_capacity / per_file) return max_capacity;
return @max(min_capacity, file_count * per_file);
}

fn writeTreeJsonEscaped(alloc: std.mem.Allocator, out: *std.ArrayList(u8), explorer: *Explorer, paths: []const []const u8) !void {
const w = cio.listWriter(out, alloc);
var seen_dirs = std.StringHashMap(void).init(alloc);
defer seen_dirs.deinit();

for (paths) |path| {
const outline = explorer.outlines.get(path) orelse continue;

var prefix_end: usize = 0;
while (std.mem.indexOfScalarPos(u8, path, prefix_end, '/')) |sep| {
const dir = path[0 .. sep + 1];
if (!seen_dirs.contains(dir)) {
try seen_dirs.put(dir, {});
const depth = std.mem.count(u8, dir[0..sep], "/");
for (0..depth) |_| try w.writeAll(" ");
const dir_name = path[if (depth > 0) std.mem.lastIndexOfScalar(u8, dir[0..sep], '/').? + 1 else 0..sep];
try writeJsonEscaped(alloc, out, dir_name);
try w.writeAll("/\\n");
}
prefix_end = sep + 1;
}

const depth = std.mem.count(u8, path, "/");
for (0..depth) |_| try w.writeAll(" ");
const basename = if (std.mem.lastIndexOfScalar(u8, path, '/')) |pos| path[pos + 1 ..] else path;
try writeJsonEscaped(alloc, out, basename);
try w.print(" {s} {d}L {d} sym\\n", .{
@tagName(outline.language),
outline.line_count,
outline.symbols.items.len,
});
}
}

fn writeJsonEscaped(alloc: std.mem.Allocator, out: *std.ArrayList(u8), s: []const u8) !void {
for (s) |c| {
var start: usize = 0;
for (s, 0..) |c, i| {
switch (c) {
'"' => try out.appendSlice(alloc, "\\\""),
'\\' => try out.appendSlice(alloc, "\\\\"),
'\n' => try out.appendSlice(alloc, "\\n"),
'\r' => try out.appendSlice(alloc, "\\r"),
'\t' => try out.appendSlice(alloc, "\\t"),
'"' => {
if (i > start) try out.appendSlice(alloc, s[start..i]);
try out.appendSlice(alloc, "\\\"");
start = i + 1;
},
'\\' => {
if (i > start) try out.appendSlice(alloc, s[start..i]);
try out.appendSlice(alloc, "\\\\");
start = i + 1;
},
'\n' => {
if (i > start) try out.appendSlice(alloc, s[start..i]);
try out.appendSlice(alloc, "\\n");
start = i + 1;
},
'\r' => {
if (i > start) try out.appendSlice(alloc, s[start..i]);
try out.appendSlice(alloc, "\\r");
start = i + 1;
},
'\t' => {
if (i > start) try out.appendSlice(alloc, s[start..i]);
try out.appendSlice(alloc, "\\t");
start = i + 1;
},
else => if (c < 0x20) {
if (i > start) try out.appendSlice(alloc, s[start..i]);
const hex = "0123456789abcdef";
const esc = [6]u8{ '\\', 'u', '0', '0', hex[c >> 4], hex[c & 0x0f] };
try out.appendSlice(alloc, &esc);
} else {
try out.append(alloc, c);
start = i + 1;
},
}
}
if (start < s.len) try out.appendSlice(alloc, s[start..]);
}
8 changes: 8 additions & 0 deletions src/tests.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1647,6 +1647,14 @@ test "snapshot_json: snapshot builds and is valid JSON" {
try testing.expect(parsed.value.object.contains("outlines"));
try testing.expect(parsed.value.object.contains("symbol_index"));
try testing.expect(parsed.value.object.contains("dep_graph"));

const tree = parsed.value.object.get("tree").?.string;
try testing.expect(std.mem.indexOf(u8, tree, "src/") != null);
try testing.expect(std.mem.indexOf(u8, tree, "main.zig") != null);

const symbol_index = parsed.value.object.get("symbol_index").?.object;
try testing.expect(symbol_index.contains("main"));
try testing.expect(symbol_index.contains("version"));
}

// ── Deep copy correctness tests ─────────────────────────────
Expand Down
Loading