Skip to content

Commit 31862f0

Browse files
committed
[kensai] skip .git dirs and binary extensions during walk
PathIndex walked every file including .git internals and opened every file for line counting. On repos with millions of files (e.g. game engines with binary assets), session_start exceeded MCP tool timeout. Two configurable constants control the behavior: - WALK_EXCLUDE_DIRS: directories never descended into (.git) - BINARY_EXTENSIONS: ~100 extensions that get stat-only indexing (size captured, marked isBinary, no fd opened) Binary-extension files remain in the index for find_files and grep -- only line counting is skipped.
1 parent 6b0d8c9 commit 31862f0

2 files changed

Lines changed: 177 additions & 26 deletions

File tree

claude/kensai/mcp/review/src/repofs/pathindex/pathindex.test.ts

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ describe("PathIndex.new", () => {
284284
expect(ix.length).toBe(4);
285285
});
286286

287-
it("includes all files without exclusions", async () => {
287+
it("excludes .git directory", async () => {
288288
await createFiles(tempDir, [
289289
"src/main.go",
290290
".git/HEAD",
@@ -293,16 +293,9 @@ describe("PathIndex.new", () => {
293293
".hidden",
294294
]);
295295
const ix = await PathIndex.new(tempDir);
296-
expect(ix.paths).toEqual(
297-
expect.arrayContaining([
298-
"src/main.go",
299-
".git/HEAD",
300-
".git/objects/ab/cdef",
301-
"node_modules/lib/index.js",
302-
".hidden",
303-
]),
304-
);
305-
expect(ix.length).toBe(5);
296+
expect(ix.paths).toEqual(expect.arrayContaining(["src/main.go", "node_modules/lib/index.js", ".hidden"]));
297+
expect(ix.paths.filter((p) => p.startsWith(".git/"))).toHaveLength(0);
298+
expect(ix.length).toBe(3);
306299
});
307300

308301
it("returns paths in sorted order", async () => {

claude/kensai/mcp/review/src/repofs/pathindex/pathindex.ts

Lines changed: 173 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,142 @@ import picomatch from "picomatch";
88

99
import { ErrBinaryContent, LineIter } from "../lineiter/lineiter.ts";
1010

11+
/** Directory names skipped during filesystem walk. Never descended into. */
12+
const WALK_EXCLUDE_DIRS: ReadonlySet<string> = new Set([".git"]);
13+
14+
/** Extensions treated as binary without opening the file. Stat-only — no line counting, no fd consumed. */
15+
const BINARY_EXTENSIONS: ReadonlySet<string> = new Set([
16+
// Images
17+
".png",
18+
".jpg",
19+
".jpeg",
20+
".gif",
21+
".bmp",
22+
".ico",
23+
".svg",
24+
".webp",
25+
".tiff",
26+
".tif",
27+
".psd",
28+
".dds",
29+
".tga",
30+
".hdr",
31+
".exr",
32+
".ktx",
33+
".ktx2",
34+
".astc",
35+
".pvr",
36+
".basis",
37+
// Audio
38+
".wav",
39+
".mp3",
40+
".ogg",
41+
".flac",
42+
".aac",
43+
".wma",
44+
".m4a",
45+
".opus",
46+
// Video
47+
".mp4",
48+
".avi",
49+
".mkv",
50+
".mov",
51+
".wmv",
52+
".webm",
53+
".flv",
54+
// 3D / geometry
55+
".fbx",
56+
".obj",
57+
".gltf",
58+
".glb",
59+
".blend",
60+
".dae",
61+
".3ds",
62+
".stl",
63+
".usd",
64+
".usda",
65+
".usdc",
66+
".usdz",
67+
// Compiled / binary
68+
".exe",
69+
".dll",
70+
".so",
71+
".dylib",
72+
".o",
73+
".a",
74+
".lib",
75+
".pdb",
76+
".wasm",
77+
".class",
78+
".jar",
79+
".pyc",
80+
".pyo",
81+
// Archives
82+
".zip",
83+
".tar",
84+
".gz",
85+
".bz2",
86+
".xz",
87+
".7z",
88+
".rar",
89+
".zst",
90+
".lz4",
91+
// Fonts
92+
".ttf",
93+
".otf",
94+
".woff",
95+
".woff2",
96+
".eot",
97+
// Documents / data
98+
".pdf",
99+
".doc",
100+
".docx",
101+
".xls",
102+
".xlsx",
103+
".ppt",
104+
".pptx",
105+
".sqlite",
106+
".db",
107+
".mdb",
108+
// Game engine assets (general)
109+
".bin",
110+
".pak",
111+
".dat",
112+
".res",
113+
".asset",
114+
".bundle",
115+
".bank",
116+
".bsp",
117+
".nav",
118+
".lightmap",
119+
".cubemap",
120+
".mdl",
121+
".vtf",
122+
".vpk",
123+
".pck",
124+
// Compiled shaders
125+
".spv",
126+
".cso",
127+
".dxbc",
128+
".metallib",
129+
// Unreal Engine
130+
".uasset",
131+
".umap",
132+
".ubulk",
133+
".upk",
134+
// Audio middleware (FMOD / Wwise)
135+
".fsb",
136+
".fev",
137+
".bnk",
138+
// Dagor Engine
139+
".dag",
140+
".dynmodel",
141+
".rendinst",
142+
".composit",
143+
".gameobj",
144+
".lod",
145+
]);
146+
11147
export type IndexEntryFile = {
12148
type: "file";
13149
name: string;
@@ -168,6 +304,8 @@ export class PathIndex {
168304
const work: Promise<void>[] = [];
169305

170306
for (const dirent of dirEntries) {
307+
if (WALK_EXCLUDE_DIRS.has(dirent.name)) continue;
308+
171309
const relPath = path.join(dir, dirent.name);
172310
const absPath = nativePath.join(this.absRoot, relPath);
173311

@@ -194,21 +332,41 @@ export class PathIndex {
194332
continue;
195333
}
196334

197-
work.push(
198-
Promise.all([fs.stat(absPath), this.#countLines(absPath)]).then(([s, lc]) => {
199-
const entry: IndexEntryFile = {
200-
type: "file",
201-
name: dirent.name,
202-
size: s.size,
203-
lineCount: lc.lineCount,
204-
maxLineLen: lc.maxLineLen,
205-
isBinary: lc.isBinary,
206-
};
207-
dirEntry.children.push(entry);
208-
this.paths.push(relPath);
209-
this.#entries.set(relPath, entry);
210-
}),
211-
);
335+
const ext = path.extname(dirent.name).toLowerCase();
336+
337+
if (BINARY_EXTENSIONS.has(ext)) {
338+
work.push(
339+
fs.stat(absPath).then((s) => {
340+
const entry: IndexEntryFile = {
341+
type: "file",
342+
name: dirent.name,
343+
size: s.size,
344+
lineCount: 0,
345+
maxLineLen: 0,
346+
isBinary: true,
347+
};
348+
dirEntry.children.push(entry);
349+
this.paths.push(relPath);
350+
this.#entries.set(relPath, entry);
351+
}),
352+
);
353+
} else {
354+
work.push(
355+
Promise.all([fs.stat(absPath), this.#countLines(absPath)]).then(([s, lc]) => {
356+
const entry: IndexEntryFile = {
357+
type: "file",
358+
name: dirent.name,
359+
size: s.size,
360+
lineCount: lc.lineCount,
361+
maxLineLen: lc.maxLineLen,
362+
isBinary: lc.isBinary,
363+
};
364+
dirEntry.children.push(entry);
365+
this.paths.push(relPath);
366+
this.#entries.set(relPath, entry);
367+
}),
368+
);
369+
}
212370
}
213371

214372
await Promise.all(work);

0 commit comments

Comments
 (0)