From 2328caeaf93314f093aed77be973c0357ecf2c10 Mon Sep 17 00:00:00 2001 From: xobotyi Date: Tue, 26 May 2026 17:32:42 +0200 Subject: [PATCH] [kensai] skip .git dirs and binary extensions during walk PathIndex walked every file including .git internals and opened every file for line counting. On repos with millions of files (e.g. game engines with binary assets), session_start exceeded MCP tool timeout. Two configurable constants control the behavior: - WALK_EXCLUDE_DIRS: directories never descended into (.git) - BINARY_EXTENSIONS: ~100 extensions that get stat-only indexing (size captured, marked isBinary, no fd opened) Binary-extension files remain in the index for find_files and grep -- only line counting is skipped. --- .../src/repofs/pathindex/pathindex.test.ts | 15 +- .../review/src/repofs/pathindex/pathindex.ts | 188 ++++++++++++++++-- 2 files changed, 177 insertions(+), 26 deletions(-) diff --git a/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.test.ts b/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.test.ts index 1766ab4..1659744 100644 --- a/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.test.ts +++ b/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.test.ts @@ -284,7 +284,7 @@ describe("PathIndex.new", () => { expect(ix.length).toBe(4); }); - it("includes all files without exclusions", async () => { + it("excludes .git directory", async () => { await createFiles(tempDir, [ "src/main.go", ".git/HEAD", @@ -293,16 +293,9 @@ describe("PathIndex.new", () => { ".hidden", ]); const ix = await PathIndex.new(tempDir); - expect(ix.paths).toEqual( - expect.arrayContaining([ - "src/main.go", - ".git/HEAD", - ".git/objects/ab/cdef", - "node_modules/lib/index.js", - ".hidden", - ]), - ); - expect(ix.length).toBe(5); + expect(ix.paths).toEqual(expect.arrayContaining(["src/main.go", "node_modules/lib/index.js", ".hidden"])); + expect(ix.paths.filter((p) => p.startsWith(".git/"))).toHaveLength(0); + expect(ix.length).toBe(3); }); it("returns paths in sorted order", async () => { diff --git a/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.ts b/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.ts index f663757..b41cae6 100644 --- a/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.ts +++ b/claude/kensai/mcp/review/src/repofs/pathindex/pathindex.ts @@ -8,6 +8,142 @@ import picomatch from "picomatch"; import { ErrBinaryContent, LineIter } from "../lineiter/lineiter.ts"; +/** Directory names skipped during filesystem walk. Never descended into. */ +const WALK_EXCLUDE_DIRS: ReadonlySet = new Set([".git"]); + +/** Extensions treated as binary without opening the file. Stat-only — no line counting, no fd consumed. */ +const BINARY_EXTENSIONS: ReadonlySet = new Set([ + // Images + ".png", + ".jpg", + ".jpeg", + ".gif", + ".bmp", + ".ico", + ".svg", + ".webp", + ".tiff", + ".tif", + ".psd", + ".dds", + ".tga", + ".hdr", + ".exr", + ".ktx", + ".ktx2", + ".astc", + ".pvr", + ".basis", + // Audio + ".wav", + ".mp3", + ".ogg", + ".flac", + ".aac", + ".wma", + ".m4a", + ".opus", + // Video + ".mp4", + ".avi", + ".mkv", + ".mov", + ".wmv", + ".webm", + ".flv", + // 3D / geometry + ".fbx", + ".obj", + ".gltf", + ".glb", + ".blend", + ".dae", + ".3ds", + ".stl", + ".usd", + ".usda", + ".usdc", + ".usdz", + // Compiled / binary + ".exe", + ".dll", + ".so", + ".dylib", + ".o", + ".a", + ".lib", + ".pdb", + ".wasm", + ".class", + ".jar", + ".pyc", + ".pyo", + // Archives + ".zip", + ".tar", + ".gz", + ".bz2", + ".xz", + ".7z", + ".rar", + ".zst", + ".lz4", + // Fonts + ".ttf", + ".otf", + ".woff", + ".woff2", + ".eot", + // Documents / data + ".pdf", + ".doc", + ".docx", + ".xls", + ".xlsx", + ".ppt", + ".pptx", + ".sqlite", + ".db", + ".mdb", + // Game engine assets (general) + ".bin", + ".pak", + ".dat", + ".res", + ".asset", + ".bundle", + ".bank", + ".bsp", + ".nav", + ".lightmap", + ".cubemap", + ".mdl", + ".vtf", + ".vpk", + ".pck", + // Compiled shaders + ".spv", + ".cso", + ".dxbc", + ".metallib", + // Unreal Engine + ".uasset", + ".umap", + ".ubulk", + ".upk", + // Audio middleware (FMOD / Wwise) + ".fsb", + ".fev", + ".bnk", + // Dagor Engine + ".dag", + ".dynmodel", + ".rendinst", + ".composit", + ".gameobj", + ".lod", +]); + export type IndexEntryFile = { type: "file"; name: string; @@ -168,6 +304,8 @@ export class PathIndex { const work: Promise[] = []; for (const dirent of dirEntries) { + if (WALK_EXCLUDE_DIRS.has(dirent.name)) continue; + const relPath = path.join(dir, dirent.name); const absPath = nativePath.join(this.absRoot, relPath); @@ -194,21 +332,41 @@ export class PathIndex { continue; } - work.push( - Promise.all([fs.stat(absPath), this.#countLines(absPath)]).then(([s, lc]) => { - const entry: IndexEntryFile = { - type: "file", - name: dirent.name, - size: s.size, - lineCount: lc.lineCount, - maxLineLen: lc.maxLineLen, - isBinary: lc.isBinary, - }; - dirEntry.children.push(entry); - this.paths.push(relPath); - this.#entries.set(relPath, entry); - }), - ); + const ext = path.extname(dirent.name).toLowerCase(); + + if (BINARY_EXTENSIONS.has(ext)) { + work.push( + fs.stat(absPath).then((s) => { + const entry: IndexEntryFile = { + type: "file", + name: dirent.name, + size: s.size, + lineCount: 0, + maxLineLen: 0, + isBinary: true, + }; + dirEntry.children.push(entry); + this.paths.push(relPath); + this.#entries.set(relPath, entry); + }), + ); + } else { + work.push( + Promise.all([fs.stat(absPath), this.#countLines(absPath)]).then(([s, lc]) => { + const entry: IndexEntryFile = { + type: "file", + name: dirent.name, + size: s.size, + lineCount: lc.lineCount, + maxLineLen: lc.maxLineLen, + isBinary: lc.isBinary, + }; + dirEntry.children.push(entry); + this.paths.push(relPath); + this.#entries.set(relPath, entry); + }), + ); + } } await Promise.all(work);