Skip to content

Commit b4ac84f

Browse files
aron-cfaron-cf
committed
dofs: Add bounded grep options
Keep literal, case-sensitive defaults while adding regular expressions, explicit case handling, numbered context, and limit and offset controls for tool callers.
1 parent 9761634 commit b4ac84f

3 files changed

Lines changed: 226 additions & 55 deletions

File tree

‎packages/dofs/src/fs/grep.test.ts‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,65 @@ describe("grep", () => {
4343
});
4444
});
4545

46-
it("respects ignoreCase", async () => {
46+
it("respects explicit case options and the ignoreCase alias", async () => {
4747
await withDB(async (db) => {
4848
await writeFile(db, "/a.txt", "todo\nTODO\nTodo\n", {}, () => 0);
4949
expect((await grep(db, "TODO", "/a.txt", { ignoreCase: true })).length).toBe(3);
50+
expect((await grep(db, "TODO", "/a.txt", { caseSensitive: false })).length).toBe(3);
51+
expect((await grep(db, "TODO", "/a.txt", { caseSensitive: true })).length).toBe(1);
5052
expect((await grep(db, "TODO", "/a.txt")).length).toBe(1);
5153
});
5254
});
5355

56+
it("supports regular expressions and fixed strings", async () => {
57+
await withDB(async (db) => {
58+
await writeFile(db, "/a.txt", "task 12\ntask \\d+\ntask xx\n", {}, () => 0);
59+
expect(
60+
(await grep(db, String.raw`task \d+`, "/a.txt", { fixedString: false })).map(
61+
(match) => match.line,
62+
),
63+
).toEqual([1]);
64+
expect(
65+
(await grep(db, String.raw`task \d+`, "/a.txt", { fixedString: true })).map(
66+
(match) => match.line,
67+
),
68+
).toEqual([2]);
69+
await expect(grep(db, "[", "/a.txt", { fixedString: false })).rejects.toThrow(
70+
"Invalid regular expression",
71+
);
72+
});
73+
});
74+
75+
it("returns numbered context around matches", async () => {
76+
await withDB(async (db) => {
77+
await writeFile(db, "/a.txt", "one\ntwo\nTODO\nfour\nfive\n", {}, () => 0);
78+
expect(await grep(db, "TODO", "/a.txt", { contextLines: 1 })).toEqual([
79+
{
80+
path: "/a.txt",
81+
line: 3,
82+
text: "TODO",
83+
context: [
84+
{ line: 2, text: "two", isMatch: false },
85+
{ line: 3, text: "TODO", isMatch: true },
86+
{ line: 4, text: "four", isMatch: false },
87+
],
88+
},
89+
]);
90+
});
91+
});
92+
93+
it("applies offset and limit across files in path and line order", async () => {
94+
await withDB(async (db) => {
95+
await writeFile(db, "/a.txt", "TODO a1\nTODO a2\n", {}, () => 0);
96+
await writeFile(db, "/b.txt", "TODO b1\nTODO b2\n", {}, () => 0);
97+
expect(
98+
(await grep(db, "TODO", "/", { offset: 1, limit: 2 })).map(
99+
(match) => `${match.path}:${match.line}`,
100+
),
101+
).toEqual(["/a.txt:2", "/b.txt:1"]);
102+
});
103+
});
104+
54105
it("matches across a chunk boundary", async () => {
55106
await withDB(async (db) => {
56107
// Lay out a file whose line straddles the 512KiB chunk boundary.

‎packages/dofs/src/fs/grep.ts‎

Lines changed: 169 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,47 @@ import { find } from "./find.js";
55
import { readFile } from "./readFile.js";
66
import { resolveInode } from "./resolve.js";
77

8+
export interface WorkspaceGrepContextLine {
9+
line: number;
10+
text: string;
11+
isMatch: boolean;
12+
}
13+
814
export interface WorkspaceGrepMatch {
915
path: string;
1016
line: number;
1117
text: string;
18+
context?: WorkspaceGrepContextLine[];
1219
}
1320

1421
export interface GrepOptions {
22+
/** Compatibility alias for `caseSensitive: false`. */
1523
ignoreCase?: boolean;
24+
/** Match letter case. Defaults to true. */
25+
caseSensitive?: boolean;
26+
/** Treat the pattern as plain text. Defaults to true. */
27+
fixedString?: boolean;
28+
/** Lines of context to include before and after each match. */
29+
contextLines?: number;
30+
/** Maximum matches to return. */
31+
limit?: number;
32+
/** Matching lines to skip before collecting results. */
33+
offset?: number;
34+
}
35+
36+
interface ScanState {
37+
seen: number;
38+
accepted: number;
39+
}
40+
41+
interface NumberedLine {
42+
line: number;
43+
text: string;
44+
}
45+
46+
interface PendingMatch {
47+
match: WorkspaceGrepMatch;
48+
remaining: number;
1649
}
1750

1851
export async function grep(
@@ -27,81 +60,164 @@ export async function grep(
2760
throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical);
2861
}
2962

63+
const settings = normalizeOptions(options);
64+
if (settings.limit === 0) return [];
65+
const matcher = compileMatcher(pattern, settings.fixedString, settings.caseSensitive);
3066
const filePaths =
3167
node.type === "file"
3268
? [canonical]
3369
: find(db, canonical)
3470
.filter((entry) => entry.type === "file")
35-
.map((entry) => entry.path);
71+
.map((entry) => entry.path)
72+
.sort();
3673

3774
const matches: WorkspaceGrepMatch[] = [];
75+
const state: ScanState = { seen: 0, accepted: 0 };
3876
for (const filePath of filePaths) {
39-
await scanFile(db, filePath, pattern, options, matches);
77+
const complete = await scanFile(
78+
db,
79+
filePath,
80+
matcher,
81+
settings.contextLines,
82+
settings.offset,
83+
settings.limit,
84+
state,
85+
matches,
86+
);
87+
if (complete) break;
4088
}
4189
return matches;
4290
}
4391

44-
// Stream the file in chunks so very large files don't load fully into
45-
// memory. Carry a partial-line tail between chunks (everything after
46-
// the last '\n') so a line that straddles a chunk boundary still
47-
// matches as one line. Line numbers are 1-indexed.
92+
function normalizeOptions(options: GrepOptions): {
93+
caseSensitive: boolean;
94+
fixedString: boolean;
95+
contextLines: number;
96+
limit: number;
97+
offset: number;
98+
} {
99+
if (
100+
options.caseSensitive !== undefined &&
101+
options.ignoreCase !== undefined &&
102+
options.caseSensitive === options.ignoreCase
103+
) {
104+
throw new TypeError("caseSensitive conflicts with ignoreCase");
105+
}
106+
const contextLines = options.contextLines ?? 0;
107+
if (!Number.isSafeInteger(contextLines) || contextLines < 0) {
108+
throw new TypeError("grep contextLines must be a non-negative safe integer");
109+
}
110+
const limit = options.limit ?? Number.MAX_SAFE_INTEGER;
111+
if (!Number.isSafeInteger(limit) || limit < 0) {
112+
throw new TypeError("grep limit must be a non-negative safe integer");
113+
}
114+
const offset = options.offset ?? 0;
115+
if (!Number.isSafeInteger(offset) || offset < 0) {
116+
throw new TypeError("grep offset must be a non-negative safe integer");
117+
}
118+
return {
119+
caseSensitive: options.caseSensitive ?? options.ignoreCase !== true,
120+
fixedString: options.fixedString ?? true,
121+
contextLines,
122+
limit,
123+
offset,
124+
};
125+
}
126+
127+
function compileMatcher(pattern: string, fixedString: boolean, caseSensitive: boolean): RegExp {
128+
const source = fixedString ? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") : pattern;
129+
try {
130+
return new RegExp(source, caseSensitive ? "" : "i");
131+
} catch (error) {
132+
throw new TypeError(
133+
`Invalid regular expression: ${error instanceof Error ? error.message : String(error)}`,
134+
);
135+
}
136+
}
137+
48138
async function scanFile(
49139
db: Database,
50140
path: string,
51-
pattern: string,
52-
options: GrepOptions,
141+
matcher: RegExp,
142+
contextLines: number,
143+
offset: number,
144+
limit: number,
145+
state: ScanState,
53146
out: WorkspaceGrepMatch[],
54-
): Promise<void> {
55-
const stream = await readFile(db, path);
56-
const reader = stream.getReader();
57-
const decoder = new TextDecoder("utf-8", { fatal: false });
58-
const needle = options.ignoreCase ? pattern.toUpperCase() : pattern;
147+
): Promise<boolean> {
148+
const before: NumberedLine[] = [];
149+
const pending: PendingMatch[] = [];
59150

60-
let tail = "";
61-
let lineNo = 1;
62-
while (true) {
63-
const { value, done } = await reader.read();
64-
if (done) break;
65-
if (value === undefined) continue;
66-
const text = tail + decoder.decode(value, { stream: true });
67-
const newlineIdx = text.lastIndexOf("\n");
68-
const ready = newlineIdx === -1 ? "" : text.slice(0, newlineIdx);
69-
tail = newlineIdx === -1 ? text : text.slice(newlineIdx + 1);
70-
if (ready.length > 0) {
71-
lineNo = scanLines(ready, lineNo, needle, options.ignoreCase === true, path, out);
151+
for await (const current of readLines(db, path)) {
152+
for (const item of pending) {
153+
item.match.context?.push({ ...current, isMatch: false });
154+
item.remaining -= 1;
72155
}
156+
flushReady(pending, out);
157+
if (state.accepted >= limit && pending.length === 0) return true;
158+
159+
if (matcher.test(current.text)) {
160+
const matchIndex = state.seen;
161+
state.seen += 1;
162+
if (matchIndex >= offset && state.accepted < limit) {
163+
const match: WorkspaceGrepMatch = { path, ...current };
164+
if (contextLines > 0) {
165+
match.context = [
166+
...before.map((line) => ({ ...line, isMatch: false })),
167+
{ ...current, isMatch: true },
168+
];
169+
pending.push({ match, remaining: contextLines });
170+
} else {
171+
out.push(match);
172+
}
173+
state.accepted += 1;
174+
}
175+
}
176+
177+
before.push(current);
178+
if (before.length > contextLines) before.shift();
179+
if (state.accepted >= limit && pending.length === 0) return true;
73180
}
74-
// Drain the decoder and scan whatever's left (final line without a
75-
// trailing newline).
76-
tail += decoder.decode();
77-
if (tail.length > 0) {
78-
scanLines(tail, lineNo, needle, options.ignoreCase === true, path, out);
181+
182+
for (const item of pending) out.push(item.match);
183+
return state.accepted >= limit;
184+
}
185+
186+
function flushReady(pending: PendingMatch[], out: WorkspaceGrepMatch[]): void {
187+
while (pending[0]?.remaining === 0) {
188+
const item = pending.shift();
189+
if (item !== undefined) out.push(item.match);
79190
}
80191
}
81192

82-
// Walk `block` line-by-line, push matches into `out`, return the next
83-
// 1-indexed line number to use for the following block.
84-
function scanLines(
85-
block: string,
86-
startLine: number,
87-
needle: string,
88-
ignoreCase: boolean,
89-
path: string,
90-
out: WorkspaceGrepMatch[],
91-
): number {
92-
let line = startLine;
93-
let cursor = 0;
94-
while (cursor <= block.length) {
95-
const next = block.indexOf("\n", cursor);
96-
const end = next === -1 ? block.length : next;
97-
const text = block.slice(cursor, end);
98-
const haystack = ignoreCase ? text.toUpperCase() : text;
99-
if (haystack.includes(needle)) {
100-
out.push({ path, line, text });
193+
async function* readLines(db: Database, path: string): AsyncIterable<NumberedLine> {
194+
const stream = await readFile(db, path);
195+
const reader = stream.getReader();
196+
const decoder = new TextDecoder("utf-8", { fatal: false });
197+
let tail = "";
198+
let line = 1;
199+
let completed = false;
200+
try {
201+
while (true) {
202+
const { value, done } = await reader.read();
203+
if (done) {
204+
completed = true;
205+
break;
206+
}
207+
if (value === undefined) continue;
208+
tail += decoder.decode(value, { stream: true });
209+
let newline = tail.indexOf("\n");
210+
while (newline !== -1) {
211+
yield { line, text: tail.slice(0, newline) };
212+
line += 1;
213+
tail = tail.slice(newline + 1);
214+
newline = tail.indexOf("\n");
215+
}
101216
}
102-
line += 1;
103-
if (next === -1) break;
104-
cursor = next + 1;
217+
tail += decoder.decode();
218+
if (tail.length > 0) yield { line, text: tail };
219+
} finally {
220+
if (!completed) await reader.cancel();
221+
reader.releaseLock();
105222
}
106-
return line;
107223
}

‎packages/dofs/src/index.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,11 @@ export {
66
type WorkspaceFilesystemOptions,
77
} from "./fs/filesystem.js";
88
export type { WorkspaceFoundEntry } from "./fs/find.js";
9-
export type { GrepOptions, WorkspaceGrepMatch } from "./fs/grep.js";
9+
export type {
10+
GrepOptions,
11+
WorkspaceGrepContextLine,
12+
WorkspaceGrepMatch,
13+
} from "./fs/grep.js";
1014
export { link } from "./fs/link.js";
1115
export type { MkdirOptions } from "./fs/mkdir.js";
1216
// Read-only mount enforcement. The workspace-side indexer writes

0 commit comments

Comments
 (0)