diff --git a/spec/MultiTabWorkerBroker.spec.ts b/spec/MultiTabWorkerBroker.spec.ts index f60761c..2c17943 100644 --- a/spec/MultiTabWorkerBroker.spec.ts +++ b/spec/MultiTabWorkerBroker.spec.ts @@ -235,17 +235,31 @@ describe("MultiTabWorkerBroker", () => { // Wait for follower to become leader and initialize its worker await broker2LeaderPromise; - // Wait for broker2 to be fully ready as leader await waitFor( () => broker2.isLeader, (isLeader) => isLeader === true, - { timeout: 500, message: "Broker2 did not become leader" } + { timeout: 3000, message: "Broker2 did not become leader" } ); - // Give worker additional time to be fully ready - await new Promise((resolve) => setTimeout(resolve, 100)); expect(broker2.isLeader).toBe(true); - // Verify new leader can communicate + // Verify worker is responsive before proceeding — avoids flaky fixed delays + const probeMessages: any[] = []; + const probeConn = broker2.createConnection(); + probeConn.reader.listen((msg) => probeMessages.push(msg)); + await probeConn.writer.write({ + jsonrpc: "2.0", + id: 9999, + method: "echo", + params: { probe: true }, + } as any); + await waitFor( + () => probeMessages, + (msgs) => msgs.some((m) => m.id === 9999), + { timeout: 3000, message: "Worker not responsive after leader promotion" } + ); + probeConn.dispose(); + + // Now verify new leader can communicate on the original connection await conn2.writer.write({ jsonrpc: "2.0", id: 1, @@ -253,11 +267,10 @@ describe("MultiTabWorkerBroker", () => { params: { promoted: true }, } as any); - // Wait for the response with retry logic await waitFor( () => followerMessages, (msgs) => msgs.some((m) => m.id === 1 && m.result?.promoted === true), - { timeout: 2000, message: "Expected response from promoted leader not received" } + { timeout: 3000, message: "Expected response from promoted leader not received" } ); expect(followerMessages).toContainEqual(expect.objectContaining({ id: 1, result: { promoted: true } })); diff --git a/spec/vfs.spec.ts b/spec/vfs.spec.ts index eff0d0c..3b095c6 100644 --- a/spec/vfs.spec.ts +++ b/spec/vfs.spec.ts @@ -430,3 +430,177 @@ describe.each([ expect(vfs.readFileSync(link, { encoding: "utf8" })).toBe("data"); }); }); + +describe("sync opfs gc", () => { + let vfs: SyncOPFSFileSystem; + + beforeEach(async () => { + vfs = new SyncOPFSFileSystem("gadget-gc"); + await vfs.init(); + }); + + afterEach(async () => { + await vfs.close(); + }); + + test("getStats reports arena usage", async () => { + await vfs.writeFileEnsuringDirectories("/stats/file.ts", "hello world"); + const stats = vfs.getStats(); + expect(stats.arenaBytes).toBeGreaterThan(0); + expect(stats.allocatedBytes).toBeGreaterThan(0); + }); + + test("arena shrinks when trailing pages are freed", async () => { + // Write a file to grow the arena, then delete it + const bigData = "x".repeat(200_000); // ~200KB, multiple pages + await vfs.writeFileEnsuringDirectories("/shrink/big.ts", bigData); + const afterWrite = vfs.getStats(); + + await vfs.deleteFile("/shrink/big.ts"); + const afterDelete = vfs.getStats(); + + // Arena should have shrunk after freeing trailing pages + expect(afterDelete.arenaBytes).toBeLessThanOrEqual(afterWrite.arenaBytes); + expect(afterDelete.allocatedBytes).toBe(0); + }); + + test("overwriting a file does not leak arena space indefinitely", async () => { + const path = "/overwrite/file.ts"; + const data = "x".repeat(100_000); + + await vfs.writeFileEnsuringDirectories(path, data); + const baseline = vfs.getStats(); + + // Overwrite many times — arena should not grow unboundedly + for (let i = 0; i < 20; i++) { + vfs.writeFileSync(path, data + i); + } + const afterOverwrites = vfs.getStats(); + + // Arena should not be significantly larger than baseline + // (some growth is expected due to copy-on-write, but not 20x) + expect(afterOverwrites.arenaBytes).toBeLessThan(baseline.arenaBytes * 3); + }); + + test("many sequential small edits to a single file do not grow arena disproportionately", async () => { + const path = "/edits/document.ts"; + // Start with a ~10KB file, simulating a source file + const baseContent = "// line\n".repeat(1250); + await vfs.writeFileEnsuringDirectories(path, baseContent); + const baseline = vfs.getStats(); + + // Simulate 500 small incremental edits (typo fixes, adding a line, etc.) + // Each edit changes only a few characters but rewrites the whole file + for (let i = 0; i < 500; i++) { + const edited = baseContent.slice(0, 100) + `// edit ${i}\n` + baseContent.slice(100); + vfs.writeFileSync(path, edited); + } + + const afterEdits = vfs.getStats(); + + // The file is ~10KB. After 500 overwrites, a naive system would accumulate + // ~5MB of dead data. With GC, the arena should stay close to baseline. + // We allow up to 2x the baseline to account for copy-on-write transients + // and page alignment overhead, but certainly not 500x. + expect(afterEdits.arenaBytes).toBeLessThan(baseline.arenaBytes * 2); + + // Allocated bytes should reflect only the single live file (~10KB, page-aligned) + expect(afterEdits.allocatedBytes).toBeLessThanOrEqual(baseline.allocatedBytes * 2); + + // The file content should be the last edit, proving correctness + const finalContent = baseContent.slice(0, 100) + `// edit 499\n` + baseContent.slice(100); + expect(vfs.readFileSync(path, { encoding: "utf8" })).toBe(finalContent); + }); + + test("many sequential small edits across multiple files do not grow arena disproportionately", async () => { + const fileCount = 10; + const editsPerFile = 100; + const baseContent = "x".repeat(5_000); // 5KB per file + + // Create all files + for (let f = 0; f < fileCount; f++) { + await vfs.writeFileEnsuringDirectories(`/multi/file${f}.ts`, baseContent); + } + const baseline = vfs.getStats(); + + // Round-robin edits across files — simulates a dev editing multiple open files + for (let round = 0; round < editsPerFile; round++) { + for (let f = 0; f < fileCount; f++) { + vfs.writeFileSync(`/multi/file${f}.ts`, baseContent + `// r${round}`); + } + } + + const afterEdits = vfs.getStats(); + + // 10 files × 5KB = 50KB live data. 1000 total writes would be 5MB without GC. + // Arena should stay reasonable — well under 4x the baseline. + expect(afterEdits.arenaBytes).toBeLessThan(baseline.arenaBytes * 4); + + // Verify every file has the correct final content + for (let f = 0; f < fileCount; f++) { + expect(vfs.readFileSync(`/multi/file${f}.ts`, { encoding: "utf8" })).toBe(baseContent + `// r${editsPerFile - 1}`); + } + }); + + test("gc() compacts fragmented arena", async () => { + // Create several files, delete alternating ones to create fragmentation + for (let i = 0; i < 10; i++) { + await vfs.writeFileEnsuringDirectories(`/frag/file${i}.ts`, "x".repeat(70_000)); + } + // Delete even-numbered files to fragment + for (let i = 0; i < 10; i += 2) { + await vfs.deleteFile(`/frag/file${i}.ts`); + } + const beforeGc = vfs.getStats(); + + vfs.gc(); + const afterGc = vfs.getStats(); + + // After compaction, arena should be smaller or equal + expect(afterGc.arenaBytes).toBeLessThanOrEqual(beforeGc.arenaBytes); + // Fragmentation should be reduced (0 or 1 free fragment) + expect(afterGc.freeFragments).toBeLessThanOrEqual(1); + + // All remaining files should still be readable + for (let i = 1; i < 10; i += 2) { + const content = vfs.readFileSync(`/frag/file${i}.ts`, { encoding: "utf8" }); + expect(content).toBe("x".repeat(70_000)); + } + }); + + test("gc() on empty filesystem is a no-op", () => { + vfs.gc(); + const stats = vfs.getStats(); + expect(stats.allocatedBytes).toBe(0); + }); + + test("exportIndex/importIndex preserves data after gc", async () => { + for (let i = 0; i < 5; i++) { + await vfs.writeFileEnsuringDirectories(`/persist/file${i}.ts`, `content-${i}`); + } + await vfs.deleteFile("/persist/file2.ts"); + + vfs.gc(); + const snapshot = vfs.exportIndex(); + + // Create a fresh instance, import the snapshot + const vfs2 = new SyncOPFSFileSystem("gadget-gc"); + // Re-use the same arena handle by closing and reopening + await vfs.close(); + await vfs2.init(); + vfs2.importIndex(snapshot); + + for (let i = 0; i < 5; i++) { + if (i === 2) { + expect(vfs2.existsSync(`/persist/file${i}.ts`)).toBe(false); + } else { + expect(vfs2.readFileSync(`/persist/file${i}.ts`, { encoding: "utf8" })).toBe(`content-${i}`); + } + } + + await vfs2.close(); + // Re-open original for teardown + vfs = new SyncOPFSFileSystem("gadget-gc"); + await vfs.init(); + }); +}); diff --git a/src/SyncOPFSFileSystem.ts b/src/SyncOPFSFileSystem.ts index ed18917..e4f4677 100644 --- a/src/SyncOPFSFileSystem.ts +++ b/src/SyncOPFSFileSystem.ts @@ -363,6 +363,94 @@ export class SyncOPFSFileSystem implements VirtualFileSystem { private freeExtents(extents: FileExtent[]) { for (const e of extents) this.freeList.push({ startPage: e.startPage, pageCount: e.pageCount }); this.coalesceFreeList(); + this.shrinkArenaTrailingFreeSpace(); + } + + /** Shrink the arena file by truncating trailing free pages */ + private shrinkArenaTrailingFreeSpace() { + if (this.freeList.length === 0) return; + const arenaBytes = this.arenaHandle.getSize(); + const totalPages = Math.floor(arenaBytes / PAGE_SIZE); + if (totalPages === 0) return; + + // The free list is sorted after coalesce — check if the last range reaches the arena end + const last = this.freeList[this.freeList.length - 1]; + if (last.startPage + last.pageCount !== totalPages) return; + + // Keep a minimum arena size to avoid thrashing (e.g. 1 MiB / PAGE_SIZE pages) + const minPages = Math.ceil((1024 * 1024) / PAGE_SIZE); + const reclaimablePages = last.pageCount; + const newTotalPages = Math.max(totalPages - reclaimablePages, minPages); + const pagesReclaimed = totalPages - newTotalPages; + + if (pagesReclaimed <= 0) return; + + this.arenaHandle.truncate(newTotalPages * PAGE_SIZE); + + if (pagesReclaimed === last.pageCount) { + this.freeList.pop(); + } else { + last.pageCount -= pagesReclaimed; + } + } + + /** + * Full compaction: relocates all live file data to the start of the arena, + * eliminates fragmentation, and truncates the arena file. + * Call this periodically or after large batch deletions for maximum space savings. + */ + gc() { + // Collect all live file extents in the tree + const liveFiles: { node: FileNode; oldExtents: FileExtent[] }[] = []; + const walk = (node: Node) => { + if (node.type === "file" && node.extents.length > 0) { + liveFiles.push({ node, oldExtents: node.extents.map((e) => ({ ...e })) }); + } else if (node.type === "dir") { + for (const child of node.children.values()) walk(child); + } + }; + walk(this.root); + + // Sort live files by their first extent's start page for sequential reads + liveFiles.sort((a, b) => a.oldExtents[0].startPage - b.oldExtents[0].startPage); + + // Relocate each file's data contiguously from page 0 onward + let nextPage = 0; + for (const { node, oldExtents } of liveFiles) { + const data = this.readFileBytes(node); + if (!data) continue; + + const pagesNeeded = Math.ceil(node.size / PAGE_SIZE); + const newExtent: FileExtent = { startPage: nextPage, pageCount: pagesNeeded }; + + // Write data to its new location + this.writeBytesToExtents([newExtent], data); + node.extents = [newExtent]; + nextPage += pagesNeeded; + } + + // Rebuild free list and truncate + const usedPages = nextPage; + const minPages = Math.max(usedPages, Math.ceil((1024 * 1024) / PAGE_SIZE)); + this.arenaHandle.truncate(minPages * PAGE_SIZE); + this.allocatedBytes = usedPages * PAGE_SIZE; + + this.freeList = []; + if (minPages > usedPages) { + this.freeList.push({ startPage: usedPages, pageCount: minPages - usedPages }); + } + } + + /** Returns stats about arena usage for monitoring */ + getStats(): { arenaBytes: number; allocatedBytes: number; freeBytes: number; freeFragments: number } { + const arenaBytes = this.arenaHandle.getSize(); + const freePages = this.freeList.reduce((sum, r) => sum + r.pageCount, 0); + return { + arenaBytes, + allocatedBytes: this.allocatedBytes, + freeBytes: freePages * PAGE_SIZE, + freeFragments: this.freeList.length, + }; } private coalesceFreeList() {