Add garbage collection to SyncOPFSFileSystem to prevent unbounded disk growth

spec/MultiTabWorkerBroker.spec.ts

@@ -235,29 +235,42 @@ describe("MultiTabWorkerBroker", () => {
 
       // Wait for follower to become leader and initialize its worker
       await broker2LeaderPromise;
-      // Wait for broker2 to be fully ready as leader
       await waitFor(
         () => broker2.isLeader,
         (isLeader) => isLeader === true,
-        { timeout: 500, message: "Broker2 did not become leader" }
+        { timeout: 3000, message: "Broker2 did not become leader" }
       );
-      // Give worker additional time to be fully ready
-      await new Promise((resolve) => setTimeout(resolve, 100));
       expect(broker2.isLeader).toBe(true);
 
-      // Verify new leader can communicate
+      // Verify worker is responsive before proceeding — avoids flaky fixed delays
+      const probeMessages: any[] = [];
+      const probeConn = broker2.createConnection();
+      probeConn.reader.listen((msg) => probeMessages.push(msg));
+      await probeConn.writer.write({
+        jsonrpc: "2.0",
+        id: 9999,
+        method: "echo",
+        params: { probe: true },
+      } as any);
+      await waitFor(
+        () => probeMessages,
+        (msgs) => msgs.some((m) => m.id === 9999),
+        { timeout: 3000, message: "Worker not responsive after leader promotion" }
+      );
+      probeConn.dispose();
+
+      // Now verify new leader can communicate on the original connection
       await conn2.writer.write({
         jsonrpc: "2.0",
         id: 1,
         method: "echo",
         params: { promoted: true },
       } as any);
 
-      // Wait for the response with retry logic
       await waitFor(
         () => followerMessages,
         (msgs) => msgs.some((m) => m.id === 1 && m.result?.promoted === true),
-        { timeout: 2000, message: "Expected response from promoted leader not received" }
+        { timeout: 3000, message: "Expected response from promoted leader not received" }
       );
 
       expect(followerMessages).toContainEqual(expect.objectContaining({ id: 1, result: { promoted: true } }));

spec/vfs.spec.ts

@@ -430,3 +430,177 @@ describe.each([
     expect(vfs.readFileSync(link, { encoding: "utf8" })).toBe("data");
   });
 });
+
+describe("sync opfs gc", () => {
+  let vfs: SyncOPFSFileSystem;
+
+  beforeEach(async () => {
+    vfs = new SyncOPFSFileSystem("gadget-gc");
+    await vfs.init();
+  });
+
+  afterEach(async () => {
+    await vfs.close();
+  });
+
+  test("getStats reports arena usage", async () => {
+    await vfs.writeFileEnsuringDirectories("/stats/file.ts", "hello world");
+    const stats = vfs.getStats();
+    expect(stats.arenaBytes).toBeGreaterThan(0);
+    expect(stats.allocatedBytes).toBeGreaterThan(0);
+  });
+
+  test("arena shrinks when trailing pages are freed", async () => {
+    // Write a file to grow the arena, then delete it
+    const bigData = "x".repeat(200_000); // ~200KB, multiple pages
+    await vfs.writeFileEnsuringDirectories("/shrink/big.ts", bigData);
+    const afterWrite = vfs.getStats();
+
+    await vfs.deleteFile("/shrink/big.ts");
+    const afterDelete = vfs.getStats();
+
+    // Arena should have shrunk after freeing trailing pages
+    expect(afterDelete.arenaBytes).toBeLessThanOrEqual(afterWrite.arenaBytes);
+    expect(afterDelete.allocatedBytes).toBe(0);
+  });
+
+  test("overwriting a file does not leak arena space indefinitely", async () => {
+    const path = "/overwrite/file.ts";
+    const data = "x".repeat(100_000);
+
+    await vfs.writeFileEnsuringDirectories(path, data);
+    const baseline = vfs.getStats();
+
+    // Overwrite many times — arena should not grow unboundedly
+    for (let i = 0; i < 20; i++) {
+      vfs.writeFileSync(path, data + i);
+    }
+    const afterOverwrites = vfs.getStats();
+
+    // Arena should not be significantly larger than baseline
+    // (some growth is expected due to copy-on-write, but not 20x)
+    expect(afterOverwrites.arenaBytes).toBeLessThan(baseline.arenaBytes * 3);
+  });
+
+  test("many sequential small edits to a single file do not grow arena disproportionately", async () => {
+    const path = "/edits/document.ts";
+    // Start with a ~10KB file, simulating a source file
+    const baseContent = "// line\n".repeat(1250);
+    await vfs.writeFileEnsuringDirectories(path, baseContent);
+    const baseline = vfs.getStats();
+
+    // Simulate 500 small incremental edits (typo fixes, adding a line, etc.)
+    // Each edit changes only a few characters but rewrites the whole file
+    for (let i = 0; i < 500; i++) {
+      const edited = baseContent.slice(0, 100) + `// edit ${i}\n` + baseContent.slice(100);
+      vfs.writeFileSync(path, edited);
+    }
+
+    const afterEdits = vfs.getStats();
+
+    // The file is ~10KB. After 500 overwrites, a naive system would accumulate
+    // ~5MB of dead data. With GC, the arena should stay close to baseline.
+    // We allow up to 2x the baseline to account for copy-on-write transients
+    // and page alignment overhead, but certainly not 500x.
+    expect(afterEdits.arenaBytes).toBeLessThan(baseline.arenaBytes * 2);
+
+    // Allocated bytes should reflect only the single live file (~10KB, page-aligned)
+    expect(afterEdits.allocatedBytes).toBeLessThanOrEqual(baseline.allocatedBytes * 2);
+
+    // The file content should be the last edit, proving correctness
+    const finalContent = baseContent.slice(0, 100) + `// edit 499\n` + baseContent.slice(100);
+    expect(vfs.readFileSync(path, { encoding: "utf8" })).toBe(finalContent);
+  });
+
+  test("many sequential small edits across multiple files do not grow arena disproportionately", async () => {
+    const fileCount = 10;
+    const editsPerFile = 100;
+    const baseContent = "x".repeat(5_000); // 5KB per file
+
+    // Create all files
+    for (let f = 0; f < fileCount; f++) {
+      await vfs.writeFileEnsuringDirectories(`/multi/file${f}.ts`, baseContent);
+    }
+    const baseline = vfs.getStats();
+
+    // Round-robin edits across files — simulates a dev editing multiple open files
+    for (let round = 0; round < editsPerFile; round++) {
+      for (let f = 0; f < fileCount; f++) {
+        vfs.writeFileSync(`/multi/file${f}.ts`, baseContent + `// r${round}`);
+      }
+    }
+
+    const afterEdits = vfs.getStats();
+
+    // 10 files × 5KB = 50KB live data. 1000 total writes would be 5MB without GC.
+    // Arena should stay reasonable — well under 4x the baseline.
+    expect(afterEdits.arenaBytes).toBeLessThan(baseline.arenaBytes * 4);
+
+    // Verify every file has the correct final content
+    for (let f = 0; f < fileCount; f++) {
+      expect(vfs.readFileSync(`/multi/file${f}.ts`, { encoding: "utf8" })).toBe(baseContent + `// r${editsPerFile - 1}`);
+    }
+  });
+
+  test("gc() compacts fragmented arena", async () => {
+    // Create several files, delete alternating ones to create fragmentation
+    for (let i = 0; i < 10; i++) {
+      await vfs.writeFileEnsuringDirectories(`/frag/file${i}.ts`, "x".repeat(70_000));
+    }
+    // Delete even-numbered files to fragment
+    for (let i = 0; i < 10; i += 2) {
+      await vfs.deleteFile(`/frag/file${i}.ts`);
+    }
+    const beforeGc = vfs.getStats();
+
+    vfs.gc();
+    const afterGc = vfs.getStats();
+
+    // After compaction, arena should be smaller or equal
+    expect(afterGc.arenaBytes).toBeLessThanOrEqual(beforeGc.arenaBytes);
+    // Fragmentation should be reduced (0 or 1 free fragment)
+    expect(afterGc.freeFragments).toBeLessThanOrEqual(1);
+
+    // All remaining files should still be readable
+    for (let i = 1; i < 10; i += 2) {
+      const content = vfs.readFileSync(`/frag/file${i}.ts`, { encoding: "utf8" });
+      expect(content).toBe("x".repeat(70_000));
+    }
+  });
+
+  test("gc() on empty filesystem is a no-op", () => {
+    vfs.gc();
+    const stats = vfs.getStats();
+    expect(stats.allocatedBytes).toBe(0);
+  });
+
+  test("exportIndex/importIndex preserves data after gc", async () => {
+    for (let i = 0; i < 5; i++) {
+      await vfs.writeFileEnsuringDirectories(`/persist/file${i}.ts`, `content-${i}`);
+    }
+    await vfs.deleteFile("/persist/file2.ts");
+
+    vfs.gc();
+    const snapshot = vfs.exportIndex();
+
+    // Create a fresh instance, import the snapshot
+    const vfs2 = new SyncOPFSFileSystem("gadget-gc");
+    // Re-use the same arena handle by closing and reopening
+    await vfs.close();
+    await vfs2.init();
+    vfs2.importIndex(snapshot);
+
+    for (let i = 0; i < 5; i++) {
+      if (i === 2) {
+        expect(vfs2.existsSync(`/persist/file${i}.ts`)).toBe(false);
+      } else {
+        expect(vfs2.readFileSync(`/persist/file${i}.ts`, { encoding: "utf8" })).toBe(`content-${i}`);
+      }
+    }
+
+    await vfs2.close();
+    // Re-open original for teardown
+    vfs = new SyncOPFSFileSystem("gadget-gc");
+    await vfs.init();
+  });
+});

src/SyncOPFSFileSystem.ts

@@ -363,6 +363,94 @@ export class SyncOPFSFileSystem implements VirtualFileSystem {
   private freeExtents(extents: FileExtent[]) {
     for (const e of extents) this.freeList.push({ startPage: e.startPage, pageCount: e.pageCount });
     this.coalesceFreeList();
+    this.shrinkArenaTrailingFreeSpace();
+  }
+
+  /** Shrink the arena file by truncating trailing free pages */
+  private shrinkArenaTrailingFreeSpace() {
+    if (this.freeList.length === 0) return;
+    const arenaBytes = this.arenaHandle.getSize();
+    const totalPages = Math.floor(arenaBytes / PAGE_SIZE);
+    if (totalPages === 0) return;
+
+    // The free list is sorted after coalesce — check if the last range reaches the arena end
+    const last = this.freeList[this.freeList.length - 1];
+    if (last.startPage + last.pageCount !== totalPages) return;
+
+    // Keep a minimum arena size to avoid thrashing (e.g. 1 MiB / PAGE_SIZE pages)
+    const minPages = Math.ceil((1024 * 1024) / PAGE_SIZE);
+    const reclaimablePages = last.pageCount;
+    const newTotalPages = Math.max(totalPages - reclaimablePages, minPages);
+    const pagesReclaimed = totalPages - newTotalPages;
+
+    if (pagesReclaimed <= 0) return;
+
+    this.arenaHandle.truncate(newTotalPages * PAGE_SIZE);
+
+    if (pagesReclaimed === last.pageCount) {
+      this.freeList.pop();
+    } else {
+      last.pageCount -= pagesReclaimed;
+    }
+  }
+
+  /**
+   * Full compaction: relocates all live file data to the start of the arena,
+   * eliminates fragmentation, and truncates the arena file.
+   * Call this periodically or after large batch deletions for maximum space savings.
+   */
+  gc() {
+    // Collect all live file extents in the tree
+    const liveFiles: { node: FileNode; oldExtents: FileExtent[] }[] = [];
+    const walk = (node: Node) => {
+      if (node.type === "file" && node.extents.length > 0) {
+        liveFiles.push({ node, oldExtents: node.extents.map((e) => ({ ...e })) });
+      } else if (node.type === "dir") {
+        for (const child of node.children.values()) walk(child);
+      }
+    };
+    walk(this.root);
+
+    // Sort live files by their first extent's start page for sequential reads
+    liveFiles.sort((a, b) => a.oldExtents[0].startPage - b.oldExtents[0].startPage);
+
+    // Relocate each file's data contiguously from page 0 onward
+    let nextPage = 0;
+    for (const { node, oldExtents } of liveFiles) {
+      const data = this.readFileBytes(node);
+      if (!data) continue;
+
+      const pagesNeeded = Math.ceil(node.size / PAGE_SIZE);
+      const newExtent: FileExtent = { startPage: nextPage, pageCount: pagesNeeded };
+
+      // Write data to its new location
+      this.writeBytesToExtents([newExtent], data);
+      node.extents = [newExtent];
+      nextPage += pagesNeeded;
+    }
+
+    // Rebuild free list and truncate
+    const usedPages = nextPage;
+    const minPages = Math.max(usedPages, Math.ceil((1024 * 1024) / PAGE_SIZE));
+    this.arenaHandle.truncate(minPages * PAGE_SIZE);
+    this.allocatedBytes = usedPages * PAGE_SIZE;
+
+    this.freeList = [];
+    if (minPages > usedPages) {
+      this.freeList.push({ startPage: usedPages, pageCount: minPages - usedPages });
+    }
+  }
+
+  /** Returns stats about arena usage for monitoring */
+  getStats(): { arenaBytes: number; allocatedBytes: number; freeBytes: number; freeFragments: number } {
+    const arenaBytes = this.arenaHandle.getSize();
+    const freePages = this.freeList.reduce((sum, r) => sum + r.pageCount, 0);
+    return {
+      arenaBytes,
+      allocatedBytes: this.allocatedBytes,
+      freeBytes: freePages * PAGE_SIZE,
+      freeFragments: this.freeList.length,
+    };
   }
 
   private coalesceFreeList() {