feat(workspace): configure pull ignore paths

Author: mattzcarey

docs/02_sync_protocol.md

@@ -407,14 +407,12 @@ first-class conflict primitives.
 
 ## Ignore lists
 
-
-The `ignore` option hides path segments from the pull. Excluded
-paths are still written and read inside the container — the bytes just
-never cross the wire back to the DO. This is essential for any large
-directory of derived files: `node_modules`, `.next`, `target`,
-`__pycache__`, `dist`. Without an ignore, a single `npm install` would
-push tens of thousands of small files through the sync wire on the
-next pull.
+`WorkspaceOptions.ignore` omits matching path segments from each backend
+pull. Filtered paths remain available on the backend that created them, but
+their bytes do not cross the wire into the Durable Object. This is essential
+for any large directory of derived files: `node_modules`, `.next`, `target`,
+`__pycache__`, `dist`. Without an ignore, a single `npm install` would send
+tens of thousands of small files through the sync wire on the next pull.
 
 The default is `["node_modules"]`, applied server-side when `ignore` is
 omitted. A caller-supplied list **replaces** the default — it does not
@@ -423,12 +421,11 @@ list (including `"node_modules"` if you still want it) to customise.
 
 ### Ignored entries
 
-Ignored paths are **invisible to the `Workspace.fs` API**. They do not
-appear in `readdir`, `stat` returns `ENOENT`, and `readFile` returns
-`ENOENT`. The bytes still live inside the container, so anything that
-*uses* the ignored files — `exec("node ...")`, build tools, anything
-running container-side — keeps working. The exclusion only affects what
-crosses the wire **and** what the DO-side API surfaces.
+Paths filtered from a pull do not appear in `Workspace.fs`: they are absent
+from `readdir`, and `stat` and `readFile` return `ENOENT`. Commands on the
+backend that created the paths can still use them. The setting only filters
+backend-to-host pulls; matching files written through `Workspace.fs` remain
+visible and can still be pushed to backends.
 
 This is a deliberately narrow surface for the initial release. Whether
 ignored entries should be representable to the DO at all (as stubs, as
@@ -442,7 +439,7 @@ case depends on a particular resolution.
 
 ### Representing ignored entries to the DO
 
-Today ignored paths are entirely invisible to `Workspace.fs`. That is
+Today backend-only ignored paths are invisible to `Workspace.fs`. That is
 the simplest contract but it loses one piece of information: tools that
 want to enumerate "everything the agent's exec can see" can't get it
 from the DO. Two options worth weighing later:
@@ -454,8 +451,8 @@ from the DO. Two options worth weighing later:
   returns container-only entries, `workspace.fs.readdir` stays clean.
   Cleaner separation, larger API surface.
 
-Either way, the bytes never cross the wire; the question is purely how
-much the DO admits exists.
+Either way, backend-only bytes stay out of the DO; the question is purely
+how much the DO admits exists.
 
 ### Bloom/cuckoo filter over `vfs_blobs.hash`
 

docs/03_filesystem_schema.md

@@ -74,8 +74,8 @@ The `vfs_nodes_by_rev` index supports `coalesceChanges`'s cursor scan
 over live inodes, which the sync protocol calls once per pull to
 enumerate everything modified after the last fetch cursor.
 
-There is no `ignored` column: ignored paths are entirely invisible to
-the DO-side filesystem API (see
+There is no `ignored` column: backend-only paths filtered from a pull are
+not represented in the DO-side filesystem API (see
 [02. Sync Protocol → Ignored entries](./02_sync_protocol.md#ignored-entries)).
 
 ### `vfs_dirents` — name → inode mapping

docs/08_capnweb_interface.md

@@ -309,7 +309,7 @@ type WireError = {
 
 | Code | Meaning |
 | --- | --- |
-| `ENOENT` | Path does not exist on the receiver (covers ignored paths, which are invisible to `Workspace.fs`), or `getExec` / `disposeExec` referenced an unknown id. |
+| `ENOENT` | Path does not exist on the receiver (including backend-only paths filtered from a pull), or `getExec` / `disposeExec` referenced an unknown id. |
 | `EUNKNOWN_HASH` | **(reserved, planned)** `fetchObjects` or `pushObjects` referenced a hash the receiver has no record of. Reserved in `WireErrorCode` but not raised today; `pushObjects` should throw it via `createWorkspaceError`. |
 | `EEXEC_BUSY` | `exec` was called with an `id` that's already in use by a live run. |
 | `ELOG_TRUNCATED` | `getExec` resume point is older than the retained log. |

examples/think/src/agent.ts

@@ -205,6 +205,9 @@ export class TriageAgent extends withWorkspaceContainer(TriageBase) {
         }),
         this.#containerBackend,
       ],
+      // Keep generated output on the backend that created it while source
+      // files still pull into the durable workspace.
+      ignore: ["node_modules", "dist", "build", ".cache"],
       // Mount the shared skills bucket at /workspace/.agents. The
       // R2 keys live under `.agents/` (e.g.
       // `.agents/skills/triage/SKILL.md`); the prefix is stripped

packages/rpc/src/sync-driver.ts

@@ -65,15 +65,21 @@ const PULL_BATCH_SIZE = 256;
 // (rev, path), so a retry can resume inside a single large rev. The
 // cursor is read and written per backend so concurrent backends keep
 // independent resume points.
+export interface PullOptions {
+  backend?: string;
+  // Path segments the remote should omit from this pull.
+  ignore?: string[];
+}
+
 export async function pullOnce(
   db: Database,
   remote: SyncRPC,
-  backend?: string,
+  options: PullOptions = {},
 ): Promise<ApplyResult> {
   // Delegate to the inner implementation with retried=false. See
   // pullOnceImpl for the fetchChanges round trip, invariant check,
   // reset-and-retry path, and batched apply loop.
-  return pullOnceImpl(db, remote, backend, false);
+  return pullOnceImpl(db, remote, options, false);
 }
 
 // Inner pullOnce that knows whether it is already a retry. The
@@ -84,9 +90,10 @@ export async function pullOnce(
 async function pullOnceImpl(
   db: Database,
   remote: SyncRPC,
-  backend: string | undefined,
+  options: PullOptions,
   retried: boolean,
 ): Promise<ApplyResult> {
+  const { backend, ignore } = options;
   const after = readFetchCursor(db, backend);
   const localPushRev = readWatermark(db, "pushRev", backend);
   // fetchChanges hands back the remote's currentCursor (cursor we
@@ -103,7 +110,7 @@ async function pullOnceImpl(
   // invariant trip, and any throw inside the batch loop. Disposing the
   // envelope tears down the contained stream stub, releasing the
   // remote iterator.
-  const fetchResult = await remote.fetchChanges({ after });
+  const fetchResult = await remote.fetchChanges({ after, ignore });
   try {
     const { currentCursor, appliedPushCursor } = fetchResult;
     // Cross-side watermark divergence. Two shapes are recoverable:
@@ -163,7 +170,7 @@ async function pullOnceImpl(
       if (fetchDiverged) {
         writeFetchCursor(db, { rev: 0, path: null }, backend);
       }
-      return pullOnceImpl(db, remote, backend, true);
+      return pullOnceImpl(db, remote, options, true);
     }
     // After the retry path above, this assertion guards a
     // divergence that survived a reset. Tear down rather than loop.

packages/rpc/tests/wire.test.ts

@@ -232,10 +232,12 @@ describe("SyncRPC pull convergence", () => {
     harness = undefined;
   });
 
-  it("client pulls a file written via writeFileSync on the server", async () => {
+  it("client applies pull ignore paths across the wire", async () => {
     harness = await startHarness();
     const provider = new SQLiteWorkspaceProvider(harness.db, { now: () => 1500 });
     provider.writeFileSync("/whole.txt", "whole-file write");
+    provider.mkdirSync("/dist");
+    provider.writeFileSync("/dist/generated.js", "generated");
 
     // Receiver DB — the host's local store in the production setup.
     const recvStorage = new SQLiteTestStorage();
@@ -245,10 +247,11 @@ describe("SyncRPC pull convergence", () => {
     const client = createSyncClient({ url: harness.url });
     try {
       const { pullOnce } = await import("../src/sync-driver.js");
-      const applied = await pullOnce(recvDb, client);
+      const applied = await pullOnce(recvDb, client, { ignore: ["dist"] });
       expect(applied.applied).toBeGreaterThan(0);
       const recvProvider = new SQLiteWorkspaceProvider(recvDb, { now: () => 2500 });
       expect(recvProvider.readFileSync("/whole.txt", "utf8")).toBe("whole-file write");
+      expect(() => recvProvider.statSync("/dist/generated.js")).toThrow();
     } finally {
       await client.close();
       recvStorage.close();

packages/workspace/README.md

@@ -239,6 +239,28 @@ A workspace with two backends that both write into
 [`docs/05_shell_interface.md`](../../docs/05_shell_interface.md)
 for the caveat.
 
+## Keep generated paths on execution backends
+
+Use `ignore` to omit generated directories when pulling changes into
+`Workspace.fs`:
+
+```ts
+const workspace = new Workspace({
+  storage: ctx.storage,
+  backends: [containerBackend],
+  ignore: ["node_modules", "dist", "build", ".cache"],
+});
+```
+
+Patterns match whole path segments and apply to every backend. Omit `ignore` to
+use the remote default (`["node_modules"]`). A supplied list replaces that
+default; pass `[]` to pull every path.
+
+Filtered files remain available to commands on the backend where they were
+created. Pushes are unaffected: matching files written through `Workspace.fs`
+still sync to backends. Choose the list before the first pull; changing it later
+does not backfill previously filtered changes or remove local files.
+
 ## Worker-side consumption
 
 ```ts

packages/workspace/src/workspace.test.ts

@@ -1,4 +1,6 @@
+import { Database, initializeSchema, SQLiteWorkspaceProvider } from "@cloudflare/dofs";
 import { SQLiteTestStorage } from "@cloudflare/dofs/testing";
+import { createSyncServer } from "@cloudflare/workspace-rpc/server";
 import { describe, expect, it, vi } from "vitest";
 
 import type { BackendHandle, WorkspaceBackend } from "./backend.js";
@@ -9,6 +11,21 @@ function makeStorage(): SQLiteTestStorage {
   return new SQLiteTestStorage();
 }
 
+function makeRemoteFiles(files: Record<string, string>): {
+  rpc: import("@cloudflare/workspace-rpc").SyncRPC;
+  close(): void;
+} {
+  const storage = makeStorage();
+  const db = new Database(storage);
+  initializeSchema(db, () => 1);
+  const provider = new SQLiteWorkspaceProvider(db, { now: () => 1 });
+  for (const [path, content] of Object.entries(files)) {
+    provider.mkdirSync(path.slice(0, path.lastIndexOf("/")), { recursive: true });
+    provider.writeFileSync(path, content);
+  }
+  return { rpc: createSyncServer(db), close: () => storage.close() };
+}
+
 // In-process fakes. We never spawn anything from the package
 // code; the backend's only contract is "produce a SyncRPC
 // stub that wsd would speak". A plain object is enough.
@@ -652,6 +669,56 @@ describe("Workspace.fs against the local store", () => {
   });
 });
 
+describe("Workspace.pull ignore", () => {
+  it("replaces the remote default with the configured path segments", async () => {
+    const remote = makeRemoteFiles({
+      "/workspace/src/index.ts": "source",
+      "/workspace/dist/index.js": "build",
+      "/workspace/node_modules/pkg/index.js": "dependency",
+    });
+    try {
+      const ws = new Workspace({
+        storage: makeStorage(),
+        backends: [makeBackend("container", remote.rpc)],
+        ignore: ["dist"],
+      });
+
+      await ws.pull();
+
+      expect(await ws.fs.readFile("/workspace/src/index.ts", "utf8")).toBe("source");
+      expect(await ws.fs.readFile("/workspace/node_modules/pkg/index.js", "utf8")).toBe(
+        "dependency",
+      );
+      await expect(ws.fs.stat("/workspace/dist/index.js")).rejects.toMatchObject({
+        code: "ENOENT",
+      });
+    } finally {
+      remote.close();
+    }
+  });
+
+  it("pulls node_modules when the configured list is empty", async () => {
+    const remote = makeRemoteFiles({
+      "/workspace/node_modules/pkg/index.js": "dependency",
+    });
+    try {
+      const ws = new Workspace({
+        storage: makeStorage(),
+        backends: [makeBackend("container", remote.rpc)],
+        ignore: [],
+      });
+
+      await ws.pull();
+
+      expect(await ws.fs.readFile("/workspace/node_modules/pkg/index.js", "utf8")).toBe(
+        "dependency",
+      );
+    } finally {
+      remote.close();
+    }
+  });
+});
+
 describe("Workspace.pull return shape", () => {
   it("resolves to the dofs ApplyResult shape", async () => {
     // The fake SyncRPC's fetchChanges returns an empty stream, so

packages/workspace/src/workspace.ts

@@ -58,6 +58,12 @@ export interface WorkspaceOptions {
   // factories via MountContext.sessionId. Optional; defaults to "".
   sessionId?: string;
 
+  // Path segments omitted from changes pulled from every backend.
+  // Omit to use the remote default (`node_modules`). A supplied list
+  // replaces that default; pass [] to pull every path. Pushes are
+  // unaffected.
+  ignore?: string[];
+
   // Mounts to register against the workspace. Keys are absolute
   // mount roots (no trailing slash, no nesting). Values are either
   // bare Mount objects or factories that take a MountContext and
@@ -112,6 +118,7 @@ export class Workspace {
   readonly #observer: WorkspaceObserver;
   readonly #now: () => number;
   readonly #sessionId: string;
+  readonly #ignore: string[] | undefined;
   readonly #defaultGitIdentity: GitIdentity | undefined;
   readonly #assets: AssetsClient | undefined;
   readonly #artifacts: ArtifactClient;
@@ -143,6 +150,7 @@ export class Workspace {
   constructor(options: WorkspaceOptions) {
     this.#now = options.now ?? Date.now;
     this.#sessionId = options.sessionId ?? "";
+    this.#ignore = options.ignore?.slice();
     this.#defaultGitIdentity = options.defaultGitIdentity;
     this.#artifacts = options.artifacts
       ? createArtifact(
@@ -419,7 +427,10 @@ export class Workspace {
           const handle = await this.#handleFor(resolvedId);
           if (handle.sync === "none") return { applied: 0, skipped: [] };
           return this.#runWithInvalidation(resolvedId, handle, () =>
-            pullOnce(this.#db, handle.rpc.sync, resolvedId),
+            pullOnce(this.#db, handle.rpc.sync, {
+              backend: resolvedId,
+              ignore: this.#ignore,
+            }),
           );
         },
         (span, outcome) => {