From ef102b50217d30949dcf5f8f364afdc5a53eb48a Mon Sep 17 00:00:00 2001 From: Matt Carey Date: Wed, 8 Jul 2026 10:31:32 +0100 Subject: [PATCH] feat(workspace): configure pull ignore paths --- docs/02_sync_protocol.md | 31 +++++------ docs/03_filesystem_schema.md | 4 +- docs/08_capnweb_interface.md | 2 +- examples/think/src/agent.ts | 3 ++ packages/rpc/src/sync-driver.ts | 17 ++++-- packages/rpc/tests/wire.test.ts | 7 ++- packages/workspace/README.md | 22 ++++++++ packages/workspace/src/workspace.test.ts | 67 ++++++++++++++++++++++++ packages/workspace/src/workspace.ts | 13 ++++- 9 files changed, 138 insertions(+), 28 deletions(-) diff --git a/docs/02_sync_protocol.md b/docs/02_sync_protocol.md index d5699e09..1410fe30 100644 --- a/docs/02_sync_protocol.md +++ b/docs/02_sync_protocol.md @@ -407,14 +407,12 @@ first-class conflict primitives. ## Ignore lists - -The `ignore` option hides path segments from the pull. Excluded -paths are still written and read inside the container — the bytes just -never cross the wire back to the DO. This is essential for any large -directory of derived files: `node_modules`, `.next`, `target`, -`__pycache__`, `dist`. Without an ignore, a single `npm install` would -push tens of thousands of small files through the sync wire on the -next pull. +`WorkspaceOptions.ignore` omits matching path segments from each backend +pull. Filtered paths remain available on the backend that created them, but +their bytes do not cross the wire into the Durable Object. This is essential +for any large directory of derived files: `node_modules`, `.next`, `target`, +`__pycache__`, `dist`. Without an ignore, a single `npm install` would send +tens of thousands of small files through the sync wire on the next pull. The default is `["node_modules"]`, applied server-side when `ignore` is omitted. A caller-supplied list **replaces** the default — it does not @@ -423,12 +421,11 @@ list (including `"node_modules"` if you still want it) to customise. ### Ignored entries -Ignored paths are **invisible to the `Workspace.fs` API**. They do not -appear in `readdir`, `stat` returns `ENOENT`, and `readFile` returns -`ENOENT`. The bytes still live inside the container, so anything that -*uses* the ignored files — `exec("node ...")`, build tools, anything -running container-side — keeps working. The exclusion only affects what -crosses the wire **and** what the DO-side API surfaces. +Paths filtered from a pull do not appear in `Workspace.fs`: they are absent +from `readdir`, and `stat` and `readFile` return `ENOENT`. Commands on the +backend that created the paths can still use them. The setting only filters +backend-to-host pulls; matching files written through `Workspace.fs` remain +visible and can still be pushed to backends. This is a deliberately narrow surface for the initial release. Whether ignored entries should be representable to the DO at all (as stubs, as @@ -442,7 +439,7 @@ case depends on a particular resolution. ### Representing ignored entries to the DO -Today ignored paths are entirely invisible to `Workspace.fs`. That is +Today backend-only ignored paths are invisible to `Workspace.fs`. That is the simplest contract but it loses one piece of information: tools that want to enumerate "everything the agent's exec can see" can't get it from the DO. Two options worth weighing later: @@ -454,8 +451,8 @@ from the DO. Two options worth weighing later: returns container-only entries, `workspace.fs.readdir` stays clean. Cleaner separation, larger API surface. -Either way, the bytes never cross the wire; the question is purely how -much the DO admits exists. +Either way, backend-only bytes stay out of the DO; the question is purely +how much the DO admits exists. ### Bloom/cuckoo filter over `vfs_blobs.hash` diff --git a/docs/03_filesystem_schema.md b/docs/03_filesystem_schema.md index 9df4dfaa..031dabaf 100644 --- a/docs/03_filesystem_schema.md +++ b/docs/03_filesystem_schema.md @@ -74,8 +74,8 @@ The `vfs_nodes_by_rev` index supports `coalesceChanges`'s cursor scan over live inodes, which the sync protocol calls once per pull to enumerate everything modified after the last fetch cursor. -There is no `ignored` column: ignored paths are entirely invisible to -the DO-side filesystem API (see +There is no `ignored` column: backend-only paths filtered from a pull are +not represented in the DO-side filesystem API (see [02. Sync Protocol → Ignored entries](./02_sync_protocol.md#ignored-entries)). ### `vfs_dirents` — name → inode mapping diff --git a/docs/08_capnweb_interface.md b/docs/08_capnweb_interface.md index b28aa5e8..08f9229b 100644 --- a/docs/08_capnweb_interface.md +++ b/docs/08_capnweb_interface.md @@ -309,7 +309,7 @@ type WireError = { | Code | Meaning | | --- | --- | -| `ENOENT` | Path does not exist on the receiver (covers ignored paths, which are invisible to `Workspace.fs`), or `getExec` / `disposeExec` referenced an unknown id. | +| `ENOENT` | Path does not exist on the receiver (including backend-only paths filtered from a pull), or `getExec` / `disposeExec` referenced an unknown id. | | `EUNKNOWN_HASH` | **(reserved, planned)** `fetchObjects` or `pushObjects` referenced a hash the receiver has no record of. Reserved in `WireErrorCode` but not raised today; `pushObjects` should throw it via `createWorkspaceError`. | | `EEXEC_BUSY` | `exec` was called with an `id` that's already in use by a live run. | | `ELOG_TRUNCATED` | `getExec` resume point is older than the retained log. | diff --git a/examples/think/src/agent.ts b/examples/think/src/agent.ts index 347d9db9..1a6d7532 100644 --- a/examples/think/src/agent.ts +++ b/examples/think/src/agent.ts @@ -205,6 +205,9 @@ export class TriageAgent extends withWorkspaceContainer(TriageBase) { }), this.#containerBackend, ], + // Keep generated output on the backend that created it while source + // files still pull into the durable workspace. + ignore: ["node_modules", "dist", "build", ".cache"], // Mount the shared skills bucket at /workspace/.agents. The // R2 keys live under `.agents/` (e.g. // `.agents/skills/triage/SKILL.md`); the prefix is stripped diff --git a/packages/rpc/src/sync-driver.ts b/packages/rpc/src/sync-driver.ts index 75028ba3..660279fc 100644 --- a/packages/rpc/src/sync-driver.ts +++ b/packages/rpc/src/sync-driver.ts @@ -65,15 +65,21 @@ const PULL_BATCH_SIZE = 256; // (rev, path), so a retry can resume inside a single large rev. The // cursor is read and written per backend so concurrent backends keep // independent resume points. +export interface PullOptions { + backend?: string; + // Path segments the remote should omit from this pull. + ignore?: string[]; +} + export async function pullOnce( db: Database, remote: SyncRPC, - backend?: string, + options: PullOptions = {}, ): Promise { // Delegate to the inner implementation with retried=false. See // pullOnceImpl for the fetchChanges round trip, invariant check, // reset-and-retry path, and batched apply loop. - return pullOnceImpl(db, remote, backend, false); + return pullOnceImpl(db, remote, options, false); } // Inner pullOnce that knows whether it is already a retry. The @@ -84,9 +90,10 @@ export async function pullOnce( async function pullOnceImpl( db: Database, remote: SyncRPC, - backend: string | undefined, + options: PullOptions, retried: boolean, ): Promise { + const { backend, ignore } = options; const after = readFetchCursor(db, backend); const localPushRev = readWatermark(db, "pushRev", backend); // fetchChanges hands back the remote's currentCursor (cursor we @@ -103,7 +110,7 @@ async function pullOnceImpl( // invariant trip, and any throw inside the batch loop. Disposing the // envelope tears down the contained stream stub, releasing the // remote iterator. - const fetchResult = await remote.fetchChanges({ after }); + const fetchResult = await remote.fetchChanges({ after, ignore }); try { const { currentCursor, appliedPushCursor } = fetchResult; // Cross-side watermark divergence. Two shapes are recoverable: @@ -163,7 +170,7 @@ async function pullOnceImpl( if (fetchDiverged) { writeFetchCursor(db, { rev: 0, path: null }, backend); } - return pullOnceImpl(db, remote, backend, true); + return pullOnceImpl(db, remote, options, true); } // After the retry path above, this assertion guards a // divergence that survived a reset. Tear down rather than loop. diff --git a/packages/rpc/tests/wire.test.ts b/packages/rpc/tests/wire.test.ts index c432ce03..e0bd755a 100644 --- a/packages/rpc/tests/wire.test.ts +++ b/packages/rpc/tests/wire.test.ts @@ -232,10 +232,12 @@ describe("SyncRPC pull convergence", () => { harness = undefined; }); - it("client pulls a file written via writeFileSync on the server", async () => { + it("client applies pull ignore paths across the wire", async () => { harness = await startHarness(); const provider = new SQLiteWorkspaceProvider(harness.db, { now: () => 1500 }); provider.writeFileSync("/whole.txt", "whole-file write"); + provider.mkdirSync("/dist"); + provider.writeFileSync("/dist/generated.js", "generated"); // Receiver DB — the host's local store in the production setup. const recvStorage = new SQLiteTestStorage(); @@ -245,10 +247,11 @@ describe("SyncRPC pull convergence", () => { const client = createSyncClient({ url: harness.url }); try { const { pullOnce } = await import("../src/sync-driver.js"); - const applied = await pullOnce(recvDb, client); + const applied = await pullOnce(recvDb, client, { ignore: ["dist"] }); expect(applied.applied).toBeGreaterThan(0); const recvProvider = new SQLiteWorkspaceProvider(recvDb, { now: () => 2500 }); expect(recvProvider.readFileSync("/whole.txt", "utf8")).toBe("whole-file write"); + expect(() => recvProvider.statSync("/dist/generated.js")).toThrow(); } finally { await client.close(); recvStorage.close(); diff --git a/packages/workspace/README.md b/packages/workspace/README.md index 7bf386ca..69bce828 100644 --- a/packages/workspace/README.md +++ b/packages/workspace/README.md @@ -239,6 +239,28 @@ A workspace with two backends that both write into [`docs/05_shell_interface.md`](../../docs/05_shell_interface.md) for the caveat. +## Keep generated paths on execution backends + +Use `ignore` to omit generated directories when pulling changes into +`Workspace.fs`: + +```ts +const workspace = new Workspace({ + storage: ctx.storage, + backends: [containerBackend], + ignore: ["node_modules", "dist", "build", ".cache"], +}); +``` + +Patterns match whole path segments and apply to every backend. Omit `ignore` to +use the remote default (`["node_modules"]`). A supplied list replaces that +default; pass `[]` to pull every path. + +Filtered files remain available to commands on the backend where they were +created. Pushes are unaffected: matching files written through `Workspace.fs` +still sync to backends. Choose the list before the first pull; changing it later +does not backfill previously filtered changes or remove local files. + ## Worker-side consumption ```ts diff --git a/packages/workspace/src/workspace.test.ts b/packages/workspace/src/workspace.test.ts index fb192f13..827c0231 100644 --- a/packages/workspace/src/workspace.test.ts +++ b/packages/workspace/src/workspace.test.ts @@ -1,4 +1,6 @@ +import { Database, initializeSchema, SQLiteWorkspaceProvider } from "@cloudflare/dofs"; import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; +import { createSyncServer } from "@cloudflare/workspace-rpc/server"; import { describe, expect, it, vi } from "vitest"; import type { BackendHandle, WorkspaceBackend } from "./backend.js"; @@ -9,6 +11,21 @@ function makeStorage(): SQLiteTestStorage { return new SQLiteTestStorage(); } +function makeRemoteFiles(files: Record): { + rpc: import("@cloudflare/workspace-rpc").SyncRPC; + close(): void; +} { + const storage = makeStorage(); + const db = new Database(storage); + initializeSchema(db, () => 1); + const provider = new SQLiteWorkspaceProvider(db, { now: () => 1 }); + for (const [path, content] of Object.entries(files)) { + provider.mkdirSync(path.slice(0, path.lastIndexOf("/")), { recursive: true }); + provider.writeFileSync(path, content); + } + return { rpc: createSyncServer(db), close: () => storage.close() }; +} + // In-process fakes. We never spawn anything from the package // code; the backend's only contract is "produce a SyncRPC // stub that wsd would speak". A plain object is enough. @@ -652,6 +669,56 @@ describe("Workspace.fs against the local store", () => { }); }); +describe("Workspace.pull ignore", () => { + it("replaces the remote default with the configured path segments", async () => { + const remote = makeRemoteFiles({ + "/workspace/src/index.ts": "source", + "/workspace/dist/index.js": "build", + "/workspace/node_modules/pkg/index.js": "dependency", + }); + try { + const ws = new Workspace({ + storage: makeStorage(), + backends: [makeBackend("container", remote.rpc)], + ignore: ["dist"], + }); + + await ws.pull(); + + expect(await ws.fs.readFile("/workspace/src/index.ts", "utf8")).toBe("source"); + expect(await ws.fs.readFile("/workspace/node_modules/pkg/index.js", "utf8")).toBe( + "dependency", + ); + await expect(ws.fs.stat("/workspace/dist/index.js")).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + remote.close(); + } + }); + + it("pulls node_modules when the configured list is empty", async () => { + const remote = makeRemoteFiles({ + "/workspace/node_modules/pkg/index.js": "dependency", + }); + try { + const ws = new Workspace({ + storage: makeStorage(), + backends: [makeBackend("container", remote.rpc)], + ignore: [], + }); + + await ws.pull(); + + expect(await ws.fs.readFile("/workspace/node_modules/pkg/index.js", "utf8")).toBe( + "dependency", + ); + } finally { + remote.close(); + } + }); +}); + describe("Workspace.pull return shape", () => { it("resolves to the dofs ApplyResult shape", async () => { // The fake SyncRPC's fetchChanges returns an empty stream, so diff --git a/packages/workspace/src/workspace.ts b/packages/workspace/src/workspace.ts index 77e66eac..60818514 100644 --- a/packages/workspace/src/workspace.ts +++ b/packages/workspace/src/workspace.ts @@ -58,6 +58,12 @@ export interface WorkspaceOptions { // factories via MountContext.sessionId. Optional; defaults to "". sessionId?: string; + // Path segments omitted from changes pulled from every backend. + // Omit to use the remote default (`node_modules`). A supplied list + // replaces that default; pass [] to pull every path. Pushes are + // unaffected. + ignore?: string[]; + // Mounts to register against the workspace. Keys are absolute // mount roots (no trailing slash, no nesting). Values are either // bare Mount objects or factories that take a MountContext and @@ -112,6 +118,7 @@ export class Workspace { readonly #observer: WorkspaceObserver; readonly #now: () => number; readonly #sessionId: string; + readonly #ignore: string[] | undefined; readonly #defaultGitIdentity: GitIdentity | undefined; readonly #assets: AssetsClient | undefined; readonly #artifacts: ArtifactClient; @@ -143,6 +150,7 @@ export class Workspace { constructor(options: WorkspaceOptions) { this.#now = options.now ?? Date.now; this.#sessionId = options.sessionId ?? ""; + this.#ignore = options.ignore?.slice(); this.#defaultGitIdentity = options.defaultGitIdentity; this.#artifacts = options.artifacts ? createArtifact( @@ -419,7 +427,10 @@ export class Workspace { const handle = await this.#handleFor(resolvedId); if (handle.sync === "none") return { applied: 0, skipped: [] }; return this.#runWithInvalidation(resolvedId, handle, () => - pullOnce(this.#db, handle.rpc.sync, resolvedId), + pullOnce(this.#db, handle.rpc.sync, { + backend: resolvedId, + ignore: this.#ignore, + }), ); }, (span, outcome) => {