From ba4646af8f711cbad013d823e47239584ae7fc1e Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:54:10 +0100 Subject: [PATCH 01/20] feat(core): add guarded collection deletion foundation --- packages/cloudflare/src/db/d1.ts | 177 +++++++++++ packages/cloudflare/src/db/do-sql-class.ts | 76 +++++ packages/cloudflare/src/db/do-sql-types.ts | 3 + packages/cloudflare/src/db/do-sql.ts | 13 + packages/cloudflare/src/index.ts | 2 + .../db/do-sql-collection-deletion.test.ts | 86 ++++++ packages/cloudflare/tests/do-config.test.ts | 2 + .../src/astro/integration/virtual-modules.ts | 10 +- .../core/src/astro/integration/vite-config.ts | 2 + .../065_media_usage_collection_deletion.ts | 85 ++++++ .../core/src/database/migrations/runner.ts | 2 + packages/core/src/database/types.ts | 19 ++ packages/core/src/db/adapters.ts | 28 ++ packages/core/src/db/index.ts | 3 + packages/core/src/index.ts | 3 + .../src/media/usage/collection-deletion.ts | 282 ++++++++++++++++++ packages/core/src/virtual-modules.d.ts | 2 + ...age-collection-deletion-foundation.test.ts | 176 +++++++++++ ...sage-collection-deletion-migration.test.ts | 73 +++++ ...a-usage-incremental-work-migration.test.ts | 6 + .../integration/database/migrations.test.ts | 2 + .../astro/integration/virtual-modules.test.ts | 7 + ...media-usage-collection-deletion-d1.test.ts | 87 ++++++ ...edia-usage-projection-admission-d1.test.ts | 4 +- 24 files changed, 1148 insertions(+), 2 deletions(-) create mode 100644 packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts create mode 100644 packages/core/src/database/migrations/065_media_usage_collection_deletion.ts create mode 100644 packages/core/src/media/usage/collection-deletion.ts create mode 100644 packages/core/tests/integration/database/media-usage-collection-deletion-foundation.test.ts create mode 100644 packages/core/tests/integration/database/media-usage-collection-deletion-migration.test.ts create mode 100644 packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts diff --git a/packages/cloudflare/src/db/d1.ts b/packages/cloudflare/src/db/d1.ts index 2b7357967b..422650331a 100644 --- a/packages/cloudflare/src/db/d1.ts +++ b/packages/cloudflare/src/db/d1.ts @@ -9,6 +9,7 @@ */ import { env } from "cloudflare:workers"; +import type { CollectionDeletionGuardInput, CollectionDeletionGuardResult } from "emdash"; import { kyselyLogOption } from "emdash/database/instrumentation"; import { type Dialect, Kysely } from "kysely"; @@ -26,6 +27,8 @@ interface D1Config { } const DEFAULT_BOOKMARK_COOKIE = "__em_d1_bookmark"; +const COLLECTION_SLUG_PATTERN = /^[a-z][a-z0-9_]*$/; +const STALE_DELETION_GUARD_PATTERN = /collection_id.*not null|not null.*collection_id/i; /** * One-shot guard so the "coalesce opted in but the binding can't do sessions @@ -143,6 +146,180 @@ export interface RequestScopedDb { commit: () => void; } +export async function executeCollectionDeletionGuard( + config: D1Config, + input: CollectionDeletionGuardInput, +): Promise { + assertCollectionDeletionInput(input); + const binding = getBinding(config); + if (!binding) throw new Error(`D1 binding "${config.binding}" not found in environment.`); + return input.action === "fence" + ? executeFenceBatch(binding, input) + : executeDropBatch(binding, input); +} + +async function executeFenceBatch( + binding: D1Database, + input: Extract, +): Promise { + const tableName = `ec_${input.collectionSlug}`; + const contentPredicate = input.forceDelete + ? "" + : `AND NOT EXISTS (SELECT 1 FROM "${tableName}" LIMIT 1)`; + const update = binding + .prepare(` + UPDATE _emdash_media_usage_index_status + SET capture_state = 'deleting', + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE adapter_id = 'content-media' + AND scope_type = 'collection' + AND scope_key = ? + AND collection_id = ? + AND capture_state = 'active' + AND EXISTS ( + SELECT 1 FROM _emdash_collections + WHERE id = ? AND slug = ? + ) + AND EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions + WHERE collection_id = ? + AND collection_slug = ? + AND state = 'leased' + AND phase = 'fence' + AND lease_token = ? + AND lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + ) + ${contentPredicate} + RETURNING collection_id + `) + .bind( + input.collectionSlug, + input.collectionId, + input.collectionId, + input.collectionSlug, + input.collectionId, + input.collectionSlug, + input.leaseToken, + ); + const diagnostic = binding + .prepare(` + SELECT CASE + WHEN EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions + WHERE collection_id = ? + AND collection_slug = ? + AND state = 'leased' + AND phase = 'fence' + AND lease_token = ? + AND lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + ) + AND EXISTS ( + SELECT 1 FROM _emdash_media_usage_index_status + WHERE adapter_id = 'content-media' + AND scope_type = 'collection' + AND scope_key = ? + AND collection_id = ? + AND capture_state = 'active' + ) + AND EXISTS (SELECT 1 FROM "${tableName}" LIMIT 1) + THEN 'has_content' + ELSE 'stale' + END AS outcome + `) + .bind( + input.collectionId, + input.collectionSlug, + input.leaseToken, + input.collectionSlug, + input.collectionId, + ); + const [updated, observed] = await binding.batch<{ collection_id: string } | { outcome: string }>([ + update, + diagnostic, + ]); + if (updated?.results.length) return { outcome: "fenced" }; + const diagnosticRow = observed?.results[0]; + return diagnosticRow && "outcome" in diagnosticRow && diagnosticRow.outcome === "has_content" + ? { outcome: "has_content" } + : { outcome: "stale" }; +} + +async function executeDropBatch( + binding: D1Database, + input: Extract, +): Promise { + const contentTable = `ec_${input.collectionSlug}`; + const ftsTable = `_emdash_fts_${input.collectionSlug}`; + const guardId = `__emdash_guard:${input.collectionId}:${input.leaseToken}`; + const guardSlug = `__emdash_guard_slug:${input.collectionId}:${input.leaseToken}`; + try { + await binding.batch([ + binding + .prepare(` + INSERT INTO _emdash_media_usage_collection_deletions ( + collection_id, collection_slug, force_delete, state, phase, + next_attempt_at, lease_token, lease_expires_at + ) + VALUES ( + ( + SELECT ? + FROM _emdash_media_usage_collection_deletions + WHERE collection_id = ? + AND collection_slug = ? + AND state = 'leased' + AND phase = 'table' + AND lease_token = ? + AND lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + ), + ?, 0, 'leased', 'table', + strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), ?, + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '+1 minute') + ) + `) + .bind( + guardId, + input.collectionId, + input.collectionSlug, + input.leaseToken, + guardSlug, + input.leaseToken, + ), + binding.prepare(`DROP TRIGGER IF EXISTS "${ftsTable}_insert"`), + binding.prepare(`DROP TRIGGER IF EXISTS "${ftsTable}_update"`), + binding.prepare(`DROP TRIGGER IF EXISTS "${ftsTable}_delete"`), + binding.prepare(`DROP TABLE IF EXISTS "${ftsTable}"`), + binding.prepare(`DROP TABLE IF EXISTS "${contentTable}"`), + binding + .prepare( + "DELETE FROM _emdash_media_usage_collection_deletions WHERE collection_id = ? AND collection_slug = ?", + ) + .bind(guardId, guardSlug), + ]); + } catch (error) { + if (STALE_DELETION_GUARD_PATTERN.test(deepErrorMessage(error))) { + return { outcome: "stale" }; + } + throw error; + } + return { outcome: "dropped" }; +} + +function assertCollectionDeletionInput(input: CollectionDeletionGuardInput): void { + if (!input.collectionId || !input.leaseToken) { + throw new Error("Collection deletion guard requires a collection ID and lease token"); + } + if (!COLLECTION_SLUG_PATTERN.test(input.collectionSlug) || input.collectionSlug.length > 63) { + throw new Error("Collection deletion guard requires a valid collection slug"); + } +} + +function deepErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.cause ? `${error.message}: ${deepErrorMessage(error.cause)}` : error.message; + } + return String(error); +} + /** * Create a per-request session-backed Kysely, or null when D1 sessions are * disabled or the binding is missing. Core middleware calls this once per diff --git a/packages/cloudflare/src/db/do-sql-class.ts b/packages/cloudflare/src/db/do-sql-class.ts index a6b25d3af2..3078fb496d 100644 --- a/packages/cloudflare/src/db/do-sql-class.ts +++ b/packages/cloudflare/src/db/do-sql-class.ts @@ -28,6 +28,7 @@ */ import { DurableObject } from "cloudflare:workers"; +import type { CollectionDeletionGuardInput, CollectionDeletionGuardResult } from "emdash"; import type { DOQueryResult, DOQueryStatement, EmDashDBStub } from "./do-sql-types.js"; import { isPragmaStatement, isReadStatement } from "./do-sql-types.js"; @@ -52,6 +53,7 @@ interface ReplicationStorage { } const READONLY_ERROR_PATTERN = /readonly database/i; +const COLLECTION_SLUG_PATTERN = /^[a-z][a-z0-9_]*$/; /** * Upper bound on how long a read will block waiting for a replica to catch up to @@ -185,6 +187,71 @@ export class EmDashDB extends DurableObject { return { rows, changes: cursor.rowsWritten, bookmark: await this.#currentBookmark() }; } + async executeCollectionDeletionGuard( + input: CollectionDeletionGuardInput, + ): Promise { + this.#ensureReplication(); + if (this.#isReplica) { + return this.#primaryStub!.executeCollectionDeletionGuard(input); + } + assertCollectionDeletionInput(input); + return this.ctx.storage.transactionSync(() => { + const phase = input.action === "fence" ? "fence" : "table"; + const guard = this.ctx.storage.sql.exec( + `SELECT collection_id + FROM _emdash_media_usage_collection_deletions + WHERE collection_id = ? + AND collection_slug = ? + AND state = 'leased' + AND phase = ? + AND lease_token = ? + AND lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`, + input.collectionId, + input.collectionSlug, + phase, + input.leaseToken, + ); + if (guard.toArray().length !== 1) return { outcome: "stale" }; + + const contentTable = `ec_${input.collectionSlug}`; + if (input.action === "fence") { + if (!input.forceDelete) { + const content = this.ctx.storage.sql.exec( + `SELECT 1 AS present FROM "${contentTable}" LIMIT 1`, + ); + if (content.toArray().length > 0) return { outcome: "has_content" }; + } + const updated = this.ctx.storage.sql.exec( + `UPDATE _emdash_media_usage_index_status + SET capture_state = 'deleting', + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE adapter_id = 'content-media' + AND scope_type = 'collection' + AND scope_key = ? + AND collection_id = ? + AND capture_state = 'active' + AND EXISTS ( + SELECT 1 FROM _emdash_collections + WHERE id = ? AND slug = ? + )`, + input.collectionSlug, + input.collectionId, + input.collectionId, + input.collectionSlug, + ); + return updated.rowsWritten === 1 ? { outcome: "fenced" } : { outcome: "stale" }; + } + + const ftsTable = `_emdash_fts_${input.collectionSlug}`; + this.ctx.storage.sql.exec(`DROP TRIGGER IF EXISTS "${ftsTable}_insert"`); + this.ctx.storage.sql.exec(`DROP TRIGGER IF EXISTS "${ftsTable}_update"`); + this.ctx.storage.sql.exec(`DROP TRIGGER IF EXISTS "${ftsTable}_delete"`); + this.ctx.storage.sql.exec(`DROP TABLE IF EXISTS "${ftsTable}"`); + this.ctx.storage.sql.exec(`DROP TABLE IF EXISTS "${contentTable}"`); + return { outcome: "dropped" }; + }); + } + /** * Execute several read statements in a single RPC, returning one result per * statement in order. This is the round-trip win: a page that issues ~17 @@ -223,3 +290,12 @@ export class EmDashDB extends DurableObject { }); } } + +function assertCollectionDeletionInput(input: CollectionDeletionGuardInput): void { + if (!input.collectionId || !input.leaseToken) { + throw new Error("Collection deletion guard requires a collection ID and lease token"); + } + if (!COLLECTION_SLUG_PATTERN.test(input.collectionSlug) || input.collectionSlug.length > 63) { + throw new Error("Collection deletion guard requires a valid collection slug"); + } +} diff --git a/packages/cloudflare/src/db/do-sql-types.ts b/packages/cloudflare/src/db/do-sql-types.ts index 001a387fbd..cb08d8c7e5 100644 --- a/packages/cloudflare/src/db/do-sql-types.ts +++ b/packages/cloudflare/src/db/do-sql-types.ts @@ -86,6 +86,9 @@ export interface EmDashDBStub { statements: DOQueryStatement[], opts?: { bookmark?: string }, ): Promise; + executeCollectionDeletionGuard( + input: import("emdash").CollectionDeletionGuardInput, + ): Promise; } /** diff --git a/packages/cloudflare/src/db/do-sql.ts b/packages/cloudflare/src/db/do-sql.ts index f18c8a236d..1b5f5eb583 100644 --- a/packages/cloudflare/src/db/do-sql.ts +++ b/packages/cloudflare/src/db/do-sql.ts @@ -13,6 +13,7 @@ */ import { env } from "cloudflare:workers"; +import type { CollectionDeletionGuardInput, CollectionDeletionGuardResult } from "emdash"; import { kyselyLogOption, recordRpc } from "emdash/database/instrumentation"; import { type Dialect, Kysely } from "kysely"; @@ -61,6 +62,18 @@ function bindingError(binding: string): Error { ); } +export async function executeCollectionDeletionGuard( + config: DurableObjectsConfig, + input: CollectionDeletionGuardInput, +): Promise { + const ns = getNamespace(config); + if (!ns) throw bindingError(config.binding); + const id = ns.idFromName(config.name ?? DEFAULT_NAME); + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- Rpc type limitation with unknown row types + const stub = ns.get(id) as unknown as EmDashDBStub; + return stub.executeCollectionDeletionGuard(input); +} + /** * Bookmark sinks for the non-request-scoped dialects, keyed by DO identity. * diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index c19d46a266..41782d0e64 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -276,6 +276,7 @@ export function d1(config: D1Config): DatabaseDescriptor { type: "sqlite", supportsRequestScope: true, supportsCoalescing: true, + supportsCollectionDeletionGuard: true, }; } @@ -398,6 +399,7 @@ export function durableObjects(config: DurableObjectsConfig): DatabaseDescriptor type: "sqlite", supportsRequestScope: true, supportsCoalescing: true, + supportsCollectionDeletionGuard: true, }; } diff --git a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts new file mode 100644 index 0000000000..8aecdfa869 --- /dev/null +++ b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts @@ -0,0 +1,86 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("cloudflare:workers", () => ({ + DurableObject: class { + ctx: unknown; + + constructor(ctx: unknown) { + this.ctx = ctx; + } + }, +})); + +import { EmDashDB } from "../../src/db/do-sql-class.js"; + +interface FakeCursor { + rowsWritten: number; + toArray(): Record[]; + [Symbol.iterator](): Iterator>; +} + +function cursor(rows: Record[] = [], rowsWritten = 0): FakeCursor { + return { + rowsWritten, + toArray: () => rows, + [Symbol.iterator]: () => rows[Symbol.iterator](), + }; +} + +describe("EmDashDB collection deletion guard", () => { + let statements: string[]; + let transactionSync: ReturnType; + + beforeEach(() => { + statements = []; + transactionSync = vi.fn((operation: () => unknown) => operation()); + }); + + it("returns stale before dispatching DDL when the exact lease is absent", async () => { + const sql = { + exec: vi.fn((statement: string) => { + statements.push(statement); + return cursor(); + }), + }; + const object = new EmDashDB({ storage: { sql, transactionSync } } as never, {}); + + await expect( + object.executeCollectionDeletionGuard({ + action: "drop", + collectionId: "collection-1", + collectionSlug: "articles", + leaseToken: "stale-owner", + }), + ).resolves.toEqual({ outcome: "stale" }); + + expect(transactionSync).toHaveBeenCalledOnce(); + expect(statements).toHaveLength(1); + expect(statements[0]).toContain("SELECT collection_id"); + expect(statements.some((statement) => statement.includes("DROP TABLE"))).toBe(false); + }); + + it("executes the exact drop inside one synchronous primary transaction", async () => { + const sql = { + exec: vi.fn((statement: string) => { + statements.push(statement); + return statement.includes("SELECT collection_id") + ? cursor([{ collection_id: "collection-1" }]) + : cursor(); + }), + }; + const object = new EmDashDB({ storage: { sql, transactionSync } } as never, {}); + + await expect( + object.executeCollectionDeletionGuard({ + action: "drop", + collectionId: "collection-1", + collectionSlug: "articles", + leaseToken: "current-owner", + }), + ).resolves.toEqual({ outcome: "dropped" }); + + expect(transactionSync).toHaveBeenCalledOnce(); + expect(statements.filter((statement) => statement.includes("DROP TRIGGER"))).toHaveLength(3); + expect(statements.filter((statement) => statement.includes("DROP TABLE"))).toHaveLength(2); + }); +}); diff --git a/packages/cloudflare/tests/do-config.test.ts b/packages/cloudflare/tests/do-config.test.ts index 51ed9c004f..456f2482bc 100644 --- a/packages/cloudflare/tests/do-config.test.ts +++ b/packages/cloudflare/tests/do-config.test.ts @@ -7,6 +7,7 @@ describe("d1()", () => { const result = d1({ binding: "DB" }); expect(result.supportsRequestScope).toBe(true); expect(result.supportsCoalescing).toBe(true); + expect(result.supportsCollectionDeletionGuard).toBe(true); }); }); @@ -15,6 +16,7 @@ describe("durableObjects()", () => { const result = durableObjects({ binding: "DB_DO" }); expect(result.supportsRequestScope).toBe(true); expect(result.supportsCoalescing).toBe(true); + expect(result.supportsCollectionDeletionGuard).toBe(true); }); }); diff --git a/packages/core/src/astro/integration/virtual-modules.ts b/packages/core/src/astro/integration/virtual-modules.ts index d8ba59adef..201141ef0f 100644 --- a/packages/core/src/astro/integration/virtual-modules.ts +++ b/packages/core/src/astro/integration/virtual-modules.ts @@ -100,14 +100,17 @@ export function generateDialectModule(opts: { type?: string; supportsRequestScope: boolean; supportsCoalescing: boolean; + supportsCollectionDeletionGuard: boolean; }): string { - const { entrypoint, supportsRequestScope, supportsCoalescing } = opts; + const { entrypoint, supportsRequestScope, supportsCoalescing, supportsCollectionDeletionGuard } = + opts; if (!entrypoint) { return [ `export const createDialect = undefined;`, `export const dialectType = "sqlite";`, `export const createRequestScopedDb = (_opts) => null;`, `export const createCoalescingDialect = undefined;`, + `export const executeCollectionDeletionGuard = undefined;`, ].join("\n"); } const type = opts.type ?? "sqlite"; @@ -116,12 +119,16 @@ export function generateDialectModule(opts: { ? `import { createCoalescingDialect as _createCoalescingDialect } from "${entrypoint}"; export const createCoalescingDialect = _createCoalescingDialect;` : `export const createCoalescingDialect = undefined;`; + const collectionDeletionExport = supportsCollectionDeletionGuard + ? `export { executeCollectionDeletionGuard } from "${entrypoint}";` + : `export const executeCollectionDeletionGuard = undefined;`; if (supportsRequestScope) { return ` import { createDialect as _createDialect } from "${entrypoint}"; export { createRequestScopedDb } from "${entrypoint}"; ${coalescingExport} +${collectionDeletionExport} export const createDialect = _createDialect; export const dialectType = ${JSON.stringify(type)}; `; @@ -130,6 +137,7 @@ export const dialectType = ${JSON.stringify(type)}; return ` import { createDialect as _createDialect } from "${entrypoint}"; ${coalescingExport} +${collectionDeletionExport} export const createDialect = _createDialect; export const dialectType = ${JSON.stringify(type)}; export const createRequestScopedDb = (_opts) => null; diff --git a/packages/core/src/astro/integration/vite-config.ts b/packages/core/src/astro/integration/vite-config.ts index 67ed100c6a..d04d2e59ae 100644 --- a/packages/core/src/astro/integration/vite-config.ts +++ b/packages/core/src/astro/integration/vite-config.ts @@ -257,6 +257,8 @@ export function createVirtualModulesPlugin( type: resolvedConfig.database?.type, supportsRequestScope: resolvedConfig.database?.supportsRequestScope ?? false, supportsCoalescing: resolvedConfig.database?.supportsCoalescing ?? false, + supportsCollectionDeletionGuard: + resolvedConfig.database?.supportsCollectionDeletionGuard ?? false, }); } // Generate a module that statically imports the configured storage diff --git a/packages/core/src/database/migrations/065_media_usage_collection_deletion.ts b/packages/core/src/database/migrations/065_media_usage_collection_deletion.ts new file mode 100644 index 0000000000..d02a396553 --- /dev/null +++ b/packages/core/src/database/migrations/065_media_usage_collection_deletion.ts @@ -0,0 +1,85 @@ +import { sql, type Kysely, type RawBuilder } from "kysely"; + +import { isPostgres } from "../dialect-helpers.js"; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable("_emdash_media_usage_collection_deletions") + .ifNotExists() + .addColumn("collection_id", "text", (column) => column.notNull().primaryKey()) + .addColumn("collection_slug", "text", (column) => column.notNull().unique()) + .addColumn("force_delete", "integer", (column) => column.notNull()) + .addColumn("state", "text", (column) => column.notNull().defaultTo("pending")) + .addColumn("phase", "text", (column) => column.notNull().defaultTo("fence")) + .addColumn("work_cursor", "text") + .addColumn("source_key", "text") + .addColumn("occurrence_cursor", "text") + .addColumn("attempt_count", "integer", (column) => column.notNull().defaultTo(0)) + .addColumn("next_attempt_at", "text", (column) => column.notNull()) + .addColumn("lease_token", "text") + .addColumn("lease_expires_at", "text") + .addColumn("last_error_code", "text") + .addColumn("created_at", "text", (column) => + column.notNull().defaultTo(sortableUtcTimestamp(db)), + ) + .addColumn("updated_at", "text", (column) => + column.notNull().defaultTo(sortableUtcTimestamp(db)), + ) + .execute(); + + await db.schema + .createIndex("idx__emdash_media_usage_collection_deletions_due") + .ifNotExists() + .on("_emdash_media_usage_collection_deletions") + .columns(["state", "next_attempt_at", "updated_at", "collection_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_collection_deletions_lease") + .ifNotExists() + .on("_emdash_media_usage_collection_deletions") + .columns(["state", "lease_expires_at", "updated_at", "collection_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_collection_deletions_operator") + .ifNotExists() + .on("_emdash_media_usage_collection_deletions") + .columns(["state", "updated_at", "collection_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_sources_collection_cursor") + .ifNotExists() + .on("_emdash_media_usage_sources") + .columns(["source_type", "collection_id", "source_key"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_source_cursor") + .ifNotExists() + .on("_emdash_media_usage") + .columns(["source_key", "id"]) + .execute(); +} + +export async function down(db: Kysely): Promise { + const deletion = await sql<{ present: number }>` + SELECT 1 AS present + FROM _emdash_media_usage_collection_deletions + LIMIT 1 + `.execute(db); + if (deletion.rows.length > 0) { + throw new Error("Cannot roll back while durable collection deletion evidence exists"); + } + + await db.schema.dropIndex("idx__emdash_media_usage_source_cursor").ifExists().execute(); + await db.schema + .dropIndex("idx__emdash_media_usage_sources_collection_cursor") + .ifExists() + .execute(); + await db.schema.dropTable("_emdash_media_usage_collection_deletions").ifExists().execute(); +} + +function sortableUtcTimestamp(db: Kysely): RawBuilder { + if (isPostgres(db)) { + return sql`to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`; + } + return sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`; +} diff --git a/packages/core/src/database/migrations/runner.ts b/packages/core/src/database/migrations/runner.ts index 8bb2ce1671..640b19d2f0 100644 --- a/packages/core/src/database/migrations/runner.ts +++ b/packages/core/src/database/migrations/runner.ts @@ -67,6 +67,7 @@ import * as m061 from "./061_media_usage_cleanup.js"; import * as m062 from "./062_media_usage_cleanup_fence.js"; import * as m063 from "./063_media_usage_incremental_work.js"; import * as m064 from "./064_fts_plain_text.js"; +import * as m065 from "./065_media_usage_collection_deletion.js"; const MIGRATIONS: Readonly> = Object.freeze({ "001_initial": m001, @@ -132,6 +133,7 @@ const MIGRATIONS: Readonly> = Object.freeze({ "062_media_usage_cleanup_fence": m062, "063_media_usage_incremental_work": m063, "064_fts_plain_text": m064, + "065_media_usage_collection_deletion": m065, }); /** Total number of registered migrations. Exported for use in tests. */ diff --git a/packages/core/src/database/types.ts b/packages/core/src/database/types.ts index fc797fc640..9511f11e89 100644 --- a/packages/core/src/database/types.ts +++ b/packages/core/src/database/types.ts @@ -235,6 +235,24 @@ export interface MediaUsageWorkTable { updated_at: Generated; } +export interface MediaUsageCollectionDeletionTable { + collection_id: string; + collection_slug: string; + force_delete: number; + state: Generated; + phase: Generated; + work_cursor: Generated; + source_key: Generated; + occurrence_cursor: Generated; + attempt_count: Generated; + next_attempt_at: string; + lease_token: Generated; + lease_expires_at: Generated; + last_error_code: Generated; + created_at: Generated; + updated_at: Generated; +} + export interface UserTable { id: string; email: string; @@ -604,6 +622,7 @@ export interface Database { _emdash_media_usage_index_status: MediaUsageIndexStatusTable; _emdash_media_usage_activation: MediaUsageActivationTable; _emdash_media_usage_work: MediaUsageWorkTable; + _emdash_media_usage_collection_deletions: MediaUsageCollectionDeletionTable; users: UserTable; credentials: CredentialTable; auth_tokens: AuthTokenTable; diff --git a/packages/core/src/db/adapters.ts b/packages/core/src/db/adapters.ts index ab45459593..3802e0b513 100644 --- a/packages/core/src/db/adapters.ts +++ b/packages/core/src/db/adapters.ts @@ -26,6 +26,32 @@ */ export type DatabaseDialectType = "sqlite" | "postgres"; +export type CollectionDeletionGuardInput = + | { + action: "fence"; + collectionId: string; + collectionSlug: string; + leaseToken: string; + forceDelete: boolean; + } + | { + action: "drop"; + collectionId: string; + collectionSlug: string; + leaseToken: string; + }; + +export type CollectionDeletionGuardResult = + | { outcome: "fenced" } + | { outcome: "has_content" } + | { outcome: "stale" } + | { outcome: "dropped" }; + +export type ExecuteCollectionDeletionGuard = ( + config: unknown, + input: CollectionDeletionGuardInput, +) => Promise; + /** * Database descriptor - serializable config for virtual modules */ @@ -65,6 +91,8 @@ export interface DatabaseDescriptor { * inspecting an optional entrypoint export. */ supportsCoalescing?: boolean; + /** The runtime entrypoint exports the deletion-specific atomic guard. */ + supportsCollectionDeletionGuard?: boolean; } export interface SqliteConfig { diff --git a/packages/core/src/db/index.ts b/packages/core/src/db/index.ts index b20bfae959..cebd97424c 100644 --- a/packages/core/src/db/index.ts +++ b/packages/core/src/db/index.ts @@ -26,6 +26,9 @@ export type { SqliteConfig, LibsqlConfig, PostgresConfig, + CollectionDeletionGuardInput, + CollectionDeletionGuardResult, + ExecuteCollectionDeletionGuard, } from "./adapters.js"; // Migration utilities (used by playground, preview, and custom deployment scripts) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 88df2216cf..c2f90ecfd0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -557,4 +557,7 @@ export type { SqliteConfig, LibsqlConfig, PostgresConfig, + CollectionDeletionGuardInput, + CollectionDeletionGuardResult, + ExecuteCollectionDeletionGuard, } from "./db/adapters.js"; diff --git a/packages/core/src/media/usage/collection-deletion.ts b/packages/core/src/media/usage/collection-deletion.ts new file mode 100644 index 0000000000..6a1814e0b6 --- /dev/null +++ b/packages/core/src/media/usage/collection-deletion.ts @@ -0,0 +1,282 @@ +import { sql, type Kysely, type RawBuilder, type Selectable, type Transaction } from "kysely"; +import { ulid } from "ulidx"; + +import { isPostgres } from "../../database/dialect-helpers.js"; +import type { Database, MediaUsageCollectionDeletionTable } from "../../database/types.js"; +import { validateIdentifier } from "../../database/validate.js"; +import type { + CollectionDeletionGuardInput, + CollectionDeletionGuardResult, +} from "../../db/adapters.js"; +import { FTSManager } from "../../search/fts-manager.js"; + +export type MediaUsageCollectionDeletionState = "pending" | "retry" | "leased" | "failed"; +export type MediaUsageCollectionDeletionPhase = + | "fence" + | "registry" + | "table" + | "work" + | "sources" + | "status" + | "finalize"; + +export interface MediaUsageCollectionDeletionRecord { + collectionId: string; + collectionSlug: string; + forceDelete: boolean; + state: MediaUsageCollectionDeletionState; + phase: MediaUsageCollectionDeletionPhase; + workCursor: string | null; + sourceKey: string | null; + occurrenceCursor: string | null; + attemptCount: number; + nextAttemptAt: string; + leaseToken: string | null; + leaseExpiresAt: string | null; + lastErrorCode: string | null; + createdAt: string; + updatedAt: string; +} + +export class MediaUsageCollectionDeletionRepository { + constructor(private db: Kysely) {} + + async createTombstone(input: { + collectionId: string; + collectionSlug: string; + forceDelete: boolean; + }): Promise { + assertIdentity(input); + const now = timestampOffset(this.db, 0); + await this.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: input.collectionId, + collection_slug: input.collectionSlug, + force_delete: input.forceDelete ? 1 : 0, + state: "pending", + phase: "fence", + next_attempt_at: now, + updated_at: now, + }) + .onConflict((conflict) => conflict.column("collection_id").doNothing()) + .execute(); + + const row = await this.db + .selectFrom("_emdash_media_usage_collection_deletions") + .selectAll() + .where("collection_id", "=", input.collectionId) + .executeTakeFirstOrThrow(); + if ( + row.collection_slug !== input.collectionSlug || + Boolean(row.force_delete) !== input.forceDelete + ) { + throw new Error("Collection deletion tombstone identity conflicts with existing work"); + } + return rowToRecord(row); + } + + async claim(input: { + collectionId: string; + phase: MediaUsageCollectionDeletionPhase; + leaseDurationSeconds: number; + }): Promise<(MediaUsageCollectionDeletionRecord & { leaseToken: string }) | null> { + if (!input.collectionId || !isPhase(input.phase)) { + throw new Error("Collection deletion claim requires an exact identity and phase"); + } + if (!Number.isSafeInteger(input.leaseDurationSeconds) || input.leaseDurationSeconds < 1) { + throw new Error("Collection deletion lease duration must be a positive whole number"); + } + const leaseToken = ulid(); + const row = await this.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ + state: "leased", + lease_token: leaseToken, + lease_expires_at: timestampOffset(this.db, input.leaseDurationSeconds), + updated_at: timestampOffset(this.db, 0), + }) + .where("collection_id", "=", input.collectionId) + .where("phase", "=", input.phase) + .where((eb) => + eb.or([ + eb.and([ + eb("state", "in", ["pending", "retry"]), + timestampIsDue(this.db, "next_attempt_at"), + ]), + eb.and([eb("state", "=", "leased"), timestampIsDue(this.db, "lease_expires_at")]), + ]), + ) + .returningAll() + .executeTakeFirst(); + if (!row) return null; + return { ...rowToRecord(row), leaseToken }; + } +} + +export async function executeLocalCollectionDeletionGuard( + db: Kysely, + input: CollectionDeletionGuardInput, +): Promise { + assertGuardInput(input); + return db.transaction().execute(async (trx) => { + if (!(await lockLiveTombstone(trx, input))) return { outcome: "stale" }; + if (input.action === "fence") return fenceCollection(trx, input); + + const fts = new FTSManager(trx); + await fts.dropFtsTable(input.collectionSlug); + await sql`DROP TABLE IF EXISTS ${sql.ref(`ec_${input.collectionSlug}`)}`.execute(trx); + return { outcome: "dropped" }; + }); +} + +async function fenceCollection( + trx: Transaction, + input: Extract, +): Promise { + const tableName = `ec_${input.collectionSlug}`; + validateIdentifier(tableName, "content table"); + if (isPostgres(trx)) { + await sql`LOCK TABLE ${sql.ref(tableName)} IN SHARE ROW EXCLUSIVE MODE`.execute(trx); + } + + if (!input.forceDelete) { + const content = await sql<{ present: number }>` + SELECT 1 AS present FROM ${sql.ref(tableName)} LIMIT 1 + `.execute(trx); + if (content.rows.length > 0) return { outcome: "has_content" }; + } + + const result = await trx + .updateTable("_emdash_media_usage_index_status as status") + .set({ capture_state: "deleting", updated_at: timestampOffset(trx, 0) }) + .where("status.adapter_id", "=", "content-media") + .where("status.scope_type", "=", "collection") + .where("status.scope_key", "=", input.collectionSlug) + .where("status.collection_id", "=", input.collectionId) + .where("status.capture_state", "=", "active") + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_collections as collection") + .select("collection.id") + .where("collection.id", "=", input.collectionId) + .where("collection.slug", "=", input.collectionSlug), + ), + ) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1 ? { outcome: "fenced" } : { outcome: "stale" }; +} + +async function lockLiveTombstone( + trx: Transaction, + input: CollectionDeletionGuardInput, +): Promise { + const phase = input.action === "fence" ? "fence" : "table"; + if (isPostgres(trx)) { + const row = await trx + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .where("collection_id", "=", input.collectionId) + .where("collection_slug", "=", input.collectionSlug) + .where("state", "=", "leased") + .where("phase", "=", phase) + .where("lease_token", "=", input.leaseToken) + .where(liveLease(trx)) + .forUpdate() + .executeTakeFirst(); + return row !== undefined; + } + + const locked = await trx + .updateTable("_emdash_media_usage_collection_deletions") + .set({ updated_at: sql`updated_at` }) + .where("collection_id", "=", input.collectionId) + .where("collection_slug", "=", input.collectionSlug) + .where("state", "=", "leased") + .where("phase", "=", phase) + .where("lease_token", "=", input.leaseToken) + .where(liveLease(trx)) + .executeTakeFirst(); + return Number(locked.numUpdatedRows ?? 0) === 1; +} + +function assertIdentity(input: { collectionId: string; collectionSlug: string }): void { + if (!input.collectionId) throw new Error("Collection deletion requires a collection ID"); + validateIdentifier(input.collectionSlug, "collection slug"); +} + +function assertGuardInput(input: CollectionDeletionGuardInput): void { + assertIdentity(input); + if (!input.leaseToken) throw new Error("Collection deletion requires a lease token"); +} + +function liveLease(db: Kysely): RawBuilder { + return isPostgres(db) + ? sql`lease_expires_at::timestamptz > clock_timestamp()` + : sql`lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; +} + +function timestampIsDue( + db: Kysely, + column: "next_attempt_at" | "lease_expires_at", +): RawBuilder { + return isPostgres(db) + ? sql`${sql.ref(column)}::timestamptz <= clock_timestamp()` + : sql`${sql.ref(column)} <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; +} + +function timestampOffset(db: Kysely, offsetSeconds: number): RawBuilder { + if (isPostgres(db)) { + return sql`to_char( + (clock_timestamp() AT TIME ZONE 'UTC') + (${offsetSeconds} * INTERVAL '1 second'), + 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"' + )`; + } + return sql`strftime( + '%Y-%m-%dT%H:%M:%fZ', + 'now', + ${`${offsetSeconds >= 0 ? "+" : ""}${offsetSeconds} seconds`} + )`; +} + +function rowToRecord( + row: Selectable, +): MediaUsageCollectionDeletionRecord { + if (!isState(row.state) || !isPhase(row.phase)) { + throw new Error("Invalid media usage collection deletion lifecycle"); + } + return { + collectionId: row.collection_id, + collectionSlug: row.collection_slug, + forceDelete: Boolean(row.force_delete), + state: row.state, + phase: row.phase, + workCursor: row.work_cursor, + sourceKey: row.source_key, + occurrenceCursor: row.occurrence_cursor, + attemptCount: row.attempt_count, + nextAttemptAt: row.next_attempt_at, + leaseToken: row.lease_token, + leaseExpiresAt: row.lease_expires_at, + lastErrorCode: row.last_error_code, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + +function isState(value: string): value is MediaUsageCollectionDeletionState { + return value === "pending" || value === "retry" || value === "leased" || value === "failed"; +} + +function isPhase(value: string): value is MediaUsageCollectionDeletionPhase { + return ( + value === "fence" || + value === "registry" || + value === "table" || + value === "work" || + value === "sources" || + value === "status" || + value === "finalize" + ); +} diff --git a/packages/core/src/virtual-modules.d.ts b/packages/core/src/virtual-modules.d.ts index a4057b13c2..3cc9b60e72 100644 --- a/packages/core/src/virtual-modules.d.ts +++ b/packages/core/src/virtual-modules.d.ts @@ -35,6 +35,7 @@ declare module "virtual:emdash/dialect" { import type { Dialect, Kysely } from "kysely"; import type { DatabaseDialectType } from "./db/adapters.js"; + import type { ExecuteCollectionDeletionGuard } from "./db/adapters.js"; // Can be undefined if no database configured, or the actual function export const createDialect: ((config: unknown) => Dialect) | undefined; @@ -82,6 +83,7 @@ declare module "virtual:emdash/dialect" { close?: () => void; } export const createRequestScopedDb: (opts: RequestScopedDbOpts) => RequestScopedDb | null; + export const executeCollectionDeletionGuard: ExecuteCollectionDeletionGuard | undefined; } declare module "virtual:emdash/storage" { diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-foundation.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-foundation.test.ts new file mode 100644 index 0000000000..daa8edcdb7 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-foundation.test.ts @@ -0,0 +1,176 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { tableExists } from "../../../src/database/dialect-helpers.js"; +import { + MediaUsageCollectionDeletionRepository, + executeLocalCollectionDeletionGuard, +} from "../../../src/media/usage/collection-deletion.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage collection deletion foundation", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("fences and drops only for the exact live tombstone lease", async () => { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "articles", label: "Articles" }); + const collection = await registry.getCollection("articles"); + if (!collection) throw new Error("Expected articles collection"); + + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: collection.slug, + collection_id: collection.id, + status: "complete", + capture_state: "active", + }) + .execute(); + + const repository = new MediaUsageCollectionDeletionRepository(ctx.db); + await repository.createTombstone({ + collectionId: collection.id, + collectionSlug: collection.slug, + forceDelete: false, + }); + const claim = await repository.claim({ + collectionId: collection.id, + phase: "fence", + leaseDurationSeconds: 300, + }); + expect(claim).not.toBeNull(); + + const staleFence = await executeLocalCollectionDeletionGuard(ctx.db, { + action: "fence", + collectionId: collection.id, + collectionSlug: collection.slug, + leaseToken: "stale-token", + forceDelete: false, + }); + expect(staleFence).toEqual({ outcome: "stale" }); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select("capture_state") + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ capture_state: "active" }); + + await expect( + executeLocalCollectionDeletionGuard(ctx.db, { + action: "fence", + collectionId: collection.id, + collectionSlug: collection.slug, + leaseToken: claim!.leaseToken, + forceDelete: false, + }), + ).resolves.toEqual({ outcome: "fenced" }); + + await ctx.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ phase: "table" }) + .where("collection_id", "=", collection.id) + .execute(); + await expect( + executeLocalCollectionDeletionGuard(ctx.db, { + action: "drop", + collectionId: collection.id, + collectionSlug: collection.slug, + leaseToken: claim!.leaseToken, + }), + ).resolves.toEqual({ outcome: "dropped" }); + expect(await tableExists(ctx.db, "ec_articles")).toBe(false); + }); + + it("does not let an expired owner drop a replacement table", async () => { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "reused", label: "Reused" }); + const collection = await registry.getCollection("reused"); + if (!collection) throw new Error("Expected reused collection"); + + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: collection.id, + collection_slug: collection.slug, + force_delete: 1, + state: "leased", + phase: "table", + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: "old-owner", + lease_expires_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + + await expect( + executeLocalCollectionDeletionGuard(ctx.db, { + action: "drop", + collectionId: collection.id, + collectionSlug: collection.slug, + leaseToken: "old-owner", + }), + ).resolves.toEqual({ outcome: "stale" }); + expect(await tableExists(ctx.db, "ec_reused")).toBe(true); + }); + + it("keeps a non-empty collection unchanged when force is false", async () => { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "occupied", label: "Occupied" }); + const collection = await registry.getCollection("occupied"); + if (!collection) throw new Error("Expected occupied collection"); + await sql`INSERT INTO ${sql.ref("ec_occupied")} (id, slug) VALUES ('entry-1', 'entry-1')`.execute( + ctx.db, + ); + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: collection.slug, + collection_id: collection.id, + status: "complete", + capture_state: "active", + }) + .execute(); + + const repository = new MediaUsageCollectionDeletionRepository(ctx.db); + await repository.createTombstone({ + collectionId: collection.id, + collectionSlug: collection.slug, + forceDelete: false, + }); + const claim = await repository.claim({ + collectionId: collection.id, + phase: "fence", + leaseDurationSeconds: 300, + }); + if (!claim) throw new Error("Expected deletion claim"); + + await expect( + executeLocalCollectionDeletionGuard(ctx.db, { + action: "fence", + collectionId: collection.id, + collectionSlug: collection.slug, + leaseToken: claim.leaseToken, + forceDelete: false, + }), + ).resolves.toEqual({ outcome: "has_content" }); + expect(await tableExists(ctx.db, "ec_occupied")).toBe(true); + }); +}); diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-migration.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-migration.test.ts new file mode 100644 index 0000000000..4e869be158 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-migration.test.ts @@ -0,0 +1,73 @@ +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage collection deletion migration", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("keeps a unique exact-ID tombstone and slug lock", async () => { + const migration = await ctx.db + .selectFrom("_emdash_migrations") + .select("name") + .where("name", "=", "065_media_usage_collection_deletion") + .executeTakeFirst(); + expect(migration).toBeDefined(); + + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-1", + collection_slug: "articles", + force_delete: 0, + state: "pending", + phase: "fence", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + + await expect( + ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-2", + collection_slug: "articles", + force_delete: 1, + state: "pending", + phase: "fence", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(), + ).rejects.toThrow(); + }); + + it("refuses rollback while durable deletion evidence exists", async () => { + const migration = + await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-1", + collection_slug: "articles", + force_delete: 0, + state: "pending", + phase: "fence", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + + await expect(migration.down(ctx.db)).rejects.toThrow(/durable collection deletion/i); + }); +}); diff --git a/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts b/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts index 5c220cf729..1476ae3563 100644 --- a/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts +++ b/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts @@ -49,6 +49,9 @@ describeEachDialect("media usage incremental work migration", (dialect) => { }); it("upgrades and reruns without rewriting legacy evidence or inventing work", async () => { + const collectionDeletionMigration = + await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); + await collectionDeletionMigration.down(ctx.db); const migration = await import("../../../src/database/migrations/063_media_usage_incremental_work.js"); await migration.down(ctx.db); @@ -191,6 +194,9 @@ describeEachDialect("media usage incremental work migration", (dialect) => { }); it("purges a partially bound status if its collection is deleted or recreated before retry", async () => { + const collectionDeletionMigration = + await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); + await collectionDeletionMigration.down(ctx.db); const migration = await import("../../../src/database/migrations/063_media_usage_incremental_work.js"); await migration.down(ctx.db); diff --git a/packages/core/tests/integration/database/migrations.test.ts b/packages/core/tests/integration/database/migrations.test.ts index 3e524c6063..239bafe558 100644 --- a/packages/core/tests/integration/database/migrations.test.ts +++ b/packages/core/tests/integration/database/migrations.test.ts @@ -55,6 +55,7 @@ describe("Database Migrations (Integration)", () => { "_emdash_media_usage_generation_writes", "_emdash_media_usage_cleanup_fence", "_emdash_media_usage_index_status", + "_emdash_media_usage_collection_deletions", ]; for (const table of tables) { @@ -154,6 +155,7 @@ describe("Database Migrations (Integration)", () => { "062_media_usage_cleanup_fence", "063_media_usage_incremental_work", "064_fts_plain_text", + "065_media_usage_collection_deletion", ]; await db.deleteFrom("_emdash_migrations").where("name", "in", trailing).execute(); diff --git a/packages/core/tests/unit/astro/integration/virtual-modules.test.ts b/packages/core/tests/unit/astro/integration/virtual-modules.test.ts index e547fa283f..09a6989b10 100644 --- a/packages/core/tests/unit/astro/integration/virtual-modules.test.ts +++ b/packages/core/tests/unit/astro/integration/virtual-modules.test.ts @@ -45,6 +45,7 @@ describe("generateDialectModule", () => { const out = generateDialectModule({ supportsRequestScope: false, supportsCoalescing: false, + supportsCollectionDeletionGuard: false, }); expect(out).toContain("export const createDialect = undefined"); expect(out).toContain("export const createRequestScopedDb = (_opts) => null"); @@ -56,6 +57,7 @@ describe("generateDialectModule", () => { type: "sqlite", supportsRequestScope: false, supportsCoalescing: false, + supportsCollectionDeletionGuard: false, }); expect(out).toContain(`import { createDialect as _createDialect } from "some-adapter/dialect"`); expect(out).toContain("export const createRequestScopedDb = (_opts) => null"); @@ -70,12 +72,16 @@ describe("generateDialectModule", () => { type: "sqlite", supportsRequestScope: true, supportsCoalescing: true, + supportsCollectionDeletionGuard: true, }); expect(out).toContain(`export { createRequestScopedDb } from "@emdash-cms/cloudflare/db/d1"`); expect(out).toContain( `import { createCoalescingDialect as _createCoalescingDialect } from "@emdash-cms/cloudflare/db/d1"`, ); expect(out).toContain("export const createCoalescingDialect = _createCoalescingDialect"); + expect(out).toContain( + `export { executeCollectionDeletionGuard } from "@emdash-cms/cloudflare/db/d1"`, + ); expect(out).not.toContain("= () => null"); expect(out).not.toContain("= (_opts) => null"); }); @@ -86,6 +92,7 @@ describe("generateDialectModule", () => { type: "postgres", supportsRequestScope: false, supportsCoalescing: false, + supportsCollectionDeletionGuard: false, }); expect(out).toContain(`export const dialectType = "postgres"`); }); diff --git a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts new file mode 100644 index 0000000000..4be6b834f9 --- /dev/null +++ b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts @@ -0,0 +1,87 @@ +import { env } from "cloudflare:test"; +import { Kysely, sql } from "kysely"; +import { afterAll, beforeAll, expect, it } from "vitest"; + +import { RawBindingD1Dialect } from "../../../cloudflare/src/db/d1-dialect.js"; +import { executeCollectionDeletionGuard } from "../../../cloudflare/src/db/d1.js"; +import { runMigrations } from "../../src/database/migrations/runner.js"; +import type { Database } from "../../src/database/types.js"; + +declare module "cloudflare:test" { + interface ProvidedEnv { + DB: D1Database; + } +} + +let db: Kysely; + +beforeAll(async () => { + db = new Kysely({ dialect: new RawBindingD1Dialect({ database: env.DB }) }); + await runMigrations(db); +}); + +afterAll(async () => { + await db.destroy(); +}); + +it("rolls back a stale guarded batch before any collection DDL", async () => { + await sql`CREATE TABLE ec_d1_guarded (id TEXT PRIMARY KEY)`.execute(db); + await db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-d1", + collection_slug: "d1_guarded", + force_delete: 1, + state: "leased", + phase: "table", + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: "current-owner", + lease_expires_at: "2999-01-01T00:00:00.000Z", + }) + .execute(); + + await expect( + executeCollectionDeletionGuard( + { binding: "DB" }, + { + action: "drop", + collectionId: "collection-d1", + collectionSlug: "d1_guarded", + leaseToken: "stale-owner", + }, + ), + ).resolves.toEqual({ outcome: "stale" }); + + const table = await sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'ec_d1_guarded' + `.execute(db); + expect(table.rows).toEqual([{ name: "ec_d1_guarded" }]); + expect( + await db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .execute(), + ).toEqual([{ collection_id: "collection-d1" }]); + + await expect( + executeCollectionDeletionGuard( + { binding: "DB" }, + { + action: "drop", + collectionId: "collection-d1", + collectionSlug: "d1_guarded", + leaseToken: "current-owner", + }, + ), + ).resolves.toEqual({ outcome: "dropped" }); + const dropped = await sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'ec_d1_guarded' + `.execute(db); + expect(dropped.rows).toEqual([]); + expect( + await db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .execute(), + ).toEqual([{ collection_id: "collection-d1" }]); +}); diff --git a/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts b/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts index e1b94be650..6997e707a9 100644 --- a/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts @@ -65,7 +65,9 @@ it("reports real D1 metadata for the approved full runtime boundary", async () = ); expect(measurement.value.success).toBe(true); expect(measurement.d1Queries).toBeLessThanOrEqual(40); - expect(measurement.rowsWritten).toBeLessThanOrEqual(122); + // The collection/source cleanup cursors add one index write for this source + // and each of its 12 occurrences; keep the bound exact so further growth fails. + expect(measurement.rowsWritten).toBeLessThanOrEqual(135); expect(measurement.maxBinds).toBeLessThanOrEqual(100); expect(measurement.maxSqlBytes).toBeLessThan(100 * 1024); const { value: _value, ...evidence } = measurement; From 302ac9ecd6e2407c00113aa4164db0f88925d35f Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:29:39 +0100 Subject: [PATCH 02/20] feat(core): safely detach activated collections --- .../db/do-sql-collection-deletion.test.ts | 39 ++ .../src/database/repositories/media-usage.ts | 42 +- .../src/media/usage/collection-deletion.ts | 312 ++++++++++- packages/core/src/schema/registry.ts | 54 +- ...sage-collection-deletion-lifecycle.test.ts | 490 ++++++++++++++++++ .../usage-collection-deletion-virtual.test.ts | 18 + ...media-usage-collection-deletion-d1.test.ts | 75 +++ 7 files changed, 1009 insertions(+), 21 deletions(-) create mode 100644 packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts create mode 100644 packages/core/tests/unit/media/usage-collection-deletion-virtual.test.ts diff --git a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts index 8aecdfa869..16bffedbb4 100644 --- a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts +++ b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts @@ -83,4 +83,43 @@ describe("EmDashDB collection deletion guard", () => { expect(statements.filter((statement) => statement.includes("DROP TRIGGER"))).toHaveLength(3); expect(statements.filter((statement) => statement.includes("DROP TABLE"))).toHaveLength(2); }); + + it("preserves non-forced content and fences the collection once empty", async () => { + let contentPresent = true; + const sql = { + exec: vi.fn((statement: string) => { + statements.push(statement); + if (statement.includes("SELECT collection_id")) { + return cursor([{ collection_id: "collection-1" }]); + } + if (statement.includes("SELECT 1 AS present")) { + return cursor(contentPresent ? [{ present: 1 }] : []); + } + if (statement.includes("UPDATE _emdash_media_usage_index_status")) { + return cursor([], 1); + } + return cursor(); + }), + }; + const object = new EmDashDB({ storage: { sql, transactionSync } } as never, {}); + const input = { + action: "fence" as const, + collectionId: "collection-1", + collectionSlug: "articles", + leaseToken: "current-owner", + forceDelete: false, + }; + + await expect(object.executeCollectionDeletionGuard(input)).resolves.toEqual({ + outcome: "has_content", + }); + expect(statements.some((statement) => statement.includes("UPDATE _emdash"))).toBe(false); + + statements = []; + contentPresent = false; + await expect(object.executeCollectionDeletionGuard(input)).resolves.toEqual({ + outcome: "fenced", + }); + expect(statements.some((statement) => statement.includes("UPDATE _emdash"))).toBe(true); + }); }); diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index 657d431b9f..c07ab18a06 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -460,6 +460,9 @@ export class MediaUsageRepository { await this.withGenerationWriteLease(source.sourceKey, generation, async (leaseToken, now) => { await withTransaction(this.db, async (trx) => { + if (!(await this.lockCanonicalSourceCollection(trx, source))) { + throw new Error(`Media usage collection is no longer current for ${source.sourceKey}`); + } await this.insertOccurrences(trx, source.sourceKey, generation, occurrences, now); const promoted = await this.upsertSource(trx, source, generation, now, leaseToken); if (!promoted) { @@ -491,6 +494,7 @@ export class MediaUsageRepository { await this.withGenerationWriteLease(source.sourceKey, generation, async (leaseToken, now) => { const row = this.buildSourceRow(source, generation, now); await withTransaction(this.db, async (trx) => { + if (!(await this.lockCanonicalSourceCollection(trx, source))) return; await this.insertOccurrences(trx, source.sourceKey, generation, occurrences, now); if (expectedCurrentGeneration === null) { replaced = await this.insertSourceIfAbsent(trx, row, leaseToken); @@ -591,6 +595,7 @@ export class MediaUsageRepository { await this.withGenerationWriteLease(source.sourceKey, generation, async (leaseToken, now) => { const row = this.buildSourceRow(source, generation, now); await withTransaction(this.db, async (trx) => { + if (!(await this.lockCanonicalSourceCollection(trx, source))) return; await this.insertOccurrences(trx, source.sourceKey, generation, occurrences, now); if (expectedSource === null) { replaced = await this.insertSourceIfAbsent(trx, row, leaseToken); @@ -652,16 +657,22 @@ export class MediaUsageRepository { if (expectedSource === null) { await this.withGenerationWriteLease(source.sourceKey, generation, async (leaseToken, now) => { const row = this.buildAttemptedSourceRow(source, generation, now); - attempted = await this.persistSourceIfWriteLease( - this.db, - row, - leaseToken, - sql`ON CONFLICT (source_key) DO NOTHING`, - ); + await withTransaction(this.db, async (trx) => { + if (!(await this.lockCanonicalSourceCollection(trx, source))) return; + attempted = await this.persistSourceIfWriteLease( + trx, + row, + leaseToken, + sql`ON CONFLICT (source_key) DO NOTHING`, + ); + }); }); } else { const row = this.buildAttemptedSourceRow(source, generation, new Date().toISOString()); - attempted = await this.updateAttemptedSourceIfMatching(this.db, source, row, expectedSource); + await withTransaction(this.db, async (trx) => { + if (!(await this.lockCanonicalSourceCollection(trx, source))) return; + attempted = await this.updateAttemptedSourceIfMatching(trx, source, row, expectedSource); + }); } return { @@ -2174,6 +2185,23 @@ export class MediaUsageRepository { } } + private async lockCanonicalSourceCollection( + db: DatabaseExecutor, + source: MediaUsageSourceInput, + ): Promise { + if (source.collectionId === undefined || source.collectionId === null) return true; + if (!source.collectionSlug) return false; + if (!isPostgres(this.db)) return true; + const collection = await db + .selectFrom("_emdash_collections") + .select("id") + .where("id", "=", source.collectionId) + .where("slug", "=", source.collectionSlug) + .forKeyShare() + .executeTakeFirst(); + return collection !== undefined; + } + private async upsertSource( db: DatabaseExecutor, source: MediaUsageSourceInput, diff --git a/packages/core/src/media/usage/collection-deletion.ts b/packages/core/src/media/usage/collection-deletion.ts index 6a1814e0b6..08990f2fad 100644 --- a/packages/core/src/media/usage/collection-deletion.ts +++ b/packages/core/src/media/usage/collection-deletion.ts @@ -2,6 +2,7 @@ import { sql, type Kysely, type RawBuilder, type Selectable, type Transaction } import { ulid } from "ulidx"; import { isPostgres } from "../../database/dialect-helpers.js"; +import { withTransaction } from "../../database/transaction.js"; import type { Database, MediaUsageCollectionDeletionTable } from "../../database/types.js"; import { validateIdentifier } from "../../database/validate.js"; import type { @@ -9,6 +10,10 @@ import type { CollectionDeletionGuardResult, } from "../../db/adapters.js"; import { FTSManager } from "../../search/fts-manager.js"; +import { verifyMediaUsageCaptureTriggers } from "./capture-triggers.js"; + +const ACTIVATION_KEY = "incremental_capture"; +const VIRTUAL_DIALECT_ID = "virtual:emdash/dialect"; export type MediaUsageCollectionDeletionState = "pending" | "retry" | "leased" | "failed"; export type MediaUsageCollectionDeletionPhase = @@ -112,6 +117,306 @@ export class MediaUsageCollectionDeletionRepository { if (!row) return null; return { ...rowToRecord(row), leaseToken }; } + + async findBySlug(collectionSlug: string): Promise { + validateIdentifier(collectionSlug, "collection slug"); + const row = await this.db + .selectFrom("_emdash_media_usage_collection_deletions") + .selectAll() + .where("collection_slug", "=", collectionSlug) + .executeTakeFirst(); + return row ? rowToRecord(row) : null; + } + + async checkpoint(input: { + collectionId: string; + leaseToken: string; + fromPhase: MediaUsageCollectionDeletionPhase; + toPhase: MediaUsageCollectionDeletionPhase; + }): Promise { + const result = await this.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ phase: input.toPhase, updated_at: timestampOffset(this.db, 0) }) + .where("collection_id", "=", input.collectionId) + .where("state", "=", "leased") + .where("phase", "=", input.fromPhase) + .where("lease_token", "=", input.leaseToken) + .where(liveLease(this.db)) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async release(input: { collectionId: string; leaseToken: string }): Promise { + const now = timestampOffset(this.db, 0); + const result = await this.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ + state: "pending", + next_attempt_at: now, + lease_token: null, + lease_expires_at: null, + updated_at: now, + }) + .where("collection_id", "=", input.collectionId) + .where("state", "=", "leased") + .where("lease_token", "=", input.leaseToken) + .where(liveLease(this.db)) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async cancelFence(input: { collectionId: string; leaseToken: string }): Promise { + const result = await this.db + .deleteFrom("_emdash_media_usage_collection_deletions") + .where("collection_id", "=", input.collectionId) + .where("state", "=", "leased") + .where("phase", "=", "fence") + .where("lease_token", "=", input.leaseToken) + .where(liveLease(this.db)) + .executeTakeFirst(); + return Number(result.numDeletedRows ?? 0) === 1; + } + + async deleteRegistryAndCheckpoint(input: { + collectionId: string; + collectionSlug: string; + leaseToken: string; + }): Promise { + return withTransaction(this.db, async (trx) => { + await trx + .deleteFrom("_emdash_collections") + .where("id", "=", input.collectionId) + .where("slug", "=", input.collectionSlug) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_collection_deletions as deletion") + .select("deletion.collection_id") + .where("deletion.collection_id", "=", input.collectionId) + .where("deletion.collection_slug", "=", input.collectionSlug) + .where("deletion.state", "=", "leased") + .where("deletion.phase", "=", "registry") + .where("deletion.lease_token", "=", input.leaseToken) + .where(liveLease(this.db, "deletion.lease_expires_at")), + ), + ) + .execute(); + + const checkpoint = await trx + .updateTable("_emdash_media_usage_collection_deletions") + .set({ phase: "table", updated_at: timestampOffset(this.db, 0) }) + .where("collection_id", "=", input.collectionId) + .where("collection_slug", "=", input.collectionSlug) + .where("state", "=", "leased") + .where("phase", "=", "registry") + .where("lease_token", "=", input.leaseToken) + .where(liveLease(this.db)) + .executeTakeFirst(); + return Number(checkpoint.numUpdatedRows ?? 0) === 1; + }); + } +} + +export type ActivatedCollectionDeletionOutcome = + | "inactive" + | "not_found" + | "in_progress" + | "deleted" + | "has_content"; + +export async function deleteActivatedMediaUsageCollection( + db: Kysely, + input: { + collectionSlug: string; + collectionId?: string; + forceDelete: boolean; + }, +): Promise { + validateIdentifier(input.collectionSlug, "collection slug"); + const repository = new MediaUsageCollectionDeletionRepository(db); + let deletion = await repository.findBySlug(input.collectionSlug); + if (deletion && input.collectionId && deletion.collectionId !== input.collectionId) { + throw new Error("Collection deletion tombstone identity conflict"); + } + if (!deletion) { + const activation = await db + .selectFrom("_emdash_media_usage_activation") + .select("state") + .where("task_key", "=", ACTIVATION_KEY) + .executeTakeFirst(); + if (!activation || activation.state === "expanded") return "inactive"; + if (activation.state !== "active") { + throw new Error("Media usage activation must be active before collection deletion"); + } + if (!input.collectionId) return "not_found"; + await assertActivatedCollectionDeletionReady(db, { + collectionId: input.collectionId, + collectionSlug: input.collectionSlug, + }); + deletion = await repository.createTombstone({ + collectionId: input.collectionId, + collectionSlug: input.collectionSlug, + forceDelete: input.forceDelete, + }); + } + + if (deletion.phase !== "fence" && deletion.phase !== "registry" && deletion.phase !== "table") { + return "deleted"; + } + const claim = await repository.claim({ + collectionId: deletion.collectionId, + phase: deletion.phase, + leaseDurationSeconds: 5 * 60, + }); + if (!claim) return "in_progress"; + let phase = claim.phase; + const lease = { collectionId: claim.collectionId, leaseToken: claim.leaseToken }; + + if (phase === "fence") { + const captureState = await findCollectionCaptureState(db, claim); + if (captureState !== "active" && captureState !== "deleting") { + throw new Error("Activated collection deletion requires a fenced capture lifecycle"); + } + if (!(await verifyMediaUsageCaptureTriggers(db, claim))) { + throw new Error("Activated collection deletion requires the exact capture trigger set"); + } + if (captureState === "active") { + const result = await executeCollectionDeletionGuard(db, { + action: "fence", + collectionId: claim.collectionId, + collectionSlug: claim.collectionSlug, + leaseToken: claim.leaseToken, + forceDelete: claim.forceDelete, + }); + if (result.outcome === "has_content") { + if (!(await repository.cancelFence(lease))) { + throw new Error("Collection deletion lost its fence while preserving content"); + } + return "has_content"; + } + if (result.outcome !== "fenced") throw new Error("Collection deletion lost its fence"); + } + if ( + !(await repository.checkpoint({ + ...lease, + fromPhase: "fence", + toPhase: "registry", + })) + ) { + throw new Error("Collection deletion lost its fence checkpoint"); + } + phase = "registry"; + } + + if (phase === "registry") { + if ( + !(await repository.deleteRegistryAndCheckpoint({ + ...lease, + collectionSlug: claim.collectionSlug, + })) + ) { + throw new Error("Collection deletion lost its registry checkpoint"); + } + phase = "table"; + } + + if (phase === "table") { + const result = await executeCollectionDeletionGuard(db, { + action: "drop", + collectionId: claim.collectionId, + collectionSlug: claim.collectionSlug, + leaseToken: claim.leaseToken, + }); + if (result.outcome !== "dropped") throw new Error("Collection deletion lost its table fence"); + if ( + !(await repository.checkpoint({ + ...lease, + fromPhase: "table", + toPhase: "work", + })) + ) { + throw new Error("Collection deletion lost its table checkpoint"); + } + } + + if (!(await repository.release(lease))) { + throw new Error("Collection deletion lost its cleanup handoff"); + } + return "deleted"; +} + +export async function isMediaUsageCollectionSlugDeleting( + db: Kysely, + collectionSlug: string, +): Promise { + validateIdentifier(collectionSlug, "collection slug"); + const row = await db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .where("collection_slug", "=", collectionSlug) + .executeTakeFirst(); + return row !== undefined; +} + +async function assertActivatedCollectionDeletionReady( + db: Kysely, + identity: { collectionId: string; collectionSlug: string }, +): Promise { + const captureState = await findCollectionCaptureState(db, identity); + if (captureState !== "active") { + throw new Error("Activated collection deletion requires an active capture lifecycle"); + } + if (!(await verifyMediaUsageCaptureTriggers(db, identity))) { + throw new Error("Activated collection deletion requires the exact capture trigger set"); + } +} + +async function findCollectionCaptureState( + db: Kysely, + identity: { collectionId: string; collectionSlug: string }, +): Promise { + const lifecycle = await db + .selectFrom("_emdash_media_usage_index_status") + .select("capture_state") + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", identity.collectionSlug) + .where("collection_id", "=", identity.collectionId) + .executeTakeFirst(); + return lifecycle?.capture_state ?? null; +} + +async function executeCollectionDeletionGuard( + db: Kysely, + input: CollectionDeletionGuardInput, +): Promise { + const executeAdapterGuard = await loadAdapterCollectionDeletionGuard(); + if (executeAdapterGuard) { + const { default: config } = await import("virtual:emdash/config"); + return executeAdapterGuard(config.database?.config, input); + } + return executeLocalCollectionDeletionGuard(db, input); +} + +async function loadAdapterCollectionDeletionGuard(): Promise< + import("../../db/adapters.js").ExecuteCollectionDeletionGuard | undefined +> { + try { + const dialect = await import("virtual:emdash/dialect"); + return dialect.executeCollectionDeletionGuard; + } catch (error) { + if (isVirtualDialectUnavailableError(error)) return undefined; + throw error; + } +} + +export function isVirtualDialectUnavailableError(error: unknown): boolean { + if (!(error instanceof Error) || !("code" in error)) return false; + if (error.code === "ERR_MODULE_NOT_FOUND") return error.message.includes(VIRTUAL_DIALECT_ID); + return ( + error.code === "ERR_UNSUPPORTED_ESM_URL_SCHEME" && + error.message.includes("Received protocol 'virtual:'") + ); } export async function executeLocalCollectionDeletionGuard( @@ -211,10 +516,11 @@ function assertGuardInput(input: CollectionDeletionGuardInput): void { if (!input.leaseToken) throw new Error("Collection deletion requires a lease token"); } -function liveLease(db: Kysely): RawBuilder { +function liveLease(db: Kysely, column = "lease_expires_at"): RawBuilder { + const leaseExpiresAt = sql.ref(column); return isPostgres(db) - ? sql`lease_expires_at::timestamptz > clock_timestamp()` - : sql`lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; + ? sql`${leaseExpiresAt}::timestamptz > clock_timestamp()` + : sql`${leaseExpiresAt} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; } function timestampIsDue( diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index 0fe03b2b34..f4db23ef71 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -13,6 +13,10 @@ import { markMediaUsageCollectionCaptureReady, prepareMediaUsageCollectionCapture, } from "../media/usage/activation.js"; +import { + deleteActivatedMediaUsageCollection, + isMediaUsageCollectionSlugDeleting, +} from "../media/usage/collection-deletion.js"; import { deleteContentMediaUsageCollection, invalidateContentMediaUsageSchemaChange, @@ -325,9 +329,15 @@ export class SchemaRegistry { if (RESERVED_COLLECTION_SLUGS.includes(input.slug)) { throw new SchemaError(`Collection slug "${input.slug}" is reserved`, "RESERVED_SLUG"); } + if (await isMediaUsageCollectionSlugDeleting(this.db, input.slug)) { + throw new SchemaError(`Collection "${input.slug}" already exists`, "COLLECTION_EXISTS"); + } // Check if collection already exists const existing = await this.getCollection(input.slug); + if (await isMediaUsageCollectionSlugDeleting(this.db, input.slug)) { + throw new SchemaError(`Collection "${input.slug}" already exists`, "COLLECTION_EXISTS"); + } if ( existing && !(await canResumeMediaUsageCollectionCapture(this.db, { @@ -428,6 +438,9 @@ export class SchemaRegistry { if (RESERVED_COLLECTION_SLUGS.includes(input.slug)) { throw new SchemaError(`Collection slug "${input.slug}" is reserved`, "RESERVED_SLUG"); } + if (await isMediaUsageCollectionSlugDeleting(this.db, input.slug)) { + throw new SchemaError(`Collection "${input.slug}" already exists`, "COLLECTION_EXISTS"); + } const fieldSlugs = new Set(); for (const field of fields) { @@ -448,6 +461,9 @@ export class SchemaRegistry { const hasSeo = input.hasSeo ?? supports.includes("seo") ?? false; const creationFingerprint = await buildSeedCollectionCaptureFingerprint(input, fields); const existing = await this.getCollection(input.slug); + if (await isMediaUsageCollectionSlugDeleting(this.db, input.slug)) { + throw new SchemaError(`Collection "${input.slug}" already exists`, "COLLECTION_EXISTS"); + } if ( existing && !(await canResumeMediaUsageCollectionCapture(this.db, { @@ -708,21 +724,31 @@ export class SchemaRegistry { */ async deleteCollection(slug: string, options?: { force?: boolean }): Promise { const existing = await this.getCollection(slug); + if (existing && !options?.force && (await this.collectionHasContent(slug))) { + throw new SchemaError( + `Collection "${slug}" has content. Use force: true to delete.`, + "COLLECTION_HAS_CONTENT", + ); + } + const activated = await deleteActivatedMediaUsageCollection(this.db, { + collectionId: existing?.id, + collectionSlug: slug, + forceDelete: options?.force === true, + }); + if (activated === "has_content") { + throw new SchemaError( + `Collection "${slug}" has content. Use force: true to delete.`, + "COLLECTION_HAS_CONTENT", + ); + } + if (activated === "in_progress") { + throw new SchemaError(`Collection "${slug}" deletion is already in progress`, "CONFLICT"); + } + if (activated === "deleted") return; if (!existing) { throw new SchemaError(`Collection "${slug}" not found`, "COLLECTION_NOT_FOUND"); } - // Check if collection has content - if (!options?.force) { - const hasContent = await this.collectionHasContent(slug); - if (hasContent) { - throw new SchemaError( - `Collection "${slug}" has content. Use force: true to delete.`, - "COLLECTION_HAS_CONTENT", - ); - } - } - let contentTableDropped = false; try { await withTransaction(this.db, async (trx) => { @@ -1687,6 +1713,9 @@ export class SchemaRegistry { ): Promise { // Verify table exists const tableName = this.getTableName(slug); + if (await isMediaUsageCollectionSlugDeleting(this.db, slug)) { + throw new SchemaError(`Collection "${slug}" is already registered`, "COLLECTION_EXISTS"); + } const exists = await tableExists(this.db, tableName); if (!exists) { @@ -1695,6 +1724,9 @@ export class SchemaRegistry { // Check if already registered const existing = await this.getCollection(slug); + if (await isMediaUsageCollectionSlugDeleting(this.db, slug)) { + throw new SchemaError(`Collection "${slug}" is already registered`, "COLLECTION_EXISTS"); + } if ( existing && !(await canResumeMediaUsageCollectionCapture(this.db, { diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts new file mode 100644 index 0000000000..1e92644c52 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts @@ -0,0 +1,490 @@ +import BetterSqlite3 from "better-sqlite3"; +import { Kysely, SqliteDialect, sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { tableExists } from "../../../src/database/dialect-helpers.js"; +import { runMigrations } from "../../../src/database/migrations/runner.js"; +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import type { Database } from "../../../src/database/types.js"; +import { activateMediaUsageCapture } from "../../../src/media/usage/activation.js"; +import { removeMediaUsageCaptureTriggers } from "../../../src/media/usage/capture-triggers.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage activated collection deletion", (dialect) => { + let ctx: DialectTestContext; + let registry: SchemaRegistry; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + registry = new SchemaRegistry(ctx.db); + await activateMediaUsageCapture(ctx.db, { writersDrained: true }); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("detaches an empty activated collection and leaves bounded cleanup pending", async () => { + const collection = await registry.createCollection({ slug: "articles", label: "Articles" }); + + await registry.deleteCollection("articles"); + + expect(await registry.getCollection("articles")).toBeNull(); + expect(await tableExists(ctx.db, "ec_articles")).toBe(false); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select(["collection_id", "capture_state"]) + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toEqual({ collection_id: collection.id, capture_state: "deleting" }); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select(["collection_id", "collection_slug", "state", "phase", "lease_token"]) + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toEqual({ + collection_id: collection.id, + collection_slug: "articles", + state: "pending", + phase: "work", + lease_token: null, + }); + }); + + it("preserves the collection-not-found contract after activation", async () => { + await expect(registry.deleteCollection("missing")).rejects.toMatchObject({ + code: "COLLECTION_NOT_FOUND", + }); + }); + + it("reports a live front-phase lease as a stable conflict", async () => { + const collection = await registry.createCollection({ slug: "leased", label: "Leased" }); + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: collection.id, + collection_slug: collection.slug, + force_delete: 1, + state: "leased", + phase: "fence", + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: "live-owner", + lease_expires_at: "2999-01-01T00:00:00.000Z", + }) + .execute(); + + await expect(registry.deleteCollection("leased", { force: true })).rejects.toMatchObject({ + code: "CONFLICT", + }); + }); + + it("does not fence or detach a non-empty collection without force", async () => { + const collection = await registry.createCollection({ slug: "occupied", label: "Occupied" }); + await sql`INSERT INTO ${sql.ref("ec_occupied")} (id, slug) VALUES ('entry-1', 'entry-1')`.execute( + ctx.db, + ); + + await expect(registry.deleteCollection("occupied")).rejects.toThrow(/has content/i); + + expect(await registry.getCollection("occupied")).not.toBeNull(); + expect(await tableExists(ctx.db, "ec_occupied")).toBe(true); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select("capture_state") + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toEqual({ capture_state: "active" }); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .execute(), + ).toEqual([]); + }); + + it("detaches a non-empty activated collection only when force is explicit", async () => { + const collection = await registry.createCollection({ slug: "forced", label: "Forced" }); + await sql`INSERT INTO ${sql.ref("ec_forced")} (id, slug) VALUES ('entry-1', 'entry-1')`.execute( + ctx.db, + ); + + await registry.deleteCollection("forced", { force: true }); + + expect(await registry.getCollection("forced")).toBeNull(); + expect(await tableExists(ctx.db, "ec_forced")).toBe(false); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select(["collection_id", "force_delete", "phase"]) + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toEqual({ collection_id: collection.id, force_delete: 1, phase: "work" }); + }); + + it("fails closed before a tombstone when exact capture triggers are missing", async () => { + const collection = await registry.createCollection({ slug: "unfenced", label: "Unfenced" }); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ capture_state: "deleting" }) + .where("collection_id", "=", collection.id) + .execute(); + await removeMediaUsageCaptureTriggers(ctx.db, { + collectionId: collection.id, + collectionSlug: collection.slug, + }); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ capture_state: "active" }) + .where("collection_id", "=", collection.id) + .execute(); + + await expect(registry.deleteCollection("unfenced", { force: true })).rejects.toThrow( + /capture trigger/i, + ); + + expect(await registry.getCollection("unfenced")).not.toBeNull(); + expect(await tableExists(ctx.db, "ec_unfenced")).toBe(true); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .execute(), + ).toEqual([]); + }); + + it("holds the deleted slug until durable cleanup finalizes", async () => { + await registry.createCollection({ slug: "reserved", label: "Reserved" }); + await registry.deleteCollection("reserved", { force: true }); + + await expect( + registry.createCollection({ slug: "reserved", label: "Replacement" }), + ).rejects.toThrow(); + await expect( + registry.createSeedCollection({ slug: "reserved", label: "Replacement" }, []), + ).rejects.toThrow(); + await sql`CREATE TABLE ${sql.ref("ec_reserved")} (id text primary key)`.execute(ctx.db); + await expect(registry.registerOrphanedTable("reserved")).rejects.toThrow(); + + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_slug") + .where("collection_slug", "=", "reserved") + .executeTakeFirst(), + ).toEqual({ collection_slug: "reserved" }); + }); + + it("rejects a replacement identity that bypasses the slug producer fence", async () => { + const deleted = await registry.createCollection({ slug: "conflicted", label: "Conflicted" }); + await registry.deleteCollection("conflicted", { force: true }); + await ctx.db + .insertInto("_emdash_collections") + .values({ id: "replacement-id", slug: "conflicted", label: "Replacement" }) + .execute(); + + await expect(registry.deleteCollection("conflicted", { force: true })).rejects.toThrow( + /identity conflict/i, + ); + + expect(await registry.getCollection("conflicted")).toEqual( + expect.objectContaining({ id: "replacement-id" }), + ); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .where("collection_slug", "=", "conflicted") + .executeTakeFirst(), + ).toEqual({ collection_id: deleted.id }); + }); + + it("resumes after the lifecycle fence commits before its checkpoint", async () => { + const collection = await registry.createCollection({ slug: "resuming", label: "Resuming" }); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ capture_state: "deleting" }) + .where("collection_id", "=", collection.id) + .execute(); + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: collection.id, + collection_slug: collection.slug, + force_delete: 1, + state: "leased", + phase: "fence", + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: "expired-owner", + lease_expires_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + + await registry.deleteCollection("resuming", { force: true }); + + expect(await registry.getCollection("resuming")).toBeNull(); + expect(await tableExists(ctx.db, "ec_resuming")).toBe(false); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select(["state", "phase"]) + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toEqual({ state: "pending", phase: "work" }); + }); + + it("resumes after registry or table removal commits before its checkpoint", async () => { + for (const phase of ["registry", "table"] as const) { + const slug = `resume_${phase}`; + const collection = await registry.createCollection({ slug, label: slug }); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ capture_state: "deleting" }) + .where("collection_id", "=", collection.id) + .execute(); + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: collection.id, + collection_slug: collection.slug, + force_delete: 1, + state: "leased", + phase, + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: "expired-owner", + lease_expires_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + await ctx.db.deleteFrom("_emdash_collections").where("id", "=", collection.id).execute(); + if (phase === "table") { + await sql`DROP TABLE ${sql.ref(`ec_${slug}`)}`.execute(ctx.db); + } + + await registry.deleteCollection(slug, { force: true }); + await registry.deleteCollection(slug, { force: true }); + + expect(await tableExists(ctx.db, `ec_${slug}`)).toBe(false); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select(["state", "phase"]) + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toEqual({ state: "pending", phase: "work" }); + } + }); + + it.runIf(dialect === "sqlite")( + "persists the tombstone and fence before removing registry identity or table", + async () => { + const collection = await registry.createCollection({ slug: "ordered", label: "Ordered" }); + await sql` + CREATE TRIGGER assert_collection_deletion_order + BEFORE DELETE ON _emdash_collections + WHEN OLD.id = ${sql.lit(collection.id)} + AND ( + NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions + WHERE collection_id = OLD.id + AND collection_slug = OLD.slug + AND state = 'leased' + AND phase = 'registry' + ) + OR NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_index_status + WHERE collection_id = OLD.id AND capture_state = 'deleting' + ) + OR NOT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'ec_ordered' + ) + ) + BEGIN + SELECT RAISE(ABORT, 'collection deletion order violated'); + END + `.execute(ctx.db); + + await registry.deleteCollection("ordered", { force: true }); + + expect(await registry.getCollection("ordered")).toBeNull(); + expect(await tableExists(ctx.db, "ec_ordered")).toBe(false); + }, + ); + + it.runIf(dialect === "postgres")( + "waits for an already-authorized canonical projection before registry removal", + async () => { + const collection = await registry.createCollection({ + slug: "projecting", + label: "Projecting", + }); + const advisoryKey = 8642031; + await sql + .raw(` + CREATE FUNCTION pause_collection_projection() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + PERFORM pg_advisory_xact_lock(8642031); + RETURN NEW; + END; + $$ + `) + .execute(ctx.db); + await sql + .raw(` + CREATE TRIGGER pause_collection_projection + BEFORE INSERT ON _emdash_media_usage + FOR EACH ROW + EXECUTE FUNCTION pause_collection_projection() + `) + .execute(ctx.db); + + let releaseBlocker!: () => void; + let blockerReady!: () => void; + const blockerGate = new Promise((resolve) => { + releaseBlocker = resolve; + }); + const ready = new Promise((resolve) => { + blockerReady = resolve; + }); + const blocker = ctx.db.transaction().execute(async (trx) => { + await sql`SELECT pg_advisory_xact_lock(${advisoryKey})`.execute(trx); + blockerReady(); + await blockerGate; + }); + await ready; + + const sourceKey = `content:v1:${collection.id}:entry-1:columns`; + const projection = new MediaUsageRepository(ctx.db).replaceSource( + { + sourceKey, + sourceType: "content", + collectionId: collection.id, + collectionSlug: collection.slug, + contentId: "entry-1", + sourceVariant: "columns", + identityVersion: 1, + }, + [ + { + fieldSlug: "hero", + fieldPath: "hero", + referenceType: "local", + mediaId: "media-1", + provider: "local", + providerAssetId: "media-1", + }, + ], + ); + + let projectionWaiting = false; + for (let attempt = 0; attempt < 100; attempt++) { + const waiting = await sql<{ present: boolean }>` + SELECT EXISTS ( + SELECT 1 FROM pg_locks + WHERE locktype = 'advisory' + AND objid = ${advisoryKey} + AND NOT granted + ) AS present + `.execute(ctx.db); + if (waiting.rows[0]?.present) { + projectionWaiting = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(projectionWaiting).toBe(true); + + let deletionSettled = false; + const deletion = registry + .deleteCollection("projecting", { force: true }) + .finally(() => (deletionSettled = true)); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(deletionSettled).toBe(false); + + releaseBlocker(); + await blocker; + await projection; + await deletion; + + expect( + await ctx.db + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .where("source_key", "=", sourceKey) + .executeTakeFirst(), + ).toEqual({ source_key: sourceKey }); + }, + ); +}); + +it.each(["collection", "seed", "orphan"] as const)( + "rechecks the durable slug lock before %s producer mutation", + async (producer) => { + const sqlite = new BetterSqlite3(":memory:"); + const prepare = sqlite.prepare.bind(sqlite); + let armed = false; + let inserted = false; + sqlite.prepare = ((source: string) => { + const statement = prepare(source); + if ( + !statement.reader || + !source.toLowerCase().includes("select") || + !source.includes("_emdash_media_usage_collection_deletions") || + !source.includes("collection_slug") + ) { + return statement; + } + return new Proxy(statement, { + get(target, property) { + if (property === "all") { + return (parameters?: unknown[]) => { + const rows = target.all(parameters ?? []); + if (armed && !inserted) { + inserted = true; + prepare(` + INSERT INTO _emdash_media_usage_collection_deletions ( + collection_id, collection_slug, force_delete, state, phase, + next_attempt_at + ) VALUES (?, ?, 1, 'pending', 'work', ?) + `).run("old-collection", "raced", "2000-01-01T00:00:00.000Z"); + } + return rows; + }; + } + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }) as typeof sqlite.prepare; + const db = new Kysely({ dialect: new SqliteDialect({ database: sqlite }) }); + await runMigrations(db); + if (producer === "orphan") { + await sql`CREATE TABLE ec_raced (id text primary key)`.execute(db); + } + armed = true; + const registry = new SchemaRegistry(db); + + const operation = + producer === "collection" + ? registry.createCollection({ slug: "raced", label: "Raced" }) + : producer === "seed" + ? registry.createSeedCollection({ slug: "raced", label: "Raced" }, []) + : registry.registerOrphanedTable("raced"); + await expect(operation).rejects.toThrow(); + expect(inserted).toBe(true); + expect(await registry.getCollection("raced")).toBeNull(); + + await db.destroy(); + }, +); diff --git a/packages/core/tests/unit/media/usage-collection-deletion-virtual.test.ts b/packages/core/tests/unit/media/usage-collection-deletion-virtual.test.ts new file mode 100644 index 0000000000..0b8b53770a --- /dev/null +++ b/packages/core/tests/unit/media/usage-collection-deletion-virtual.test.ts @@ -0,0 +1,18 @@ +import { expect, it } from "vitest"; + +import { isVirtualDialectUnavailableError } from "../../../src/media/usage/collection-deletion.js"; + +it("recognizes Node's unsupported virtual URL without swallowing unrelated loader failures", () => { + const unsupported = Object.assign( + new Error( + "Only URLs with a scheme in: file, data, and node are supported by the default ESM loader. Received protocol 'virtual:'", + ), + { code: "ERR_UNSUPPORTED_ESM_URL_SCHEME" }, + ); + const unrelated = Object.assign(new Error("Unsupported https URL"), { + code: "ERR_UNSUPPORTED_ESM_URL_SCHEME", + }); + + expect(isVirtualDialectUnavailableError(unsupported)).toBe(true); + expect(isVirtualDialectUnavailableError(unrelated)).toBe(false); +}); diff --git a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts index 4be6b834f9..12f57284c2 100644 --- a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts @@ -85,3 +85,78 @@ it("rolls back a stale guarded batch before any collection DDL", async () => { .execute(), ).toEqual([{ collection_id: "collection-d1" }]); }); + +it("atomically preserves content or fences an empty collection", async () => { + await sql`CREATE TABLE ec_d1_fence (id TEXT PRIMARY KEY)`.execute(db); + await ctxInsertCollection(); + await db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "d1_fence", + collection_id: "collection-d1-fence", + status: "complete", + capture_state: "active", + }) + .execute(); + await db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-d1-fence", + collection_slug: "d1_fence", + force_delete: 0, + state: "leased", + phase: "fence", + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: "fence-owner", + lease_expires_at: "2999-01-01T00:00:00.000Z", + }) + .execute(); + await sql`INSERT INTO ec_d1_fence (id) VALUES ('entry-1')`.execute(db); + + await expect( + executeCollectionDeletionGuard( + { binding: "DB" }, + { + action: "fence", + collectionId: "collection-d1-fence", + collectionSlug: "d1_fence", + leaseToken: "fence-owner", + forceDelete: false, + }, + ), + ).resolves.toEqual({ outcome: "has_content" }); + expect(await captureState()).toBe("active"); + + await sql`DELETE FROM ec_d1_fence`.execute(db); + await expect( + executeCollectionDeletionGuard( + { binding: "DB" }, + { + action: "fence", + collectionId: "collection-d1-fence", + collectionSlug: "d1_fence", + leaseToken: "fence-owner", + forceDelete: false, + }, + ), + ).resolves.toEqual({ outcome: "fenced" }); + expect(await captureState()).toBe("deleting"); +}); + +async function ctxInsertCollection(): Promise { + await db + .insertInto("_emdash_collections") + .values({ id: "collection-d1-fence", slug: "d1_fence", label: "D1 fence" }) + .execute(); +} + +async function captureState(): Promise { + const row = await db + .selectFrom("_emdash_media_usage_index_status") + .select("capture_state") + .where("collection_id", "=", "collection-d1-fence") + .executeTakeFirst(); + return row?.capture_state ?? null; +} From a576d1039a92f993f420421cdf955c422ffe09d8 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:00:37 +0100 Subject: [PATCH 03/20] feat(core): process bounded collection deletion cleanup --- packages/core/src/emdash-runtime.ts | 9 + .../usage/collection-deletion-processor.ts | 345 ++++++++++++++++++ .../src/media/usage/collection-deletion.ts | 113 +++++- ...sage-collection-deletion-processor.test.ts | 331 +++++++++++++++++ .../media-usage-scheduled-driver.test.ts | 30 ++ ...media-usage-collection-deletion-d1.test.ts | 39 ++ 6 files changed, 860 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/media/usage/collection-deletion-processor.ts create mode 100644 packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index 3dc7f6789e..f77d00089b 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -42,6 +42,7 @@ import { getI18nConfig } from "./i18n/config.js"; import { repairLocaleCasing } from "./i18n/repair-locale-casing.js"; import { normalizeMediaValue } from "./media/normalize.js"; import type { MediaProvider, MediaProviderCapabilities } from "./media/types.js"; +import { processDueMediaUsageCollectionDeletions } from "./media/usage/collection-deletion-processor.js"; import { deleteContentMediaUsage, findNonTranslatableSiblingContentIds, @@ -540,6 +541,14 @@ async function runScheduledMediaUsageWork(db: Kysely): Promise { } catch (error) { console.error("[media-usage:work] Scheduled processing failed:", error); } + try { + const result = await processDueMediaUsageCollectionDeletions(db); + if (result.candidateCount > 0) { + console.info("[media-usage:collection-deletion] Scheduled processing", result); + } + } catch (error) { + console.error("[media-usage:collection-deletion] Scheduled processing failed:", error); + } } /** diff --git a/packages/core/src/media/usage/collection-deletion-processor.ts b/packages/core/src/media/usage/collection-deletion-processor.ts new file mode 100644 index 0000000000..8a45696ab4 --- /dev/null +++ b/packages/core/src/media/usage/collection-deletion-processor.ts @@ -0,0 +1,345 @@ +import { sql, type Kysely, type RawBuilder, type Transaction, type Updateable } from "kysely"; + +import { isPostgres, tableExists } from "../../database/dialect-helpers.js"; +import { withTransaction } from "../../database/transaction.js"; +import type { Database, MediaUsageCollectionDeletionTable } from "../../database/types.js"; +import { + deleteActivatedMediaUsageCollection, + MediaUsageCollectionDeletionRepository, + type MediaUsageCollectionDeletionRecord, +} from "./collection-deletion.js"; + +export const MEDIA_USAGE_COLLECTION_DELETION_LIMITS = Object.freeze({ + candidatesPerTick: 4, + deletionsPerTick: 1, + rowsPerBatch: 50, + leaseDurationSeconds: 5 * 60, + maxAttempts: 5, + retryBaseSeconds: 30, + retryMaxSeconds: 15 * 60, +}); + +export interface MediaUsageCollectionDeletionTickResult { + candidateCount: number; + claimedCount: number; + outcome: "idle" | "progress" | "finalized" | "retry" | "failed" | "claim_lost"; +} + +type DatabaseExecutor = Kysely | Transaction; + +export async function processDueMediaUsageCollectionDeletions( + db: Kysely, +): Promise { + const repository = new MediaUsageCollectionDeletionRepository(db); + const candidates = await repository.findDue( + MEDIA_USAGE_COLLECTION_DELETION_LIMITS.candidatesPerTick, + ); + if (candidates.length === 0) return { candidateCount: 0, claimedCount: 0, outcome: "idle" }; + + let claim: (MediaUsageCollectionDeletionRecord & { leaseToken: string }) | null = null; + for (const candidate of candidates) { + claim = await repository.claim({ + collectionId: candidate.collectionId, + phase: candidate.phase, + leaseDurationSeconds: MEDIA_USAGE_COLLECTION_DELETION_LIMITS.leaseDurationSeconds, + }); + if (claim?.leaseToken) break; + } + if (!claim?.leaseToken) { + return { candidateCount: candidates.length, claimedCount: 0, outcome: "claim_lost" }; + } + + try { + const processed = await processClaimedDeletion(db, claim); + if (!processed.finalized && !processed.released && !(await repository.release(claim))) { + return { candidateCount: candidates.length, claimedCount: 1, outcome: "claim_lost" }; + } + return { + candidateCount: candidates.length, + claimedCount: 1, + outcome: processed.finalized ? "finalized" : "progress", + }; + } catch (error) { + const terminal = claim.attemptCount + 1 >= MEDIA_USAGE_COLLECTION_DELETION_LIMITS.maxAttempts; + const recorded = await repository.recordFailure({ + collectionId: claim.collectionId, + leaseToken: claim.leaseToken, + errorCode: "MEDIA_USAGE_COLLECTION_DELETION_FAILED", + terminal, + retryDelaySeconds: retryDelaySeconds(claim.attemptCount), + }); + if (!recorded) { + return { candidateCount: candidates.length, claimedCount: 1, outcome: "claim_lost" }; + } + console.error("[media-usage:collection-deletion] Processing failed:", error); + return { + candidateCount: candidates.length, + claimedCount: 1, + outcome: terminal ? "failed" : "retry", + }; + } +} + +async function processClaimedDeletion( + db: Kysely, + claim: MediaUsageCollectionDeletionRecord & { leaseToken: string }, +): Promise<{ finalized: boolean; released: boolean }> { + if (claim.phase === "fence" || claim.phase === "registry" || claim.phase === "table") { + await deleteActivatedMediaUsageCollection( + db, + { + collectionId: claim.collectionId, + collectionSlug: claim.collectionSlug, + forceDelete: claim.forceDelete, + }, + { frontPhaseLimit: 1, claimed: claim }, + ); + return { finalized: false, released: true }; + } + if (claim.phase === "work") await processWorkBatch(db, claim); + if (claim.phase === "sources") await processSourceBatch(db, claim); + if (claim.phase === "status") await processStatus(db, claim); + if (claim.phase === "finalize") { + await finalizeDeletion(db, claim); + return { finalized: true, released: true }; + } + return { finalized: false, released: false }; +} + +async function processWorkBatch( + db: Kysely, + claim: MediaUsageCollectionDeletionRecord & { leaseToken: string }, +): Promise { + await withTransaction(db, async (trx) => { + const rows = await trx + .selectFrom("_emdash_media_usage_work") + .select("content_id") + .where("collection_id", "=", claim.collectionId) + .$if(claim.workCursor !== null, (query) => query.where("content_id", ">", claim.workCursor!)) + .orderBy("content_id", "asc") + .limit(MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch + 1) + .execute(); + const batch = rows.slice(0, MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch); + if (batch.length > 0) { + await trx + .deleteFrom("_emdash_media_usage_work") + .where("collection_id", "=", claim.collectionId) + .where( + "content_id", + "in", + batch.map((row) => row.content_id), + ) + .where(liveLeaseGuard(db, claim)) + .execute(); + } + await updateDeletion(trx, claim, { + phase: rows.length > MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch ? "work" : "sources", + work_cursor: + rows.length > MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch + ? batch.at(-1)!.content_id + : null, + }); + }); + return false; +} + +async function processSourceBatch( + db: Kysely, + claim: MediaUsageCollectionDeletionRecord & { leaseToken: string }, +): Promise { + await withTransaction(db, async (trx) => { + let sourceKey = claim.sourceKey; + if (!sourceKey) { + const source = await trx + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .where("source_type", "=", "content") + .where("collection_id", "=", claim.collectionId) + .orderBy("source_key", "asc") + .limit(1) + .executeTakeFirst(); + if (!source) { + await updateDeletion(trx, claim, { + phase: "status", + source_key: null, + occurrence_cursor: null, + }); + return; + } + sourceKey = source.source_key; + await updateDeletion(trx, claim, { source_key: sourceKey, occurrence_cursor: null }); + } + + const occurrences = await trx + .selectFrom("_emdash_media_usage") + .select("id") + .where("source_key", "=", sourceKey) + .$if(claim.occurrenceCursor !== null, (query) => + query.where("id", ">", claim.occurrenceCursor!), + ) + .orderBy("id", "asc") + .limit(MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch + 1) + .execute(); + const batch = occurrences.slice(0, MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch); + if (batch.length > 0) { + await trx + .deleteFrom("_emdash_media_usage") + .where("source_key", "=", sourceKey) + .where( + "id", + "in", + batch.map((row) => row.id), + ) + .where(liveLeaseGuard(db, claim)) + .execute(); + } + if (occurrences.length > MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch) { + await updateDeletion(trx, claim, { + source_key: sourceKey, + occurrence_cursor: batch.at(-1)!.id, + }); + return; + } + await trx + .deleteFrom("_emdash_media_usage_sources") + .where("source_key", "=", sourceKey) + .where("source_type", "=", "content") + .where("collection_id", "=", claim.collectionId) + .where(liveLeaseGuard(db, claim)) + .execute(); + await updateDeletion(trx, claim, { source_key: null, occurrence_cursor: null }); + }); + return false; +} + +async function processStatus( + db: Kysely, + claim: MediaUsageCollectionDeletionRecord & { leaseToken: string }, +): Promise { + await withTransaction(db, async (trx) => { + if (await exactCleanupRowsRemain(trx, claim, false)) { + throw new Error("Collection deletion cleanup is incomplete"); + } + await trx + .deleteFrom("_emdash_media_usage_index_status") + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", claim.collectionSlug) + .where("collection_id", "=", claim.collectionId) + .where(liveLeaseGuard(db, claim)) + .execute(); + await updateDeletion(trx, claim, { phase: "finalize" }); + }); + return false; +} + +async function finalizeDeletion( + db: Kysely, + claim: MediaUsageCollectionDeletionRecord & { leaseToken: string }, +): Promise { + if (await tableExists(db, `ec_${claim.collectionSlug}`)) { + throw new Error("Collection table still exists during deletion finalization"); + } + if (await exactCleanupRowsRemain(db, claim)) throw new Error("Collection deletion is incomplete"); + const registry = await db + .selectFrom("_emdash_collections") + .select("id") + .where("id", "=", claim.collectionId) + .where("slug", "=", claim.collectionSlug) + .executeTakeFirst(); + if (registry) throw new Error("Collection registry identity still exists"); + const result = await db + .deleteFrom("_emdash_media_usage_collection_deletions") + .where("collection_id", "=", claim.collectionId) + .where("collection_slug", "=", claim.collectionSlug) + .where("state", "=", "leased") + .where("phase", "=", "finalize") + .where("lease_token", "=", claim.leaseToken) + .where(liveLeaseGuard(db, claim)) + .executeTakeFirst(); + if (Number(result.numDeletedRows ?? 0) !== 1) + throw new Error("Collection deletion lost finalization"); + return true; +} + +async function exactCleanupRowsRemain( + db: DatabaseExecutor, + claim: Pick, + includeStatus = true, +): Promise { + const [work, source, status] = await Promise.all([ + db + .selectFrom("_emdash_media_usage_work") + .select("content_id") + .where("collection_id", "=", claim.collectionId) + .limit(1) + .executeTakeFirst(), + db + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .where("source_type", "=", "content") + .where("collection_id", "=", claim.collectionId) + .limit(1) + .executeTakeFirst(), + db + .selectFrom("_emdash_media_usage_index_status") + .select("collection_id") + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", claim.collectionSlug) + .where("collection_id", "=", claim.collectionId) + .limit(1) + .executeTakeFirst(), + ]); + return work !== undefined || source !== undefined || (includeStatus && status !== undefined); +} + +async function updateDeletion( + db: DatabaseExecutor, + claim: MediaUsageCollectionDeletionRecord & { leaseToken: string }, + values: Updateable, +): Promise { + const result = await db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ + ...values, + attempt_count: 0, + last_error_code: null, + updated_at: new Date().toISOString(), + }) + .where("collection_id", "=", claim.collectionId) + .where("state", "=", "leased") + .where("lease_token", "=", claim.leaseToken) + .where(liveLeaseGuard(db, claim)) + .executeTakeFirst(); + if (Number(result.numUpdatedRows ?? 0) !== 1) + throw new Error("Collection deletion lease was lost"); +} + +function liveLeaseGuard( + db: DatabaseExecutor, + claim: Pick & { leaseToken: string }, +): RawBuilder { + return isPostgres(db) + ? sql`EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = ${claim.collectionId} + AND deletion.state = 'leased' + AND deletion.lease_token = ${claim.leaseToken} + AND deletion.lease_expires_at::timestamptz > clock_timestamp() + )` + : sql`EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = ${claim.collectionId} + AND deletion.state = 'leased' + AND deletion.lease_token = ${claim.leaseToken} + AND deletion.lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + )`; +} + +function retryDelaySeconds(attemptCount: number): number { + return Math.min( + MEDIA_USAGE_COLLECTION_DELETION_LIMITS.retryMaxSeconds, + MEDIA_USAGE_COLLECTION_DELETION_LIMITS.retryBaseSeconds * 2 ** attemptCount, + ); +} diff --git a/packages/core/src/media/usage/collection-deletion.ts b/packages/core/src/media/usage/collection-deletion.ts index 08990f2fad..39e6b00528 100644 --- a/packages/core/src/media/usage/collection-deletion.ts +++ b/packages/core/src/media/usage/collection-deletion.ts @@ -128,6 +128,42 @@ export class MediaUsageCollectionDeletionRepository { return row ? rowToRecord(row) : null; } + async findDue(limit: number): Promise { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) { + throw new Error("Collection deletion candidate limit must be from 1 to 100"); + } + const nextAttemptIsDue = timestampIsDue(this.db, "next_attempt_at"); + const leaseIsDue = timestampIsDue(this.db, "lease_expires_at"); + const result = await sql>` + WITH pending_candidates AS ( + SELECT * FROM _emdash_media_usage_collection_deletions + WHERE state = 'pending' AND ${nextAttemptIsDue} + ORDER BY next_attempt_at, updated_at, collection_id + LIMIT ${limit} + ), retry_candidates AS ( + SELECT * FROM _emdash_media_usage_collection_deletions + WHERE state = 'retry' AND ${nextAttemptIsDue} + ORDER BY next_attempt_at, updated_at, collection_id + LIMIT ${limit} + ), leased_candidates AS ( + SELECT * FROM _emdash_media_usage_collection_deletions + WHERE state = 'leased' AND ${leaseIsDue} + ORDER BY lease_expires_at, updated_at, collection_id + LIMIT ${limit} + ), candidates AS ( + SELECT * FROM pending_candidates + UNION ALL SELECT * FROM retry_candidates + UNION ALL SELECT * FROM leased_candidates + ) + SELECT * FROM candidates + ORDER BY CASE WHEN state = 'leased' THEN lease_expires_at ELSE next_attempt_at END, + updated_at, + collection_id + LIMIT ${limit} + `.execute(this.db); + return result.rows.map(rowToRecord); + } + async checkpoint(input: { collectionId: string; leaseToken: string; @@ -136,7 +172,12 @@ export class MediaUsageCollectionDeletionRepository { }): Promise { const result = await this.db .updateTable("_emdash_media_usage_collection_deletions") - .set({ phase: input.toPhase, updated_at: timestampOffset(this.db, 0) }) + .set({ + phase: input.toPhase, + attempt_count: 0, + last_error_code: null, + updated_at: timestampOffset(this.db, 0), + }) .where("collection_id", "=", input.collectionId) .where("state", "=", "leased") .where("phase", "=", input.fromPhase) @@ -155,6 +196,8 @@ export class MediaUsageCollectionDeletionRepository { next_attempt_at: now, lease_token: null, lease_expires_at: null, + attempt_count: 0, + last_error_code: null, updated_at: now, }) .where("collection_id", "=", input.collectionId) @@ -177,6 +220,32 @@ export class MediaUsageCollectionDeletionRepository { return Number(result.numDeletedRows ?? 0) === 1; } + async recordFailure(input: { + collectionId: string; + leaseToken: string; + errorCode: string; + terminal: boolean; + retryDelaySeconds: number; + }): Promise { + const result = await this.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ + state: input.terminal ? "failed" : "retry", + attempt_count: sql`attempt_count + 1`, + next_attempt_at: timestampOffset(this.db, input.retryDelaySeconds), + lease_token: null, + lease_expires_at: null, + last_error_code: input.errorCode, + updated_at: timestampOffset(this.db, 0), + }) + .where("collection_id", "=", input.collectionId) + .where("state", "=", "leased") + .where("lease_token", "=", input.leaseToken) + .where(liveLease(this.db)) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + async deleteRegistryAndCheckpoint(input: { collectionId: string; collectionSlug: string; @@ -221,6 +290,7 @@ export type ActivatedCollectionDeletionOutcome = | "inactive" | "not_found" | "in_progress" + | "advanced" | "deleted" | "has_content"; @@ -231,7 +301,15 @@ export async function deleteActivatedMediaUsageCollection( collectionId?: string; forceDelete: boolean; }, + options: { + frontPhaseLimit?: number; + claimed?: MediaUsageCollectionDeletionRecord & { leaseToken: string }; + } = {}, ): Promise { + const frontPhaseLimit = options.frontPhaseLimit ?? 3; + if (!Number.isSafeInteger(frontPhaseLimit) || frontPhaseLimit < 1 || frontPhaseLimit > 3) { + throw new Error("Collection deletion front-phase limit must be from 1 to 3"); + } validateIdentifier(input.collectionSlug, "collection slug"); const repository = new MediaUsageCollectionDeletionRepository(db); let deletion = await repository.findBySlug(input.collectionSlug); @@ -263,13 +341,19 @@ export async function deleteActivatedMediaUsageCollection( if (deletion.phase !== "fence" && deletion.phase !== "registry" && deletion.phase !== "table") { return "deleted"; } - const claim = await repository.claim({ - collectionId: deletion.collectionId, - phase: deletion.phase, - leaseDurationSeconds: 5 * 60, - }); + const claim = + options.claimed ?? + (await repository.claim({ + collectionId: deletion.collectionId, + phase: deletion.phase, + leaseDurationSeconds: 5 * 60, + })); if (!claim) return "in_progress"; + if (claim.collectionId !== deletion.collectionId || claim.phase !== deletion.phase) { + throw new Error("Collection deletion claim identity conflict"); + } let phase = claim.phase; + let processedFrontPhases = 0; const lease = { collectionId: claim.collectionId, leaseToken: claim.leaseToken }; if (phase === "fence") { @@ -306,6 +390,12 @@ export async function deleteActivatedMediaUsageCollection( throw new Error("Collection deletion lost its fence checkpoint"); } phase = "registry"; + processedFrontPhases++; + if (processedFrontPhases >= frontPhaseLimit) { + if (!(await repository.release(lease))) + throw new Error("Collection deletion lost its handoff"); + return "advanced"; + } } if (phase === "registry") { @@ -318,6 +408,12 @@ export async function deleteActivatedMediaUsageCollection( throw new Error("Collection deletion lost its registry checkpoint"); } phase = "table"; + processedFrontPhases++; + if (processedFrontPhases >= frontPhaseLimit) { + if (!(await repository.release(lease))) + throw new Error("Collection deletion lost its handoff"); + return "advanced"; + } } if (phase === "table") { @@ -528,7 +624,10 @@ function timestampIsDue( column: "next_attempt_at" | "lease_expires_at", ): RawBuilder { return isPostgres(db) - ? sql`${sql.ref(column)}::timestamptz <= clock_timestamp()` + ? sql`${sql.ref(column)} <= to_char( + statement_timestamp() AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"' + )` : sql`${sql.ref(column)} <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; } diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts new file mode 100644 index 0000000000..082dc6bcb3 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts @@ -0,0 +1,331 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { activateMediaUsageCapture } from "../../../src/media/usage/activation.js"; +import { processDueMediaUsageCollectionDeletions } from "../../../src/media/usage/collection-deletion-processor.js"; +import { MediaUsageCollectionDeletionRepository } from "../../../src/media/usage/collection-deletion.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + setupTestDatabaseWithCompoundSelectLimit, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage collection deletion processor", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("resumes bounded exact-ID cleanup and removes the tombstone last", async () => { + await insertDeletion("old-id", "articles", "work"); + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "articles", + collection_id: "old-id", + status: "stale", + capture_state: "deleting", + }) + .execute(); + await ctx.db + .insertInto("_emdash_media_usage_work") + .values( + Array.from({ length: 55 }, (_, index) => ({ + collection_id: "old-id", + collection_slug: "articles", + content_id: `entry-${String(index).padStart(3, "0")}`, + change_epoch: 1, + next_attempt_at: "2000-01-01T00:00:00.000Z", + })), + ) + .execute(); + await insertSource("old-source", "old-id", "articles"); + await ctx.db + .insertInto("_emdash_media_usage") + .values( + Array.from({ length: 55 }, (_, index) => ({ + id: `usage-${String(index).padStart(3, "0")}`, + source_key: "old-source", + generation: "generation-old", + field_slug: "hero", + field_path: `hero[${index}]`, + occurrence_index: index, + reference_type: "local", + media_id: `media-${index}`, + provider_asset_id: `media-${index}`, + })), + ) + .execute(); + await insertSource("legacy-source", null, "articles"); + await insertSource("replacement-source", "replacement-id", "articles"); + + await expect(runTick()).resolves.toMatchObject({ claimedCount: 1, outcome: "progress" }); + expect(await workCount("old-id")).toBe(5); + expect(await deletionState("old-id")).toEqual( + expect.objectContaining({ phase: "work", work_cursor: "entry-049" }), + ); + + await runTick(); + expect(await workCount("old-id")).toBe(0); + expect(await deletionState("old-id")).toEqual(expect.objectContaining({ phase: "sources" })); + + await runTick(); + expect(await usageCount("old-source")).toBe(5); + expect(await deletionState("old-id")).toEqual( + expect.objectContaining({ + phase: "sources", + source_key: "old-source", + occurrence_cursor: "usage-049", + }), + ); + await ctx.db + .deleteFrom("_emdash_media_usage_sources") + .where("source_key", "=", "old-source") + .execute(); + + await runTick(); + expect(await usageCount("old-source")).toBe(0); + expect(await sourceExists("old-source")).toBe(false); + await runTick(); + expect(await deletionState("old-id")).toEqual(expect.objectContaining({ phase: "status" })); + await runTick(); + expect(await deletionState("old-id")).toEqual(expect.objectContaining({ phase: "finalize" })); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select("collection_id") + .where("collection_id", "=", "old-id") + .executeTakeFirst(), + ).toBeUndefined(); + await runTick(); + expect(await deletionState("old-id")).toBeNull(); + expect(await sourceExists("legacy-source")).toBe(true); + expect(await sourceExists("replacement-source")).toBe(true); + }); + + it("recovers exactly one interrupted front phase per tick", async () => { + await activateMediaUsageCapture(ctx.db, { writersDrained: true }); + const registry = new SchemaRegistry(ctx.db); + const collection = await registry.createCollection({ slug: "front", label: "Front" }); + await new MediaUsageCollectionDeletionRepository(ctx.db).createTombstone({ + collectionId: collection.id, + collectionSlug: collection.slug, + forceDelete: true, + }); + + await runTick(); + expect(await deletionState(collection.id)).toEqual( + expect.objectContaining({ state: "pending", phase: "registry" }), + ); + expect(await registry.getCollection("front")).not.toBeNull(); + + await runTick(); + expect(await deletionState(collection.id)).toEqual( + expect.objectContaining({ state: "pending", phase: "table" }), + ); + expect(await registry.getCollection("front")).toBeNull(); + + await runTick(); + expect(await deletionState(collection.id)).toEqual( + expect.objectContaining({ state: "pending", phase: "work" }), + ); + }); + + it("makes five consecutive phase failures visible", async () => { + await insertDeletion("failed-id", "failed", "status"); + await ctx.db + .insertInto("_emdash_media_usage_work") + .values({ + collection_id: "failed-id", + collection_slug: "failed", + content_id: "still-present", + change_epoch: 1, + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + for (let attempt = 1; attempt <= 5; attempt++) { + const result = await runTick(); + expect(result.outcome).toBe(attempt === 5 ? "failed" : "retry"); + if (attempt < 5) { + await ctx.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ next_attempt_at: "2000-01-01T00:00:00.000Z" }) + .where("collection_id", "=", "failed-id") + .execute(); + } + } + + expect(await deletionState("failed-id")).toEqual( + expect.objectContaining({ + state: "failed", + phase: "status", + attempt_count: 5, + last_error_code: "MEDIA_USAGE_COLLECTION_DELETION_FAILED", + }), + ); + error.mockRestore(); + }); + + it("inspects at most four candidates and claims at most one deletion", async () => { + for (let index = 0; index < 5; index++) { + await insertDeletion(`collection-${index}`, `collection_${index}`, "work"); + } + + const result = await runTick(); + + expect(result.candidateCount).toBe(4); + expect(result.claimedCount).toBe(1); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("phase") + .where("phase", "=", "sources") + .execute(), + ).toHaveLength(1); + }); + + it.runIf(dialect === "sqlite")("tries the next due candidate after a claim race", async () => { + await insertDeletion("first-id", "first", "work"); + await insertDeletion("second-id", "second", "work"); + await sql + .raw(` + CREATE TRIGGER lose_first_collection_deletion_claim + BEFORE UPDATE OF state ON _emdash_media_usage_collection_deletions + WHEN OLD.collection_id = 'first-id' AND NEW.state = 'leased' + BEGIN + SELECT RAISE(IGNORE); + END + `) + .execute(ctx.db); + + const result = await runTick(); + + expect(result).toMatchObject({ candidateCount: 2, claimedCount: 1, outcome: "progress" }); + expect(await deletionState("first-id")).toEqual(expect.objectContaining({ phase: "work" })); + expect(await deletionState("second-id")).toEqual(expect.objectContaining({ phase: "sources" })); + }); + + function runTick() { + return processDueMediaUsageCollectionDeletions(ctx.db); + } + + async function insertDeletion(collectionId: string, slug: string, phase: string) { + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: collectionId, + collection_slug: slug, + force_delete: 1, + state: "pending", + phase, + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + } + + async function insertSource(sourceKey: string, collectionId: string | null, slug: string) { + await ctx.db + .insertInto("_emdash_media_usage_sources") + .values({ + source_key: sourceKey, + source_type: "content", + collection_id: collectionId, + collection_slug: slug, + content_id: "entry", + source_variant: "columns", + current_generation: "generation-old", + }) + .execute(); + } + + async function workCount(collectionId: string) { + const row = await ctx.db + .selectFrom("_emdash_media_usage_work") + .select((eb) => eb.fn.countAll().as("count")) + .where("collection_id", "=", collectionId) + .executeTakeFirstOrThrow(); + return Number(row.count); + } + + async function usageCount(sourceKey: string) { + const row = await ctx.db + .selectFrom("_emdash_media_usage") + .select((eb) => eb.fn.countAll().as("count")) + .where("source_key", "=", sourceKey) + .executeTakeFirstOrThrow(); + return Number(row.count); + } + + async function sourceExists(sourceKey: string) { + return ( + (await ctx.db + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .where("source_key", "=", sourceKey) + .executeTakeFirst()) !== undefined + ); + } + + async function deletionState(collectionId: string) { + return ( + (await ctx.db + .selectFrom("_emdash_media_usage_collection_deletions") + .selectAll() + .where("collection_id", "=", collectionId) + .executeTakeFirst()) ?? null + ); + } +}); + +it("selects one globally bounded due-candidate window", async () => { + const fixture = await setupTestDatabaseWithCompoundSelectLimit(null); + try { + for (const [state, timestamp] of [ + ["pending", "next_attempt_at"], + ["retry", "next_attempt_at"], + ["leased", "lease_expires_at"], + ] as const) { + for (let index = 0; index < 4; index++) { + await fixture.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: `${state}-${index}`, + collection_slug: `${state}_${index}`, + force_delete: 1, + state, + phase: "work", + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: state === "leased" ? `lease-${index}` : null, + lease_expires_at: timestamp === "lease_expires_at" ? "2000-01-01T00:00:00.000Z" : null, + }) + .execute(); + } + } + fixture.statements.length = 0; + + const due = await new MediaUsageCollectionDeletionRepository(fixture.db).findDue(4); + + expect(due).toHaveLength(4); + expect( + fixture.statements.filter( + (statement) => + /^(?:select|with)/i.test(statement.trim()) && + statement.includes("_emdash_media_usage_collection_deletions"), + ), + ).toHaveLength(1); + } finally { + await fixture.db.destroy(); + } +}); diff --git a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts index 2d535c4585..fdf2b50032 100644 --- a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts +++ b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts @@ -48,6 +48,27 @@ describe("media usage scheduled drivers", () => { ).not.toBeNull(); }); + it("advances bounded collection deletion from the Cloudflare scheduled entry point", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const fixture = await activateCollection(runtime, "cloudflare_delete"); + await runtime.schemaRegistry.deleteCollection("cloudflare_delete", { force: true }); + + await runtime.runScheduledTasks(); + + expect(await deletionPhase(runtime, fixture.collectionId)).toBe("sources"); + }); + + it("advances bounded collection deletion from the Node maintenance callback", async () => { + const scheduler = new CapturingScheduler(); + runtime = await EmDashRuntime.create(createDeps(() => scheduler)); + const fixture = await activateCollection(runtime, "node_delete"); + await runtime.schemaRegistry.deleteCollection("node_delete", { force: true }); + + await scheduler.runMaintenance(); + + expect(await deletionPhase(runtime, fixture.collectionId)).toBe("sources"); + }); + it("processes a trigger-created job before returning from an authenticated write", async () => { runtime = await EmDashRuntime.create(createDeps(null)); const fixture = await activateCollection(runtime, "fast_posts"); @@ -165,6 +186,15 @@ async function countWork(runtime: EmDashRuntime): Promise { return Number(row.count); } +async function deletionPhase(runtime: EmDashRuntime, collectionId: string): Promise { + const row = await runtime.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("phase") + .where("collection_id", "=", collectionId) + .executeTakeFirst(); + return row?.phase ?? null; +} + function canonicalSourceKey(collectionId: string, contentId: string): string { return `content:${collectionId}:${contentId}:columns`; } diff --git a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts index 12f57284c2..33117d7132 100644 --- a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts @@ -6,6 +6,7 @@ import { RawBindingD1Dialect } from "../../../cloudflare/src/db/d1-dialect.js"; import { executeCollectionDeletionGuard } from "../../../cloudflare/src/db/d1.js"; import { runMigrations } from "../../src/database/migrations/runner.js"; import type { Database } from "../../src/database/types.js"; +import { processDueMediaUsageCollectionDeletions } from "../../src/media/usage/collection-deletion-processor.js"; declare module "cloudflare:test" { interface ProvidedEnv { @@ -145,6 +146,44 @@ it("atomically preserves content or fences an empty collection", async () => { expect(await captureState()).toBe("deleting"); }); +it("drains at most fifty exact-ID work rows in a real D1 tick", async () => { + await db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-d1-cleanup", + collection_slug: "d1_cleanup", + force_delete: 1, + state: "pending", + phase: "work", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + const work = Array.from({ length: 51 }, (_, index) => ({ + collection_id: "collection-d1-cleanup", + collection_slug: "d1_cleanup", + content_id: `d1-entry-${String(index).padStart(3, "0")}`, + change_epoch: 1, + next_attempt_at: "2000-01-01T00:00:00.000Z", + })); + for (let index = 0; index < work.length; index += 10) { + await db + .insertInto("_emdash_media_usage_work") + .values(work.slice(index, index + 10)) + .execute(); + } + + await expect(processDueMediaUsageCollectionDeletions(db)).resolves.toMatchObject({ + claimedCount: 1, + outcome: "progress", + }); + const remaining = await db + .selectFrom("_emdash_media_usage_work") + .select("content_id") + .where("collection_id", "=", "collection-d1-cleanup") + .execute(); + expect(remaining).toEqual([{ content_id: "d1-entry-050" }]); +}); + async function ctxInsertCollection(): Promise { await db .insertInto("_emdash_collections") From 83dbf0c7bfa3f3a609c7942effabbbe5358334c5 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:27:11 +0100 Subject: [PATCH 04/20] feat(core): expose collection deletion recovery controls --- .changeset/safe-collection-deletion.md | 5 + docs/src/content/docs/reference/rest-api.mdx | 24 +++ packages/core/src/api/errors.ts | 2 + .../core/src/api/handlers/media-usage-work.ts | 75 ++++++++ packages/core/src/api/openapi/document.ts | 53 ++++++ packages/core/src/api/schemas/media-usage.ts | 55 ++++++ packages/core/src/astro/integration/routes.ts | 8 + .../media-usage/collection-deletions/index.ts | 22 +++ .../media-usage/collection-deletions/retry.ts | 22 +++ packages/core/src/client/index.ts | 48 +++++ .../src/media/usage/collection-deletion.ts | 162 +++++++++++++++-- ...usage-collection-deletion-operator.test.ts | 85 +++++++++ ...sage-collection-deletion-processor.test.ts | 40 +++++ ...ia-usage-collection-deletion-route.test.ts | 55 ++++++ ...media-usage-collection-deletion-d1.test.ts | 165 ++++++++++++++++++ 15 files changed, 809 insertions(+), 12 deletions(-) create mode 100644 .changeset/safe-collection-deletion.md create mode 100644 packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/index.ts create mode 100644 packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/retry.ts create mode 100644 packages/core/tests/integration/database/media-usage-collection-deletion-operator.test.ts create mode 100644 packages/core/tests/unit/api/media-usage-collection-deletion-route.test.ts diff --git a/.changeset/safe-collection-deletion.md b/.changeset/safe-collection-deletion.md new file mode 100644 index 0000000000..8facf4d0c7 --- /dev/null +++ b/.changeset/safe-collection-deletion.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Adds crash-safe collection deletion with bounded Media Usage cleanup and operator recovery controls. diff --git a/docs/src/content/docs/reference/rest-api.mdx b/docs/src/content/docs/reference/rest-api.mdx index 7f819dd9b0..593d6da617 100644 --- a/docs/src/content/docs/reference/rest-api.mdx +++ b/docs/src/content/docs/reference/rest-api.mdx @@ -432,6 +432,30 @@ collection-scoped Media Usage repair after imports or direct database writes. When scheduled maintenance is disabled, failed jobs remain visible and manually retryable, but no automatic freshness deadline is promised. +### Recover Collection Deletion + +```http +GET /_emdash/api/admin/media-usage/collection-deletions?state=failed&limit=50&cursor=... +``` + +Returns a bounded page of durable collection-deletion work. The list defaults to failed work; +`limit` defaults to 50 and is capped at 100. Items include the immutable collection ID, slug, +phase, attempts, eligibility/lease timestamps, stable error code, and update time. Lease tokens, +raw database errors, content, media references, and exact backlog counts are never returned. + +```http +POST /_emdash/api/admin/media-usage/collection-deletions/retry +Content-Type: application/json +X-EmDash-Request: 1 + +{ "collectionId": "01COLLECTION..." } +``` + +Retry reopens failed, retrying, or expired-leased work without changing its phase. A live lease +returns `409 WORK_LEASE_ACTIVE`; a concurrent state change returns `409 WORK_CHANGED`. Both routes +require `schema:manage`, and bearer tokens also require the `admin` scope. They recover internal +index cleanup only and never delete media assets. + ### Repair Media Usage ```http diff --git a/packages/core/src/api/errors.ts b/packages/core/src/api/errors.ts index 92790f774c..e31b99a9bf 100644 --- a/packages/core/src/api/errors.ts +++ b/packages/core/src/api/errors.ts @@ -90,6 +90,8 @@ export const ErrorCode = { MEDIA_USAGE_REPAIR_ERROR: "MEDIA_USAGE_REPAIR_ERROR", MEDIA_USAGE_WORK_LIST_ERROR: "MEDIA_USAGE_WORK_LIST_ERROR", MEDIA_USAGE_WORK_RETRY_ERROR: "MEDIA_USAGE_WORK_RETRY_ERROR", + MEDIA_USAGE_COLLECTION_DELETION_LIST_ERROR: "MEDIA_USAGE_COLLECTION_DELETION_LIST_ERROR", + MEDIA_USAGE_COLLECTION_DELETION_RETRY_ERROR: "MEDIA_USAGE_COLLECTION_DELETION_RETRY_ERROR", WORK_LEASE_ACTIVE: "WORK_LEASE_ACTIVE", WORK_CHANGED: "WORK_CHANGED", NO_STORAGE: "NO_STORAGE", diff --git a/packages/core/src/api/handlers/media-usage-work.ts b/packages/core/src/api/handlers/media-usage-work.ts index 95381c8b8e..2498f344da 100644 --- a/packages/core/src/api/handlers/media-usage-work.ts +++ b/packages/core/src/api/handlers/media-usage-work.ts @@ -3,12 +3,17 @@ import type { Kysely } from "kysely"; import { MediaUsageWorkRepository } from "../../database/repositories/media-usage-work.js"; import { InvalidCursorError } from "../../database/repositories/types.js"; import type { Database } from "../../database/types.js"; +import { MediaUsageCollectionDeletionRepository } from "../../media/usage/collection-deletion.js"; import { ErrorCode } from "../errors.js"; import type { MediaUsageWorkListQuery, MediaUsageWorkListResponse, MediaUsageWorkRetryRequest, MediaUsageWorkRetryResponse, + MediaUsageCollectionDeletionListQuery, + MediaUsageCollectionDeletionListResponse, + MediaUsageCollectionDeletionRetryRequest, + MediaUsageCollectionDeletionRetryResponse, } from "../schemas/media-usage.js"; import type { ApiResult } from "../types.js"; @@ -62,6 +67,76 @@ export async function handleMediaUsageWorkList( } } +export async function handleMediaUsageCollectionDeletionList( + db: Kysely, + query: MediaUsageCollectionDeletionListQuery, +): Promise> { + try { + return { + success: true, + data: await new MediaUsageCollectionDeletionRepository(db).findOperatorPage(query), + }; + } catch (error) { + if (error instanceof InvalidCursorError) { + return { + success: false, + error: { code: ErrorCode.INVALID_CURSOR, message: "Invalid cursor" }, + }; + } + console.error("[media-usage:collection-deletion] list failed:", error); + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_COLLECTION_DELETION_LIST_ERROR, + message: "Failed to list collection deletions", + }, + }; + } +} + +export async function handleMediaUsageCollectionDeletionRetry( + db: Kysely, + input: MediaUsageCollectionDeletionRetryRequest, +): Promise> { + try { + const result = await new MediaUsageCollectionDeletionRepository(db).retryOperatorDeletion( + input, + ); + if (result.outcome === "pending") { + return { success: true, data: { changed: result.changed, item: result.item } }; + } + if (result.outcome === "lease_active") { + return { + success: false, + error: { + code: ErrorCode.WORK_LEASE_ACTIVE, + message: "Collection deletion is currently leased", + details: { leaseExpiresAt: result.leaseExpiresAt }, + }, + }; + } + return { + success: false, + error: { + code: result.outcome === "not_found" ? ErrorCode.NOT_FOUND : ErrorCode.WORK_CHANGED, + message: + result.outcome === "not_found" + ? "Collection deletion not found" + : "Collection deletion changed", + }, + }; + } catch (error) { + console.error("[media-usage:collection-deletion] retry failed:", error); + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_COLLECTION_DELETION_RETRY_ERROR, + message: "Failed to retry collection deletion", + }, + }; + } +} + export async function handleMediaUsageWorkRetry( db: Kysely, input: MediaUsageWorkRetryRequest, diff --git a/packages/core/src/api/openapi/document.ts b/packages/core/src/api/openapi/document.ts index 8fe7221c8a..73530181fb 100644 --- a/packages/core/src/api/openapi/document.ts +++ b/packages/core/src/api/openapi/document.ts @@ -40,6 +40,10 @@ import { import { mediaUsageDetailsQuery, mediaUsageDetailsResponseSchema, + mediaUsageCollectionDeletionListQuery, + mediaUsageCollectionDeletionListResponseSchema, + mediaUsageCollectionDeletionRetryBody, + mediaUsageCollectionDeletionRetryResponseSchema, mediaUsageRepairBody, mediaUsageRepairResponseSchema, mediaUsageWorkListQuery, @@ -842,6 +846,55 @@ function buildMediaPaths(maxUploadSize: number) { }, }, }, + "/_emdash/api/admin/media-usage/collection-deletions": { + get: { + operationId: "listMediaUsageCollectionDeletions", + summary: "List durable collection deletions", + description: + "Returns a bounded, redacted cursor page of collection deletion work. Requires `schema:manage`; bearer tokens also require the `admin` scope.", + tags: ["Media"], + requestParams: { query: mediaUsageCollectionDeletionListQuery }, + responses: { + "200": { + description: "Bounded collection deletion page", + content: { + [JSON_CONTENT]: { + schema: successEnvelope(mediaUsageCollectionDeletionListResponseSchema), + }, + }, + }, + ...authErrors, + ...standardErrors(400, 500), + }, + }, + }, + "/_emdash/api/admin/media-usage/collection-deletions/retry": { + post: { + operationId: "retryMediaUsageCollectionDeletion", + summary: "Retry one collection deletion", + tags: ["Media"], + requestBody: { + required: true, + content: { [JSON_CONTENT]: { schema: mediaUsageCollectionDeletionRetryBody } }, + }, + responses: { + "200": { + description: "Pending collection deletion state", + content: { + [JSON_CONTENT]: { + schema: successEnvelope(mediaUsageCollectionDeletionRetryResponseSchema), + }, + }, + }, + ...authErrors, + ...standardErrors(400, 404, 500), + "409": { + description: "The deletion has a live lease or changed concurrently", + content: { [JSON_CONTENT]: { schema: mediaUsageWorkRetryConflictSchema } }, + }, + }, + }, + }, "/_emdash/api/media/upload-url": { post: { operationId: "getMediaUploadUrl", diff --git a/packages/core/src/api/schemas/media-usage.ts b/packages/core/src/api/schemas/media-usage.ts index cfec6256a8..51d0caae96 100644 --- a/packages/core/src/api/schemas/media-usage.ts +++ b/packages/core/src/api/schemas/media-usage.ts @@ -182,6 +182,49 @@ export const mediaUsageWorkRetryConflictSchema = z.object({ ]), }); +export const mediaUsageCollectionDeletionStateSchema = z.enum([ + "pending", + "retry", + "leased", + "failed", +]); +export const mediaUsageCollectionDeletionPhaseSchema = z.enum([ + "fence", + "registry", + "table", + "work", + "sources", + "status", + "finalize", +]); +export const mediaUsageCollectionDeletionListQuery = z.object({ + state: mediaUsageCollectionDeletionStateSchema.optional().default("failed"), + cursor: z.string().min(1).max(2048).optional(), + limit: z.coerce.number().int().min(1).max(100).optional().default(50), +}); +export const mediaUsageCollectionDeletionItemSchema = z.object({ + collectionId: z.string(), + collectionSlug: z.string(), + state: mediaUsageCollectionDeletionStateSchema, + phase: mediaUsageCollectionDeletionPhaseSchema, + attemptCount: z.number().int().min(0), + nextAttemptAt: z.string(), + leaseExpiresAt: z.string().nullable(), + lastErrorCode: z.string().nullable(), + updatedAt: z.string(), +}); +export const mediaUsageCollectionDeletionListResponseSchema = z.object({ + items: z.array(mediaUsageCollectionDeletionItemSchema), + nextCursor: z.string().optional(), +}); +export const mediaUsageCollectionDeletionRetryBody = z + .object({ collectionId: boundedOpaqueMediaUsageId }) + .strict(); +export const mediaUsageCollectionDeletionRetryResponseSchema = z.object({ + changed: z.boolean(), + item: mediaUsageCollectionDeletionItemSchema, +}); + export type MediaUsageRepairRequest = z.infer; export type MediaUsageRepairResponse = z.infer; export type MediaUsageWorkListQuery = z.infer; @@ -189,6 +232,18 @@ export type MediaUsageWorkItem = z.infer; export type MediaUsageWorkListResponse = z.infer; export type MediaUsageWorkRetryRequest = z.infer; export type MediaUsageWorkRetryResponse = z.infer; +export type MediaUsageCollectionDeletionListQuery = z.infer< + typeof mediaUsageCollectionDeletionListQuery +>; +export type MediaUsageCollectionDeletionListResponse = z.infer< + typeof mediaUsageCollectionDeletionListResponseSchema +>; +export type MediaUsageCollectionDeletionRetryRequest = z.infer< + typeof mediaUsageCollectionDeletionRetryBody +>; +export type MediaUsageCollectionDeletionRetryResponse = z.infer< + typeof mediaUsageCollectionDeletionRetryResponseSchema +>; export type MediaUsageCoverageStatus = z.infer; export type MediaUsageCoverage = z.infer; export type MediaUsageSummary = z.infer; diff --git a/packages/core/src/astro/integration/routes.ts b/packages/core/src/astro/integration/routes.ts index 8364c2fd47..f1b70a6f93 100644 --- a/packages/core/src/astro/integration/routes.ts +++ b/packages/core/src/astro/integration/routes.ts @@ -258,6 +258,14 @@ export function injectCoreRoutes( pattern: "/_emdash/api/admin/media-usage/work/retry", entrypoint: resolveRoute("api/admin/media-usage/work/retry.ts"), }); + injectRoute({ + pattern: "/_emdash/api/admin/media-usage/collection-deletions", + entrypoint: resolveRoute("api/admin/media-usage/collection-deletions/index.ts"), + }); + injectRoute({ + pattern: "/_emdash/api/admin/media-usage/collection-deletions/retry", + entrypoint: resolveRoute("api/admin/media-usage/collection-deletions/retry.ts"), + }); // Import API routes injectRoute({ diff --git a/packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/index.ts b/packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/index.ts new file mode 100644 index 0000000000..75369e89ac --- /dev/null +++ b/packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/index.ts @@ -0,0 +1,22 @@ +import type { APIRoute } from "astro"; + +import { requirePerm } from "#api/authorize.js"; +import { requireDb, unwrapResult } from "#api/error.js"; +import { handleMediaUsageCollectionDeletionList } from "#api/handlers/media-usage-work.js"; +import { isParseError, parseQuery } from "#api/parse.js"; +import { mediaUsageCollectionDeletionListQuery } from "#api/schemas.js"; +import { requireScope } from "#auth/scopes.js"; + +export const prerender = false; + +export const GET: APIRoute = async ({ request, locals }) => { + const dbErr = requireDb(locals.emdash?.db); + if (dbErr) return dbErr; + const denied = requirePerm(locals.user, "schema:manage"); + if (denied) return denied; + const scopeDenied = requireScope(locals, "admin"); + if (scopeDenied) return scopeDenied; + const query = parseQuery(new URL(request.url), mediaUsageCollectionDeletionListQuery); + if (isParseError(query)) return query; + return unwrapResult(await handleMediaUsageCollectionDeletionList(locals.emdash.db, query)); +}; diff --git a/packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/retry.ts b/packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/retry.ts new file mode 100644 index 0000000000..a6dd1dd319 --- /dev/null +++ b/packages/core/src/astro/routes/api/admin/media-usage/collection-deletions/retry.ts @@ -0,0 +1,22 @@ +import type { APIRoute } from "astro"; + +import { requirePerm } from "#api/authorize.js"; +import { requireDb, unwrapResult } from "#api/error.js"; +import { handleMediaUsageCollectionDeletionRetry } from "#api/handlers/media-usage-work.js"; +import { isParseError, parseBody } from "#api/parse.js"; +import { mediaUsageCollectionDeletionRetryBody } from "#api/schemas.js"; +import { requireScope } from "#auth/scopes.js"; + +export const prerender = false; + +export const POST: APIRoute = async ({ request, locals }) => { + const dbErr = requireDb(locals.emdash?.db); + if (dbErr) return dbErr; + const denied = requirePerm(locals.user, "schema:manage"); + if (denied) return denied; + const scopeDenied = requireScope(locals, "admin"); + if (scopeDenied) return scopeDenied; + const body = await parseBody(request, mediaUsageCollectionDeletionRetryBody); + if (isParseError(body)) return body; + return unwrapResult(await handleMediaUsageCollectionDeletionRetry(locals.emdash.db, body)); +}; diff --git a/packages/core/src/client/index.ts b/packages/core/src/client/index.ts index c226413c53..298ada864c 100644 --- a/packages/core/src/client/index.ts +++ b/packages/core/src/client/index.ts @@ -271,6 +271,31 @@ export interface MediaUsageWorkRetryResponse { item: MediaUsageWorkItem; } +export type MediaUsageCollectionDeletionState = "pending" | "retry" | "leased" | "failed"; +export type MediaUsageCollectionDeletionPhase = + | "fence" + | "registry" + | "table" + | "work" + | "sources" + | "status" + | "finalize"; +export interface MediaUsageCollectionDeletionItem { + collectionId: string; + collectionSlug: string; + state: MediaUsageCollectionDeletionState; + phase: MediaUsageCollectionDeletionPhase; + attemptCount: number; + nextAttemptAt: string; + leaseExpiresAt: string | null; + lastErrorCode: string | null; + updatedAt: string; +} +export interface MediaUsageCollectionDeletionListResponse { + items: MediaUsageCollectionDeletionItem[]; + nextCursor?: string; +} + /** Search result */ export interface SearchResult { id: string; @@ -881,6 +906,29 @@ export class EmDashClient { ); } + async mediaListCollectionDeletions( + options: { + state?: MediaUsageCollectionDeletionState; + limit?: number; + cursor?: string; + } = {}, + ): Promise { + const params = new URLSearchParams(); + if (options.state) params.set("state", options.state); + if (options.limit !== undefined) params.set("limit", String(options.limit)); + if (options.cursor) params.set("cursor", options.cursor); + return this.request("GET", `/admin/media-usage/collection-deletions?${params}`); + } + + async mediaRetryCollectionDeletion(collectionId: string): Promise<{ + changed: boolean; + item: MediaUsageCollectionDeletionItem; + }> { + return this.request("POST", "/admin/media-usage/collection-deletions/retry", { + collectionId, + }); + } + // ----------------------------------------------------------------------- // Search // ----------------------------------------------------------------------- diff --git a/packages/core/src/media/usage/collection-deletion.ts b/packages/core/src/media/usage/collection-deletion.ts index 39e6b00528..e0d3e71910 100644 --- a/packages/core/src/media/usage/collection-deletion.ts +++ b/packages/core/src/media/usage/collection-deletion.ts @@ -2,6 +2,11 @@ import { sql, type Kysely, type RawBuilder, type Selectable, type Transaction } import { ulid } from "ulidx"; import { isPostgres } from "../../database/dialect-helpers.js"; +import { + decodeCursor, + encodeCursor, + type FindManyResult, +} from "../../database/repositories/types.js"; import { withTransaction } from "../../database/transaction.js"; import type { Database, MediaUsageCollectionDeletionTable } from "../../database/types.js"; import { validateIdentifier } from "../../database/validate.js"; @@ -10,6 +15,7 @@ import type { CollectionDeletionGuardResult, } from "../../db/adapters.js"; import { FTSManager } from "../../search/fts-manager.js"; +import { isMissingTableError } from "../../utils/db-errors.js"; import { verifyMediaUsageCaptureTriggers } from "./capture-triggers.js"; const ACTIVATION_KEY = "incremental_capture"; @@ -43,6 +49,24 @@ export interface MediaUsageCollectionDeletionRecord { updatedAt: string; } +export interface MediaUsageCollectionDeletionOperatorItem { + collectionId: string; + collectionSlug: string; + state: MediaUsageCollectionDeletionState; + phase: MediaUsageCollectionDeletionPhase; + attemptCount: number; + nextAttemptAt: string; + leaseExpiresAt: string | null; + lastErrorCode: string | null; + updatedAt: string; +} + +export type MediaUsageCollectionDeletionRetryResult = + | { outcome: "pending"; changed: boolean; item: MediaUsageCollectionDeletionOperatorItem } + | { outcome: "lease_active"; leaseExpiresAt: string } + | { outcome: "not_found" } + | { outcome: "conflict" }; + export class MediaUsageCollectionDeletionRepository { constructor(private db: Kysely) {} @@ -120,12 +144,17 @@ export class MediaUsageCollectionDeletionRepository { async findBySlug(collectionSlug: string): Promise { validateIdentifier(collectionSlug, "collection slug"); - const row = await this.db - .selectFrom("_emdash_media_usage_collection_deletions") - .selectAll() - .where("collection_slug", "=", collectionSlug) - .executeTakeFirst(); - return row ? rowToRecord(row) : null; + try { + const row = await this.db + .selectFrom("_emdash_media_usage_collection_deletions") + .selectAll() + .where("collection_slug", "=", collectionSlug) + .executeTakeFirst(); + return row ? rowToRecord(row) : null; + } catch (error) { + if (isMissingTableError(error)) return null; + throw error; + } } async findDue(limit: number): Promise { @@ -164,6 +193,93 @@ export class MediaUsageCollectionDeletionRepository { return result.rows.map(rowToRecord); } + async findOperatorPage(options: { + state?: MediaUsageCollectionDeletionState; + limit?: number; + cursor?: string; + }): Promise> { + const state = options.state ?? "failed"; + if (!isState(state)) throw new Error("Invalid collection deletion state"); + const limit = options.limit ?? 50; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) { + throw new Error("Collection deletion operator limit must be from 1 to 100"); + } + const cursor = options.cursor ? decodeCursor(options.cursor) : null; + let query = this.db + .selectFrom("_emdash_media_usage_collection_deletions") + .selectAll() + .where("state", "=", state); + if (cursor) { + query = query.where((eb) => + eb.or([ + eb("updated_at", "<", cursor.orderValue), + eb.and([eb("updated_at", "=", cursor.orderValue), eb("collection_id", "<", cursor.id)]), + ]), + ); + } + const rows = await query + .orderBy("updated_at", "desc") + .orderBy("collection_id", "desc") + .limit(limit + 1) + .execute(); + const items = rows.slice(0, limit).map(rowToOperatorItem); + const result: FindManyResult = { items }; + if (rows.length > limit && items.length > 0) { + const last = items.at(-1)!; + result.nextCursor = encodeCursor(last.updatedAt, last.collectionId); + } + return result; + } + + async retryOperatorDeletion(input: { + collectionId: string; + }): Promise { + if (!input.collectionId) throw new Error("Collection deletion retry requires an ID"); + const observed = await this.db + .selectFrom("_emdash_media_usage_collection_deletions") + .selectAll() + .where("collection_id", "=", input.collectionId) + .executeTakeFirst(); + if (!observed) return { outcome: "not_found" }; + if (observed.state === "leased" && observed.lease_expires_at) { + const live = await this.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("lease_expires_at") + .where("collection_id", "=", input.collectionId) + .where("state", "=", "leased") + .where(liveLease(this.db)) + .executeTakeFirst(); + if (live?.lease_expires_at) { + return { outcome: "lease_active", leaseExpiresAt: live.lease_expires_at }; + } + } + const now = timestampOffset(this.db, 0); + const reopened = await this.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ + state: "pending", + attempt_count: 0, + next_attempt_at: now, + lease_token: null, + lease_expires_at: null, + last_error_code: null, + updated_at: now, + }) + .where("collection_id", "=", input.collectionId) + .where((eb) => + eb.or([ + eb("state", "in", ["failed", "retry"]), + eb.and([eb("state", "=", "leased"), timestampIsDue(this.db, "lease_expires_at")]), + ]), + ) + .returningAll() + .executeTakeFirst(); + if (reopened) { + return { outcome: "pending", changed: true, item: rowToOperatorItem(reopened) }; + } + return { outcome: "conflict" }; + } + async checkpoint(input: { collectionId: string; leaseToken: string; @@ -446,12 +562,17 @@ export async function isMediaUsageCollectionSlugDeleting( collectionSlug: string, ): Promise { validateIdentifier(collectionSlug, "collection slug"); - const row = await db - .selectFrom("_emdash_media_usage_collection_deletions") - .select("collection_id") - .where("collection_slug", "=", collectionSlug) - .executeTakeFirst(); - return row !== undefined; + try { + const row = await db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .where("collection_slug", "=", collectionSlug) + .executeTakeFirst(); + return row !== undefined; + } catch (error) { + if (isMissingTableError(error)) return false; + throw error; + } } async function assertActivatedCollectionDeletionReady( @@ -670,6 +791,23 @@ function rowToRecord( }; } +function rowToOperatorItem( + row: Selectable, +): MediaUsageCollectionDeletionOperatorItem { + const deletion = rowToRecord(row); + return { + collectionId: deletion.collectionId, + collectionSlug: deletion.collectionSlug, + state: deletion.state, + phase: deletion.phase, + attemptCount: deletion.attemptCount, + nextAttemptAt: deletion.nextAttemptAt, + leaseExpiresAt: deletion.leaseExpiresAt, + lastErrorCode: deletion.lastErrorCode, + updatedAt: deletion.updatedAt, + }; +} + function isState(value: string): value is MediaUsageCollectionDeletionState { return value === "pending" || value === "retry" || value === "leased" || value === "failed"; } diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-operator.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-operator.test.ts new file mode 100644 index 0000000000..01e8d3e6bf --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-operator.test.ts @@ -0,0 +1,85 @@ +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { MediaUsageCollectionDeletionRepository } from "../../../src/media/usage/collection-deletion.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage collection deletion operator repository", (dialect) => { + let ctx: DialectTestContext; + let repository: MediaUsageCollectionDeletionRepository; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + repository = new MediaUsageCollectionDeletionRepository(ctx.db); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("returns a bounded redacted failed page with an opaque cursor", async () => { + await insert("failed-a", "failed_a", "failed", "lease-a"); + await insert("failed-b", "failed_b", "failed", "lease-b"); + + const first = await repository.findOperatorPage({ state: "failed", limit: 1 }); + expect(first.items).toHaveLength(1); + expect(first.nextCursor).toBeDefined(); + expect(first.items[0]).not.toHaveProperty("leaseToken"); + const second = await repository.findOperatorPage({ + state: "failed", + limit: 1, + cursor: first.nextCursor, + }); + expect(second.items).toHaveLength(1); + expect(second.items[0]?.collectionId).not.toBe(first.items[0]?.collectionId); + }); + + it("reopens failed, retry, and expired exact work but preserves a live lease", async () => { + await insert("failed-id", "failed", "failed", null); + await insert("retry-id", "retry", "retry", null); + await insert("expired-id", "expired", "leased", "expired-token", "2000-01-01T00:00:00.000Z"); + await insert("live-id", "live", "leased", "private-live-token", "2999-01-01T00:00:00.000Z"); + + for (const collectionId of ["failed-id", "retry-id", "expired-id"]) { + const result = await repository.retryOperatorDeletion({ collectionId }); + expect(result).toEqual( + expect.objectContaining({ outcome: "pending", changed: true, item: expect.any(Object) }), + ); + expect(result).not.toHaveProperty("leaseToken"); + } + expect(await repository.retryOperatorDeletion({ collectionId: "live-id" })).toEqual({ + outcome: "lease_active", + leaseExpiresAt: "2999-01-01T00:00:00.000Z", + }); + expect(await repository.retryOperatorDeletion({ collectionId: "missing" })).toEqual({ + outcome: "not_found", + }); + }); + + async function insert( + collectionId: string, + slug: string, + state: "failed" | "retry" | "leased", + leaseToken: string | null, + leaseExpiresAt: string | null = null, + ) { + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: collectionId, + collection_slug: slug, + force_delete: 1, + state, + phase: "work", + next_attempt_at: "2000-01-01T00:00:00.000Z", + lease_token: leaseToken, + lease_expires_at: leaseExpiresAt, + last_error_code: "MEDIA_USAGE_COLLECTION_DELETION_FAILED", + }) + .execute(); + } +}); diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts index 082dc6bcb3..b0d65098ac 100644 --- a/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts @@ -217,6 +217,46 @@ describeEachDialect("media usage collection deletion processor", (dialect) => { expect(await deletionState("second-id")).toEqual(expect.objectContaining({ phase: "sources" })); }); + it.runIf(dialect === "sqlite")("uses every declared deletion selector index", async () => { + const plans = await Promise.all([ + sql + .raw( + "EXPLAIN QUERY PLAN SELECT collection_id FROM _emdash_media_usage_collection_deletions WHERE state = 'pending' AND next_attempt_at <= '2026-01-01T00:00:00.000Z' ORDER BY next_attempt_at, updated_at, collection_id LIMIT 4", + ) + .execute(ctx.db), + sql + .raw( + "EXPLAIN QUERY PLAN SELECT collection_id FROM _emdash_media_usage_collection_deletions WHERE state = 'leased' AND lease_expires_at <= '2026-01-01T00:00:00.000Z' ORDER BY lease_expires_at, updated_at, collection_id LIMIT 4", + ) + .execute(ctx.db), + sql + .raw( + "EXPLAIN QUERY PLAN SELECT collection_id FROM _emdash_media_usage_collection_deletions WHERE state = 'failed' ORDER BY updated_at DESC, collection_id DESC LIMIT 50", + ) + .execute(ctx.db), + sql + .raw( + "EXPLAIN QUERY PLAN SELECT source_key FROM _emdash_media_usage_sources WHERE source_type = 'content' AND collection_id = 'old-id' ORDER BY source_key LIMIT 1", + ) + .execute(ctx.db), + sql + .raw( + "EXPLAIN QUERY PLAN SELECT id FROM _emdash_media_usage WHERE source_key = 'source' AND id > 'cursor' ORDER BY id LIMIT 51", + ) + .execute(ctx.db), + ]); + const indexes = [ + "idx__emdash_media_usage_collection_deletions_due", + "idx__emdash_media_usage_collection_deletions_lease", + "idx__emdash_media_usage_collection_deletions_operator", + "idx__emdash_media_usage_sources_collection_cursor", + "idx__emdash_media_usage_source_cursor", + ]; + for (const [index, plan] of plans.entries()) { + expect(plan.rows.map((row) => JSON.stringify(row)).join(" ")).toContain(indexes[index]); + } + }); + function runTick() { return processDueMediaUsageCollectionDeletions(ctx.db); } diff --git a/packages/core/tests/unit/api/media-usage-collection-deletion-route.test.ts b/packages/core/tests/unit/api/media-usage-collection-deletion-route.test.ts new file mode 100644 index 0000000000..00957c4547 --- /dev/null +++ b/packages/core/tests/unit/api/media-usage-collection-deletion-route.test.ts @@ -0,0 +1,55 @@ +import { Role, type RoleLevel } from "@emdash-cms/auth"; +import { expect, it } from "vitest"; + +import { GET } from "../../../src/astro/routes/api/admin/media-usage/collection-deletions/index.js"; +import { POST } from "../../../src/astro/routes/api/admin/media-usage/collection-deletions/retry.js"; +import { setupTestDatabase, teardownTestDatabase } from "../../utils/test-db.js"; + +it("requires schema permission and admin scope for collection deletion recovery", async () => { + const db = await setupTestDatabase(); + try { + const list = new Request( + "http://localhost/_emdash/api/admin/media-usage/collection-deletions?state=failed", + ); + await expectError(await GET(context(db, list, Role.EDITOR, ["admin"])), 403, "FORBIDDEN"); + await expectError( + await GET(context(db, list, Role.ADMIN, ["content:read"])), + 403, + "INSUFFICIENT_SCOPE", + ); + const retry = new Request( + "http://localhost/_emdash/api/admin/media-usage/collection-deletions/retry", + { + method: "POST", + headers: { "Content-Type": "application/json", "X-EmDash-Request": "1" }, + body: JSON.stringify({ collectionId: "missing" }), + }, + ); + await expectError(await POST(context(db, retry, Role.EDITOR, ["admin"])), 403, "FORBIDDEN"); + } finally { + await teardownTestDatabase(db); + } +}); + +function context( + db: Awaited>, + request: Request, + role: RoleLevel, + scopes: string[], +) { + return { + request, + locals: { + emdash: { db }, + user: { id: "user-1", email: "admin@example.com", name: "Admin", role }, + tokenScopes: scopes, + }, + } as never; +} + +async function expectError(response: Response, status: number, code: string) { + expect(response.status).toBe(status); + expect((await response.json()) as { error: { code: string } }).toMatchObject({ + error: expect.objectContaining({ code }), + }); +} diff --git a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts index 33117d7132..15ac46a9d4 100644 --- a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts @@ -182,6 +182,113 @@ it("drains at most fifty exact-ID work rows in a real D1 tick", async () => { .where("collection_id", "=", "collection-d1-cleanup") .execute(); expect(remaining).toEqual([{ content_id: "d1-entry-050" }]); + await db + .deleteFrom("_emdash_media_usage_work") + .where("collection_id", "=", "collection-d1-cleanup") + .execute(); + await db + .deleteFrom("_emdash_media_usage_collection_deletions") + .where("collection_id", "=", "collection-d1-cleanup") + .execute(); +}); + +it("records bounded real-D1 cost evidence through finalization", async () => { + await db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-d1-measure", + collection_slug: "d1_measure", + force_delete: 1, + state: "pending", + phase: "work", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + const measuredWork = Array.from({ length: 51 }, (_, index) => ({ + collection_id: "collection-d1-measure", + collection_slug: "d1_measure", + content_id: `measured-entry-${String(index).padStart(3, "0")}`, + change_epoch: 1, + next_attempt_at: "2000-01-01T00:00:00.000Z", + })); + for (let index = 0; index < measuredWork.length; index += 10) { + await db + .insertInto("_emdash_media_usage_work") + .values(measuredWork.slice(index, index + 10)) + .execute(); + } + await db + .insertInto("_emdash_media_usage_sources") + .values({ + source_key: "d1-measured-source", + source_type: "content", + collection_id: "collection-d1-measure", + collection_slug: "d1_measure", + content_id: "entry", + source_variant: "columns", + current_generation: "generation", + }) + .execute(); + const measuredOccurrences = Array.from({ length: 51 }, (_, index) => ({ + id: `d1-measured-usage-${String(index).padStart(3, "0")}`, + source_key: "d1-measured-source", + generation: "generation", + field_slug: "hero", + field_path: `hero[${index}]`, + occurrence_index: index, + reference_type: "local", + media_id: `media-${index}`, + provider_asset_id: `media-${index}`, + })); + for (let index = 0; index < measuredOccurrences.length; index += 5) { + await db + .insertInto("_emdash_media_usage") + .values(measuredOccurrences.slice(index, index + 5)) + .execute(); + } + await db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "d1_measure", + collection_id: "collection-d1-measure", + status: "stale", + capture_state: "deleting", + }) + .execute(); + + const evidence: Array> = []; + for (let tick = 0; tick < 8; tick++) { + const deletion = await db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("phase") + .where("collection_id", "=", "collection-d1-measure") + .executeTakeFirst(); + if (!deletion) break; + const measurement = emptyMeasurement(); + const measuredDb = new Kysely({ + dialect: new RawBindingD1Dialect({ database: captureD1(env.DB, measurement) }), + }); + const startedAt = performance.now(); + await processDueMediaUsageCollectionDeletions(measuredDb); + await measuredDb.destroy(); + const wallDurationMs = performance.now() - startedAt; + expect(measurement.queries).toBeLessThanOrEqual(40); + expect(measurement.maxBinds).toBeLessThanOrEqual(100); + expect(measurement.maxSqlBytes).toBeLessThan(100 * 1024); + expect(measurement.rowsWritten).toBeLessThanOrEqual(70); + expect(wallDurationMs).toBeLessThan(2500); + evidence.push({ phase: deletion.phase, ...measurement, wallDurationMs }); + } + expect( + await db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .where("collection_id", "=", "collection-d1-measure") + .executeTakeFirst(), + ).toBeUndefined(); + console.info(`PR2_D1_COLLECTION_DELETION=${JSON.stringify(evidence)}`); }); async function ctxInsertCollection(): Promise { @@ -199,3 +306,61 @@ async function captureState(): Promise { .executeTakeFirst(); return row?.capture_state ?? null; } + +interface D1Measurement { + queries: number; + rowsRead: number; + rowsWritten: number; + durationMs: number; + maxBinds: number; + maxSqlBytes: number; +} + +function emptyMeasurement(): D1Measurement { + return { queries: 0, rowsRead: 0, rowsWritten: 0, durationMs: 0, maxBinds: 0, maxSqlBytes: 0 }; +} + +function captureD1(database: D1Database, measurement: D1Measurement): D1Database { + return new Proxy(database, { + get(target, property) { + if (property === "prepare") { + return (query: string) => captureStatement(target.prepare(query), query, [], measurement); + } + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +function captureStatement( + statement: D1PreparedStatement, + query: string, + binds: unknown[], + measurement: D1Measurement, +): D1PreparedStatement { + return new Proxy(statement, { + get(target, property) { + if (property === "bind") { + return (...values: unknown[]) => + captureStatement(target.bind(...values), query, values, measurement); + } + if (property === "all") { + return async () => { + const result = await target.all(); + measurement.queries++; + measurement.rowsRead += result.meta.rows_read; + measurement.rowsWritten += result.meta.rows_written; + measurement.durationMs += result.meta.duration; + measurement.maxBinds = Math.max(measurement.maxBinds, binds.length); + measurement.maxSqlBytes = Math.max( + measurement.maxSqlBytes, + new TextEncoder().encode(query).byteLength, + ); + return result; + }; + } + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} From 2d36e13f6cf58c2d7b747f94930f2b55e4fb44f9 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:40:36 +0100 Subject: [PATCH 05/20] fix(cloudflare): harden collection deletion guards --- packages/cloudflare/src/db/d1.ts | 7 ++++- .../db/do-sql-collection-deletion.test.ts | 17 +++++++++++ ...media-usage-collection-deletion-d1.test.ts | 28 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/packages/cloudflare/src/db/d1.ts b/packages/cloudflare/src/db/d1.ts index 422650331a..2e2ad25fcd 100644 --- a/packages/cloudflare/src/db/d1.ts +++ b/packages/cloudflare/src/db/d1.ts @@ -28,7 +28,8 @@ interface D1Config { const DEFAULT_BOOKMARK_COOKIE = "__em_d1_bookmark"; const COLLECTION_SLUG_PATTERN = /^[a-z][a-z0-9_]*$/; -const STALE_DELETION_GUARD_PATTERN = /collection_id.*not null|not null.*collection_id/i; +const STALE_DELETION_GUARD_PATTERN = + /not null constraint failed:\s*_emdash_media_usage_collection_deletions\.collection_id/i; /** * One-shot guard so the "coalesce opted in but the binding can't do sessions @@ -233,6 +234,10 @@ async function executeFenceBatch( input.collectionSlug, input.collectionId, ); + if (input.forceDelete) { + const updated = await update.all<{ collection_id: string }>(); + return updated.results.length > 0 ? { outcome: "fenced" } : { outcome: "stale" }; + } const [updated, observed] = await binding.batch<{ collection_id: string } | { outcome: string }>([ update, diagnostic, diff --git a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts index 16bffedbb4..b2855e861d 100644 --- a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts +++ b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts @@ -35,6 +35,23 @@ describe("EmDashDB collection deletion guard", () => { transactionSync = vi.fn((operation: () => unknown) => operation()); }); + it("rejects interpolated identifiers before opening a transaction", async () => { + const object = new EmDashDB( + { storage: { sql: { exec: vi.fn() }, transactionSync } } as never, + {}, + ); + + await expect( + object.executeCollectionDeletionGuard({ + action: "drop", + collectionId: "collection-1", + collectionSlug: 'posts";drop_table', + leaseToken: "owner", + }), + ).rejects.toThrow(/valid collection slug/i); + expect(transactionSync).not.toHaveBeenCalled(); + }); + it("returns stale before dispatching DDL when the exact lease is absent", async () => { const sql = { exec: vi.fn((statement: string) => { diff --git a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts index 15ac46a9d4..83fb7be143 100644 --- a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts @@ -25,6 +25,20 @@ afterAll(async () => { await db.destroy(); }); +it("rejects interpolated collection identifiers before issuing D1 SQL", async () => { + await expect( + executeCollectionDeletionGuard( + { binding: "DB" }, + { + action: "drop", + collectionId: "collection-invalid", + collectionSlug: 'posts";drop_table', + leaseToken: "owner", + }, + ), + ).rejects.toThrow(/valid collection slug/i); +}); + it("rolls back a stale guarded batch before any collection DDL", async () => { await sql`CREATE TABLE ec_d1_guarded (id TEXT PRIMARY KEY)`.execute(db); await db @@ -129,6 +143,20 @@ it("atomically preserves content or fences an empty collection", async () => { ), ).resolves.toEqual({ outcome: "has_content" }); expect(await captureState()).toBe("active"); + await db.deleteFrom("_emdash_collections").where("id", "=", "collection-d1-fence").execute(); + await expect( + executeCollectionDeletionGuard( + { binding: "DB" }, + { + action: "fence", + collectionId: "collection-d1-fence", + collectionSlug: "d1_fence", + leaseToken: "fence-owner", + forceDelete: true, + }, + ), + ).resolves.toEqual({ outcome: "stale" }); + await ctxInsertCollection(); await sql`DELETE FROM ec_d1_fence`.execute(db); await expect( From 725ef86d9d6e9d25c473c3ae9a6e314744da77b3 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:43:03 +0100 Subject: [PATCH 06/20] fix(core): bound collection deletion completion checks --- .../usage/collection-deletion-processor.ts | 53 ++++++++++--------- ...sage-collection-deletion-processor.test.ts | 42 +++++++++++++++ 2 files changed, 70 insertions(+), 25 deletions(-) diff --git a/packages/core/src/media/usage/collection-deletion-processor.ts b/packages/core/src/media/usage/collection-deletion-processor.ts index 8a45696ab4..52ec277c5f 100644 --- a/packages/core/src/media/usage/collection-deletion-processor.ts +++ b/packages/core/src/media/usage/collection-deletion-processor.ts @@ -267,31 +267,34 @@ async function exactCleanupRowsRemain( claim: Pick, includeStatus = true, ): Promise { - const [work, source, status] = await Promise.all([ - db - .selectFrom("_emdash_media_usage_work") - .select("content_id") - .where("collection_id", "=", claim.collectionId) - .limit(1) - .executeTakeFirst(), - db - .selectFrom("_emdash_media_usage_sources") - .select("source_key") - .where("source_type", "=", "content") - .where("collection_id", "=", claim.collectionId) - .limit(1) - .executeTakeFirst(), - db - .selectFrom("_emdash_media_usage_index_status") - .select("collection_id") - .where("adapter_id", "=", "content-media") - .where("scope_type", "=", "collection") - .where("scope_key", "=", claim.collectionSlug) - .where("collection_id", "=", claim.collectionId) - .limit(1) - .executeTakeFirst(), - ]); - return work !== undefined || source !== undefined || (includeStatus && status !== undefined); + const result = await sql<{ + work_present: boolean | number; + source_present: boolean | number; + status_present: boolean | number; + }>` + SELECT + EXISTS ( + SELECT 1 FROM _emdash_media_usage_work + WHERE collection_id = ${claim.collectionId} + ) AS work_present, + EXISTS ( + SELECT 1 FROM _emdash_media_usage_sources + WHERE source_type = 'content' AND collection_id = ${claim.collectionId} + ) AS source_present, + EXISTS ( + SELECT 1 FROM _emdash_media_usage_index_status + WHERE adapter_id = 'content-media' + AND scope_type = 'collection' + AND scope_key = ${claim.collectionSlug} + AND collection_id = ${claim.collectionId} + ) AS status_present + `.execute(db); + const row = result.rows[0]; + return ( + Boolean(row?.work_present) || + Boolean(row?.source_present) || + (includeStatus && Boolean(row?.status_present)) + ); } async function updateDeletion( diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts index b0d65098ac..a5f6912924 100644 --- a/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts @@ -369,3 +369,45 @@ it("selects one globally bounded due-candidate window", async () => { await fixture.db.destroy(); } }); + +it("checks cleanup completion in one bounded query", async () => { + const fixture = await setupTestDatabaseWithCompoundSelectLimit(null); + try { + await fixture.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "status-id", + collection_slug: "status_slug", + force_delete: 1, + state: "pending", + phase: "status", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + await fixture.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "status_slug", + collection_id: "status-id", + status: "stale", + capture_state: "deleting", + }) + .execute(); + fixture.statements.length = 0; + + await processDueMediaUsageCollectionDeletions(fixture.db); + + const probes = fixture.statements.filter( + (statement) => + /^select/i.test(statement.trim()) && + (statement.includes("_emdash_media_usage_work") || + statement.includes("_emdash_media_usage_sources") || + statement.includes("_emdash_media_usage_index_status")), + ); + expect(probes).toHaveLength(1); + } finally { + await fixture.db.destroy(); + } +}); From 110f94591cf305e5b176509b933c66fc73003f5d Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:44:01 +0100 Subject: [PATCH 07/20] fix(core): use database time for deletion progress --- .../usage/collection-deletion-processor.ts | 3 ++- .../src/media/usage/collection-deletion.ts | 4 +++ ...sage-collection-deletion-processor.test.ts | 26 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/core/src/media/usage/collection-deletion-processor.ts b/packages/core/src/media/usage/collection-deletion-processor.ts index 52ec277c5f..313962b332 100644 --- a/packages/core/src/media/usage/collection-deletion-processor.ts +++ b/packages/core/src/media/usage/collection-deletion-processor.ts @@ -4,6 +4,7 @@ import { isPostgres, tableExists } from "../../database/dialect-helpers.js"; import { withTransaction } from "../../database/transaction.js"; import type { Database, MediaUsageCollectionDeletionTable } from "../../database/types.js"; import { + collectionDeletionCurrentTimestamp, deleteActivatedMediaUsageCollection, MediaUsageCollectionDeletionRepository, type MediaUsageCollectionDeletionRecord, @@ -308,7 +309,7 @@ async function updateDeletion( ...values, attempt_count: 0, last_error_code: null, - updated_at: new Date().toISOString(), + updated_at: collectionDeletionCurrentTimestamp(db), }) .where("collection_id", "=", claim.collectionId) .where("state", "=", "leased") diff --git a/packages/core/src/media/usage/collection-deletion.ts b/packages/core/src/media/usage/collection-deletion.ts index e0d3e71910..98745a5b50 100644 --- a/packages/core/src/media/usage/collection-deletion.ts +++ b/packages/core/src/media/usage/collection-deletion.ts @@ -766,6 +766,10 @@ function timestampOffset(db: Kysely, offsetSeconds: number): RawBuilde )`; } +export function collectionDeletionCurrentTimestamp(db: Kysely): RawBuilder { + return timestampOffset(db, 0); +} + function rowToRecord( row: Selectable, ): MediaUsageCollectionDeletionRecord { diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts index a5f6912924..d528386e44 100644 --- a/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-processor.test.ts @@ -257,6 +257,32 @@ describeEachDialect("media usage collection deletion processor", (dialect) => { } }); + it.runIf(dialect === "sqlite")( + "uses database time when a progress handoff is interrupted", + async () => { + await insertDeletion("clock-id", "clock", "work"); + await sql + .raw(` + CREATE TRIGGER interrupt_collection_deletion_release + BEFORE UPDATE OF state ON _emdash_media_usage_collection_deletions + WHEN OLD.collection_id = 'clock-id' AND NEW.state = 'pending' + BEGIN + SELECT RAISE(IGNORE); + END + `) + .execute(ctx.db); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2099-01-01T00:00:00.000Z")); + try { + await runTick(); + } finally { + vi.useRealTimers(); + } + + expect((await deletionState("clock-id"))?.updated_at.startsWith("2099-")).toBe(false); + }, + ); + function runTick() { return processDueMediaUsageCollectionDeletions(ctx.db); } From 1d81a961a22ba45aa3d49851d581c6d0ce273ae5 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:44:37 +0100 Subject: [PATCH 08/20] chore(core): register collection deletion schemas --- packages/core/src/api/schemas/media-usage.ts | 80 ++++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/core/src/api/schemas/media-usage.ts b/packages/core/src/api/schemas/media-usage.ts index 51d0caae96..c2b7358d98 100644 --- a/packages/core/src/api/schemas/media-usage.ts +++ b/packages/core/src/api/schemas/media-usage.ts @@ -182,48 +182,48 @@ export const mediaUsageWorkRetryConflictSchema = z.object({ ]), }); -export const mediaUsageCollectionDeletionStateSchema = z.enum([ - "pending", - "retry", - "leased", - "failed", -]); -export const mediaUsageCollectionDeletionPhaseSchema = z.enum([ - "fence", - "registry", - "table", - "work", - "sources", - "status", - "finalize", -]); -export const mediaUsageCollectionDeletionListQuery = z.object({ - state: mediaUsageCollectionDeletionStateSchema.optional().default("failed"), - cursor: z.string().min(1).max(2048).optional(), - limit: z.coerce.number().int().min(1).max(100).optional().default(50), -}); -export const mediaUsageCollectionDeletionItemSchema = z.object({ - collectionId: z.string(), - collectionSlug: z.string(), - state: mediaUsageCollectionDeletionStateSchema, - phase: mediaUsageCollectionDeletionPhaseSchema, - attemptCount: z.number().int().min(0), - nextAttemptAt: z.string(), - leaseExpiresAt: z.string().nullable(), - lastErrorCode: z.string().nullable(), - updatedAt: z.string(), -}); -export const mediaUsageCollectionDeletionListResponseSchema = z.object({ - items: z.array(mediaUsageCollectionDeletionItemSchema), - nextCursor: z.string().optional(), -}); +export const mediaUsageCollectionDeletionStateSchema = z + .enum(["pending", "retry", "leased", "failed"]) + .meta({ id: "MediaUsageCollectionDeletionState" }); +export const mediaUsageCollectionDeletionPhaseSchema = z + .enum(["fence", "registry", "table", "work", "sources", "status", "finalize"]) + .meta({ id: "MediaUsageCollectionDeletionPhase" }); +export const mediaUsageCollectionDeletionListQuery = z + .object({ + state: mediaUsageCollectionDeletionStateSchema.optional().default("failed"), + cursor: z.string().min(1).max(2048).optional(), + limit: z.coerce.number().int().min(1).max(100).optional().default(50), + }) + .meta({ id: "MediaUsageCollectionDeletionListQuery" }); +export const mediaUsageCollectionDeletionItemSchema = z + .object({ + collectionId: z.string(), + collectionSlug: z.string(), + state: mediaUsageCollectionDeletionStateSchema, + phase: mediaUsageCollectionDeletionPhaseSchema, + attemptCount: z.number().int().min(0), + nextAttemptAt: z.string(), + leaseExpiresAt: z.string().nullable(), + lastErrorCode: z.string().nullable(), + updatedAt: z.string(), + }) + .meta({ id: "MediaUsageCollectionDeletionItem" }); +export const mediaUsageCollectionDeletionListResponseSchema = z + .object({ + items: z.array(mediaUsageCollectionDeletionItemSchema), + nextCursor: z.string().optional(), + }) + .meta({ id: "MediaUsageCollectionDeletionListResponse" }); export const mediaUsageCollectionDeletionRetryBody = z .object({ collectionId: boundedOpaqueMediaUsageId }) - .strict(); -export const mediaUsageCollectionDeletionRetryResponseSchema = z.object({ - changed: z.boolean(), - item: mediaUsageCollectionDeletionItemSchema, -}); + .strict() + .meta({ id: "MediaUsageCollectionDeletionRetryBody" }); +export const mediaUsageCollectionDeletionRetryResponseSchema = z + .object({ + changed: z.boolean(), + item: mediaUsageCollectionDeletionItemSchema, + }) + .meta({ id: "MediaUsageCollectionDeletionRetryResponse" }); export type MediaUsageRepairRequest = z.infer; export type MediaUsageRepairResponse = z.infer; From 808d03e5b7d5e351136e7fd4e06006b9c054484d Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:50:13 +0100 Subject: [PATCH 09/20] chore(core): keep deletion lease guards transaction-scoped --- .../core/src/media/usage/collection-deletion-processor.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/src/media/usage/collection-deletion-processor.ts b/packages/core/src/media/usage/collection-deletion-processor.ts index 313962b332..99749ee5ce 100644 --- a/packages/core/src/media/usage/collection-deletion-processor.ts +++ b/packages/core/src/media/usage/collection-deletion-processor.ts @@ -130,7 +130,7 @@ async function processWorkBatch( "in", batch.map((row) => row.content_id), ) - .where(liveLeaseGuard(db, claim)) + .where(liveLeaseGuard(trx, claim)) .execute(); } await updateDeletion(trx, claim, { @@ -191,7 +191,7 @@ async function processSourceBatch( "in", batch.map((row) => row.id), ) - .where(liveLeaseGuard(db, claim)) + .where(liveLeaseGuard(trx, claim)) .execute(); } if (occurrences.length > MEDIA_USAGE_COLLECTION_DELETION_LIMITS.rowsPerBatch) { @@ -206,7 +206,7 @@ async function processSourceBatch( .where("source_key", "=", sourceKey) .where("source_type", "=", "content") .where("collection_id", "=", claim.collectionId) - .where(liveLeaseGuard(db, claim)) + .where(liveLeaseGuard(trx, claim)) .execute(); await updateDeletion(trx, claim, { source_key: null, occurrence_cursor: null }); }); @@ -227,7 +227,7 @@ async function processStatus( .where("scope_type", "=", "collection") .where("scope_key", "=", claim.collectionSlug) .where("collection_id", "=", claim.collectionId) - .where(liveLeaseGuard(db, claim)) + .where(liveLeaseGuard(trx, claim)) .execute(); await updateDeletion(trx, claim, { phase: "finalize" }); }); From 26e4b26785eb26b7803b4303690f57a4a4d7db0d Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:42:12 +0100 Subject: [PATCH 10/20] fix(core): preserve collection deletion compatibility --- packages/cloudflare/src/db/d1.ts | 4 ++-- packages/cloudflare/src/db/do-sql-class.ts | 2 +- .../db/do-sql-collection-deletion.test.ts | 9 +++++---- .../src/media/usage/collection-deletion.ts | 19 +++++++++++++------ ...sage-collection-deletion-lifecycle.test.ts | 13 +++++++++++++ ...a-usage-incremental-work-migration.test.ts | 15 +++++++++++++++ ...media-usage-collection-deletion-d1.test.ts | 6 +++--- 7 files changed, 52 insertions(+), 16 deletions(-) diff --git a/packages/cloudflare/src/db/d1.ts b/packages/cloudflare/src/db/d1.ts index 2e2ad25fcd..de816cda40 100644 --- a/packages/cloudflare/src/db/d1.ts +++ b/packages/cloudflare/src/db/d1.ts @@ -166,7 +166,7 @@ async function executeFenceBatch( const tableName = `ec_${input.collectionSlug}`; const contentPredicate = input.forceDelete ? "" - : `AND NOT EXISTS (SELECT 1 FROM "${tableName}" LIMIT 1)`; + : `AND NOT EXISTS (SELECT 1 FROM "${tableName}" WHERE deleted_at IS NULL LIMIT 1)`; const update = binding .prepare(` UPDATE _emdash_media_usage_index_status @@ -222,7 +222,7 @@ async function executeFenceBatch( AND collection_id = ? AND capture_state = 'active' ) - AND EXISTS (SELECT 1 FROM "${tableName}" LIMIT 1) + AND EXISTS (SELECT 1 FROM "${tableName}" WHERE deleted_at IS NULL LIMIT 1) THEN 'has_content' ELSE 'stale' END AS outcome diff --git a/packages/cloudflare/src/db/do-sql-class.ts b/packages/cloudflare/src/db/do-sql-class.ts index 3078fb496d..67eba6b15d 100644 --- a/packages/cloudflare/src/db/do-sql-class.ts +++ b/packages/cloudflare/src/db/do-sql-class.ts @@ -217,7 +217,7 @@ export class EmDashDB extends DurableObject { if (input.action === "fence") { if (!input.forceDelete) { const content = this.ctx.storage.sql.exec( - `SELECT 1 AS present FROM "${contentTable}" LIMIT 1`, + `SELECT 1 AS present FROM "${contentTable}" WHERE deleted_at IS NULL LIMIT 1`, ); if (content.toArray().length > 0) return { outcome: "has_content" }; } diff --git a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts index b2855e861d..3077a0d269 100644 --- a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts +++ b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts @@ -101,8 +101,8 @@ describe("EmDashDB collection deletion guard", () => { expect(statements.filter((statement) => statement.includes("DROP TABLE"))).toHaveLength(2); }); - it("preserves non-forced content and fences the collection once empty", async () => { - let contentPresent = true; + it("preserves active content and fences the collection once content is trashed", async () => { + let contentState: "active" | "trashed" = "active"; const sql = { exec: vi.fn((statement: string) => { statements.push(statement); @@ -110,7 +110,8 @@ describe("EmDashDB collection deletion guard", () => { return cursor([{ collection_id: "collection-1" }]); } if (statement.includes("SELECT 1 AS present")) { - return cursor(contentPresent ? [{ present: 1 }] : []); + const visible = contentState === "active" || !statement.includes("deleted_at IS NULL"); + return cursor(visible ? [{ present: 1 }] : []); } if (statement.includes("UPDATE _emdash_media_usage_index_status")) { return cursor([], 1); @@ -133,7 +134,7 @@ describe("EmDashDB collection deletion guard", () => { expect(statements.some((statement) => statement.includes("UPDATE _emdash"))).toBe(false); statements = []; - contentPresent = false; + contentState = "trashed"; await expect(object.executeCollectionDeletionGuard(input)).resolves.toEqual({ outcome: "fenced", }); diff --git a/packages/core/src/media/usage/collection-deletion.ts b/packages/core/src/media/usage/collection-deletion.ts index 98745a5b50..0b7e66dbad 100644 --- a/packages/core/src/media/usage/collection-deletion.ts +++ b/packages/core/src/media/usage/collection-deletion.ts @@ -433,11 +433,16 @@ export async function deleteActivatedMediaUsageCollection( throw new Error("Collection deletion tombstone identity conflict"); } if (!deletion) { - const activation = await db - .selectFrom("_emdash_media_usage_activation") - .select("state") - .where("task_key", "=", ACTIVATION_KEY) - .executeTakeFirst(); + let activation: { state: string } | undefined; + try { + activation = await db + .selectFrom("_emdash_media_usage_activation") + .select("state") + .where("task_key", "=", ACTIVATION_KEY) + .executeTakeFirst(); + } catch (error) { + if (!isMissingTableError(error)) throw error; + } if (!activation || activation.state === "expanded") return "inactive"; if (activation.state !== "active") { throw new Error("Media usage activation must be active before collection deletion"); @@ -664,7 +669,9 @@ async function fenceCollection( if (!input.forceDelete) { const content = await sql<{ present: number }>` - SELECT 1 AS present FROM ${sql.ref(tableName)} LIMIT 1 + SELECT 1 AS present FROM ${sql.ref(tableName)} + WHERE deleted_at IS NULL + LIMIT 1 `.execute(trx); if (content.rows.length > 0) return { outcome: "has_content" }; } diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts index 1e92644c52..5e032093fa 100644 --- a/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts @@ -111,6 +111,19 @@ describeEachDialect("media usage activated collection deletion", (dialect) => { ).toEqual([]); }); + it("detaches a collection whose only entries are trashed without force", async () => { + await registry.createCollection({ slug: "trashed", label: "Trashed" }); + await sql` + INSERT INTO ${sql.ref("ec_trashed")} (id, slug, deleted_at) + VALUES ('entry-1', 'entry-1', '2026-08-12T00:00:00.000Z') + `.execute(ctx.db); + + await registry.deleteCollection("trashed"); + + expect(await registry.getCollection("trashed")).toBeNull(); + expect(await tableExists(ctx.db, "ec_trashed")).toBe(false); + }); + it("detaches a non-empty activated collection only when force is explicit", async () => { const collection = await registry.createCollection({ slug: "forced", label: "Forced" }); await sql`INSERT INTO ${sql.ref("ec_forced")} (id, slug) VALUES ('entry-1', 'entry-1')`.execute( diff --git a/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts b/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts index 1476ae3563..d27ec993e5 100644 --- a/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts +++ b/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts @@ -48,6 +48,21 @@ describeEachDialect("media usage incremental work migration", (dialect) => { expect(work).toEqual([]); }); + it("keeps V1 collection deletion available after rolling back incremental capture", async () => { + const collectionDeletionMigration = + await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); + await collectionDeletionMigration.down(ctx.db); + const incrementalMigration = + await import("../../../src/database/migrations/063_media_usage_incremental_work.js"); + await incrementalMigration.down(ctx.db); + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "legacy", label: "Legacy" }); + + await registry.deleteCollection("legacy", { force: true }); + + expect(await registry.getCollection("legacy")).toBeNull(); + }); + it("upgrades and reruns without rewriting legacy evidence or inventing work", async () => { const collectionDeletionMigration = await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); diff --git a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts index 83fb7be143..9f89ca72f4 100644 --- a/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-collection-deletion-d1.test.ts @@ -101,8 +101,8 @@ it("rolls back a stale guarded batch before any collection DDL", async () => { ).toEqual([{ collection_id: "collection-d1" }]); }); -it("atomically preserves content or fences an empty collection", async () => { - await sql`CREATE TABLE ec_d1_fence (id TEXT PRIMARY KEY)`.execute(db); +it("atomically preserves active content or fences a trashed collection", async () => { + await sql`CREATE TABLE ec_d1_fence (id TEXT PRIMARY KEY, deleted_at TEXT)`.execute(db); await ctxInsertCollection(); await db .insertInto("_emdash_media_usage_index_status") @@ -158,7 +158,7 @@ it("atomically preserves content or fences an empty collection", async () => { ).resolves.toEqual({ outcome: "stale" }); await ctxInsertCollection(); - await sql`DELETE FROM ec_d1_fence`.execute(db); + await sql`UPDATE ec_d1_fence SET deleted_at = '2026-08-12T00:00:00.000Z'`.execute(db); await expect( executeCollectionDeletionGuard( { binding: "DB" }, From 90f01f88cd58157f88a08abb708d03ed876ee818 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:25:50 +0100 Subject: [PATCH 11/20] fix(cloudflare): pin mutation reads to DO primary --- .../cloudflare/src/db/coalescing-do-sql.ts | 19 ++++-- packages/cloudflare/src/db/do-sql-class.ts | 21 ++++--- packages/cloudflare/src/db/do-sql-dialect.ts | 12 +++- packages/cloudflare/src/db/do-sql-types.ts | 13 ++-- packages/cloudflare/src/db/do-sql.ts | 39 ++++++------ .../tests/db/coalescing-do-sql.test.ts | 18 ++++++ .../db/do-sql-collection-deletion.test.ts | 27 ++++++++ .../tests/db/do-sql-dialect.test.ts | 16 +++++ .../tests/db/do-sql-request-scope.test.ts | 62 +++++++++++++++++++ packages/core/src/astro/middleware.ts | 13 ++-- .../unit/astro/with-emdash-runtime.test.ts | 17 +++-- 11 files changed, 206 insertions(+), 51 deletions(-) create mode 100644 packages/cloudflare/tests/db/do-sql-request-scope.test.ts diff --git a/packages/cloudflare/src/db/coalescing-do-sql.ts b/packages/cloudflare/src/db/coalescing-do-sql.ts index 260a29b6ad..fa8e9678e1 100644 --- a/packages/cloudflare/src/db/coalescing-do-sql.ts +++ b/packages/cloudflare/src/db/coalescing-do-sql.ts @@ -30,7 +30,7 @@ import { import { D1Introspector } from "./d1-introspector.js"; import type { DOSqlDialectConfig } from "./do-sql-dialect.js"; -import type { EmDashDBStub } from "./do-sql-types.js"; +import type { DOQueryOptions, EmDashDBStub } from "./do-sql-types.js"; import { isReadStatement } from "./do-sql-types.js"; /** @@ -83,6 +83,15 @@ class CoalescingDOSqlConnection implements DatabaseConnection { return this.#config.bookmarkSink?.latest ?? this.#config.readBookmark; } + #readOptions(): DOQueryOptions | undefined { + const bookmark = this.#effectiveBookmark(); + if (!bookmark && !this.#config.forcePrimary) return undefined; + return { + ...(bookmark ? { bookmark } : {}), + ...(this.#config.forcePrimary ? { primary: true } : {}), + }; + } + /** * Run `op` after all previously-enqueued RPCs settle, so only one physical * RPC is ever in flight. We report `supportsMultipleConnections: true` to @@ -101,9 +110,9 @@ class CoalescingDOSqlConnection implements DatabaseConnection { /** Single-statement path: full `query()` semantics (bookmark, sink, writes). */ async #single(sql: string, params: unknown[]): Promise> { - const bookmark = isReadStatement(sql) ? this.#effectiveBookmark() : undefined; + const opts = isReadStatement(sql) ? this.#readOptions() : undefined; this.#config.onRpc?.(); - const result = await this.#stub.query(sql, params, bookmark ? { bookmark } : undefined); + const result = await this.#stub.query(sql, params, opts); if (result.bookmark && this.#config.bookmarkSink) { this.#config.bookmarkSink.latest = result.bookmark; } @@ -184,13 +193,13 @@ class CoalescingDOSqlConnection implements DatabaseConnection { // Compute the bookmark inside the enqueued op so it reflects any write // that ran just before this flush. - const bookmark = this.#effectiveBookmark(); + const opts = this.#readOptions(); let results; try { this.#config.onRpc?.(); results = await this.#stub.batchQuery( pending.map((p) => ({ sql: p.sql, params: p.params })), - bookmark ? { bookmark } : undefined, + opts, ); } catch { // The batch RPC failed as a unit. Fall back to running each buffered diff --git a/packages/cloudflare/src/db/do-sql-class.ts b/packages/cloudflare/src/db/do-sql-class.ts index 67eba6b15d..4ee4fb67ef 100644 --- a/packages/cloudflare/src/db/do-sql-class.ts +++ b/packages/cloudflare/src/db/do-sql-class.ts @@ -30,7 +30,12 @@ import { DurableObject } from "cloudflare:workers"; import type { CollectionDeletionGuardInput, CollectionDeletionGuardResult } from "emdash"; -import type { DOQueryResult, DOQueryStatement, EmDashDBStub } from "./do-sql-types.js"; +import type { + DOQueryOptions, + DOQueryResult, + DOQueryStatement, + EmDashDBStub, +} from "./do-sql-types.js"; import { isPragmaStatement, isReadStatement } from "./do-sql-types.js"; /** @@ -133,13 +138,12 @@ export class EmDashDB extends DurableObject { * @param opts.bookmark On a replica read, wait until this instance has * caught up to the given bookmark before serving (read-your-writes). */ - async query( - sql: string, - params?: unknown[], - opts?: { bookmark?: string }, - ): Promise { + async query(sql: string, params?: unknown[], opts?: DOQueryOptions): Promise { this.#ensureReplication(); const isRead = isReadStatement(sql); + if (opts?.primary && this.#isReplica) { + return this.#primaryStub!.query(sql, params, opts); + } // Writes must hit the primary. On a replica, proxy to it. if (!isRead && this.#isReplica) { @@ -270,9 +274,12 @@ export class EmDashDB extends DurableObject { */ async batchQuery( statements: DOQueryStatement[], - opts?: { bookmark?: string }, + opts?: DOQueryOptions, ): Promise { this.#ensureReplication(); + if (opts?.primary && this.#isReplica) { + return this.#primaryStub!.batchQuery(statements, opts); + } if (opts?.bookmark && this.#isReplica) { await this.#waitForBookmarkBounded(opts.bookmark); diff --git a/packages/cloudflare/src/db/do-sql-dialect.ts b/packages/cloudflare/src/db/do-sql-dialect.ts index 51638fc0fb..2889528e2f 100644 --- a/packages/cloudflare/src/db/do-sql-dialect.ts +++ b/packages/cloudflare/src/db/do-sql-dialect.ts @@ -29,7 +29,7 @@ import type { import { SqliteAdapter, SqliteQueryCompiler } from "kysely"; import { D1Introspector } from "./d1-introspector.js"; -import type { EmDashDBStub } from "./do-sql-types.js"; +import type { DOQueryOptions, EmDashDBStub } from "./do-sql-types.js"; import { isReadStatement } from "./do-sql-types.js"; /** Mutable holder for the latest write bookmark, read by the request `commit()`. */ @@ -65,6 +65,8 @@ export interface DOSqlDialectConfig { * rather than imported so the dialect stays decoupled from core. */ onRpc?: () => void; + /** Route reads through the primary for mutation and maintenance scopes. */ + forcePrimary?: boolean; } export class DOSqlDialect implements Dialect { @@ -153,10 +155,14 @@ class DOSqlConnection implements DatabaseConnection { // read-after-write (e.g. create() then findById()) on a replica would // miss the just-written row. Writes always proxy to the primary and mint // a fresh bookmark, so they don't carry one inbound. - let opts: { bookmark: string } | undefined; + let opts: DOQueryOptions | undefined; if (isReadStatement(sqlText)) { const bookmark = this.#config.bookmarkSink?.latest ?? this.#config.readBookmark; - if (bookmark) opts = { bookmark }; + if (bookmark || this.#config.forcePrimary) { + opts = {}; + if (bookmark) opts.bookmark = bookmark; + if (this.#config.forcePrimary) opts.primary = true; + } } this.#config.onRpc?.(); diff --git a/packages/cloudflare/src/db/do-sql-types.ts b/packages/cloudflare/src/db/do-sql-types.ts index cb08d8c7e5..81bb4d6a20 100644 --- a/packages/cloudflare/src/db/do-sql-types.ts +++ b/packages/cloudflare/src/db/do-sql-types.ts @@ -72,6 +72,12 @@ export interface DOQueryResult { bookmark?: string; } +export interface DOQueryOptions { + bookmark?: string; + /** Execute read statements on the primary instead of a replica. */ + primary?: boolean; +} + /** * Minimal RPC surface of an `EmDashDB` Durable Object stub. * @@ -81,11 +87,8 @@ export interface DOQueryResult { * driver and request-scoped code free of `cloudflare:workers` types. */ export interface EmDashDBStub { - query(sql: string, params?: unknown[], opts?: { bookmark?: string }): Promise; - batchQuery( - statements: DOQueryStatement[], - opts?: { bookmark?: string }, - ): Promise; + query(sql: string, params?: unknown[], opts?: DOQueryOptions): Promise; + batchQuery(statements: DOQueryStatement[], opts?: DOQueryOptions): Promise; executeCollectionDeletionGuard( input: import("emdash").CollectionDeletionGuardInput, ): Promise; diff --git a/packages/cloudflare/src/db/do-sql.ts b/packages/cloudflare/src/db/do-sql.ts index 1b5f5eb583..117252d148 100644 --- a/packages/cloudflare/src/db/do-sql.ts +++ b/packages/cloudflare/src/db/do-sql.ts @@ -160,9 +160,9 @@ export function createCoalescingDialect(config: DurableObjectsConfig): Dialect { // Read-replica request scoping // // createRequestScopedDb is called by the core middleware on each request. -// When session is "auto" it returns a per-request Kysely that holds one DO -// stub for the whole request, plus a commit() that persists the resulting -// replication bookmark as a cookie for authenticated users (read-your-writes). +// Replica sessions get a per-request Kysely and authenticated bookmark cookie. +// Mutation requests also get a scope when sessions are disabled so every +// safety read is explicitly routed to the primary. // ========================================================================= interface CookieJar { @@ -174,13 +174,9 @@ export interface RequestScopedDbOpts { config: DurableObjectsConfig; isAuthenticated: boolean; /** - * Whether this request mutates. Part of the shared adapter contract (the D1 - * adapter pins writes to `first-primary`). The DO backend does NOT use it for - * routing: DO exposes no Worker-side "give me the primary" handle -- a write - * is proxied to the primary by the DO itself, and read-your-writes is - * provided by the per-request bookmark feedback (a write records its bookmark - * in the sink; later reads in the same request wait for it). So correctness - * doesn't depend on knowing up front that the request writes. + * Whether this request mutates. Mutation scopes route their reads through the + * primary so destructive preconditions cannot be evaluated against a lagging + * replica. */ isWrite: boolean; cookies: CookieJar; @@ -193,7 +189,8 @@ export interface RequestScopedDb { } export function createRequestScopedDb(opts: RequestScopedDbOpts): RequestScopedDb | null { - if (opts.config?.session !== "auto") return null; + const sessionEnabled = opts.config?.session === "auto"; + if (!sessionEnabled && !opts.isWrite) return null; const ns = getNamespace(opts.config); if (!ns) return null; @@ -213,7 +210,7 @@ export function createRequestScopedDb(opts: RequestScopedDbOpts): RequestScopedD // so a replica waits until it has caught up before serving. Anonymous // readers can't resume across requests, so they always read nearest-replica. let readBookmark: string | undefined; - if (opts.isAuthenticated) { + if (sessionEnabled && opts.isAuthenticated) { const bookmark = opts.cookies.get(cookieName)?.value; if ( bookmark && @@ -230,13 +227,17 @@ export function createRequestScopedDb(opts: RequestScopedDbOpts): RequestScopedD // createDialect uses the plain DOSqlDialect -- it must never coalesce, since // concurrent requests would share a buffer.) const bookmarkSink: BookmarkSink = {}; + const dialectConfig = { + resolveStub, + readBookmark, + bookmarkSink, + onRpc: recordRpc, + forcePrimary: opts.isWrite, + }; const db = new Kysely({ - dialect: new CoalescingDOSqlDialect({ - resolveStub, - readBookmark, - bookmarkSink, - onRpc: recordRpc, - }), + dialect: sessionEnabled + ? new CoalescingDOSqlDialect(dialectConfig) + : new DOSqlDialect(dialectConfig), log: kyselyLogOption(), }); @@ -244,7 +245,7 @@ export function createRequestScopedDb(opts: RequestScopedDbOpts): RequestScopedD db, commit() { // Only authenticated users benefit from resuming a bookmark. - if (!opts.isAuthenticated) return; + if (!sessionEnabled || !opts.isAuthenticated) return; const newBookmark = bookmarkSink.latest; if (!newBookmark) return; // Don't emit a cookie the browser will silently drop (~4 KB limit), diff --git a/packages/cloudflare/tests/db/coalescing-do-sql.test.ts b/packages/cloudflare/tests/db/coalescing-do-sql.test.ts index 6ec99bee30..286f53cc72 100644 --- a/packages/cloudflare/tests/db/coalescing-do-sql.test.ts +++ b/packages/cloudflare/tests/db/coalescing-do-sql.test.ts @@ -12,6 +12,7 @@ function setup( readBookmark?: string; bookmarkSink?: BookmarkSink; onRpc?: () => void; + forcePrimary?: boolean; } = {}, ) { const query = opts.query ?? vi.fn().mockResolvedValue({ rows: [] }); @@ -22,6 +23,7 @@ function setup( readBookmark: opts.readBookmark, bookmarkSink: opts.bookmarkSink, onRpc: opts.onRpc, + forcePrimary: opts.forcePrimary, }); return { query, batchQuery, dialect }; } @@ -118,6 +120,22 @@ describe("CoalescingDOSqlDialect", () => { expect(batchQuery).toHaveBeenCalledWith(expect.any(Array), { bookmark: "bm-fresh" }); }); + it("forces lone and coalesced reads to the primary when the scope mutates", async () => { + const query = vi.fn().mockResolvedValue({ rows: [] }); + const batchQuery = vi.fn().mockResolvedValue([{ rows: [] }, { rows: [] }] as DOQueryResult[]); + const { dialect } = setup({ query, batchQuery, forcePrimary: true }); + const conn = await dialect.createDriver().acquireConnection(); + + await conn.executeQuery(CompiledQuery.raw("SELECT * FROM solo")); + expect(query).toHaveBeenCalledWith("SELECT * FROM solo", [], { primary: true }); + + await Promise.all([ + conn.executeQuery(CompiledQuery.raw("SELECT * FROM a")), + conn.executeQuery(CompiledQuery.raw("SELECT * FROM b")), + ]); + expect(batchQuery).toHaveBeenCalledWith(expect.any(Array), { primary: true }); + }); + it("falls back to individual query() calls when the batch RPC fails", async () => { const batchQuery = vi.fn().mockRejectedValue(new Error("batch boom")); const query = vi diff --git a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts index 3077a0d269..491d6b05f3 100644 --- a/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts +++ b/packages/cloudflare/tests/db/do-sql-collection-deletion.test.ts @@ -141,3 +141,30 @@ describe("EmDashDB collection deletion guard", () => { expect(statements.some((statement) => statement.includes("UPDATE _emdash"))).toBe(true); }); }); + +describe("EmDashDB primary read routing", () => { + it("proxies primary-forced reads and batches instead of serving replica state", async () => { + const primary = { + query: vi.fn().mockResolvedValue({ rows: [{ source: "primary" }] }), + batchQuery: vi.fn().mockResolvedValue([{ rows: [{ source: "primary" }] }]), + }; + const exec = vi.fn(() => cursor([{ source: "replica" }])); + const object = new EmDashDB( + { storage: { primary, sql: { exec }, transactionSync: vi.fn() } } as never, + {}, + ); + + await expect(object.query("SELECT 1", [], { primary: true })).resolves.toEqual({ + rows: [{ source: "primary" }], + }); + await expect(object.batchQuery([{ sql: "SELECT 1" }], { primary: true })).resolves.toEqual([ + { rows: [{ source: "primary" }] }, + ]); + + expect(primary.query).toHaveBeenCalledWith("SELECT 1", [], { primary: true }); + expect(primary.batchQuery).toHaveBeenCalledWith([{ sql: "SELECT 1" }], { + primary: true, + }); + expect(exec).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cloudflare/tests/db/do-sql-dialect.test.ts b/packages/cloudflare/tests/db/do-sql-dialect.test.ts index d45f66593d..a993aeb4e6 100644 --- a/packages/cloudflare/tests/db/do-sql-dialect.test.ts +++ b/packages/cloudflare/tests/db/do-sql-dialect.test.ts @@ -157,4 +157,20 @@ describe("DOSqlConnection", () => { expect(queryFn).toHaveBeenLastCalledWith("SELECT * FROM posts", [], { bookmark: "bm-cookie" }); }); + + it("forces reads to the primary when the scope mutates", async () => { + const queryFn = vi.fn().mockResolvedValue({ rows: [] }); + const { config } = createConfig(queryFn, { + forcePrimary: true, + readBookmark: "bm-cookie", + }); + const conn = await new DOSqlDialect(config).createDriver().acquireConnection(); + + await conn.executeQuery(CompiledQuery.raw("SELECT * FROM posts")); + + expect(queryFn).toHaveBeenLastCalledWith("SELECT * FROM posts", [], { + bookmark: "bm-cookie", + primary: true, + }); + }); }); diff --git a/packages/cloudflare/tests/db/do-sql-request-scope.test.ts b/packages/cloudflare/tests/db/do-sql-request-scope.test.ts new file mode 100644 index 0000000000..8be37f1bf0 --- /dev/null +++ b/packages/cloudflare/tests/db/do-sql-request-scope.test.ts @@ -0,0 +1,62 @@ +import { sql } from "kysely"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const query = vi.fn().mockResolvedValue({ rows: [] }); + const batchQuery = vi.fn().mockResolvedValue([]); + const stub = { query, batchQuery }; + const namespace = { + idFromName: vi.fn(() => "emdash-id"), + get: vi.fn(() => stub), + }; + return { query, batchQuery, stub, namespace }; +}); + +vi.mock("cloudflare:workers", () => ({ + DurableObject: class { + ctx: unknown; + + constructor(ctx: unknown) { + this.ctx = ctx; + } + }, + env: { DB_DO: mocks.namespace }, +})); + +import { createRequestScopedDb } from "../../src/db/do-sql.js"; + +describe("DO SQL request scoping", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.query.mockResolvedValue({ rows: [] }); + }); + + it("creates a primary-forced scope for a write when replica sessions are disabled", async () => { + const cookies = { get: vi.fn(), set: vi.fn() }; + const scoped = createRequestScopedDb({ + config: { binding: "DB_DO", session: "disabled" }, + isAuthenticated: true, + isWrite: true, + cookies, + url: new URL("https://example.com/_emdash/api/schema"), + }); + + expect(scoped).not.toBeNull(); + await sql`SELECT 1`.execute(scoped!.db); + expect(mocks.query).toHaveBeenCalledWith("SELECT 1", [], { primary: true }); + scoped!.commit(); + expect(cookies.set).not.toHaveBeenCalled(); + }); + + it("keeps anonymous reads on the singleton when replica sessions are disabled", () => { + expect( + createRequestScopedDb({ + config: { binding: "DB_DO", session: "disabled" }, + isAuthenticated: false, + isWrite: false, + cookies: { get: vi.fn(), set: vi.fn() }, + url: new URL("https://example.com/"), + }), + ).toBeNull(); + }); +}); diff --git a/packages/core/src/astro/middleware.ts b/packages/core/src/astro/middleware.ts index cf01c40fd9..49bd8e259a 100644 --- a/packages/core/src/astro/middleware.ts +++ b/packages/core/src/astro/middleware.ts @@ -337,16 +337,16 @@ export async function withEmDashRuntime( /** * Shared plumbing for request-free entry points (`runScheduledTasks`, * `withEmDashRuntime`): resolve the runtime singleton, then run the callback - * under an event-scoped db connection when the adapter needs one. + * under an event-scoped db when the adapter needs one. * * Connection-backed adapters (e.g. Postgres over Hyperdrive) cannot reuse * the per-isolate singleton from a platform event: its socket belongs to the * request that opened it, and workerd rejects cross-event I/O. Open an * event-scoped connection and run the callback under it in ALS — the * runtime's db getter, the cron executor, and plugin contexts all resolve - * the connection from ALS — then close it. Gated on the adapter being - * connection-backed (it exposes `close()`); stateless adapters (D1, Node - * SQLite) return null or a close-less scope and keep using the singleton. + * the connection from ALS — then close it when required. Stateless adapters + * that need primary routing can return a close-less scope; adapters with no + * event scoping return null and keep using the singleton. */ async function runOutsideRequest( config: EmDashConfig, @@ -364,9 +364,8 @@ async function runOutsideRequest( cookies: NOOP_COOKIE_JAR, url: CRON_EVENT_URL, }); - if (!scoped?.close) { - // Stateless adapter (or no per-request scoping): the singleton is safe - // outside a request. Any close-less scope created above is discarded. + if (!scoped) { + // This adapter needs no event-specific routing or connection. return fn(runtime); } const { closed, deferredTasks, lifecycle } = coordinateScopedDbLifecycle(scoped); diff --git a/packages/core/tests/unit/astro/with-emdash-runtime.test.ts b/packages/core/tests/unit/astro/with-emdash-runtime.test.ts index 1baa69dcc2..8f54ebb221 100644 --- a/packages/core/tests/unit/astro/with-emdash-runtime.test.ts +++ b/packages/core/tests/unit/astro/with-emdash-runtime.test.ts @@ -184,15 +184,22 @@ describe("withEmDashRuntime (#1887)", () => { expect(close).toHaveBeenCalledTimes(1); }); - it("uses the singleton path for a close-less scope", async () => { + it("runs outside-request work under a close-less scoped db", async () => { const commit = vi.fn(); + const scopedDb = { _marker: "scoped" }; vi.mocked(createRequestScopedDb).mockReturnValue({ - db: { _marker: "scoped" } as never, + db: scopedDb as never, commit, }); - await expect(withEmDashRuntime(() => "ok")).resolves.toBe("ok"); - // Close-less scope is discarded — nothing to commit outside a request - expect(commit).not.toHaveBeenCalled(); + let dbSeenByCallback: unknown; + await expect( + withEmDashRuntime(() => { + dbSeenByCallback = getRequestContext()?.db; + return "ok"; + }), + ).resolves.toBe("ok"); + expect(dbSeenByCallback).toBe(scopedDb); + expect(commit).toHaveBeenCalledOnce(); }); }); From e2a18b64cb1dbeab33e981de1e23f1efba9403bf Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:15:28 +0100 Subject: [PATCH 12/20] feat(media): add reconciliation coordinator state --- .../066_media_usage_reconciliation.ts | 115 +++++ .../core/src/database/migrations/runner.ts | 2 + packages/core/src/database/types.ts | 23 + .../core/src/media/usage/reconciliation.ts | 439 ++++++++++++++++++ ...a-usage-incremental-work-migration.test.ts | 9 + ...ia-usage-reconciliation-foundation.test.ts | 291 ++++++++++++ 6 files changed, 879 insertions(+) create mode 100644 packages/core/src/database/migrations/066_media_usage_reconciliation.ts create mode 100644 packages/core/src/media/usage/reconciliation.ts create mode 100644 packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts diff --git a/packages/core/src/database/migrations/066_media_usage_reconciliation.ts b/packages/core/src/database/migrations/066_media_usage_reconciliation.ts new file mode 100644 index 0000000000..c97e409a35 --- /dev/null +++ b/packages/core/src/database/migrations/066_media_usage_reconciliation.ts @@ -0,0 +1,115 @@ +import { sql, type Kysely, type RawBuilder } from "kysely"; + +import { columnExists, isPostgres, tableExists } from "../dialect-helpers.js"; + +const ACTIVATION_KEY = "incremental_capture"; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable("_emdash_media_usage_reconciliations") + .ifNotExists() + .addColumn("collection_id", "text", (column) => column.notNull().primaryKey()) + .addColumn("collection_slug", "text", (column) => column.notNull()) + .addColumn("run_token", "text", (column) => column.notNull()) + .addColumn("target_epoch", "bigint") + .addColumn("field_fingerprint", "text") + .addColumn("state", "text", (column) => column.notNull().defaultTo("pending")) + .addColumn("phase", "text", (column) => column.notNull().defaultTo("scan")) + .addColumn("scan_cursor", "text") + .addColumn("scan_upper_id", "text") + .addColumn("source_cursor", "text") + .addColumn("source_upper_key", "text") + .addColumn("attempt_count", "integer", (column) => column.notNull().defaultTo(0)) + .addColumn("next_attempt_at", "text", (column) => column.notNull()) + .addColumn("lease_token", "text") + .addColumn("lease_expires_at", "text") + .addColumn("last_error_code", "text") + .addColumn("created_at", "text", (column) => + column.notNull().defaultTo(sortableUtcTimestamp(db)), + ) + .addColumn("updated_at", "text", (column) => + column.notNull().defaultTo(sortableUtcTimestamp(db)), + ) + .execute(); + + await db.schema + .createIndex("idx__emdash_media_usage_reconciliations_due") + .ifNotExists() + .on("_emdash_media_usage_reconciliations") + .columns(["state", "next_attempt_at", "updated_at", "collection_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_reconciliations_lease") + .ifNotExists() + .on("_emdash_media_usage_reconciliations") + .columns(["state", "lease_expires_at", "updated_at", "collection_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_reconciliations_failed") + .ifNotExists() + .on("_emdash_media_usage_reconciliations") + .columns(["state", "updated_at", "collection_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_status_reconciliation") + .ifNotExists() + .on("_emdash_media_usage_index_status") + .columns([ + "adapter_id", + "scope_type", + "capture_state", + "reconciliation_required", + "collection_id", + ]) + .execute(); + + if (!(await columnExists(db, "_emdash_media_usage_activation", "media_usage_maintenance_turn"))) { + await db.schema + .alterTable("_emdash_media_usage_activation") + .addColumn("media_usage_maintenance_turn", "integer", (column) => + column.notNull().defaultTo(2), + ) + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (await tableExists(db, "_emdash_media_usage_reconciliations")) { + const evidence = await sql<{ present: number }>` + SELECT 1 AS present + FROM _emdash_media_usage_reconciliations + LIMIT 1 + `.execute(db); + if (evidence.rows.length > 0) { + throw new Error("Cannot roll back while durable reconciliation evidence exists"); + } + } + + const activation = await sql<{ state: string }>` + SELECT state + FROM _emdash_media_usage_activation + WHERE task_key = ${ACTIVATION_KEY} + `.execute(db); + if (activation.rows[0]?.state !== "expanded") { + throw new Error("Cannot roll back media usage reconciliation after activation has started"); + } + + await db.schema.dropIndex("idx__emdash_media_usage_status_reconciliation").ifExists().execute(); + await db.schema.dropIndex("idx__emdash_media_usage_reconciliations_failed").ifExists().execute(); + await db.schema.dropIndex("idx__emdash_media_usage_reconciliations_lease").ifExists().execute(); + await db.schema.dropIndex("idx__emdash_media_usage_reconciliations_due").ifExists().execute(); + await db.schema.dropTable("_emdash_media_usage_reconciliations").ifExists().execute(); + if (await columnExists(db, "_emdash_media_usage_activation", "media_usage_maintenance_turn")) { + await db.schema + .alterTable("_emdash_media_usage_activation") + .dropColumn("media_usage_maintenance_turn") + .execute(); + } +} + +function sortableUtcTimestamp(db: Kysely): RawBuilder { + if (isPostgres(db)) { + return sql`to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`; + } + return sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`; +} diff --git a/packages/core/src/database/migrations/runner.ts b/packages/core/src/database/migrations/runner.ts index 640b19d2f0..5fc302a5b2 100644 --- a/packages/core/src/database/migrations/runner.ts +++ b/packages/core/src/database/migrations/runner.ts @@ -68,6 +68,7 @@ import * as m062 from "./062_media_usage_cleanup_fence.js"; import * as m063 from "./063_media_usage_incremental_work.js"; import * as m064 from "./064_fts_plain_text.js"; import * as m065 from "./065_media_usage_collection_deletion.js"; +import * as m066 from "./066_media_usage_reconciliation.js"; const MIGRATIONS: Readonly> = Object.freeze({ "001_initial": m001, @@ -134,6 +135,7 @@ const MIGRATIONS: Readonly> = Object.freeze({ "063_media_usage_incremental_work": m063, "064_fts_plain_text": m064, "065_media_usage_collection_deletion": m065, + "066_media_usage_reconciliation": m066, }); /** Total number of registered migrations. Exported for use in tests. */ diff --git a/packages/core/src/database/types.ts b/packages/core/src/database/types.ts index 9511f11e89..71f18fbf97 100644 --- a/packages/core/src/database/types.ts +++ b/packages/core/src/database/types.ts @@ -216,6 +216,7 @@ export interface MediaUsageActivationTable { activated_at: Generated; created_at: Generated; updated_at: Generated; + media_usage_maintenance_turn: Generated; } export interface MediaUsageWorkTable { @@ -253,6 +254,27 @@ export interface MediaUsageCollectionDeletionTable { updated_at: Generated; } +export interface MediaUsageReconciliationTable { + collection_id: string; + collection_slug: string; + run_token: string; + target_epoch: Generated; + field_fingerprint: Generated; + state: Generated; + phase: Generated; + scan_cursor: Generated; + scan_upper_id: Generated; + source_cursor: Generated; + source_upper_key: Generated; + attempt_count: Generated; + next_attempt_at: string; + lease_token: Generated; + lease_expires_at: Generated; + last_error_code: Generated; + created_at: Generated; + updated_at: Generated; +} + export interface UserTable { id: string; email: string; @@ -623,6 +645,7 @@ export interface Database { _emdash_media_usage_activation: MediaUsageActivationTable; _emdash_media_usage_work: MediaUsageWorkTable; _emdash_media_usage_collection_deletions: MediaUsageCollectionDeletionTable; + _emdash_media_usage_reconciliations: MediaUsageReconciliationTable; users: UserTable; credentials: CredentialTable; auth_tokens: AuthTokenTable; diff --git a/packages/core/src/media/usage/reconciliation.ts b/packages/core/src/media/usage/reconciliation.ts new file mode 100644 index 0000000000..36b7e7fc7f --- /dev/null +++ b/packages/core/src/media/usage/reconciliation.ts @@ -0,0 +1,439 @@ +import { sql, type Kysely, type RawBuilder, type Selectable } from "kysely"; +import { ulid } from "ulidx"; + +import { isPostgres } from "../../database/dialect-helpers.js"; +import type { Database, MediaUsageReconciliationTable } from "../../database/types.js"; + +const ACTIVATION_KEY = "incremental_capture"; +const CONTENT_ADAPTER_ID = "content-media"; +const COLLECTION_SCOPE = "collection"; +const MAX_CANDIDATES = 100; +const MAX_PORTABLE_DURATION_SECONDS = 365 * 24 * 60 * 60; +const STABLE_ERROR_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; + +export type MediaUsageReconciliationState = "pending" | "retry" | "leased" | "failed"; +export type MediaUsageReconciliationPhase = "scan" | "sources"; + +export interface MediaUsageReconciliationRecord { + collectionId: string; + collectionSlug: string; + runToken: string; + targetEpoch: number | string | null; + fieldFingerprint: string | null; + state: MediaUsageReconciliationState; + phase: MediaUsageReconciliationPhase; + scanCursor: string | null; + scanUpperId: string | null; + sourceCursor: string | null; + sourceUpperKey: string | null; + attemptCount: number; + nextAttemptAt: string; + leaseToken: string | null; + leaseExpiresAt: string | null; + lastErrorCode: string | null; + createdAt: string; + updatedAt: string; +} + +export interface MediaUsageReconciliationClaim extends MediaUsageReconciliationRecord { + leaseToken: string; +} + +export class MediaUsageReconciliationRepository { + constructor(private db: Kysely) {} + + async seedNextCandidate(): Promise { + const runToken = ulid(); + const now = timestampOffset(this.db, 0); + const result = await sql<{ collection_id: string }>` + INSERT INTO _emdash_media_usage_reconciliations ( + collection_id, + collection_slug, + run_token, + next_attempt_at, + updated_at + ) + SELECT status.collection_id, status.scope_key, ${runToken}, ${now}, ${now} + FROM _emdash_media_usage_index_status AS status + INNER JOIN _emdash_collections AS collection + ON collection.id = status.collection_id + AND collection.slug = status.scope_key + WHERE status.adapter_id = ${CONTENT_ADAPTER_ID} + AND status.scope_type = ${COLLECTION_SCOPE} + AND status.capture_state = 'active' + AND status.reconciliation_required = 1 + AND status.collection_id IS NOT NULL + AND EXISTS ( + SELECT 1 FROM _emdash_media_usage_activation AS activation + WHERE activation.task_key = ${ACTIVATION_KEY} + AND activation.state = 'active' + ) + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_reconciliations AS existing + WHERE existing.collection_id = status.collection_id + ) + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = status.collection_id + ) + ORDER BY status.collection_id + LIMIT 1 + ON CONFLICT (collection_id) DO NOTHING + RETURNING collection_id + `.execute(this.db); + return result.rows.length === 1; + } + + async findDue(limit: number): Promise { + assertLimit(limit); + const nextAttemptIsDue = timestampIsDue(this.db, "next_attempt_at"); + const leaseIsDue = timestampIsDue(this.db, "lease_expires_at"); + const result = await sql>` + WITH pending_candidates AS ( + SELECT * FROM _emdash_media_usage_reconciliations + WHERE state = 'pending' AND ${nextAttemptIsDue} + ORDER BY next_attempt_at, updated_at, collection_id + LIMIT ${limit} + ), retry_candidates AS ( + SELECT * FROM _emdash_media_usage_reconciliations + WHERE state = 'retry' AND ${nextAttemptIsDue} + ORDER BY next_attempt_at, updated_at, collection_id + LIMIT ${limit} + ), leased_candidates AS ( + SELECT * FROM _emdash_media_usage_reconciliations + WHERE state = 'leased' AND ${leaseIsDue} + ORDER BY lease_expires_at, updated_at, collection_id + LIMIT ${limit} + ), candidates AS ( + SELECT * FROM pending_candidates + UNION ALL SELECT * FROM retry_candidates + UNION ALL SELECT * FROM leased_candidates + ) + SELECT * FROM candidates + ORDER BY CASE WHEN state = 'leased' THEN lease_expires_at ELSE next_attempt_at END, + updated_at, + collection_id + LIMIT ${limit} + `.execute(this.db); + return result.rows.map(rowToRecord); + } + + async findFailed(limit: number): Promise { + assertLimit(limit); + const rows = await this.db + .selectFrom("_emdash_media_usage_reconciliations as reconciliation") + .innerJoin("_emdash_media_usage_index_status as status", (join) => + join + .onRef("status.collection_id", "=", "reconciliation.collection_id") + .onRef("status.scope_key", "=", "reconciliation.collection_slug"), + ) + .selectAll("reconciliation") + .where("reconciliation.state", "=", "failed") + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .orderBy("reconciliation.updated_at") + .orderBy("reconciliation.collection_id") + .limit(limit) + .execute(); + return rows.map(rowToRecord); + } + + async claim(input: { + collectionId: string; + runToken: string; + leaseDurationSeconds: number; + }): Promise { + assertIdentity(input); + assertDuration(input.leaseDurationSeconds, "lease duration"); + const leaseToken = ulid(); + const row = await this.db + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + state: "leased", + lease_token: leaseToken, + lease_expires_at: timestampOffset(this.db, input.leaseDurationSeconds), + updated_at: timestampOffset(this.db, 0), + }) + .where("reconciliation.collection_id", "=", input.collectionId) + .where("reconciliation.run_token", "=", input.runToken) + .where((eb) => + eb.or([ + eb.and([ + eb("reconciliation.state", "in", ["pending", "retry"]), + timestampIsDue(this.db, "reconciliation.next_attempt_at"), + ]), + eb.and([ + eb("reconciliation.state", "=", "leased"), + timestampIsDue(this.db, "reconciliation.lease_expires_at"), + ]), + ]), + ) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_activation as activation") + .select("activation.task_key") + .where("activation.task_key", "=", ACTIVATION_KEY) + .where("activation.state", "=", "active"), + ), + ) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_index_status as status") + .innerJoin("_emdash_collections as collection", (join) => + join + .onRef("collection.id", "=", "status.collection_id") + .onRef("collection.slug", "=", "status.scope_key"), + ) + .select("status.collection_id") + .whereRef("status.collection_id", "=", "reconciliation.collection_id") + .whereRef("status.scope_key", "=", "reconciliation.collection_slug") + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .where("status.capture_state", "=", "active") + .where("status.reconciliation_required", "=", 1), + ), + ) + .where((eb) => + eb.not( + eb.exists( + eb + .selectFrom("_emdash_media_usage_collection_deletions as deletion") + .select("deletion.collection_id") + .whereRef("deletion.collection_id", "=", "reconciliation.collection_id"), + ), + ), + ) + .returningAll() + .executeTakeFirst(); + return row + ? ({ ...rowToRecord(row), leaseToken } satisfies MediaUsageReconciliationClaim) + : null; + } + + async release(input: { + collectionId: string; + runToken: string; + leaseToken: string; + delaySeconds: number; + }): Promise { + assertLeaseIdentity(input); + assertDuration(input.delaySeconds, "release delay", true); + const result = await this.db + .updateTable("_emdash_media_usage_reconciliations") + .set({ + state: "pending", + next_attempt_at: timestampOffset(this.db, input.delaySeconds), + lease_token: null, + lease_expires_at: null, + updated_at: timestampOffset(this.db, 0), + }) + .where("collection_id", "=", input.collectionId) + .where("run_token", "=", input.runToken) + .where("state", "=", "leased") + .where("lease_token", "=", input.leaseToken) + .where(liveLease(this.db)) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async recordFailure(input: { + collectionId: string; + runToken: string; + leaseToken: string; + errorCode: string; + retryDelaySeconds: number; + terminal: boolean; + }): Promise { + assertLeaseIdentity(input); + if (!STABLE_ERROR_CODE_PATTERN.test(input.errorCode)) { + throw new Error("Reconciliation failure requires a stable error code"); + } + assertDuration(input.retryDelaySeconds, "retry delay", true); + const result = await this.db + .updateTable("_emdash_media_usage_reconciliations") + .set({ + state: input.terminal + ? "failed" + : sql`CASE WHEN attempt_count >= 4 THEN 'failed' ELSE 'retry' END`, + attempt_count: sql`attempt_count + 1`, + next_attempt_at: timestampOffset(this.db, input.retryDelaySeconds), + lease_token: null, + lease_expires_at: null, + last_error_code: input.errorCode, + updated_at: timestampOffset(this.db, 0), + }) + .where("collection_id", "=", input.collectionId) + .where("run_token", "=", input.runToken) + .where("state", "=", "leased") + .where("lease_token", "=", input.leaseToken) + .where(liveLease(this.db)) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async deleteObsoleteFailed( + observed: Selectable, + ): Promise { + const result = await this.db + .deleteFrom("_emdash_media_usage_reconciliations as reconciliation") + .where("reconciliation.collection_id", "=", observed.collection_id) + .where("reconciliation.run_token", "=", observed.run_token) + .where("reconciliation.state", "=", "failed") + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_index_status as status") + .select("status.collection_id") + .whereRef("status.collection_id", "=", "reconciliation.collection_id") + .whereRef("status.scope_key", "=", "reconciliation.collection_slug") + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .where("status.reconciliation_required", "=", 0), + ), + ) + .executeTakeFirst(); + return Number(result.numDeletedRows ?? 0) === 1; + } + + async resetFailedForNewEpoch( + observed: Selectable, + ): Promise { + if (observed.target_epoch === null) return false; + const now = timestampOffset(this.db, 0); + const result = await this.db + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + state: "pending", + phase: "scan", + target_epoch: null, + field_fingerprint: null, + scan_cursor: null, + scan_upper_id: null, + source_cursor: null, + source_upper_key: null, + attempt_count: 0, + next_attempt_at: now, + lease_token: null, + lease_expires_at: null, + last_error_code: null, + updated_at: now, + }) + .where("reconciliation.collection_id", "=", observed.collection_id) + .where("reconciliation.run_token", "=", observed.run_token) + .where("reconciliation.state", "=", "failed") + .where("reconciliation.target_epoch", "=", observed.target_epoch) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_index_status as status") + .select("status.collection_id") + .whereRef("status.collection_id", "=", "reconciliation.collection_id") + .whereRef("status.scope_key", "=", "reconciliation.collection_slug") + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .where("status.capture_state", "=", "active") + .where("status.reconciliation_required", "=", 1) + .where("status.cursor", "is", null) + .where("status.change_epoch", ">", observed.target_epoch!), + ), + ) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } +} + +function rowToRecord( + row: Selectable, +): MediaUsageReconciliationRecord { + if (!isState(row.state) || !isPhase(row.phase) || !Number.isSafeInteger(row.attempt_count)) { + throw new Error("Invalid media usage reconciliation lifecycle"); + } + return { + collectionId: row.collection_id, + collectionSlug: row.collection_slug, + runToken: row.run_token, + targetEpoch: row.target_epoch, + fieldFingerprint: row.field_fingerprint, + state: row.state, + phase: row.phase, + scanCursor: row.scan_cursor, + scanUpperId: row.scan_upper_id, + sourceCursor: row.source_cursor, + sourceUpperKey: row.source_upper_key, + attemptCount: row.attempt_count, + nextAttemptAt: row.next_attempt_at, + leaseToken: row.lease_token, + leaseExpiresAt: row.lease_expires_at, + lastErrorCode: row.last_error_code, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + +function isState(state: string): state is MediaUsageReconciliationState { + return state === "pending" || state === "retry" || state === "leased" || state === "failed"; +} + +function isPhase(phase: string): phase is MediaUsageReconciliationPhase { + return phase === "scan" || phase === "sources"; +} + +function assertLimit(limit: number): void { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_CANDIDATES) { + throw new Error("Reconciliation candidate limit must be from 1 to 100"); + } +} + +function assertIdentity(input: { collectionId: string; runToken: string }): void { + if (!input.collectionId || !input.runToken) { + throw new Error("Reconciliation requires an exact collection and run token"); + } +} + +function assertLeaseIdentity(input: { + collectionId: string; + runToken: string; + leaseToken: string; +}): void { + assertIdentity(input); + if (!input.leaseToken) throw new Error("Reconciliation requires a lease token"); +} + +function assertDuration(value: number, label: string, allowZero = false): void { + if ( + !Number.isSafeInteger(value) || + value < (allowZero ? 0 : 1) || + value > MAX_PORTABLE_DURATION_SECONDS + ) { + throw new Error(`Reconciliation ${label} is outside the portable range`); + } +} + +function liveLease(db: Kysely): RawBuilder { + return isPostgres(db) + ? sql`lease_expires_at > to_char(statement_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + : sql`lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; +} + +function timestampIsDue(db: Kysely, column: string): RawBuilder { + const value = sql.ref(column); + return isPostgres(db) + ? sql`${value} <= to_char(statement_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + : sql`${value} <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; +} + +function timestampOffset(db: Kysely, offsetSeconds: number): RawBuilder { + if (isPostgres(db)) { + return sql`to_char( + (clock_timestamp() AT TIME ZONE 'UTC') + (${offsetSeconds} * INTERVAL '1 second'), + 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"' + )`; + } + return sql`strftime( + '%Y-%m-%dT%H:%M:%fZ', + 'now', + ${`${offsetSeconds >= 0 ? "+" : ""}${offsetSeconds} seconds`} + )`; +} diff --git a/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts b/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts index d27ec993e5..0ba28c2207 100644 --- a/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts +++ b/packages/core/tests/integration/database/media-usage-incremental-work-migration.test.ts @@ -49,6 +49,9 @@ describeEachDialect("media usage incremental work migration", (dialect) => { }); it("keeps V1 collection deletion available after rolling back incremental capture", async () => { + const reconciliationMigration = + await import("../../../src/database/migrations/066_media_usage_reconciliation.js"); + await reconciliationMigration.down(ctx.db); const collectionDeletionMigration = await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); await collectionDeletionMigration.down(ctx.db); @@ -64,6 +67,9 @@ describeEachDialect("media usage incremental work migration", (dialect) => { }); it("upgrades and reruns without rewriting legacy evidence or inventing work", async () => { + const reconciliationMigration = + await import("../../../src/database/migrations/066_media_usage_reconciliation.js"); + await reconciliationMigration.down(ctx.db); const collectionDeletionMigration = await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); await collectionDeletionMigration.down(ctx.db); @@ -209,6 +215,9 @@ describeEachDialect("media usage incremental work migration", (dialect) => { }); it("purges a partially bound status if its collection is deleted or recreated before retry", async () => { + const reconciliationMigration = + await import("../../../src/database/migrations/066_media_usage_reconciliation.js"); + await reconciliationMigration.down(ctx.db); const collectionDeletionMigration = await import("../../../src/database/migrations/065_media_usage_collection_deletion.js"); await collectionDeletionMigration.down(ctx.db); diff --git a/packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts b/packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts new file mode 100644 index 0000000000..6a74ac5884 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts @@ -0,0 +1,291 @@ +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { indexExists, tableExists } from "../../../src/database/dialect-helpers.js"; +import { MediaUsageReconciliationRepository } from "../../../src/media/usage/reconciliation.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage reconciliation foundation", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("installs the durable coordinator schema with dormant scheduling", async () => { + expect(await tableExists(ctx.db, "_emdash_media_usage_reconciliations")).toBe(true); + expect(await indexExists(ctx.db, "idx__emdash_media_usage_reconciliations_due")).toBe(true); + expect(await indexExists(ctx.db, "idx__emdash_media_usage_reconciliations_lease")).toBe(true); + expect(await indexExists(ctx.db, "idx__emdash_media_usage_reconciliations_failed")).toBe(true); + expect(await indexExists(ctx.db, "idx__emdash_media_usage_status_reconciliation")).toBe(true); + + const activation = await ctx.db + .selectFrom("_emdash_media_usage_activation") + .select(["state", "media_usage_maintenance_turn"]) + .where("task_key", "=", "incremental_capture") + .executeTakeFirstOrThrow(); + expect(activation).toEqual({ state: "expanded", media_usage_maintenance_turn: 2 }); + }); + + it("seeds at most one exact active collection per invocation", async () => { + const first = await createActiveCollection(ctx, "articles"); + const second = await createActiveCollection(ctx, "pages"); + const repository = new MediaUsageReconciliationRepository(ctx.db); + + await expect(repository.seedNextCandidate()).resolves.toBe(false); + await ctx.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + + await expect(repository.seedNextCandidate()).resolves.toBe(true); + expect(await reconciliationIdentities(ctx)).toHaveLength(1); + await expect(repository.seedNextCandidate()).resolves.toBe(true); + expect(await reconciliationIdentities(ctx)).toEqual( + [first, second].toSorted((left, right) => left.id.localeCompare(right.id)), + ); + await expect(repository.seedNextCandidate()).resolves.toBe(false); + }); + + it("excludes inactive, complete, and deleting collections from discovery", async () => { + await createActiveCollection(ctx, "inactive", { captureState: "installing" }); + await createActiveCollection(ctx, "complete", { reconciliationRequired: 0 }); + await createActiveCollection(ctx, "deleting", { captureState: "deleting" }); + await ctx.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + + const repository = new MediaUsageReconciliationRepository(ctx.db); + await expect(repository.seedNextCandidate()).resolves.toBe(false); + expect(await reconciliationIdentities(ctx)).toEqual([]); + }); + + it("claims due work once and fences stale lease owners", async () => { + const collection = await createActiveCollection(ctx, "articles"); + await activateAndSeed(ctx); + const repository = new MediaUsageReconciliationRepository(ctx.db); + const [candidate] = await repository.findDue(4); + if (!candidate) throw new Error("Expected a due reconciliation candidate"); + + const claim = await repository.claim({ + collectionId: collection.id, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }); + expect(claim).toMatchObject({ state: "leased", collectionId: collection.id }); + await expect( + repository.claim({ + collectionId: collection.id, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }), + ).resolves.toBeNull(); + + await expect( + repository.release({ + collectionId: collection.id, + runToken: candidate.runToken, + leaseToken: "stale-owner", + delaySeconds: 30, + }), + ).resolves.toBe(false); + await expect( + repository.release({ + collectionId: collection.id, + runToken: candidate.runToken, + leaseToken: claim!.leaseToken, + delaySeconds: 30, + }), + ).resolves.toBe(true); + }); + + it("does not claim a coordinator after coverage becomes complete", async () => { + const collection = await createActiveCollection(ctx, "articles"); + await activateAndSeed(ctx); + const repository = new MediaUsageReconciliationRepository(ctx.db); + const [candidate] = await repository.findDue(4); + if (!candidate) throw new Error("Expected a due reconciliation candidate"); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "complete", reconciliation_required: 0 }) + .where("collection_id", "=", collection.id) + .execute(); + + await expect( + repository.claim({ + collectionId: collection.id, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }), + ).resolves.toBeNull(); + }); + + it("persists bounded retry and terminal failure without hot-looping", async () => { + await createActiveCollection(ctx, "articles"); + await activateAndSeed(ctx); + const repository = new MediaUsageReconciliationRepository(ctx.db); + const [candidate] = await repository.findDue(4); + if (!candidate) throw new Error("Expected a due reconciliation candidate"); + const firstClaim = await repository.claim({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }); + if (!firstClaim) throw new Error("Expected the first reconciliation claim"); + + await expect( + repository.recordFailure({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseToken: firstClaim.leaseToken, + errorCode: "MEDIA_USAGE_RECONCILIATION_FAILED", + retryDelaySeconds: 30, + terminal: false, + }), + ).resolves.toBe(true); + expect(await repository.findDue(4)).toEqual([]); + + await ctx.db + .updateTable("_emdash_media_usage_reconciliations") + .set({ next_attempt_at: "2000-01-01T00:00:00.000Z" }) + .where("collection_id", "=", candidate.collectionId) + .execute(); + const retryClaim = await repository.claim({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }); + if (!retryClaim) throw new Error("Expected the retry reconciliation claim"); + await expect( + repository.recordFailure({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseToken: retryClaim.leaseToken, + errorCode: "MEDIA_USAGE_RECONCILIATION_INVALID_SOURCE", + retryDelaySeconds: 0, + terminal: true, + }), + ).resolves.toBe(true); + + expect(await repository.findDue(4)).toEqual([]); + expect(await repository.findFailed(4)).toEqual([ + expect.objectContaining({ + collectionId: candidate.collectionId, + state: "failed", + attemptCount: 2, + lastErrorCode: "MEDIA_USAGE_RECONCILIATION_INVALID_SOURCE", + }), + ]); + }); + + it("reopens failed work only after a newer coverage epoch", async () => { + const collection = await createActiveCollection(ctx, "articles"); + await activateAndSeed(ctx); + const repository = new MediaUsageReconciliationRepository(ctx.db); + const row = await ctx.db + .updateTable("_emdash_media_usage_reconciliations") + .set({ + state: "failed", + target_epoch: 4, + attempt_count: 5, + last_error_code: "MEDIA_USAGE_RECONCILIATION_FAILED", + }) + .where("collection_id", "=", collection.id) + .returningAll() + .executeTakeFirstOrThrow(); + + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ change_epoch: 4, cursor: null }) + .where("collection_id", "=", collection.id) + .execute(); + await expect(repository.resetFailedForNewEpoch(row)).resolves.toBe(false); + + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ change_epoch: 5 }) + .where("collection_id", "=", collection.id) + .execute(); + await expect(repository.resetFailedForNewEpoch(row)).resolves.toBe(true); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["state", "target_epoch", "attempt_count", "last_error_code"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ state: "pending", target_epoch: null, attempt_count: 0, last_error_code: null }); + }); + + it("refuses rollback while reconciliation evidence exists", async () => { + const migration = + await import("../../../src/database/migrations/066_media_usage_reconciliation.js"); + await ctx.db + .insertInto("_emdash_media_usage_reconciliations") + .values({ + collection_id: "collection-1", + collection_slug: "articles", + run_token: "run-1", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + + await expect(migration.down(ctx.db)).rejects.toThrow(/reconciliation evidence/i); + }); +}); + +async function createActiveCollection( + ctx: DialectTestContext, + slug: string, + options: { captureState?: string; reconciliationRequired?: number } = {}, +): Promise<{ id: string; slug: string }> { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug, label: slug }); + const collection = await registry.getCollection(slug); + if (!collection) throw new Error(`Expected ${slug} collection`); + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: collection.slug, + collection_id: collection.id, + status: "stale", + capture_state: options.captureState ?? "active", + reconciliation_required: options.reconciliationRequired ?? 1, + }) + .execute(); + return { id: collection.id, slug: collection.slug }; +} + +async function activateAndSeed(ctx: DialectTestContext): Promise { + await ctx.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + const repository = new MediaUsageReconciliationRepository(ctx.db); + await expect(repository.seedNextCandidate()).resolves.toBe(true); +} + +async function reconciliationIdentities( + ctx: DialectTestContext, +): Promise<{ id: string; slug: string }[]> { + const rows = await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["collection_id", "collection_slug"]) + .orderBy("collection_id") + .execute(); + return rows.map((row) => ({ id: row.collection_id, slug: row.collection_slug })); +} From 136ece46db8578cf8d0826647754e89e0602d23e Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:37:43 +0100 Subject: [PATCH 13/20] feat(media): add bounded reconciliation scan --- .../database/repositories/media-usage-work.ts | 93 +++++ .../src/database/repositories/media-usage.ts | 44 +++ .../core/src/media/usage/content-fields.ts | 35 ++ .../core/src/media/usage/content-snapshots.ts | 3 +- .../src/media/usage/projection-fingerprint.ts | 12 +- .../media/usage/reconciliation-processor.ts | 87 +++++ .../core/src/media/usage/reconciliation.ts | 208 +++++++++++- packages/core/src/media/usage/types.ts | 2 + .../media-usage-content-fields.test.ts | 43 +++ .../media-usage-projection-admission.test.ts | 30 ++ .../media-usage-read-repository.test.ts | 32 ++ .../media-usage-reconciliation-scan.test.ts | 320 ++++++++++++++++++ .../database/media-usage-repository.test.ts | 31 ++ 13 files changed, 933 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/media/usage/reconciliation-processor.ts create mode 100644 packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts diff --git a/packages/core/src/database/repositories/media-usage-work.ts b/packages/core/src/database/repositories/media-usage-work.ts index ffa48e1474..fb9d1a6378 100644 --- a/packages/core/src/database/repositories/media-usage-work.ts +++ b/packages/core/src/database/repositories/media-usage-work.ts @@ -15,6 +15,7 @@ const STABLE_ERROR_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; export const MEDIA_USAGE_WORK_OPERATOR_DEFAULT_LIMIT = 50; export const MEDIA_USAGE_WORK_OPERATOR_MAX_LIMIT = 100; +export const MEDIA_USAGE_RECONCILIATION_PAGE_LIMIT = 50; const MEDIA_USAGE_WORK_STATES = ["pending", "retry", "leased", "failed"] as const; export interface MediaUsageWorkIdentity { @@ -63,6 +64,91 @@ export type MediaUsageOperatorRetryResult = export class MediaUsageWorkRepository { constructor(private db: Kysely) {} + async enqueueReconciliationPage(input: { + collectionId: string; + collectionSlug: string; + runToken: string; + leaseToken: string; + changeEpoch: number | string; + contentIds: readonly string[]; + }): Promise { + if (!input.collectionId || !input.collectionSlug || !input.runToken || !input.leaseToken) { + throw new Error("Reconciliation enqueue requires exact collection and lease identity"); + } + assertNonNegativeDecimal(input.changeEpoch, "change epoch"); + const contentIds = [...new Set(input.contentIds)]; + if (contentIds.length < 1 || contentIds.length > MEDIA_USAGE_RECONCILIATION_PAGE_LIMIT) { + throw new Error("Reconciliation enqueue requires from 1 to 50 content IDs"); + } + if (contentIds.some((contentId) => !contentId)) { + throw new Error("Reconciliation enqueue content IDs must not be empty"); + } + const now = this.timestampOffset(0); + const pageValues = sql.join( + contentIds.map((contentId) => sql`(${contentId})`), + sql`, `, + ); + await sql` + WITH page(content_id) AS (VALUES ${pageValues}) + INSERT INTO _emdash_media_usage_work ( + collection_id, collection_slug, content_id, change_epoch, work_version, + state, attempt_count, next_attempt_at, lease_token, lease_expires_at, + last_attempted_at, last_error_code, created_at, updated_at + ) + SELECT + ${input.collectionId}, ${input.collectionSlug}, page.content_id, ${input.changeEpoch}, 1, + 'pending', 0, ${now}, NULL, NULL, NULL, NULL, ${now}, ${now} + FROM page + WHERE EXISTS ( + SELECT 1 + FROM _emdash_media_usage_reconciliations AS reconciliation + INNER JOIN _emdash_media_usage_index_status AS status + ON status.collection_id = reconciliation.collection_id + AND status.scope_key = reconciliation.collection_slug + INNER JOIN _emdash_collections AS collection + ON collection.id = reconciliation.collection_id + AND collection.slug = reconciliation.collection_slug + WHERE reconciliation.collection_id = ${input.collectionId} + AND reconciliation.collection_slug = ${input.collectionSlug} + AND reconciliation.run_token = ${input.runToken} + AND reconciliation.target_epoch = ${input.changeEpoch} + AND reconciliation.state = 'leased' + AND reconciliation.phase = 'scan' + AND reconciliation.lease_token = ${input.leaseToken} + AND ${this.qualifiedLeaseIsLive("reconciliation.lease_expires_at")} + AND status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.capture_state = 'active' + AND status.reconciliation_required = 1 + AND status.status = 'running' + AND status.cursor = ${input.runToken} + AND status.change_epoch = ${input.changeEpoch} + AND EXISTS ( + SELECT 1 FROM _emdash_media_usage_activation AS activation + WHERE activation.task_key = 'incremental_capture' + AND activation.state = 'active' + ) + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = reconciliation.collection_id + ) + ) + ON CONFLICT (collection_id, content_id) DO UPDATE SET + collection_slug = excluded.collection_slug, + change_epoch = excluded.change_epoch, + work_version = _emdash_media_usage_work.work_version + 1, + state = 'pending', + attempt_count = 0, + next_attempt_at = excluded.next_attempt_at, + lease_token = NULL, + lease_expires_at = NULL, + last_attempted_at = NULL, + last_error_code = NULL, + updated_at = excluded.updated_at + WHERE _emdash_media_usage_work.change_epoch < excluded.change_epoch + `.execute(this.db); + } + async findOperatorPage(options: { collectionSlug: string; state?: MediaUsageWorkState; @@ -604,6 +690,13 @@ export class MediaUsageWorkRepository { : sql`lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; } + private qualifiedLeaseIsLive(column: string): RawBuilder { + const expiry = sql.ref(column); + return isPostgres(this.db) + ? sql`${expiry} > to_char(statement_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + : sql`${expiry} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; + } + private timestampIsDue( column: "next_attempt_at" | "lease_expires_at" | "work.lease_expires_at", ): RawBuilder { diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index c07ab18a06..44baa159ad 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -2328,6 +2328,7 @@ export class MediaUsageRepository { AND ${this.generationWriteLeaseExpiryIsInFuture("expires_at")} ) AND ${this.currentCollectionExists(row.collection_id, row.collection_slug)} + AND ${this.currentCanonicalContentExists(row)} ${conflict} `.execute(db); return Number(result.numAffectedRows ?? 0) > 0; @@ -2423,6 +2424,7 @@ export class MediaUsageRepository { .where("current_generation", "=", expectedCurrentGeneration) .where(this.generationWriteLeaseExpression(row, leaseToken)) .where(this.currentCollectionExists(row.collection_id, row.collection_slug)) + .where(this.currentCanonicalContentExists(row)) .executeTakeFirst(); return Number(result.numUpdatedRows ?? 0) > 0; } @@ -2440,6 +2442,7 @@ export class MediaUsageRepository { .where(this.sourceMatchExpression(expectedSource)) .where(this.generationWriteLeaseExpression(row, leaseToken)) .where(this.currentCollectionExists(row.collection_id, row.collection_slug)) + .where(this.currentCanonicalContentExists(row)) .executeTakeFirst(); return Number(result.numUpdatedRows ?? 0) > 0; } @@ -2456,6 +2459,7 @@ export class MediaUsageRepository { .where("source_key", "=", row.source_key) .where(this.sourceMatchExpression(expectedSource)) .where(this.currentCollectionExists(row.collection_id, row.collection_slug)) + .where(this.currentCanonicalContentExists(row)) .executeTakeFirst(); return Number(result.numUpdatedRows ?? 0) > 0; } @@ -2544,6 +2548,46 @@ export class MediaUsageRepository { )`; } + private currentCanonicalContentExists( + row: + | ReturnType + | ReturnType, + ): RawBuilder { + if (row.collection_id === null || row.identity_version !== 1 || row.source_type !== "content") { + return sql`1 = 1`; + } + if ( + !row.collection_slug || + !row.content_id || + row.source_version === null || + row.source_updated_at === null + ) { + return sql`1 = 0`; + } + validateIdentifier(row.collection_slug, "collection slug"); + const tableName = `ec_${row.collection_slug}`; + validateIdentifier(tableName, "content table"); + const revisionColumn = + row.source_variant === "columns" + ? "live_revision_id" + : row.source_variant === "draft_overlay" + ? "draft_revision_id" + : null; + if (!revisionColumn) return sql`1 = 0`; + const revision = sql.ref(`content.${revisionColumn}`); + return sql`EXISTS ( + SELECT 1 + FROM ${sql.ref(tableName)} AS content + WHERE content.id = ${row.content_id} + AND content.version = ${row.source_version} + AND content.updated_at = ${row.source_updated_at} + AND ( + ${revision} = ${row.revision_id} + OR (${revision} IS NULL AND ${row.revision_id} IS NULL) + ) + )`; + } + private nullableNumberExpression( eb: ExpressionBuilder, column: "source_version" | "identity_version", diff --git a/packages/core/src/media/usage/content-fields.ts b/packages/core/src/media/usage/content-fields.ts index 9dc5146a74..82afa3d5c2 100644 --- a/packages/core/src/media/usage/content-fields.ts +++ b/packages/core/src/media/usage/content-fields.ts @@ -2,7 +2,9 @@ import type { Kysely } from "kysely"; import type { Database } from "../../database/types.js"; import { validateIdentifier } from "../../database/validate.js"; +import { buildCanonicalSha256Fingerprint } from "./projection-fingerprint.js"; import type { MediaUsageExtractionField, MediaUsageExtractionSubField } from "./types.js"; +import { CONTENT_SOURCE_SCHEMA_VERSION } from "./types.js"; export type ContentMediaUsageField = MediaUsageExtractionField; @@ -11,6 +13,31 @@ export interface ContentMediaUsageFieldDiscovery { displayFieldSlugs: string[]; } +export async function buildContentMediaUsageFieldFingerprint( + discovery: ContentMediaUsageFieldDiscovery, +): Promise { + const extractionFields = discovery.extractionFields + .map((field) => ({ + slug: field.slug, + type: field.type, + ...(field.type === "repeater" + ? { + subFields: (field.validation?.subFields ?? []) + .map((subField) => ({ slug: subField.slug, type: subField.type })) + .toSorted(compareFieldIdentity), + } + : {}), + })) + .toSorted(compareFieldIdentity); + const result = await buildCanonicalSha256Fingerprint("media-usage-fields:v1:sha256:", { + fingerprintVersion: 1, + contentSourceSchemaVersion: CONTENT_SOURCE_SCHEMA_VERSION, + extractionFields, + displayFieldSlugs: discovery.displayFieldSlugs.toSorted(compareStrings), + }); + return result.fingerprint; +} + export class MediaUsageFieldDiscoveryError extends Error { constructor( message: string, @@ -119,3 +146,11 @@ function isSupportedTopLevelType(value: string): value is SupportedTopLevelType function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } + +function compareFieldIdentity(a: { slug: string }, b: { slug: string }): number { + return compareStrings(a.slug, b.slug); +} + +function compareStrings(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} diff --git a/packages/core/src/media/usage/content-snapshots.ts b/packages/core/src/media/usage/content-snapshots.ts index 0011ab2bde..6757a3375c 100644 --- a/packages/core/src/media/usage/content-snapshots.ts +++ b/packages/core/src/media/usage/content-snapshots.ts @@ -17,8 +17,9 @@ import { buildContentMediaUsageSourceKey, type MediaUsageContentSourceVariant, } from "./source-key.js"; +import { CONTENT_SOURCE_SCHEMA_VERSION } from "./types.js"; -export const CONTENT_SOURCE_SCHEMA_VERSION = 1; +export { CONTENT_SOURCE_SCHEMA_VERSION } from "./types.js"; const CONTENT_COLLECTION_ID_RESULT = "__emdash_media_usage_collection_id"; const CONTENT_SYSTEM_COLUMNS = [ diff --git a/packages/core/src/media/usage/projection-fingerprint.ts b/packages/core/src/media/usage/projection-fingerprint.ts index bb40063e9b..d3eb16e1a8 100644 --- a/packages/core/src/media/usage/projection-fingerprint.ts +++ b/packages/core/src/media/usage/projection-fingerprint.ts @@ -41,7 +41,7 @@ export async function buildMediaUsageProjectionFingerprint( .map((occurrence) => ({ occurrence, key: canonicalJson(occurrence) })) .toSorted((a, b) => compareCanonicalStrings(a.key, b.key)) .map(({ occurrence }) => occurrence); - const payload = canonicalJson({ + return buildCanonicalSha256Fingerprint(FINGERPRINT_PREFIX, { fingerprintVersion: MEDIA_USAGE_PROJECTION_FINGERPRINT_VERSION, collectionId: input.collectionId, extractionSchema: normalizeExtractionFields(input.extractionFields), @@ -64,13 +64,19 @@ export async function buildMediaUsageProjectionFingerprint( }, occurrences: canonicalOccurrences, }); - const encodedPayload = new TextEncoder().encode(payload); +} + +export async function buildCanonicalSha256Fingerprint( + prefix: string, + payload: unknown, +): Promise { + const encodedPayload = new TextEncoder().encode(canonicalJson(payload)); const digest = await crypto.subtle.digest("SHA-256", encodedPayload); const hex = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join( "", ); return { - fingerprint: `${FINGERPRINT_PREFIX}${hex}`, + fingerprint: `${prefix}${hex}`, byteLength: encodedPayload.byteLength, }; } diff --git a/packages/core/src/media/usage/reconciliation-processor.ts b/packages/core/src/media/usage/reconciliation-processor.ts new file mode 100644 index 0000000000..1bff60148d --- /dev/null +++ b/packages/core/src/media/usage/reconciliation-processor.ts @@ -0,0 +1,87 @@ +import type { Kysely } from "kysely"; + +import { MediaUsageWorkRepository } from "../../database/repositories/media-usage-work.js"; +import type { Database } from "../../database/types.js"; +import { + buildContentMediaUsageFieldFingerprint, + loadContentMediaUsageFields, +} from "./content-fields.js"; +import { + MediaUsageReconciliationRepository, + type MediaUsageReconciliationClaim, +} from "./reconciliation.js"; + +export type MediaUsageReconciliationScanOutcome = "advanced" | "exhausted" | "deferred"; + +export async function processClaimedMediaUsageReconciliationScan( + db: Kysely, + claim: MediaUsageReconciliationClaim, +): Promise { + const reconciliation = new MediaUsageReconciliationRepository(db); + let current = await reconciliation.findByIdentity(claim.collectionId, claim.runToken); + if (!current || current.leaseToken !== claim.leaseToken || current.phase !== "scan") { + return "deferred"; + } + + let fields; + let fieldFingerprint: string; + if (current.targetEpoch === null) { + const targetEpoch = await reconciliation.beginRun(claim); + if (targetEpoch === null) { + await reconciliation.release({ ...claim, delaySeconds: 30 }); + return "deferred"; + } + fields = await loadContentMediaUsageFields(db, claim.collectionSlug, claim.collectionId); + fieldFingerprint = await buildContentMediaUsageFieldFingerprint(fields); + const scanUpperId = + fields.extractionFields.length === 0 ? null : await reconciliation.findScanUpperId(claim); + if ( + !(await reconciliation.initializeScan({ + claim, + targetEpoch, + fieldFingerprint, + scanUpperId, + })) + ) { + return "deferred"; + } + current = await reconciliation.findByIdentity(claim.collectionId, claim.runToken); + if (!current) return "deferred"; + } else { + fields = await loadContentMediaUsageFields(db, claim.collectionSlug, claim.collectionId); + fieldFingerprint = await buildContentMediaUsageFieldFingerprint(fields); + } + + if (current.fieldFingerprint !== fieldFingerprint || current.targetEpoch === null) { + await reconciliation.release({ ...claim, delaySeconds: 30 }); + return "deferred"; + } + const contentIds = await reconciliation.findScanPage(current, 50); + if (contentIds.length === 0) { + await reconciliation.release({ ...claim, delaySeconds: 30 }); + return "exhausted"; + } + + const work = new MediaUsageWorkRepository(db); + await work.enqueueReconciliationPage({ + collectionId: claim.collectionId, + collectionSlug: claim.collectionSlug, + runToken: claim.runToken, + leaseToken: claim.leaseToken, + changeEpoch: current.targetEpoch, + contentIds, + }); + const nextCursor = contentIds.at(-1)!; + if ( + !(await reconciliation.checkpointScan({ + claim, + targetEpoch: current.targetEpoch, + previousCursor: current.scanCursor, + nextCursor, + })) + ) { + return "deferred"; + } + if (!(await reconciliation.release({ ...claim, delaySeconds: 0 }))) return "deferred"; + return "advanced"; +} diff --git a/packages/core/src/media/usage/reconciliation.ts b/packages/core/src/media/usage/reconciliation.ts index 36b7e7fc7f..4a1a74281f 100644 --- a/packages/core/src/media/usage/reconciliation.ts +++ b/packages/core/src/media/usage/reconciliation.ts @@ -3,6 +3,7 @@ import { ulid } from "ulidx"; import { isPostgres } from "../../database/dialect-helpers.js"; import type { Database, MediaUsageReconciliationTable } from "../../database/types.js"; +import { validateIdentifier } from "../../database/validate.js"; const ACTIVATION_KEY = "incremental_capture"; const CONTENT_ADAPTER_ID = "content-media"; @@ -42,6 +43,199 @@ export interface MediaUsageReconciliationClaim extends MediaUsageReconciliationR export class MediaUsageReconciliationRepository { constructor(private db: Kysely) {} + async findByIdentity( + collectionId: string, + runToken: string, + ): Promise { + assertIdentity({ collectionId, runToken }); + const row = await this.db + .selectFrom("_emdash_media_usage_reconciliations") + .selectAll() + .where("collection_id", "=", collectionId) + .where("run_token", "=", runToken) + .executeTakeFirst(); + return row ? rowToRecord(row) : null; + } + + async beginRun(claim: MediaUsageReconciliationClaim): Promise { + const now = timestampOffset(this.db, 0); + const sameRun = sql`status = 'running' AND cursor = ${claim.runToken}`; + const row = await this.db + .updateTable("_emdash_media_usage_index_status as status") + .set({ + status: "running", + started_at: sql`CASE WHEN ${sameRun} THEN started_at ELSE ${now} END`, + completed_at: null, + cursor: claim.runToken, + indexed_source_count: 0, + failed_source_count: 0, + last_error_code: null, + change_epoch: sql`CASE WHEN ${sameRun} THEN change_epoch ELSE change_epoch + 1 END`, + reconciliation_required: 1, + updated_at: now, + }) + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .where("status.collection_id", "=", claim.collectionId) + .where("status.scope_key", "=", claim.collectionSlug) + .where("status.capture_state", "=", "active") + .where("status.reconciliation_required", "=", 1) + .where((eb) => + eb.or([eb("status.status", "!=", "running"), eb("status.cursor", "=", claim.runToken)]), + ) + .where(this.liveClaimExists(claim)) + .returning("change_epoch") + .executeTakeFirst(); + return row?.change_epoch ?? null; + } + + async findScanUpperId(claim: MediaUsageReconciliationClaim): Promise { + const tableName = contentTableName(claim.collectionSlug); + const result = await sql<{ id: string }>` + SELECT content.id + FROM ${sql.ref(tableName)} AS content + WHERE ${this.liveClaimExistsSql(claim)} + ORDER BY content.id DESC + LIMIT 1 + `.execute(this.db); + return result.rows[0]?.id ?? null; + } + + async initializeScan(input: { + claim: MediaUsageReconciliationClaim; + targetEpoch: number | string; + fieldFingerprint: string; + scanUpperId: string | null; + }): Promise { + const result = await this.db + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + target_epoch: input.targetEpoch, + field_fingerprint: input.fieldFingerprint, + phase: "scan", + scan_cursor: null, + scan_upper_id: input.scanUpperId, + source_cursor: null, + source_upper_key: null, + attempt_count: 0, + last_error_code: null, + updated_at: timestampOffset(this.db, 0), + }) + .where("reconciliation.collection_id", "=", input.claim.collectionId) + .where("reconciliation.run_token", "=", input.claim.runToken) + .where("reconciliation.target_epoch", "is", null) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.lease_token", "=", input.claim.leaseToken) + .where(liveLease(this.db)) + .where(this.statusOwnsRun(input.claim, input.targetEpoch)) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async findScanPage( + reconciliation: MediaUsageReconciliationRecord, + limit: number, + ): Promise { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50) { + throw new Error("Reconciliation scan page limit must be from 1 to 50"); + } + if (!reconciliation.leaseToken || reconciliation.targetEpoch === null) return []; + const tableName = contentTableName(reconciliation.collectionSlug); + const lowerBound = reconciliation.scanCursor + ? sql`AND content.id > ${reconciliation.scanCursor}` + : sql``; + const upperBound = reconciliation.scanUpperId + ? sql`AND content.id <= ${reconciliation.scanUpperId}` + : sql`AND 1 = 0`; + const result = await sql<{ id: string }>` + SELECT content.id + FROM ${sql.ref(tableName)} AS content + WHERE 1 = 1 + ${lowerBound} + ${upperBound} + AND ${this.liveClaimExistsSql(reconciliation as MediaUsageReconciliationClaim)} + ORDER BY content.id ASC + LIMIT ${limit} + `.execute(this.db); + return result.rows.map((row) => row.id); + } + + async checkpointScan(input: { + claim: MediaUsageReconciliationClaim; + targetEpoch: number | string; + previousCursor: string | null; + nextCursor: string; + }): Promise { + let query = this.db + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + scan_cursor: input.nextCursor, + attempt_count: 0, + last_error_code: null, + updated_at: timestampOffset(this.db, 0), + }) + .where("reconciliation.collection_id", "=", input.claim.collectionId) + .where("reconciliation.run_token", "=", input.claim.runToken) + .where("reconciliation.target_epoch", "=", input.targetEpoch) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.phase", "=", "scan") + .where("reconciliation.lease_token", "=", input.claim.leaseToken) + .where(liveLease(this.db)) + .where(this.statusOwnsRun(input.claim, input.targetEpoch)); + query = input.previousCursor + ? query.where("reconciliation.scan_cursor", "=", input.previousCursor) + : query.where("reconciliation.scan_cursor", "is", null); + const result = await query.executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + private liveClaimExists(claim: MediaUsageReconciliationClaim): RawBuilder { + return this.liveClaimExistsSql(claim); + } + + private liveClaimExistsSql(claim: MediaUsageReconciliationClaim): RawBuilder { + return sql`EXISTS ( + SELECT 1 + FROM _emdash_media_usage_reconciliations AS reconciliation + INNER JOIN _emdash_collections AS collection + ON collection.id = reconciliation.collection_id + AND collection.slug = reconciliation.collection_slug + WHERE reconciliation.collection_id = ${claim.collectionId} + AND reconciliation.collection_slug = ${claim.collectionSlug} + AND reconciliation.run_token = ${claim.runToken} + AND reconciliation.state = 'leased' + AND reconciliation.lease_token = ${claim.leaseToken} + AND ${liveLease(this.db, "reconciliation.lease_expires_at")} + AND EXISTS ( + SELECT 1 FROM _emdash_media_usage_activation AS activation + WHERE activation.task_key = ${ACTIVATION_KEY} + AND activation.state = 'active' + ) + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = reconciliation.collection_id + ) + )`; + } + + private statusOwnsRun( + claim: MediaUsageReconciliationClaim, + targetEpoch: number | string, + ): RawBuilder { + return sql`EXISTS ( + SELECT 1 FROM _emdash_media_usage_index_status AS status + WHERE status.adapter_id = ${CONTENT_ADAPTER_ID} + AND status.scope_type = ${COLLECTION_SCOPE} + AND status.collection_id = ${claim.collectionId} + AND status.scope_key = ${claim.collectionSlug} + AND status.capture_state = 'active' + AND status.reconciliation_required = 1 + AND status.status = 'running' + AND status.cursor = ${claim.runToken} + AND status.change_epoch = ${targetEpoch} + )`; + } + async seedNextCandidate(): Promise { const runToken = ulid(); const now = timestampOffset(this.db, 0); @@ -411,10 +605,11 @@ function assertDuration(value: number, label: string, allowZero = false): void { } } -function liveLease(db: Kysely): RawBuilder { +function liveLease(db: Kysely, column = "lease_expires_at"): RawBuilder { + const expiry = sql.ref(column); return isPostgres(db) - ? sql`lease_expires_at > to_char(statement_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` - : sql`lease_expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; + ? sql`${expiry} > to_char(statement_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + : sql`${expiry} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`; } function timestampIsDue(db: Kysely, column: string): RawBuilder { @@ -437,3 +632,10 @@ function timestampOffset(db: Kysely, offsetSeconds: number): RawBuilde ${`${offsetSeconds >= 0 ? "+" : ""}${offsetSeconds} seconds`} )`; } + +function contentTableName(collectionSlug: string): string { + validateIdentifier(collectionSlug, "collection slug"); + const tableName = `ec_${collectionSlug}`; + validateIdentifier(tableName, "content table"); + return tableName; +} diff --git a/packages/core/src/media/usage/types.ts b/packages/core/src/media/usage/types.ts index a76281c7db..d4fc944058 100644 --- a/packages/core/src/media/usage/types.ts +++ b/packages/core/src/media/usage/types.ts @@ -1,5 +1,7 @@ import type { FieldType } from "../../schema/types.js"; +export const CONTENT_SOURCE_SCHEMA_VERSION = 1; + export type MediaKind = | "image" | "video" diff --git a/packages/core/tests/integration/database/media-usage-content-fields.test.ts b/packages/core/tests/integration/database/media-usage-content-fields.test.ts index eee9d89044..50c3d4b2bf 100644 --- a/packages/core/tests/integration/database/media-usage-content-fields.test.ts +++ b/packages/core/tests/integration/database/media-usage-content-fields.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, expect, it } from "vitest"; import { IdentifierError } from "../../../src/database/validate.js"; import { + buildContentMediaUsageFieldFingerprint, loadContentMediaUsageFields, MediaUsageFieldDiscoveryError, } from "../../../src/media/usage/content-fields.js"; @@ -152,4 +153,46 @@ describeEachDialect("content media usage field discovery", (dialect) => { await expect(loadContentMediaUsageFields(ctx.db, "posts")).rejects.toThrow(IdentifierError); }); + + it("fingerprints exact extraction and display-field definitions independent of row order", async () => { + const first = await buildContentMediaUsageFieldFingerprint({ + extractionFields: [ + { + slug: "sections", + type: "repeater", + validation: { + subFields: [ + { slug: "secondary", type: "image" }, + { slug: "primary", type: "image" }, + ], + }, + }, + { slug: "hero", type: "image" }, + ], + displayFieldSlugs: ["title", "name"], + }); + const reordered = await buildContentMediaUsageFieldFingerprint({ + extractionFields: [ + { slug: "hero", type: "image" }, + { + slug: "sections", + type: "repeater", + validation: { + subFields: [ + { slug: "primary", type: "image" }, + { slug: "secondary", type: "image" }, + ], + }, + }, + ], + displayFieldSlugs: ["name", "title"], + }); + const changed = await buildContentMediaUsageFieldFingerprint({ + extractionFields: [{ slug: "hero", type: "file" }], + displayFieldSlugs: ["name", "title"], + }); + + expect(first).toBe(reordered); + expect(changed).not.toBe(first); + }); }); diff --git a/packages/core/tests/integration/database/media-usage-projection-admission.test.ts b/packages/core/tests/integration/database/media-usage-projection-admission.test.ts index c5fc2f5e38..e98b744ff0 100644 --- a/packages/core/tests/integration/database/media-usage-projection-admission.test.ts +++ b/packages/core/tests/integration/database/media-usage-projection-admission.test.ts @@ -1,3 +1,4 @@ +import { sql } from "kysely"; import { afterEach, beforeEach, expect, it } from "vitest"; import { @@ -38,6 +39,15 @@ describeEachDialect("content media usage projection admission", (dialect) => { .insertInto("_emdash_collections") .values({ id: COLLECTION_ID, slug: COLLECTION_SLUG, label: "Admission" }) .execute(); + await sql` + CREATE TABLE ${sql.ref("ec_admission")} ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + updated_at TEXT NOT NULL, + live_revision_id TEXT, + draft_revision_id TEXT + ) + `.execute(ctx.db); }); afterEach(async () => { @@ -60,6 +70,7 @@ describeEachDialect("content media usage projection admission", (dialect) => { it("recomputes an oversized mixed plan after proving its replacement is unchanged", async () => { const contentId = "mixed-no-op-delete"; + await insertContentIdentity(ctx, contentId, true); const unchangedColumns = await snapshot(contentId, "columns", 0, "é".repeat(270_000)); const absentDraft = await snapshot(contentId, "draft_overlay", 1, "Small draft"); await repo.replaceSource(unchangedColumns.source, unchangedColumns.occurrences); @@ -89,6 +100,7 @@ describeEachDialect("content media usage projection admission", (dialect) => { it("rejects deletion when the stored source row alone exceeds the byte limit", async () => { const contentId = "oversized-source-delete"; + await insertContentIdentity(ctx, contentId, true); const absentDraft = await snapshot(contentId, "draft_overlay", 0, "é".repeat(300_000)); await repo.replaceSource(absentDraft.source, []); const observedSources = await repo.findSources(canonicalSourceKeys(contentId)); @@ -151,6 +163,24 @@ describeEachDialect("content media usage projection admission", (dialect) => { } }); +async function insertContentIdentity( + ctx: DialectTestContext, + contentId: string, + withDraft: boolean, +): Promise { + await sql` + INSERT INTO ${sql.ref("ec_admission")} ( + id, version, updated_at, live_revision_id, draft_revision_id + ) VALUES ( + ${contentId}, + 1, + '2026-08-11T00:00:00.000Z', + NULL, + ${withDraft ? `revision-${contentId}` : null} + ) + `.execute(ctx.db); +} + async function snapshot( contentId: string, sourceVariant: MediaUsageContentSourceVariant, diff --git a/packages/core/tests/integration/database/media-usage-read-repository.test.ts b/packages/core/tests/integration/database/media-usage-read-repository.test.ts index 7112a1834f..51503d836b 100644 --- a/packages/core/tests/integration/database/media-usage-read-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-read-repository.test.ts @@ -1,3 +1,4 @@ +import { sql } from "kysely"; import { afterEach, beforeEach, expect, it } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; @@ -190,6 +191,7 @@ describeEachDialect("MediaUsageRepository reads", (dialect) => { .insertInto("_emdash_collections") .values({ id: "collection-posts-old", slug: "posts", label: "Old posts" }) .execute(); + await installCanonicalContentFixture(ctx, "posts", "old-only", "rev-old-only-columns"); await repo.replaceSource( contentSource("old-only", "columns", { sourceKey: buildContentMediaUsageSourceKey({ @@ -200,6 +202,8 @@ describeEachDialect("MediaUsageRepository reads", (dialect) => { }), collectionId: "collection-posts-old", identityVersion: 1, + sourceVersion: 1, + sourceUpdatedAt: "2026-08-12T00:00:00.000Z", }), [occurrence("hero", "media-shared")], ); @@ -239,6 +243,7 @@ describeEachDialect("MediaUsageRepository reads", (dialect) => { }), [occurrence("unversioned", "media-shared")], ); + await installCanonicalContentFixture(ctx, "posts", "current", "rev-current-columns"); await repo.replaceSource( contentSource("current", "columns", { sourceKey: buildContentMediaUsageSourceKey({ @@ -250,6 +255,8 @@ describeEachDialect("MediaUsageRepository reads", (dialect) => { collectionId: "collection-posts", contentStatus: "draft", identityVersion: 1, + sourceVersion: 1, + sourceUpdatedAt: "2026-08-12T00:00:00.000Z", }), [occurrence("canonical", "media-shared")], ); @@ -492,6 +499,31 @@ async function registerCollection(ctx: DialectTestContext, slug: string): Promis .execute(); } +async function installCanonicalContentFixture( + ctx: DialectTestContext, + collectionSlug: string, + contentId: string, + liveRevisionId: string, +): Promise { + const tableName = `ec_${collectionSlug}`; + await sql` + CREATE TABLE IF NOT EXISTS ${sql.ref(tableName)} ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + updated_at TEXT NOT NULL, + live_revision_id TEXT, + draft_revision_id TEXT + ) + `.execute(ctx.db); + await sql` + INSERT INTO ${sql.ref(tableName)} ( + id, version, updated_at, live_revision_id, draft_revision_id + ) VALUES ( + ${contentId}, 1, '2026-08-12T00:00:00.000Z', ${liveRevisionId}, NULL + ) + `.execute(ctx.db); +} + function entryIdentity(entry: { collectionSlug: string; contentId: string }): [string, string] { return [entry.collectionSlug, entry.contentId]; } diff --git a/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts b/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts new file mode 100644 index 0000000000..edb528fd67 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts @@ -0,0 +1,320 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { processClaimedMediaUsageReconciliationScan } from "../../../src/media/usage/reconciliation-processor.js"; +import { MediaUsageReconciliationRepository } from "../../../src/media/usage/reconciliation.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage reconciliation scan", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("pages through the captured bound without resetting equal or newer work", async () => { + const collection = await createScanCollection(ctx, 51); + await ctx.db + .insertInto("_emdash_media_usage_work") + .values([ + { + collection_id: collection.id, + collection_slug: collection.slug, + content_id: "entry-001", + change_epoch: 0, + state: "failed", + next_attempt_at: "2000-01-01T00:00:00.000Z", + last_error_code: "OLD_FAILURE", + }, + { + collection_id: collection.id, + collection_slug: collection.slug, + content_id: "entry-002", + change_epoch: 99, + state: "failed", + next_attempt_at: "2000-01-01T00:00:00.000Z", + last_error_code: "NEWER_FAILURE", + }, + ]) + .execute(); + + await expect(claimAndScan(ctx)).resolves.toBe("advanced"); + const coordinator = await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["target_epoch", "field_fingerprint", "scan_upper_id", "scan_cursor"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(); + expect(coordinator).toMatchObject({ + target_epoch: 1, + scan_upper_id: "entry-050", + scan_cursor: "entry-049", + }); + expect(coordinator.field_fingerprint).toMatch(/^media-usage-fields:v1:sha256:[a-f0-9]{64}$/); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_work") + .select((eb) => eb.fn.countAll().as("count")) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ count: 50 }); + expect(await workState(ctx, collection.id, "entry-001")).toMatchObject({ + change_epoch: 1, + work_version: 2, + state: "pending", + last_error_code: null, + }); + expect(await workState(ctx, collection.id, "entry-002")).toMatchObject({ + change_epoch: 99, + work_version: 1, + state: "failed", + last_error_code: "NEWER_FAILURE", + }); + + const versionsBeforeReplay = await workVersions(ctx, collection.id); + await ctx.db + .updateTable("_emdash_media_usage_reconciliations") + .set({ scan_cursor: null, state: "pending", next_attempt_at: "2000-01-01T00:00:00.000Z" }) + .where("collection_id", "=", collection.id) + .execute(); + await expect(claimAndScan(ctx)).resolves.toBe("advanced"); + expect(await workVersions(ctx, collection.id)).toEqual(versionsBeforeReplay); + + await expect(claimAndScan(ctx)).resolves.toBe("advanced"); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("scan_cursor") + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ scan_cursor: "entry-050" }); + expect(await claimAndScan(ctx)).toBe("exhausted"); + }); + + it("invalidates stale canonical publication after deletion or a newer version", async () => { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "guarded", label: "Guarded" }); + const collection = await registry.getCollection("guarded"); + if (!collection) throw new Error("Expected guarded collection"); + await sql` + INSERT INTO ${sql.ref("ec_guarded")} (id, slug, version, updated_at) + VALUES ('entry-1', 'entry-1', 1, '2026-08-12T10:00:00.000Z') + `.execute(ctx.db); + const repository = new MediaUsageRepository(ctx.db); + const source = canonicalSource(collection.id, 1, "2026-08-12T10:00:00.000Z"); + + await sql`DELETE FROM ${sql.ref("ec_guarded")} WHERE id = 'entry-1'`.execute(ctx.db); + await expect(repository.replaceSourceIfMatching(source, [], null)).resolves.toMatchObject({ + replaced: false, + }); + expect(await repository.findSource(source.sourceKey)).toBeNull(); + + await sql` + INSERT INTO ${sql.ref("ec_guarded")} (id, slug, version, updated_at) + VALUES ('entry-1', 'entry-1', 2, '2026-08-12T10:05:00.000Z') + `.execute(ctx.db); + await expect(repository.replaceSourceIfMatching(source, [], null)).resolves.toMatchObject({ + replaced: false, + }); + expect(await repository.findSource(source.sourceKey)).toBeNull(); + + await ctx.db + .insertInto("revisions") + .values({ + id: "live-2", + collection: "guarded", + entry_id: "entry-1", + data: "{}", + author_id: null, + }) + .execute(); + await sql` + UPDATE ${sql.ref("ec_guarded")} + SET live_revision_id = 'live-2' + WHERE id = 'entry-1' + `.execute(ctx.db); + const staleRevision = canonicalSource( + collection.id, + 2, + "2026-08-12T10:05:00.000Z", + "guarded", + "live-1", + ); + await expect( + repository.replaceSourceIfMatching(staleRevision, [], null), + ).resolves.toMatchObject({ replaced: false }); + + const current = canonicalSource( + collection.id, + 2, + "2026-08-12T10:05:00.000Z", + "guarded", + "live-2", + ); + await expect(repository.replaceSourceIfMatching(current, [], null)).resolves.toMatchObject({ + replaced: true, + }); + }); + + it("skips content work when the collection has no media-bearing fields", async () => { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "plain", label: "Plain" }); + const collection = await registry.getCollection("plain"); + if (!collection) throw new Error("Expected plain collection"); + await sql`INSERT INTO ${sql.ref("ec_plain")} (id, slug) VALUES ('entry-1', 'entry-1')`.execute( + ctx.db, + ); + await activateCollection(ctx, collection); + + await expect(claimAndScan(ctx)).resolves.toBe("exhausted"); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_work") + .select("content_id") + .where("collection_id", "=", collection.id) + .execute(), + ).toEqual([]); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("scan_upper_id") + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ scan_upper_id: null }); + }); + + it("applies the content identity guard to attempted canonical sources", async () => { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "attempted", label: "Attempted" }); + const collection = await registry.getCollection("attempted"); + if (!collection) throw new Error("Expected attempted collection"); + await sql` + INSERT INTO ${sql.ref("ec_attempted")} (id, slug, version, updated_at) + VALUES ('entry-1', 'entry-1', 2, '2026-08-12T10:05:00.000Z') + `.execute(ctx.db); + const repository = new MediaUsageRepository(ctx.db); + const stale = { + ...canonicalSource(collection.id, 1, "2026-08-12T10:00:00.000Z", "attempted"), + lastErrorCode: "DRAFT_REVISION_NOT_FOUND", + }; + + await expect(repository.markSourceAttemptedIfMatching(stale, null)).resolves.toMatchObject({ + attempted: false, + }); + expect(await repository.findSource(stale.sourceKey)).toBeNull(); + }); +}); + +async function createScanCollection( + ctx: DialectTestContext, + entryCount: number, +): Promise<{ id: string; slug: string }> { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "articles", label: "Articles" }); + await registry.createField("articles", { slug: "hero", label: "Hero", type: "image" }); + const collection = await registry.getCollection("articles"); + if (!collection) throw new Error("Expected articles collection"); + for (let index = 0; index < entryCount; index++) { + const id = `entry-${String(index).padStart(3, "0")}`; + await sql`INSERT INTO ${sql.ref("ec_articles")} (id, slug) VALUES (${id}, ${id})`.execute( + ctx.db, + ); + } + await activateCollection(ctx, collection); + return { id: collection.id, slug: collection.slug }; +} + +async function activateCollection( + ctx: DialectTestContext, + collection: { id: string; slug: string }, +): Promise { + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: collection.slug, + collection_id: collection.id, + status: "stale", + capture_state: "active", + reconciliation_required: 1, + }) + .onConflict((conflict) => + conflict.columns(["adapter_id", "scope_type", "scope_key"]).doUpdateSet({ + collection_id: collection.id, + status: "stale", + capture_state: "active", + reconciliation_required: 1, + }), + ) + .execute(); + await ctx.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + const repository = new MediaUsageReconciliationRepository(ctx.db); + await repository.seedNextCandidate(); +} + +async function claimAndScan(ctx: DialectTestContext) { + const repository = new MediaUsageReconciliationRepository(ctx.db); + const [candidate] = await repository.findDue(4); + if (!candidate) throw new Error("Expected a due reconciliation candidate"); + const claim = await repository.claim({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }); + if (!claim) throw new Error("Expected a reconciliation claim"); + return processClaimedMediaUsageReconciliationScan(ctx.db, claim); +} + +function canonicalSource( + collectionId: string, + version: number, + updatedAt: string, + collectionSlug = "guarded", + revisionId: string | null = null, +) { + return { + sourceKey: `content-id:${collectionId}:entry-1:columns`, + sourceType: "content", + collectionId, + collectionSlug, + contentId: "entry-1", + sourceVariant: "columns" as const, + revisionId, + sourceVersion: version, + sourceUpdatedAt: updatedAt, + identityVersion: 1, + }; +} + +async function workState(ctx: DialectTestContext, collectionId: string, contentId: string) { + return ctx.db + .selectFrom("_emdash_media_usage_work") + .select(["change_epoch", "work_version", "state", "last_error_code"]) + .where("collection_id", "=", collectionId) + .where("content_id", "=", contentId) + .executeTakeFirstOrThrow(); +} + +async function workVersions(ctx: DialectTestContext, collectionId: string) { + return ctx.db + .selectFrom("_emdash_media_usage_work") + .select(["content_id", "work_version"]) + .where("collection_id", "=", collectionId) + .orderBy("content_id") + .execute(); +} diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts index 038f81635d..146cb2fe87 100644 --- a/packages/core/tests/integration/database/media-usage-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -98,6 +98,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { .insertInto("_emdash_collections") .values({ id: collectionId, slug: "posts", label: "Posts" }) .execute(); + await installCanonicalContentFixture(ctx, "posts", "entry1", "rev-entry1-columns"); const sourceKey = buildContentMediaUsageSourceKey({ collectionId, collectionSlug: "posts", @@ -110,6 +111,8 @@ describeEachDialect("MediaUsageRepository", (dialect) => { sourceKey, collectionId, identityVersion: 1, + sourceVersion: 1, + sourceUpdatedAt: "2026-08-12T00:00:00.000Z", }), [occurrence("hero", "media-old")], ); @@ -131,6 +134,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { .insertInto("_emdash_collections") .values({ id: collectionId, slug: "posts", label: "Posts" }) .execute(); + await installCanonicalContentFixture(ctx, "posts", "entry1", "rev-entry1-columns"); const source = contentSource("entry1", "columns", { sourceKey: buildContentMediaUsageSourceKey({ collectionId, @@ -140,6 +144,8 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }), collectionId, identityVersion: 1, + sourceVersion: 1, + sourceUpdatedAt: "2026-08-12T00:00:00.000Z", }); const observed = await repo.replaceSource(source, [occurrence("hero", "media-old")]); await ctx.db.deleteFrom("_emdash_collections").where("id", "=", collectionId).execute(); @@ -1581,6 +1587,31 @@ function occurrence( }; } +async function installCanonicalContentFixture( + ctx: DialectTestContext, + collectionSlug: string, + contentId: string, + liveRevisionId: string, +): Promise { + const tableName = `ec_${collectionSlug}`; + await sql` + CREATE TABLE IF NOT EXISTS ${sql.ref(tableName)} ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + updated_at TEXT NOT NULL, + live_revision_id TEXT, + draft_revision_id TEXT + ) + `.execute(ctx.db); + await sql` + INSERT INTO ${sql.ref(tableName)} ( + id, version, updated_at, live_revision_id, draft_revision_id + ) VALUES ( + ${contentId}, 1, '2026-08-12T00:00:00.000Z', ${liveRevisionId}, NULL + ) + `.execute(ctx.db); +} + async function insertOccurrenceGeneration( ctx: DialectTestContext, sourceKey: string, From e3d35a267ce417b9962079151f49d785db8804c6 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:05:34 +0100 Subject: [PATCH 14/20] feat(media): finalize automatic reconciliation --- .../database/repositories/media-usage-work.ts | 3 +- .../src/database/repositories/media-usage.ts | 14 +- .../usage/collection-deletion-processor.ts | 6 + .../media/usage/reconciliation-processor.ts | 292 +++++++++- .../core/src/media/usage/reconciliation.ts | 513 +++++++++++++++++- ...-usage-reconciliation-finalization.test.ts | 369 +++++++++++++ ...ia-usage-reconciliation-foundation.test.ts | 21 +- 7 files changed, 1183 insertions(+), 35 deletions(-) create mode 100644 packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts diff --git a/packages/core/src/database/repositories/media-usage-work.ts b/packages/core/src/database/repositories/media-usage-work.ts index fb9d1a6378..115967a4ab 100644 --- a/packages/core/src/database/repositories/media-usage-work.ts +++ b/packages/core/src/database/repositories/media-usage-work.ts @@ -70,6 +70,7 @@ export class MediaUsageWorkRepository { runToken: string; leaseToken: string; changeEpoch: number | string; + phase: "scan" | "sources"; contentIds: readonly string[]; }): Promise { if (!input.collectionId || !input.collectionSlug || !input.runToken || !input.leaseToken) { @@ -113,7 +114,7 @@ export class MediaUsageWorkRepository { AND reconciliation.run_token = ${input.runToken} AND reconciliation.target_epoch = ${input.changeEpoch} AND reconciliation.state = 'leased' - AND reconciliation.phase = 'scan' + AND reconciliation.phase = ${input.phase} AND reconciliation.lease_token = ${input.leaseToken} AND ${this.qualifiedLeaseIsLive("reconciliation.lease_expires_at")} AND status.adapter_id = 'content-media' diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index 44baa159ad..edce2b272c 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -1716,19 +1716,31 @@ export class MediaUsageRepository { }, ): Promise { const now = this.sortableUtcTimestamp(); + const automaticRunOwnsCoverage = sql`EXISTS ( + SELECT 1 + FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = ${input.collectionId} + AND reconciliation.run_token = cursor + )`; const result = await this.db .updateTable("_emdash_media_usage_index_status") .set({ status: sql`CASE + WHEN ${automaticRunOwnsCoverage} THEN status WHEN reconciliation_required = 0 THEN 'partial' WHEN status = 'running' THEN 'stale' ELSE status END`, completed_at: sql`CASE + WHEN ${automaticRunOwnsCoverage} THEN completed_at WHEN reconciliation_required = 0 OR status = 'running' THEN NULL ELSE completed_at END`, - cursor: sql`CASE WHEN status = 'running' THEN NULL ELSE cursor END`, + cursor: sql`CASE + WHEN ${automaticRunOwnsCoverage} THEN cursor + WHEN status = 'running' THEN NULL + ELSE cursor + END`, last_error_code: input.errorCode, updated_at: now, }) diff --git a/packages/core/src/media/usage/collection-deletion-processor.ts b/packages/core/src/media/usage/collection-deletion-processor.ts index 99749ee5ce..3afe9fd3e3 100644 --- a/packages/core/src/media/usage/collection-deletion-processor.ts +++ b/packages/core/src/media/usage/collection-deletion-processor.ts @@ -221,6 +221,12 @@ async function processStatus( if (await exactCleanupRowsRemain(trx, claim, false)) { throw new Error("Collection deletion cleanup is incomplete"); } + await trx + .deleteFrom("_emdash_media_usage_reconciliations") + .where("collection_id", "=", claim.collectionId) + .where("collection_slug", "=", claim.collectionSlug) + .where(liveLeaseGuard(trx, claim)) + .execute(); await trx .deleteFrom("_emdash_media_usage_index_status") .where("adapter_id", "=", "content-media") diff --git a/packages/core/src/media/usage/reconciliation-processor.ts b/packages/core/src/media/usage/reconciliation-processor.ts index 1bff60148d..914c0ce586 100644 --- a/packages/core/src/media/usage/reconciliation-processor.ts +++ b/packages/core/src/media/usage/reconciliation-processor.ts @@ -9,13 +9,94 @@ import { import { MediaUsageReconciliationRepository, type MediaUsageReconciliationClaim, + type MediaUsageReconciliationRecord, } from "./reconciliation.js"; +import { CONTENT_SOURCE_SCHEMA_VERSION } from "./types.js"; -export type MediaUsageReconciliationScanOutcome = "advanced" | "exhausted" | "deferred"; +export const MEDIA_USAGE_RECONCILIATION_LIMITS = Object.freeze({ + candidatesPerTick: 4, + pageSize: 50, + leaseDurationSeconds: 60, + maxAttempts: 5, + retryBaseSeconds: 30, + retryMaxSeconds: 15 * 60, + retryJitterRatio: 0.25, +}); + +export type MediaUsageReconciliationOutcome = + | "inactive" + | "not_due" + | "claim_lost" + | "advanced" + | "deferred" + | "completed" + | "retry" + | "failed"; + +export type MediaUsageReconciliationScanOutcome = + | "advanced" + | "exhausted" + | "deferred" + | "restart_required"; + +export async function processDueMediaUsageReconciliation( + db: Kysely, +): Promise { + const activation = await db + .selectFrom("_emdash_media_usage_activation") + .select("state") + .where("task_key", "=", "incremental_capture") + .executeTakeFirst(); + if (activation?.state !== "active") return "inactive"; + + const reconciliation = new MediaUsageReconciliationRepository(db); + if (await reconciliation.deleteOneObsolete()) return "completed"; + const [failed] = await reconciliation.findFailed(1); + if (failed) { + if (await reconciliation.finishFailedCoverage(failed.collectionId, failed.runToken)) { + return "failed"; + } + if (await reconciliation.resetFailedForNewEpoch(failed)) return "advanced"; + } + + await reconciliation.seedNextCandidate(); + const candidates = await reconciliation.findDue( + MEDIA_USAGE_RECONCILIATION_LIMITS.candidatesPerTick, + ); + let claim: MediaUsageReconciliationClaim | null = null; + for (const candidate of candidates) { + claim = await reconciliation.claim({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseDurationSeconds: MEDIA_USAGE_RECONCILIATION_LIMITS.leaseDurationSeconds, + }); + if (claim) break; + } + if (!claim) return candidates.length === 0 ? "not_due" : "claim_lost"; + + try { + return await processClaimedReconciliation(db, claim); + } catch (error) { + const terminal = claim.attemptCount + 1 >= MEDIA_USAGE_RECONCILIATION_LIMITS.maxAttempts; + const recorded = await reconciliation.recordFailure({ + collectionId: claim.collectionId, + runToken: claim.runToken, + leaseToken: claim.leaseToken, + errorCode: "MEDIA_USAGE_RECONCILIATION_FAILED", + retryDelaySeconds: retryDelaySeconds(claim.attemptCount), + terminal, + }); + if (!recorded) return "claim_lost"; + if (terminal) await reconciliation.finishFailedCoverage(claim.collectionId, claim.runToken); + console.error("[media-usage:reconciliation] Processing failed:", error); + return terminal ? "failed" : "retry"; + } +} export async function processClaimedMediaUsageReconciliationScan( db: Kysely, claim: MediaUsageReconciliationClaim, + options: { releaseOnExhausted?: boolean } = {}, ): Promise { const reconciliation = new MediaUsageReconciliationRepository(db); let current = await reconciliation.findByIdentity(claim.collectionId, claim.runToken); @@ -53,12 +134,13 @@ export async function processClaimedMediaUsageReconciliationScan( } if (current.fieldFingerprint !== fieldFingerprint || current.targetEpoch === null) { - await reconciliation.release({ ...claim, delaySeconds: 30 }); - return "deferred"; + return "restart_required"; } const contentIds = await reconciliation.findScanPage(current, 50); if (contentIds.length === 0) { - await reconciliation.release({ ...claim, delaySeconds: 30 }); + if (options.releaseOnExhausted ?? true) { + await reconciliation.release({ ...claim, delaySeconds: 30 }); + } return "exhausted"; } @@ -69,6 +151,7 @@ export async function processClaimedMediaUsageReconciliationScan( runToken: claim.runToken, leaseToken: claim.leaseToken, changeEpoch: current.targetEpoch, + phase: "scan", contentIds, }); const nextCursor = contentIds.at(-1)!; @@ -85,3 +168,204 @@ export async function processClaimedMediaUsageReconciliationScan( if (!(await reconciliation.release({ ...claim, delaySeconds: 0 }))) return "deferred"; return "advanced"; } + +async function processClaimedReconciliation( + db: Kysely, + claim: MediaUsageReconciliationClaim, +): Promise { + const reconciliation = new MediaUsageReconciliationRepository(db); + let current = await reconciliation.findByIdentity(claim.collectionId, claim.runToken); + if (!current || current.leaseToken !== claim.leaseToken) return "claim_lost"; + if (current.targetEpoch !== null && !(await reconciliation.ownsRun(claim, current.targetEpoch))) { + return restartReconciliation(db, reconciliation, claim, current); + } + + if (current.phase === "scan") { + const outcome = await processClaimedMediaUsageReconciliationScan(db, claim, { + releaseOnExhausted: false, + }); + if (outcome === "restart_required") { + current = + (await reconciliation.findByIdentity(claim.collectionId, claim.runToken)) ?? current; + return restartReconciliation(db, reconciliation, claim, current); + } + if (outcome !== "exhausted") return outcome; + current = (await reconciliation.findByIdentity(claim.collectionId, claim.runToken)) ?? current; + const barrier = await reconciliation.findWorkBarrier(claim.collectionId); + if (barrier.state === "failed") { + return failReconciliation(reconciliation, claim, barrier.errorCode, true); + } + if (barrier.state === "pending") { + await reconciliation.release({ ...claim, delaySeconds: 30 }); + return "deferred"; + } + if (current.targetEpoch === null || current.fieldFingerprint === null) return "claim_lost"; + const sourceUpperKey = await reconciliation.findSourceUpperKey(claim, current.targetEpoch); + if ( + !(await reconciliation.transitionToSources({ + claim, + targetEpoch: current.targetEpoch, + fieldFingerprint: current.fieldFingerprint, + sourceUpperKey, + })) + ) { + return "claim_lost"; + } + if (!(await reconciliation.release({ ...claim, delaySeconds: 0 }))) return "claim_lost"; + return "advanced"; + } + + return processSourcePhase(db, reconciliation, claim, current); +} + +async function processSourcePhase( + db: Kysely, + reconciliation: MediaUsageReconciliationRepository, + claim: MediaUsageReconciliationClaim, + current: MediaUsageReconciliationRecord, +): Promise { + if (current.targetEpoch === null || current.fieldFingerprint === null) return "claim_lost"; + const fields = await loadContentMediaUsageFields(db, claim.collectionSlug, claim.collectionId); + const fieldFingerprint = await buildContentMediaUsageFieldFingerprint(fields); + if (fieldFingerprint !== current.fieldFingerprint) { + return restartReconciliation(db, reconciliation, claim, current, fields, fieldFingerprint); + } + + const page = await reconciliation.findSourcePage( + current, + MEDIA_USAGE_RECONCILIATION_LIMITS.pageSize, + ); + if (page.length > 0) { + const malformed = page.some( + (source) => + !source.contentId || + (source.sourceVariant !== "columns" && source.sourceVariant !== "draft_overlay"), + ); + if (malformed) { + return failReconciliation( + reconciliation, + claim, + "MEDIA_USAGE_RECONCILIATION_INVALID_SOURCE", + false, + ); + } + const contentIds = [...new Set(page.map((source) => source.contentId!))]; + const enqueueIds = + fields.extractionFields.length === 0 + ? contentIds + : await reconciliation.findMissingContentIds(claim.collectionSlug, contentIds); + if (enqueueIds.length > 0) { + await new MediaUsageWorkRepository(db).enqueueReconciliationPage({ + collectionId: claim.collectionId, + collectionSlug: claim.collectionSlug, + runToken: claim.runToken, + leaseToken: claim.leaseToken, + changeEpoch: current.targetEpoch, + phase: "sources", + contentIds: enqueueIds, + }); + } + if ( + !(await reconciliation.checkpointSources({ + claim, + targetEpoch: current.targetEpoch, + previousCursor: current.sourceCursor, + nextCursor: page.at(-1)!.sourceKey, + })) + ) { + return "claim_lost"; + } + if (!(await reconciliation.release({ ...claim, delaySeconds: 0 }))) return "claim_lost"; + return "advanced"; + } + + const barrier = await reconciliation.findWorkBarrier(claim.collectionId); + if (barrier.state === "failed") { + return failReconciliation(reconciliation, claim, barrier.errorCode, true); + } + if (barrier.state === "pending") { + await reconciliation.release({ ...claim, delaySeconds: 30 }); + return "deferred"; + } + if ( + !(await reconciliation.finalizeCoverage({ + claim, + targetEpoch: current.targetEpoch, + fieldFingerprint, + schemaVersion: CONTENT_SOURCE_SCHEMA_VERSION, + })) + ) { + return "claim_lost"; + } + if (!(await reconciliation.deleteFinalized(claim))) return "claim_lost"; + return "completed"; +} + +async function restartReconciliation( + db: Kysely, + reconciliation: MediaUsageReconciliationRepository, + claim: MediaUsageReconciliationClaim, + current: MediaUsageReconciliationRecord, + fields?: Awaited>, + fieldFingerprint?: string, +): Promise { + if (current.targetEpoch === null) return "claim_lost"; + const discoveredFields = + fields ?? (await loadContentMediaUsageFields(db, claim.collectionSlug, claim.collectionId)); + const fingerprint = + fieldFingerprint ?? (await buildContentMediaUsageFieldFingerprint(discoveredFields)); + const targetEpoch = await reconciliation.restartRun(claim, current.targetEpoch); + if (targetEpoch === null) { + await reconciliation.release({ ...claim, delaySeconds: 30 }); + return "deferred"; + } + const scanUpperId = + discoveredFields.extractionFields.length === 0 + ? null + : await reconciliation.findScanUpperId(claim); + if ( + !(await reconciliation.restartScan({ + claim, + previousEpoch: current.targetEpoch, + targetEpoch, + fieldFingerprint: fingerprint, + scanUpperId, + })) + ) { + return "claim_lost"; + } + if (!(await reconciliation.release({ ...claim, delaySeconds: 0 }))) return "claim_lost"; + return "advanced"; +} + +async function failReconciliation( + reconciliation: MediaUsageReconciliationRepository, + claim: MediaUsageReconciliationClaim, + errorCode: string, + entryFailure: boolean, +): Promise { + const recorded = entryFailure + ? await reconciliation.recordEntryFailure(claim) + : await reconciliation.recordFailure({ + collectionId: claim.collectionId, + runToken: claim.runToken, + leaseToken: claim.leaseToken, + errorCode, + retryDelaySeconds: 0, + terminal: true, + }); + if (!recorded) return "claim_lost"; + await reconciliation.finishFailedCoverage(claim.collectionId, claim.runToken); + return "failed"; +} + +function retryDelaySeconds(attemptCount: number): number { + const exponential = Math.min( + MEDIA_USAGE_RECONCILIATION_LIMITS.retryMaxSeconds, + MEDIA_USAGE_RECONCILIATION_LIMITS.retryBaseSeconds * 2 ** attemptCount, + ); + const jitter = Math.floor( + exponential * MEDIA_USAGE_RECONCILIATION_LIMITS.retryJitterRatio * Math.random(), + ); + return Math.min(MEDIA_USAGE_RECONCILIATION_LIMITS.retryMaxSeconds, exponential + jitter); +} diff --git a/packages/core/src/media/usage/reconciliation.ts b/packages/core/src/media/usage/reconciliation.ts index 4a1a74281f..2a16de217f 100644 --- a/packages/core/src/media/usage/reconciliation.ts +++ b/packages/core/src/media/usage/reconciliation.ts @@ -40,6 +40,17 @@ export interface MediaUsageReconciliationClaim extends MediaUsageReconciliationR leaseToken: string; } +export interface MediaUsageReconciliationSourceCandidate { + sourceKey: string; + contentId: string | null; + sourceVariant: string; +} + +export type MediaUsageReconciliationWorkBarrier = + | { state: "empty" } + | { state: "pending" } + | { state: "failed"; errorCode: string }; + export class MediaUsageReconciliationRepository { constructor(private db: Kysely) {} @@ -189,6 +200,429 @@ export class MediaUsageReconciliationRepository { return Number(result.numUpdatedRows ?? 0) === 1; } + async ownsRun( + claim: MediaUsageReconciliationClaim, + targetEpoch: number | string, + ): Promise { + const result = await sql<{ owned: boolean | number }>` + SELECT ${this.statusOwnsRun(claim, targetEpoch)} AS owned + `.execute(this.db); + return Boolean(result.rows[0]?.owned); + } + + async restartRun( + claim: MediaUsageReconciliationClaim, + previousEpoch: number | string, + ): Promise { + const now = timestampOffset(this.db, 0); + const interruptedRestart = sql`status = 'running' + AND cursor = ${claim.runToken} + AND change_epoch > ${previousEpoch}`; + const row = await this.db + .updateTable("_emdash_media_usage_index_status as status") + .set({ + status: "running", + started_at: sql< + string | null + >`CASE WHEN ${interruptedRestart} THEN started_at ELSE ${now} END`, + completed_at: null, + cursor: claim.runToken, + last_error_code: null, + change_epoch: sql`CASE WHEN ${interruptedRestart} THEN change_epoch ELSE change_epoch + 1 END`, + reconciliation_required: 1, + updated_at: now, + }) + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .where("status.collection_id", "=", claim.collectionId) + .where("status.scope_key", "=", claim.collectionSlug) + .where("status.capture_state", "=", "active") + .where("status.reconciliation_required", "=", 1) + .where((eb) => + eb.or([eb("status.status", "!=", "running"), eb("status.cursor", "=", claim.runToken)]), + ) + .where(this.liveClaimExists(claim)) + .returning("change_epoch") + .executeTakeFirst(); + return row?.change_epoch ?? null; + } + + async restartScan(input: { + claim: MediaUsageReconciliationClaim; + previousEpoch: number | string; + targetEpoch: number | string; + fieldFingerprint: string; + scanUpperId: string | null; + }): Promise { + const result = await this.db + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + target_epoch: input.targetEpoch, + field_fingerprint: input.fieldFingerprint, + phase: "scan", + scan_cursor: null, + scan_upper_id: input.scanUpperId, + source_cursor: null, + source_upper_key: null, + attempt_count: 0, + next_attempt_at: timestampOffset(this.db, 0), + last_error_code: null, + updated_at: timestampOffset(this.db, 0), + }) + .where("reconciliation.collection_id", "=", input.claim.collectionId) + .where("reconciliation.run_token", "=", input.claim.runToken) + .where("reconciliation.target_epoch", "=", input.previousEpoch) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.lease_token", "=", input.claim.leaseToken) + .where(liveLease(this.db)) + .where(this.statusOwnsRun(input.claim, input.targetEpoch)) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async findWorkBarrier(collectionId: string): Promise { + if (!collectionId) throw new Error("Reconciliation work barrier requires a collection ID"); + const failed = await this.db + .selectFrom("_emdash_media_usage_work") + .select("last_error_code") + .where("collection_id", "=", collectionId) + .where("state", "=", "failed") + .orderBy("content_id") + .limit(1) + .executeTakeFirst(); + if (failed) { + return { + state: "failed", + errorCode: failed.last_error_code ?? "MEDIA_USAGE_PROCESSING_FAILED", + }; + } + const pending = await this.db + .selectFrom("_emdash_media_usage_work") + .select("content_id") + .where("collection_id", "=", collectionId) + .limit(1) + .executeTakeFirst(); + return pending ? { state: "pending" } : { state: "empty" }; + } + + async findSourceUpperKey( + claim: MediaUsageReconciliationClaim, + targetEpoch: number | string, + ): Promise { + const row = await this.db + .selectFrom("_emdash_media_usage_sources as source") + .select("source.source_key") + .where("source.source_type", "=", "content") + .where("source.collection_id", "=", claim.collectionId) + .where("source.identity_version", "=", 1) + .where(this.liveClaimExists(claim)) + .where(this.statusOwnsRun(claim, targetEpoch)) + .orderBy("source.source_key", "desc") + .limit(1) + .executeTakeFirst(); + return row?.source_key ?? null; + } + + async transitionToSources(input: { + claim: MediaUsageReconciliationClaim; + targetEpoch: number | string; + fieldFingerprint: string; + sourceUpperKey: string | null; + }): Promise { + const result = await this.db + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + phase: "sources", + source_cursor: null, + source_upper_key: input.sourceUpperKey, + attempt_count: 0, + last_error_code: null, + updated_at: timestampOffset(this.db, 0), + }) + .where("reconciliation.collection_id", "=", input.claim.collectionId) + .where("reconciliation.run_token", "=", input.claim.runToken) + .where("reconciliation.target_epoch", "=", input.targetEpoch) + .where("reconciliation.field_fingerprint", "=", input.fieldFingerprint) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.phase", "=", "scan") + .where("reconciliation.lease_token", "=", input.claim.leaseToken) + .where(liveLease(this.db)) + .where(this.statusOwnsRun(input.claim, input.targetEpoch)) + .where((eb) => + eb.not( + eb.exists( + eb + .selectFrom("_emdash_media_usage_work as work") + .select("work.content_id") + .where("work.collection_id", "=", input.claim.collectionId), + ), + ), + ) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async findSourcePage( + reconciliation: MediaUsageReconciliationRecord, + limit: number, + ): Promise { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50) { + throw new Error("Reconciliation source page limit must be from 1 to 50"); + } + if (!reconciliation.leaseToken || reconciliation.targetEpoch === null) return []; + let query = this.db + .selectFrom("_emdash_media_usage_sources as source") + .select(["source.source_key", "source.content_id", "source.source_variant"]) + .where("source.source_type", "=", "content") + .where("source.collection_id", "=", reconciliation.collectionId) + .where("source.identity_version", "=", 1) + .where(this.liveClaimExists(reconciliation as MediaUsageReconciliationClaim)) + .where( + this.statusOwnsRun( + reconciliation as MediaUsageReconciliationClaim, + reconciliation.targetEpoch, + ), + ); + if (reconciliation.sourceCursor) { + query = query.where("source.source_key", ">", reconciliation.sourceCursor); + } + if (reconciliation.sourceUpperKey) { + query = query.where("source.source_key", "<=", reconciliation.sourceUpperKey); + } else { + query = query.where(sql`1 = 0`); + } + const rows = await query.orderBy("source.source_key").limit(limit).execute(); + return rows.map((row) => ({ + sourceKey: row.source_key, + contentId: row.content_id, + sourceVariant: row.source_variant, + })); + } + + async findMissingContentIds( + collectionSlug: string, + contentIds: readonly string[], + ): Promise { + const unique = [...new Set(contentIds)]; + if (unique.length === 0) return []; + if (unique.length > 50 || unique.some((contentId) => !contentId)) { + throw new Error("Reconciliation source page has invalid content identity"); + } + const tableName = contentTableName(collectionSlug); + const existing = await sql<{ id: string }>` + SELECT id FROM ${sql.ref(tableName)} WHERE id IN (${sql.join(unique)}) + `.execute(this.db); + const present = new Set(existing.rows.map((row) => row.id)); + return unique.filter((contentId) => !present.has(contentId)); + } + + async checkpointSources(input: { + claim: MediaUsageReconciliationClaim; + targetEpoch: number | string; + previousCursor: string | null; + nextCursor: string; + }): Promise { + let query = this.db + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + source_cursor: input.nextCursor, + attempt_count: 0, + last_error_code: null, + updated_at: timestampOffset(this.db, 0), + }) + .where("reconciliation.collection_id", "=", input.claim.collectionId) + .where("reconciliation.run_token", "=", input.claim.runToken) + .where("reconciliation.target_epoch", "=", input.targetEpoch) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.phase", "=", "sources") + .where("reconciliation.lease_token", "=", input.claim.leaseToken) + .where(liveLease(this.db)) + .where(this.statusOwnsRun(input.claim, input.targetEpoch)); + query = input.previousCursor + ? query.where("reconciliation.source_cursor", "=", input.previousCursor) + : query.where("reconciliation.source_cursor", "is", null); + const result = await query.executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async finishFailedCoverage(collectionId: string, runToken: string): Promise { + assertIdentity({ collectionId, runToken }); + const now = timestampOffset(this.db, 0); + const result = await this.db + .updateTable("_emdash_media_usage_index_status as status") + .set({ + status: sql`CASE WHEN EXISTS ( + SELECT 1 FROM _emdash_media_usage_sources AS source + WHERE source.source_type = 'content' + AND source.collection_id = ${collectionId} + AND source.identity_version = 1 + LIMIT 1 + ) THEN 'partial' ELSE 'failed' END`, + completed_at: null, + cursor: null, + last_error_code: sql`CASE WHEN ( + SELECT reconciliation.last_error_code + FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = ${collectionId} + AND reconciliation.run_token = ${runToken} + ) = 'MEDIA_USAGE_RECONCILIATION_ENTRY_FAILED' + THEN COALESCE( + (SELECT work.last_error_code + FROM _emdash_media_usage_work AS work + WHERE work.collection_id = ${collectionId} AND work.state = 'failed' + ORDER BY work.content_id LIMIT 1), + 'MEDIA_USAGE_RECONCILIATION_ENTRY_FAILED' + ) + ELSE ( + SELECT reconciliation.last_error_code + FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = ${collectionId} + AND reconciliation.run_token = ${runToken} + ) END`, + reconciliation_required: 1, + updated_at: now, + }) + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .where("status.collection_id", "=", collectionId) + .where("status.status", "=", "running") + .where("status.cursor", "=", runToken) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_reconciliations as reconciliation") + .select("reconciliation.collection_id") + .where("reconciliation.collection_id", "=", collectionId) + .where("reconciliation.run_token", "=", runToken) + .where("reconciliation.state", "=", "failed"), + ), + ) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async finalizeCoverage(input: { + claim: MediaUsageReconciliationClaim; + targetEpoch: number | string; + fieldFingerprint: string; + schemaVersion: number; + }): Promise { + const now = timestampOffset(this.db, 0); + const result = await this.db + .updateTable("_emdash_media_usage_index_status as status") + .set({ + status: "complete", + schema_version: input.schemaVersion, + completed_at: now, + cursor: null, + last_error_code: null, + reconciliation_required: 0, + updated_at: now, + }) + .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) + .where("status.scope_type", "=", COLLECTION_SCOPE) + .where("status.collection_id", "=", input.claim.collectionId) + .where("status.scope_key", "=", input.claim.collectionSlug) + .where("status.capture_state", "=", "active") + .where("status.reconciliation_required", "=", 1) + .where("status.status", "=", "running") + .where("status.cursor", "=", input.claim.runToken) + .where("status.change_epoch", "=", input.targetEpoch) + .where((eb) => + eb.not( + eb.exists( + eb + .selectFrom("_emdash_media_usage_work as work") + .select("work.content_id") + .where("work.collection_id", "=", input.claim.collectionId), + ), + ), + ) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_reconciliations as reconciliation") + .innerJoin("_emdash_collections as collection", (join) => + join + .onRef("collection.id", "=", "reconciliation.collection_id") + .onRef("collection.slug", "=", "reconciliation.collection_slug"), + ) + .select("reconciliation.collection_id") + .where("reconciliation.collection_id", "=", input.claim.collectionId) + .where("reconciliation.run_token", "=", input.claim.runToken) + .where("reconciliation.target_epoch", "=", input.targetEpoch) + .where("reconciliation.field_fingerprint", "=", input.fieldFingerprint) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.phase", "=", "sources") + .where("reconciliation.lease_token", "=", input.claim.leaseToken) + .where(liveLease(this.db, "reconciliation.lease_expires_at")) + .where((inner) => + inner.not( + inner.exists( + inner + .selectFrom("_emdash_media_usage_collection_deletions as deletion") + .select("deletion.collection_id") + .where("deletion.collection_id", "=", input.claim.collectionId), + ), + ), + ), + ), + ) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_activation as activation") + .select("activation.task_key") + .where("activation.task_key", "=", ACTIVATION_KEY) + .where("activation.state", "=", "active"), + ), + ) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) === 1; + } + + async deleteFinalized(claim: MediaUsageReconciliationClaim): Promise { + const result = await this.db + .deleteFrom("_emdash_media_usage_reconciliations as reconciliation") + .where("reconciliation.collection_id", "=", claim.collectionId) + .where("reconciliation.run_token", "=", claim.runToken) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.lease_token", "=", claim.leaseToken) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_index_status as status") + .select("status.collection_id") + .where("status.collection_id", "=", claim.collectionId) + .where("status.scope_key", "=", claim.collectionSlug) + .where("status.status", "=", "complete") + .where("status.reconciliation_required", "=", 0), + ), + ) + .executeTakeFirst(); + return Number(result.numDeletedRows ?? 0) === 1; + } + + async deleteOneObsolete(): Promise { + const result = await sql<{ collection_id: string }>` + DELETE FROM _emdash_media_usage_reconciliations + WHERE (collection_id, run_token) IN ( + SELECT reconciliation.collection_id, reconciliation.run_token + FROM _emdash_media_usage_reconciliations AS reconciliation + INNER JOIN _emdash_media_usage_index_status AS status + ON status.collection_id = reconciliation.collection_id + AND status.scope_key = reconciliation.collection_slug + WHERE status.adapter_id = ${CONTENT_ADAPTER_ID} + AND status.scope_type = ${COLLECTION_SCOPE} + AND status.reconciliation_required = 0 + ORDER BY reconciliation.updated_at, reconciliation.collection_id + LIMIT 1 + ) + RETURNING collection_id + `.execute(this.db); + return result.rows.length === 1; + } + private liveClaimExists(claim: MediaUsageReconciliationClaim): RawBuilder { return this.liveClaimExistsSql(claim); } @@ -325,6 +759,19 @@ export class MediaUsageReconciliationRepository { .where("reconciliation.state", "=", "failed") .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) .where("status.scope_type", "=", COLLECTION_SCOPE) + .where((eb) => + eb.or([ + eb("status.reconciliation_required", "=", 0), + eb.and([ + eb("status.status", "=", "running"), + eb("status.cursor", "=", eb.ref("reconciliation.run_token")), + ]), + eb.and([ + eb("status.cursor", "is", null), + eb("status.change_epoch", ">", eb.ref("reconciliation.target_epoch")), + ]), + ]), + ) .orderBy("reconciliation.updated_at") .orderBy("reconciliation.collection_id") .limit(limit) @@ -451,6 +898,20 @@ export class MediaUsageReconciliationRepository { state: input.terminal ? "failed" : sql`CASE WHEN attempt_count >= 4 THEN 'failed' ELSE 'retry' END`, + ...(input.terminal + ? { + target_epoch: sql`COALESCE( + target_epoch, + (SELECT status.change_epoch + FROM _emdash_media_usage_index_status AS status + WHERE status.adapter_id = ${CONTENT_ADAPTER_ID} + AND status.scope_type = ${COLLECTION_SCOPE} + AND status.collection_id = ${input.collectionId} + AND status.status = 'running' + AND status.cursor = ${input.runToken}) + )`, + } + : {}), attempt_count: sql`attempt_count + 1`, next_attempt_at: timestampOffset(this.db, input.retryDelaySeconds), lease_token: null, @@ -467,34 +928,44 @@ export class MediaUsageReconciliationRepository { return Number(result.numUpdatedRows ?? 0) === 1; } - async deleteObsoleteFailed( - observed: Selectable, - ): Promise { + async recordEntryFailure(claim: MediaUsageReconciliationClaim): Promise { const result = await this.db - .deleteFrom("_emdash_media_usage_reconciliations as reconciliation") - .where("reconciliation.collection_id", "=", observed.collection_id) - .where("reconciliation.run_token", "=", observed.run_token) - .where("reconciliation.state", "=", "failed") + .updateTable("_emdash_media_usage_reconciliations as reconciliation") + .set({ + state: "failed", + attempt_count: sql`attempt_count + 1`, + next_attempt_at: timestampOffset(this.db, 0), + lease_token: null, + lease_expires_at: null, + last_error_code: "MEDIA_USAGE_RECONCILIATION_ENTRY_FAILED", + updated_at: timestampOffset(this.db, 0), + }) + .where("reconciliation.collection_id", "=", claim.collectionId) + .where("reconciliation.run_token", "=", claim.runToken) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.lease_token", "=", claim.leaseToken) + .where(liveLease(this.db)) .where((eb) => eb.exists( eb - .selectFrom("_emdash_media_usage_index_status as status") - .select("status.collection_id") - .whereRef("status.collection_id", "=", "reconciliation.collection_id") - .whereRef("status.scope_key", "=", "reconciliation.collection_slug") - .where("status.adapter_id", "=", CONTENT_ADAPTER_ID) - .where("status.scope_type", "=", COLLECTION_SCOPE) - .where("status.reconciliation_required", "=", 0), + .selectFrom("_emdash_media_usage_work as work") + .select("work.content_id") + .where("work.collection_id", "=", claim.collectionId) + .where("work.state", "=", "failed"), ), ) .executeTakeFirst(); - return Number(result.numDeletedRows ?? 0) === 1; + return Number(result.numUpdatedRows ?? 0) === 1; } async resetFailedForNewEpoch( - observed: Selectable, + observed: Selectable | MediaUsageReconciliationRecord, ): Promise { - if (observed.target_epoch === null) return false; + const collectionId = + "collection_id" in observed ? observed.collection_id : observed.collectionId; + const runToken = "run_token" in observed ? observed.run_token : observed.runToken; + const targetEpoch = "target_epoch" in observed ? observed.target_epoch : observed.targetEpoch; + if (targetEpoch === null) return false; const now = timestampOffset(this.db, 0); const result = await this.db .updateTable("_emdash_media_usage_reconciliations as reconciliation") @@ -514,10 +985,10 @@ export class MediaUsageReconciliationRepository { last_error_code: null, updated_at: now, }) - .where("reconciliation.collection_id", "=", observed.collection_id) - .where("reconciliation.run_token", "=", observed.run_token) + .where("reconciliation.collection_id", "=", collectionId) + .where("reconciliation.run_token", "=", runToken) .where("reconciliation.state", "=", "failed") - .where("reconciliation.target_epoch", "=", observed.target_epoch) + .where("reconciliation.target_epoch", "=", targetEpoch) .where((eb) => eb.exists( eb @@ -530,7 +1001,7 @@ export class MediaUsageReconciliationRepository { .where("status.capture_state", "=", "active") .where("status.reconciliation_required", "=", 1) .where("status.cursor", "is", null) - .where("status.change_epoch", ">", observed.target_epoch!), + .where("status.change_epoch", ">", targetEpoch), ), ) .executeTakeFirst(); diff --git a/packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts b/packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts new file mode 100644 index 0000000000..c3ae649d77 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts @@ -0,0 +1,369 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { processDueMediaUsageCollectionDeletions } from "../../../src/media/usage/collection-deletion-processor.js"; +import { loadContentMediaUsageSnapshots } from "../../../src/media/usage/content-snapshots.js"; +import { processDueMediaUsageReconciliation } from "../../../src/media/usage/reconciliation-processor.js"; +import { MediaUsageReconciliationRepository } from "../../../src/media/usage/reconciliation.js"; +import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; +import { processDueMediaUsageWork } from "../../../src/media/usage/work-processor.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage reconciliation finalization", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("reconciles historical missing content through existing entry work", async () => { + const collection = await createCollection(ctx, "articles", true); + await sql` + INSERT INTO ${sql.ref("ec_articles")} (id, slug) + VALUES ('historical', 'historical') + `.execute(ctx.db); + const snapshots = await loadContentMediaUsageSnapshots( + ctx.db, + collection.slug, + "historical", + undefined, + { collectionId: collection.id, identityVersion: 1 }, + ); + if (!snapshots.success) throw new Error(snapshots.error); + const usage = new MediaUsageRepository(ctx.db); + for (const snapshot of snapshots.snapshots) { + await usage.replaceSourceIfMatching(snapshot.source, snapshot.occurrences, null); + } + const quarantinedSourceKey = buildContentMediaUsageSourceKey({ + collectionId: collection.id, + collectionSlug: collection.slug, + contentId: "quarantined", + sourceVariant: "columns", + }); + await ctx.db + .insertInto("_emdash_media_usage_sources") + .values({ + source_key: quarantinedSourceKey, + source_type: "content", + collection_id: collection.id, + collection_slug: collection.slug, + content_id: "quarantined", + source_variant: "columns", + current_generation: "legacy-generation", + identity_version: null, + }) + .execute(); + await sql`DELETE FROM ${sql.ref("ec_articles")} WHERE id = 'historical'`.execute(ctx.db); + await activateCollection(ctx, collection); + + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("advanced"); + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("advanced"); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_work") + .select(["content_id", "state"]) + .where("collection_id", "=", collection.id) + .execute(), + ).toEqual([{ content_id: "historical", state: "pending" }]); + + await expect(processDueMediaUsageWork(ctx.db)).resolves.toMatchObject({ completedCount: 1 }); + expect(await usage.findSource(snapshots.snapshots[0]!.source.sourceKey)).toBeNull(); + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("completed"); + + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select(["status", "reconciliation_required", "cursor", "last_error_code"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ + status: "complete", + reconciliation_required: 0, + cursor: null, + last_error_code: null, + }); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("collection_id") + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toBeUndefined(); + expect(await usage.findSource(quarantinedSourceKey)).not.toBeNull(); + }); + + it("preserves automatic ownership until failed work reaches terminal coverage", async () => { + const collection = await createCollection(ctx, "articles", true); + await sql` + INSERT INTO ${sql.ref("ec_articles")} (id, slug) + VALUES ('entry-1', 'entry-1') + `.execute(ctx.db); + await activateCollection(ctx, collection); + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("advanced"); + const coordinator = await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["run_token", "target_epoch"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(); + const work = await ctx.db + .updateTable("_emdash_media_usage_work") + .set({ state: "failed", last_error_code: "MEDIA_USAGE_RESOURCE_LIMIT" }) + .where("collection_id", "=", collection.id) + .returning(["content_id", "work_version"]) + .executeTakeFirstOrThrow(); + await new MediaUsageRepository(ctx.db).recordIncrementalFailure({ + collectionId: collection.id, + collectionSlug: collection.slug, + contentId: work.content_id, + workVersion: work.work_version, + errorCode: "MEDIA_USAGE_RESOURCE_LIMIT", + }); + + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select(["status", "cursor", "last_error_code"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ + status: "running", + cursor: coordinator.run_token, + last_error_code: "MEDIA_USAGE_RESOURCE_LIMIT", + }); + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("failed"); + + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["state", "last_error_code", "target_epoch"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ + state: "failed", + last_error_code: "MEDIA_USAGE_RECONCILIATION_ENTRY_FAILED", + target_epoch: coordinator.target_epoch, + }); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select(["status", "reconciliation_required", "cursor", "last_error_code"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ + status: "failed", + reconciliation_required: 1, + cursor: null, + last_error_code: "MEDIA_USAGE_RESOURCE_LIMIT", + }); + }); + + it("collection deletion removes the exact reconciliation row before status", async () => { + const collection = await createCollection(ctx, "articles", false); + await activateCollection(ctx, collection); + await processDueMediaUsageReconciliation(ctx.db); + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: collection.id, + collection_slug: collection.slug, + force_delete: 1, + state: "pending", + phase: "status", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + + await expect(processDueMediaUsageCollectionDeletions(ctx.db)).resolves.toMatchObject({ + outcome: "progress", + }); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("collection_id") + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toBeUndefined(); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select("collection_id") + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toBeUndefined(); + }); + + it("does not persist terminal entry failure after work has been reopened", async () => { + const collection = await createCollection(ctx, "articles", true); + await sql` + INSERT INTO ${sql.ref("ec_articles")} (id, slug) + VALUES ('entry-1', 'entry-1') + `.execute(ctx.db); + await activateCollection(ctx, collection); + await processDueMediaUsageReconciliation(ctx.db); + const repository = new MediaUsageReconciliationRepository(ctx.db); + const [candidate] = await repository.findDue(4); + if (!candidate) throw new Error("Expected reconciliation work"); + const claim = await repository.claim({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }); + if (!claim) throw new Error("Expected reconciliation claim"); + + await expect(repository.recordEntryFailure(claim)).resolves.toBe(false); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("state") + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ state: "leased" }); + }); + + it("restarts from a new epoch after a schema fingerprint changes", async () => { + const collection = await createCollection(ctx, "articles", true); + await sql` + INSERT INTO ${sql.ref("ec_articles")} (id, slug) + VALUES ('entry-1', 'entry-1') + `.execute(ctx.db); + await activateCollection(ctx, collection); + await processDueMediaUsageReconciliation(ctx.db); + await processDueMediaUsageWork(ctx.db); + await processDueMediaUsageReconciliation(ctx.db); + const before = await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["target_epoch", "field_fingerprint", "phase"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(); + + await new SchemaRegistry(ctx.db).createField("articles", { + slug: "attachment", + label: "Attachment", + type: "file", + }); + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("advanced"); + const after = await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["target_epoch", "field_fingerprint", "phase", "scan_cursor", "source_cursor"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(); + + expect(Number(after.target_epoch)).toBeGreaterThan(Number(before.target_epoch)); + expect(after.field_fingerprint).not.toBe(before.field_fingerprint); + expect(after).toMatchObject({ phase: "scan", scan_cursor: null, source_cursor: null }); + }); + + it("defers without taking coverage from a manual repair owner", async () => { + const collection = await createCollection(ctx, "articles", true); + await sql` + INSERT INTO ${sql.ref("ec_articles")} (id, slug) + VALUES ('entry-1', 'entry-1') + `.execute(ctx.db); + await activateCollection(ctx, collection); + await processDueMediaUsageReconciliation(ctx.db); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "running", cursor: "manual-repair" }) + .where("collection_id", "=", collection.id) + .execute(); + + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("deferred"); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select(["status", "cursor"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ status: "running", cursor: "manual-repair" }); + }); + + it("retains the claimed epoch when initialization fails terminally", async () => { + const collection = await createCollection(ctx, "articles", true); + await activateCollection(ctx, collection); + const repository = new MediaUsageReconciliationRepository(ctx.db); + await repository.seedNextCandidate(); + const [candidate] = await repository.findDue(4); + if (!candidate) throw new Error("Expected reconciliation candidate"); + const claim = await repository.claim({ + collectionId: candidate.collectionId, + runToken: candidate.runToken, + leaseDurationSeconds: 60, + }); + if (!claim) throw new Error("Expected reconciliation claim"); + const epoch = await repository.beginRun(claim); + if (epoch === null) throw new Error("Expected a claimed coverage epoch"); + + await repository.recordFailure({ + collectionId: claim.collectionId, + runToken: claim.runToken, + leaseToken: claim.leaseToken, + errorCode: "MEDIA_USAGE_RECONCILIATION_FAILED", + retryDelaySeconds: 0, + terminal: true, + }); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["state", "target_epoch"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ state: "failed", target_epoch: epoch }); + }); +}); + +async function createCollection( + ctx: DialectTestContext, + slug: string, + withMediaField: boolean, +): Promise<{ id: string; slug: string }> { + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug, label: slug }); + if (withMediaField) { + await registry.createField(slug, { slug: "hero", label: "Hero", type: "image" }); + } + const collection = await registry.getCollection(slug); + if (!collection) throw new Error(`Expected ${slug} collection`); + return { id: collection.id, slug: collection.slug }; +} + +async function activateCollection( + ctx: DialectTestContext, + collection: { id: string; slug: string }, +): Promise { + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: collection.slug, + collection_id: collection.id, + status: "stale", + capture_state: "active", + reconciliation_required: 1, + }) + .onConflict((conflict) => + conflict.columns(["adapter_id", "scope_type", "scope_key"]).doUpdateSet({ + collection_id: collection.id, + status: "stale", + capture_state: "active", + reconciliation_required: 1, + }), + ) + .execute(); + await ctx.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); +} diff --git a/packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts b/packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts index 6a74ac5884..8366209922 100644 --- a/packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts +++ b/packages/core/tests/integration/database/media-usage-reconciliation-foundation.test.ts @@ -180,14 +180,19 @@ describeEachDialect("media usage reconciliation foundation", (dialect) => { ).resolves.toBe(true); expect(await repository.findDue(4)).toEqual([]); - expect(await repository.findFailed(4)).toEqual([ - expect.objectContaining({ - collectionId: candidate.collectionId, - state: "failed", - attemptCount: 2, - lastErrorCode: "MEDIA_USAGE_RECONCILIATION_INVALID_SOURCE", - }), - ]); + expect(await repository.findFailed(4)).toEqual([]); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select(["collection_id", "state", "attempt_count", "last_error_code"]) + .where("collection_id", "=", candidate.collectionId) + .executeTakeFirstOrThrow(), + ).toEqual({ + collection_id: candidate.collectionId, + state: "failed", + attempt_count: 2, + last_error_code: "MEDIA_USAGE_RECONCILIATION_INVALID_SOURCE", + }); }); it("reopens failed work only after a newer coverage epoch", async () => { From da3101d4c536a56af51c8a4dce0ac1fb1f55f611 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:05:42 +0100 Subject: [PATCH 15/20] feat(media): schedule automatic reconciliation --- .changeset/calm-files-reconcile.md | 6 + demos/cloudflare/src/worker.ts | 5 +- demos/cloudflare/wrangler.jsonc | 2 +- .../content/docs/deployment/cloudflare.mdx | 23 +++- packages/cloudflare/src/worker.ts | 52 ++++++-- .../cloudflare/tests/worker-scheduled.test.ts | 70 +++++++++++ packages/core/src/astro/middleware.ts | 37 +++++- packages/core/src/emdash-runtime.ts | 93 +++++++++++--- .../usage/collection-deletion-processor.ts | 1 + .../media/usage/reconciliation-processor.ts | 1 + .../core/src/media/usage/reconciliation.ts | 27 ++-- packages/core/src/plugins/scheduler/node.ts | 14 ++- packages/core/src/plugins/scheduler/types.ts | 2 + .../integration/database/migrations.test.ts | 1 + .../media-usage-scheduled-driver.test.ts | 119 +++++++++++++++++- templates/blog-cloudflare/src/worker.ts | 15 ++- templates/blog-cloudflare/wrangler.jsonc | 4 +- templates/marketing-cloudflare/src/worker.ts | 15 ++- templates/marketing-cloudflare/wrangler.jsonc | 4 +- templates/portfolio-cloudflare/src/worker.ts | 15 ++- templates/portfolio-cloudflare/wrangler.jsonc | 4 +- templates/starter-cloudflare/src/worker.ts | 15 ++- templates/starter-cloudflare/wrangler.jsonc | 4 +- 23 files changed, 450 insertions(+), 79 deletions(-) create mode 100644 .changeset/calm-files-reconcile.md create mode 100644 packages/cloudflare/tests/worker-scheduled.test.ts diff --git a/.changeset/calm-files-reconcile.md b/.changeset/calm-files-reconcile.md new file mode 100644 index 0000000000..9526041310 --- /dev/null +++ b/.changeset/calm-files-reconcile.md @@ -0,0 +1,6 @@ +--- +"emdash": patch +"@emdash-cms/cloudflare": patch +--- + +Adds bounded automatic Media Usage reconciliation and a dedicated Cloudflare maintenance lane for controlled activation. diff --git a/demos/cloudflare/src/worker.ts b/demos/cloudflare/src/worker.ts index d154c752d9..834b1c1b0a 100644 --- a/demos/cloudflare/src/worker.ts +++ b/demos/cloudflare/src/worker.ts @@ -13,5 +13,8 @@ export { PluginBridge }; export default { ...handler, - scheduled: createScheduledHandler(), + scheduled: createScheduledHandler({ + generalCron: "* * * * *", + mediaUsageCron: "*/2 * * * *", + }), } satisfies ExportedHandler; diff --git a/demos/cloudflare/wrangler.jsonc b/demos/cloudflare/wrangler.jsonc index 9e10c0f6bc..71bbfb825c 100644 --- a/demos/cloudflare/wrangler.jsonc +++ b/demos/cloudflare/wrangler.jsonc @@ -36,7 +36,7 @@ ], // Cron trigger drives the AI Search reindex queue flush. "triggers": { - "crons": ["* * * * *"], + "crons": ["* * * * *", "*/2 * * * *"], }, // Observability "observability": { diff --git a/docs/src/content/docs/deployment/cloudflare.mdx b/docs/src/content/docs/deployment/cloudflare.mdx index d3b9b0bb1b..19fe0e31bc 100644 --- a/docs/src/content/docs/deployment/cloudflare.mdx +++ b/docs/src/content/docs/deployment/cloudflare.mdx @@ -74,24 +74,37 @@ To change the schema or content model of a site that is already deployed, see [E ## Scheduled Publishing -On Cloudflare Workers, scheduled publishing, plugin cron, and maintenance tasks run from a Worker Cron Trigger. New Cloudflare templates include this setup automatically. If you are adding updating an existing project, export the EmDash Worker entry from `@emdash-cms/cloudflare/worker`: +On Cloudflare Workers, scheduled publishing, plugin cron, and maintenance tasks run from Worker Cron Triggers. New Cloudflare templates include both required schedules automatically. When updating an existing project, configure distinct general and Media Usage lanes: ```ts title="src/worker.ts" -export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; +import handler, { + createScheduledHandler, + PluginBridge, +} from "@emdash-cms/cloudflare/worker"; + +export { PluginBridge }; + +export default { + ...handler, + scheduled: createScheduledHandler({ + generalCron: "* * * * *", + mediaUsageCron: "*/2 * * * *", + }), +} satisfies ExportedHandler; ``` -Then add a Cron Trigger to `wrangler.jsonc`: +Then add both Cron Triggers to `wrangler.jsonc` using the same expressions: ```jsonc title="wrangler.jsonc" { "triggers": { - "crons": ["* * * * *"], + "crons": ["* * * * *", "*/2 * * * *"], }, } ``` ## Deploy diff --git a/packages/cloudflare/src/worker.ts b/packages/cloudflare/src/worker.ts index 12b258d6e9..8ae664194e 100644 --- a/packages/cloudflare/src/worker.ts +++ b/packages/cloudflare/src/worker.ts @@ -2,17 +2,17 @@ * Cloudflare Worker entry for EmDash sites. * * Wraps the Astro Cloudflare server handler with a `scheduled()` handler so a - * Cron Trigger drives scheduled publishing, plugin cron, and system cleanup - * without any request side effects. Re-exports the `PluginBridge` Durable - * Object so the sandbox binding resolves against the entry module. + * Cron Triggers drive general maintenance and the separately bounded Media + * Usage lane without request side effects. Re-exports the `PluginBridge` + * Durable Object so the sandbox binding resolves against the entry module. * - * Templates use this as their entire `src/worker.ts`: + * Existing sites can keep the default general-maintenance handler: * * export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; * - * and add a Cron Trigger to wrangler.jsonc: + * New sites configure distinct expressions through `createScheduledHandler`. * - * "triggers": { "crons": ["* * * * *"] } + * Configure one general expression and one distinct Media Usage expression. * * The `@astrojs/cloudflare/entrypoints/server` import is resolved by the * consuming app's Astro build (it pulls the build-time `virtual:astro:app` @@ -22,7 +22,7 @@ // @ts-ignore - resolved against the consuming app's Astro build import astroHandler from "@astrojs/cloudflare/entrypoints/server"; import { createApp } from "astro/app/entrypoint"; -import { runScheduledTasks } from "emdash/middleware"; +import { runScheduledMediaUsageTasks, runScheduledTasks } from "emdash/middleware"; export { PluginBridge } from "./sandbox/index.js"; @@ -48,13 +48,39 @@ async function invalidatePublishedTags( } /** - * Build a Worker `scheduled()` handler that runs EmDash's scheduled - * maintenance batch and purges edge-cache tags for anything it published. - * Exported for sites that assemble their own Worker object; most sites get it - * via this module's default export. + * Build a Worker `scheduled()` handler. Without options every expression runs + * the backwards-compatible general lane. Configured handlers dispatch exact, + * distinct expressions to general or Media Usage maintenance. */ -export function createScheduledHandler(): ExportedHandlerScheduledHandler { - return (_controller, _env, ctx) => { +export interface ScheduledHandlerOptions { + generalCron: string; + mediaUsageCron: string; +} + +export function createScheduledHandler( + options?: ScheduledHandlerOptions, +): ExportedHandlerScheduledHandler { + if (options) { + if (!options.generalCron.trim() || !options.mediaUsageCron.trim()) { + throw new Error("Configured scheduled-handler expressions must be non-empty"); + } + if (options.generalCron === options.mediaUsageCron) { + throw new Error("General and Media Usage Cron expressions must differ"); + } + } + return (controller, _env, ctx) => { + if (options && controller.cron === options.mediaUsageCron) { + ctx.waitUntil( + runScheduledMediaUsageTasks().catch((error: unknown) => { + console.error("[scheduled] Media Usage maintenance failed:", error); + }), + ); + return; + } + if (options && controller.cron !== options.generalCron) { + console.warn(`[scheduled] Ignoring unexpected Cron expression: ${controller.cron}`); + return; + } ctx.waitUntil( // Invalidate incrementally as each collection batch publishes, so a // scheduled() invocation killed mid-sweep (CPU/wall-clock limits on a diff --git a/packages/cloudflare/tests/worker-scheduled.test.ts b/packages/cloudflare/tests/worker-scheduled.test.ts new file mode 100644 index 0000000000..a880d84872 --- /dev/null +++ b/packages/cloudflare/tests/worker-scheduled.test.ts @@ -0,0 +1,70 @@ +import { beforeEach, expect, it, vi } from "vitest"; + +const scheduled = vi.hoisted(() => ({ + general: vi.fn(async () => ({ published: [] })), + mediaUsage: vi.fn(async () => ({ outcome: "inactive", taskClass: null, turn: null })), +})); + +vi.mock("@astrojs/cloudflare/entrypoints/server", () => ({ default: { fetch: vi.fn() } })); +vi.mock("astro/app/entrypoint", () => ({ + createApp: () => ({ pipeline: { getCacheProvider: async () => null } }), +})); +vi.mock("emdash/middleware", () => ({ + runScheduledTasks: scheduled.general, + runScheduledMediaUsageTasks: scheduled.mediaUsage, +})); +vi.mock("../src/sandbox/index.js", () => ({ PluginBridge: vi.fn() })); + +import { createScheduledHandler } from "../src/worker.js"; + +beforeEach(() => { + scheduled.general.mockClear(); + scheduled.mediaUsage.mockClear(); +}); + +it("keeps the unconfigured handler backwards-compatible", async () => { + await invoke(createScheduledHandler(), "custom expression"); + expect(scheduled.general).toHaveBeenCalledOnce(); + expect(scheduled.mediaUsage).not.toHaveBeenCalled(); +}); + +it("dispatches distinct configured cron expressions to exactly one lane", async () => { + const handler = createScheduledHandler({ + generalCron: "* * * * *", + mediaUsageCron: "*/2 * * * *", + }); + + await invoke(handler, "* * * * *"); + expect(scheduled.general).toHaveBeenCalledOnce(); + expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + + scheduled.general.mockClear(); + await invoke(handler, "*/2 * * * *"); + expect(scheduled.general).not.toHaveBeenCalled(); + expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); + + scheduled.mediaUsage.mockClear(); + await invoke(handler, "0 0 * * *"); + expect(scheduled.general).not.toHaveBeenCalled(); + expect(scheduled.mediaUsage).not.toHaveBeenCalled(); +}); + +it("rejects empty or aliased configured expressions", () => { + expect(() => + createScheduledHandler({ generalCron: "* * * * *", mediaUsageCron: "* * * * *" }), + ).toThrow(/must differ/i); + expect(() => createScheduledHandler({ generalCron: "", mediaUsageCron: "*/2 * * * *" })).toThrow( + /non-empty/i, + ); +}); + +async function invoke(handler: ExportedHandlerScheduledHandler, cron: string): Promise { + const pending: Promise[] = []; + const context = { + waitUntil(promise: Promise) { + pending.push(promise); + }, + }; + Reflect.apply(handler, undefined, [{ cron }, {}, context]); + await Promise.all(pending); +} diff --git a/packages/core/src/astro/middleware.ts b/packages/core/src/astro/middleware.ts index 49bd8e259a..eab7aa1245 100644 --- a/packages/core/src/astro/middleware.ts +++ b/packages/core/src/astro/middleware.ts @@ -39,9 +39,11 @@ import { flushRecorder, isInstrumentationEnabled, } from "../database/instrumentation.js"; +import { createDeferredTaskTracker } from "../deferred-tasks.js"; import { DB_INIT_DEADLINE_MS, EmDashRuntime, + type MediaUsageMaintenanceResult, type RuntimeDependencies, type SandboxedPluginEntry, type MediaProviderEntry, @@ -287,6 +289,12 @@ export async function runScheduledTasks( return runOutsideRequest(config, (runtime) => runtime.runScheduledTasks(options)); } +export async function runScheduledMediaUsageTasks(): Promise { + const config = getConfig(); + if (!config) return { outcome: "inactive", taskClass: null, turn: null }; + return runOutsideRequest(config, (runtime) => runtime.runScheduledMediaUsageTasks()); +} + /** * Run a callback against the EmDash runtime outside any HTTP request — from a * Cloudflare Queue consumer, a `scheduled()` handler, or any other @@ -352,8 +360,35 @@ async function runOutsideRequest( config: EmDashConfig, fn: (runtime: EmDashRuntime) => Promise, ): Promise { - const runtime = await getRuntime(config); + if (getRequestContext()) { + const runtime = await getRuntime(config); + return runOutsideRequestWithRuntime(config, runtime, fn); + } + const deferredTasks = createDeferredTaskTracker(() => {}); + const context = { + editMode: false, + metrics: createRequestMetrics(performance.now()), + deferredTasks, + }; + return runWithContext(context, async () => { + const runtime = await (async () => { + try { + return await getRuntime(config); + } finally { + deferredTasks.settle(); + await deferredTasks.settled; + } + })(); + return runOutsideRequestWithRuntime(config, runtime, fn); + }); +} + +async function runOutsideRequestWithRuntime( + config: EmDashConfig, + runtime: EmDashRuntime, + fn: (runtime: EmDashRuntime) => Promise, +): Promise { const scoped = createRequestScopedDb({ config: config.database?.config, isAuthenticated: false, diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index f77d00089b..4a51b8937c 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -9,7 +9,7 @@ import { Permissions } from "@emdash-cms/auth"; import type { Element } from "@emdash-cms/blocks"; -import { Kysely, type Dialect } from "kysely"; +import { Kysely, sql, type Dialect } from "kysely"; import virtualConfig from "virtual:emdash/config"; import { z } from "zod"; @@ -42,7 +42,10 @@ import { getI18nConfig } from "./i18n/config.js"; import { repairLocaleCasing } from "./i18n/repair-locale-casing.js"; import { normalizeMediaValue } from "./media/normalize.js"; import type { MediaProvider, MediaProviderCapabilities } from "./media/types.js"; -import { processDueMediaUsageCollectionDeletions } from "./media/usage/collection-deletion-processor.js"; +import { + MEDIA_USAGE_COLLECTION_DELETION_LIMITS, + processDueMediaUsageCollectionDeletions, +} from "./media/usage/collection-deletion-processor.js"; import { deleteContentMediaUsage, findNonTranslatableSiblingContentIds, @@ -50,6 +53,11 @@ import { refreshContentMediaUsageAfterWrite, } from "./media/usage/content-refresh.js"; import { + MEDIA_USAGE_RECONCILIATION_LIMITS, + processDueMediaUsageReconciliation, +} from "./media/usage/reconciliation-processor.js"; +import { + MEDIA_USAGE_WORK_PROCESSING_LIMITS, processDueMediaUsageWork, processMediaUsageWorkAfterWrite, } from "./media/usage/work-processor.js"; @@ -532,23 +540,60 @@ const marketplaceManifestCache = new Map< const sandboxedRouteMetaCache = new Map>(); let sandboxRunner: SandboxRunner | null = null; -async function runScheduledMediaUsageWork(db: Kysely): Promise { - try { - const result = await processDueMediaUsageWork(db); - if (result.candidateCount > 0) { - console.info("[media-usage:work] Scheduled processing", result); - } - } catch (error) { - console.error("[media-usage:work] Scheduled processing failed:", error); +export const MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS = Object.freeze({ + entryWork: MEDIA_USAGE_WORK_PROCESSING_LIMITS.ordinaryStatementsPerJob, + collectionDeletion: MEDIA_USAGE_COLLECTION_DELETION_LIMITS.maxQueriesPerTick, + reconciliation: MEDIA_USAGE_RECONCILIATION_LIMITS.maxQueriesPerTick, + maxClassQueries: Math.max( + MEDIA_USAGE_WORK_PROCESSING_LIMITS.ordinaryStatementsPerJob, + MEDIA_USAGE_COLLECTION_DELETION_LIMITS.maxQueriesPerTick, + MEDIA_USAGE_RECONCILIATION_LIMITS.maxQueriesPerTick, + ), + eventCeiling: 40, +}); + +export type MediaUsageMaintenanceTaskClass = + | "entry_work" + | "collection_deletion" + | "reconciliation"; + +export type MediaUsageMaintenanceResult = + | { outcome: "inactive" | "admission_closed"; taskClass: null; turn: null } + | { outcome: "processed"; taskClass: MediaUsageMaintenanceTaskClass; turn: number }; + +async function runScheduledMediaUsageLane( + db: Kysely, +): Promise { + const queriesAlreadySpent = getRequestContext()?.metrics?.dbCount ?? 0; + if ( + queriesAlreadySpent + 1 + MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.maxClassQueries > + MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.eventCeiling + ) { + return { outcome: "admission_closed", taskClass: null, turn: null }; } - try { - const result = await processDueMediaUsageCollectionDeletions(db); - if (result.candidateCount > 0) { - console.info("[media-usage:collection-deletion] Scheduled processing", result); - } - } catch (error) { - console.error("[media-usage:collection-deletion] Scheduled processing failed:", error); + + const activation = await db + .updateTable("_emdash_media_usage_activation") + .set({ + media_usage_maintenance_turn: sql`(media_usage_maintenance_turn + 1) % 3`, + }) + .where("task_key", "=", "incremental_capture") + .where("state", "=", "active") + .returning("media_usage_maintenance_turn") + .executeTakeFirst(); + if (!activation) return { outcome: "inactive", taskClass: null, turn: null }; + + const turn = activation.media_usage_maintenance_turn; + if (turn === 0) { + await processDueMediaUsageWork(db); + return { outcome: "processed", taskClass: "entry_work", turn }; } + if (turn === 1) { + await processDueMediaUsageCollectionDeletions(db); + return { outcome: "processed", taskClass: "collection_deletion", turn }; + } + await processDueMediaUsageReconciliation(db); + return { outcome: "processed", taskClass: "reconciliation", turn }; } /** @@ -743,7 +788,6 @@ export class EmDashRuntime { console.error("[cleanup] System cleanup failed:", error); } - await runScheduledMediaUsageWork(this.db); try { await this.syncPluginStorageIndexesOnce(); } catch (error) { @@ -756,6 +800,10 @@ export class EmDashRuntime { return { published }; } + async runScheduledMediaUsageTasks(): Promise { + return runScheduledMediaUsageLane(this.db); + } + /** * Materialize plugin-declared storage indexes, once per process. * @@ -1700,7 +1748,6 @@ export class EmDashRuntime { // by runSystemCleanup. This catches unexpected errors. console.error("[cleanup] System cleanup failed:", error); } - await runScheduledMediaUsageWork(db); try { await runtimeRef.current?.syncPluginStorageIndexesOnce(); } catch (error) { @@ -1709,6 +1756,14 @@ export class EmDashRuntime { // Never throws; no-op unless scheduled backups are enabled and due. await maybeRunScheduledBackup(db, storage ?? undefined); }); + scheduler.setMediaUsageMaintenance?.(async () => { + const runtime = runtimeRef.current; + if (runtime) { + await runtime.runScheduledMediaUsageTasks(); + } else { + await runScheduledMediaUsageLane(db); + } + }); // start() is void on the timer scheduler but the interface // allows a promise (alarm-backed schedulers); we don't block on it. diff --git a/packages/core/src/media/usage/collection-deletion-processor.ts b/packages/core/src/media/usage/collection-deletion-processor.ts index 3afe9fd3e3..f0adb7ce98 100644 --- a/packages/core/src/media/usage/collection-deletion-processor.ts +++ b/packages/core/src/media/usage/collection-deletion-processor.ts @@ -18,6 +18,7 @@ export const MEDIA_USAGE_COLLECTION_DELETION_LIMITS = Object.freeze({ maxAttempts: 5, retryBaseSeconds: 30, retryMaxSeconds: 15 * 60, + maxQueriesPerTick: 30, }); export interface MediaUsageCollectionDeletionTickResult { diff --git a/packages/core/src/media/usage/reconciliation-processor.ts b/packages/core/src/media/usage/reconciliation-processor.ts index 914c0ce586..3bf944f437 100644 --- a/packages/core/src/media/usage/reconciliation-processor.ts +++ b/packages/core/src/media/usage/reconciliation-processor.ts @@ -21,6 +21,7 @@ export const MEDIA_USAGE_RECONCILIATION_LIMITS = Object.freeze({ retryBaseSeconds: 30, retryMaxSeconds: 15 * 60, retryJitterRatio: 0.25, + maxQueriesPerTick: 20, }); export type MediaUsageReconciliationOutcome = diff --git a/packages/core/src/media/usage/reconciliation.ts b/packages/core/src/media/usage/reconciliation.ts index 2a16de217f..2fc7bcddcc 100644 --- a/packages/core/src/media/usage/reconciliation.ts +++ b/packages/core/src/media/usage/reconciliation.ts @@ -164,7 +164,7 @@ export class MediaUsageReconciliationRepository { WHERE 1 = 1 ${lowerBound} ${upperBound} - AND ${this.liveClaimExistsSql(reconciliation as MediaUsageReconciliationClaim)} + AND ${this.liveClaimExistsSql(reconciliation)} ORDER BY content.id ASC LIMIT ${limit} `.execute(this.db); @@ -376,13 +376,8 @@ export class MediaUsageReconciliationRepository { .where("source.source_type", "=", "content") .where("source.collection_id", "=", reconciliation.collectionId) .where("source.identity_version", "=", 1) - .where(this.liveClaimExists(reconciliation as MediaUsageReconciliationClaim)) - .where( - this.statusOwnsRun( - reconciliation as MediaUsageReconciliationClaim, - reconciliation.targetEpoch, - ), - ); + .where(this.liveClaimExists(reconciliation)) + .where(this.statusOwnsRun(reconciliation, reconciliation.targetEpoch)); if (reconciliation.sourceCursor) { query = query.where("source.source_key", ">", reconciliation.sourceCursor); } @@ -623,11 +618,21 @@ export class MediaUsageReconciliationRepository { return result.rows.length === 1; } - private liveClaimExists(claim: MediaUsageReconciliationClaim): RawBuilder { + private liveClaimExists( + claim: Pick< + MediaUsageReconciliationRecord, + "collectionId" | "collectionSlug" | "runToken" | "leaseToken" + >, + ): RawBuilder { return this.liveClaimExistsSql(claim); } - private liveClaimExistsSql(claim: MediaUsageReconciliationClaim): RawBuilder { + private liveClaimExistsSql( + claim: Pick< + MediaUsageReconciliationRecord, + "collectionId" | "collectionSlug" | "runToken" | "leaseToken" + >, + ): RawBuilder { return sql`EXISTS ( SELECT 1 FROM _emdash_media_usage_reconciliations AS reconciliation @@ -653,7 +658,7 @@ export class MediaUsageReconciliationRepository { } private statusOwnsRun( - claim: MediaUsageReconciliationClaim, + claim: Pick, targetEpoch: number | string, ): RawBuilder { return sql`EXISTS ( diff --git a/packages/core/src/plugins/scheduler/node.ts b/packages/core/src/plugins/scheduler/node.ts index c4486c4b3d..32e63d9ded 100644 --- a/packages/core/src/plugins/scheduler/node.ts +++ b/packages/core/src/plugins/scheduler/node.ts @@ -29,6 +29,7 @@ export class NodeCronScheduler implements CronScheduler { private timer: ReturnType | null = null; private running = false; private systemCleanup: SystemCleanupFn | null = null; + private mediaUsageMaintenance: SystemCleanupFn | null = null; constructor(private executor: CronExecutor) {} @@ -36,6 +37,10 @@ export class NodeCronScheduler implements CronScheduler { this.systemCleanup = fn; } + setMediaUsageMaintenance(fn: SystemCleanupFn): void { + this.mediaUsageMaintenance = fn; + } + start(): void { this.running = true; this.arm(); @@ -112,12 +117,19 @@ export class NodeCronScheduler implements CronScheduler { } void Promise.allSettled(tasks) - .then((results) => { + .then(async (results) => { for (const r of results) { if (r.status === "rejected") { console.error("[cron:node] Tick task failed:", r.reason); } } + if (this.mediaUsageMaintenance) { + try { + await this.mediaUsageMaintenance(); + } catch (error) { + console.error("[cron:node] Media Usage maintenance failed:", error); + } + } return undefined; }) .finally(() => { diff --git a/packages/core/src/plugins/scheduler/types.ts b/packages/core/src/plugins/scheduler/types.ts index 511f9b8fcc..f4ee58afbb 100644 --- a/packages/core/src/plugins/scheduler/types.ts +++ b/packages/core/src/plugins/scheduler/types.ts @@ -18,6 +18,8 @@ export interface CronScheduler { reschedule(): void; /** Register a system cleanup function to run alongside each tick. */ setSystemCleanup(fn: SystemCleanupFn): void; + /** Register bounded Media Usage maintenance to run after the general tick settles. */ + setMediaUsageMaintenance?(fn: SystemCleanupFn): void; } /** diff --git a/packages/core/tests/integration/database/migrations.test.ts b/packages/core/tests/integration/database/migrations.test.ts index 239bafe558..852f0c4177 100644 --- a/packages/core/tests/integration/database/migrations.test.ts +++ b/packages/core/tests/integration/database/migrations.test.ts @@ -156,6 +156,7 @@ describe("Database Migrations (Integration)", () => { "063_media_usage_incremental_work", "064_fts_plain_text", "065_media_usage_collection_deletion", + "066_media_usage_reconciliation", ]; await db.deleteFrom("_emdash_migrations").where("name", "in", trailing).execute(); diff --git a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts index fdf2b50032..c37d3bcc6f 100644 --- a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts +++ b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts @@ -5,9 +5,14 @@ import { sql, SqliteDialect } from "kysely"; import { afterEach, describe, expect, it } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; -import { EmDashRuntime, type RuntimeDependencies } from "../../../src/emdash-runtime.js"; +import { + EmDashRuntime, + MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS, + type RuntimeDependencies, +} from "../../../src/emdash-runtime.js"; import { installMediaUsageCaptureTriggers } from "../../../src/media/usage/capture-triggers.js"; import type { CronScheduler, SystemCleanupFn } from "../../../src/plugins/scheduler/types.js"; +import { createRequestMetrics, runWithContext } from "../../../src/request-context.js"; describe("media usage scheduled drivers", () => { let runtime: EmDashRuntime | null = null; @@ -17,12 +22,17 @@ describe("media usage scheduled drivers", () => { runtime = null; }); - it("drains bounded work from the Cloudflare scheduled entry point", async () => { + it("keeps general maintenance unchanged and drains work from the dedicated lane", async () => { runtime = await EmDashRuntime.create(createDeps(null)); const fixture = await activateCollection(runtime, "cloudflare_posts"); await insertEntry(runtime, fixture.tableName, "entry-1"); await runtime.runScheduledTasks(); + expect(await countWork(runtime)).toBe(1); + await expect(runtime.runScheduledMediaUsageTasks()).resolves.toMatchObject({ + outcome: "processed", + taskClass: "entry_work", + }); expect(await countWork(runtime)).toBe(0); expect( @@ -53,7 +63,8 @@ describe("media usage scheduled drivers", () => { const fixture = await activateCollection(runtime, "cloudflare_delete"); await runtime.schemaRegistry.deleteCollection("cloudflare_delete", { force: true }); - await runtime.runScheduledTasks(); + await runtime.runScheduledMediaUsageTasks(); + await runtime.runScheduledMediaUsageTasks(); expect(await deletionPhase(runtime, fixture.collectionId)).toBe("sources"); }); @@ -64,6 +75,7 @@ describe("media usage scheduled drivers", () => { const fixture = await activateCollection(runtime, "node_delete"); await runtime.schemaRegistry.deleteCollection("node_delete", { force: true }); + await scheduler.runMaintenance(); await scheduler.runMaintenance(); expect(await deletionPhase(runtime, fixture.collectionId)).toBe("sources"); @@ -89,14 +101,104 @@ describe("media usage scheduled drivers", () => { ), ).not.toBeNull(); }); + + it("persists a fair entry, deletion, reconciliation turn sequence", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const fixture = await activateCollection(runtime, "fair_posts"); + await runtime.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "stale", reconciliation_required: 1 }) + .where("collection_id", "=", fixture.collectionId) + .execute(); + + const classes = []; + for (let index = 0; index < 3; index++) { + const result = await runtime.runScheduledMediaUsageTasks(); + classes.push(result.taskClass); + } + expect(classes).toEqual(["entry_work", "collection_deletion", "reconciliation"]); + expect( + await runtime.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("collection_id") + .where("collection_id", "=", fixture.collectionId) + .executeTakeFirst(), + ).toBeDefined(); + }); + + it("does not advance the turn or spend class queries before activation", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const before = await maintenanceTurn(runtime); + await expect(runtime.runScheduledMediaUsageTasks()).resolves.toEqual({ + outcome: "inactive", + taskClass: null, + turn: null, + }); + expect(await maintenanceTurn(runtime)).toBe(before); + }); + + it("reserves ten queries of headroom before changing the persisted turn", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + await runtime.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + const before = await maintenanceTurn(runtime); + const metrics = createRequestMetrics(performance.now()); + metrics.dbCount = + MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.eventCeiling - + MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.maxClassQueries; + + const result = await runWithContext({ editMode: false, metrics }, () => + runtime!.runScheduledMediaUsageTasks(), + ); + expect(result).toEqual({ outcome: "admission_closed", taskClass: null, turn: null }); + expect(await maintenanceTurn(runtime)).toBe(before); + }); + + it("measures every mutation class within its exported event reservation", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const work = await activateCollection(runtime, "measure_work"); + await insertEntry(runtime, work.tableName, "entry-1"); + await activateCollection(runtime, "measure_delete"); + await runtime.schemaRegistry.deleteCollection("measure_delete", { force: true }); + const reconciliation = await activateCollection(runtime, "measure_reconcile"); + await runtime.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "stale", reconciliation_required: 1 }) + .where("collection_id", "=", reconciliation.collectionId) + .execute(); + + const expected = [ + ["entry_work", MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.entryWork], + ["collection_deletion", MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.collectionDeletion], + ["reconciliation", MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.reconciliation], + ] as const; + for (const [taskClass, reservation] of expected) { + const metrics = createRequestMetrics(performance.now()); + const result = await runWithContext({ editMode: false, metrics }, () => + runtime!.runScheduledMediaUsageTasks(), + ); + expect(result.taskClass).toBe(taskClass); + expect(metrics.dbCount).toBeLessThanOrEqual(1 + reservation); + expect(metrics.dbCount).toBeLessThanOrEqual( + MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.eventCeiling, + ); + } + }); }); class CapturingScheduler implements CronScheduler { private maintenance: SystemCleanupFn | null = null; + private mediaUsageMaintenance: SystemCleanupFn | null = null; setSystemCleanup(fn: SystemCleanupFn): void { this.maintenance = fn; } + setMediaUsageMaintenance(fn: SystemCleanupFn): void { + this.mediaUsageMaintenance = fn; + } start(): void {} stop(): void {} @@ -105,6 +207,8 @@ class CapturingScheduler implements CronScheduler { async runMaintenance(): Promise { if (!this.maintenance) throw new Error("Expected Node maintenance callback"); await this.maintenance(); + if (!this.mediaUsageMaintenance) throw new Error("Expected Media Usage maintenance callback"); + await this.mediaUsageMaintenance(); } } @@ -195,6 +299,15 @@ async function deletionPhase(runtime: EmDashRuntime, collectionId: string): Prom return row?.phase ?? null; } +async function maintenanceTurn(runtime: EmDashRuntime): Promise { + const row = await runtime.db + .selectFrom("_emdash_media_usage_activation") + .select("media_usage_maintenance_turn") + .where("task_key", "=", "incremental_capture") + .executeTakeFirstOrThrow(); + return row.media_usage_maintenance_turn; +} + function canonicalSourceKey(collectionId: string, contentId: string): string { return `content:${collectionId}:${contentId}:columns`; } diff --git a/templates/blog-cloudflare/src/worker.ts b/templates/blog-cloudflare/src/worker.ts index df373a0194..ad5efeee42 100644 --- a/templates/blog-cloudflare/src/worker.ts +++ b/templates/blog-cloudflare/src/worker.ts @@ -1,4 +1,11 @@ -// Worker entry: Astro's fetch handler plus EmDash's scheduled() handler, which -// the Cron Trigger in wrangler.jsonc drives. PluginBridge is the sandbox -// Durable Object, re-exported here so its binding resolves. -export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; +import handler, { createScheduledHandler, PluginBridge } from "@emdash-cms/cloudflare/worker"; + +export { PluginBridge }; + +export default { + ...handler, + scheduled: createScheduledHandler({ + generalCron: "* * * * *", + mediaUsageCron: "*/2 * * * *", + }), +}; diff --git a/templates/blog-cloudflare/wrangler.jsonc b/templates/blog-cloudflare/wrangler.jsonc index f6baeeca71..23632b0e98 100644 --- a/templates/blog-cloudflare/wrangler.jsonc +++ b/templates/blog-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // Drives scheduled publishing, plugin cron, and maintenance (see src/worker.ts) + // General maintenance plus the bounded Media Usage lane (see src/worker.ts) "triggers": { - "crons": ["* * * * *"], + "crons": ["* * * * *", "*/2 * * * *"], }, } diff --git a/templates/marketing-cloudflare/src/worker.ts b/templates/marketing-cloudflare/src/worker.ts index df373a0194..ad5efeee42 100644 --- a/templates/marketing-cloudflare/src/worker.ts +++ b/templates/marketing-cloudflare/src/worker.ts @@ -1,4 +1,11 @@ -// Worker entry: Astro's fetch handler plus EmDash's scheduled() handler, which -// the Cron Trigger in wrangler.jsonc drives. PluginBridge is the sandbox -// Durable Object, re-exported here so its binding resolves. -export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; +import handler, { createScheduledHandler, PluginBridge } from "@emdash-cms/cloudflare/worker"; + +export { PluginBridge }; + +export default { + ...handler, + scheduled: createScheduledHandler({ + generalCron: "* * * * *", + mediaUsageCron: "*/2 * * * *", + }), +}; diff --git a/templates/marketing-cloudflare/wrangler.jsonc b/templates/marketing-cloudflare/wrangler.jsonc index c020df0791..e5e23e764d 100644 --- a/templates/marketing-cloudflare/wrangler.jsonc +++ b/templates/marketing-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // Drives scheduled publishing, plugin cron, and maintenance (see src/worker.ts) + // General maintenance plus the bounded Media Usage lane (see src/worker.ts) "triggers": { - "crons": ["* * * * *"], + "crons": ["* * * * *", "*/2 * * * *"], }, } diff --git a/templates/portfolio-cloudflare/src/worker.ts b/templates/portfolio-cloudflare/src/worker.ts index df373a0194..ad5efeee42 100644 --- a/templates/portfolio-cloudflare/src/worker.ts +++ b/templates/portfolio-cloudflare/src/worker.ts @@ -1,4 +1,11 @@ -// Worker entry: Astro's fetch handler plus EmDash's scheduled() handler, which -// the Cron Trigger in wrangler.jsonc drives. PluginBridge is the sandbox -// Durable Object, re-exported here so its binding resolves. -export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; +import handler, { createScheduledHandler, PluginBridge } from "@emdash-cms/cloudflare/worker"; + +export { PluginBridge }; + +export default { + ...handler, + scheduled: createScheduledHandler({ + generalCron: "* * * * *", + mediaUsageCron: "*/2 * * * *", + }), +}; diff --git a/templates/portfolio-cloudflare/wrangler.jsonc b/templates/portfolio-cloudflare/wrangler.jsonc index 62ee430e60..792c1fda4d 100644 --- a/templates/portfolio-cloudflare/wrangler.jsonc +++ b/templates/portfolio-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // Drives scheduled publishing, plugin cron, and maintenance (see src/worker.ts) + // General maintenance plus the bounded Media Usage lane (see src/worker.ts) "triggers": { - "crons": ["* * * * *"], + "crons": ["* * * * *", "*/2 * * * *"], }, } diff --git a/templates/starter-cloudflare/src/worker.ts b/templates/starter-cloudflare/src/worker.ts index df373a0194..ad5efeee42 100644 --- a/templates/starter-cloudflare/src/worker.ts +++ b/templates/starter-cloudflare/src/worker.ts @@ -1,4 +1,11 @@ -// Worker entry: Astro's fetch handler plus EmDash's scheduled() handler, which -// the Cron Trigger in wrangler.jsonc drives. PluginBridge is the sandbox -// Durable Object, re-exported here so its binding resolves. -export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; +import handler, { createScheduledHandler, PluginBridge } from "@emdash-cms/cloudflare/worker"; + +export { PluginBridge }; + +export default { + ...handler, + scheduled: createScheduledHandler({ + generalCron: "* * * * *", + mediaUsageCron: "*/2 * * * *", + }), +}; diff --git a/templates/starter-cloudflare/wrangler.jsonc b/templates/starter-cloudflare/wrangler.jsonc index f6baeeca71..23632b0e98 100644 --- a/templates/starter-cloudflare/wrangler.jsonc +++ b/templates/starter-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // Drives scheduled publishing, plugin cron, and maintenance (see src/worker.ts) + // General maintenance plus the bounded Media Usage lane (see src/worker.ts) "triggers": { - "crons": ["* * * * *"], + "crons": ["* * * * *", "*/2 * * * *"], }, } From 5ce7f25e71e5aaa0a7a7f5e228cd7188390877c8 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:57:57 +0100 Subject: [PATCH 16/20] fix(media): preserve scheduled maintenance compatibility --- packages/cloudflare/src/worker.ts | 7 ++++ .../cloudflare/tests/worker-scheduled.test.ts | 2 +- packages/core/src/emdash-runtime.ts | 22 +++++++++---- .../media-usage-scheduled-driver.test.ts | 33 +++++++++++++++++++ 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/packages/cloudflare/src/worker.ts b/packages/cloudflare/src/worker.ts index 8ae664194e..a268af67d0 100644 --- a/packages/cloudflare/src/worker.ts +++ b/packages/cloudflare/src/worker.ts @@ -81,6 +81,13 @@ export function createScheduledHandler( console.warn(`[scheduled] Ignoring unexpected Cron expression: ${controller.cron}`); return; } + if (!options) { + ctx.waitUntil( + runScheduledMediaUsageTasks().catch((error: unknown) => { + console.error("[scheduled] Media Usage maintenance failed:", error); + }), + ); + } ctx.waitUntil( // Invalidate incrementally as each collection batch publishes, so a // scheduled() invocation killed mid-sweep (CPU/wall-clock limits on a diff --git a/packages/cloudflare/tests/worker-scheduled.test.ts b/packages/cloudflare/tests/worker-scheduled.test.ts index a880d84872..0aff339620 100644 --- a/packages/cloudflare/tests/worker-scheduled.test.ts +++ b/packages/cloudflare/tests/worker-scheduled.test.ts @@ -25,7 +25,7 @@ beforeEach(() => { it("keeps the unconfigured handler backwards-compatible", async () => { await invoke(createScheduledHandler(), "custom expression"); expect(scheduled.general).toHaveBeenCalledOnce(); - expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); }); it("dispatches distinct configured cron expressions to exactly one lane", async () => { diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index 4a51b8937c..7c754e2818 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -1723,6 +1723,14 @@ export class EmDashRuntime { if (deps.createScheduler) { const scheduler = deps.createScheduler(cronExecutor); cronScheduler = scheduler; + const runMediaUsageMaintenance = async () => { + const runtime = runtimeRef.current; + if (runtime) { + await runtime.runScheduledMediaUsageTasks(); + } else { + await runScheduledMediaUsageLane(db); + } + }; // Run scheduled publishing and system cleanup alongside each tick. // Pass storage so cleanupPendingUploads can delete orphaned files. @@ -1755,15 +1763,15 @@ export class EmDashRuntime { } // Never throws; no-op unless scheduled backups are enabled and due. await maybeRunScheduledBackup(db, storage ?? undefined); - }); - scheduler.setMediaUsageMaintenance?.(async () => { - const runtime = runtimeRef.current; - if (runtime) { - await runtime.runScheduledMediaUsageTasks(); - } else { - await runScheduledMediaUsageLane(db); + if (!scheduler.setMediaUsageMaintenance) { + try { + await runMediaUsageMaintenance(); + } catch (error) { + console.error("[media-usage] Scheduled maintenance failed:", error); + } } }); + scheduler.setMediaUsageMaintenance?.(runMediaUsageMaintenance); // start() is void on the timer scheduler but the interface // allows a promise (alarm-backed schedulers); we don't block on it. diff --git a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts index c37d3bcc6f..c5445fdda0 100644 --- a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts +++ b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts @@ -58,6 +58,22 @@ describe("media usage scheduled drivers", () => { ).not.toBeNull(); }); + it("drains bounded work through a legacy Node scheduler cleanup callback", async () => { + const scheduler = new LegacyCapturingScheduler(); + runtime = await EmDashRuntime.create(createDeps(() => scheduler)); + const fixture = await activateCollection(runtime, "legacy_node_posts"); + await insertEntry(runtime, fixture.tableName, "entry-1"); + + await scheduler.runMaintenance(); + + expect(await countWork(runtime)).toBe(0); + expect( + await new MediaUsageRepository(runtime.db).findSource( + canonicalSourceKey(fixture.collectionId, "entry-1"), + ), + ).not.toBeNull(); + }); + it("advances bounded collection deletion from the Cloudflare scheduled entry point", async () => { runtime = await EmDashRuntime.create(createDeps(null)); const fixture = await activateCollection(runtime, "cloudflare_delete"); @@ -212,6 +228,23 @@ class CapturingScheduler implements CronScheduler { } } +class LegacyCapturingScheduler implements CronScheduler { + private maintenance: SystemCleanupFn | null = null; + + setSystemCleanup(fn: SystemCleanupFn): void { + this.maintenance = fn; + } + + start(): void {} + stop(): void {} + reschedule(): void {} + + async runMaintenance(): Promise { + if (!this.maintenance) throw new Error("Expected Node maintenance callback"); + await this.maintenance(); + } +} + function createDeps(createScheduler: RuntimeDependencies["createScheduler"]): RuntimeDependencies { return { config: { From 8c8c0e0cca9740308c15e722075a54b52e88eca0 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:29:32 +0100 Subject: [PATCH 17/20] docs: clarify automatic reconciliation changeset --- .changeset/calm-files-reconcile.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/calm-files-reconcile.md b/.changeset/calm-files-reconcile.md index 9526041310..f1b97eb9c7 100644 --- a/.changeset/calm-files-reconcile.md +++ b/.changeset/calm-files-reconcile.md @@ -3,4 +3,4 @@ "@emdash-cms/cloudflare": patch --- -Adds bounded automatic Media Usage reconciliation and a dedicated Cloudflare maintenance lane for controlled activation. +Adds automatic, resumable background indexing so Media Usage can safely catch up on existing content without processing the whole site at once. From 1a96dda60fb3fa6a8dcb28988335b51bc9b4c885 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:39:28 +0100 Subject: [PATCH 18/20] fix(core): guard null media usage revisions on postgres --- .../src/database/repositories/media-usage.ts | 9 ++++---- ...sage-collection-deletion-lifecycle.test.ts | 8 +++++++ .../media-usage-reconciliation-scan.test.ts | 22 +++++++++++-------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index daa9538407..49ccb3f6f4 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -2600,16 +2600,17 @@ export class MediaUsageRepository { : null; if (!revisionColumn) return sql`1 = 0`; const revision = sql.ref(`content.${revisionColumn}`); + const revisionMatches = + row.revision_id === null + ? sql`${revision} IS NULL` + : sql`${revision} = ${row.revision_id}`; return sql`EXISTS ( SELECT 1 FROM ${sql.ref(tableName)} AS content WHERE content.id = ${row.content_id} AND content.version = ${row.source_version} AND content.updated_at = ${row.source_updated_at} - AND ( - ${revision} = ${row.revision_id} - OR (${revision} IS NULL AND ${row.revision_id} IS NULL) - ) + AND ${revisionMatches} )`; } diff --git a/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts b/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts index 5e032093fa..285ad93c6c 100644 --- a/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts +++ b/packages/core/tests/integration/database/media-usage-collection-deletion-lifecycle.test.ts @@ -339,6 +339,11 @@ describeEachDialect("media usage activated collection deletion", (dialect) => { slug: "projecting", label: "Projecting", }); + const sourceUpdatedAt = "2026-08-12T10:00:00.000Z"; + await sql` + INSERT INTO ${sql.ref("ec_projecting")} (id, slug, version, updated_at) + VALUES ('entry-1', 'entry-1', 1, ${sourceUpdatedAt}) + `.execute(ctx.db); const advisoryKey = 8642031; await sql .raw(` @@ -386,6 +391,9 @@ describeEachDialect("media usage activated collection deletion", (dialect) => { collectionSlug: collection.slug, contentId: "entry-1", sourceVariant: "columns", + revisionId: null, + sourceVersion: 1, + sourceUpdatedAt, identityVersion: 1, }, [ diff --git a/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts b/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts index edb528fd67..f5b98de0cf 100644 --- a/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts +++ b/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts @@ -55,19 +55,18 @@ describeEachDialect("media usage reconciliation scan", (dialect) => { .select(["target_epoch", "field_fingerprint", "scan_upper_id", "scan_cursor"]) .where("collection_id", "=", collection.id) .executeTakeFirstOrThrow(); + expect(Number(coordinator.target_epoch)).toBe(1); expect(coordinator).toMatchObject({ - target_epoch: 1, scan_upper_id: "entry-050", scan_cursor: "entry-049", }); expect(coordinator.field_fingerprint).toMatch(/^media-usage-fields:v1:sha256:[a-f0-9]{64}$/); - expect( - await ctx.db - .selectFrom("_emdash_media_usage_work") - .select((eb) => eb.fn.countAll().as("count")) - .where("collection_id", "=", collection.id) - .executeTakeFirstOrThrow(), - ).toEqual({ count: 50 }); + const workCount = await ctx.db + .selectFrom("_emdash_media_usage_work") + .select((eb) => eb.fn.countAll().as("count")) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(); + expect(Number(workCount.count)).toBe(50); expect(await workState(ctx, collection.id, "entry-001")).toMatchObject({ change_epoch: 1, work_version: 2, @@ -302,12 +301,17 @@ function canonicalSource( } async function workState(ctx: DialectTestContext, collectionId: string, contentId: string) { - return ctx.db + const row = await ctx.db .selectFrom("_emdash_media_usage_work") .select(["change_epoch", "work_version", "state", "last_error_code"]) .where("collection_id", "=", collectionId) .where("content_id", "=", contentId) .executeTakeFirstOrThrow(); + return { + ...row, + change_epoch: Number(row.change_epoch), + work_version: Number(row.work_version), + }; } async function workVersions(ctx: DialectTestContext, collectionId: string) { From 7a3ff56a803a399f9b2a413ce9686e611ea3efd8 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:45:28 +0100 Subject: [PATCH 19/20] fix(cloudflare): default scheduled cron routing --- demos/cloudflare/src/worker.ts | 5 +- .../content/docs/deployment/cloudflare.mdx | 9 ++-- packages/cloudflare/src/worker.ts | 47 +++++++++---------- .../cloudflare/tests/worker-scheduled.test.ts | 41 +++++++++++++++- templates/blog-cloudflare/src/worker.ts | 5 +- templates/marketing-cloudflare/src/worker.ts | 5 +- templates/portfolio-cloudflare/src/worker.ts | 5 +- templates/starter-cloudflare/src/worker.ts | 5 +- 8 files changed, 70 insertions(+), 52 deletions(-) diff --git a/demos/cloudflare/src/worker.ts b/demos/cloudflare/src/worker.ts index 834b1c1b0a..d154c752d9 100644 --- a/demos/cloudflare/src/worker.ts +++ b/demos/cloudflare/src/worker.ts @@ -13,8 +13,5 @@ export { PluginBridge }; export default { ...handler, - scheduled: createScheduledHandler({ - generalCron: "* * * * *", - mediaUsageCron: "*/2 * * * *", - }), + scheduled: createScheduledHandler(), } satisfies ExportedHandler; diff --git a/docs/src/content/docs/deployment/cloudflare.mdx b/docs/src/content/docs/deployment/cloudflare.mdx index 19fe0e31bc..f08ac5fb4f 100644 --- a/docs/src/content/docs/deployment/cloudflare.mdx +++ b/docs/src/content/docs/deployment/cloudflare.mdx @@ -86,14 +86,11 @@ export { PluginBridge }; export default { ...handler, - scheduled: createScheduledHandler({ - generalCron: "* * * * *", - mediaUsageCron: "*/2 * * * *", - }), + scheduled: createScheduledHandler(), } satisfies ExportedHandler; ``` -Then add both Cron Triggers to `wrangler.jsonc` using the same expressions: +By default, `*/2 * * * *` runs Media Usage maintenance and every other expression runs general maintenance. Then add both Cron Triggers to `wrangler.jsonc`: ```jsonc title="wrangler.jsonc" { @@ -103,6 +100,8 @@ Then add both Cron Triggers to `wrangler.jsonc` using the same expressions: } ``` +To use different schedules, set the corresponding `generalCron` or `mediaUsageCron` option in `createScheduledHandler()` and use the same expression in `wrangler.jsonc`. + diff --git a/packages/cloudflare/src/worker.ts b/packages/cloudflare/src/worker.ts index a268af67d0..91525e5f73 100644 --- a/packages/cloudflare/src/worker.ts +++ b/packages/cloudflare/src/worker.ts @@ -6,13 +6,13 @@ * Usage lane without request side effects. Re-exports the `PluginBridge` * Durable Object so the sandbox binding resolves against the entry module. * - * Existing sites can keep the default general-maintenance handler: + * Existing sites can keep the default scheduled handler: * * export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; * - * New sites configure distinct expressions through `createScheduledHandler`. - * - * Configure one general expression and one distinct Media Usage expression. + * By default the every-two-minutes expression runs Media Usage maintenance + * and every other expression runs general maintenance. Sites only pass + * options when changing either expression. * * The `@astrojs/cloudflare/entrypoints/server` import is resolved by the * consuming app's Astro build (it pulls the build-time `virtual:astro:app` @@ -48,28 +48,32 @@ async function invalidatePublishedTags( } /** - * Build a Worker `scheduled()` handler. Without options every expression runs - * the backwards-compatible general lane. Configured handlers dispatch exact, - * distinct expressions to general or Media Usage maintenance. + * Build a Worker `scheduled()` handler. By default the every-two-minutes + * expression runs Media Usage maintenance and every other expression runs + * general maintenance. Configuring a general expression changes that lane + * from catch-all to exact. */ export interface ScheduledHandlerOptions { - generalCron: string; - mediaUsageCron: string; + generalCron?: string; + mediaUsageCron?: string; } +const DEFAULT_MEDIA_USAGE_CRON = "*/2 * * * *"; + export function createScheduledHandler( options?: ScheduledHandlerOptions, ): ExportedHandlerScheduledHandler { - if (options) { - if (!options.generalCron.trim() || !options.mediaUsageCron.trim()) { - throw new Error("Configured scheduled-handler expressions must be non-empty"); - } - if (options.generalCron === options.mediaUsageCron) { - throw new Error("General and Media Usage Cron expressions must differ"); - } + const generalCron = options?.generalCron?.trim(); + const mediaUsageCron = options?.mediaUsageCron?.trim() ?? DEFAULT_MEDIA_USAGE_CRON; + if ((options?.generalCron !== undefined && !generalCron) || !mediaUsageCron) { + throw new Error("Configured scheduled-handler expressions must be non-empty"); + } + if (generalCron === mediaUsageCron) { + throw new Error("General and Media Usage Cron expressions must differ"); } + return (controller, _env, ctx) => { - if (options && controller.cron === options.mediaUsageCron) { + if (controller.cron === mediaUsageCron) { ctx.waitUntil( runScheduledMediaUsageTasks().catch((error: unknown) => { console.error("[scheduled] Media Usage maintenance failed:", error); @@ -77,17 +81,10 @@ export function createScheduledHandler( ); return; } - if (options && controller.cron !== options.generalCron) { + if (generalCron !== undefined && controller.cron !== generalCron) { console.warn(`[scheduled] Ignoring unexpected Cron expression: ${controller.cron}`); return; } - if (!options) { - ctx.waitUntil( - runScheduledMediaUsageTasks().catch((error: unknown) => { - console.error("[scheduled] Media Usage maintenance failed:", error); - }), - ); - } ctx.waitUntil( // Invalidate incrementally as each collection batch publishes, so a // scheduled() invocation killed mid-sweep (CPU/wall-clock limits on a diff --git a/packages/cloudflare/tests/worker-scheduled.test.ts b/packages/cloudflare/tests/worker-scheduled.test.ts index 0aff339620..034844f207 100644 --- a/packages/cloudflare/tests/worker-scheduled.test.ts +++ b/packages/cloudflare/tests/worker-scheduled.test.ts @@ -22,9 +22,16 @@ beforeEach(() => { scheduled.mediaUsage.mockClear(); }); -it("keeps the unconfigured handler backwards-compatible", async () => { - await invoke(createScheduledHandler(), "custom expression"); +it("uses the default Media Usage expression and treats every other expression as general", async () => { + const handler = createScheduledHandler(); + + await invoke(handler, "custom expression"); expect(scheduled.general).toHaveBeenCalledOnce(); + expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + + scheduled.general.mockClear(); + await invoke(handler, "*/2 * * * *"); + expect(scheduled.general).not.toHaveBeenCalled(); expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); }); @@ -49,6 +56,34 @@ it("dispatches distinct configured cron expressions to exactly one lane", async expect(scheduled.mediaUsage).not.toHaveBeenCalled(); }); +it("allows either default expression to be overridden independently", async () => { + const customMedia = createScheduledHandler({ mediaUsageCron: "*/5 * * * *" }); + await invoke(customMedia, "*/5 * * * *"); + expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); + expect(scheduled.general).not.toHaveBeenCalled(); + + scheduled.mediaUsage.mockClear(); + await invoke(customMedia, "15 * * * *"); + expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + expect(scheduled.general).toHaveBeenCalledOnce(); + + scheduled.general.mockClear(); + const customGeneral = createScheduledHandler({ generalCron: "0 * * * *" }); + await invoke(customGeneral, "*/2 * * * *"); + expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); + expect(scheduled.general).not.toHaveBeenCalled(); + + scheduled.mediaUsage.mockClear(); + await invoke(customGeneral, "0 * * * *"); + expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + expect(scheduled.general).toHaveBeenCalledOnce(); + + scheduled.general.mockClear(); + await invoke(customGeneral, "15 * * * *"); + expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + expect(scheduled.general).not.toHaveBeenCalled(); +}); + it("rejects empty or aliased configured expressions", () => { expect(() => createScheduledHandler({ generalCron: "* * * * *", mediaUsageCron: "* * * * *" }), @@ -56,6 +91,8 @@ it("rejects empty or aliased configured expressions", () => { expect(() => createScheduledHandler({ generalCron: "", mediaUsageCron: "*/2 * * * *" })).toThrow( /non-empty/i, ); + expect(() => createScheduledHandler({ mediaUsageCron: " " })).toThrow(/non-empty/i); + expect(() => createScheduledHandler({ generalCron: " */2 * * * * " })).toThrow(/must differ/i); }); async function invoke(handler: ExportedHandlerScheduledHandler, cron: string): Promise { diff --git a/templates/blog-cloudflare/src/worker.ts b/templates/blog-cloudflare/src/worker.ts index ad5efeee42..75fe3e2743 100644 --- a/templates/blog-cloudflare/src/worker.ts +++ b/templates/blog-cloudflare/src/worker.ts @@ -4,8 +4,5 @@ export { PluginBridge }; export default { ...handler, - scheduled: createScheduledHandler({ - generalCron: "* * * * *", - mediaUsageCron: "*/2 * * * *", - }), + scheduled: createScheduledHandler(), }; diff --git a/templates/marketing-cloudflare/src/worker.ts b/templates/marketing-cloudflare/src/worker.ts index ad5efeee42..75fe3e2743 100644 --- a/templates/marketing-cloudflare/src/worker.ts +++ b/templates/marketing-cloudflare/src/worker.ts @@ -4,8 +4,5 @@ export { PluginBridge }; export default { ...handler, - scheduled: createScheduledHandler({ - generalCron: "* * * * *", - mediaUsageCron: "*/2 * * * *", - }), + scheduled: createScheduledHandler(), }; diff --git a/templates/portfolio-cloudflare/src/worker.ts b/templates/portfolio-cloudflare/src/worker.ts index ad5efeee42..75fe3e2743 100644 --- a/templates/portfolio-cloudflare/src/worker.ts +++ b/templates/portfolio-cloudflare/src/worker.ts @@ -4,8 +4,5 @@ export { PluginBridge }; export default { ...handler, - scheduled: createScheduledHandler({ - generalCron: "* * * * *", - mediaUsageCron: "*/2 * * * *", - }), + scheduled: createScheduledHandler(), }; diff --git a/templates/starter-cloudflare/src/worker.ts b/templates/starter-cloudflare/src/worker.ts index ad5efeee42..75fe3e2743 100644 --- a/templates/starter-cloudflare/src/worker.ts +++ b/templates/starter-cloudflare/src/worker.ts @@ -4,8 +4,5 @@ export { PluginBridge }; export default { ...handler, - scheduled: createScheduledHandler({ - generalCron: "* * * * *", - mediaUsageCron: "*/2 * * * *", - }), + scheduled: createScheduledHandler(), }; From 29ae6966a7497aa098f47ee648b3bca759d94b69 Mon Sep 17 00:00:00 2001 From: khoinguyenpham04 <137921741+khoinguyenpham04@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:14:39 +0100 Subject: [PATCH 20/20] docs(cloudflare): keep worker comments current --- packages/cloudflare/src/worker.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/cloudflare/src/worker.ts b/packages/cloudflare/src/worker.ts index 91525e5f73..d2272d38a3 100644 --- a/packages/cloudflare/src/worker.ts +++ b/packages/cloudflare/src/worker.ts @@ -6,14 +6,6 @@ * Usage lane without request side effects. Re-exports the `PluginBridge` * Durable Object so the sandbox binding resolves against the entry module. * - * Existing sites can keep the default scheduled handler: - * - * export { default, PluginBridge } from "@emdash-cms/cloudflare/worker"; - * - * By default the every-two-minutes expression runs Media Usage maintenance - * and every other expression runs general maintenance. Sites only pass - * options when changing either expression. - * * The `@astrojs/cloudflare/entrypoints/server` import is resolved by the * consuming app's Astro build (it pulls the build-time `virtual:astro:app` * module), so this package keeps the adapter external.