fix(core): reconcile media usage indexes automatically

.changeset/calm-files-reconcile.md

@@ -0,0 +1,6 @@
+---
+"emdash": patch
+"@emdash-cms/cloudflare": patch
+---
+
+Adds automatic, resumable background indexing so Media Usage can safely catch up on existing content without processing the whole site at once.

demos/cloudflare/src/worker.ts

@@ -13,5 +13,8 @@ export { PluginBridge };
 
 export default {
 	...handler,
-	scheduled: createScheduledHandler(),
+	scheduled: createScheduledHandler({
+		generalCron: "* * * * *",
+		mediaUsageCron: "*/2 * * * *",
+	}),
 } satisfies ExportedHandler<Env>;

demos/cloudflare/wrangler.jsonc

@@ -36,7 +36,7 @@
 	],
 	// Cron trigger drives the AI Search reindex queue flush.
 	"triggers": {
-		"crons": ["* * * * *"],
+		"crons": ["* * * * *", "*/2 * * * *"],
 	},
 	// Observability
 	"observability": {

docs/src/content/docs/deployment/cloudflare.mdx

@@ -74,24 +74,37 @@ To change the schema or content model of a site that is already deployed, see [E
 
 ## Scheduled Publishing
 
-On Cloudflare Workers, scheduled publishing, plugin cron, and maintenance tasks run from a Worker Cron Trigger. New Cloudflare templates include this setup automatically. If you are adding updating an existing project, export the EmDash Worker entry from `@emdash-cms/cloudflare/worker`:
+On Cloudflare Workers, scheduled publishing, plugin cron, and maintenance tasks run from Worker Cron Triggers. New Cloudflare templates include both required schedules automatically. When updating an existing project, configure distinct general and Media Usage lanes:
 
 ```ts title="src/worker.ts"
-export { default, PluginBridge } from "@emdash-cms/cloudflare/worker";
+import handler, {
+	createScheduledHandler,
+	PluginBridge,
+} from "@emdash-cms/cloudflare/worker";
+
+export { PluginBridge };
+
+export default {
+	...handler,
+	scheduled: createScheduledHandler({
+		generalCron: "* * * * *",
+		mediaUsageCron: "*/2 * * * *",
+	}),
+} satisfies ExportedHandler;
 ```
 
-Then add a Cron Trigger to `wrangler.jsonc`:
+Then add both Cron Triggers to `wrangler.jsonc` using the same expressions:
 
 ```jsonc title="wrangler.jsonc"
 {
 	"triggers": {
-		"crons": ["* * * * *"],
+		"crons": ["* * * * *", "*/2 * * * *"],
 	},
 }
 ```
 
 <Aside type="caution">
-	Without the Cron Trigger, content scheduled in the admin panel does not publish on Cloudflare Workers, and plugin cron jobs do not run. Local `astro dev` still uses the in-process scheduler.
+	Without the general trigger, scheduled publishing and plugin cron do not run. Without the dedicated Media Usage trigger, automatic historical reconciliation cannot progress. Local `astro dev` still uses the in-process scheduler.
 </Aside>
 
 ## Deploy

packages/cloudflare/src/worker.ts

@@ -2,17 +2,17 @@
  * Cloudflare Worker entry for EmDash sites.
  *
  * Wraps the Astro Cloudflare server handler with a `scheduled()` handler so a
- * Cron Trigger drives scheduled publishing, plugin cron, and system cleanup
- * without any request side effects. Re-exports the `PluginBridge` Durable
- * Object so the sandbox binding resolves against the entry module.
+ * Cron Triggers drive general maintenance and the separately bounded Media
+ * Usage lane without request side effects. Re-exports the `PluginBridge`
+ * Durable Object so the sandbox binding resolves against the entry module.
  *
- * Templates use this as their entire `src/worker.ts`:
+ * Existing sites can keep the default general-maintenance handler:
  *
  *   export { default, PluginBridge } from "@emdash-cms/cloudflare/worker";
  *
- * and add a Cron Trigger to wrangler.jsonc:
+ * New sites configure distinct expressions through `createScheduledHandler`.
  *
- *   "triggers": { "crons": ["* * * * *"] }
+ *   Configure one general expression and one distinct Media Usage expression.
  *
  * The `@astrojs/cloudflare/entrypoints/server` import is resolved by the
  * consuming app's Astro build (it pulls the build-time `virtual:astro:app`
@@ -22,7 +22,7 @@
 // @ts-ignore - resolved against the consuming app's Astro build
 import astroHandler from "@astrojs/cloudflare/entrypoints/server";
 import { createApp } from "astro/app/entrypoint";
-import { runScheduledTasks } from "emdash/middleware";
+import { runScheduledMediaUsageTasks, runScheduledTasks } from "emdash/middleware";
 
 export { PluginBridge } from "./sandbox/index.js";
 
@@ -48,13 +48,46 @@ async function invalidatePublishedTags(
 }
 
 /**
- * Build a Worker `scheduled()` handler that runs EmDash's scheduled
- * maintenance batch and purges edge-cache tags for anything it published.
- * Exported for sites that assemble their own Worker object; most sites get it
- * via this module's default export.
+ * Build a Worker `scheduled()` handler. Without options every expression runs
+ * the backwards-compatible general lane. Configured handlers dispatch exact,
+ * distinct expressions to general or Media Usage maintenance.
  */
-export function createScheduledHandler(): ExportedHandlerScheduledHandler {
-	return (_controller, _env, ctx) => {
+export interface ScheduledHandlerOptions {
+	generalCron: string;
+	mediaUsageCron: string;
+}
+
+export function createScheduledHandler(
+	options?: ScheduledHandlerOptions,
+): ExportedHandlerScheduledHandler {
+	if (options) {
+		if (!options.generalCron.trim() || !options.mediaUsageCron.trim()) {
+			throw new Error("Configured scheduled-handler expressions must be non-empty");
+		}
+		if (options.generalCron === options.mediaUsageCron) {
+			throw new Error("General and Media Usage Cron expressions must differ");
+		}
+	}
+	return (controller, _env, ctx) => {
+		if (options && controller.cron === options.mediaUsageCron) {
+			ctx.waitUntil(
+				runScheduledMediaUsageTasks().catch((error: unknown) => {
+					console.error("[scheduled] Media Usage maintenance failed:", error);
+				}),
+			);
+			return;
+		}
+		if (options && controller.cron !== options.generalCron) {
+			console.warn(`[scheduled] Ignoring unexpected Cron expression: ${controller.cron}`);
+			return;
+		}
+		if (!options) {
+			ctx.waitUntil(
+				runScheduledMediaUsageTasks().catch((error: unknown) => {
+					console.error("[scheduled] Media Usage maintenance failed:", error);
+				}),
+			);
+		}
 		ctx.waitUntil(
 			// Invalidate incrementally as each collection batch publishes, so a
 			// scheduled() invocation killed mid-sweep (CPU/wall-clock limits on a

packages/cloudflare/tests/worker-scheduled.test.ts

@@ -0,0 +1,70 @@
+import { beforeEach, expect, it, vi } from "vitest";
+
+const scheduled = vi.hoisted(() => ({
+	general: vi.fn(async () => ({ published: [] })),
+	mediaUsage: vi.fn(async () => ({ outcome: "inactive", taskClass: null, turn: null })),
+}));
+
+vi.mock("@astrojs/cloudflare/entrypoints/server", () => ({ default: { fetch: vi.fn() } }));
+vi.mock("astro/app/entrypoint", () => ({
+	createApp: () => ({ pipeline: { getCacheProvider: async () => null } }),
+}));
+vi.mock("emdash/middleware", () => ({
+	runScheduledTasks: scheduled.general,
+	runScheduledMediaUsageTasks: scheduled.mediaUsage,
+}));
+vi.mock("../src/sandbox/index.js", () => ({ PluginBridge: vi.fn() }));
+
+import { createScheduledHandler } from "../src/worker.js";
+
+beforeEach(() => {
+	scheduled.general.mockClear();
+	scheduled.mediaUsage.mockClear();
+});
+
+it("keeps the unconfigured handler backwards-compatible", async () => {
+	await invoke(createScheduledHandler(), "custom expression");
+	expect(scheduled.general).toHaveBeenCalledOnce();
+	expect(scheduled.mediaUsage).toHaveBeenCalledOnce();
+});
+
+it("dispatches distinct configured cron expressions to exactly one lane", async () => {
+	const handler = createScheduledHandler({
+		generalCron: "* * * * *",
+		mediaUsageCron: "*/2 * * * *",
+	});
+
+	await invoke(handler, "* * * * *");
+	expect(scheduled.general).toHaveBeenCalledOnce();
+	expect(scheduled.mediaUsage).not.toHaveBeenCalled();
+
+	scheduled.general.mockClear();
+	await invoke(handler, "*/2 * * * *");
+	expect(scheduled.general).not.toHaveBeenCalled();
+	expect(scheduled.mediaUsage).toHaveBeenCalledOnce();
+
+	scheduled.mediaUsage.mockClear();
+	await invoke(handler, "0 0 * * *");
+	expect(scheduled.general).not.toHaveBeenCalled();
+	expect(scheduled.mediaUsage).not.toHaveBeenCalled();
+});
+
+it("rejects empty or aliased configured expressions", () => {
+	expect(() =>
+		createScheduledHandler({ generalCron: "* * * * *", mediaUsageCron: "* * * * *" }),
+	).toThrow(/must differ/i);
+	expect(() => createScheduledHandler({ generalCron: "", mediaUsageCron: "*/2 * * * *" })).toThrow(
+		/non-empty/i,
+	);
+});
+
+async function invoke(handler: ExportedHandlerScheduledHandler, cron: string): Promise<void> {
+	const pending: Promise<unknown>[] = [];
+	const context = {
+		waitUntil(promise: Promise<unknown>) {
+			pending.push(promise);
+		},
+	};
+	Reflect.apply(handler, undefined, [{ cron }, {}, context]);
+	await Promise.all(pending);
+}

packages/core/src/astro/middleware.ts

@@ -39,9 +39,11 @@ import {
 	flushRecorder,
 	isInstrumentationEnabled,
 } from "../database/instrumentation.js";
+import { createDeferredTaskTracker } from "../deferred-tasks.js";
 import {
 	DB_INIT_DEADLINE_MS,
 	EmDashRuntime,
+	type MediaUsageMaintenanceResult,
 	type RuntimeDependencies,
 	type SandboxedPluginEntry,
 	type MediaProviderEntry,
@@ -287,6 +289,12 @@ export async function runScheduledTasks(
 	return runOutsideRequest(config, (runtime) => runtime.runScheduledTasks(options));
 }
 
+export async function runScheduledMediaUsageTasks(): Promise<MediaUsageMaintenanceResult> {
+	const config = getConfig();
+	if (!config) return { outcome: "inactive", taskClass: null, turn: null };
+	return runOutsideRequest(config, (runtime) => runtime.runScheduledMediaUsageTasks());
+}
+
 /**
  * Run a callback against the EmDash runtime outside any HTTP request — from a
  * Cloudflare Queue consumer, a `scheduled()` handler, or any other
@@ -352,8 +360,35 @@ async function runOutsideRequest<T>(
 	config: EmDashConfig,
 	fn: (runtime: EmDashRuntime) => Promise<T>,
 ): Promise<T> {
-	const runtime = await getRuntime(config);
+	if (getRequestContext()) {
+		const runtime = await getRuntime(config);
+		return runOutsideRequestWithRuntime(config, runtime, fn);
+	}
 
+	const deferredTasks = createDeferredTaskTracker(() => {});
+	const context = {
+		editMode: false,
+		metrics: createRequestMetrics(performance.now()),
+		deferredTasks,
+	};
+	return runWithContext(context, async () => {
+		const runtime = await (async () => {
+			try {
+				return await getRuntime(config);
+			} finally {
+				deferredTasks.settle();
+				await deferredTasks.settled;
+			}
+		})();
+		return runOutsideRequestWithRuntime(config, runtime, fn);
+	});
+}
+
+async function runOutsideRequestWithRuntime<T>(
+	config: EmDashConfig,
+	runtime: EmDashRuntime,
+	fn: (runtime: EmDashRuntime) => Promise<T>,
+): Promise<T> {
 	const scoped = createRequestScopedDb({
 		config: config.database?.config,
 		isAuthenticated: false,