[Test][CI] Run the store-emdash contract and races on D1 under the workers pool, nightly

.github/workflows/ci.yml

@@ -4,6 +4,13 @@ on:
   push:
     branches: [main]
   pull_request:
+  # The D1 tier is nightly, not per-PR: it boots workerd, runs every migration
+  # against a fresh D1 database per test file, and drives 250 reserves through
+  # the race shape, which costs several minutes of runner time for evidence that
+  # changes only when the adapter or the host build changes.
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
 
 # Cancel a PR's superseded runs when new commits land; never cancel an
 # in-flight main-branch validation (each merge should finish its own check).
@@ -13,6 +20,9 @@ concurrency:
 
 jobs:
   unit:
+    # The nightly schedule exists for the `d1` job alone; the per-commit jobs
+    # already ran on the commit that is being re-tested.
+    if: github.event_name != 'schedule'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -63,3 +73,24 @@ jobs:
       # Scoped to the files that actually touch Postgres (scripts/pg-test-files.sh) —
       # the sqlite/fake-only suite already ran in `unit`, no need to redo it here.
       - run: pnpm test:pg
+
+  # Tier T3: the contract suites and the race over REAL D1, inside workerd, via
+  # the workers pool. This is the dialect the storefront ships on and the only
+  # tier that exercises the host's own Kysely wiring (`createDialect` reading the
+  # `DB` binding). Everything is local — the miniflare D1 simulator — so no
+  # Cloudflare account, token or remote database is involved.
+  d1:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    # The tier takes a couple of minutes; anything near this ceiling is a hang,
+    # not a slow run.
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm test:d1

package.json

@@ -7,6 +7,7 @@
 		"typecheck": "tsc -b && tsc -p tsconfig.e2e.json",
 		"test": "vitest run",
 		"test:pg": "vitest run $(./scripts/pg-test-files.sh)",
+		"test:d1": "pnpm -C packages/store-emdash test:d1",
 		"test:e2e": "playwright test",
 		"format": "oxfmt",
 		"format:check": "oxfmt --check",

packages/store-emdash/README.md

@@ -34,6 +34,89 @@ The schema always comes from the host's `runMigrations`; never hand-create the
 storage table. Revisions come from a trigger that migration creates — which is
 also why cases reset by emptying the table rather than recreating it.
 
+## The D1 tier
+
+D1 is the dialect a deployed storefront actually runs on, and the two Node tiers
+never touch it: the conditional-write primitives ride the host's **SQLite branch**
+there by inference. `updateIf` is one
+`UPDATE … SET data = json_set(…) WHERE … RETURNING data`; revisions are stamped by
+the `AFTER INSERT` / `AFTER UPDATE` triggers the conditional-write migration
+creates on that branch. `better-sqlite3` runs the same SQL against a different
+engine build, in a different process model. So this tier exists to answer, rather
+than assume, whether D1 agrees.
+
+```bash
+pnpm test:d1        # from the repo root, or from this package
+```
+
+It runs under the Cloudflare workers vitest pool on the **local miniflare D1
+simulator** — no Cloudflare account, API token, remote database or deployment is
+involved, and nothing here can reach one. It is wired as its own vitest project
+(`store-emdash-d1`, `vitest.d1.config.ts`) rather than into the default battery:
+it boots `workerd`, migrates a fresh database per test file, and takes a couple of
+minutes. CI runs it **nightly** and on manual dispatch, never per PR. Miniflare is
+given the **storefront's own** compatibility date and flags
+(`sites/staging/wrangler.jsonc`), so a divergence found here means something about
+production rather than about an invented runtime.
+
+**What the toolchain costs, stated plainly.** `@cloudflare/vitest-plugin` pins its
+`wrangler` and `miniflare` versions **exactly**, and that `miniflare` in turn pins
+its own `workerd` exactly. So installing it adds a third `workerd` build (~150 MB)
+that only the nightly job ever executes, and **every** install — including every
+per-PR CI install — pays for it. It also moves the version `sites/staging`'s
+`@astrojs/cloudflare` peer-resolves `workerd` to, because pnpm picks the highest
+`workerd` in the graph: the storefront build now runs the newer one. Overriding
+`wrangler` back to the catalog version was tried and does **not** undo either
+effect — `miniflare`'s exact `workerd` pin is what carries it — so the override is
+deliberately absent rather than forgotten. The honest fix is upstream ranges or a
+separate install for the nightly; until then the whole toolchain is enumerated in
+`pnpm-workspace.yaml`'s `minimumReleaseAgeExclude` so nothing about it is
+implicit.
+
+**How the tier is built.** `test/d1/describe-d1.ts` is a sibling of
+`test/describe-each-dialect.ts`, not an extension of it. The split is structural:
+the Node harness imports `better-sqlite3` and `pg` at module scope, and neither
+exists inside `workerd`. What the two share is imported — the collection layout,
+the document helpers, the fault-injection wrappers, the domain contract itself —
+so only the test-surface plumbing is restated. The D1 files are named `*.spec.ts`
+so the default project's `test/**/*.test.ts` glob cannot pick them up, and so
+`scripts/pg-test-files.sh` never selects them.
+
+The dialect comes from the host's own `createDialect` reading the `DB` binding out
+of `cloudflare:workers` — the same call a real site makes — which makes this the
+only tier that observes the host's wiring rather than Otta's. The schema comes
+from the host's full `runMigrations` set, and the suite asserts that the revision
+triggers really exist on D1 and really fire for a writer that supplies no
+revision.
+
+**What it proves.** The primitive suite (`updateIf`'s `RETURNING` and `json_set`,
+`getVersioned`, `compareAndSet`'s revision assignment, `compareAndDelete`, the
+query allow-list, the 100-row page ceiling) behaves on D1 exactly as it does on
+better-sqlite3 and Postgres — case for case, no divergence. `inventoryStoreContract`
+passes in full, with no skips — including the W1 crash-window case, which needs the
+harness's `abandonPending` hook and silently asserts nothing without it.
+Representative crash seams — (a), (c), (e) and the cross-SKU `commitMany` of (g) —
+heal on D1 under the same real fault injection.
+
+**What it does NOT prove, and where that is proved instead.** Miniflare runs a
+test file in one `workerd` isolate on one thread, so concurrent promises
+**interleave** but no two statements execute at the same instant. The race file
+therefore runs the M=5/N=50 shape as an interleaving check — strictly stronger
+than the sequential contract path, strictly weaker than simultaneity. Atomicity
+under genuinely simultaneous writers is the **Postgres** tier's job, and it stays
+the no-oversell gate. A staging site on real D1 has many isolates at once, so the
+race this tier cannot run is real in production.
+
+The crash tier is also not reused wholesale: the eighteen cases in
+`test/inventory-crash-seams.dialects.test.ts` live inside a closure passed to
+`describeEachDialect`, so running all of them on D1 means first splitting that
+harness into a driver-agnostic binder plus two driver modules — a change to the
+Node tiers, and its own change rather than a rider on this one. Seams (b),
+(d-release), (f) and (g-`adoptMany`) are therefore Node-only today; they exercise
+the same two injection mechanisms this tier already proves on D1, so what is
+missing is logic coverage the Node tiers give on every commit — but it is a gap,
+not a non-issue.
+
 ## Known gap: no physical indexes
 
 Declared indexes reach a collection through the repository's `indexes`

packages/store-emdash/package.json

@@ -28,12 +28,15 @@
 		}
 	},
 	"scripts": {
-		"build": "tsdown"
+		"build": "tsdown",
+		"test:d1": "vitest run --config vitest.d1.config.ts"
 	},
 	"dependencies": {
 		"@otta-sh/domain": "workspace:*"
 	},
 	"devDependencies": {
+		"@cloudflare/vitest-plugin": "^1.1.8",
+		"@emdash-cms/cloudflare": "0.37.0",
 		"@types/better-sqlite3": "catalog:",
 		"@types/pg": "catalog:",
 		"better-sqlite3": "catalog:",

packages/store-emdash/test/d1/describe-d1.ts

@@ -0,0 +1,163 @@
+/**
+ * The **D1** harness for `@otta-sh/store-emdash` — tier T3.
+ *
+ * It is the sibling of `test/describe-each-dialect.ts`, not an extension of it,
+ * and the split is structural rather than stylistic: this file runs INSIDE
+ * `workerd`, where `better-sqlite3`, `pg` and `node:fs` do not exist. The Node
+ * harness imports both drivers at module scope, so importing it here would fail
+ * before a single case ran. What the two share is the part that matters — the
+ * collection layout (`test/inventory-collections.ts`) and the contract wiring the
+ * suites do themselves.
+ *
+ * What this tier is for: D1 is the dialect Otta actually ships on, and it is the
+ * only tier that exercises the host's OWN Kysely wiring — `createDialect` from
+ * `@emdash-cms/cloudflare/db/d1`, reading the `DB` binding out of
+ * `cloudflare:workers`, which is precisely what a deployed site does. The Node
+ * tiers construct Otta's own dialects instead.
+ *
+ * Three rules carry over from the Node harness unchanged, for the same reasons:
+ *
+ * 1. **The schema always comes from `runMigrations(db)`.** Revisions are assigned
+ *    by triggers the conditional-write migration creates — on the SQLite branch,
+ *    an `AFTER INSERT` and an `AFTER UPDATE` trigger per table that stamp
+ *    `lower(hex(randomblob(16)))`. A hand-created `_plugin_storage` has no
+ *    triggers, so every `compareAndSet` would see an unchanging revision and
+ *    quietly agree with itself. (Upstream's own D1 suites DO hand-create the
+ *    table and then apply migration 077 to it; this harness runs the whole set,
+ *    which is what a real site's database has been through.)
+ * 2. **The database is per FILE, and rows are cleared per TEST.** The pool gives
+ *    each test file its own D1 database and keeps its contents for the file's
+ *    lifetime. Isolation between cases comes from emptying the storage table —
+ *    the only reset that KEEPS the triggers rule (1) depends on.
+ * 3. **Each collection gets the declared `indexes` *and* `uniqueIndexes` as its
+ *    constructor argument**, exactly as the host's own `createStorageAccess`
+ *    does. No physical index is created in any tier, so uniqueness is never
+ *    enforced here and no adapter may depend on it being.
+ *
+ * The one thing this tier CANNOT do is race: miniflare runs the test file in a
+ * single `workerd` isolate on a single thread, so concurrent promises interleave
+ * at `await` points but no two statements ever execute at the same instant. See
+ * `no-oversell.d1.spec.ts` for what that does and does not prove.
+ */
+import { createDialect } from "@emdash-cms/cloudflare/db/d1";
+import { PluginStorageRepository } from "emdash";
+import { runMigrations } from "emdash/db";
+import { Kysely, sql } from "kysely";
+import { afterAll, beforeAll, beforeEach } from "vitest";
+import type { StorageAccess, StorageCollection } from "../../src/index.js";
+import { collectionOf } from "../../src/index.js";
+
+/** The binding name the D1 vitest config declares. */
+const BINDING = "DB";
+
+/** The plugin id every harness collection is namespaced under. */
+const PLUGIN_ID = "otta";
+
+/** The one table the repositories write. Emptied between cases, never dropped. */
+const STORAGE_TABLE = "_plugin_storage";
+
+/**
+ * One collection as the plugin descriptor declares it.
+ *
+ * Structurally identical to the Node harness's `CollectionLayout` on purpose:
+ * `test/inventory-collections.ts` is typed against that one and is consumed here
+ * without a cast.
+ */
+export interface CollectionLayout {
+	indexes?: Array<string | string[]>;
+	uniqueIndexes?: Array<string | string[]>;
+}
+
+/** The declared storage layout: collection name → its declared indexes. */
+export type StorageLayout = Record<string, CollectionLayout>;
+
+/** The schema is the host's — name its own database type rather than restate it. */
+type HostDb = Parameters<typeof runMigrations>[0];
+
+/** What a suite reads its collections out of, after the file's `beforeAll`. */
+export interface D1Storage {
+	/** The injected `StorageAccess`, keyed exactly as the layout was. */
+	readonly storage: StorageAccess;
+	/** One collection, typed to the document it holds. */
+	collection<T>(name: string): StorageCollection<T>;
+	/** The migrated Kysely instance, for the schema-level assertions. */
+	readonly db: HostDb;
+}
+
+/** Build the collections the way the host builds `ctx.storage`. */
+function buildStorage(db: HostDb, layout: StorageLayout): StorageAccess {
+	const storage: StorageAccess = {};
+	for (const [name, config] of Object.entries(layout)) {
+		// Exactly the argument the host passes: declared indexes AND unique
+		// indexes are both queryable fields.
+		const indexes = [...(config.indexes ?? []), ...(config.uniqueIndexes ?? [])];
+		storage[name] = new PluginStorageRepository(db, PLUGIN_ID, name, indexes);
+	}
+	return storage;
+}
+
+/**
+ * Open the file's D1 database through the host's own dialect and migrate it.
+ *
+ * Exported so the race file can build its own instance without the per-test
+ * `DELETE` a shared binding would impose on it.
+ */
+export async function openD1(layout: StorageLayout): Promise<{
+	db: HostDb;
+	storage: StorageAccess;
+	reset(): Promise<void>;
+	close(): Promise<void>;
+}> {
+	const db = new Kysely({ dialect: createDialect({ binding: BINDING }) }) as HostDb;
+	await runMigrations(db);
+	return {
+		db,
+		storage: buildStorage(db, layout),
+		async reset() {
+			// D1 has no TRUNCATE; the DELETE leaves the revision triggers in place.
+			await sql.raw(`DELETE FROM ${STORAGE_TABLE}`).execute(db);
+		},
+		async close() {
+			await db.destroy();
+		},
+	};
+}
+
+/**
+ * Call once at the top of a suite file. Registers the `beforeAll` that migrates
+ * the binding, a `beforeEach` that empties the storage table, and an `afterAll`
+ * that closes the Kysely instance.
+ */
+export function useD1Storage(layout: StorageLayout): D1Storage {
+	let open: Awaited<ReturnType<typeof openD1>> | undefined;
+
+	beforeAll(async () => {
+		open = await openD1(layout);
+	});
+
+	beforeEach(async () => {
+		await open?.reset();
+	});
+
+	afterAll(async () => {
+		const held = open;
+		open = undefined;
+		await held?.close();
+	});
+
+	const current = (): NonNullable<typeof open> => {
+		if (open === undefined) throw new Error("storage is only available inside a test");
+		return open;
+	};
+	return {
+		get storage() {
+			return current().storage;
+		},
+		get db() {
+			return current().db;
+		},
+		collection<T>(name: string): StorageCollection<T> {
+			return collectionOf<T>(current().storage, name);
+		},
+	};
+}