diff --git a/apps/web/src/db/bootstrap.ts b/apps/web/src/db/bootstrap.ts index 62967688..807c51aa 100644 --- a/apps/web/src/db/bootstrap.ts +++ b/apps/web/src/db/bootstrap.ts @@ -1,7 +1,9 @@ +import Database from "better-sqlite3"; + /** * Idempotent schema bootstrap, run by the app at startup. Self-hosted software * should never make its user run a migration tool; new columns arrive as - * additive `ALTER TABLE` guards in `MIGRATION_STEPS` below. + * additive `ALTER TABLE` guards in `migrate` below. * * Keep in sync with schema.ts. */ @@ -197,3 +199,64 @@ CREATE TABLE IF NOT EXISTS import_batches ( ); CREATE UNIQUE INDEX IF NOT EXISTS import_batch_unique ON import_batches(account_id, format, fingerprint); `; + +const hasColumn = (sqlite: Database.Database, table: string, name: string) => + (sqlite.pragma(`table_info(${table})`) as { name: string }[]).some( + (column) => column.name === name, + ); + +function migrate(sqlite: Database.Database) { + sqlite.exec(BOOTSTRAP_SQL); + if (!hasColumn(sqlite, "folders", "deleted_at")) { + sqlite.exec("ALTER TABLE folders ADD COLUMN deleted_at TEXT"); + } + if (!hasColumn(sqlite, "accounts", "ibkr_sync_time_zone")) { + sqlite.exec("ALTER TABLE accounts ADD COLUMN ibkr_sync_time_zone TEXT"); + } + // Additive upgrade: existing executions retain their fields and dedup hashes. + if (!hasColumn(sqlite, "executions", "import_metadata_json")) { + sqlite.exec("ALTER TABLE executions ADD COLUMN import_metadata_json TEXT"); + } + // Materialize CSV bounds once so connection and range lookups never scan candle JSON. + for (const name of ["bar_count", "first_time", "last_time"]) { + if (!hasColumn(sqlite, "market_csv_datasets", name)) + sqlite.exec(`ALTER TABLE market_csv_datasets ADD COLUMN ${name} INTEGER NOT NULL DEFAULT 0`); + } + sqlite.exec(`UPDATE market_csv_datasets SET + bar_count = json_array_length(bars_json), + first_time = json_extract(bars_json, '$[0].time'), + last_time = json_extract(bars_json, '$[#-1].time') WHERE bar_count = 0`); +} + +const SETUP_DEADLINE_MS = 15_000; +const sleeper = new Int32Array(new SharedArrayBuffer(4)); + +/** + * `next build` imports route modules from several workers at once and each one + * opens the journal. SQLite refuses some lock upgrades immediately instead of + * waiting out busy_timeout (switching a fresh file to WAL, for one), so setup + * retries those with a short, jittered backoff. + */ +function retryWhileBusy(step: () => T): T { + const deadline = Date.now() + SETUP_DEADLINE_MS; + for (let delay = 5; ; delay = Math.min(delay * 2, 250)) { + try { + return step(); + } catch (error) { + const busy = error instanceof Database.SqliteError && error.code.startsWith("SQLITE_BUSY"); + if (!busy || Date.now() >= deadline) throw error; + Atomics.wait(sleeper, 0, 0, delay * (0.5 + Math.random())); + } + } +} + +/** Opens the journal and brings its schema up to date; safe to call from many processes at once. */ +export function openJournalDatabase(file: string): Database.Database { + const sqlite = new Database(file); + retryWhileBusy(() => sqlite.pragma("journal_mode = WAL")); + sqlite.pragma("foreign_keys = ON"); + // One write lock covers every column check and ALTER, so concurrent first runs + // cannot both decide a column is missing. + retryWhileBusy(() => sqlite.transaction(() => migrate(sqlite)).immediate()); + return sqlite; +} diff --git a/apps/web/src/db/index.ts b/apps/web/src/db/index.ts index 9593d016..08f3fefb 100644 --- a/apps/web/src/db/index.ts +++ b/apps/web/src/db/index.ts @@ -1,9 +1,8 @@ -import Database from "better-sqlite3"; import { drizzle } from "drizzle-orm/better-sqlite3"; import { mkdirSync } from "node:fs"; import { join } from "node:path"; import * as schema from "./schema"; -import { BOOTSTRAP_SQL } from "./bootstrap"; +import { openJournalDatabase } from "./bootstrap"; export const dataDir = (): string => process.env.JOURNAL_DATA_DIR ?? join(process.cwd(), "data"); @@ -12,42 +11,7 @@ const globalForDb = globalThis as unknown as { __journalDb?: ReturnType { const dir = dataDir(); mkdirSync(dir, { recursive: true }); - const sqlite = new Database(join(dir, "journal.db")); - sqlite.pragma("journal_mode = WAL"); - sqlite.pragma("foreign_keys = ON"); - sqlite.exec(BOOTSTRAP_SQL); - const folderColumns = sqlite.pragma("table_info(folders)") as { name: string }[]; - if (!folderColumns.some((column) => column.name === "deleted_at")) { - sqlite.exec("ALTER TABLE folders ADD COLUMN deleted_at TEXT"); - } - sqlite - .transaction(() => { - const columns = sqlite.pragma("table_info(accounts)") as { name: string }[]; - if (!columns.some((column) => column.name === "ibkr_sync_time_zone")) { - sqlite.exec("ALTER TABLE accounts ADD COLUMN ibkr_sync_time_zone TEXT"); - } - }) - .immediate(); - // Additive upgrade: existing executions retain their fields and dedup hashes. - const executionColumns = sqlite.pragma("table_info(executions)") as { name: string }[]; - if (!executionColumns.some((column) => column.name === "import_metadata_json")) { - sqlite.exec("ALTER TABLE executions ADD COLUMN import_metadata_json TEXT"); - } - // Materialize CSV bounds once so connection and range lookups never scan candle JSON. - const csvColumns = sqlite.pragma("table_info(market_csv_datasets)") as { name: string }[]; - sqlite.transaction(() => { - for (const name of ["bar_count", "first_time", "last_time"]) { - if (!csvColumns.some((column) => column.name === name)) - sqlite.exec( - `ALTER TABLE market_csv_datasets ADD COLUMN ${name} INTEGER NOT NULL DEFAULT 0`, - ); - } - sqlite.exec(`UPDATE market_csv_datasets SET - bar_count = json_array_length(bars_json), - first_time = json_extract(bars_json, '$[0].time'), - last_time = json_extract(bars_json, '$[#-1].time') WHERE bar_count = 0`); - })(); - return drizzle(sqlite, { schema }); + return drizzle(openJournalDatabase(join(dir, "journal.db")), { schema }); }; /** Singleton across Next dev hot reloads. */ diff --git a/apps/web/tests/db-concurrent-init.test.ts b/apps/web/tests/db-concurrent-init.test.ts new file mode 100644 index 00000000..4cad44e1 --- /dev/null +++ b/apps/web/tests/db-concurrent-init.test.ts @@ -0,0 +1,88 @@ +import { afterAll, expect, it } from "vitest"; +import Database from "better-sqlite3"; +import { execFile } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { BOOTSTRAP_SQL } from "../src/db/bootstrap"; + +// `next build` opens the journal from several worker processes at once. Each child +// here loads the real bootstrap module and spins until a shared start time, so their +// first-run setup overlaps the way build workers do. +const run = promisify(execFile); +const bootstrapUrl = pathToFileURL( + fileURLToPath(new URL("../src/db/bootstrap.ts", import.meta.url)), +).href; +const child = ` +const [file, start, url] = process.argv.slice(1); +const { openJournalDatabase } = await import(url); +while (Date.now() < Number(start)); +openJournalDatabase(file).close(); +`; +const scratch = mkdtempSync(join(tmpdir(), "journal-concurrent-init-")); +const PROCESSES = 12; + +async function openConcurrently(file: string) { + const start = Date.now() + 1500; + const results = await Promise.allSettled( + Array.from({ length: PROCESSES }, () => + run( + process.execPath, + [ + "--experimental-strip-types", + "--no-warnings", + "--input-type=module", + "-e", + child, + file, + String(start), + bootstrapUrl, + ], + { timeout: 30_000 }, + ), + ), + ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [String((result.reason as { stderr?: string }).stderr)] : [], + ); + expect(failures).toEqual([]); + const sqlite = new Database(file, { readonly: true }); + try { + expect(sqlite.pragma("journal_mode", { simple: true })).toBe("wal"); + expect(sqlite.pragma("integrity_check", { simple: true })).toBe("ok"); + const columns = (table: string) => + (sqlite.pragma(`table_info(${table})`) as { name: string }[]).map((column) => column.name); + expect(columns("accounts")).toContain("ibkr_sync_time_zone"); + expect(columns("executions")).toContain("import_metadata_json"); + expect(columns("folders")).toContain("deleted_at"); + expect(sqlite.prepare("SELECT count(*) AS n FROM folders").get()).toEqual({ n: 5 }); + } finally { + sqlite.close(); + } +} + +it.each([1, 2, 3])( + "creates a fresh journal from concurrent processes without SQLITE_BUSY (batch %i)", + async (round) => { + await openConcurrently(join(scratch, `fresh-${round}.db`)); + }, + 60_000, +); + +it("upgrades a legacy rollback-journal database from concurrent processes", async () => { + const file = join(scratch, "legacy.db"); + const legacy = new Database(file); + legacy.exec( + BOOTSTRAP_SQL.replace(" ibkr_sync_time_zone TEXT,\n", "").replace( + " import_metadata_json TEXT,\n", + "", + ), + ); + expect(legacy.pragma("journal_mode", { simple: true })).toBe("delete"); + legacy.close(); + await openConcurrently(file); +}, 60_000); + +afterAll(() => rmSync(scratch, { recursive: true, force: true }));