Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 64 additions & 1 deletion apps/web/src/db/bootstrap.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import Database from "better-sqlite3";

/**
* Idempotent schema bootstrap, run by the app at startup. Self-hosted software
* should never make its user run a migration tool; new columns arrive as
* additive `ALTER TABLE` guards in `MIGRATION_STEPS` below.
* additive `ALTER TABLE` guards in `migrate` below.
*
* Keep in sync with schema.ts.
*/
Expand Down Expand Up @@ -197,3 +199,64 @@ CREATE TABLE IF NOT EXISTS import_batches (
);
CREATE UNIQUE INDEX IF NOT EXISTS import_batch_unique ON import_batches(account_id, format, fingerprint);
`;

const hasColumn = (sqlite: Database.Database, table: string, name: string) =>
(sqlite.pragma(`table_info(${table})`) as { name: string }[]).some(
(column) => column.name === name,
);

function migrate(sqlite: Database.Database) {
sqlite.exec(BOOTSTRAP_SQL);
if (!hasColumn(sqlite, "folders", "deleted_at")) {
sqlite.exec("ALTER TABLE folders ADD COLUMN deleted_at TEXT");
}
if (!hasColumn(sqlite, "accounts", "ibkr_sync_time_zone")) {
sqlite.exec("ALTER TABLE accounts ADD COLUMN ibkr_sync_time_zone TEXT");
}
// Additive upgrade: existing executions retain their fields and dedup hashes.
if (!hasColumn(sqlite, "executions", "import_metadata_json")) {
sqlite.exec("ALTER TABLE executions ADD COLUMN import_metadata_json TEXT");
}
// Materialize CSV bounds once so connection and range lookups never scan candle JSON.
for (const name of ["bar_count", "first_time", "last_time"]) {
if (!hasColumn(sqlite, "market_csv_datasets", name))
sqlite.exec(`ALTER TABLE market_csv_datasets ADD COLUMN ${name} INTEGER NOT NULL DEFAULT 0`);
}
sqlite.exec(`UPDATE market_csv_datasets SET
bar_count = json_array_length(bars_json),
first_time = json_extract(bars_json, '$[0].time'),
last_time = json_extract(bars_json, '$[#-1].time') WHERE bar_count = 0`);
}

const SETUP_DEADLINE_MS = 15_000;
const sleeper = new Int32Array(new SharedArrayBuffer(4));

/**
* `next build` imports route modules from several workers at once and each one
* opens the journal. SQLite refuses some lock upgrades immediately instead of
* waiting out busy_timeout (switching a fresh file to WAL, for one), so setup
* retries those with a short, jittered backoff.
*/
function retryWhileBusy<T>(step: () => T): T {
const deadline = Date.now() + SETUP_DEADLINE_MS;
for (let delay = 5; ; delay = Math.min(delay * 2, 250)) {
try {
return step();
} catch (error) {
const busy = error instanceof Database.SqliteError && error.code.startsWith("SQLITE_BUSY");
if (!busy || Date.now() >= deadline) throw error;
Atomics.wait(sleeper, 0, 0, delay * (0.5 + Math.random()));
}
}
}

/** Opens the journal and brings its schema up to date; safe to call from many processes at once. */
export function openJournalDatabase(file: string): Database.Database {
const sqlite = new Database(file);
retryWhileBusy(() => sqlite.pragma("journal_mode = WAL"));
sqlite.pragma("foreign_keys = ON");
// One write lock covers every column check and ALTER, so concurrent first runs
// cannot both decide a column is missing.
retryWhileBusy(() => sqlite.transaction(() => migrate(sqlite)).immediate());
return sqlite;
}
40 changes: 2 additions & 38 deletions apps/web/src/db/index.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import Database from "better-sqlite3";
import { drizzle } from "drizzle-orm/better-sqlite3";
import { mkdirSync } from "node:fs";
import { join } from "node:path";
import * as schema from "./schema";
import { BOOTSTRAP_SQL } from "./bootstrap";
import { openJournalDatabase } from "./bootstrap";

export const dataDir = (): string => process.env.JOURNAL_DATA_DIR ?? join(process.cwd(), "data");

Expand All @@ -12,42 +11,7 @@ const globalForDb = globalThis as unknown as { __journalDb?: ReturnType<typeof c
const createDb = () => {
const dir = dataDir();
mkdirSync(dir, { recursive: true });
const sqlite = new Database(join(dir, "journal.db"));
sqlite.pragma("journal_mode = WAL");
sqlite.pragma("foreign_keys = ON");
sqlite.exec(BOOTSTRAP_SQL);
const folderColumns = sqlite.pragma("table_info(folders)") as { name: string }[];
if (!folderColumns.some((column) => column.name === "deleted_at")) {
sqlite.exec("ALTER TABLE folders ADD COLUMN deleted_at TEXT");
}
sqlite
.transaction(() => {
const columns = sqlite.pragma("table_info(accounts)") as { name: string }[];
if (!columns.some((column) => column.name === "ibkr_sync_time_zone")) {
sqlite.exec("ALTER TABLE accounts ADD COLUMN ibkr_sync_time_zone TEXT");
}
})
.immediate();
// Additive upgrade: existing executions retain their fields and dedup hashes.
const executionColumns = sqlite.pragma("table_info(executions)") as { name: string }[];
if (!executionColumns.some((column) => column.name === "import_metadata_json")) {
sqlite.exec("ALTER TABLE executions ADD COLUMN import_metadata_json TEXT");
}
// Materialize CSV bounds once so connection and range lookups never scan candle JSON.
const csvColumns = sqlite.pragma("table_info(market_csv_datasets)") as { name: string }[];
sqlite.transaction(() => {
for (const name of ["bar_count", "first_time", "last_time"]) {
if (!csvColumns.some((column) => column.name === name))
sqlite.exec(
`ALTER TABLE market_csv_datasets ADD COLUMN ${name} INTEGER NOT NULL DEFAULT 0`,
);
}
sqlite.exec(`UPDATE market_csv_datasets SET
bar_count = json_array_length(bars_json),
first_time = json_extract(bars_json, '$[0].time'),
last_time = json_extract(bars_json, '$[#-1].time') WHERE bar_count = 0`);
})();
return drizzle(sqlite, { schema });
return drizzle(openJournalDatabase(join(dir, "journal.db")), { schema });
};

/** Singleton across Next dev hot reloads. */
Expand Down
88 changes: 88 additions & 0 deletions apps/web/tests/db-concurrent-init.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { afterAll, expect, it } from "vitest";
import Database from "better-sqlite3";
import { execFile } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { BOOTSTRAP_SQL } from "../src/db/bootstrap";

// `next build` opens the journal from several worker processes at once. Each child
// here loads the real bootstrap module and spins until a shared start time, so their
// first-run setup overlaps the way build workers do.
const run = promisify(execFile);
const bootstrapUrl = pathToFileURL(
fileURLToPath(new URL("../src/db/bootstrap.ts", import.meta.url)),
).href;
const child = `
const [file, start, url] = process.argv.slice(1);
const { openJournalDatabase } = await import(url);
while (Date.now() < Number(start));
openJournalDatabase(file).close();
`;
const scratch = mkdtempSync(join(tmpdir(), "journal-concurrent-init-"));
const PROCESSES = 12;

async function openConcurrently(file: string) {
const start = Date.now() + 1500;
const results = await Promise.allSettled(
Array.from({ length: PROCESSES }, () =>
run(
process.execPath,
[
"--experimental-strip-types",
"--no-warnings",
"--input-type=module",
"-e",
child,
file,
String(start),
bootstrapUrl,
],
{ timeout: 30_000 },
),
),
);
const failures = results.flatMap((result) =>
result.status === "rejected" ? [String((result.reason as { stderr?: string }).stderr)] : [],
);
expect(failures).toEqual([]);
const sqlite = new Database(file, { readonly: true });
try {
expect(sqlite.pragma("journal_mode", { simple: true })).toBe("wal");
expect(sqlite.pragma("integrity_check", { simple: true })).toBe("ok");
const columns = (table: string) =>
(sqlite.pragma(`table_info(${table})`) as { name: string }[]).map((column) => column.name);
expect(columns("accounts")).toContain("ibkr_sync_time_zone");
expect(columns("executions")).toContain("import_metadata_json");
expect(columns("folders")).toContain("deleted_at");
expect(sqlite.prepare("SELECT count(*) AS n FROM folders").get()).toEqual({ n: 5 });
} finally {
sqlite.close();
}
}

it.each([1, 2, 3])(
"creates a fresh journal from concurrent processes without SQLITE_BUSY (batch %i)",
async (round) => {
await openConcurrently(join(scratch, `fresh-${round}.db`));
},
60_000,
);

it("upgrades a legacy rollback-journal database from concurrent processes", async () => {
const file = join(scratch, "legacy.db");
const legacy = new Database(file);
legacy.exec(
BOOTSTRAP_SQL.replace(" ibkr_sync_time_zone TEXT,\n", "").replace(
" import_metadata_json TEXT,\n",
"",
),
);
expect(legacy.pragma("journal_mode", { simple: true })).toBe("delete");
legacy.close();
await openConcurrently(file);
}, 60_000);

afterAll(() => rmSync(scratch, { recursive: true, force: true }));
Loading