Skip to content

Commit 18d591e

Browse files
authored
fix(postgres): Serialize schema initialization (#17)
1 parent ffc3712 commit 18d591e

7 files changed

Lines changed: 234 additions & 18 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,20 @@ jobs:
1616
fail-fast: false
1717
matrix:
1818
node: [22, 24]
19+
services:
20+
postgres:
21+
image: postgres:17-alpine
22+
env:
23+
POSTGRES_HOST_AUTH_METHOD: trust
24+
ports:
25+
- 127.0.0.1:5432:5432
26+
options: >-
27+
--health-cmd "pg_isready -U postgres"
28+
--health-interval 5s
29+
--health-timeout 5s
30+
--health-retries 10
31+
env:
32+
TEST_DATABASE_URL: postgres://postgres@127.0.0.1:5432/postgres
1933
steps:
2034
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
2135
with:

‎docs/schema.md‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@ Everything lives in a `workos` schema by default. Set `WORKOS_SYNC_SCHEMA` or
44
pass `--schema` to change it; the setting applies to both the DDL and every
55
query the engine issues.
66

7+
Initialization runs in one transaction under a per-schema advisory lock,
8+
taken before even creating the schema. Concurrent initializers wait, then
9+
check the committed fingerprint rather than racing to create catalog objects.
10+
Commit or rollback releases the lock; a fatal setup error rolls back its DDL.
11+
Repair attempts use savepoints, so a refused view replacement or column repair
12+
can still be handled without aborting the migration or releasing its lock.
13+
714
## Columns every table has
815

916
| Column | Type | Meaning |
@@ -86,7 +93,7 @@ Things to know:
8693
| Table | Purpose |
8794
| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
8895
| `_sync_state` | Cursors and checkpoints, keyed by `(sync_id, env, stream)`. The tail position is the row with `stream = '__tail__'`; backfill progress is one row per stream. |
89-
| `_migrations` | Fingerprints of the schema versions that have been applied, so `init` is a single `SELECT` when nothing changed. |
96+
| `_migrations` | Fingerprints of the schema versions that have been applied, so unchanged catalogs skip per-stream DDL. |
9097
| `_sync_lease` | The single-writer lease for the events tail: one row per `(sync_id, env)` naming the holder and when its claim expires. `listen` and `tick` take it; the cursor checkpoint is refused unless the writer still holds it. See [Operations](operations.md#deployment). |
9198
| `_rejected` | Events the tail could not apply: one row per event that failed deterministically on two consecutive polls, with the error and the whole event in `_raw`. Keyed by `(sync_id, env, event_id)`; `rejected --replay` drains it. |
9299

@@ -268,10 +275,10 @@ replaced in place (see [Live views](#live-views)); as a library the two are
268275
separate options, `repairColumns` and `repairViews`.
269276

270277
`--repair` drops each drifted column and adds it back from the catalog, each
271-
in its own transaction, then recreates its index. `STORED` recomputes the new
272-
expression for every row already present, so this is also the backfill, and
273-
nothing is lost because `_raw` holds all of it. As a library, pass
274-
`repairColumns: true` to `PostgresDestination`.
278+
in its own savepoint within the migration transaction, then recreates its
279+
index. `STORED` recomputes the new expression for every row already present,
280+
so this is also the backfill. Nothing is lost because `_raw` holds all of it.
281+
As a library, pass `repairColumns: true` to `PostgresDestination`.
275282

276283
Rebuilding a column also forgets every fingerprint recorded before it. A
277284
fingerprint says the live columns were checked against that catalog, and that

‎packages/sync-postgres/src/schema/drift.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,10 @@ async function expectedColumns(db: SqlClient, stream: StreamDef, schema: string)
141141
const tmp = quoteIdent(safeIdentifier(`_drift_${stream.name}`))
142142
const cols = [`${quoteIdent(RAW)} jsonb`, ...stream.columns.map((c) => generatedColumnSql(c, schema))]
143143
await tx.exec(`create temp table ${tmp} (${cols.join(', ')}) on commit drop;`)
144-
return liveColumns(tx, `pg_temp.${tmp}`)
144+
const columns = await liveColumns(tx, `pg_temp.${tmp}`)
145+
// Migration savepoints do not trigger ON COMMIT DROP before the next drift check.
146+
await tx.exec(`drop table pg_temp.${tmp};`)
147+
return columns
145148
})
146149
}
147150

@@ -258,7 +261,8 @@ export interface RepairResult {
258261
/**
259262
* Accepted for repair, but the DDL failed — typically a view depends on the
260263
* column (SQLSTATE 2BP01). Nothing changed for these: each column's DDL runs
261-
* in its own transaction, so a failed re-add never leaves a column dropped.
264+
* in its own transaction or migration savepoint, so a failed re-add never
265+
* leaves a column dropped.
262266
*/
263267
failed: { drift: ColumnDrift; error: string }[]
264268
}
@@ -271,10 +275,10 @@ export interface RepairResult {
271275
*
272276
* Several processes can share one database and start together, each
273277
* detecting the same drift before any of them repairs it. Each column's
274-
* transaction therefore takes an advisory lock on the table and reads the
275-
* column again before dropping it: the second process to arrive finds the
276-
* column already rebuilt and leaves it alone, rather than rewriting the whole
277-
* table a second time behind an exclusive lock. A transaction-scoped
278+
* transaction or migration savepoint takes an advisory lock on the table and
279+
* reads the column again before dropping it: the second process finds it
280+
* already rebuilt and leaves it alone, rather than rewriting the whole table
281+
* a second time behind an exclusive lock. A transaction-scoped
278282
* advisory lock is used rather than a table lock so that nothing is blocked
279283
* until the drop itself.
280284
*

‎packages/sync-postgres/src/schema/migrate.ts‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
* files, one per upstream schema change, each needing a release; deriving the
77
* DDL instead means adding a projected column is a catalog edit.
88
*
9-
* Setup is gated on a fingerprint of the catalog, so a restart with no changes
10-
* is a single SELECT rather than dozens of DDL round trips.
9+
* A fingerprint gates per-stream DDL, so unchanged catalogs skip that work.
10+
* A per-schema transaction lock serializes setup, including the bootstrap DDL.
1111
*
1212
* The DDL is additive, which cannot change a column that already exists. So
1313
* after it runs, and only when the fingerprint is new, the live tables are
@@ -90,6 +90,44 @@ export async function migrate(
9090
schema: string,
9191
logger?: Logger,
9292
options: MigrateOptions = {},
93+
): Promise<MigrateResult> {
94+
return db.transaction(async (tx) => {
95+
// Lock before even CREATE SCHEMA: IF NOT EXISTS does not serialize catalog writes.
96+
await tx.query(
97+
`select pg_advisory_xact_lock(hashtext('workos-sync-postgres:migrate'), hashtext($1::name::text))`,
98+
[schema],
99+
)
100+
// Repairs must recover from failed DDL without aborting the migration or releasing its lock.
101+
const nested: SqlClient = {
102+
query: tx.query.bind(tx),
103+
exec: tx.exec.bind(tx),
104+
close: tx.close.bind(tx),
105+
async transaction<T>(fn: (client: SqlClient) => Promise<T>): Promise<T> {
106+
await tx.exec('savepoint workos_sync_migrate')
107+
try {
108+
const result = await fn(nested)
109+
await tx.exec('release savepoint workos_sync_migrate')
110+
return result
111+
} catch (err) {
112+
try {
113+
await tx.exec('rollback to savepoint workos_sync_migrate; release savepoint workos_sync_migrate')
114+
} catch {
115+
// The outer transaction still rolls back; preserve the original error.
116+
}
117+
throw err
118+
}
119+
},
120+
}
121+
return applySchema(nested, streams, schema, logger, options)
122+
})
123+
}
124+
125+
async function applySchema(
126+
db: SqlClient,
127+
streams: StreamDef[],
128+
schema: string,
129+
logger: Logger | undefined,
130+
options: MigrateOptions,
93131
): Promise<MigrateResult> {
94132
const fp = fingerprint(streams, schema)
95133

‎packages/sync-postgres/src/schema/views.ts‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,10 @@ export interface ApplyViewOptions {
112112
* place; the views are documented as not to be edited.
113113
*
114114
* The drop is not `cascade` — a user's view built on top of ours is theirs,
115-
* and taking it down silently would be worse than failing. This relies on
116-
* `migrate` running each statement outside a transaction; inside one, the
117-
* failed create would already have aborted it. The repair itself is one
118-
* transaction: DDL is transactional in Postgres, so a create that fails after
119-
* the drop — or a process that dies between the two — rolls back to the old
115+
* and taking it down silently would be worse than failing. During migration,
116+
* these inner transactions use savepoints so a failed create is rolled back
117+
* before the repair starts. The repair itself is atomic: a create that fails
118+
* after the drop — or a process that dies between the two — rolls back to the old
120119
* view instead of leaving readers with none.
121120
*/
122121
export async function applyViewStatement(
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Database tests
2+
3+
`pnpm test` runs the PGlite-backed suite without a database service.
4+
Multi-connection regressions also run when `TEST_DATABASE_URL` is set:
5+
6+
```sh
7+
TEST_DATABASE_URL=postgres://postgres@127.0.0.1:5432/postgres pnpm test
8+
```
9+
10+
Use a disposable PostgreSQL database with permission to create schemas. Each
11+
integration test creates a uniquely named schema and drops it on completion.
12+
CI supplies PostgreSQL 17 and runs these tests on both supported Node versions.
Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import { randomUUID } from 'node:crypto'
2+
import { Pool } from 'pg'
3+
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
4+
import { streamByName } from '@workos/sync-engine'
5+
import { fromPgPool, type SqlClient } from '../src/db/client.js'
6+
import { migrate } from '../src/schema/migrate.js'
7+
import { quoteIdent } from '../src/schema/ddl.js'
8+
import { detectColumnDrift } from '../src/schema/drift.js'
9+
import { GENERATED_BY } from '../src/schema/views.js'
10+
11+
function pauseAfterSchema(client: SqlClient, created: () => void, resume: Promise<void>): SqlClient {
12+
return {
13+
...client,
14+
async exec(sql) {
15+
await client.exec(sql)
16+
if (sql.startsWith('create schema')) {
17+
created()
18+
await resume
19+
}
20+
},
21+
transaction: (fn) => client.transaction((tx) => fn(pauseAfterSchema(tx, created, resume))),
22+
}
23+
}
24+
25+
describe.skipIf(!process.env.TEST_DATABASE_URL)('schema initialization on PostgreSQL', () => {
26+
const users = streamByName('users')!
27+
const changed = { ...users, columns: users.columns.map((column) =>
28+
column.name === 'email' ? { ...column, path: 'primary_email' } : column,
29+
) }
30+
let pools: Pool[]
31+
let db: SqlClient
32+
let other: SqlClient
33+
let observer: SqlClient
34+
let otherPid: number
35+
let schema: string
36+
37+
beforeEach(async () => {
38+
schema = `migrate_test_${randomUUID().replaceAll('-', '')}`
39+
pools = Array.from({ length: 3 }, () => new Pool({
40+
connectionString: process.env.TEST_DATABASE_URL, max: 1, statement_timeout: 10_000,
41+
}))
42+
db = fromPgPool(pools[0]!)
43+
other = fromPgPool(pools[1]!)
44+
observer = fromPgPool(pools[2]!)
45+
otherPid = (await other.query<{ pid: number }>('select pg_backend_pid() as pid')).rows[0]!.pid
46+
})
47+
48+
afterEach(async () => {
49+
try {
50+
await observer.exec(`drop schema if exists "${schema}" cascade`)
51+
} finally {
52+
await Promise.all(pools.map((pool) => pool.end()))
53+
}
54+
})
55+
56+
it('waits for an in-progress first migration before creating any catalog objects', async () => {
57+
let created!: () => void
58+
let resume!: () => void
59+
const ready = new Promise<void>((resolve) => { created = resolve })
60+
const paused = new Promise<void>((resolve) => { resume = resolve })
61+
const first = db.transaction((tx) => migrate(pauseAfterSchema(tx, created, paused), [users], schema))
62+
let second: ReturnType<typeof migrate> | undefined
63+
try {
64+
await Promise.race([ready, first])
65+
second = migrate(other, [users], schema)
66+
await expect.poll(async () => {
67+
const result = await observer.query<{ wait_event_type: string }>(
68+
'select wait_event_type from pg_stat_activity where pid = $1', [otherPid],
69+
)
70+
return result.rows[0]?.wait_event_type
71+
}, { timeout: 5_000, interval: 10 }).toBe('Lock')
72+
resume()
73+
const results = await Promise.all([first, second])
74+
expect(results.map((result) => result.applied)).toEqual([true, false])
75+
expect((await other.query(`select count(*)::int as n from "${schema}"._migrations`)).rows).toEqual([{ n: 1 }])
76+
expect((await other.query(`select * from "${schema}".users_live`)).rows).toEqual([])
77+
} finally {
78+
resume()
79+
await Promise.allSettled(second ? [first, second] : [first])
80+
}
81+
})
82+
83+
it('allows concurrent warm startup with single-connection pools', async () => {
84+
await migrate(db, [users], schema)
85+
const results = await Promise.all([migrate(db, [users], schema), migrate(other, [users], schema)])
86+
expect(results.map((result) => result.applied)).toEqual([false, false])
87+
})
88+
89+
it('rolls back failed bootstrap and releases the lock for another initializer', async () => {
90+
await expect(migrate(db, [{ ...users, key: ['missing_column'] }], schema)).rejects.toThrow(/does not exist/)
91+
expect((await observer.query('select to_regnamespace($1) as name', [schema])).rows).toEqual([{ name: null }])
92+
expect((await migrate(other, [users], schema)).applied).toBe(true)
93+
})
94+
95+
it('recovers from a refused view replacement without losing grants', async () => {
96+
await migrate(db, [users], schema)
97+
const columns = await db.query<{ column_name: string }>(
98+
`select column_name from information_schema.columns
99+
where table_schema = $1 and table_name = 'users' order by ordinal_position desc`, [schema],
100+
)
101+
await db.exec(`
102+
drop view "${schema}".users_live;
103+
create view "${schema}".users_live as
104+
select ${columns.rows.map((column) => quoteIdent(column.column_name)).join(', ')} from "${schema}".users;
105+
comment on view "${schema}".users_live is '${GENERATED_BY}: older definition';
106+
grant select on "${schema}".users_live to public;
107+
delete from "${schema}"._migrations;
108+
`)
109+
expect((await migrate(other, [users], schema, undefined, { repairViews: true })).applied).toBe(true)
110+
const grants = await observer.query(
111+
`select privilege_type from information_schema.table_privileges
112+
where table_schema = $1 and table_name = 'users_live' and grantee = 'PUBLIC'`, [schema],
113+
)
114+
expect(grants.rows).toEqual([{ privilege_type: 'SELECT' }])
115+
})
116+
117+
it('can check column drift again after repairing it in the migration transaction', async () => {
118+
await migrate(db, [users], schema)
119+
await db.query(`insert into "${schema}".users (_env, _raw, _updated_at)
120+
values ('prod', '{"id":"user_1","email":"old@example.test","primary_email":"new@example.test"}', now())`)
121+
const result = await other.transaction(async (tx) => {
122+
const repaired = await migrate(tx, [changed], schema, undefined, { repair: true })
123+
expect(await detectColumnDrift(tx, [changed], schema)).toEqual([])
124+
return repaired
125+
})
126+
expect(result.repaired.map((column) => column.column)).toEqual(['email'])
127+
expect(result.drift).toEqual([])
128+
expect((await observer.query(`select email from "${schema}".users_live`)).rows).toEqual([{ email: 'new@example.test' }])
129+
expect((await migrate(db, [changed], schema)).applied).toBe(false)
130+
})
131+
132+
it('reports a refused column repair without aborting the migration transaction', async () => {
133+
await migrate(db, [users], schema)
134+
await db.exec(`create view "${schema}".custom_emails as select email from "${schema}".users`)
135+
const result = await migrate(other, [changed], schema, undefined, { repair: true })
136+
expect(result.repaired).toEqual([])
137+
expect(result.drift).toHaveLength(1)
138+
expect((await observer.query(`select * from "${schema}".custom_emails`)).rows).toEqual([])
139+
expect((await observer.query(`select * from "${schema}".users_live`)).rows).toEqual([])
140+
expect((await migrate(db, [users], schema)).applied).toBe(false)
141+
})
142+
})

0 commit comments

Comments
 (0)