Skip to content

Commit 68b9e3f

Browse files
authored
refactor: lift the CLI core, health and write helpers into the engine (#55)
1 parent 613ae96 commit 68b9e3f

55 files changed

Lines changed: 1858 additions & 4136 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎packages/sync-clickhouse/src/cli.ts‎

Lines changed: 60 additions & 518 deletions
Large diffs are not rendered by default.

‎packages/sync-clickhouse/src/db/lease.ts‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
* that equal cursors compare equal as text, as they do as jsonb.
2828
*/
2929
import type { Lease, LeaseResult } from '@workos/sync-engine'
30+
import { canonicalJson } from '@workos/sync-engine'
3031
import { isoOf, KEEPER_SETTINGS, type ChClient } from './client.js'
3132
import { isKeeperConflict } from './errors.js'
3233
import { quoteIdent } from '../schema/ddl.js'
@@ -52,24 +53,6 @@ export interface SyncRow {
5253
now: string
5354
}
5455

55-
/** `value` as JSON with every object's keys sorted, so equal values serialise equally. */
56-
export function canonicalJson(value: unknown): string {
57-
return JSON.stringify(sortKeys(value))
58-
}
59-
60-
function sortKeys(value: unknown): unknown {
61-
if (Array.isArray(value)) return value.map(sortKeys)
62-
if (value !== null && typeof value === 'object') {
63-
const out: Record<string, unknown> = {}
64-
for (const key of Object.keys(value as Record<string, unknown>).sort()) {
65-
const v = (value as Record<string, unknown>)[key]
66-
if (v !== undefined) out[key] = sortKeys(v)
67-
}
68-
return out
69-
}
70-
return value
71-
}
72-
7356
export const syncKey = (syncId: string, env: string) => `${syncId}${SEP}${env}`
7457

7558
const table = (schema: string) => `${quoteIdent(schema)}.\`_sync\``

‎packages/sync-clickhouse/src/db/reconcile.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@
1414
* replica, and on a replicated deployment the next request may land
1515
* elsewhere. They are dropped when the diff ends, however it ends.
1616
*/
17-
import { encodeKey, type IdDiff, type TableDef } from '@workos/sync-engine'
17+
import { encodeKey, type IdDiff, type TableDef, chunk, keyValues } from '@workos/sync-engine'
1818
import { randomBytes } from 'node:crypto'
1919
import type { ChClient } from './client.js'
20-
import { chunk, keyExpression, keyValues, tableRef, KEY_SEPARATOR, type WriteOptions } from './upsert.js'
20+
import { keyExpression, tableRef, KEY_SEPARATOR, type WriteOptions } from './upsert.js'
2121
import { quoteIdent, ENV } from '../schema/ddl.js'
2222

2323
export interface ChDiff extends IdDiff {

‎packages/sync-clickhouse/src/db/upsert.ts‎

Lines changed: 13 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,19 @@
2323
* re-delivery a no-op; the table, a `ReplacingMergeTree` keyed by the id,
2424
* collapses a copy that got past that anyway.
2525
*/
26-
import { emptyStats, type Row, type TableDef, type WriteMode, type WriteStats } from '@workos/sync-engine'
26+
import {
27+
emptyStats,
28+
type Row,
29+
type TableDef,
30+
type WriteMode,
31+
type WriteStats,
32+
keyValues,
33+
dedupeByKey,
34+
wavesByKey,
35+
chunk,
36+
parseUpdatedAt,
37+
jsonEqual,
38+
} from '@workos/sync-engine'
2739
import { hostname } from 'node:os'
2840
import { fromCh, toCh, type ChClient } from './client.js'
2941
import { lockKey, withStreamLock, type HeldCheck, type LockOptions } from './lock.js'
@@ -32,140 +44,6 @@ import { quoteIdent, safeIdentifier, ENV, RAW } from '../schema/ddl.js'
3244
/** Joins the components of a composite key, on both sides of every lookup. */
3345
export const KEY_SEPARATOR = '\u0001'
3446

35-
export function readPath(data: Record<string, unknown>, path: string | string[]): unknown {
36-
const parts = Array.isArray(path) ? path : [path]
37-
let cur: unknown = data
38-
for (const p of parts) {
39-
if (cur == null || typeof cur !== 'object') return undefined
40-
cur = (cur as Record<string, unknown>)[p]
41-
}
42-
return cur
43-
}
44-
45-
/** Key values for a row, in `stream.key` order. Undefined if any part is missing. */
46-
export function keyValues(stream: TableDef, data: Record<string, unknown>): string[] | undefined {
47-
const out: string[] = []
48-
for (const k of stream.key) {
49-
const col = stream.columns.find((c) => c.name === k)
50-
const v = readPath(data, col ? col.path : k)
51-
if (v == null || v === '') return undefined
52-
out.push(String(v))
53-
}
54-
return out
55-
}
56-
57-
/**
58-
* Collapse rows that share a primary key, keeping the newest. It serves the
59-
* delete, relink and unlink paths, whose rows are whole; upserts use
60-
* `wavesByKey`.
61-
*/
62-
export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropped: number } {
63-
const byKey = new Map<string, { row: Row; at: number }>()
64-
let dropped = 0
65-
for (const row of rows) {
66-
const key = keyValues(stream, row.data)
67-
if (!key) {
68-
dropped++
69-
continue
70-
}
71-
const k = key.join(KEY_SEPARATOR)
72-
const at = micros(row.updatedAt)
73-
const existing = byKey.get(k)
74-
if (!existing || !isBefore(row.updatedAt, at, existing.row.updatedAt, existing.at)) byKey.set(k, { row, at })
75-
}
76-
return { rows: [...byKey.values()].map((e) => e.row), dropped }
77-
}
78-
79-
/**
80-
* The instant `iso` names, in microseconds since the epoch, or NaN.
81-
* `Date.parse` reads three fractional digits, and so does the column
82-
* (`DateTime64(3)`), so two rows within a millisecond tie at the staleness
83-
* gate and the last one written wins; reading the rest of the fraction
84-
* here is what makes that last one the newer of the two.
85-
*/
86-
function micros(iso: string): number {
87-
const frac = /\.(\d+)/.exec(iso)?.[1] ?? ''
88-
const ms = Date.parse(frac.length > 3 ? iso.replace(`.${frac}`, `.${frac.slice(0, 3)}`) : iso)
89-
if (Number.isNaN(ms)) return ms
90-
return ms * 1000 + Number(frac.slice(3, 6).padEnd(3, '0'))
91-
}
92-
93-
/**
94-
* Whether one timestamp is strictly earlier than another, by the instant
95-
* each names: `01:00:00+01:00` sorts after `00:30:00Z` as text and is half an
96-
* hour earlier as a time. A value that is not a date is ordered as text;
97-
* `parseUpdatedAt` refuses it on the write either way.
98-
*/
99-
function isBefore(a: string, atA: number, b: string, atB: number): boolean {
100-
if (Number.isNaN(atA) || Number.isNaN(atB)) return a < b
101-
return atA < atB
102-
}
103-
104-
/**
105-
* Split a batch into waves in which no key appears twice, keeping order: the
106-
* n-th row for a key goes in wave n. Written one after another, the waves
107-
* are exactly the rows written one at a time — each row meets the staleness
108-
* gate against what the row before it left stored, and a merge overlays it
109-
* on that. See the Postgres destination for why keeping only the newest row
110-
* would be wrong for a merge.
111-
*/
112-
export function wavesByKey(stream: TableDef, rows: Row[]): { waves: Row[][]; dropped: number } {
113-
const waves: Row[][] = []
114-
const seen = new Map<string, number>()
115-
let dropped = 0
116-
for (const row of rows) {
117-
const key = keyValues(stream, row.data)
118-
if (!key) {
119-
dropped++
120-
continue
121-
}
122-
const k = key.join(KEY_SEPARATOR)
123-
const n = seen.get(k) ?? 0
124-
seen.set(k, n + 1)
125-
const wave = waves[n] ?? (waves[n] = [])
126-
wave.push(row)
127-
}
128-
return { waves, dropped }
129-
}
130-
131-
export const chunk = <T>(items: T[], size: number): T[][] => {
132-
const out: T[][] = []
133-
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))
134-
return out
135-
}
136-
137-
/**
138-
* `Row.updatedAt` as a `Date`. Postgres casts the string in SQL and a bad one
139-
* fails the statement; here the same verdict is reached before the
140-
* statement, with a code the error classifier reports as deterministic, so
141-
* the event is quarantined rather than retried forever.
142-
*/
143-
export function parseUpdatedAt(value: string): Date {
144-
const d = new Date(value)
145-
if (typeof value !== 'string' || Number.isNaN(d.getTime())) {
146-
throw Object.assign(new Error(`invalid updatedAt timestamp: ${JSON.stringify(value)}`), {
147-
code: 'INVALID_TIMESTAMP',
148-
})
149-
}
150-
return d
151-
}
152-
153-
/** Structural equality with key order ignored, which is what `is distinct from` means for jsonb. */
154-
export function jsonEqual(a: unknown, b: unknown): boolean {
155-
if (a === b) return true
156-
if (typeof a !== typeof b || a === null || b === null || typeof a !== 'object') return false
157-
if (Array.isArray(a) !== Array.isArray(b)) return false
158-
if (Array.isArray(a)) {
159-
const bb = b as unknown[]
160-
return a.length === bb.length && a.every((v, i) => jsonEqual(v, bb[i]))
161-
}
162-
const ao = a as Record<string, unknown>
163-
const bo = b as Record<string, unknown>
164-
const keys = Object.keys(ao)
165-
if (keys.length !== Object.keys(bo).length) return false
166-
return keys.every((k) => Object.hasOwn(bo, k) && jsonEqual(ao[k], bo[k]))
167-
}
168-
16947
/** Shallow merge: the jsonb `||` of the Postgres destination. */
17048
const merge = (stored: Record<string, unknown>, incoming: Record<string, unknown>): Record<string, unknown> => ({
17149
...stored,

‎packages/sync-clickhouse/src/index.ts‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,21 +33,10 @@ export {
3333
} from './schema/ddl.js'
3434
export { liveViewSql, liveViewName } from './schema/views.js'
3535

36-
export {
37-
upsertRows,
38-
deleteRows,
39-
relinkRows,
40-
unlinkRows,
41-
dedupeByKey,
42-
wavesByKey,
43-
keyValues,
44-
parseUpdatedAt,
45-
jsonEqual,
46-
} from './db/upsert.js'
36+
export { upsertRows, deleteRows, relinkRows, unlinkRows } from './db/upsert.js'
4737
export { diffIdsInClickHouse } from './db/reconcile.js'
4838
export type { ChDiff } from './db/reconcile.js'
4939
export { rejectEvents, listRejected, countRejected, clearRejected } from './db/rejected.js'
50-
export { canonicalJson } from './db/lease.js'
5140
export { isTransientClickHouseError } from './db/errors.js'
5241
export { connect, fromClient, clientConfig, APPLICATION_NAME, DEFAULT_SETTINGS, KEEPER_SETTINGS } from './db/client.js'
5342
export type { ConnectOptions, ChClient } from './db/client.js'

‎packages/sync-clickhouse/src/otel.ts‎

Lines changed: 0 additions & 75 deletions
This file was deleted.

0 commit comments

Comments
 (0)