Skip to content

Commit 05e8fb4

Browse files
authored
fix(mysql): run the installed command, dedupe by instant, refuse out-of-range offsets (#50)
1 parent e667d58 commit 05e8fb4

6 files changed

Lines changed: 116 additions & 6 deletions

File tree

‎packages/sync-mysql/src/cli.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ import {
3232
type SyncEngineConfig,
3333
} from '@workos/sync-engine'
3434
import { X509Certificate } from 'node:crypto'
35+
import { realpathSync } from 'node:fs'
36+
import { fileURLToPath } from 'node:url'
3537
import { MysqlDestination } from './destination.js'
3638
import { buildSchemaSql } from './schema/ddl.js'
3739
import { fingerprint } from './schema/migrate.js'
@@ -538,9 +540,23 @@ async function runCommand(command: string, flags: Record<string, string | boolea
538540
}
539541
}
540542

541-
// Run when invoked directly, not when imported by a test.
542-
const invokedDirectly = process.argv[1] && (process.argv[1].endsWith('cli.js') || process.argv[1].endsWith('cli.ts'))
543-
if (invokedDirectly) {
543+
/**
544+
* Whether this module is the script Node was started with, as opposed to an
545+
* import by a test. npm installs a `bin` as a symlink named after the
546+
* command, so `process.argv[1]` ends in `workos-sync-mysql` rather than
547+
* `cli.js`; both sides are resolved to real paths so the installed command
548+
* counts.
549+
*/
550+
export function invokedDirectly(argv1: string | undefined, moduleUrl: string): boolean {
551+
if (!argv1) return false
552+
try {
553+
return realpathSync(argv1) === realpathSync(fileURLToPath(moduleUrl))
554+
} catch {
555+
return false
556+
}
557+
}
558+
559+
if (invokedDirectly(process.argv[1], import.meta.url)) {
544560
main().then(
545561
(code) => process.exit(code),
546562
(err) => {

‎packages/sync-mysql/src/db/upsert.ts‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ export function keyValues(stream: TableDef, data: Record<string, unknown>): stri
5656
* are whole; upserts use `wavesByKey`.
5757
*/
5858
export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropped: number } {
59-
const byKey = new Map<string, Row>()
59+
const byKey = new Map<string, { row: Row; at: number }>()
6060
let dropped = 0
6161
for (const row of rows) {
6262
const key = keyValues(stream, row.data)
@@ -65,10 +65,35 @@ export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropp
6565
continue
6666
}
6767
const k = key.join('\0')
68+
const at = micros(row.updatedAt)
6869
const existing = byKey.get(k)
69-
if (!existing || row.updatedAt >= existing.updatedAt) byKey.set(k, row)
70+
if (!existing || !isBefore(row.updatedAt, at, existing.row.updatedAt, existing.at)) byKey.set(k, { row, at })
7071
}
71-
return { rows: [...byKey.values()], dropped }
72+
return { rows: [...byKey.values()].map((e) => e.row), dropped }
73+
}
74+
75+
/**
76+
* The instant `iso` names, in microseconds since the epoch, or NaN.
77+
* `Date.parse` reads three fractional digits; a `DATETIME(6)` keeps six, so
78+
* the rest of the fraction is read here and two rows a microsecond apart
79+
* stay ordered the way the staleness gate will order them.
80+
*/
81+
function micros(iso: string): number {
82+
const frac = /\.(\d+)/.exec(iso)?.[1] ?? ''
83+
const ms = Date.parse(frac.length > 3 ? iso.replace(`.${frac}`, `.${frac.slice(0, 3)}`) : iso)
84+
if (Number.isNaN(ms)) return ms
85+
return ms * 1000 + Number(frac.slice(3, 6).padEnd(3, '0'))
86+
}
87+
88+
/**
89+
* Whether one timestamp is strictly earlier than another, by the instant
90+
* each names: `01:00:00+01:00` sorts after `00:30:00Z` as text and is half an
91+
* hour earlier as a time. A value that is not a date is ordered as text; the
92+
* database refuses it on the write either way.
93+
*/
94+
function isBefore(a: string, atA: number, b: string, atB: number): boolean {
95+
if (Number.isNaN(atA) || Number.isNaN(atB)) return a < b
96+
return atA < atB
7297
}
7398

7499
/**

‎packages/sync-mysql/src/schema/ddl.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,16 @@ export function iso8601Expression(v: string): string {
8686
// `+05:30` means the wall time is 5h30 ahead of UTC, so UTC is that much earlier.
8787
// `signed`, not `unsigned`: MySQL refuses to multiply an unsigned value by -1.
8888
const offsetMinutes = `(case when ${offset} is null then 0 else (case when left(${offset}, 1) = '-' then 1 else -1 end) * (cast(substring(${offset}, 2, 2) as signed) * 60 + cast(right(${offset}, 2) as signed)) end)`
89+
// An offset is hours and minutes too; `+00:99` names no zone.
90+
const offsetOutOfRange = `(${offset} is not null and (cast(substring(${offset}, 2, 2) as unsigned) > 23 or cast(right(${offset}, 2) as unsigned) > 59))`
8991
const fraction = `coalesce(regexp_substr(${v}, '\\\\.[0-9]{1,6}'), '.0')`
9092
const wall = `concat(substring(${v}, 1, 10), ' ', substring(${v}, 12, 8), ${fraction})`
9193
return `(case
9294
when ${v} is null or not (${v} regexp '${ISO_PATTERN}') then null
9395
when ${year} < 1 or ${month} < 1 or ${month} > 12 then null
9496
when ${day} < 1 or ${day} > ${daysInMonth} then null
9597
when ${field(12)} > 23 or ${field(15)} > 59 or ${field(18)} > 59 then null
98+
when ${offsetOutOfRange} then null
9699
else timestampadd(minute, ${offsetMinutes}, str_to_date(${wall}, '%Y-%m-%d %H:%i:%s.%f'))
97100
end)`
98101
.replace(/\s+/g, ' ')

‎packages/sync-mysql/test/cli.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
2+
import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
3+
import { tmpdir } from 'node:os'
4+
import { join } from 'node:path'
5+
import { pathToFileURL } from 'node:url'
26
import { SyncEngine, WorkOSClient, selectStreams, silentLogger } from '@workos/sync-engine'
37
import {
8+
invokedDirectly,
49
main,
510
parseHydrate,
611
parseExcludeStreams,
@@ -16,6 +21,42 @@ import { FaultyDestination } from './helpers/faulty.js'
1621
import { TEST_CA_PEM } from './helpers/ca.js'
1722
import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js'
1823

24+
describe('entrypoint', () => {
25+
let dir: string
26+
beforeEach(() => {
27+
dir = mkdtempSync(join(tmpdir(), 'workos-sync-mysql-bin-'))
28+
})
29+
afterEach(() => {
30+
rmSync(dir, { recursive: true, force: true })
31+
})
32+
33+
it('recognises the script itself and the symlink npm installs as the command', () => {
34+
const script = join(dir, 'cli.js')
35+
writeFileSync(script, '')
36+
const url = pathToFileURL(script).href
37+
expect(invokedDirectly(script, url)).toBe(true)
38+
39+
// `npm install` links `node_modules/.bin/workos-sync-mysql` at dist/cli.js,
40+
// so argv[1] is the link, not the file.
41+
const link = join(dir, 'workos-sync-mysql')
42+
symlinkSync(script, link)
43+
expect(invokedDirectly(link, url)).toBe(true)
44+
})
45+
46+
it('is false for an import, a missing path, or another script', () => {
47+
const script = join(dir, 'cli.js')
48+
writeFileSync(script, '')
49+
const url = pathToFileURL(script).href
50+
expect(invokedDirectly(undefined, url)).toBe(false)
51+
expect(invokedDirectly(join(dir, 'missing'), url)).toBe(false)
52+
const other = join(dir, 'vitest.mjs')
53+
writeFileSync(other, '')
54+
expect(invokedDirectly(other, url)).toBe(false)
55+
// This test file was started by the test runner, not by the CLI.
56+
expect(invokedDirectly(process.argv[1], url)).toBe(false)
57+
})
58+
})
59+
1960
describe('WORKOS_SYNC_HYDRATE', () => {
2061
it('leaves the engine default in place when unset', () => {
2162
expect(parseHydrate(undefined)).toBeUndefined()

‎packages/sync-mysql/test/schema.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,10 @@ describeDb('migrations against real MySQL', () => {
210210
['2026-01-01T23:60:00Z', null],
211211
['2026-01-01T23:59:60Z', null],
212212
['0000-01-01T00:00:00Z', null],
213+
// The offset's fields are checked like the clock's.
214+
['2026-01-01T00:00:00+00:99', null],
215+
['2026-01-01T00:00:00-24:00', null],
216+
['2026-01-01T00:00:00+23:59', '2025-12-31T00:01:00.000Z'],
213217
]
214218
for (const [input, expected] of cases) {
215219
const id = `user_${String(input)}`

‎packages/sync-mysql/test/upsert.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,27 @@ describe('key extraction', () => {
5050
expect(dropped).toBe(1)
5151
})
5252

53+
it('judges the newest row by its instant, not by the text of its timestamp', () => {
54+
// "01:00:00+01:00" sorts after "00:30:00Z" as text and is half an hour earlier as a time.
55+
const { rows } = dedupeByKey(users, [
56+
{ data: userPayload({ email: 'older@example.com' }), updatedAt: '2026-01-01T01:00:00+01:00' },
57+
{ data: userPayload({ email: 'newer@example.com' }), updatedAt: '2026-01-01T00:30:00Z' },
58+
])
59+
expect(rows.map((r) => r.data['email'])).toEqual(['newer@example.com'])
60+
// The database keeps six fractional digits, so two rows a microsecond apart stay ordered.
61+
const fine = dedupeByKey(users, [
62+
{ data: userPayload({ email: 'newer@example.com' }), updatedAt: '2026-01-01T00:00:00.000900Z' },
63+
{ data: userPayload({ email: 'older@example.com' }), updatedAt: '2026-01-01T00:00:00.000100Z' },
64+
])
65+
expect(fine.rows.map((r) => r.data['email'])).toEqual(['newer@example.com'])
66+
// What is not a date is ordered as text, and the database refuses it on the write.
67+
const junk = dedupeByKey(users, [
68+
{ data: userPayload({ email: 'a@example.com' }), updatedAt: 'later' },
69+
{ data: userPayload({ email: 'b@example.com' }), updatedAt: 'yesterday' },
70+
])
71+
expect(junk.rows.map((r) => r.data['email'])).toEqual(['b@example.com'])
72+
})
73+
5374
it('splits repeated keys into ordered waves and leaves distinct keys in one', () => {
5475
const first = { data: userPayload({ external_id: 'ext_1' }), updatedAt: '2026-01-01T00:00:00.000Z' }
5576
const other = { data: userPayload({ id: 'user_2' }), updatedAt: '2026-01-01T00:00:00.000Z' }

0 commit comments

Comments
 (0)