Skip to content

Commit 613ae96

Browse files
authored
fix: redact every column that names a user, verify unrecorded MySQL columns (#52)
1 parent 272a149 commit 613ae96

14 files changed

Lines changed: 271 additions & 55 deletions

File tree

‎packages/sync-clickhouse/src/destination.ts‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import type {
1616
WriteMode,
1717
WriteStats,
1818
} from '@workos/sync-engine'
19-
import { encodeKey } from '@workos/sync-engine'
19+
import { encodeKey, userReferenceColumns, userReferencePaths } from '@workos/sync-engine'
2020
import { connect, isoOf, type ChClient } from './db/client.js'
2121
import { upsertRows, deleteRows, relinkRows, unlinkRows, keyExpression, tableRef, KEY_SEPARATOR } from './db/upsert.js'
2222
import { diffIdsInClickHouse } from './db/reconcile.js'
@@ -28,14 +28,6 @@ import { jsonPathArgs, quoteIdent, safeIdentifier } from './schema/ddl.js'
2828

2929
const ident = (name: string) => quoteIdent(safeIdentifier(name))
3030

31-
/**
32-
* The columns that name a user: a membership's or sign-in's `user_id`, and
33-
* an invitation's inviter and the user who accepted it. `redact` erases a
34-
* row that names the subject in any of them, and an archived or quarantined
35-
* event whose payload does.
36-
*/
37-
const USER_REFERENCES = ['user_id', 'inviter_user_id', 'accepted_user_id']
38-
3931
export interface ClickHouseDestinationOptions {
4032
/**
4133
* The HTTP interface, `https://user:password@host:8443`. Alternatively
@@ -229,9 +221,7 @@ export class ClickHouseDestination implements Destination {
229221
out.push({ table: label, rows: n })
230222
}
231223
for (const stream of streams) {
232-
const refs = stream.columns.filter(
233-
(c) => USER_REFERENCES.includes(c.name) || (c.name === 'id' && stream.name === 'users'),
234-
)
224+
const refs = userReferenceColumns(stream)
235225
if (refs.length === 0) continue
236226
await erase(
237227
stream.name,
@@ -241,15 +231,15 @@ export class ClickHouseDestination implements Destination {
241231
{ env, id: userId },
242232
)
243233
}
244-
const namedInData = USER_REFERENCES.map(
245-
(field) => `JSONExtractString(\`_raw\`, ${jsonPathArgs(['data', field])}) = {id:String}`,
246-
).join(' OR ')
234+
const namedInData = userReferencePaths(streams)
235+
.map((p) => ` OR JSONExtractString(\`_raw\`, ${jsonPathArgs(['data', ...p])}) = {id:String}`)
236+
.join('')
247237
// Archived events embed the payload, so they must be scrubbed too.
248238
await erase(
249239
'events',
250240
`${quoteIdent(this.schema)}.\`events\``,
251241
' FINAL',
252-
`\`_env\` = {env:String} AND (\`object_id\` = {id:String} OR ${namedInData})`,
242+
`\`_env\` = {env:String} AND (\`object_id\` = {id:String}${namedInData})`,
253243
{ env, id: userId },
254244
)
255245
// Quarantined events embed the payload too, and they may be the only copy
@@ -259,7 +249,7 @@ export class ClickHouseDestination implements Destination {
259249
'_rejected',
260250
`${quoteIdent(this.schema)}.\`_rejected\``,
261251
' FINAL',
262-
`sync_id = {syncId:String} AND env = {env:String} AND (object_id = {id:String} OR ${namedInData})`,
252+
`sync_id = {syncId:String} AND env = {env:String} AND (object_id = {id:String}${namedInData})`,
263253
{ syncId: this.syncId, env, id: userId },
264254
)
265255
return out

‎packages/sync-engine/src/catalog.ts‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ export const STREAMS: StreamDef[] = [
116116
prefixes: ['om_'],
117117
columns: [
118118
{ name: 'id', type: 'text', path: 'id' },
119-
{ name: 'user_id', type: 'text', path: 'user_id', index: true },
119+
{ name: 'user_id', type: 'text', path: 'user_id', index: true, references: 'users' },
120120
{ name: 'organization_id', type: 'text', path: 'organization_id', index: true },
121121
{ name: 'status', type: 'text', path: 'status' },
122122
{ name: 'role_slug', type: 'text', path: ['role', 'slug'] },
@@ -183,8 +183,8 @@ export const STREAMS: StreamDef[] = [
183183
{ name: 'email', type: 'text', path: 'email', index: true },
184184
{ name: 'state', type: 'text', path: 'state' },
185185
{ name: 'organization_id', type: 'text', path: 'organization_id', index: true },
186-
{ name: 'inviter_user_id', type: 'text', path: 'inviter_user_id' },
187-
{ name: 'accepted_user_id', type: 'text', path: 'accepted_user_id' },
186+
{ name: 'inviter_user_id', type: 'text', path: 'inviter_user_id', references: 'users' },
187+
{ name: 'accepted_user_id', type: 'text', path: 'accepted_user_id', references: 'users' },
188188
{ name: 'role_slug', type: 'text', path: ['role', 'slug'] },
189189
{ name: 'expires_at', type: 'timestamptz', path: 'expires_at' },
190190
{ name: 'accepted_at', type: 'timestamptz', path: 'accepted_at' },
@@ -251,7 +251,7 @@ export const STREAMS: StreamDef[] = [
251251
{ name: 'event', type: 'text', path: 'event', index: true },
252252
{ name: 'type', type: 'text', path: 'type' },
253253
{ name: 'status', type: 'text', path: 'status' },
254-
{ name: 'user_id', type: 'text', path: 'user_id', index: true },
254+
{ name: 'user_id', type: 'text', path: 'user_id', index: true, references: 'users' },
255255
{ name: 'email', type: 'text', path: 'email', index: true },
256256
{ name: 'ip_address', type: 'text', path: 'ip_address' },
257257
{ name: 'user_agent', type: 'text', path: 'user_agent' },
@@ -271,7 +271,7 @@ export const STREAMS: StreamDef[] = [
271271
prefixes: ['session_'],
272272
columns: [
273273
{ name: 'id', type: 'text', path: 'id' },
274-
{ name: 'user_id', type: 'text', path: 'user_id', index: true },
274+
{ name: 'user_id', type: 'text', path: 'user_id', index: true, references: 'users' },
275275
{ name: 'organization_id', type: 'text', path: 'organization_id', index: true },
276276
{ name: 'status', type: 'text', path: 'status' },
277277
{ name: 'auth_method', type: 'text', path: 'auth_method' },
@@ -591,6 +591,48 @@ export const STREAMS: StreamDef[] = [
591591
},
592592
] satisfies CatalogStreamDef[]
593593

594+
/**
595+
* Whether `col` holds a user id: declared with `references: 'users'`, or
596+
* named `user_id`, which an `extraColumns` entry may be without declaring
597+
* anything, and which always counted.
598+
*/
599+
const namesUser = (col: ColumnDef): boolean => col.references === 'users' || col.name === 'user_id'
600+
601+
/**
602+
* The columns of `stream` that name a user, and the `users` stream's own
603+
* key: a membership's or sign-in's `user_id`, an invitation's inviter and
604+
* the user who accepted it, an extra column declared as referencing
605+
* `users`. `redact` erases a row that names the subject in any of them.
606+
*/
607+
export function userReferenceColumns(stream: StreamDef): ColumnDef[] {
608+
return stream.columns.filter((c) => namesUser(c) || (stream.name === 'users' && stream.key.includes(c.name)))
609+
}
610+
611+
/**
612+
* The payload paths that name a user, each as its steps under the object:
613+
* `['user_id']`, or `['metadata', 'owner_user_id']` for a nested extra
614+
* column. Every stream of the catalog contributes, whether selected or
615+
* not, because the archive keeps every event type the tail subscribed to;
616+
* `streams` adds the extra columns of the selection. An event's `data`
617+
* carries the object, so `redact` scrubs archived and quarantined events
618+
* by `data` followed by each.
619+
*/
620+
export function userReferencePaths(streams: StreamDef[]): string[][] {
621+
const seen = new Set<string>()
622+
const out: string[][] = []
623+
for (const stream of [...STREAMS, ...streams]) {
624+
for (const col of stream.columns) {
625+
if (!namesUser(col)) continue
626+
const path = Array.isArray(col.path) ? col.path : [col.path]
627+
const key = path.join('\u0000')
628+
if (seen.has(key)) continue
629+
seen.add(key)
630+
out.push(path)
631+
}
632+
}
633+
return out
634+
}
635+
594636
export const streamByName = (name: string): StreamDef | undefined => STREAMS.find((s) => s.name === name)
595637

596638
/**

‎packages/sync-engine/src/index.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,16 @@ export type {
3737
export { StateStore } from './state.js'
3838
export { redactPayload, redactRows, redactEvent, RedactingDestination, redacting } from './redact.js'
3939

40-
export { STREAMS, streamByName, streamForId, defaultStreams, selectStreams, subscribedEventTypes } from './catalog.js'
40+
export {
41+
STREAMS,
42+
streamByName,
43+
streamForId,
44+
defaultStreams,
45+
selectStreams,
46+
subscribedEventTypes,
47+
userReferenceColumns,
48+
userReferencePaths,
49+
} from './catalog.js'
4150
export { routeEvent, sourceUpdatedAt } from './router.js'
4251
export type { RouteAction } from './router.js'
4352

‎packages/sync-engine/src/types.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ export interface ColumnDef {
1717
path: string | string[]
1818
/** Create a btree index on this column. */
1919
index?: boolean
20+
/**
21+
* The stream whose key this column holds, when it holds one: `user_id`
22+
* references `users`. What `redact` follows to every row that names a
23+
* subject, on every destination alike.
24+
*/
25+
references?: string
2026
}
2127

2228
/**

‎packages/sync-engine/test/catalog.test.ts‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
import { describe, it, expect } from 'vitest'
2-
import { STREAMS, defaultStreams, selectStreams } from '../src/catalog.js'
2+
import {
3+
streamByName,
4+
STREAMS,
5+
defaultStreams,
6+
selectStreams,
7+
userReferenceColumns,
8+
userReferencePaths,
9+
} from '../src/catalog.js'
310

411
describe('selectStreams', () => {
512
it('selects the defaults, with events, when given nothing', () => {
@@ -28,3 +35,39 @@ describe('selectStreams', () => {
2835
)
2936
})
3037
})
38+
39+
describe('user references', () => {
40+
it('names the columns of a stream that hold a user id, and the users key itself', () => {
41+
expect(userReferenceColumns(streamByName('users')!).map((c) => c.name)).toEqual(['id'])
42+
expect(userReferenceColumns(streamByName('organization_memberships')!).map((c) => c.name)).toEqual(['user_id'])
43+
expect(userReferenceColumns(streamByName('invitations')!).map((c) => c.name)).toEqual([
44+
'inviter_user_id',
45+
'accepted_user_id',
46+
])
47+
expect(userReferenceColumns(streamByName('organizations')!)).toEqual([])
48+
})
49+
50+
it('collects the payload paths that name a user across the given streams, extra and nested ones included', () => {
51+
expect(
52+
userReferencePaths(STREAMS)
53+
.map((p) => p.join('.'))
54+
.sort(),
55+
).toEqual(['accepted_user_id', 'inviter_user_id', 'user_id'])
56+
const withExtra = selectStreams(['organizations'], {
57+
organizations: [
58+
{ name: 'owner', type: 'text', path: ['metadata', 'owner_user_id'], references: 'users' },
59+
// Named `user_id` without declaring anything: counted, as it always was.
60+
{ name: 'user_id', type: 'text', path: 'user_id' },
61+
],
62+
})
63+
const organizations = withExtra.find((s) => s.name === 'organizations')!
64+
expect(userReferenceColumns(organizations).map((c) => c.name)).toEqual(['owner', 'user_id'])
65+
// The catalog's own paths stay: the archive holds every event type,
66+
// whichever streams were selected.
67+
expect(
68+
userReferencePaths(withExtra)
69+
.map((p) => p.join('.'))
70+
.sort(),
71+
).toEqual(['accepted_user_id', 'inviter_user_id', 'metadata.owner_user_id', 'user_id'])
72+
})
73+
})

‎packages/sync-mongodb/src/destination.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import type {
1616
WriteMode,
1717
WriteStats,
1818
} from '@workos/sync-engine'
19-
import { encodeKey } from '@workos/sync-engine'
19+
import { encodeKey, userReferenceColumns, userReferencePaths } from '@workos/sync-engine'
2020
import type { Collection, Db, Document, MongoClient } from 'mongodb'
2121
import { connect, databaseFromUrl } from './db/client.js'
2222
import { isTransientMongoError } from './db/errors.js'
@@ -240,23 +240,27 @@ export class MongoDestination implements Destination {
240240
async redact(streams: TableDef[], env: string, userId: string): Promise<{ table: string; rows: number }[]> {
241241
const out: { table: string; rows: number }[] = []
242242
for (const stream of streams) {
243-
const col = stream.columns.find((c) => c.name === 'user_id' || (c.name === 'id' && stream.name === 'users'))
244-
if (!col) continue
245-
const r = await this.mirror(stream).deleteMany({ _env: env, [fieldPath(col.path)]: userId })
243+
const refs = userReferenceColumns(stream)
244+
if (refs.length === 0) continue
245+
const r = await this.mirror(stream).deleteMany({
246+
_env: env,
247+
$or: refs.map((c) => ({ [fieldPath(c.path)]: userId })),
248+
})
246249
if (r.deletedCount > 0) out.push({ table: stream.name, rows: r.deletedCount })
247250
}
251+
const namedInData = userReferencePaths(streams).map((p) => ({ [`_raw.data.${p.join('.')}`]: userId }))
248252
// Archived events embed the payload, so they must be scrubbed too.
249253
const ev = await this.store
250254
.collection<MirrorDoc>(this.names.mirror('events'))
251-
.deleteMany({ _env: env, $or: [{ '_raw.data.id': userId }, { '_raw.data.user_id': userId }] })
255+
.deleteMany({ _env: env, $or: [{ '_raw.data.id': userId }, ...namedInData] })
252256
if (ev.deletedCount > 0) out.push({ table: 'events', rows: ev.deletedCount })
253257
// Quarantined events embed the payload too, and they may be the only copy
254258
// of it: an event rejected before its archive write succeeded is nowhere
255259
// in `events`.
256260
const rj = await this.quarantine.deleteMany({
257261
sync_id: this.syncId,
258262
env,
259-
$or: [{ object_id: userId }, { '_raw.data.user_id': userId }],
263+
$or: [{ object_id: userId }, ...namedInData],
260264
})
261265
if (rj.deletedCount > 0) out.push({ table: '_rejected', rows: rj.deletedCount })
262266
return out

‎packages/sync-mongodb/test/engine.test.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -785,12 +785,20 @@ describe('redaction', () => {
785785
it('erases a subject from the mirror and the event archive', async () => {
786786
const user = fake.createUser({ email: 'forget@example.com' })
787787
fake.emitAuthAttempt(user['id'] as string, true)
788+
// Invitations name a user as the one who accepted and as the inviter.
789+
const org = fake.createOrganization()
790+
const accepted = fake.createInvitation(org['id'] as string)
791+
fake.acceptInvitation(accepted['id'] as string, user['id'] as string)
792+
fake.createInvitation(org['id'] as string, { inviter_user_id: user['id'] })
793+
fake.createInvitation(org['id'] as string)
788794
const engine = makeEngine(t, fake)
789795
await engine.backfill()
790796
await engine.tick()
791797

792798
const removed = await engine.redact(user['id'] as string)
793799
expect(removed.length).toBeGreaterThan(0)
800+
expect(removed).toContainEqual({ table: 'invitations', rows: 2 })
801+
expect(await count(t, 'invitations')).toBe(1)
794802
expect(await count(t, 'users')).toBe(0)
795803
expect(await count(t, 'auth_attempts')).toBe(0)
796804
// The archived payloads carry the same personal data and must go too.

‎packages/sync-mysql/README.md‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,10 +105,14 @@ npx workos-sync-mysql init --print-sql --schema <branch database> > schema.sql
105105
npx workos-sync-mysql init --schema <branch database>
106106
```
107107

108-
Once the tables and views are in place, `init` runs no DDL at all: it checks
109-
the live columns, records what it finds, and records the fingerprint. A later
110-
catalog change that adds a column is the one thing that needs another deploy
111-
request; `init` reports the missing column and the DDL for it.
108+
Once the tables and views are in place, `init` changes no schema: it checks
109+
the live columns, records what it finds, and records the fingerprint. To
110+
check a column it has no record of, it creates a temporary table from the
111+
catalog's clauses and drops it again within the session, which is session
112+
state rather than a schema change, needs only the `CREATE TEMPORARY TABLES`
113+
grant, and passes a host that gates schema changes. A later catalog change
114+
that adds a column is the one thing that needs another deploy request;
115+
`init` reports the missing column and the DDL for it.
112116

113117
`listen` with a health port answers `GET /healthz` with `200` while the tail
114118
is polling and `503` once a poll is overdue, and `GET /metrics` in Prometheus

‎packages/sync-mysql/src/destination.ts‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import type {
1616
WriteMode,
1717
WriteStats,
1818
} from '@workos/sync-engine'
19-
import { encodeKey } from '@workos/sync-engine'
19+
import { encodeKey, userReferenceColumns, userReferencePaths } from '@workos/sync-engine'
2020
import { connect, type SqlClient } from './db/client.js'
2121
import { upsertRows, deleteRows, relinkRows, unlinkRows } from './db/upsert.js'
2222
import { diffIdsInSql } from './db/reconcile.js'
@@ -205,29 +205,35 @@ export class MysqlDestination implements Destination {
205205
async redact(streams: TableDef[], env: string, userId: string): Promise<{ table: string; rows: number }[]> {
206206
const out: { table: string; rows: number }[] = []
207207
for (const stream of streams) {
208-
const col = stream.columns.find((c) => c.name === 'user_id' || (c.name === 'id' && stream.name === 'users'))
209-
if (!col) continue
208+
const refs = userReferenceColumns(stream)
209+
if (refs.length === 0) continue
210210
const r = await this.client.query(
211-
`delete from ${this.table(stream)} where \`_env\` = ? and ${ident(col.name)} = ?`,
212-
[env, userId],
211+
`delete from ${this.table(stream)} where \`_env\` = ? and (${refs.map((c) => `${ident(c.name)} = ?`).join(' or ')})`,
212+
[env, ...refs.map(() => userId)],
213213
)
214214
if (r.affectedRows > 0) out.push({ table: stream.name, rows: r.affectedRows })
215215
}
216-
const userIdInData = `json_value(\`_raw\`, ${jsonPath(['data', 'user_id'])} returning char(${KEY_LENGTH}) null on error)`
216+
const paths = userReferencePaths(streams)
217+
const namedInData = paths
218+
.map(
219+
(p) => ` or json_value(\`_raw\`, ${jsonPath(['data', ...p])} returning char(${KEY_LENGTH}) null on error) = ?`,
220+
)
221+
.join('')
222+
const named = paths.map(() => userId)
217223
// Archived events embed the payload, so they must be scrubbed too.
218224
const ev = await this.client.query(
219225
`delete from ${quoteIdent(this.schema)}.\`events\`
220-
where \`_env\` = ? and (\`object_id\` = ? or ${userIdInData} = ?)`,
221-
[env, userId, userId],
226+
where \`_env\` = ? and (\`object_id\` = ?${namedInData})`,
227+
[env, userId, ...named],
222228
)
223229
if (ev.affectedRows > 0) out.push({ table: 'events', rows: ev.affectedRows })
224230
// Quarantined events embed the payload too, and they may be the only copy
225231
// of it: an event rejected before its archive write succeeded is nowhere
226232
// in `events`.
227233
const rj = await this.client.query(
228234
`delete from ${quoteIdent(this.schema)}.\`_rejected\`
229-
where sync_id = ? and env = ? and (object_id = ? or ${userIdInData} = ?)`,
230-
[this.syncId, env, userId, userId],
235+
where sync_id = ? and env = ? and (object_id = ?${namedInData})`,
236+
[this.syncId, env, userId, ...named],
231237
)
232238
if (rj.affectedRows > 0) out.push({ table: '_rejected', rows: rj.affectedRows })
233239
return out

0 commit comments

Comments
 (0)