Skip to content

Commit bac1bfc

Browse files
authored
fix(mysql): publish releases, port --exclude-streams and the oversized verdict (#47)
1 parent 5df9add commit bac1bfc

5 files changed

Lines changed: 129 additions & 17 deletions

File tree

‎.github/workflows/release-please.yml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: Release Please
22

3-
# Both packages have npm trusted publishers that name THIS file, not
3+
# Every package has an npm trusted publisher that names THIS file, not
44
# release.yml: when a publish runs inside a workflow_call, npm validates the
55
# calling workflow's filename. Provenance is attached automatically once the
66
# repo is public; nothing here changes for that.
@@ -35,6 +35,8 @@ jobs:
3535
sync-engine--tag_name: ${{ steps.release.outputs['packages/sync-engine--tag_name'] }}
3636
sync-postgres--release_created: ${{ steps.release.outputs['packages/sync-postgres--release_created'] }}
3737
sync-postgres--tag_name: ${{ steps.release.outputs['packages/sync-postgres--tag_name'] }}
38+
sync-mysql--release_created: ${{ steps.release.outputs['packages/sync-mysql--release_created'] }}
39+
sync-mysql--tag_name: ${{ steps.release.outputs['packages/sync-mysql--tag_name'] }}
3840
steps:
3941
# Release PRs and tags come from the SDK Automation Bot, as in workos-node.
4042
# Events raised with GITHUB_TOKEN start no workflow runs, so a release PR
@@ -80,3 +82,15 @@ jobs:
8082
tag_name: ${{ needs.release-please.outputs.sync-postgres--tag_name }}
8183
secrets:
8284
PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}
85+
86+
publish-sync-mysql:
87+
needs: [release-please, publish-sync-engine]
88+
if: >-
89+
${{ !cancelled()
90+
&& needs.publish-sync-engine.result != 'failure'
91+
&& needs.release-please.outputs.sync-mysql--release_created == 'true' }}
92+
uses: ./.github/workflows/release.yml
93+
with:
94+
tag_name: ${{ needs.release-please.outputs.sync-mysql--tag_name }}
95+
secrets:
96+
PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}

‎packages/sync-mysql/README.md‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,18 @@ npx workos-sync-mysql backfill # load everything that already exists
3838
npx workos-sync-mysql listen # tail the Events API, forever
3939
```
4040

41-
| Command | What it does |
42-
| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
43-
| `init [--repair] [--print-sql]` | Create the database and bookkeeping tables. Idempotent; safe on every deploy. Exits 2 if a generated column has drifted from the catalog, and fails if a view under a generated `_live` name is not the generated shape; `--repair` rebuilds columns from `_raw` and replaces such views. `--print-sql` prints the DDL for a fresh database instead of running it; apply it, then run `init` once. |
44-
| `backfill [--resume] [--max-pages <n>]` | Load current state from the REST list endpoints. `--resume` continues each stream from its checkpoint and skips streams already complete. `--max-pages` stops after `n` pages across all streams, for a host with a time limit; run again with `--resume` until every stream reports `complete`. |
45-
| `listen [--health-port <n>]` | Poll the Events API continuously as the sole writer; waits if another listener holds the lease. Finishes the current batch on `SIGINT` or `SIGTERM`. With a port, serves `GET /healthz` and `/metrics` on it. |
46-
| `tick` | One poll, then exit. For a cron job instead of a long-running process. Exits 3 if a listener holds the lease. |
47-
| `reconcile [--repair] [--resume] [--max-pages <n>]` | Diff ids against the API. Exits 2 on drift; `--repair` tombstones missed deletes and inserts missed creates. `--max-pages` stops starting streams once `n` pages have been fetched; run again with `--resume` until every stream reports `complete`. |
48-
| `refresh [--resume] [--max-pages <n>]` | Re-list top-level streams from the REST API, replacing each row; `--max-pages` and `--resume` slice it like `backfill`, with which it shares checkpoints. Re-run `backfill` for per-parent streams. |
49-
| `status` | Cursor, time since the last successful poll, and per-stream row counts, tombstones and lag. |
50-
| `redact --user <id>` | Erase one subject from the mirror and the event archive. |
51-
| `rejected [--replay] [--id <event_id>]` | Show events the tail quarantined; exits 2 if there are any. `--replay` re-applies them (one, with `--id`) once the cause is fixed. |
52-
| `streams` | List what this build can mirror, with group, mode, and whether it is on by default. |
41+
| Command | What it does |
42+
| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
43+
| `init [--repair] [--print-sql]` | Create the database and bookkeeping tables. Idempotent; safe on every deploy. Exits 2 if a generated column has drifted from the catalog, and fails if a view under a generated `_live` name is not the generated shape; `--repair` rebuilds columns from `_raw` and replaces such views. `--print-sql` prints the DDL for a fresh database instead of running it; apply it, then run `init` once. |
44+
| `backfill [--resume] [--max-pages <n>]` | Load current state from the REST list endpoints. `--resume` continues each stream from its checkpoint and skips streams already complete. `--max-pages` stops after `n` pages across all streams, for a host with a time limit; run again with `--resume` until every stream reports `complete`. |
45+
| `listen [--health-port <n>]` | Poll the Events API continuously as the sole writer; waits if another listener holds the lease. Finishes the current batch on `SIGINT` or `SIGTERM`. With a port, serves `GET /healthz` and `/metrics` on it. |
46+
| `tick` | One poll, then exit. For a cron job instead of a long-running process. Exits 3 if a listener holds the lease. |
47+
| `reconcile [--repair] [--resume] [--max-pages <n>] [--exclude-streams <a,b>]` | Diff ids against the API. Exits 2 on drift; `--repair` tombstones missed deletes and inserts missed creates. `--max-pages` stops starting streams once `n` pages have been fetched, for a host with a time limit; a stream that starts runs to its last page, since a diff needs the whole listing, unless it alone needs more than `n`: it is then abandoned, reported `oversized`, and skipped by `--resume` until a run with a larger budget or none. Run again with `--resume` until every stream reports `complete`. `--exclude-streams` leaves streams out, for one that has to run somewhere without a time limit. |
48+
| `refresh [--resume] [--max-pages <n>]` | Re-list top-level streams from the REST API, replacing each row; `--max-pages` and `--resume` slice it like `backfill`, with which it shares checkpoints. Re-run `backfill` for per-parent streams. |
49+
| `status` | Cursor, time since the last successful poll, streams `reconcile` has abandoned as oversized, and per-stream row counts, tombstones and lag. |
50+
| `redact --user <id>` | Erase one subject from the mirror and the event archive. |
51+
| `rejected [--replay] [--id <event_id>]` | Show events the tail quarantined; exits 2 if there are any. `--replay` re-applies them (one, with `--id`) once the cause is fixed. |
52+
| `streams` | List what this build can mirror, with group, mode, and whether it is on by default. |
5353

5454
Flags every command takes:
5555

‎packages/sync-mysql/src/cli.ts‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* page the list endpoints, optionally n pages at a time
1010
* workos-sync-mysql listen tail the Events API
1111
* workos-sync-mysql tick one poll, then exit (for cron)
12-
* workos-sync-mysql reconcile [--repair] detect (and optionally fix) drift
12+
* workos-sync-mysql reconcile [--repair] [--exclude-streams <a,b>] detect (and optionally fix) drift
1313
* workos-sync-mysql refresh re-list to refresh partial event fields
1414
* workos-sync-mysql status cursor age and per-stream freshness
1515
* workos-sync-mysql redact --user <id> erase one subject
@@ -244,6 +244,27 @@ export function parseMaxPages(value: string | boolean | undefined): number | und
244244
return n
245245
}
246246

247+
/**
248+
* `--exclude-streams <a,b>` names streams `reconcile` leaves out, for one
249+
* whose listing has to run somewhere without a time limit. A bare flag is
250+
* refused here: excluding nothing by accident is the mistake the flag exists
251+
* to prevent. The names themselves are checked by the engine.
252+
*/
253+
export function parseExcludeStreams(value: string | boolean | undefined): string[] | undefined {
254+
if (value === undefined || value === false) return undefined
255+
const names =
256+
typeof value === 'string'
257+
? value
258+
.split(',')
259+
.map((s) => s.trim())
260+
.filter(Boolean)
261+
: []
262+
if (names.length === 0) {
263+
throw new Error(`invalid --exclude-streams "${String(value)}": expected a comma-separated list of stream names`)
264+
}
265+
return names
266+
}
267+
247268
function configFromEnv(
248269
flags: Record<string, string | boolean>,
249270
destination: MysqlDestination,
@@ -289,12 +310,16 @@ Commands
289310
with a port, serve GET /healthz and /metrics on it
290311
tick One poll of the Events API, then exit (for a cron job);
291312
exits 3 if a listener holds the lease
292-
reconcile [--repair] [--resume] [--max-pages <n>]
313+
reconcile [--repair] [--resume] [--max-pages <n>] [--exclude-streams <a,b>]
293314
Compare the mirror against the API; --repair fixes drift.
294315
--max-pages stops starting streams once n pages have been
295316
fetched (a stream that starts runs to its last page, since
296-
a diff needs the whole listing); run again with --resume
297-
until every stream reports complete
317+
a diff needs the whole listing, unless it alone needs more
318+
than n: it is then abandoned and reported oversized until
319+
a run with a larger budget or none); run again with
320+
--resume until every stream reports complete.
321+
--exclude-streams leaves streams out, for one that has to
322+
run somewhere without a time limit
298323
refresh [--resume] [--max-pages <n>]
299324
Re-list top-level streams, replacing each row;
300325
--max-pages and --resume slice it like backfill, with
@@ -459,6 +484,7 @@ async function runCommand(command: string, flags: Record<string, string | boolea
459484
repair: Boolean(flags['repair']),
460485
resume: Boolean(flags['resume']),
461486
maxPages: parseMaxPages(flags['max-pages']),
487+
excludeStreams: parseExcludeStreams(flags['exclude-streams']),
462488
})
463489
process.stdout.write(JSON.stringify(r, null, 2) + '\n')
464490
// A non-zero exit makes this usable as a scheduled check.

0 commit comments

Comments
 (0)