Skip to content

Commit aeff598

Browse files
carlassmanncarlassmann
committed
fix: recover from root covalue corruption
1 parent dccd21b commit aeff598

3 files changed

Lines changed: 203 additions & 8 deletions

File tree

‎scripts/check-worker-credentials.ts‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#!/usr/bin/env tsx
2+
3+
import { startWorker } from "jazz-tools/worker"
4+
import { ServerAccount } from "../src/shared/schema/server"
5+
6+
type CliArgs = {
7+
syncServer: string
8+
accountId: string
9+
accountSecret: string
10+
timeoutMs: number
11+
}
12+
13+
async function main() {
14+
if (process.argv.includes("--help") || process.argv.includes("-h")) {
15+
printUsage()
16+
process.exit(0)
17+
}
18+
19+
let args = parseArgs(process.argv.slice(2))
20+
if (!args) {
21+
printUsage()
22+
process.exit(1)
23+
}
24+
25+
console.log("Checking Jazz worker credentials...")
26+
console.log(`- sync server: ${args.syncServer}`)
27+
console.log(`- expected account: ${args.accountId}`)
28+
29+
let timeout = new Promise<never>((_, reject) => {
30+
setTimeout(
31+
() => reject(new Error(`Timed out after ${args.timeoutMs}ms`)),
32+
args.timeoutMs,
33+
)
34+
})
35+
36+
let workerPromise = startWorker({
37+
AccountSchema: ServerAccount,
38+
syncServer: args.syncServer,
39+
accountID: args.accountId,
40+
accountSecret: args.accountSecret,
41+
skipInboxLoad: true,
42+
asActiveAccount: false,
43+
})
44+
45+
let workerResult = await Promise.race([workerPromise, timeout])
46+
let actualAccountId = workerResult.worker.$jazz.id
47+
48+
console.log(`- loaded account: ${actualAccountId}`)
49+
50+
if (actualAccountId !== args.accountId) {
51+
console.error("\nFAIL: secret authenticated a different account")
52+
console.error(`expected: ${args.accountId}`)
53+
console.error(`actual: ${actualAccountId}`)
54+
await workerResult.shutdownWorker?.()
55+
process.exit(2)
56+
}
57+
58+
await workerResult.shutdownWorker?.()
59+
console.log("\nOK: worker account + secret match")
60+
}
61+
62+
function parseArgs(argv: string[]): CliArgs | null {
63+
let syncServer =
64+
readFlag(argv, "--sync-server") || process.env.PUBLIC_JAZZ_SYNC_SERVER
65+
let accountId =
66+
readFlag(argv, "--account-id") || process.env.PUBLIC_JAZZ_WORKER_ACCOUNT
67+
let accountSecret =
68+
readFlag(argv, "--account-secret") || process.env.JAZZ_WORKER_SECRET
69+
let timeoutRaw = readFlag(argv, "--timeout-ms")
70+
let timeoutMs = timeoutRaw ? Number(timeoutRaw) : 30_000
71+
72+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
73+
console.error("Invalid --timeout-ms value")
74+
return null
75+
}
76+
77+
if (!syncServer || !accountId || !accountSecret) {
78+
console.error("Missing required inputs")
79+
return null
80+
}
81+
82+
return {
83+
syncServer,
84+
accountId,
85+
accountSecret,
86+
timeoutMs,
87+
}
88+
}
89+
90+
function readFlag(argv: string[], key: string): string | undefined {
91+
let index = argv.indexOf(key)
92+
if (index === -1) return undefined
93+
let value = argv[index + 1]
94+
if (!value || value.startsWith("--")) return undefined
95+
return value
96+
}
97+
98+
function printUsage() {
99+
console.log(`
100+
Usage:
101+
tsx scripts/check-worker-credentials.ts \\
102+
--sync-server <ws://...> \\
103+
--account-id <co_...> \\
104+
--account-secret <sealerSecret_.../signerSecret_...>
105+
106+
Or use env vars:
107+
PUBLIC_JAZZ_SYNC_SERVER
108+
PUBLIC_JAZZ_WORKER_ACCOUNT
109+
JAZZ_WORKER_SECRET
110+
111+
Optional:
112+
--timeout-ms <number> (default: 30000)
113+
`)
114+
}
115+
116+
main().catch(error => {
117+
let message = error instanceof Error ? error.message : String(error)
118+
console.error("\nFAIL:", message)
119+
process.exit(1)
120+
})

‎src/server/lib/utils.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,27 @@ async function getServerWorker(): Promise<ServerWorker> {
3535
}
3636

3737
let timeoutId: ReturnType<typeof setTimeout> | undefined
38+
let workerAccountId = PUBLIC_JAZZ_WORKER_ACCOUNT
39+
let syncServer = PUBLIC_JAZZ_SYNC_SERVER
3840

3941
let workerInitPromise = startWorker({
4042
AccountSchema: ServerAccount,
41-
syncServer: PUBLIC_JAZZ_SYNC_SERVER,
42-
accountID: PUBLIC_JAZZ_WORKER_ACCOUNT,
43+
syncServer,
44+
accountID: workerAccountId,
4345
accountSecret: JAZZ_WORKER_SECRET,
4446
skipInboxLoad: true,
4547
asActiveAccount: false,
4648
})
4749
.then(result => result.worker)
4850
.catch(error => {
51+
console.error("[PushWorker] Worker start failed", {
52+
workerAccountId,
53+
syncServer,
54+
error:
55+
error instanceof Error
56+
? { name: error.name, message: error.message, stack: error.stack }
57+
: String(error),
58+
})
4959
throw error
5060
})
5161

‎src/shared/schema/server.ts‎

Lines changed: 71 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@ export let NotificationSettingsRefsRecord = co.record(
1212
NotificationSettingsRef,
1313
)
1414

15+
function isRootAuthCorruptionError(error: unknown) {
16+
if (!(error instanceof Error)) return false
17+
let message = error.message || ""
18+
return message.includes("Failed to deeply load CoValue")
19+
}
20+
1521
export let ServerAccountRoot = co.map({
1622
notificationSettingsRefs: co.list(NotificationSettingsRef).optional(),
1723
notificationSettingsRefsV2: NotificationSettingsRefsRecord.optional(),
@@ -26,16 +32,75 @@ export let ServerAccount = co
2632
if (!account.$jazz.has("root")) {
2733
let newRoot = ServerAccountRoot.create(
2834
{
29-
notificationSettingsRefs: co.list(NotificationSettingsRef).create([]),
30-
notificationSettingsRefsV2: NotificationSettingsRefsRecord.create({}),
35+
notificationSettingsRefs: co
36+
.list(NotificationSettingsRef)
37+
.create([], account),
38+
notificationSettingsRefsV2: NotificationSettingsRefsRecord.create(
39+
{},
40+
account,
41+
),
3142
},
32-
account.$jazz.owner,
43+
account,
3344
)
3445
account.$jazz.set("root", newRoot)
3546
} else {
36-
let { root } = await account.$jazz.ensureLoaded({
37-
resolve: { root: true },
38-
})
47+
let loaded
48+
try {
49+
loaded = await account.$jazz.ensureLoaded({
50+
resolve: { root: true },
51+
})
52+
} catch (error) {
53+
let currentRoot = Reflect.get(account, "root")
54+
console.error("[ServerAccount] Failed to load root", {
55+
accountId: account.$jazz.id,
56+
rootValueType: typeof currentRoot,
57+
rootId:
58+
currentRoot &&
59+
typeof currentRoot === "object" &&
60+
"$jazz" in currentRoot
61+
? Reflect.get(Reflect.get(currentRoot, "$jazz"), "id")
62+
: undefined,
63+
error:
64+
error instanceof Error
65+
? {
66+
name: error.name,
67+
message: error.message,
68+
stack: error.stack,
69+
}
70+
: String(error),
71+
})
72+
73+
if (isRootAuthCorruptionError(error)) {
74+
let repairedRoot = ServerAccountRoot.create(
75+
{
76+
notificationSettingsRefs: co
77+
.list(NotificationSettingsRef)
78+
.create([], account),
79+
notificationSettingsRefsV2: NotificationSettingsRefsRecord.create(
80+
{},
81+
account,
82+
),
83+
},
84+
account,
85+
)
86+
account.$jazz.set("root", repairedRoot)
87+
console.warn("[ServerAccount] Repaired unreadable root", {
88+
accountId: account.$jazz.id,
89+
oldRootId:
90+
currentRoot &&
91+
typeof currentRoot === "object" &&
92+
"$jazz" in currentRoot
93+
? Reflect.get(Reflect.get(currentRoot, "$jazz"), "id")
94+
: undefined,
95+
newRootId: repairedRoot.$jazz.id,
96+
})
97+
return
98+
}
99+
100+
throw error
101+
}
102+
103+
let { root } = loaded
39104
if (root.notificationSettingsRefs === undefined) {
40105
root.$jazz.set(
41106
"notificationSettingsRefs",

0 commit comments

Comments
 (0)