Skip to content

Commit 09f4f00

Browse files
author
tlonny
committed
Add optional "useUnsafeChunkedAsyncGenerator"
if this flag is set to true, pullChanges is expected to be an AsyncGenerator that yields multiple SyncPullResults pullChanges is expected to be an AsyncGenerator that yields multiple SyncPullResults.
1 parent f5c34eb commit 09f4f00

File tree

4 files changed

+125
-97
lines changed

4 files changed

+125
-97
lines changed

‎src/sync/impl/synchronize.d.ts‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,3 @@
11
import type { SyncArgs } from '../index'
22

3-
export default function synchronize({
4-
database,
5-
pullChanges,
6-
onDidPullChanges,
7-
pushChanges,
8-
sendCreatedAsUpdated,
9-
migrationsEnabledAtVersion,
10-
log,
11-
conflictResolver,
12-
_unsafeBatchPerCollection,
13-
unsafeTurbo,
14-
}: SyncArgs): Promise<void>
3+
export default function synchronize(params: SyncArgs): Promise<void>

‎src/sync/impl/synchronize.js‎

Lines changed: 82 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,23 @@ import {
1414
import { ensureSameDatabase, isChangeSetEmpty, changeSetCount } from './helpers'
1515
import type { SyncArgs, Timestamp, SyncPullStrategy } from '../index'
1616

17-
export default async function synchronize({
18-
database,
19-
pullChanges,
20-
onWillApplyRemoteChanges,
21-
onDidPullChanges,
22-
pushChanges,
23-
sendCreatedAsUpdated = false,
24-
migrationsEnabledAtVersion,
25-
log,
26-
conflictResolver,
27-
_unsafeBatchPerCollection,
28-
unsafeTurbo,
29-
}: SyncArgs): Promise<void> {
17+
async function* liftToAsyncGenerator<T>(promise : Promise<T>) : AsyncGenerator<T, void, void> {
18+
yield await promise
19+
}
20+
21+
export default async function synchronize(params : SyncArgs) : Promise<void> {
22+
const {
23+
database,
24+
onWillApplyRemoteChanges,
25+
onDidPullChanges,
26+
pushChanges,
27+
sendCreatedAsUpdated = false,
28+
migrationsEnabledAtVersion,
29+
log,
30+
conflictResolver,
31+
_unsafeBatchPerCollection,
32+
unsafeTurbo,
33+
} = params
3034
const resetCount = database._resetCount
3135
log && (log.startedAt = new Date())
3236
log && (log.phase = 'starting')
@@ -46,78 +50,84 @@ export default async function synchronize({
4650
log && (log.phase = 'ready to pull')
4751

4852
// $FlowFixMe
49-
const pullResult = await pullChanges({
50-
lastPulledAt,
51-
schemaVersion,
52-
migration,
53-
})
53+
const pullChunks = params.useUnsafeChunkedAsyncGenerator
54+
? params.pullChanges({ lastPulledAt, schemaVersion, migration })
55+
: liftToAsyncGenerator(params.pullChanges({ lastPulledAt, schemaVersion, migration }))
5456
log && (log.phase = 'pulled')
5557

56-
let newLastPulledAt: Timestamp = (pullResult: any).timestamp
57-
const remoteChangeCount = pullResult.changes ? changeSetCount(pullResult.changes) : NaN
58+
let newLastPulledAt: Timestamp | null = null
59+
for await (const pullResult of pullChunks) {
60+
let newLastPulledAt: Timestamp = (pullResult: any).timestamp
61+
const remoteChangeCount = pullResult.changes ? changeSetCount(pullResult.changes) : NaN
5862

59-
if (onWillApplyRemoteChanges) {
60-
await onWillApplyRemoteChanges({ remoteChangeCount })
61-
}
62-
63-
await database.write(async () => {
64-
ensureSameDatabase(database, resetCount)
65-
invariant(
66-
lastPulledAt === (await getLastPulledAt(database)),
67-
'[Sync] Concurrent synchronization is not allowed. More than one synchronize() call was running at the same time, and the later one was aborted before committing results to local database.',
68-
)
63+
if (onWillApplyRemoteChanges) {
64+
await onWillApplyRemoteChanges({ remoteChangeCount })
65+
}
6966

70-
if (unsafeTurbo) {
67+
await database.write(async () => {
68+
ensureSameDatabase(database, resetCount)
7169
invariant(
72-
!_unsafeBatchPerCollection,
73-
'unsafeTurbo must not be used with _unsafeBatchPerCollection',
70+
lastPulledAt === (await getLastPulledAt(database)),
71+
'[Sync] Concurrent synchronization is not allowed. More than one synchronize() call was running at the same time, and the later one was aborted before committing results to local database.',
7472
)
73+
74+
if (unsafeTurbo) {
75+
invariant(
76+
!_unsafeBatchPerCollection,
77+
'unsafeTurbo must not be used with _unsafeBatchPerCollection',
78+
)
79+
invariant(
80+
'syncJson' in pullResult || 'syncJsonId' in pullResult,
81+
'missing syncJson/syncJsonId',
82+
)
83+
invariant(lastPulledAt === null, 'unsafeTurbo can only be used as the first sync')
84+
85+
const syncJsonId = pullResult.syncJsonId || Math.floor(Math.random() * 1000000000)
86+
87+
if (pullResult.syncJson) {
88+
await database.adapter.provideSyncJson(syncJsonId, pullResult.syncJson)
89+
}
90+
91+
const resultRest = await database.adapter.unsafeLoadFromSync(syncJsonId)
92+
newLastPulledAt = resultRest.timestamp
93+
onDidPullChanges && onDidPullChanges(resultRest)
94+
}
95+
96+
log && (log.newLastPulledAt = newLastPulledAt)
7597
invariant(
76-
'syncJson' in pullResult || 'syncJsonId' in pullResult,
77-
'missing syncJson/syncJsonId',
98+
typeof newLastPulledAt === 'number' && newLastPulledAt > 0,
99+
`pullChanges() returned invalid timestamp ${newLastPulledAt}. timestamp must be a non-zero number`,
78100
)
79-
invariant(lastPulledAt === null, 'unsafeTurbo can only be used as the first sync')
80101

81-
const syncJsonId = pullResult.syncJsonId || Math.floor(Math.random() * 1000000000)
82-
83-
if (pullResult.syncJson) {
84-
await database.adapter.provideSyncJson(syncJsonId, pullResult.syncJson)
102+
if (!unsafeTurbo) {
103+
// $FlowFixMe
104+
const { changes: remoteChanges, ...resultRest } = pullResult
105+
log && (log.remoteChangeCount = remoteChangeCount)
106+
// $FlowFixMe
107+
await applyRemoteChanges(remoteChanges, {
108+
db: database,
109+
strategy: ((pullResult: any).experimentalStrategy: ?SyncPullStrategy),
110+
sendCreatedAsUpdated,
111+
log,
112+
conflictResolver,
113+
_unsafeBatchPerCollection,
114+
})
115+
onDidPullChanges && onDidPullChanges(resultRest)
85116
}
86117

87-
const resultRest = await database.adapter.unsafeLoadFromSync(syncJsonId)
88-
newLastPulledAt = resultRest.timestamp
89-
onDidPullChanges && onDidPullChanges(resultRest)
90-
}
118+
log && (log.phase = 'applied remote changes')
119+
await setLastPulledAt(database, newLastPulledAt)
91120

92-
log && (log.newLastPulledAt = newLastPulledAt)
93-
invariant(
94-
typeof newLastPulledAt === 'number' && newLastPulledAt > 0,
95-
`pullChanges() returned invalid timestamp ${newLastPulledAt}. timestamp must be a non-zero number`,
96-
)
97-
98-
if (!unsafeTurbo) {
99-
// $FlowFixMe
100-
const { changes: remoteChanges, ...resultRest } = pullResult
101-
log && (log.remoteChangeCount = remoteChangeCount)
102-
// $FlowFixMe
103-
await applyRemoteChanges(remoteChanges, {
104-
db: database,
105-
strategy: ((pullResult: any).experimentalStrategy: ?SyncPullStrategy),
106-
sendCreatedAsUpdated,
107-
log,
108-
conflictResolver,
109-
_unsafeBatchPerCollection,
110-
})
111-
onDidPullChanges && onDidPullChanges(resultRest)
112-
}
121+
if (shouldSaveSchemaVersion) {
122+
await setLastPulledSchemaVersion(database, schemaVersion)
123+
}
124+
}, 'sync-synchronize-apply')
113125

114-
log && (log.phase = 'applied remote changes')
115-
await setLastPulledAt(database, newLastPulledAt)
126+
}
116127

117-
if (shouldSaveSchemaVersion) {
118-
await setLastPulledSchemaVersion(database, schemaVersion)
119-
}
120-
}, 'sync-synchronize-apply')
128+
if(newLastPulledAt === null) {
129+
throw new Error('An empty generator was used')
130+
}
121131

122132
// push phase
123133
if (pushChanges) {
@@ -145,4 +155,4 @@ export default async function synchronize({
145155

146156
log && (log.finishedAt = new Date())
147157
log && (log.phase = 'done')
148-
}
158+
}

‎src/sync/index.d.ts‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -57,35 +57,49 @@ export type SyncConflictResolver = (
5757
) => DirtyRaw
5858

5959
export type SyncArgs = $Exact<{
60-
database: Database
61-
pullChanges: (_: SyncPullArgs) => Promise<SyncPullResult>
62-
pushChanges?: (_: SyncPushArgs) => Promise<SyncPushResult | undefined | void>
60+
database: Database,
61+
pushChanges?: (SyncPushArgs) => Promise<?SyncPushResult>,
6362
// version at which support for migration syncs was added - the version BEFORE first syncable migration
64-
migrationsEnabledAtVersion?: SchemaVersion
65-
sendCreatedAsUpdated?: boolean
66-
log?: SyncLog
63+
migrationsEnabledAtVersion?: SchemaVersion,
64+
sendCreatedAsUpdated?: boolean,
65+
log?: SyncLog,
6766
// Advanced (unsafe) customization point. Useful when you have subtle invariants between multiple
6867
// columns and want to have them updated consistently, or to implement partial sync
6968
// It's called for every record being updated locally, so be sure that this function is FAST.
7069
// If you don't want to change default behavior for a given record, return `resolved` as is
7170
// Note that it's safe to mutate `resolved` object, so you can skip copying it for performance.
72-
conflictResolver?: SyncConflictResolver
71+
conflictResolver?: SyncConflictResolver,
7372
// commits changes in multiple batches, and not one - temporary workaround for memory issue
74-
_unsafeBatchPerCollection?: boolean
73+
_unsafeBatchPerCollection?: boolean,
7574
// Advanced optimization - pullChanges must return syncJson or syncJsonId to be processed by native code.
7675
// This can only be used on initial (login) sync, not for incremental syncs.
7776
// This can only be used with SQLiteAdapter with JSI enabled.
7877
// The exact API may change between versions of WatermelonDB.
7978
// See documentation for more details.
80-
unsafeTurbo?: boolean
81-
// Called after pullChanges with whatever was returned by pullChanges, minus `changes`. Useful
79+
unsafeTurbo?: boolean,
80+
// Called after changes are pulled with whatever was returned by pullChanges, minus `changes`. Useful
8281
// when using turbo mode
83-
onDidPullChanges?: (_: Object) => Promise<void>
82+
onDidPullChanges?: (Object) => Promise<void>,
8483
// Called after pullChanges is done, but before these changes are applied. Some stats about the pulled
8584
// changes are passed as arguments. An advanced user can use this for example to show some UI to the user
8685
// when processing a very large sync (could be useful for replacement syncs). Note that remote change count
8786
// is NaN in turbo mode.
88-
onWillApplyRemoteChanges?: (info: $Exact<{ remoteChangeCount: number }>) => Promise<void>
87+
onWillApplyRemoteChanges?: (info: $Exact<{ remoteChangeCount: number }>) => Promise<void>,
88+
useUnsafeChunkedAsyncGenerator: true,
89+
pullChanges: (SyncPullArgs) => AsyncGenerator<SyncPullResult, void, void>,
90+
}> | $Exact<{
91+
database: Database,
92+
pushChanges?: (SyncPushArgs) => Promise<?SyncPushResult>,
93+
migrationsEnabledAtVersion?: SchemaVersion,
94+
sendCreatedAsUpdated?: boolean,
95+
log?: SyncLog,
96+
conflictResolver?: SyncConflictResolver,
97+
_unsafeBatchPerCollection?: boolean,
98+
unsafeTurbo?: boolean,
99+
onDidPullChanges?: (Object) => Promise<void>,
100+
onWillApplyRemoteChanges?: (info: $Exact<{ remoteChangeCount: number }>) => Promise<void>,
101+
useUnsafeChunkedAsyncGenerator: false,
102+
pullChanges: (SyncPullArgs) => Promise<SyncPullResult>,
89103
}>
90104

91105
export function synchronize(args: SyncArgs): Promise<void>

‎src/sync/index.js‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ export type SyncConflictResolver = (
8585
// TODO: JSDoc'ify this
8686
export type SyncArgs = $Exact<{
8787
database: Database,
88-
pullChanges: (SyncPullArgs) => Promise<SyncPullResult>,
8988
pushChanges?: (SyncPushArgs) => Promise<?SyncPushResult>,
9089
// version at which support for migration syncs was added - the version BEFORE first syncable migration
9190
migrationsEnabledAtVersion?: SchemaVersion,
@@ -113,6 +112,22 @@ export type SyncArgs = $Exact<{
113112
// when processing a very large sync (could be useful for replacement syncs). Note that remote change count
114113
// is NaN in turbo mode.
115114
onWillApplyRemoteChanges?: (info: $Exact<{ remoteChangeCount: number }>) => Promise<void>,
115+
// If this flag is set to true, then pullChanges is expected to return an async generator that yields multiple SyncPullResults. This allows WatermelonDB to process incoming sync data in chunks, which can be useful for syncs that end up being particularly large and might otherwise cause an OOM if held in memory all at once. It is not recommended to use this flag unless you are sure - should a sync fail midway through, the database will be in a "partially synced" and potentially inconsistent state. Although it will eventually become consistent after a subsequent sync is completed, it might cause unexpected issues depending on your particular use case.
116+
useUnsafeChunkedAsyncGenerator?: false,
117+
pullChanges: (SyncPullArgs) => Promise<SyncPullResult>,
118+
}> | $Exact<{
119+
database: Database,
120+
pushChanges?: (SyncPushArgs) => Promise<?SyncPushResult>,
121+
migrationsEnabledAtVersion?: SchemaVersion,
122+
sendCreatedAsUpdated?: boolean,
123+
log?: SyncLog,
124+
conflictResolver?: SyncConflictResolver,
125+
_unsafeBatchPerCollection?: boolean,
126+
unsafeTurbo?: boolean,
127+
onDidPullChanges?: (Object) => Promise<void>,
128+
onWillApplyRemoteChanges?: (info: $Exact<{ remoteChangeCount: number }>) => Promise<void>,
129+
useUnsafeChunkedAsyncGenerator: true,
130+
pullChanges: (SyncPullArgs) => AsyncGenerator<SyncPullResult, void, void>,
116131
}>
117132

118133
/**

0 commit comments

Comments
 (0)