|
1 | 1 | import type { DexieCloudDB } from '../db/DexieCloudDB'; |
2 | | -import { BlobRef } from './blobResolve'; |
| 2 | +import { BlobRef, ResolvedBlob } from './blobResolve'; |
| 3 | +import { BlobSavingQueue } from './BlobSavingQueue'; |
3 | 4 | import { loadCachedAccessToken } from './loadCachedAccessToken'; |
4 | 5 |
|
5 | 6 | /** |
6 | | - * Deduplicates in-flight blob downloads. |
| 7 | + * Owns the full lifecycle of downloaded blobs: |
| 8 | + * 1. Deduplicates concurrent downloads for the same ref. |
| 9 | + * 2. Bounds the number of concurrent network fetches (MAX_CONCURRENT) |
| 10 | + * so that ad-hoc reads can't starve the HTTP connection pool. Calls |
| 11 | + * beyond the cap queue in FIFO order as slots free. The slot is held |
| 12 | + * only for the duration of the fetch — NOT until persistence — to |
| 13 | + * avoid deadlocks when a single object contains more blob refs than |
| 14 | + * MAX_CONCURRENT (a sequential resolver would otherwise hold every |
| 15 | + * slot itself while waiting for the next). |
| 16 | + * 3. Keeps the in-flight promise alive after the network fetch completes, |
| 17 | + * until the blob has been persisted back to IndexedDB. This way, |
| 18 | + * readers that ask for the same ref while it is queued for saving |
| 19 | + * can piggyback on the existing promise instead of refetching. |
| 20 | + * In-flight membership and slot ownership are independent: a piggyback |
| 21 | + * reader consumes neither a slot nor extra memory beyond the existing |
| 22 | + * cached Uint8Array. |
| 23 | + * 4. Persists resolved blobs via an internal BlobSavingQueue, and |
| 24 | + * releases the in-flight entry when persistence completes. |
7 | 25 | * |
8 | | - * Both the blob-resolve middleware and the eager blob downloader may |
9 | | - * try to fetch the same blob concurrently. This tracker ensures each |
10 | | - * unique blob ref is only downloaded once — subsequent requests for |
11 | | - * the same ref piggyback on the existing promise. |
| 26 | + * Both the blob-resolve middleware and the eager blob downloader use this |
| 27 | + * tracker. Instantiate once per DexieCloudDB. |
| 28 | + */ |
| 29 | + |
| 30 | +/** |
| 31 | + * Maximum number of concurrent blob fetches. |
12 | 32 | * |
13 | | - * Instantiate once per DexieCloudDB. |
| 33 | + * Historically 6 to match the HTTP/1.1 same-origin connection cap that |
| 34 | + * browsers enforce. With HTTP/2 (the typical transport for Dexie Cloud |
| 35 | + * today) many streams multiplex over a single TCP connection, so the |
| 36 | + * old cap is overly conservative. 10 is a modest bump that still keeps |
| 37 | + * memory pressure (in-flight Uint8Arrays) and server load bounded. |
| 38 | + * Can be made configurable via DexieCloudOptions if a real need arises. |
14 | 39 | */ |
| 40 | +export const MAX_CONCURRENT = 10; |
| 41 | + |
15 | 42 | export class BlobDownloadTracker { |
16 | 43 | private inFlight = new Map<string, Promise<Uint8Array>>(); |
17 | 44 | private db: DexieCloudDB; |
| 45 | + private savingQueue: BlobSavingQueue; |
| 46 | + private activeFetches = 0; |
| 47 | + private waiting: Array<() => void> = []; |
18 | 48 |
|
19 | 49 | constructor(db: DexieCloudDB) { |
20 | 50 | this.db = db; |
| 51 | + this.savingQueue = new BlobSavingQueue(db, (refs) => { |
| 52 | + // Called by the queue when a save transaction has completed |
| 53 | + // (regardless of success). Drop the in-flight cache entries now — |
| 54 | + // any future reader will go through IndexedDB instead. |
| 55 | + for (const ref of refs) { |
| 56 | + this.inFlight.delete(ref); |
| 57 | + } |
| 58 | + }); |
21 | 59 | } |
22 | 60 |
|
23 | 61 | /** |
24 | | - * Download a blob, deduplicating concurrent requests for the same ref. |
| 62 | + * Download a blob, deduplicating concurrent requests for the same ref |
| 63 | + * and respecting the global fetch concurrency cap. |
| 64 | + * |
| 65 | + * Lifecycle: |
| 66 | + * - Slot is acquired before the fetch and released as soon as the |
| 67 | + * fetch settles (success or failure). |
| 68 | + * - The in-flight entry survives a successful fetch and lives on |
| 69 | + * until persistence completes (via enqueueSave) or releaseRefs |
| 70 | + * is called. On fetch failure, the entry is removed immediately |
| 71 | + * so a future call can retry. |
25 | 72 | * |
26 | 73 | * @param blobRef - The BlobRef to download |
27 | 74 | * @param dbUrl - Base URL for the database (e.g., 'https://mydb.dexie.cloud') |
28 | 75 | */ |
29 | 76 | download(blobRef: BlobRef, dbUrl: string): Promise<Uint8Array> { |
30 | 77 | let promise = this.inFlight.get(blobRef.ref); |
31 | 78 | if (!promise) { |
32 | | - promise = loadCachedAccessToken(this.db) |
33 | | - .then((accessToken) => { |
34 | | - // accessToken may be null for anonymous/unauthenticated users. |
35 | | - // Public realm blobs (rlm-public) are accessible without auth. |
36 | | - // downloadBlob will omit the Authorization header when token is null. |
37 | | - return downloadBlob(blobRef, dbUrl, accessToken); |
38 | | - }) |
39 | | - .finally(() => this.inFlight.delete(blobRef.ref)); |
40 | | - // When the promise settles (either fulfilled or rejected), remove it from the in-flight map |
| 79 | + promise = this.acquireSlot() |
| 80 | + .then(() => |
| 81 | + this.downloadBlob(blobRef, dbUrl).finally(() => this.releaseSlot()) |
| 82 | + ) |
| 83 | + .catch((err) => { |
| 84 | + // On error, remove immediately so a future call can retry. |
| 85 | + // (Slot already released by the .finally above.) |
| 86 | + this.inFlight.delete(blobRef.ref); |
| 87 | + throw err; |
| 88 | + }); |
41 | 89 | this.inFlight.set(blobRef.ref, promise); |
42 | 90 | } |
43 | 91 | return promise; |
44 | 92 | } |
45 | | -} |
46 | | -/** |
47 | | - * Download blob data from server via proxy endpoint. |
48 | | - * Uses auth header for authentication (same as sync). |
49 | | - * When accessToken is null, the request is made without Authorization header — |
50 | | - * this allows downloading blobs from public realms (rlm-public) for |
51 | | - * unauthenticated users. |
52 | | - * |
53 | | - * @param blobRef - The BlobRef to download |
54 | | - * @param dbUrl - Base URL for the database (e.g., 'https://mydb.dexie.cloud') |
55 | | - * @param accessToken - Access token for authentication, or null for anonymous access |
56 | | - */ |
57 | 93 |
|
58 | | -export async function downloadBlob( |
59 | | - blobRef: BlobRef, |
60 | | - dbUrl: string, |
61 | | - accessToken: string | null |
62 | | -): Promise<Uint8Array> { |
63 | | - const downloadUrl = `${dbUrl}/blob/${blobRef.ref}`; |
64 | | - const headers: HeadersInit = {}; |
65 | | - if (accessToken) { |
66 | | - headers['Authorization'] = `Bearer ${accessToken}`; |
| 94 | + /** |
| 95 | + * Queue resolved blobs for persisting back to IndexedDB. |
| 96 | + * When the save transaction completes, the corresponding in-flight |
| 97 | + * entries are released. |
| 98 | + */ |
| 99 | + enqueueSave( |
| 100 | + tableName: string, |
| 101 | + primaryKey: any, |
| 102 | + resolvedBlobs: ResolvedBlob[] |
| 103 | + ): void { |
| 104 | + this.savingQueue.saveBlobs(tableName, primaryKey, resolvedBlobs); |
67 | 105 | } |
68 | | - const response = await fetch(downloadUrl, { headers }); |
69 | 106 |
|
70 | | - if (!response.ok) { |
71 | | - throw new Error( |
72 | | - `Failed to download blob ${blobRef.ref}: ${response.status} ${response.statusText}` |
73 | | - ); |
| 107 | + /** |
| 108 | + * Wait until all previously enqueued saves have been persisted to |
| 109 | + * IndexedDB. Used by callers that need to make decisions based on |
| 110 | + * on-disk state — e.g., the eager downloader looping over rows with |
| 111 | + * `_hasBlobRefs=1` in chunks, where each iteration must see the |
| 112 | + * previous chunk's writes before re-querying. |
| 113 | + * |
| 114 | + * New saves enqueued AFTER drainPendingSaves() is called do NOT extend |
| 115 | + * the wait. |
| 116 | + */ |
| 117 | + drainPendingSaves(): Promise<void> { |
| 118 | + return this.savingQueue.drain(); |
74 | 119 | } |
75 | 120 |
|
76 | | - const arrayBuffer = await response.arrayBuffer(); |
77 | | - return new Uint8Array(arrayBuffer); |
| 121 | + /** |
| 122 | + * Release in-flight entries without going through the internal saving |
| 123 | + * queue. Used when the caller persists the blobs itself, or when no |
| 124 | + * primary key was available and the data won't be persisted at all. |
| 125 | + */ |
| 126 | + releaseRefs(refs: string[]): void { |
| 127 | + for (const ref of refs) { |
| 128 | + this.inFlight.delete(ref); |
| 129 | + } |
| 130 | + } |
| 131 | + |
| 132 | + private acquireSlot(): Promise<void> { |
| 133 | + if (this.activeFetches < MAX_CONCURRENT) { |
| 134 | + this.activeFetches++; |
| 135 | + return Promise.resolve(); |
| 136 | + } |
| 137 | + return new Promise<void>((resolve) => { |
| 138 | + this.waiting.push(() => { |
| 139 | + this.activeFetches++; |
| 140 | + resolve(); |
| 141 | + }); |
| 142 | + }); |
| 143 | + } |
| 144 | + |
| 145 | + private releaseSlot(): void { |
| 146 | + this.activeFetches--; |
| 147 | + const next = this.waiting.shift(); |
| 148 | + if (next) next(); |
| 149 | + } |
| 150 | + |
| 151 | + /** |
| 152 | + * Download blob data from server via proxy endpoint. |
| 153 | + * Uses auth header for authentication (same as sync). |
| 154 | + * When accessToken is null, the request is made without Authorization header — |
| 155 | + * this allows downloading blobs from public realms (rlm-public) for |
| 156 | + * unauthenticated users. |
| 157 | + * |
| 158 | + * @param blobRef - The BlobRef to download |
| 159 | + * @param dbUrl - Base URL for the database (e.g., 'https://mydb.dexie.cloud') |
| 160 | + */ |
| 161 | + |
| 162 | + private async downloadBlob( |
| 163 | + blobRef: BlobRef, |
| 164 | + dbUrl: string |
| 165 | + ): Promise<Uint8Array> { |
| 166 | + const accessToken = await loadCachedAccessToken(this.db); |
| 167 | + const downloadUrl = `${dbUrl}/blob/${blobRef.ref}`; |
| 168 | + const headers: HeadersInit = {}; |
| 169 | + if (accessToken) { |
| 170 | + // accessToken may be null for anonymous/unauthenticated users. |
| 171 | + // Public realm blobs (rlm-public) are accessible without auth. |
| 172 | + // downloadBlob will omit the Authorization header when token is null. |
| 173 | + headers['Authorization'] = `Bearer ${accessToken}`; |
| 174 | + } |
| 175 | + // cache: 'no-store' prevents the browser from storing this response in its |
| 176 | + // HTTP cache. The server sets a long Expires/Cache-Control header on blob |
| 177 | + // responses (blobs are immutable and content-addressed), which would |
| 178 | + // otherwise cause the browser to keep a copy in its disk cache in addition |
| 179 | + // to the copy we persist to IndexedDB — doubling storage for every blob. |
| 180 | + // Since we always persist to IndexedDB and subsequent reads go through |
| 181 | + // IndexedDB (never re-fetch), the browser cache copy is pure overhead. |
| 182 | + const response = await fetch(downloadUrl, { headers, cache: 'no-store' }); |
| 183 | + |
| 184 | + if (!response.ok) { |
| 185 | + throw new Error( |
| 186 | + `Failed to download blob ${blobRef.ref}: ${response.status} ${response.statusText}` |
| 187 | + ); |
| 188 | + } |
| 189 | + |
| 190 | + const arrayBuffer = await response.arrayBuffer(); |
| 191 | + return new Uint8Array(arrayBuffer); |
| 192 | + } |
78 | 193 | } |
0 commit comments