Merge pull request #124 from rhashimoto/file-export

Add export to file demo.

Author: rhashimoto

demo/file/index.html

@@ -4,6 +4,7 @@
   <meta charset="utf-8">
   <title>Local file transfer</title>
   <style>
+    button { margin-top: 1em; }
     pre { margin: 0 }
   </style>
   <script type="module" defer src="./index.js"></script>
@@ -13,6 +14,9 @@ <h1>Sample database importer</h1>
   <div>
     <input type="file" id="file-import">
   </div>
+  <div>
+    <button id="file-export" disabled>Download</button>
+  </div>
   <hr>
 </body>
 </html>

demo/file/index.js

@@ -7,6 +7,33 @@ const DB_NAME = SEARCH_PARAMS.get('db') ?? 'sqlite.db';
 
 const DBFILE_MAGIC = 'SQLite format 3\x00';
 
+// Use a service worker for downloading. This is currently the only
+// cross-browser way to stream to a local file.
+navigator.serviceWorker.register('service-worker.js', { type: 'module' });
+(async function() {
+  // Enable the export button when the service worker is responding.
+  while (true) {
+    let delay = 25;
+    const response = await fetch('./export?check=true');
+    if (response.ok) {
+      // @ts-ignore
+      document.getElementById('file-export').disabled = false;
+      return;
+    }
+    await new Promise(resolve => setTimeout(resolve, delay));
+    delay = Math.min(delay * 2, 5000);
+  }
+})();
+
+document.getElementById('file-export').addEventListener('click', async () => {
+  // Fetch from the special URL handled by the service worker. All
+  // the magic happens there.
+  const url = new URL('./export', location.href);
+  url.searchParams.set('idb', IDB_NAME);
+  url.searchParams.set('db', DB_NAME);
+  window.open(url);
+});
+
 document.getElementById('file-import').addEventListener('change', async event => {
   let vfs;
   try {
@@ -16,6 +43,7 @@ document.getElementById('file-import').addEventListener('change', async event =>
     await importDatabase(vfs, DB_NAME, event.target.files[0].stream());
     log('Import complete');
 
+    // Use a Worker to verify the database with SQLite.
     log('Verifying database integrity');
     const url = new URL('./verifier.js', location.href);
     url.searchParams.set('idb', IDB_NAME);
@@ -45,6 +73,8 @@ document.getElementById('file-import').addEventListener('change', async event =>
  * @param {ReadableStream} stream 
  */
 async function importDatabase(vfs, path, stream) {
+  // This generator converts arbitrary sized chunks from the stream
+  // into SQLite pages.
   async function* pagify() {
     /** @type {Uint8Array[]} */ const chunks = [];
     const reader = stream.getReader();
@@ -57,7 +87,7 @@ async function importDatabase(vfs, path, stream) {
       chunks.push(value);
     }
 
-    // Assemble the file header.
+    // Consolidate the header into a single DataView.
     let copyOffset = 0;
     const header = new DataView(new ArrayBuffer(32));
     for (const chunk of chunks) {
@@ -71,11 +101,12 @@ async function importDatabase(vfs, path, stream) {
       throw new Error('Not a SQLite database file');
     }
 
-    // Extract page parameters.
+    // Extract page fields.
     const pageSize = (field => field === 1 ? 65536 : field)(header.getUint16(16));
     const pageCount = header.getUint32(28);
     log(`${pageCount} pages, ${pageSize} bytes each, ${pageCount * pageSize} bytes total`);
 
+    // Yield each page in sequence.
     log('Copying pages...');
     for (let i = 0; i < pageCount; ++i) {
       // Read enough chunks to produce the next page.
@@ -85,7 +116,7 @@ async function importDatabase(vfs, path, stream) {
         chunks.push(value);
       }
 
-      // Assemble the page.
+      // Assemble the page into a single Uint8Array.
       // TODO: Optimize case where first chunk has >= pageSize bytes.
       let copyOffset = 0;
       const page = new Uint8Array(pageSize);
@@ -94,8 +125,8 @@ async function importDatabase(vfs, path, stream) {
         const src = chunks[0].subarray(0, pageSize - copyOffset);
         const dst = new Uint8Array(page.buffer, copyOffset);
         dst.set(src);
-
         copyOffset += src.byteLength;
+
         if (src.byteLength === chunks[0].byteLength) {
           // All the bytes in the chunk were consumed.
           chunks.shift();
@@ -130,8 +161,8 @@ async function importDatabase(vfs, path, stream) {
     onFinally.push(() => vfs.xUnlock(fileId, VFS.SQLITE_LOCK_SHARED));
     await check(vfs.xLock(fileId, VFS.SQLITE_LOCK_EXCLUSIVE));
 
-    const empty = new DataView(new ArrayBuffer(4));
-    await vfs.xFileControl(fileId, VFS.SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, empty);
+    const ignored = new DataView(new ArrayBuffer(4));
+    await vfs.xFileControl(fileId, VFS.SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, ignored);
 
     // Write pages.
     let iOffset = 0;
@@ -140,8 +171,8 @@ async function importDatabase(vfs, path, stream) {
       iOffset += page.byteLength;
     }
 
-    await vfs.xFileControl(fileId, VFS.SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, empty);
-    await vfs.xFileControl(fileId, VFS.SQLITE_FCNTL_SYNC, empty);
+    await vfs.xFileControl(fileId, VFS.SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, ignored);
+    await vfs.xFileControl(fileId, VFS.SQLITE_FCNTL_SYNC, ignored);
     await vfs.xSync(fileId, VFS.SQLITE_SYNC_NORMAL);
   } finally {
     while (onFinally.length) {

demo/file/service-worker.js

@@ -0,0 +1,105 @@
+import * as VFS from "../../src/VFS.js";
+import { IDBBatchAtomicVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+
+// Install the service worker as soon as possible.
+globalThis.addEventListener('install', (/** @type {ExtendableEvent} */ event) => {
+  event.waitUntil(globalThis.skipWaiting());
+});
+globalThis.addEventListener('activate', (/** @type {ExtendableEvent} */ event) => {
+  event.waitUntil(globalThis.clients.claim());
+});
+
+globalThis.addEventListener('fetch', async (/** @type {FetchEvent} */ event) => {
+  const url = new URL(event.request.url);
+  if (!url.href.includes(globalThis.registration.scope)) return;
+  if (!url.pathname.endsWith('export')) return;
+
+  if (url.searchParams.has('check')) {
+    return event.respondWith(new Response('OK'));
+  }
+
+  const vfs = new IDBBatchAtomicVFS(url.searchParams.get('idb'));
+  const path = url.searchParams.get('db');
+  const source = new DatabaseSource(vfs, path);
+  event.waitUntil(source.isDone.finally(() => vfs.close()));
+  return event.respondWith(new Response(new ReadableStream(source), {
+    headers: {
+      "Content-Type": 'application/octet-stream',
+      "Content-Disposition": `attachment; filename=sqlite.db`
+    }
+  }));
+});
+
+class DatabaseSource {
+  isDone;
+
+  #vfs;
+  #path;
+  #fileId = Math.floor(Math.random() * 0x100000000);
+  #iOffset = 0;
+  #bytesRemaining = 0;
+
+  #onDone = [];
+  #resolve;
+  #reject;
+
+  constructor(vfs, path) {
+    this.#vfs = vfs;
+    this.#path = path;
+    this.isDone = new Promise((resolve, reject) => {
+      this.#resolve = resolve;
+      this.#reject = reject;
+    }).finally(async () => {
+      while (this.#onDone.length) {
+        await this.#onDone.pop()();
+      }
+    });
+  }
+
+  async start(controller) {
+    try {
+      // Open the file for reading.
+      const flags = VFS.SQLITE_OPEN_MAIN_DB | VFS.SQLITE_OPEN_READONLY;
+      await check(this.#vfs.xOpen(this.#path, this.#fileId, flags, {setInt32(){}}));
+      this.#onDone.push(() => this.#vfs.xClose(this.#fileId));
+      await check(this.#vfs.xLock(this.#fileId, VFS.SQLITE_LOCK_SHARED));
+      this.#onDone.push(() => this.#vfs.xUnlock(this.#fileId, VFS.SQLITE_LOCK_NONE));
+
+      // Get the file size.
+      const fileSize = new DataView(new ArrayBuffer(8));
+      await check(this.#vfs.xFileSize(this.#fileId, fileSize));
+      this.#bytesRemaining = Number(fileSize.getBigUint64(0, true));
+    } catch (e) {
+      controller.error(e);
+      this.#reject(e);
+    }
+  }
+
+  async pull(controller) {
+    try {
+      const buffer = new Uint8Array(Math.min(this.#bytesRemaining, 65536));
+      await check(this.#vfs.xRead(this.#fileId, buffer, this.#iOffset));
+      controller.enqueue(buffer);
+
+      this.#iOffset += buffer.byteLength;
+      this.#bytesRemaining -= buffer.byteLength;
+      if (this.#bytesRemaining === 0) {
+        controller.close();
+        this.#resolve();
+      }
+    } catch (e) {
+      controller.error(e);
+      this.#reject(e);
+    }
+  }
+
+  cancel(reason) {
+    this.#reject(new Error(reason));
+  }
+};
+
+async function check(code) {
+  if (await code !== VFS.SQLITE_OK) {
+    throw new Error(`Error code: ${code}`);
+  }
+}