refactor: set up remote wasm loading and local caching;

Author: eveffer

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@inspatial/cloud",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "license": "Apache-2.0",
   "exports": {
     ".": "./mod.ts",

src/orm/db/postgres/in-pg/in-pg.ts

@@ -2,12 +2,10 @@ import { normalizePath } from "./src/convert.ts";
 import { FileManager } from "./src/fileManager/in-pg-files.ts";
 import { PGMem } from "./src/pgMem.ts";
 import { SysCalls } from "./src/syscalls.ts";
-import { ExitStatus } from "./src/utils.ts";
+import { ExitStatus, getTempDirBase } from "./src/utils.ts";
 import { WasmLoader } from "./src/wasmLoader.ts";
 import type { InPgOptions } from "./types.ts";
 
-const fileData = Deno.readFileSync(import.meta.dirname + `/src/inpg.data`);
-const wasmData = Deno.readFileSync(import.meta.dirname + `/src/inpg.wasm`);
 export class InPG implements Deno.Conn {
   pgMem: PGMem;
   wasmLoader;
@@ -38,6 +36,15 @@ export class InPG implements Deno.Conn {
   debug?: boolean;
   #onStdErr: (message: any) => void;
   #onStdOut: (message: any) => void;
+  #wasmData: Uint8Array = new Uint8Array(0);
+  #fileData: Uint8Array = new Uint8Array(0);
+  get wasmData() {
+    return this.#wasmData;
+  }
+
+  get fileData() {
+    return this.#fileData;
+  }
 
   log(type: "out" | "err", chunk: Uint8Array) {
     if (!this.debug) {
@@ -89,11 +96,10 @@ export class InPG implements Deno.Conn {
     this.#bufferData = new Uint8Array(0);
     this.runtimeInitialized = false;
     this.pgMem = new PGMem(this);
-    this.wasmLoader = new WasmLoader(this, wasmData);
+    this.wasmLoader = new WasmLoader(this);
     this.readEmAsmArgsArray = [];
     this.fileManager = new FileManager(this, {
       debug: options?.debug,
-      fileData,
     });
     this.sysCalls = new SysCalls(this);
 
@@ -112,6 +118,7 @@ export class InPG implements Deno.Conn {
 
     await this.wasmLoader.load();
   }
+
   sendQuery(message: Uint8Array) {
     this.wasmLoader.callExportFunction("use_wire", 1);
     const msg_len = message.length;
@@ -143,6 +150,7 @@ export class InPG implements Deno.Conn {
   }
 
   async run() {
+    await this.loadRemoteFiles();
     await this.#setup();
     await this.initRuntime();
     this.#callMain(this.args);
@@ -178,7 +186,39 @@ export class InPG implements Deno.Conn {
       Deno.exit(1);
     }
   }
+  async loadRemoteFiles() {
+    const wasmUrl =
+      "https://github.com/inspatiallabs/inspatial-cloud/releases/download/0.2.2/inpg.wasm";
+    const dataUrl =
+      "https://github.com/inspatiallabs/inspatial-cloud/releases/download/0.2.2/inpg.data";
+    const tmpDirBase = getTempDirBase();
+    const tmpDir = `${tmpDirBase}/in-pg`;
+
+    Deno.mkdirSync(tmpDir, { recursive: true });
 
+    const wasmFile = `${tmpDir}/inpg.0.2.2.wasm`;
+    const dataFile = `${tmpDir}/inpg.0.2.2.data`;
+    this.#wasmData = await this.loadRemoteFile(wasmFile, wasmUrl);
+    this.#fileData = await this.loadRemoteFile(dataFile, dataUrl);
+  }
+  async loadRemoteFile(fileName: string, remoteUrl: string) {
+    try {
+      Deno.statSync(fileName);
+      return Deno.readFileSync(fileName);
+    } catch (e) {
+      if (e instanceof Deno.errors.NotFound) {
+        const response = await fetch(remoteUrl);
+        if (!response.ok) {
+          throw new Error(`Failed to fetch file: ${response.statusText}`);
+        }
+        const data = new Uint8Array(await response.arrayBuffer());
+        await Deno.writeFile(fileName, data);
+        return data;
+      } else {
+        throw e;
+      }
+    }
+  }
   initDB() {
     const result = this.wasmLoader.callExportFunction("pgl_initdb");
     return result;

src/orm/db/postgres/in-pg/src/fileManager/in-pg-files.ts

@@ -4,7 +4,8 @@ import { ERRNO_CODES } from "../constants.ts";
 import { normalizePath } from "../convert.ts";
 import type { PGMem } from "../pgMem.ts";
 import { MemFile, PostgresFile } from "./pg-file.ts";
-
+const dataURL =
+  "https://github.com/inspatiallabs/inspatial-cloud/releases/download/0.2.2/inpg.data";
 export class FileManager {
   openFiles: Map<number, PGFile | PGFileMem>;
   openTmpFDs: Map<string, number>;
@@ -23,7 +24,6 @@ export class FileManager {
   postgresFiles: Map<string, PostgresFile>;
   constructor(inPg: InPG, options: {
     debug?: boolean;
-    fileData: Uint8Array;
   }) {
     this.lastDebugMessage = "";
     this.dirReadOffsets = new Map();
@@ -44,7 +44,7 @@ export class FileManager {
         truncate: true,
       });
     }
-    this.loadPostgresFiles(options.fileData);
+    this.loadPostgresFiles();
     this.setupStdStreams();
   }
   clearTmp() {
@@ -412,7 +412,8 @@ export class FileManager {
     }
     return null;
   }
-  loadPostgresFiles(data: Uint8Array) {
+  async loadPostgresFiles() {
+    const data = this.inPg.fileData;
     let offset = 0;
     const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
 

src/orm/db/postgres/in-pg/src/utils.ts

@@ -1,3 +1,4 @@
+import { normalizePath } from "./convert.ts";
 import type { PGMem } from "./pgMem.ts";
 
 export class ExitStatus {
@@ -57,3 +58,16 @@ export class ExceptionInfo {
     return this.mem.HEAPU32[(this.ptr + 16) >> 2];
   }
 }
+
+export function getTempDirBase() {
+  let path = Deno.makeTempFileSync();
+  if (Deno.build.os === "windows") {
+    const driveLetter = path.match(/^[a-zA-Z]:/)?.[0] || "";
+    path = `${driveLetter}${normalizePath(path)}`;
+  }
+  Deno.removeSync(path);
+  const parts = path.split("/");
+  parts.pop();
+  const tmpdir = parts.join("/");
+  return tmpdir;
+}

src/orm/db/postgres/in-pg/src/wasmLoader.ts

@@ -6,7 +6,9 @@ import { uleb128Encode, UTF8ArrayToString } from "./convert.ts";
 
 import type { PGMem } from "./pgMem.ts";
 import { setupInvokeImports } from "./setupInvokes.ts";
-
+import { getTempDirBase } from "./utils.ts";
+const wasmURL =
+  "https://github.com/inspatiallabs/inspatial-cloud/releases/download/0.2.2/inpg.wasm";
 class LDSO {
   loadedLibsByName: Record<string, DSO>;
   loadedLibsByHandle: Record<string, DSO>;
@@ -37,12 +39,10 @@ export class WasmLoader {
   wasmExports: WebAssembly.Exports;
   freeTableIndexes: Array<number>;
   inPg: InPG;
-  wasmData: Uint8Array;
-
   get pgMem(): PGMem {
     return this.inPg.pgMem;
   }
-  constructor(inPg: InPG, wasmData: Uint8Array) {
+  constructor(inPg: InPG) {
     this.GOT = {};
     this.inPg = inPg;
     this.wasmExports = {};
@@ -54,7 +54,6 @@ export class WasmLoader {
       initial: this.inPg.tableSize,
       element: "anyfunc",
     });
-    this.wasmData = wasmData;
 
     const GOT = this.GOT;
     const currentModuleWeakSymbols = this.currentModuleWeakSymbols;
@@ -74,6 +73,7 @@ export class WasmLoader {
       },
     };
   }
+
   async load() {
     const GOTHandler = this.GOTHandler;
     this.setupImports();
@@ -93,7 +93,7 @@ export class WasmLoader {
       ) as WebAssembly.ModuleImports,
     };
 
-    const result = await WebAssembly.instantiate(this.wasmData, info);
+    const result = await WebAssembly.instantiate(this.inPg.wasmData, info);
     this.wasmExports = result.instance.exports;
 
     for (var [sym, exp] of Object.entries(result.instance.exports)) {