From 9dd05a0f2666b48ec76f22d18d3cf25ac8d9333c Mon Sep 17 00:00:00 2001 From: bbm Date: Tue, 5 Mar 2024 14:54:05 -0500 Subject: [PATCH 1/9] adding sources for proxy worker (using WORKERFS) --- src/h5wasm.worker.ts | 5 +++ src/lib_worker.ts | 55 +++++++++++++++++++++++++++++ src/worker_proxy.ts | 82 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/h5wasm.worker.ts create mode 100644 src/lib_worker.ts create mode 100644 src/worker_proxy.ts diff --git a/src/h5wasm.worker.ts b/src/h5wasm.worker.ts new file mode 100644 index 0000000..84460cf --- /dev/null +++ b/src/h5wasm.worker.ts @@ -0,0 +1,5 @@ +import * as Comlink from 'comlink'; +import { api } from './lib_worker'; + +Comlink.expose(api); + diff --git a/src/lib_worker.ts b/src/lib_worker.ts new file mode 100644 index 0000000..1e51e20 --- /dev/null +++ b/src/lib_worker.ts @@ -0,0 +1,55 @@ +import * as h5wasm from '../dist/esm/hdf5_hl'; + +const WORKERFS_MOUNT = '/workerfs'; + +async function save_to_workerfs(file: File) { + const { FS, WORKERFS, mount } = await workerfs_promise; + const { name: filename, size } = file; + const output_path = `${WORKERFS_MOUNT}/${filename}`; + if (FS.analyzePath(output_path).exists) { + console.warn(`File ${output_path} already exists. Overwriting...`); + } + const outfile = WORKERFS.createNode(mount, filename, WORKERFS.FILE_MODE, 0, file); + return output_path; +} + +async function save_bytes_to_memfs(filename: string, bytes: Uint8Array) { + const { FS } = await h5wasm.ready; + const output_path = filename; + if (FS.analyzePath(output_path).exists) { + console.warn(`File ${output_path} already exists. Overwriting...`); + } + FS.writeFile(output_path, bytes); + return output_path; +} + +async function save_to_memfs(file: File) { + const { name: filename } = file; + const ab = await file.arrayBuffer(); + return save_bytes_to_memfs(filename, new Uint8Array(ab)); +} + +async function _mount_workerfs() { + const { FS } = await h5wasm.ready; + const { filesystems: { WORKERFS } } = FS; + if (!FS.analyzePath(WORKERFS_MOUNT).exists) { + FS.mkdir(WORKERFS_MOUNT); + } + const mount = FS.mount(WORKERFS, {}, WORKERFS_MOUNT); + return { FS, WORKERFS, mount }; +} + +const workerfs_promise = _mount_workerfs(); + +export const api = { + ready: h5wasm.ready, + save_to_workerfs, + save_to_memfs, + save_bytes_to_memfs, + H5WasmFile: h5wasm.File, + Dataset: h5wasm.Dataset, + Group: h5wasm.Group, + Datatype: h5wasm.Datatype, + BrokenSoftLink: h5wasm.BrokenSoftLink, +} + diff --git a/src/worker_proxy.ts b/src/worker_proxy.ts new file mode 100644 index 0000000..00040b9 --- /dev/null +++ b/src/worker_proxy.ts @@ -0,0 +1,82 @@ +import * as Comlink from 'comlink'; +import type { api } from './lib_worker.ts'; +// @ts-ignore (esbuild-plugin-inline-worker will rewrite this import) +import DedicatedWorker from './h5wasm.worker.ts'; + +import { ACCESS_MODES } from './hdf5_hl.ts'; +import type { File as H5WasmFile, Group, Dataset, Datatype, BrokenSoftLink } from './hdf5_hl.ts'; +export type { H5WasmFile, Group, Dataset, Datatype, BrokenSoftLink }; + +type ACCESS_MODESTRING = keyof typeof ACCESS_MODES; + +const worker = new DedicatedWorker(); // new Worker('./worker.js'); +const remote = Comlink.wrap(worker) as Comlink.Remote; + +export class GroupProxy { + proxy: Comlink.Remote; + file_id: bigint; + constructor(proxy: Comlink.Remote, file_id: bigint) { + this.proxy = proxy; + this.file_id = file_id; + } + + async keys() { + return await this.proxy.keys(); + } + + async paths() { + return await this.proxy.paths(); + } + + async get(name: string = "/") { + const dumb_obj = await this.proxy.get(name); + // convert to a proxy of the object: + if (dumb_obj?.type === "Group") { + const new_group_proxy = await new remote.Group(dumb_obj.file_id, dumb_obj.path); + return new GroupProxy(new_group_proxy, this.file_id); + } + else if (dumb_obj?.type === "Dataset") { + return new remote.Dataset(dumb_obj.file_id, dumb_obj.path); + } + else if (dumb_obj?.type === "Datatype") { + return new remote.Datatype(); + } + else if (dumb_obj?.type === "BrokenSoftLink") { + return new remote.BrokenSoftLink(dumb_obj?.target); + } + return + } +} + +export class FileProxy extends GroupProxy { + filename: string; + mode: ACCESS_MODESTRING; + constructor(proxy: Comlink.Remote, file_id: bigint, filename: string, mode: ACCESS_MODESTRING = 'r') { + super(proxy, file_id); + this.filename = filename; + this.mode = mode; + } +} + +export async function get_file_proxy(filename: string, mode: ACCESS_MODESTRING = 'r') { + const file_proxy = await new remote.H5WasmFile(filename, mode); + const file_id = await file_proxy.file_id; + return new FileProxy(file_proxy, file_id, filename, mode); +} + +export async function save_to_workerfs(file: File) { + const { name, lastModified, size } = file; + console.log(`Saving file ${name} of size ${lastModified} to workerfs...`); + return await remote.save_to_workerfs(file); +} + +export async function save_to_memfs(file: File) { + const { name, lastModified, size } = file; + console.log(`Saving file ${name} of size ${lastModified} to memfs...`); + return await remote.save_to_memfs(file); +} + +export async function save_bytes_to_memfs(filename: string, bytes: Uint8Array) { + console.log(`Saving bytes to memfs...`); + return await remote.save_bytes_to_memfs(filename, bytes); +} \ No newline at end of file From aa414c3fad58c87441a8c2cd82ec9bb859ffe6bd Mon Sep 17 00:00:00 2001 From: bbm Date: Tue, 5 Mar 2024 14:54:32 -0500 Subject: [PATCH 2/9] don't use import.meta.url to avoid issues in worker --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e14d4e..47d449e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ set_target_properties(hdf5_util PROPERTIES -s ENVIRONMENT=web,worker \ -s SINGLE_FILE \ -s EXPORT_ES6=1 \ + -s USE_ES6_IMPORT_META=0 \ -s FORCE_FILESYSTEM=1 \ -s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'FS', 'AsciiToString', 'UTF8ToString']\" \ -s EXPORTED_FUNCTIONS=\"${EXPORTED_FUNCTIONS_STRING}\"" From 769cb2acacae99f64ca490dc414acf148036467e Mon Sep 17 00:00:00 2001 From: bbm Date: Tue, 5 Mar 2024 14:55:06 -0500 Subject: [PATCH 3/9] add comlink and esbuild-plugin-inline-worker for building worker bundle --- package-lock.json | 253 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 3 + 2 files changed, 256 insertions(+) diff --git a/package-lock.json b/package-lock.json index 9b7bc3c..1f7aa10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "version": "0.7.2", "license": "SEE LICENSE IN LICENSE.txt", "devDependencies": { + "comlink": "^4.4.1", "esbuild": "^0.15.16", + "esbuild-plugin-inline-worker": "^0.1.1", "typescript": "^4.5.4" } }, @@ -45,6 +47,18 @@ "node": ">=12" } }, + "node_modules/comlink": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.4.1.tgz", + "integrity": "sha512-+1dlx0aY5Jo1vHy/tSsIGpSkN4tS9rZSW8FIhG0JH/crs9wwweswIo/POr451r7bZww3hFbPAKnTpimzL/mm4Q==", + "dev": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true + }, "node_modules/esbuild": { "version": "0.15.18", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.18.tgz", @@ -338,6 +352,16 @@ "node": ">=12" } }, + "node_modules/esbuild-plugin-inline-worker": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/esbuild-plugin-inline-worker/-/esbuild-plugin-inline-worker-0.1.1.tgz", + "integrity": "sha512-VmFqsQKxUlbM51C1y5bRiMeyc1x2yTdMXhKB6S//++g9aCBg8TfGsbKxl5ZDkCGquqLY+RmEk93TBNd0i35dPA==", + "dev": true, + "dependencies": { + "esbuild": "latest", + "find-cache-dir": "^3.3.1" + } + }, "node_modules/esbuild-sunos-64": { "version": "0.15.18", "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.18.tgz", @@ -402,6 +426,129 @@ "node": ">=12" } }, + "node_modules/find-cache-dir": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", + "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "dev": true, + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/avajs/find-cache-dir?sponsor=1" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/typescript": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.4.tgz", @@ -431,6 +578,18 @@ "dev": true, "optional": true }, + "comlink": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.4.1.tgz", + "integrity": "sha512-+1dlx0aY5Jo1vHy/tSsIGpSkN4tS9rZSW8FIhG0JH/crs9wwweswIo/POr451r7bZww3hFbPAKnTpimzL/mm4Q==", + "dev": true + }, + "commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true + }, "esbuild": { "version": "0.15.18", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.18.tgz", @@ -573,6 +732,16 @@ "dev": true, "optional": true }, + "esbuild-plugin-inline-worker": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/esbuild-plugin-inline-worker/-/esbuild-plugin-inline-worker-0.1.1.tgz", + "integrity": "sha512-VmFqsQKxUlbM51C1y5bRiMeyc1x2yTdMXhKB6S//++g9aCBg8TfGsbKxl5ZDkCGquqLY+RmEk93TBNd0i35dPA==", + "dev": true, + "requires": { + "esbuild": "latest", + "find-cache-dir": "^3.3.1" + } + }, "esbuild-sunos-64": { "version": "0.15.18", "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.18.tgz", @@ -601,6 +770,90 @@ "dev": true, "optional": true }, + "find-cache-dir": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", + "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "requires": { + "find-up": "^4.0.0" + } + }, + "semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true + }, "typescript": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.4.tgz", diff --git a/package.json b/package.json index ee841b9..9530d4e 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "build_node": "tsc src/hdf5_hl.ts --outDir dist/node --target es2020 --allowJs --esModuleInterop", "build_iife": "esbuild --bundle dist/esm/hdf5_hl.js --outfile=dist/iife/h5wasm.js --format=iife --global-name=h5wasm", "build_types": "tsc --declaration --strict --emitDeclarationOnly src/hdf5_hl.ts --target es2020", + "build_worker_bundle": "node build_worker_bundle.mjs", "test": "node ./test/test.mjs" }, "repository": { @@ -39,7 +40,9 @@ "author": "Brian B. Maranville", "license": "SEE LICENSE IN LICENSE.txt", "devDependencies": { + "comlink": "^4.4.1", "esbuild": "^0.15.16", + "esbuild-plugin-inline-worker": "^0.1.1", "typescript": "^4.5.4" } } From 17a42161fc3f8e4510d8a8be9457ca80314c0634 Mon Sep 17 00:00:00 2001 From: bbm Date: Tue, 5 Mar 2024 15:11:46 -0500 Subject: [PATCH 4/9] need to transfer bytes for writing --- src/worker_proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker_proxy.ts b/src/worker_proxy.ts index 00040b9..979ae9e 100644 --- a/src/worker_proxy.ts +++ b/src/worker_proxy.ts @@ -78,5 +78,5 @@ export async function save_to_memfs(file: File) { export async function save_bytes_to_memfs(filename: string, bytes: Uint8Array) { console.log(`Saving bytes to memfs...`); - return await remote.save_bytes_to_memfs(filename, bytes); + return await remote.save_bytes_to_memfs(filename, Comlink.transfer(bytes, [bytes.buffer])); } \ No newline at end of file From 3bc0511e918a8269000c41faaa8d37f18ed1fdc0 Mon Sep 17 00:00:00 2001 From: bbm Date: Tue, 5 Mar 2024 15:55:44 -0500 Subject: [PATCH 5/9] allow writing files from bytes with full path --- src/lib_worker.ts | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/lib_worker.ts b/src/lib_worker.ts index 1e51e20..92917e8 100644 --- a/src/lib_worker.ts +++ b/src/lib_worker.ts @@ -1,6 +1,25 @@ import * as h5wasm from '../dist/esm/hdf5_hl'; -const WORKERFS_MOUNT = '/workerfs'; +export const WORKERFS_MOUNT = '/workerfs'; + +function dirname(path: string) { + // adapted from dirname function in posixpath.py + const sep = "/"; + const sep_index = path.lastIndexOf(sep) + 1; + let head = path.slice(0, sep_index); + if (head && head !== sep.repeat(head.length)) { + // strip end slashes + head = head.replace(/(\/)+$/, ''); + } + return head; +} + +function basename(path: string) { + // adapted from basename function in posixpath.py + const sep = "/"; + const sep_index = path.lastIndexOf(sep) + 1; + return path.slice(sep_index); +} async function save_to_workerfs(file: File) { const { FS, WORKERFS, mount } = await workerfs_promise; @@ -13,12 +32,17 @@ async function save_to_workerfs(file: File) { return output_path; } -async function save_bytes_to_memfs(filename: string, bytes: Uint8Array) { +async function save_bytes_to_memfs(filepath: string, bytes: Uint8Array) { const { FS } = await h5wasm.ready; - const output_path = filename; + const path = dirname(filepath); + const filename = basename(filepath); + const output_path = filepath; if (FS.analyzePath(output_path).exists) { console.warn(`File ${output_path} already exists. Overwriting...`); } + if (!FS.analyzePath(path).exists) { + FS.mkdirTree(path); + } FS.writeFile(output_path, bytes); return output_path; } From 6b4fdd937b2cfae3a98a5b138d8c61bed2359d8e Mon Sep 17 00:00:00 2001 From: bbm Date: Wed, 6 Mar 2024 10:53:05 -0500 Subject: [PATCH 6/9] add API functions for returning H5WasmFile proxy directly from File or bytes object sent, and plugin loader --- src/lib_worker.ts | 35 ++++++++++++++++++++++++++++++++++- src/worker_proxy.ts | 45 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/src/lib_worker.ts b/src/lib_worker.ts index 92917e8..4dd1af6 100644 --- a/src/lib_worker.ts +++ b/src/lib_worker.ts @@ -1,4 +1,5 @@ import * as h5wasm from '../dist/esm/hdf5_hl'; +import type { H5Module } from './hdf5_util_helpers'; export const WORKERFS_MOUNT = '/workerfs'; @@ -47,6 +48,35 @@ async function save_bytes_to_memfs(filepath: string, bytes: Uint8Array) { return output_path; } +async function get_plugin_search_path() { + const Module = await h5wasm.ready as H5Module; + const plugin_paths = Module.get_plugin_search_paths(); + if (plugin_paths.length === 0) { + console.warn("No plugin paths found."); + return null; + } + else { + return plugin_paths[0]; + } +} + +async function get_plugin_names() { + const { FS } = await h5wasm.ready as H5Module; + const plugin_path = await get_plugin_search_path(); + return (plugin_path === null) ? null : FS.readdir(plugin_path) as string[]; +} + +async function add_plugin(filename: string, bytes: Uint8Array) { + const { FS } = await h5wasm.ready as H5Module; + const plugin_path = await get_plugin_search_path(); + if (plugin_path === null) { + console.warn("could not save plugin file: no plugin path found.") + return null; + } + const filepath = `${plugin_path}/${filename}`; + return await save_bytes_to_memfs(filepath, bytes); +} + async function save_to_memfs(file: File) { const { name: filename } = file; const ab = await file.arrayBuffer(); @@ -54,7 +84,7 @@ async function save_to_memfs(file: File) { } async function _mount_workerfs() { - const { FS } = await h5wasm.ready; + const { FS } = await h5wasm.ready as H5Module; const { filesystems: { WORKERFS } } = FS; if (!FS.analyzePath(WORKERFS_MOUNT).exists) { FS.mkdir(WORKERFS_MOUNT); @@ -70,6 +100,9 @@ export const api = { save_to_workerfs, save_to_memfs, save_bytes_to_memfs, + get_plugin_search_path, + get_plugin_names, + add_plugin, H5WasmFile: h5wasm.File, Dataset: h5wasm.Dataset, Group: h5wasm.Group, diff --git a/src/worker_proxy.ts b/src/worker_proxy.ts index 979ae9e..c0ce650 100644 --- a/src/worker_proxy.ts +++ b/src/worker_proxy.ts @@ -65,9 +65,15 @@ export async function get_file_proxy(filename: string, mode: ACCESS_MODESTRING = } export async function save_to_workerfs(file: File) { - const { name, lastModified, size } = file; - console.log(`Saving file ${name} of size ${lastModified} to workerfs...`); - return await remote.save_to_workerfs(file); + const { name } = file; + const filepath = await remote.save_to_workerfs(file); + console.log(`Saved file ${name} to workerfs at path ${filepath}`); + return filepath; +} + +export async function load_with_workerfs(file: File) { + const filepath = await remote.save_to_workerfs(file); + return await get_file_proxy(filepath, "r"); } export async function save_to_memfs(file: File) { @@ -76,7 +82,40 @@ export async function save_to_memfs(file: File) { return await remote.save_to_memfs(file); } +export async function load_with_memfs(file: File) { + const filepath = await remote.save_to_memfs(file); + return await get_file_proxy(filepath, "r"); +} + export async function save_bytes_to_memfs(filename: string, bytes: Uint8Array) { console.log(`Saving bytes to memfs...`); return await remote.save_bytes_to_memfs(filename, Comlink.transfer(bytes, [bytes.buffer])); +} + +export async function get_plugin_names() { + const plugin_names = await remote.get_plugin_names(); + if (plugin_names === null) { + throw new Error("No plugin paths found."); + } + return plugin_names; +} + +export async function add_plugin_bytes(filename: string, bytes: Uint8Array) { + const filepath = await remote.add_plugin(filename, Comlink.transfer(bytes, [bytes.buffer])); + if (filepath === null) { + throw new Error("could not save plugin file: no plugin path found."); + } + return filepath; +} + +export async function add_plugin_file(file: File) { + const { name: filename } = file; + const ab = await file.arrayBuffer(); + const bytes = new Uint8Array(ab); + return await add_plugin_bytes(filename, bytes); +} + +export async function load_from_bytes(filename: string, bytes: Uint8Array) { + const filepath = await save_bytes_to_memfs(filename, bytes); + return await get_file_proxy(filepath, "r"); } \ No newline at end of file From c7614ba9a8a70e5709fb6941527ce16a56a03e63 Mon Sep 17 00:00:00 2001 From: bbm Date: Wed, 6 Mar 2024 23:51:03 -0500 Subject: [PATCH 7/9] adding build script for bundle --- build_worker_bundle.mjs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 build_worker_bundle.mjs diff --git a/build_worker_bundle.mjs b/build_worker_bundle.mjs new file mode 100644 index 0000000..2b33234 --- /dev/null +++ b/build_worker_bundle.mjs @@ -0,0 +1,15 @@ +import * as esbuild from 'esbuild'; +import inlineWorkerPlugin from 'esbuild-plugin-inline-worker'; + +const extraConfig = { + target: 'es2020', + format: 'esm', +}; + +await esbuild.build({ + entryPoints: ['src/worker_proxy.ts'], + bundle: true, + format: 'esm', + outfile: 'dist/worker/worker_proxy_bundle.js', + plugins: [inlineWorkerPlugin(extraConfig)], +}); From 50fc9c667a51d2047c016211c94f551954178a3b Mon Sep 17 00:00:00 2001 From: bbm Date: Thu, 7 Mar 2024 09:54:08 -0500 Subject: [PATCH 8/9] use type export - don't need additional wrapper file --- src/h5wasm.worker.ts | 116 ++++++++++++++++++++++++++++++++++++++++++- src/worker_proxy.ts | 4 +- 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/h5wasm.worker.ts b/src/h5wasm.worker.ts index 84460cf..f4a51ff 100644 --- a/src/h5wasm.worker.ts +++ b/src/h5wasm.worker.ts @@ -1,5 +1,117 @@ -import * as Comlink from 'comlink'; -import { api } from './lib_worker'; +import * as Comlink from 'comlink'; +import * as h5wasm from '../dist/esm/hdf5_hl'; +import type { H5Module } from './hdf5_util_helpers'; + +const WORKERFS_MOUNT = '/workerfs'; + +function dirname(path: string) { + // adapted from dirname function in posixpath.py + const sep = "/"; + const sep_index = path.lastIndexOf(sep) + 1; + let head = path.slice(0, sep_index); + if (head && head !== sep.repeat(head.length)) { + // strip end slashes + head = head.replace(/(\/)+$/, ''); + } + return head; +} + +function basename(path: string) { + // adapted from basename function in posixpath.py + const sep = "/"; + const sep_index = path.lastIndexOf(sep) + 1; + return path.slice(sep_index); +} + +async function save_to_workerfs(file: File) { + const { FS, WORKERFS, mount } = await workerfs_promise; + const { name: filename, size } = file; + const output_path = `${WORKERFS_MOUNT}/${filename}`; + if (FS.analyzePath(output_path).exists) { + console.warn(`File ${output_path} already exists. Overwriting...`); + } + const outfile = WORKERFS.createNode(mount, filename, WORKERFS.FILE_MODE, 0, file); + return output_path; +} + +async function save_bytes_to_memfs(filepath: string, bytes: Uint8Array) { + const { FS } = await h5wasm.ready; + const path = dirname(filepath); + const filename = basename(filepath); + const output_path = filepath; + if (FS.analyzePath(output_path).exists) { + console.warn(`File ${output_path} already exists. Overwriting...`); + } + if (!FS.analyzePath(path).exists) { + FS.mkdirTree(path); + } + FS.writeFile(output_path, bytes); + return output_path; +} + +async function get_plugin_search_path() { + const Module = await h5wasm.ready as H5Module; + const plugin_paths = Module.get_plugin_search_paths(); + if (plugin_paths.length === 0) { + console.warn("No plugin paths found."); + return null; + } + else { + return plugin_paths[0]; + } +} + +async function get_plugin_names() { + const { FS } = await h5wasm.ready as H5Module; + const plugin_path = await get_plugin_search_path(); + return (plugin_path === null) ? null : FS.readdir(plugin_path) as string[]; +} + +async function add_plugin(filename: string, bytes: Uint8Array) { + const { FS } = await h5wasm.ready as H5Module; + const plugin_path = await get_plugin_search_path(); + if (plugin_path === null) { + console.warn("could not save plugin file: no plugin path found.") + return null; + } + const filepath = `${plugin_path}/${filename}`; + return await save_bytes_to_memfs(filepath, bytes); +} + +async function save_to_memfs(file: File) { + const { name: filename } = file; + const ab = await file.arrayBuffer(); + return save_bytes_to_memfs(filename, new Uint8Array(ab)); +} + +async function _mount_workerfs() { + const { FS } = await h5wasm.ready as H5Module; + const { filesystems: { WORKERFS } } = FS; + if (!FS.analyzePath(WORKERFS_MOUNT).exists) { + FS.mkdir(WORKERFS_MOUNT); + } + const mount = FS.mount(WORKERFS, {}, WORKERFS_MOUNT); + return { FS, WORKERFS, mount }; +} + +const workerfs_promise = _mount_workerfs(); + +const api = { + WORKERFS_MOUNT, + ready: h5wasm.ready, + save_to_workerfs, + save_to_memfs, + save_bytes_to_memfs, + get_plugin_search_path, + get_plugin_names, + add_plugin, + H5WasmFile: h5wasm.File, + Dataset: h5wasm.Dataset, + Group: h5wasm.Group, + Datatype: h5wasm.Datatype, + BrokenSoftLink: h5wasm.BrokenSoftLink, +}; Comlink.expose(api); +export type H5WasmWorkerAPI = typeof api; \ No newline at end of file diff --git a/src/worker_proxy.ts b/src/worker_proxy.ts index c0ce650..66255a3 100644 --- a/src/worker_proxy.ts +++ b/src/worker_proxy.ts @@ -1,5 +1,5 @@ import * as Comlink from 'comlink'; -import type { api } from './lib_worker.ts'; +import type { H5WasmWorkerAPI } from './h5wasm.worker.ts'; // @ts-ignore (esbuild-plugin-inline-worker will rewrite this import) import DedicatedWorker from './h5wasm.worker.ts'; @@ -10,7 +10,7 @@ export type { H5WasmFile, Group, Dataset, Datatype, BrokenSoftLink }; type ACCESS_MODESTRING = keyof typeof ACCESS_MODES; const worker = new DedicatedWorker(); // new Worker('./worker.js'); -const remote = Comlink.wrap(worker) as Comlink.Remote; +const remote = Comlink.wrap(worker) as Comlink.Remote; export class GroupProxy { proxy: Comlink.Remote; From 2d9a71bdc3692fa4045163f9f30b1b9a2f9d1908 Mon Sep 17 00:00:00 2001 From: bbm Date: Thu, 7 Mar 2024 09:54:31 -0500 Subject: [PATCH 9/9] business logic moved directly into worker source file --- src/lib_worker.ts | 112 ---------------------------------------------- 1 file changed, 112 deletions(-) delete mode 100644 src/lib_worker.ts diff --git a/src/lib_worker.ts b/src/lib_worker.ts deleted file mode 100644 index 4dd1af6..0000000 --- a/src/lib_worker.ts +++ /dev/null @@ -1,112 +0,0 @@ -import * as h5wasm from '../dist/esm/hdf5_hl'; -import type { H5Module } from './hdf5_util_helpers'; - -export const WORKERFS_MOUNT = '/workerfs'; - -function dirname(path: string) { - // adapted from dirname function in posixpath.py - const sep = "/"; - const sep_index = path.lastIndexOf(sep) + 1; - let head = path.slice(0, sep_index); - if (head && head !== sep.repeat(head.length)) { - // strip end slashes - head = head.replace(/(\/)+$/, ''); - } - return head; -} - -function basename(path: string) { - // adapted from basename function in posixpath.py - const sep = "/"; - const sep_index = path.lastIndexOf(sep) + 1; - return path.slice(sep_index); -} - -async function save_to_workerfs(file: File) { - const { FS, WORKERFS, mount } = await workerfs_promise; - const { name: filename, size } = file; - const output_path = `${WORKERFS_MOUNT}/${filename}`; - if (FS.analyzePath(output_path).exists) { - console.warn(`File ${output_path} already exists. Overwriting...`); - } - const outfile = WORKERFS.createNode(mount, filename, WORKERFS.FILE_MODE, 0, file); - return output_path; -} - -async function save_bytes_to_memfs(filepath: string, bytes: Uint8Array) { - const { FS } = await h5wasm.ready; - const path = dirname(filepath); - const filename = basename(filepath); - const output_path = filepath; - if (FS.analyzePath(output_path).exists) { - console.warn(`File ${output_path} already exists. Overwriting...`); - } - if (!FS.analyzePath(path).exists) { - FS.mkdirTree(path); - } - FS.writeFile(output_path, bytes); - return output_path; -} - -async function get_plugin_search_path() { - const Module = await h5wasm.ready as H5Module; - const plugin_paths = Module.get_plugin_search_paths(); - if (plugin_paths.length === 0) { - console.warn("No plugin paths found."); - return null; - } - else { - return plugin_paths[0]; - } -} - -async function get_plugin_names() { - const { FS } = await h5wasm.ready as H5Module; - const plugin_path = await get_plugin_search_path(); - return (plugin_path === null) ? null : FS.readdir(plugin_path) as string[]; -} - -async function add_plugin(filename: string, bytes: Uint8Array) { - const { FS } = await h5wasm.ready as H5Module; - const plugin_path = await get_plugin_search_path(); - if (plugin_path === null) { - console.warn("could not save plugin file: no plugin path found.") - return null; - } - const filepath = `${plugin_path}/${filename}`; - return await save_bytes_to_memfs(filepath, bytes); -} - -async function save_to_memfs(file: File) { - const { name: filename } = file; - const ab = await file.arrayBuffer(); - return save_bytes_to_memfs(filename, new Uint8Array(ab)); -} - -async function _mount_workerfs() { - const { FS } = await h5wasm.ready as H5Module; - const { filesystems: { WORKERFS } } = FS; - if (!FS.analyzePath(WORKERFS_MOUNT).exists) { - FS.mkdir(WORKERFS_MOUNT); - } - const mount = FS.mount(WORKERFS, {}, WORKERFS_MOUNT); - return { FS, WORKERFS, mount }; -} - -const workerfs_promise = _mount_workerfs(); - -export const api = { - ready: h5wasm.ready, - save_to_workerfs, - save_to_memfs, - save_bytes_to_memfs, - get_plugin_search_path, - get_plugin_names, - add_plugin, - H5WasmFile: h5wasm.File, - Dataset: h5wasm.Dataset, - Group: h5wasm.Group, - Datatype: h5wasm.Datatype, - BrokenSoftLink: h5wasm.BrokenSoftLink, -} -