diff --git a/package-lock.json b/package-lock.json index 0c55ea5bf..6a73d68d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "ascii-table": "^0.0.9", "benchmark": "^2.1.4", "browserify": "^17.0.1", + "buffer": "^6.0.3", "canvas": "^3.1.0", "command-line-args": "^3.0.5", "command-line-usage": "^4.0.1", @@ -550,7 +551,7 @@ "pako": "~1.0.5" } }, - "node_modules/buffer": { + "node_modules/browserify/node_modules/buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz", "integrity": "sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==", @@ -561,6 +562,30 @@ "ieee754": "^1.1.4" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", diff --git a/package.json b/package.json index c95352060..7e06718d4 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "ascii-table": "^0.0.9", "benchmark": "^2.1.4", "browserify": "^17.0.1", + "buffer": "^6.0.3", "canvas": "^3.1.0", "command-line-args": "^3.0.5", "command-line-usage": "^4.0.1", diff --git a/src/arr/compiler/cli-module-loader.arr b/src/arr/compiler/cli-module-loader.arr index 12a184183..217c49f36 100644 --- a/src/arr/compiler/cli-module-loader.arr +++ b/src/arr/compiler/cli-module-loader.arr @@ -6,12 +6,11 @@ import load-lib as L import either as E import json as JSON import ast as A -import pathlib as P import sha as crypto import string-dict as SD import render-error-display as RED +import filesystem as Filesystem import file as F -import filelib as FS import error as ERR import system as SYS import file("js-ast.arr") as J @@ -100,13 +99,13 @@ end # with the compiled version of the file. fun cached-available(basedir, uri, name, modified-time) -> Option: - saved-path = P.join(basedir, uri-to-path(uri, name)) + saved-path = Filesystem.join(basedir, uri-to-path(uri, name)) - if (F.file-exists(saved-path + "-static.js") and - (F.file-times(saved-path + "-static.js").mtime > modified-time)): + if (Filesystem.exists(saved-path + "-static.js") and + (Filesystem.stat(saved-path + "-static.js").mtime > modified-time)): some(split) - else if (F.file-exists(saved-path + ".js") and - (F.file-times(saved-path + ".js").mtime > modified-time)): + else if (Filesystem.exists(saved-path + ".js") and + (Filesystem.stat(saved-path + ".js").mtime > modified-time)): some(single-file) else: none @@ -114,7 +113,7 @@ fun cached-available(basedir, uri, name, modified-time) -> Option: end fun get-cached(basedir, uri, name, cache-type): - saved-path = P.join(basedir, uri-to-path(uri, name)) + saved-path = Filesystem.join(basedir, uri-to-path(uri, name)) {static-path; module-path} = cases(CachedType) cache-type: # NOTE(joe): leaving off .js because builtin-raw-locator below # expects no extension @@ -127,7 +126,6 @@ fun get-cached(basedir, uri, name, cache-type): method needs-compile(_, _): false end, method get-modified-time(self): 0 - # F.file-times(static-path + ".js").mtime end, method get-options(self, options): options.{ checks: "none" } @@ -163,7 +161,7 @@ fun get-cached(basedir, uri, name, cache-type): modules: raw-array-to-list(raw.get-raw-module-provides()) }) some(CL.module-as-string(provs, CS.no-builtins, CS.computed-none, - CS.ok(JSP.ccp-file(F.real-path(module-path + ".js"))))) + CS.ok(JSP.ccp-file(Filesystem.resolve(module-path + ".js"))))) end, method _equals(self, other, req-eq): @@ -176,7 +174,7 @@ fun get-cached-if-available(basedir, loc) block: get-cached-if-available-known-mtimes(basedir, loc, [SD.string-dict:]) end fun get-cached-if-available-known-mtimes(basedir, loc, max-dep-times) block: - saved-path = P.join(basedir, uri-to-path(loc.uri(), loc.name())) + saved-path = Filesystem.join(basedir, uri-to-path(loc.uri(), loc.name())) dependency-based-mtime = if max-dep-times.has-key(loc.uri()): max-dep-times.get-value(loc.uri()) else: loc.get-modified-time() @@ -237,7 +235,7 @@ fun get-loadable(basedir, read-only-basedirs, l, max-dep-times) -> Option none | some(found-basedir) => c = cached-available(found-basedir, l.locator.uri(), l.locator.name(), max-dep-times.get-value(locuri)) - saved-path = P.join(found-basedir, uri-to-path(locuri, l.locator.name())) + saved-path = Filesystem.join(found-basedir, uri-to-path(locuri, l.locator.name())) {static-path; module-path} = cases(CachedType) c.or-else(single-file): | split => {saved-path + "-static"; saved-path + "-module.js"} @@ -258,14 +256,14 @@ end fun set-loadable(basedir, locator, loadable) -> String block: doc: "Returns the module path of the cached file" - when not(FS.exists(basedir)): - FS.create-dir(basedir) + when not(Filesystem.exists(basedir)): + Filesystem.create-dir(basedir) end locuri = loadable.provides.from-uri cases(CS.CompileResult) loadable.result-printer block: | ok(ccp) => - save-static-path = P.join(basedir, uri-to-path(locuri, locator.name()) + "-static.js") - save-module-path = P.join(basedir, uri-to-path(locuri, locator.name()) + "-module.js") + save-static-path = Filesystem.join(basedir, uri-to-path(locuri, locator.name()) + "-static.js") + save-module-path = Filesystem.join(basedir, uri-to-path(locuri, locator.name()) + "-module.js") fs = F.output-file(save-static-path, false) fm = F.output-file(save-module-path, false) @@ -302,18 +300,18 @@ type CLIContext = { } fun get-real-path(current-load-path :: String, this-path :: String): - if P.is-absolute(this-path): - P.relative(current-load-path, this-path) + if Filesystem.is-absolute(this-path): + Filesystem.relative(current-load-path, this-path) else: - P.join(current-load-path, this-path) + Filesystem.join(current-load-path, this-path) end end fun locate-file(ctxt :: CLIContext, rel-path :: String): clp = ctxt.current-load-path real-path = get-real-path(clp, rel-path) - new-context = ctxt.{current-load-path: P.dirname(real-path)} - if F.file-exists(real-path): + new-context = ctxt.{current-load-path: Filesystem.dirname(real-path)} + if Filesystem.exists(real-path): some(CL.located(get-file-locator(ctxt.cache-base-dir, real-path), new-context)) else: none @@ -361,8 +359,8 @@ fun module-finder(ctxt :: CLIContext, dep :: CS.Dependency): else if protocol == "file-no-cache": clp = ctxt.current-load-path real-path = get-real-path(clp, args.get(0)) - new-context = ctxt.{current-load-path: P.dirname(real-path)} - if F.file-exists(real-path): + new-context = ctxt.{current-load-path: Filesystem.dirname(real-path)} + if Filesystem.exists(real-path): CL.located(FL.file-locator(real-path, CS.standard-globals), new-context) else: raise("Cannot find import " + torepr(dep)) @@ -370,7 +368,7 @@ fun module-finder(ctxt :: CLIContext, dep :: CS.Dependency): else if protocol == "js-file": clp = ctxt.current-load-path real-path = get-real-path(clp, args.get(0)) - new-context = ctxt.{current-load-path: P.dirname(real-path)} + new-context = ctxt.{current-load-path: Filesystem.dirname(real-path)} locator = JSF.make-jsfile-locator(real-path) CL.located(locator, new-context) else: @@ -382,15 +380,15 @@ fun module-finder(ctxt :: CLIContext, dep :: CS.Dependency): end default-start-context = { - current-load-path: P.resolve("./"), - cache-base-dir: P.resolve("./compiled"), + current-load-path: Filesystem.resolve("./"), + cache-base-dir: Filesystem.resolve("./compiled"), compiled-read-only-dirs: empty, url-file-mode: CS.all-remote } default-test-context = { - current-load-path: P.resolve("./"), - cache-base-dir: P.resolve("./tests/compiled"), + current-load-path: Filesystem.resolve("./"), + cache-base-dir: Filesystem.resolve("./tests/compiled"), compiled-read-only-dirs: empty, url-file-mode: CS.all-remote } @@ -398,9 +396,9 @@ default-test-context = { fun compile(path, options): base-module = CS.dependency("file", [list: path]) base = module-finder({ - current-load-path: P.resolve(options.base-dir), + current-load-path: Filesystem.resolve(options.base-dir), cache-base-dir: options.compiled-cache, - compiled-read-only-dirs: options.compiled-read-only.map(P.resolve), + compiled-read-only-dirs: options.compiled-read-only.map(Filesystem.resolve), url-file-mode: options.url-file-mode }, base-module) wl = CL.compile-worklist(module-finder, base.locator, base.context) @@ -463,9 +461,9 @@ fun build-program(path, options, stats) block: print-progress(str) base-module = CS.dependency("file", [list: path]) base = module-finder({ - current-load-path: P.resolve(options.base-dir), + current-load-path: Filesystem.resolve(options.base-dir), cache-base-dir: options.compiled-cache, - compiled-read-only-dirs: options.compiled-read-only.map(P.resolve), + compiled-read-only-dirs: options.compiled-read-only.map(Filesystem.resolve), url-file-mode: options.url-file-mode }, base-module) clear-and-print("Compiling worklist...") @@ -479,7 +477,7 @@ fun build-program(path, options, stats) block: clear-and-print("Loading existing compiled modules...") - starter-modules = CL.modules-from-worklist(wl, get-loadable(options.compiled-cache, options.compiled-read-only.map(P.resolve), _, _)) + starter-modules = CL.modules-from-worklist(wl, get-loadable(options.compiled-cache, options.compiled-read-only.map(Filesystem.resolve), _, _)) cached-modules = starter-modules.count-now() total-modules = wl.length() - cached-modules @@ -529,7 +527,7 @@ end fun build-runnable-standalone(path, require-config-path, outfile, options) block: stats = SD.make-mutable-string-dict() - config = JSON.read-json(F.file-to-string(require-config-path)).dict.unfreeze() + config = JSON.read-json(Filesystem.read-file-string(require-config-path)).dict.unfreeze() cases(Option) config.get-now("typable-builtins"): | none => nothing | some(tb) => @@ -544,11 +542,11 @@ fun build-runnable-standalone(path, require-config-path, outfile, options) block | left(problems) => handle-compilation-errors(problems, options) | right(program) => - shadow require-config-path = if not( P.is-absolute( require-config-path ) ): - P.resolve(P.join(options.base-dir, require-config-path)) + shadow require-config-path = if not( Filesystem.is-absolute( require-config-path ) ): + Filesystem.resolve(Filesystem.join(options.base-dir, require-config-path)) else: require-config-path end - config.set-now("out", JSON.j-str(P.resolve(P.join(options.base-dir, outfile)))) + config.set-now("out", JSON.j-str(Filesystem.resolve(Filesystem.join(options.base-dir, outfile)))) when not(config.has-key-now("baseUrl")): config.set-now("baseUrl", JSON.j-str(options.compiled-cache)) end diff --git a/src/arr/compiler/locators/builtin.arr b/src/arr/compiler/locators/builtin.arr index a7ad9f719..76b04c57b 100644 --- a/src/arr/compiler/locators/builtin.arr +++ b/src/arr/compiler/locators/builtin.arr @@ -1,7 +1,7 @@ provide * import builtin-modules as B import string-dict as SD -import file as F +import filesystem as FS import pathlib as P import parse-pyret as PP import file("../compile-lib.arr") as CL @@ -51,12 +51,12 @@ fun set-typable-builtins(uris :: List): end fun make-builtin-js-locator(basedir, builtin-name): - raw = B.builtin-raw-locator(P.join(basedir, builtin-name)) + raw = B.builtin-raw-locator(FS.join(basedir, builtin-name)) { method needs-compile(_, _): false end, method get-uncached(_): none end, method get-modified-time(self): - F.file-times(P.join(basedir, builtin-name + ".js")).mtime + FS.stat(FS.join(basedir, builtin-name + ".js")).mtime end, method get-options(self, options): options.{ check-mode: false, type-check: false } @@ -92,7 +92,7 @@ fun make-builtin-js-locator(basedir, builtin-name): datatypes: raw-array-to-list(raw.get-raw-datatype-provides()) }) some(CL.module-as-string(provs, CM.no-builtins, CM.computed-none, - CM.ok(JSP.ccp-file(P.join(basedir, builtin-name + ".js"))))) + CM.ok(JSP.ccp-file(FS.join(basedir, builtin-name + ".js"))))) end, method _equals(self, other, req-eq): @@ -102,11 +102,11 @@ fun make-builtin-js-locator(basedir, builtin-name): end fun make-builtin-arr-locator(basedir, builtin-name): - path = P.join(basedir, builtin-name + ".arr") + path = FS.join(basedir, builtin-name + ".arr") var ast = nothing { method get-modified-time(self): - F.file-times(path).mtime + FS.stat(path).mtime end, method get-uncached(_): none end, method get-options(self, options): @@ -115,10 +115,10 @@ fun make-builtin-arr-locator(basedir, builtin-name): end, method get-module(self) block: when ast == nothing block: - when not(F.file-exists(path)): + when not(FS.exists(path)): raise("File " + path + " does not exist") end - ast := CL.pyret-ast(PP.surface-parse(F.file-to-string(path), self.uri())) + ast := CL.pyret-ast(PP.surface-parse(FS.read-file-string(path), self.uri())) end ast end, @@ -142,9 +142,9 @@ fun make-builtin-arr-locator(basedir, builtin-name): # does not handle provides from dependencies currently # NOTE(joe): Until we serialize provides correctly, just return false here cpath = path + ".js" - if F.file-exists(path) and F.file-exists(cpath): - stimes = F.file-times(path) - ctimes = F.file-times(cpath) + if FS.exists(path) and FS.exists(cpath): + stimes = FS.stat(path) + ctimes = FS.stat(cpath) ctimes.mtime <= stimes.mtime else: true @@ -152,15 +152,15 @@ fun make-builtin-arr-locator(basedir, builtin-name): end, method get-compiled(self): cpath = path + ".js" - if F.file-exists(path) and F.file-exists(cpath): + if FS.exists(path) and FS.exists(cpath): # NOTE(joe): # Since we're not explicitly acquiring locks on files, there is a race # condition in the next few lines – a user could potentially delete or # overwrite the original file for the source while this method is # running. We can explicitly open and lock files with appropriate # APIs to mitigate this in the happy, sunny future. - stimes = F.file-times(path) - ctimes = F.file-times(cpath) + stimes = FS.stat(path) + ctimes = FS.stat(cpath) if ctimes.mtime > stimes.mtime: raw = B.builtin-raw-locator(path) provs = convert-provides(self.uri(), { @@ -185,16 +185,16 @@ end fun maybe-make-builtin-locator(builtin-name :: String) -> Option block: matching-arr-files = for map(p from builtin-arr-dirs): - full-path = P.join(p, builtin-name + ".arr") - if F.file-exists(full-path): + full-path = FS.join(p, builtin-name + ".arr") + if FS.exists(full-path): some(full-path) else: none end end.filter(is-some).map(_.value) matching-js-files = for map(p from builtin-js-dirs): - full-path = P.join(p, builtin-name + ".js") - if F.file-exists(full-path): + full-path = FS.join(p, builtin-name + ".js") + if FS.exists(full-path): some(full-path) else: none diff --git a/src/arr/compiler/locators/jsfile.arr b/src/arr/compiler/locators/jsfile.arr index 607c3a661..28d23d6b8 100644 --- a/src/arr/compiler/locators/jsfile.arr +++ b/src/arr/compiler/locators/jsfile.arr @@ -1,6 +1,6 @@ provide * import builtin-modules as B -import file as F +import filesystem as FS import pathlib as P import file("./builtin.arr") as BL import file("../compile-lib.arr") as CL @@ -17,7 +17,7 @@ fun make-jsfile-locator(path): method get-uncached(_): none end, method needs-compile(_, _): false end, method get-modified-time(self): - F.file-times(path + ".js").mtime + FS.stat(path + ".js").mtime end, method get-options(self, options): options.{ check-mode: false } @@ -40,7 +40,7 @@ fun make-jsfile-locator(path): CM.standard-globals end, - method uri(_): "jsfile://" + string-replace(F.real-path(path + ".js"), P.path-sep, "/") end, + method uri(_): "jsfile://" + string-replace(FS.resolve(path + ".js"), P.path-sep, "/") end, method name(_): P.basename(path, "") end, method set-compiled(_, _, _): nothing end, @@ -52,7 +52,7 @@ fun make-jsfile-locator(path): aliases: raw-array-to-list(raw.get-raw-alias-provides()), datatypes: raw-array-to-list(raw.get-raw-datatype-provides()) }) - some(CL.module-as-string(provs, CM.no-builtins, CM.computed-none, CM.ok(JSP.ccp-file(F.real-path(path + ".js"))))) + some(CL.module-as-string(provs, CM.no-builtins, CM.computed-none, CM.ok(JSP.ccp-file(FS.resolve(path + ".js"))))) end, method _equals(self, other, req-eq): diff --git a/src/js/base/runtime.js b/src/js/base/runtime.js index 0c78b6f93..b540d206b 100644 --- a/src/js/base/runtime.js +++ b/src/js/base/runtime.js @@ -3872,6 +3872,20 @@ function (Namespace, jsnums, codePoint, util, exnStackParser, loader, seedrandom return makePause(pause, resumer); } + function pauseAwait(p) { + if(!('then' in p)) { return p; } + + return pauseStack(async (restarter) => { + try { + const result = await p; + return restarter.resume(result); + } + catch(e) { + return restarter.error(e); + } + }); + } + function PausePackage() { this.resumeVal = null; this.errorVal = null; @@ -6118,6 +6132,7 @@ function (Namespace, jsnums, codePoint, util, exnStackParser, loader, seedrandom 'isPause' : isPause, 'pauseStack' : pauseStack, + 'await' : pauseAwait, 'schedulePause' : schedulePause, 'breakAll' : breakAll, diff --git a/src/js/trove/builtin-modules.js b/src/js/trove/builtin-modules.js index 7cf2afe18..8c938a023 100644 --- a/src/js/trove/builtin-modules.js +++ b/src/js/trove/builtin-modules.js @@ -1,9 +1,11 @@ ({ - requires: [], + requires: [ + { "import-type": "builtin", "name": "filesystem-internal" }, + ], nativeRequires: [ - "fs", "pyret-base/js/secure-loader", - "pyret-base/js/type-util" + "pyret-base/js/type-util", + "buffer" ], provides: { values: { @@ -11,7 +13,8 @@ "builtin-raw-locator-from-str": "tany" } }, - theModule: function(RUNTIME, ns, uri, fs, loader, t) { + theModule: function(RUNTIME, ns, uri, fsInternal, loader, t, buffer) { + const Buffer = buffer.Buffer; var F = RUNTIME.makeFunction; function builtinLocatorFromString(content) { @@ -163,8 +166,20 @@ console.error("Got undefined name in builtin locator"); console.trace(); } - var content = String(fs.readFileSync(fs.realpathSync(path + ".js"))); - return builtinLocatorFromString(content); + return RUNTIME.pauseStack(async (restarter) => { + try { + const fullPath = await fsInternal.resolve(path + ".js"); + const fileContents = await fsInternal.readFile(fullPath); + const content = Buffer.from(fileContents).toString('utf8'); + return restarter.resume(builtinLocatorFromString(content)); + } + catch(e) { + console.error("Error in builtin locator: ", e); + console.error("Path was: ", path); + console.trace(); + return restarter.error(RUNTIME.ffi.makeMessageException(String(e))); + } + }); } var O = RUNTIME.makeObject; return O({ diff --git a/src/js/trove/filesystem-internal.js b/src/js/trove/filesystem-internal.js new file mode 100644 index 000000000..4c49efc16 --- /dev/null +++ b/src/js/trove/filesystem-internal.js @@ -0,0 +1,115 @@ +({ + provides: { + values: {}, + types: {}, + }, + requires: [ ], + nativeRequires: ['fs', 'path'], + /** + * Provides a Pyret-specific filesystem API based on `node`. The API is + * designed to give a consistent view of the filesystem and path utilities + * across the node fs and path libraries: + * + * https://nodejs.org/docs/latest/api/fs.html + * https://nodejs.org/docs/latest/api/path.html + * + * and the VScode FileSystemProvider and vscode-uri library: + * + * https://code.visualstudio.com/api/references/vscode-api#FileSystemProvider + * https://github.com/microsoft/vscode-uri?tab=readme-ov-file#usage-util + * + * If Pyret needs to run on new environments with new filesystem + * definitions, this is the module to replace with --allow-builtin-overrides. + * + * Since these don't provide a consistent set of names, the names here don't + * always exactly match the corresponding underlying function, and sometimes + * have their inputs simplified or outputs manipulated to match a common + * interface. + */ + theModule: function(runtime, _, uri, fs, path) { + let initializedOK = true; + if(!('promises' in fs)) { + console.warn("Could not find 'promises' in node fs library, cannot initialize filesystem-internal and its functions will throw."); + initializedOK = false; + } + const fsp = fs.promises; + function wrap(f) { + if(initializedOK) { return f; } + else { + return async function(...args) { + throw runtime.ffi.makeMessageException(`filesystem-internal: Cannot call ${f.name} because fs.promises not available`) + } + } + } + async function readFile(p) { + return fsp.readFile(p); + } + async function writeFile(p, data) { + return fsp.writeFile(p, data); + } + /** + * Guaranteed fields are + * - `ctime` and `mtime` in epoch ms + * - `size` in bytes + * + * The underlying `fs` return value is in the `native` field + */ + async function stat(p) { + const stats = await fsp.stat(p); + return { + ctime: stats.ctimeMs, + mtime: stats.mtimeMs, + size: stats.size, + native: stats + }; + } + + async function resolve(...paths) { + return path.resolve(...paths); + } + + async function exists(p) { + // NOTE(joe): this is sync because the async version is deprecated + // See https://nodejs.org/dist/latest-v10.x/docs/api/fs.html#fs_fs_existssync_path + // Also, `exists` is not defined on the `fs.promises` api + return fs.existsSync(p); + } + + async function join(...paths) { + return path.join(...paths); + } + + async function createDir(p) { + /* NOTE(joe): this does not create parent dirs because other + * platforms may not support it (in particular VScode + * filesystemprovider) */ + return fsp.mkdir(p); + } + async function relative(from, to) { + return path.relative(from, to); + } + async function isAbsolute(p) { + return path.isAbsolute(p); + } + async function basename(p) { + return path.basename(p); + } + async function dirname(p) { + return path.dirname(p); + } + return runtime.makeJSModuleReturn({ + readFile: wrap(readFile), + writeFile: wrap(writeFile), + stat: wrap(stat), + resolve: wrap(resolve), + exists: wrap(exists), + join: wrap(join), + 'path-sep': path.sep, + createDir: wrap(createDir), + relative: wrap(relative), + isAbsolute: wrap(isAbsolute), + basename: wrap(basename), + dirname: wrap(dirname), + }); + } +}) \ No newline at end of file diff --git a/src/js/trove/filesystem.js b/src/js/trove/filesystem.js new file mode 100644 index 000000000..e8119efd3 --- /dev/null +++ b/src/js/trove/filesystem.js @@ -0,0 +1,133 @@ +({ + requires: [ + { "import-type": "builtin", "name": "filesystem-internal" }, + ], + provides: { + values: { + 'read-file-string': ["arrow", ["String"], "String"], + 'write-file-string': ["arrow", ["String", "String"], "Nothing"], + 'stat': ["arrow", ["String"], "Any"], + 'resolve': ["arrow", ["String"], "String"], + 'exists': ["arrow", ["String"], "Boolean"], + 'join': ["arrow", ["String", "String"], "String"], + 'create-dir': ["arrow", ["String"], "String"], + 'basename': ["arrow", ["String"], "String"], + 'dirname': ["arrow", ["String"], "String"], + 'relative': ["arrow", ["String", "String"], "String"], + 'is-absolute': ["arrow", ["String"], "Boolean"], + }, + types: {} + }, + nativeRequires: ['buffer'], + theModule: function(runtime, _, _, fsInternal, buffer) { + const Buffer = buffer.Buffer; + function readFileString(path) { + runtime.checkArgsInternal1('filesystem', 'read-file-string', path, runtime.String); + const result = fsInternal.readFile(path) + .then((contents) => Buffer.from(contents).toString('utf8')) + .catch((err) => { + throw runtime.throwMessageException(`Error reading file: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function writeFileString(path, data) { + runtime.checkArgsInternal2('filesystem', 'write-file-string', path, runtime.String, data, runtime.String); + const result = fsInternal.writeFile(path, Buffer.alloc(data.length, data, 'utf8')) + .then(() => runtime.nothing) + .catch((err) => { + throw runtime.throwMessageException(`Error writing file: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function resolve(path) { + const result = fsInternal.resolve(path) + .catch((err) => { + throw runtime.throwMessageException(`Error resolving path: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function join(path1, path2) { + const result = fsInternal.join(path1, path2) + .catch((err) => { + throw runtime.throwMessageException(`Error joining paths: ${path1}, ${path2}: ${String(err)}`); + }); + return runtime.await(result); + } + function stat(path) { + runtime.checkArgsInternal1('filesystem', 'stat', path, runtime.String); + const result = fsInternal.stat(path).then((stats) => { + return runtime.makeObject({ + ctime: stats.ctime, + mtime: stats.mtime, + size: stats.size, + native: stats + }); + }) + .catch((err) => { + throw runtime.throwMessageException(`Error getting stats for file: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function exists(path) { + runtime.checkArgsInternal1('filesystem', 'exists', path, runtime.String); + const result = fsInternal.exists(path) + .catch((err) => { + throw runtime.throwMessageException(`Error checking existence of file: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function createDir(path) { + runtime.checkArgsInternal1('filesystem', 'create-dir', path, runtime.String); + const result = fsInternal.createDir(path).then(() => runtime.nothing) + .catch(err => { + throw runtime.throwMessageException(`Error creating directory: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function basename(path) { + runtime.checkArgsInternal1('filesystem', 'basename', path, runtime.String); + const result = fsInternal.basename(path) + .catch((err) => { + throw runtime.throwMessageException(`Error getting basename of path: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function dirname(path) { + runtime.checkArgsInternal1('filesystem', 'dirname', path, runtime.String); + const result = fsInternal.dirname(path) + .catch((err) => { + throw runtime.throwMessageException(`Error getting dirname of path: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + function relative(from, to) { + runtime.checkArgsInternal2('filesystem', 'relative', from, runtime.String, to, runtime.String); + const result = fsInternal.relative(from, to) + .catch((err) => { + throw runtime.throwMessageException(`Error getting relative path from ${from} to ${to}: ${String(err)}`); + }); + return runtime.await(result); + } + function isAbsolute(path) { + runtime.checkArgsInternal1('filesystem', 'is-absolute', path, runtime.String); + const result = fsInternal.isAbsolute(path) + .catch((err) => { + throw runtime.throwMessageException(`Error checking if path is absolute: ${path}: ${String(err)}`); + }); + return runtime.await(result); + } + return runtime.makeModuleReturn({ + 'read-file-string': runtime.makeFunction(readFileString), + 'write-file-string': runtime.makeFunction(writeFileString), + 'stat': runtime.makeFunction(stat), + 'resolve': runtime.makeFunction(resolve), + 'exists': runtime.makeFunction(exists), + 'join': runtime.makeFunction(join), + 'create-dir': runtime.makeFunction(createDir), + 'basename': runtime.makeFunction(basename), + 'dirname': runtime.makeFunction(dirname), + 'relative': runtime.makeFunction(relative), + 'is-absolute': runtime.makeFunction(isAbsolute), + }, {}); + } +}) \ No newline at end of file diff --git a/src/js/trove/make-image.js b/src/js/trove/make-image.js index c6b54da3a..53dc84375 100644 --- a/src/js/trove/make-image.js +++ b/src/js/trove/make-image.js @@ -1,15 +1,16 @@ ({ requires: [ { "import-type": "builtin", "name": "image-lib" }, - { "import-type": "builtin", "name": "ffi" } + { "import-type": "builtin", "name": "ffi" }, + { "import-type": "builtin", "name": "filesystem-internal" } ], nativeRequires: [ "pyret-base/js/js-numbers", "fs", - "canvas" + "canvas", ], provides: {}, - theModule: function(runtime, namespace, uri, imageLib, ffi, jsnums, fs, canvas) { + theModule: function(runtime, namespace, uri, imageLib, ffi, fsInternal, jsnums, fs, canvas) { var image = runtime.getField(imageLib, "internal"); var Image = canvas.Image; // The polyfill for the browser Image API (passes through raw Image on CPO) @@ -162,44 +163,81 @@ const extension = path.slice(lastDot + 1).toLowerCase(); const mime = extensiontypes[extension]; if(!mime) { throw runtime.ffi.makeMessageException(`Path to image-file did not have a valid extension (got ${extension}), must be one of ${allowedExtensions.join(", ")}`); } - return runtime.pauseStack(function(restarter) { - fs.readFile(path, {}, async (err, result) => { - if(err) { restarter.error(runtime.ffi.makeMessageException(String(err))); } - else { - // create a data url from the result from readFile stored in result: - var dataURL = await bufferToBase64(result, mime); + return runtime.pauseStack(async function(restarter) { + if(fsInternal.init) { + try { + const contentsBuffer = await fsInternal.readFile(path); + const dataURL = await bufferToBase64(contentsBuffer, mime); var rawImage = new Image(); rawImage.onload = function() { restarter.resume(makeImage(image.makeFileImage(dataURL, rawImage))); }; rawImage.onerror = function(e) { - restarter.error(runtime.ffi.makeMessageException("Unable to load " + path)); + restarter.error(runtime.ffi.makeMessageException("Unable to load " + path + " " + String(e))); }; rawImage.src = dataURL; } - }) + catch(err) { + restarter.error(runtime.ffi.makeMessageException(String(err))); + } + } + else { + fs.readFile(path, {}, async (err, result) => { + if(err) { restarter.error(runtime.ffi.makeMessageException(String(err))); } + else { + // create a data url from the result from readFile stored in result: + var dataURL = await bufferToBase64(result, mime); + var rawImage = new Image(); + rawImage.onload = function() { + restarter.resume(makeImage(image.makeFileImage(dataURL, rawImage))); + }; + rawImage.onerror = function(e) { + restarter.error(runtime.ffi.makeMessageException("Unable to load " + path)); + }; + rawImage.src = dataURL; + } + }); + } }) } + async function getBuffer(canvas) { + if(canvas.toBuffer) { return canvas.toBuffer("image/png"); } + else { + return new Promise((resolve, reject) => { + try { + canvas.toBlob(async (blob) => { + const buffer = new Uint8Array(await blob.arrayBuffer()); + resolve(buffer); + }, "image/png"); + } + catch(e) { + reject(e); + } + }); + } + } + function saveImage(img, path) { - return runtime.pauseStack(function(restarter) { + return runtime.pauseStack(async function(restarter) { const canvas = image.makeCanvas(img.width, img.height); img.render(canvas.getContext("2d")); - if(canvas.toBuffer) { - fs.writeFile(path, canvas.toBuffer("image/png"), function(err) { + const buffer = await getBuffer(canvas); + if(fsInternal.init) { + try { + await fsInternal.writeFile(path, buffer); + restarter.resume(runtime.nothing); + } + catch(err) { + restarter.error(runtime.ffi.makeMessageException(String(err))); + } + } + else { + fs.writeFile(path, buffer, function(err) { if(err) { restarter.error(runtime.ffi.makeMessageException(String(err))); } else { restarter.resume(runtime.nothing); } }); } - else { - canvas.toBlob(async (blob) => { - const buffer = new Uint8Array(await blob.arrayBuffer()); - fs.writeFile(path, buffer, function(err) { - if(err) { restarter.error(runtime.ffi.makeMessageException(String(err))); } - else { restarter.resume(runtime.nothing); } - }); - }, "image/png"); - } }); } diff --git a/src/js/trove/require-node-compile-dependencies.js b/src/js/trove/require-node-compile-dependencies.js index ce6be5ffa..694b8ada1 100644 --- a/src/js/trove/require-node-compile-dependencies.js +++ b/src/js/trove/require-node-compile-dependencies.js @@ -20,6 +20,9 @@ define("seedrandom", [], function() {return seedrandom;}); sourcemap = require("source-map"); define("source-map", [], function () { return sourcemap; }); +buffer = require("buffer"); +define("buffer", [], function () { return buffer; }); + jssha256 = require("js-sha256"); define("js-sha256", [], function () { return jssha256; }); diff --git a/src/js/trove/require-node-dependencies.js b/src/js/trove/require-node-dependencies.js index 029470916..2eb345468 100644 --- a/src/js/trove/require-node-dependencies.js +++ b/src/js/trove/require-node-dependencies.js @@ -29,6 +29,9 @@ define("source-map", [], function () { return sourcemap; }); jssha256 = require("js-sha256"); define("js-sha256", [], function () { return jssha256; }); +buffer = require("buffer"); +define("buffer", [], function () { return buffer; }); + fs = nodeRequire("fs"); define("fs", [], function () { return fs; }); diff --git a/tests/pyret/main2.arr b/tests/pyret/main2.arr index 36c969d4f..2c406e753 100644 --- a/tests/pyret/main2.arr +++ b/tests/pyret/main2.arr @@ -28,6 +28,7 @@ import file("./tests/test-record-concat.arr") as _ import file("./tests/test-rec.arr") as _ import file("./tests/test-compile-errors.arr") as _ import file("./tests/test-well-formed.arr") as _ +import file("./tests/test-filesystem.arr") as _ import file("./tests/test-file.arr") as _ import file("./tests/test-path.arr") as _ import file("./tests/test-repl.arr") as _ diff --git a/tests/pyret/tests/test-filesystem.arr b/tests/pyret/tests/test-filesystem.arr new file mode 100644 index 000000000..6298843ef --- /dev/null +++ b/tests/pyret/tests/test-filesystem.arr @@ -0,0 +1,54 @@ +import filesystem as FS + +s = "fairly unique string to test this filesystem test" + +check: + contents = FS.read-file-string("./tests/pyret/tests/test-filesystem.arr") + contents satisfies string-contains(_, s) + + p = FS.resolve("./tests/../tests/pyret/tests/./test-file.arr") + expected = [list: "", "tests", "pyret", "tests", "test-file.arr"].join-str("/") + l = string-split(p, expected) + l.length() is 2 + l.get(1) is "" + FS.exists(p) is true + p2 = "./non-existing-file" + FS.exists(p2) is false + + FS.read-file-string("nonexistent") raises "ENOENT" +end + +check: + FS.relative("/a/b/c", "/d/e/f") is "../../../d/e/f" + FS.relative("a/b/c", "d/e/f") is "../../../d/e/f" + FS.relative(FS.resolve("a/b/c"), FS.resolve("d/e/f")) is "../../../d/e/f" + FS.relative("a/b/c", "file.arr") is "../../../file.arr" + FS.relative(".", "a/b/c/file.arr") is "a/b/c/file.arr" + FS.relative("a", "a/b/c/file.arr") is "b/c/file.arr" + FS.relative("a/b", "a/b/c/file.arr") is "c/file.arr" + + FS.relative("a/b/c", "a/b/file.arr") is "../file.arr" +end + +check: + "/" satisfies FS.is-absolute + "/a/b/c" satisfies FS.is-absolute + "/../a/b/c" satisfies FS.is-absolute + "/a/../c/d" satisfies FS.is-absolute + "../../../../../../../../.." violates FS.is-absolute + "." violates FS.is-absolute + ".." violates FS.is-absolute + "a/b/c" violates FS.is-absolute + "./a/b/c" violates FS.is-absolute +end + +check: + FS.basename("/a/b/c") is "c" + FS.basename("/a/b/c.arr") is "c.arr" + FS.basename("rel/dir/c.arr") is "c.arr" + FS.basename("a") is "a" + + FS.dirname("a") is "." + FS.dirname(".") is "." + FS.dirname("./a/b/c/file.txt") is "./a/b/c" +end \ No newline at end of file