Merge remote-tracking branch 'upstream/prealpha-dev' into nk/improve-workflow

Author: Niraj-Kamdar

packages/cli/src/lib/defaults/build-images/wasm/rust/Dockerfile.mustache

@@ -1,4 +1,4 @@
-FROM rustlang/rust:nightly-slim as base
+FROM rust:1.60.0 as base
 
 # Install the wasm32 rust build target
 RUN rustup target add wasm32-unknown-unknown
@@ -7,7 +7,7 @@ WORKDIR /build-deps
 
 # Install curl
 RUN apt-get update
-RUN apt-get -y install curl
+RUN apt-get -y install curl clang llvm build-essential
 
 # Install wasm-opt
 RUN curl -L https://github.com/WebAssembly/binaryen/releases/download/version_101/binaryen-version_101-x86_64-linux.tar.gz | tar -xz \
@@ -16,10 +16,13 @@ RUN curl -L https://github.com/WebAssembly/binaryen/releases/download/version_10
     && rm -rf binary-version_101
 
 # Install the toml-cli
-RUN cargo install toml-cli
+RUN cargo install -f toml-cli
 
 # Install wasm-snip
-RUN cargo install wasm-snip
+RUN cargo install -f wasm-snip
+
+# Install wasm-bindgen
+RUN cargo install -f wasm-bindgen-cli
 
 {{#polywrap_linked_packages.length}}
 WORKDIR /linked-packages
@@ -55,38 +58,50 @@ RUN PACKAGE_NAME={{name}}; \
     {{/polywrap_module}}
     true
 {{/polywrap_linked_packages}}
+
 {{/polywrap_linked_packages.length}}
 
-# Remove any Cargo.lock files
 {{#polywrap_module}}
+# Remove any Cargo.lock files
 RUN rm -rf {{dir}}/Cargo.lock
 
 # Ensure the Wasm module is configured to use imported memory
 ENV RUSTFLAGS="-C link-arg=-z -C link-arg=stack-size=65536 -C link-arg=--import-memory"
 
-# Cleanup an artifact left by the toml CLI program ("[]" -> [])
-RUN sed -i 's/"\[\]"/\[\]/g' ./{{dir}}/Cargo.toml
-
 # Ensure the module at {{dir}} has the crate-type = ["cdylib"]
-RUN toml set ./{{dir}}/Cargo.toml lib.crate-type ["cdylib","rlib"] > ./{{dir}}/Cargo-local.toml && \
+RUN toml set ./{{dir}}/Cargo.toml lib.crate-type ["cdylib"] > ./{{dir}}/Cargo-local.toml && \
     rm -rf ./{{dir}}/Cargo.toml && \
     mv ./{{dir}}/Cargo-local.toml ./{{dir}}/Cargo.toml && \
     true
 
-# Clean up artifacts left by the toml CLI program ("["cdylib", "rlib"]" -> ["cdylib", "rlib"])
-RUN sed -i 's/"\[cdylib,rlib\]"/\["cdylib","rlib"\]/g' ./{{dir}}/Cargo.toml
+# Clean up artifacts left by the toml CLI program ("["cdylib"]" -> ["cdylib"])
+RUN sed -i 's/"\[cdylib\]"/\["cdylib"\]/g' ./{{dir}}/Cargo.toml
 
-# Build the module at {{dir}}
-RUN cargo +nightly build --manifest-path ./{{dir}}/Cargo.toml \
-    --target wasm32-unknown-unknown --release
+# Ensure the package name = "module"
+RUN toml set ./{{dir}}/Cargo.toml package.name "module" > ./{{dir}}/Cargo-local.toml && \
+    rm -rf ./{{dir}}/Cargo.toml && \
+    mv ./{{dir}}/Cargo-local.toml ./{{dir}}/Cargo.toml && \
+    true
 
 # Make the build directory
 RUN rm -rf ./build
 RUN mkdir ./build
 
+# Build the module at {{dir}}
+RUN cargo build --manifest-path ./{{dir}}/Cargo.toml \
+    --target wasm32-unknown-unknown --release
+
+# Enable the "WASM_INTERFACE_TYPES" feature, which will remove the __wbindgen_throw import.
+# See: https://github.com/rustwasm/wasm-bindgen/blob/7f4663b70bd492278bf0e7bba4eeddb3d840c868/crates/cli-support/src/lib.rs#L397-L403
+ENV WASM_INTERFACE_TYPES=1
+
+# Run wasm-bindgen over the module, replacing all placeholder __wbindgen_... imports
+RUN wasm-bindgen ./{{dir}}/target/wasm32-unknown-unknown/release/module.wasm --out-dir ./build --out-name bg_module.wasm
+
+RUN wasm-snip ./build/bg_module.wasm -o ./build/snipped_module.wasm && \
+    rm -rf ./build/bg_module.wasm
+
 # Use wasm-opt to perform the "asyncify" post-processing step over all modules
-RUN WASM_MODULE=$(ls ./{{dir}}/target/wasm32-unknown-unknown/release/*.wasm); \
-    wasm-snip $WASM_MODULE -o ./build/snipped_{{name}}.wasm && \
-    wasm-opt --asyncify -Os ./build/snipped_{{name}}.wasm -o ./build/{{name}}.wasm && \
-    rm -rf ./build/snipped_{{name}}.wasm
+RUN wasm-opt --asyncify -Os ./build/snipped_module.wasm -o ./build/module.wasm && \
+    rm -rf ./build/snipped_module.wasm
 {{/polywrap_module}}

packages/js/asyncify/src/AsyncWasmInstance.ts

@@ -44,9 +44,7 @@ export class AsyncWasmInstance {
 
   private constructor() {}
 
-  public static createMemory(config: { module: ArrayBuffer }): WasmMemory {
-    const bytecode = new Uint8Array(config.module);
-
+  public static createMemory(config: { module: Uint8Array }): WasmMemory {
     // extract the initial memory page size, as it will
     // throw an error if the imported page size differs:
     // https://chromium.googlesource.com/v8/v8/+/644556e6ed0e6e4fac2dfabb441439820ec59813/src/wasm/module-instantiate.cc#924
@@ -73,7 +71,7 @@ export class AsyncWasmInstance {
       // 0x__,
     ]);
 
-    const sigIdx = indexOfArray(bytecode, envMemoryImportSignature);
+    const sigIdx = indexOfArray(config.module, envMemoryImportSignature);
 
     if (sigIdx < 0) {
       throw Error(
@@ -86,7 +84,7 @@ export class AsyncWasmInstance {
 
     // Extract the initial memory page-range size
     const memoryInitalLimits =
-      bytecode[sigIdx + envMemoryImportSignature.length + 1];
+      config.module[sigIdx + envMemoryImportSignature.length + 1];
 
     if (memoryInitalLimits === undefined) {
       throw Error(
@@ -98,7 +96,7 @@ export class AsyncWasmInstance {
   }
 
   public static async createInstance(config: {
-    module: ArrayBuffer;
+    module: Uint8Array;
     imports: WasmImports;
     requiredExports?: readonly string[];
   }): Promise<AsyncWasmInstance> {

packages/js/client/src/PolywrapClient.ts

@@ -18,6 +18,7 @@ import {
   InterfaceImplementations,
   InvokeOptions,
   InvokeResult,
+  InvokerOptions,
   PluginRegistration,
   QueryOptions,
   QueryResult,
@@ -47,6 +48,8 @@ import {
   JobRunner,
   PluginPackage,
   RunOptions,
+  msgpackEncode,
+  msgpackDecode,
 } from "@polywrap/core-js";
 import { Tracer } from "@polywrap/tracing-js";
 
@@ -197,7 +200,7 @@ export class PolywrapClient implements Client {
   public async getFile<TUri extends Uri | string>(
     uri: TUri,
     options: GetFileOptions
-  ): Promise<string | ArrayBuffer> {
+  ): Promise<string | Uint8Array> {
     const wrapper = await this._loadWrapper(this._toUri(uri), options);
     const client = contextualizeClient(this, options.contextId);
     return await wrapper.getFile(options, client);
@@ -309,14 +312,14 @@ export class PolywrapClient implements Client {
 
   @Tracer.traceMethod("PolywrapClient: invoke")
   public async invoke<TData = unknown, TUri extends Uri | string = string>(
-    options: InvokeOptions<TUri, PolywrapClientConfig>
+    options: InvokerOptions<TUri, PolywrapClientConfig>
   ): Promise<InvokeResult<TData>> {
     const { contextId, shouldClearContext } = this._setContext(
       options.contextId,
       options.config
     );
 
-    let result: InvokeResult<TData>;
+    let error: Error | undefined;
 
     try {
       const typedOptions: InvokeOptions<Uri> = {
@@ -327,18 +330,39 @@ export class PolywrapClient implements Client {
 
       const wrapper = await this._loadWrapper(typedOptions.uri, { contextId });
 
-      result = (await wrapper.invoke(
+      const invocableResult = await wrapper.invoke(
         typedOptions,
         contextualizeClient(this, contextId)
-      )) as TData;
-    } catch (error) {
-      result = { error };
+      );
+
+      if (invocableResult.data !== undefined) {
+        if (options.encodeResult && !invocableResult.encoded) {
+          return {
+            // TODO: if options.encodeResult, fix return type to Uint8Array
+            data: (msgpackEncode(invocableResult.data) as unknown) as TData,
+          };
+        } else if (invocableResult.encoded && !options.encodeResult) {
+          return {
+            // TODO: if result.encoded, fix return type to Uint8Array
+            data: msgpackDecode(invocableResult.data as Uint8Array) as TData,
+          };
+        } else {
+          return {
+            data: invocableResult.data as TData,
+          };
+        }
+      } else {
+        error = invocableResult.error;
+      }
+    } catch (e) {
+      error = e;
     }
 
     if (shouldClearContext) {
       this._clearContext(contextId);
     }
-    return result;
+
+    return { error };
   }
 
   @Tracer.traceMethod("PolywrapClient: run")

packages/js/client/src/__tests__/core/interface-impls.spec.ts

@@ -83,7 +83,7 @@ describe("interface-impls", () => {
         {
           uri: implementation4Uri,
           plugin: {
-            factory: () => ({} as PluginModule),
+            factory: () => ({} as PluginModule<{}>),
             manifest: {
               schema: "",
               implements: [],
@@ -145,7 +145,7 @@ describe("interface-impls", () => {
           {
             uri: interface1Uri,
             plugin: {
-              factory: () => ({} as PluginModule),
+              factory: () => ({} as PluginModule<{}>),
               manifest: {
                 schema: "",
                 implements: [],
@@ -155,7 +155,7 @@ describe("interface-impls", () => {
           {
             uri: interface2Uri,
             plugin: {
-              factory: () => ({} as PluginModule),
+              factory: () => ({} as PluginModule<{}>),
               manifest: {
                 schema: "",
                 implements: [],
@@ -197,7 +197,7 @@ describe("interface-impls", () => {
           {
             uri: interfaceUri,
             plugin: {
-              factory: () => ({} as PluginModule),
+              factory: () => ({} as PluginModule<{}>),
               manifest: {
                 schema: "",
                 implements: [],
@@ -295,7 +295,7 @@ describe("interface-impls", () => {
         {
           uri: implementation1Uri,
           plugin: {
-            factory: () => ({} as PluginModule),
+            factory: () => ({} as PluginModule<{}>),
             manifest: {
               schema: "",
               implements: [new Uri(interfaceUri)],
@@ -330,7 +330,7 @@ describe("interface-impls", () => {
         {
           uri: implementation1Uri,
           plugin: {
-            factory: () => ({} as PluginModule),
+            factory: () => ({} as PluginModule<{}>),
             manifest: {
               schema: "",
               implements: [],

packages/js/client/src/__tests__/core/resolveUri.spec.ts

@@ -190,7 +190,7 @@ describe("resolveUri", () => {
           uri: pluginUri.uri,
           plugin: {
             factory: () => {
-              return ({} as unknown) as PluginModule;
+              return ({} as unknown) as PluginModule<{}>;
             },
             manifest: {
               schema: "",

packages/js/client/src/__tests__/core/wasm-wrapper.spec.ts

@@ -113,13 +113,13 @@ describe("wasm-wrapper", () => {
           networkNameOrChainId: "testnet",
         },
       },
-      noDecode: true,
+      encodeResult: true,
     });
 
     expect(result.error).toBeFalsy();
     expect(result.data).toBeTruthy();
-    expect(result.data instanceof ArrayBuffer).toBeTruthy();
-    expect(msgpackDecode(result.data as ArrayBuffer)).toContain("0x");
+    expect(result.data instanceof Uint8Array).toBeTruthy();
+    expect(msgpackDecode(result.data as Uint8Array)).toContain("0x");
   });
 
   it("should invoke wrapper with custom redirects", async () => {
@@ -291,9 +291,9 @@ describe("wasm-wrapper", () => {
   ): Int!
 `);
 
-    const fileBuffer: ArrayBuffer = (await client.getFile(wrapperUri, {
+    const fileBuffer: Uint8Array = (await client.getFile(wrapperUri, {
       path: manifest.schema!,
-    })) as ArrayBuffer;
+    })) as Uint8Array;
     const decoder = new TextDecoder("utf8");
     const text = decoder.decode(fileBuffer);
     expect(text).toContain(`getData(