Move module loading out of js::core::Context, so it can be customised (#6199)

Author: eddyashton

.daily_canary

@@ -3,4 +3,4 @@
  (  V  ) /  .  \  |    +---=---'
 /--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---..
 2024__
-!"!
+!

include/ccf/js/core/context.h

@@ -5,6 +5,7 @@
 #include "ccf/js/core/runtime.h"
 #include "ccf/js/core/wrapped_value.h"
 #include "ccf/js/extensions/extension_interface.h"
+#include "ccf/js/modules/module_loader_interface.h"
 #include "ccf/js/tx_access.h"
 #include "ccf/pal/locking.h"
 
@@ -46,14 +47,16 @@ namespace ccf::js::core
     JSContext* ctx;
     Runtime rt;
 
+    js::extensions::Extensions extensions;
+    js::modules::ModuleLoaderPtr module_loader;
+
     // The interpreter can cache loaded modules so they do not need to be loaded
     // from the KV for every execution, which is particularly useful when
     // re-using interpreters. A module can only be loaded once per interpreter,
     // and the entire interpreter should be thrown away if _any_ of its modules
     // needs to be refreshed.
-    std::map<std::string, JSWrappedValue> loaded_modules_cache;
-
-    js::extensions::Extensions extensions;
+    std::map<std::string, js::core::JSWrappedValue, std::less<>>
+      loaded_modules_cache;
 
   public:
     ccf::pal::Mutex lock;
@@ -82,10 +85,13 @@ namespace ccf::js::core
       return ctx;
     }
 
-    std::optional<JSWrappedValue> get_module_from_cache(
-      const std::string& module_name);
-    void load_module_to_cache(
-      const std::string& module_name, const JSWrappedValue& module);
+    void set_module_loader(const modules::ModuleLoaderPtr& ml)
+    {
+      module_loader = ml;
+    }
+
+    virtual std::optional<JSWrappedValue> get_module(
+      std::string_view module_name);
 
     // Construct RAII wrapper around raw QuickJS value
     JSWrappedValue wrap(JSValue&& val) const;

include/ccf/js/modules.h

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/js/modules/module_loader_interface.h"
+
+namespace ccf::js::modules
+{
+  class ChainedModuleLoader : public ModuleLoaderInterface
+  {
+  protected:
+    ModuleLoaders sub_loaders;
+
+  public:
+    ChainedModuleLoader(ModuleLoaders&& ml) : sub_loaders(std::move(ml)) {}
+
+    virtual std::optional<js::core::JSWrappedValue> get_module(
+      std::string_view module_name, js::core::Context& ctx) override
+    {
+      for (auto& sub_loader : sub_loaders)
+      {
+        auto module_val = sub_loader->get_module(module_name, ctx);
+        if (module_val.has_value())
+        {
+          return module_val;
+        }
+      }
+
+      return std::nullopt;
+    }
+  };
+}

include/ccf/js/modules/chained_module_loader.h

@@ -0,0 +1,107 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/js/modules/module_loader_interface.h"
+#include "ccf/service/tables/modules.h"
+#include "ccf/tx.h"
+#include "ccf/version.h"
+
+#include <string>
+
+namespace ccf::js::modules
+{
+  class KvBytecodeModuleLoader : public ModuleLoaderInterface
+  {
+  protected:
+    ccf::ModulesQuickJsBytecode::ReadOnlyHandle* modules_bytecode_handle;
+
+    bool version_ok;
+
+  public:
+    KvBytecodeModuleLoader(
+      ccf::ModulesQuickJsBytecode::ReadOnlyHandle* mbh,
+      ccf::ModulesQuickJsVersion::ReadOnlyHandle* modules_version_handle) :
+      modules_bytecode_handle(mbh)
+    {
+      const auto version_in_kv = modules_version_handle->get();
+      const auto version_in_binary = std::string(ccf::quickjs_version);
+      if (version_in_kv != version_in_binary)
+      {
+        CCF_APP_INFO(
+          "Ignoring bytecode table, which was written for QuickJS {} (this "
+          "node is running QuickJS {})",
+          version_in_kv,
+          version_in_binary);
+        version_ok = false;
+      }
+      else
+      {
+        version_ok = true;
+      }
+    }
+
+    virtual std::optional<js::core::JSWrappedValue> get_module(
+      std::string_view module_name, js::core::Context& ctx) override
+    {
+      if (!version_ok)
+      {
+        return std::nullopt;
+      }
+
+      std::string module_name_kv(module_name);
+      if (module_name_kv[0] != '/')
+      {
+        module_name_kv.insert(0, "/");
+      }
+
+      CCF_APP_TRACE("Looking for module '{}' bytecode in KV", module_name_kv);
+
+      auto module_bytecode = modules_bytecode_handle->get(module_name_kv);
+      if (!module_bytecode.has_value())
+      {
+        CCF_APP_TRACE("Module '{}' not found", module_name_kv);
+        return std::nullopt;
+      }
+
+      auto module_val = ctx.read_object(
+        module_bytecode->data(), module_bytecode->size(), JS_READ_OBJ_BYTECODE);
+
+      const bool failed_deser = module_val.is_exception();
+      const bool failed_resolve =
+        !failed_deser && (JS_ResolveModule(ctx, module_val.val) < 0);
+
+      if (failed_deser || failed_resolve)
+      {
+        auto [reason, trace] = ctx.error_message();
+
+        auto& rt = ctx.runtime();
+        if (rt.log_exception_details)
+        {
+          CCF_APP_FAIL("{}: {}", reason, trace.value_or("<no trace>"));
+        }
+
+        if (failed_deser)
+        {
+          throw std::runtime_error(fmt::format(
+            "Failed to deserialize bytecode for module '{}': {}",
+            module_name,
+            reason));
+        }
+        else
+        {
+          throw std::runtime_error(fmt::format(
+            "Failed to resolve dependencies for module '{}': {}",
+            module_name,
+            reason));
+        }
+      }
+
+      CCF_APP_TRACE(
+        "Module '{}' bytecode found in KV (table: {})",
+        module_name_kv,
+        modules_bytecode_handle->get_name_of_map());
+      return module_val;
+    }
+  };
+}