fix: support importing bytes and text from npm packages

Author: dsherret

deno

@@ -23,7 +23,7 @@ deno_semver = "=0.8.1"
 url = "2.5"
 
 [dependencies.deno_ast]
-version = "=0.48.1"
+version = "=0.48.2"
 features = ["transpiling"]
 
 [dependencies.deno_cache_dir]
@@ -60,7 +60,11 @@ features = ["sync"]
 
 [dependencies.sys_traits]
 version = "=0.1.16"
-features = ["real","wasm"]
+features = ["real"]
+
+[target.'cfg(target_arch = "wasm32")'.dependencies.sys_traits]
+version = "=0.1.16"
+features = ["real", "wasm"]
 
 [profile.release]
 codegen-units = 1

src/rs_lib/Cargo.lock

@@ -40,8 +40,9 @@ use deno_resolver::file_fetcher::PermissionedFileFetcher;
 use deno_resolver::file_fetcher::PermissionedFileFetcherOptions;
 use deno_resolver::graph::DefaultDenoResolverRc;
 use deno_resolver::graph::ResolveWithGraphOptions;
-use deno_resolver::loader::PreparedModuleLoader;
-use deno_resolver::loader::PreparedModuleOrAsset;
+use deno_resolver::loader::LoadCodeSourceErrorKind;
+use deno_resolver::loader::LoadedModuleOrAsset;
+use deno_resolver::loader::ModuleLoader;
 use deno_resolver::loader::RequestedModuleType;
 use deno_resolver::npm::DenoInNpmPackageChecker;
 use deno_semver::SmallStackString;
@@ -293,10 +294,7 @@ impl DenoWorkspace {
       resolver_factory: self.resolver_factory.clone(),
       npm_installer_factory: self.npm_installer_factory.clone(),
       parsed_source_cache: self.resolver_factory.parsed_source_cache().clone(),
-      prepared_module_loader: self
-        .resolver_factory
-        .prepared_module_loader()?
-        .clone(),
+      module_loader: self.resolver_factory.module_loader()?.clone(),
       task_queue: Default::default(),
       graph: deno_graph::ModuleGraph::new(deno_graph::GraphKind::CodeOnly),
     })
@@ -314,8 +312,7 @@ pub struct DenoLoader {
   npm_installer_factory:
     Arc<NpmInstallerFactory<WasmHttpClient, ConsoleLogReporter, RealSys>>,
   parsed_source_cache: Arc<ParsedSourceCache>,
-  prepared_module_loader:
-    Arc<PreparedModuleLoader<DenoInNpmPackageChecker, RealSys>>,
+  module_loader: Arc<ModuleLoader<RealSys>>,
   resolver_factory: Arc<ResolverFactory<RealSys>>,
   workspace_factory: Arc<WorkspaceFactory<RealSys>>,
   graph: ModuleGraph,
@@ -501,67 +498,72 @@ impl DenoLoader {
       return Ok(create_external_repsonse(&url));
     }
 
-    let maybe_prepared_module = self
-      .prepared_module_loader
-      .load_prepared_module(&self.graph, &url, requested_module_type)
-      .await?;
-
-    match maybe_prepared_module {
-      Some(PreparedModuleOrAsset::Module(m)) => {
+    match self
+      .module_loader
+      .load(&self.graph, &url, None, requested_module_type)
+      .await
+    {
+      Ok(LoadedModuleOrAsset::Module(m)) => {
         self.parsed_source_cache.free(&m.specifier);
         Ok(create_module_response(
           &m.specifier,
           m.media_type,
           m.source.as_bytes(),
         ))
       }
-      Some(PreparedModuleOrAsset::ExternalAsset { specifier }) => {
+      Ok(LoadedModuleOrAsset::ExternalAsset {
+        specifier,
+        statically_analyzable: _,
+      }) => {
         let file = self
           .file_fetcher
-          .fetch_bypass_permissions(specifier)
+          .fetch_bypass_permissions(&specifier)
           .await?;
         let media_type = MediaType::from_specifier_and_headers(
           &file.url,
           file.maybe_headers.as_ref(),
         );
         Ok(create_module_response(&file.url, media_type, &file.source))
       }
-      None => {
-        if url.scheme() == "npm" {
-          bail!(
-            "Failed resolving '{}'\n\nResolve the npm: specifier to a file: specifier before providing it to the loader.",
-            url
-          )
-        }
-        let file = self.file_fetcher.fetch_bypass_permissions(&url).await?;
-        let media_type = MediaType::from_specifier_and_headers(
-          &url,
-          file.maybe_headers.as_ref(),
-        );
-        match requested_module_type {
-          RequestedModuleType::Text | RequestedModuleType::Bytes => {
-            Ok(create_module_response(&file.url, media_type, &file.source))
+      Err(err) => match err.as_kind() {
+        LoadCodeSourceErrorKind::LoadUnpreparedModule(_) => {
+          if url.scheme() == "npm" {
+            bail!(
+              "Failed resolving '{}'\n\nResolve the npm: specifier to a file: specifier before providing it to the loader.",
+              url
+            )
           }
-          RequestedModuleType::Json
-          | RequestedModuleType::None
-          | RequestedModuleType::Other(_) => {
-            if media_type.is_emittable() {
-              let str = String::from_utf8_lossy(&file.source);
-              let value = str.into();
-              let source = self
-                .maybe_transpile(&file.url, media_type, &value, None)
-                .await?;
-              Ok(create_module_response(
-                &file.url,
-                media_type,
-                source.as_bytes(),
-              ))
-            } else {
+          let file = self.file_fetcher.fetch_bypass_permissions(&url).await?;
+          let media_type = MediaType::from_specifier_and_headers(
+            &url,
+            file.maybe_headers.as_ref(),
+          );
+          match requested_module_type {
+            RequestedModuleType::Text | RequestedModuleType::Bytes => {
               Ok(create_module_response(&file.url, media_type, &file.source))
             }
+            RequestedModuleType::Json
+            | RequestedModuleType::None
+            | RequestedModuleType::Other(_) => {
+              if media_type.is_emittable() {
+                let str = String::from_utf8_lossy(&file.source);
+                let value = str.into();
+                let source = self
+                  .maybe_transpile(&file.url, media_type, &value, None)
+                  .await?;
+                Ok(create_module_response(
+                  &file.url,
+                  media_type,
+                  source.as_bytes(),
+                ))
+              } else {
+                Ok(create_module_response(&file.url, media_type, &file.source))
+              }
+            }
           }
         }
-      }
+        _ => return Err(err.into()),
+      },
     }
   }
 

src/rs_lib/Cargo.toml

@@ -1,3 +1,4 @@
+import { assertEquals } from "@std/assert";
 import {
   assertResponseText,
   createLoader,
@@ -29,4 +30,12 @@ Deno.test("loads jsx transpiled", async () => {
     await loader.load(dataFileUrl, RequestedModuleType.Text),
     `Hello there!`,
   );
+  const bytesResponse = await loader.load(
+    dataFileUrl,
+    RequestedModuleType.Bytes,
+  );
+  if (bytesResponse.kind !== "module") {
+    throw new Error("Fail");
+  }
+  assertEquals(bytesResponse.code, Deno.readFileSync(new URL(dataFileUrl)));
 });

src/rs_lib/lib.rs

@@ -0,0 +1,41 @@
+import { assertEquals } from "@std/assert";
+import {
+  assertResponseText,
+  createLoader,
+  RequestedModuleType,
+  ResolutionMode,
+  type WorkspaceOptions,
+} from "../helpers.ts";
+
+Deno.test("loads jsx transpiled", async () => {
+  const mainTs = import.meta.dirname + "/testdata/main.ts";
+  const createWorkspace = async (options?: WorkspaceOptions) => {
+    return await createLoader({
+      configPath: import.meta.dirname + "/testdata/deno.json",
+      ...(options ?? {}),
+    }, {
+      entrypoints: [mainTs],
+    });
+  };
+  const { loader } = await createWorkspace();
+
+  const mainTsUrl = loader.resolve(mainTs, undefined, ResolutionMode.Import);
+  const dataFileUrl = loader.resolve(
+    "package/data.txt",
+    mainTsUrl,
+    ResolutionMode.Import,
+  );
+
+  assertResponseText(
+    await loader.load(dataFileUrl, RequestedModuleType.Text),
+    `Hello there!`,
+  );
+  const bytesResponse = await loader.load(
+    dataFileUrl,
+    RequestedModuleType.Bytes,
+  );
+  if (bytesResponse.kind !== "module") {
+    throw new Error("Fail");
+  }
+  assertEquals(bytesResponse.code, Deno.readFileSync(new URL(dataFileUrl)));
+});

tests/bytes_and_text/main.test.ts

@@ -0,0 +1 @@
+Hello there!

tests/bytes_and_text_npm/main.test.ts

@@ -0,0 +1,5 @@
+{
+  "unstable": [
+    "raw-imports"
+  ]
+}

tests/bytes_and_text_npm/testdata/data_utf8_bom.txt

@@ -0,0 +1,5 @@
+import dataBytes from "package/data.txt" with { type: "bytes" };
+import dataText from "package/data.txt" with { type: "text" };
+
+console.log(dataBytes);
+console.log(dataText);