refactor(loader): dispatch parallel loaders from Rust

Author: intellild

Cargo.lock

@@ -21,6 +21,7 @@ aho-corasick    = { version = "1.1.4", default-features = false }
 anyhow          = { version = "1.0.102", default-features = false, features = ["backtrace", "std"] }
 anymap          = { package = "anymap3", version = "1.1.0", default-features = false, features = ["std"] }
 arc-swap        = { version = "1.9.1", default-features = false }
+async-channel   = { version = "2.5.0", default-features = false, features = ["std"] }
 async-recursion = { version = "1.1.1", default-features = false }
 async-trait     = { version = "0.1.89", default-features = false }
 atomic_refcell  = { version = "0.1.14", default-features = false }

Cargo.toml

@@ -513,6 +513,17 @@ export declare class VirtualFileStore {
 }
 export type JsVirtualFileStore = VirtualFileStore
 
+/** Owns one Rust loader context received from the process-wide native MPMC queue. */
+export declare class WorkerTask {
+  /**
+   * Materializes the ordinary JsLoaderContext DTO in the worker isolate. Until this boundary the
+   * queue contains only the canonical Rust LoaderContext.
+   */
+  takeContext(): JsLoaderContext
+  complete(context: JsLoaderContext): void
+  fail(error: string): void
+}
+
 export interface AssetInfoRelated {
   sourceMap?: string | null
 }
@@ -967,9 +978,14 @@ export interface JsLoaderContext {
   resource: string
   _module: Module
   hot: Readonly<boolean>
+  rootContext: string
+  mode: string
+  loaderContextExtensions: string
+  __internal__runHooksOnly: boolean
+  __internal__hookExtensions?: string
   /** Content maybe empty in pitching stage */
   content: null | Buffer
-  additionalData?: any
+  additionalData?: string
   __internal__parseMeta: Record<string, string>
   sourceMap?: Buffer
   cacheable: boolean
@@ -997,6 +1013,9 @@ export interface JsLoaderItem {
   loader: string
   type: string
   cache: boolean
+  parallel: boolean
+  optionsHandle?: number
+  ident?: string
   data: any
   normalExecuted: boolean
   pitchExecuted: boolean
@@ -2772,8 +2791,11 @@ export interface RawModuleRule {
 export interface RawModuleRuleUse {
   loader: string
   options?: string
+  /** Handle for ordinary JavaScript loader options kept in the main JS isolate. */
+  jsOptionsHandle?: number
   cache: boolean
   optionsCacheKey: string
+  parallel: boolean
 }
 
 export interface RawNewCache {
@@ -3226,6 +3248,12 @@ export interface RealDependencyLocation {
   end?: SourcePosition
 }
 
+/**
+ * Every JavaScript worker loops around this receive operation, so all workers compete for the
+ * same unbounded MPMC receiver without registration or per-pool state.
+ */
+export declare function recvWorkerTask(): Promise<WorkerTask>
+
 /** * this is a process level tracing, which means it would be shared by all compilers in the same process
  * only the first call would take effect, the following calls would be ignored
  * Some code is modified based on

crates/node_binding/napi-binding.d.ts

@@ -28,8 +28,10 @@ async fn lightningcss() {
           options: Some(json!({
             "include": 1 // lower nesting syntax
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },
@@ -81,8 +83,10 @@ async fn swc() {
               }
             }
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },
@@ -119,8 +123,10 @@ async fn react_refresh() {
           ModuleRuleUseLoader {
             loader: "builtin:react-refresh-loader".to_string(),
             options: None,
+            js_options_handle: None,
             cache: false,
-            options_cache_key: String::new(),
+            options_cache_key: Default::default(),
+            parallel: false,
           },
           ModuleRuleUseLoader {
           loader: "builtin:swc-loader".to_string(),
@@ -141,8 +147,10 @@ async fn react_refresh() {
               }
             }
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },
@@ -180,8 +188,10 @@ async fn preact_refresh() {
           ModuleRuleUseLoader {
             loader: "builtin:preact-refresh-loader".to_string(),
             options: None,
+            js_options_handle: None,
             cache: false,
-            options_cache_key: String::new(),
+            options_cache_key: Default::default(),
+            parallel: false,
           },
           ModuleRuleUseLoader {
           loader: "builtin:swc-loader".to_string(),
@@ -202,8 +212,10 @@ async fn preact_refresh() {
               }
             }
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },

crates/rspack/tests/loader.rs

@@ -79,6 +79,7 @@ mod module;
 mod module_graph;
 mod module_graph_connection;
 mod modules;
+mod native_reference;
 mod native_watcher;
 mod normal_module_factory;
 mod options;
@@ -101,6 +102,7 @@ mod swc;
 mod trace_event;
 mod utils;
 mod virtual_modules;
+mod worker;
 
 use std::{
   cell::RefCell,