perf(loader): dispatch parallel loaders from Rust

Cargo.toml

@@ -21,6 +21,7 @@ aho-corasick    = { version = "1.1.4", default-features = false }
 anyhow          = { version = "1.0.102", default-features = false, features = ["backtrace", "std"] }
 anymap          = { package = "anymap3", version = "1.1.0", default-features = false, features = ["std"] }
 arc-swap        = { version = "1.9.1", default-features = false }
+async-channel   = { version = "2.5.0", default-features = false, features = ["std"] }
 async-recursion = { version = "1.1.1", default-features = false }
 async-trait     = { version = "0.1.89", default-features = false }
 atomic_refcell  = { version = "0.1.14", default-features = false }

crates/node_binding/napi-binding.d.ts

@@ -513,6 +513,20 @@ export declare class VirtualFileStore {
 }
 export type JsVirtualFileStore = VirtualFileStore
 
+/** Owns one Rust loader context received from the process-wide native MPMC queue. */
+export declare class WorkerTask {
+  /**
+   * Materializes the ordinary JsLoaderContext DTO in the worker isolate. Until this boundary the
+   * queue contains only the canonical Rust LoaderContext.
+   */
+  takeContext(): JsLoaderContext
+  getCompilation(): JsCompilationWrapper
+  getResolver(options?: RawResolveOptionsWithDependencyType | undefined | null): JsResolver
+  log(name: string, logType: string, message?: string | undefined | null): void
+  complete(context: JsLoaderContext): void
+  fail(error: string): void
+}
+
 export interface AssetInfoRelated {
   sourceMap?: string | null
 }
@@ -967,9 +981,11 @@ export interface JsLoaderContext {
   resource: string
   _module: Module
   hot: Readonly<boolean>
+  __internal__runHooksOnly: boolean
+  __internal__hookExtensions?: string
   /** Content maybe empty in pitching stage */
   content: null | Buffer
-  additionalData?: any
+  additionalData?: number
   __internal__parseMeta: Record<string, string>
   sourceMap?: Buffer
   cacheable: boolean
@@ -997,6 +1013,9 @@ export interface JsLoaderItem {
   loader: string
   type: string
   cache: boolean
+  parallel: boolean
+  optionsHandle?: number
+  ident?: string
   data: any
   normalExecuted: boolean
   pitchExecuted: boolean
@@ -2772,8 +2791,11 @@ export interface RawModuleRule {
 export interface RawModuleRuleUse {
   loader: string
   options?: string
+  /** Handle for ordinary JavaScript loader options kept in the main JS isolate. */
+  jsOptionsHandle?: number
   cache: boolean
   optionsCacheKey: string
+  parallel: boolean
 }
 
 export interface RawNewCache {
@@ -3226,6 +3248,12 @@ export interface RealDependencyLocation {
   end?: SourcePosition
 }
 
+/**
+ * Every JavaScript worker loops around this receive operation, so all workers compete for the
+ * same unbounded MPMC receiver without registration or per-pool state.
+ */
+export declare function recvWorkerTask(): Promise<WorkerTask>
+
 /** * this is a process level tracing, which means it would be shared by all compilers in the same process
  * only the first call would take effect, the following calls would be ignored
  * Some code is modified based on
@@ -3350,6 +3378,8 @@ export interface RegisterJsTaps {
   registerRsdoctorPluginAssetsTaps: (stages: Array<number>) => Array<{ function: ((arg: JsRsdoctorAssetPatch) => Promise<boolean | undefined>); stage: number; }>
 }
 
+export declare function registerMainThreadJsValueRelease(release: (arg: number) => void): void
+
 export interface ResolveResult {
   path?: string
   error?: string

crates/rspack/tests/loader.rs

@@ -28,8 +28,10 @@ async fn lightningcss() {
           options: Some(json!({
             "include": 1 // lower nesting syntax
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },
@@ -81,8 +83,10 @@ async fn swc() {
               }
             }
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },
@@ -119,8 +123,10 @@ async fn react_refresh() {
           ModuleRuleUseLoader {
             loader: "builtin:react-refresh-loader".to_string(),
             options: None,
+            js_options_handle: None,
             cache: false,
-            options_cache_key: String::new(),
+            options_cache_key: Default::default(),
+            parallel: false,
           },
           ModuleRuleUseLoader {
           loader: "builtin:swc-loader".to_string(),
@@ -141,8 +147,10 @@ async fn react_refresh() {
               }
             }
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },
@@ -180,8 +188,10 @@ async fn preact_refresh() {
           ModuleRuleUseLoader {
             loader: "builtin:preact-refresh-loader".to_string(),
             options: None,
+            js_options_handle: None,
             cache: false,
-            options_cache_key: String::new(),
+            options_cache_key: Default::default(),
+            parallel: false,
           },
           ModuleRuleUseLoader {
           loader: "builtin:swc-loader".to_string(),
@@ -202,8 +212,10 @@ async fn preact_refresh() {
               }
             }
           }).to_string()),
+          js_options_handle: None,
           cache: false,
-          options_cache_key: String::new(),
+          options_cache_key: Default::default(),
+          parallel: false,
         }]),
           ..Default::default()
         },

crates/rspack_binding_api/Cargo.toml

@@ -47,6 +47,7 @@ rspack_util               = { workspace = true }
 rspack_watcher            = { workspace = true }
 rspack_workspace          = { workspace = true }
 
+async-channel           = { workspace = true }
 async-trait             = { workspace = true }
 cow-utils               = { workspace = true }
 rspack_tracing          = { workspace = true }

crates/rspack_binding_api/src/lib.rs

@@ -102,6 +102,7 @@ mod swc;
 mod trace_event;
 mod utils;
 mod virtual_modules;
+mod worker;
 
 use std::{
   cell::RefCell,
@@ -153,6 +154,18 @@ use crate::{
 #[napi]
 pub const EXPECTED_RSPACK_CORE_VERSION: &str = rspack_workspace::rspack_pkg_version!();
 
+#[napi]
+pub fn register_main_thread_js_value_release(release: Function<u32, ()>) -> napi::Result<()> {
+  let release = release
+    .build_threadsafe_function::<u32>()
+    .weak::<true>()
+    .callee_handled::<false>()
+    .max_queue_size::<0>()
+    .build()?;
+  rspack_napi::MainThreadJsValueHandle::register_release(release);
+  Ok(())
+}
+
 thread_local! {
   static COMPILER_REFERENCES: RefCell<FxHashMap<CompilerId, WeakReference<JsCompiler>>> = Default::default();
 }