perf(npm): run lifecycle scripts in parallel (#32666)

Author: dsherret

cli/npm.rs

@@ -1,13 +1,17 @@
 // Copyright 2018-2026 the Deno authors. MIT license.
 
 use std::borrow::Cow;
+use std::collections::HashMap;
 use std::collections::HashSet;
+use std::ffi::OsString;
+use std::num::NonZeroUsize;
 use std::path::PathBuf;
 use std::rc::Rc;
 use std::sync::Arc;
 
 use dashmap::DashMap;
 use deno_core::error::AnyError;
+use deno_core::futures::StreamExt;
 use deno_core::serde_json;
 use deno_core::url::Url;
 use deno_error::JsErrorBox;
@@ -28,6 +32,7 @@ use deno_npm_installer::lifecycle_scripts::LIFECYCLE_SCRIPTS_RUNNING_ENV_VAR;
 use deno_npm_installer::lifecycle_scripts::LifecycleScriptsExecutor;
 use deno_npm_installer::lifecycle_scripts::LifecycleScriptsExecutorOptions;
 use deno_npm_installer::lifecycle_scripts::PackageWithScript;
+use deno_npm_installer::lifecycle_scripts::compute_lifecycle_script_layers;
 use deno_npm_installer::lifecycle_scripts::is_broken_default_install_script;
 use deno_resolver::npm::ByonmNpmResolverCreateOptions;
 use deno_resolver::npm::ManagedNpmResolverRc;
@@ -367,6 +372,11 @@ pub struct DenoTaskLifeCycleScriptsExecutor {
   system_info: deno_npm::NpmSystemInfo,
 }
 
+struct PackageScriptResult<'a> {
+  package: &'a NpmResolutionPackage,
+  failed: Option<&'a PackageNv>,
+}
+
 #[async_trait::async_trait(?Send)]
 impl LifecycleScriptsExecutor for DenoTaskLifeCycleScriptsExecutor {
   async fn execute(
@@ -396,105 +406,48 @@ impl LifecycleScriptsExecutor for DenoTaskLifeCycleScriptsExecutor {
     // so the subprocess can detect that it is running as part of a lifecycle script,
     // and avoid trying to set up node_modules again
     env_vars.insert(LIFECYCLE_SCRIPTS_RUNNING_ENV_VAR.into(), "1".into());
-    // we want to pass the current state of npm resolution down to the deno subprocess
-    // (that may be running as part of the script). we do this with an inherited temp file
-    //
-    // SAFETY: we are sharing a single temp file across all of the scripts. the file position
-    // will be shared among these, which is okay since we run only one script at a time.
-    // However, if we concurrently run scripts in the future we will
-    // have to have multiple temp files.
-    let temp_file_fd = deno_runtime::deno_process::npm_process_state_tempfile(
-      options.process_state.as_bytes(),
-    )
-    .map_err(DenoTaskLifecycleScriptsError::CreateNpmProcessState)?;
-    // SAFETY: fd/handle is valid
-    let _temp_file = unsafe { std::fs::File::from_raw_io_handle(temp_file_fd) }; // make sure the file gets closed
-    env_vars.insert(
-      deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME.into(),
-      (temp_file_fd as usize).to_string().into(),
+
+    let concurrency = std::thread::available_parallelism()
+      .ok()
+      .and_then(|n| NonZeroUsize::new(n.get().saturating_sub(1)))
+      .unwrap_or_else(|| NonZeroUsize::new(2).unwrap())
+      .get();
+
+    let layers = compute_lifecycle_script_layers(
+      options.packages_with_scripts,
+      options.snapshot,
     );
-    for PackageWithScript {
-      package,
-      scripts,
-      package_folder,
-    } in options.packages_with_scripts
-    {
-      // add custom commands for binaries from the package's dependencies. this will take precedence over the
-      // baseline commands, so if the package relies on a bin that conflicts with one higher in the dependency tree, the
-      // correct bin will be used.
-      let custom_commands = self
-        .resolve_custom_commands_from_deps(
-          options.extra_info_provider,
-          base.clone(),
-          package,
-          options.snapshot,
-        )
-        .await;
-      for script_name in ["preinstall", "install", "postinstall"] {
-        if let Some(script) = scripts.get(script_name) {
-          if script_name == "install"
-            && is_broken_default_install_script(&sys, script, package_folder)
-          {
-            continue;
-          }
-          let _guard = self.progress_bar.update_with_prompt(
-            ProgressMessagePrompt::Initialize,
-            &format!("{}: running '{script_name}' script", package.id.nv),
-          );
-          let crate::task_runner::TaskResult {
-            exit_code,
-            stderr,
-            stdout,
-          } =
-            crate::task_runner::run_task(crate::task_runner::RunTaskOptions {
-              task_name: script_name,
-              script,
-              cwd: package_folder.clone(),
-              env_vars: env_vars.clone(),
-              custom_commands: custom_commands.clone(),
-              init_cwd: options.init_cwd,
-              argv: &[],
-              root_node_modules_dir: Some(options.root_node_modules_dir_path),
-              stdio: Some(crate::task_runner::TaskIo {
-                stderr: TaskStdio::piped(),
-                stdout: TaskStdio::piped(),
-              }),
-              kill_signal: kill_signal.clone(),
-            })
-            .await
-            .map_err(DenoTaskLifecycleScriptsError::Task)?;
-          let stdout = stdout.unwrap();
-          let stderr = stderr.unwrap();
-          if exit_code != 0 {
-            log::warn!(
-              "error: script '{}' in '{}' failed with exit code {}{}{}",
-              script_name,
-              package.id.nv,
-              exit_code,
-              if !stdout.trim_ascii().is_empty() {
-                format!(
-                  "\nstdout:\n{}\n",
-                  String::from_utf8_lossy(&stdout).trim()
-                )
-              } else {
-                String::new()
-              },
-              if !stderr.trim_ascii().is_empty() {
-                format!(
-                  "\nstderr:\n{}\n",
-                  String::from_utf8_lossy(&stderr).trim()
-                )
-              } else {
-                String::new()
-              },
-            );
-            failed_packages.push(&package.id.nv);
-            // assume if earlier script fails, later ones will fail too
-            break;
-          }
+
+    for layer in &layers {
+      log::debug!(
+        "Running lifecycle scripts layer: {}",
+        layer
+          .iter()
+          .map(|l| l.package.id.as_serialized())
+          .collect::<Vec<_>>()
+          .join(", ")
+      );
+
+      let mut results =
+        deno_core::futures::stream::iter(layer.iter().map(|pkg| {
+          self.run_single_package_scripts(
+            pkg,
+            &env_vars,
+            &base,
+            &options,
+            &kill_signal,
+            &sys,
+          )
+        }))
+        .buffer_unordered(concurrency);
+
+      while let Some(result) = results.next().await {
+        let result = result?;
+        if let Some(nv) = result.failed {
+          failed_packages.push(nv);
         }
+        (options.on_ran_pkg_scripts)(result.package)?;
       }
-      (options.on_ran_pkg_scripts)(package)?;
     }
 
     // re-set up bin entries for the packages which we've run scripts for.
@@ -542,6 +495,118 @@ impl DenoTaskLifeCycleScriptsExecutor {
     }
   }
 
+  /// Runs lifecycle scripts for a single package (preinstall, install,
+  /// postinstall in order). Each package gets its own temp file for
+  /// npm process state so concurrent execution is safe.
+  async fn run_single_package_scripts<'a>(
+    &self,
+    pkg: &'a PackageWithScript<'a>,
+    env_vars: &HashMap<OsString, OsString>,
+    base_custom_commands: &crate::task_runner::TaskCustomCommands,
+    options: &LifecycleScriptsExecutorOptions<'a>,
+    kill_signal: &KillSignal,
+    sys: &CliSys,
+  ) -> Result<PackageScriptResult<'a>, AnyError> {
+    let PackageWithScript {
+      package,
+      scripts,
+      package_folder,
+    } = pkg;
+
+    // each concurrent package gets its own temp file to avoid fd races
+    let temp_file_fd = deno_runtime::deno_process::npm_process_state_tempfile(
+      options.process_state.as_bytes(),
+    )
+    .map_err(DenoTaskLifecycleScriptsError::CreateNpmProcessState)?;
+    // SAFETY: fd/handle is valid
+    let _temp_file = unsafe { std::fs::File::from_raw_io_handle(temp_file_fd) };
+    let mut env_vars = env_vars.clone();
+    env_vars.insert(
+      deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME.into(),
+      (temp_file_fd as usize).to_string().into(),
+    );
+
+    // add custom commands for binaries from the package's dependencies.
+    // this will take precedence over the baseline commands, so if the
+    // package relies on a bin that conflicts with one higher in the
+    // dependency tree, the correct bin will be used.
+    let custom_commands = self
+      .resolve_custom_commands_from_deps(
+        options.extra_info_provider,
+        base_custom_commands.clone(),
+        package,
+        options.snapshot,
+      )
+      .await;
+
+    let mut failed = None;
+    for script_name in ["preinstall", "install", "postinstall"] {
+      if let Some(script) = scripts.get(script_name) {
+        if script_name == "install"
+          && is_broken_default_install_script(sys, script, package_folder)
+        {
+          continue;
+        }
+        let _guard = self.progress_bar.update_with_prompt(
+          ProgressMessagePrompt::Initialize,
+          &format!("{}: running '{script_name}' script", package.id.nv),
+        );
+        let crate::task_runner::TaskResult {
+          exit_code,
+          stderr,
+          stdout,
+        } = crate::task_runner::run_task(crate::task_runner::RunTaskOptions {
+          task_name: script_name,
+          script,
+          cwd: package_folder.clone(),
+          env_vars: env_vars.clone(),
+          custom_commands: custom_commands.clone(),
+          init_cwd: options.init_cwd,
+          argv: &[],
+          root_node_modules_dir: Some(options.root_node_modules_dir_path),
+          stdio: Some(crate::task_runner::TaskIo {
+            stderr: TaskStdio::piped(),
+            stdout: TaskStdio::piped(),
+          }),
+          kill_signal: kill_signal.clone(),
+        })
+        .await
+        .map_err(DenoTaskLifecycleScriptsError::Task)?;
+        let stdout = stdout.unwrap();
+        let stderr = stderr.unwrap();
+        if exit_code != 0 {
+          log::warn!(
+            "error: script '{}' in '{}' failed with exit code {}{}{}",
+            script_name,
+            package.id.nv,
+            exit_code,
+            if !stdout.trim_ascii().is_empty() {
+              format!(
+                "\nstdout:\n{}\n",
+                String::from_utf8_lossy(&stdout).trim()
+              )
+            } else {
+              String::new()
+            },
+            if !stderr.trim_ascii().is_empty() {
+              format!(
+                "\nstderr:\n{}\n",
+                String::from_utf8_lossy(&stderr).trim()
+              )
+            } else {
+              String::new()
+            },
+          );
+          failed = Some(&package.id.nv);
+          // assume if earlier script fails, later ones will fail too
+          break;
+        }
+      }
+    }
+
+    Ok(PackageScriptResult { package, failed })
+  }
+
   // take in all (non copy) packages from snapshot,
   // and resolve the set of available binaries to create
   // custom commands available to the task runner