Skip to content

perf(npm): run lifecycle scripts in parallel #32666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dsherret merged 1 commit into from
Mar 12, 2026
+454 −96
Merged

perf(npm): run lifecycle scripts in parallel #32666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
257 changes: 161 additions & 96 deletions cli/npm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
// Copyright 2018-2026 the Deno authors. MIT license.

use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::HashSet;
use std::ffi::OsString;
use std::num::NonZeroUsize;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;

use dashmap::DashMap;
use deno_core::error::AnyError;
use deno_core::futures::StreamExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_error::JsErrorBox;
Expand All @@ -27,6 +31,7 @@ use deno_npm_installer::lifecycle_scripts::LIFECYCLE_SCRIPTS_RUNNING_ENV_VAR;
use deno_npm_installer::lifecycle_scripts::LifecycleScriptsExecutor;
use deno_npm_installer::lifecycle_scripts::LifecycleScriptsExecutorOptions;
use deno_npm_installer::lifecycle_scripts::PackageWithScript;
use deno_npm_installer::lifecycle_scripts::compute_lifecycle_script_layers;
use deno_npm_installer::lifecycle_scripts::is_broken_default_install_script;
use deno_resolver::npm::ByonmNpmResolverCreateOptions;
use deno_resolver::npm::ManagedNpmResolverRc;
Expand Down Expand Up @@ -349,6 +354,11 @@ pub struct DenoTaskLifeCycleScriptsExecutor {
system_info: deno_npm::NpmSystemInfo,
}

struct PackageScriptResult<'a> {
package: &'a NpmResolutionPackage,
failed: Option<&'a PackageNv>,
}

#[async_trait::async_trait(?Send)]
impl LifecycleScriptsExecutor for DenoTaskLifeCycleScriptsExecutor {
async fn execute(
Expand Down Expand Up @@ -378,105 +388,48 @@ impl LifecycleScriptsExecutor for DenoTaskLifeCycleScriptsExecutor {
// so the subprocess can detect that it is running as part of a lifecycle script,
// and avoid trying to set up node_modules again
env_vars.insert(LIFECYCLE_SCRIPTS_RUNNING_ENV_VAR.into(), "1".into());
// we want to pass the current state of npm resolution down to the deno subprocess
// (that may be running as part of the script). we do this with an inherited temp file
//
// SAFETY: we are sharing a single temp file across all of the scripts. the file position
// will be shared among these, which is okay since we run only one script at a time.
// However, if we concurrently run scripts in the future we will
// have to have multiple temp files.
let temp_file_fd = deno_runtime::deno_process::npm_process_state_tempfile(
options.process_state.as_bytes(),
)
.map_err(DenoTaskLifecycleScriptsError::CreateNpmProcessState)?;
// SAFETY: fd/handle is valid
let _temp_file = unsafe { std::fs::File::from_raw_io_handle(temp_file_fd) }; // make sure the file gets closed
env_vars.insert(
deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME.into(),
(temp_file_fd as usize).to_string().into(),

let concurrency = std::thread::available_parallelism()
.ok()
.and_then(|n| NonZeroUsize::new(n.get().saturating_sub(1)))
.unwrap_or_else(|| NonZeroUsize::new(2).unwrap())
.get();

let layers = compute_lifecycle_script_layers(
options.packages_with_scripts,
options.snapshot,
);
for PackageWithScript {
package,
scripts,
package_folder,
} in options.packages_with_scripts
{
// add custom commands for binaries from the package's dependencies. this will take precedence over the
// baseline commands, so if the package relies on a bin that conflicts with one higher in the dependency tree, the
// correct bin will be used.
let custom_commands = self
.resolve_custom_commands_from_deps(
options.extra_info_provider,
base.clone(),
package,
options.snapshot,
)
.await;
for script_name in ["preinstall", "install", "postinstall"] {
if let Some(script) = scripts.get(script_name) {
if script_name == "install"
&& is_broken_default_install_script(&sys, script, package_folder)
{
continue;
}
let _guard = self.progress_bar.update_with_prompt(
ProgressMessagePrompt::Initialize,
&format!("{}: running '{script_name}' script", package.id.nv),
);
let crate::task_runner::TaskResult {
exit_code,
stderr,
stdout,
} =
crate::task_runner::run_task(crate::task_runner::RunTaskOptions {
task_name: script_name,
script,
cwd: package_folder.clone(),
env_vars: env_vars.clone(),
custom_commands: custom_commands.clone(),
init_cwd: options.init_cwd,
argv: &[],
root_node_modules_dir: Some(options.root_node_modules_dir_path),
stdio: Some(crate::task_runner::TaskIo {
stderr: TaskStdio::piped(),
stdout: TaskStdio::piped(),
}),
kill_signal: kill_signal.clone(),
})
.await
.map_err(DenoTaskLifecycleScriptsError::Task)?;
let stdout = stdout.unwrap();
let stderr = stderr.unwrap();
if exit_code != 0 {
log::warn!(
"error: script '{}' in '{}' failed with exit code {}{}{}",
script_name,
package.id.nv,
exit_code,
if !stdout.trim_ascii().is_empty() {
format!(
"\nstdout:\n{}\n",
String::from_utf8_lossy(&stdout).trim()
)
} else {
String::new()
},
if !stderr.trim_ascii().is_empty() {
format!(
"\nstderr:\n{}\n",
String::from_utf8_lossy(&stderr).trim()
)
} else {
String::new()
},
);
failed_packages.push(&package.id.nv);
// assume if earlier script fails, later ones will fail too
break;
}

for layer in &layers {
log::debug!(
"Running lifecycle scripts layer: {}",
layer
.iter()
.map(|l| l.package.id.as_serialized())
.collect::<Vec<_>>()
.join(", ")
);

let mut results =
deno_core::futures::stream::iter(layer.iter().map(|pkg| {
self.run_single_package_scripts(
pkg,
&env_vars,
&base,
&options,
&kill_signal,
&sys,
)
}))
.buffer_unordered(concurrency);

while let Some(result) = results.next().await {
let result = result?;
if let Some(nv) = result.failed {
failed_packages.push(nv);
}
(options.on_ran_pkg_scripts)(result.package)?;
}
(options.on_ran_pkg_scripts)(package)?;
}

// re-set up bin entries for the packages which we've run scripts for.
Expand Down Expand Up @@ -524,6 +477,118 @@ impl DenoTaskLifeCycleScriptsExecutor {
}
}

/// Runs lifecycle scripts for a single package (preinstall, install,
/// postinstall in order). Each package gets its own temp file for
/// npm process state so concurrent execution is safe.
async fn run_single_package_scripts<'a>(
&self,
pkg: &'a PackageWithScript<'a>,
env_vars: &HashMap<OsString, OsString>,
base_custom_commands: &crate::task_runner::TaskCustomCommands,
options: &LifecycleScriptsExecutorOptions<'a>,
kill_signal: &KillSignal,
sys: &CliSys,
) -> Result<PackageScriptResult<'a>, AnyError> {
let PackageWithScript {
package,
scripts,
package_folder,
} = pkg;

// each concurrent package gets its own temp file to avoid fd races
let temp_file_fd = deno_runtime::deno_process::npm_process_state_tempfile(
options.process_state.as_bytes(),
)
.map_err(DenoTaskLifecycleScriptsError::CreateNpmProcessState)?;
// SAFETY: fd/handle is valid
let _temp_file = unsafe { std::fs::File::from_raw_io_handle(temp_file_fd) };
let mut env_vars = env_vars.clone();
env_vars.insert(
deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME.into(),
(temp_file_fd as usize).to_string().into(),
);

// add custom commands for binaries from the package's dependencies.
// this will take precedence over the baseline commands, so if the
// package relies on a bin that conflicts with one higher in the
// dependency tree, the correct bin will be used.
let custom_commands = self
.resolve_custom_commands_from_deps(
options.extra_info_provider,
base_custom_commands.clone(),
package,
options.snapshot,
)
.await;

let mut failed = None;
for script_name in ["preinstall", "install", "postinstall"] {
if let Some(script) = scripts.get(script_name) {
if script_name == "install"
&& is_broken_default_install_script(sys, script, package_folder)
{
continue;
}
let _guard = self.progress_bar.update_with_prompt(
ProgressMessagePrompt::Initialize,
&format!("{}: running '{script_name}' script", package.id.nv),
);
let crate::task_runner::TaskResult {
exit_code,
stderr,
stdout,
} = crate::task_runner::run_task(crate::task_runner::RunTaskOptions {
task_name: script_name,
script,
cwd: package_folder.clone(),
env_vars: env_vars.clone(),
custom_commands: custom_commands.clone(),
init_cwd: options.init_cwd,
argv: &[],
root_node_modules_dir: Some(options.root_node_modules_dir_path),
stdio: Some(crate::task_runner::TaskIo {
stderr: TaskStdio::piped(),
stdout: TaskStdio::piped(),
}),
kill_signal: kill_signal.clone(),
})
.await
.map_err(DenoTaskLifecycleScriptsError::Task)?;
let stdout = stdout.unwrap();
let stderr = stderr.unwrap();
if exit_code != 0 {
log::warn!(
"error: script '{}' in '{}' failed with exit code {}{}{}",
script_name,
package.id.nv,
exit_code,
if !stdout.trim_ascii().is_empty() {
format!(
"\nstdout:\n{}\n",
String::from_utf8_lossy(&stdout).trim()
)
} else {
String::new()
},
if !stderr.trim_ascii().is_empty() {
format!(
"\nstderr:\n{}\n",
String::from_utf8_lossy(&stderr).trim()
)
} else {
String::new()
},
);
failed = Some(&package.id.nv);
// assume if earlier script fails, later ones will fail too
break;
}
}
}

Ok(PackageScriptResult { package, failed })
}

// take in all (non copy) packages from snapshot,
// and resolve the set of available binaries to create
// custom commands available to the task runner
Expand Down
Loading
Loading