perf: Deduplicate file hashing and parallelize globwalks (#11902)

## Summary

Optimizes `turbo run --dry` wall-clock time by up to 1.48x on large
monorepos by eliminating redundant file hashing work and removing a
serialization bottleneck in globwalk operations.

### Benchmarks

Tested across three repos of varying size:

| Repo | Packages | Before | After | Speedup |
|------|----------|--------|-------|---------|
| large | ~1000 | 5.903s | 3.999s | **1.48x** |
| medium | ~120 | 1.461s | 1.380s | 1.06x |
| small | ~6 | 0.659s | 0.693s | ~1.0x (noise) |

The improvement scales with repo size — specifically with how many tasks
share the same `(package, inputs)` combination.

### Changes

**File hash deduplication** — Multiple tasks in the same package with
identical `inputs` config (e.g. `build`, `lint`, `typecheck` all in one
package) previously each ran an independent globwalk + file hash
computation. Now tasks are grouped by `(package_path, globs,
include_default)` and each unique combination is computed once, with
results shared across tasks.

**Parallel globwalks via retry-on-EMFILE** — The previous `IoSemaphore`
(max=1) serialized all globwalk operations to prevent fd exhaustion,
making this the dominant bottleneck on large repos. This replaces the
semaphore with retry-with-exponential-backoff on `EMFILE` errors (the
same pattern Node's `graceful-fs` uses), allowing globwalks to run fully
parallel on rayon. If the OS returns "too many open files", the
operation sleeps briefly and retries — up to 10 times with exponential
backoff capped at 1s.

**Zero-copy lockfile dependency lookups** — `Lockfile::all_dependencies`
now returns `Cow<'_, HashMap<String, String>>` instead of cloning the
HashMap on every call. For pnpm (which pre-builds a dependency index),
this eliminates ~329k HashMap clones during transitive closure
resolution.

**Optimized transitive closure cache keys** — The `DashMap` resolve
cache now uses a single null-byte-separated `String` key built into a
reusable buffer, instead of allocating a `(String, String, String)`
tuple per lookup.

**HashMap importers for pnpm** — Converted pnpm's `importers` field from
`BTreeMap` to `HashMap` (with sorted serialization) for O(1) workspace
lookups during `resolve_package`.

Author: anthonyshew

crates/turborepo-engine/src/builder.rs

@@ -1167,7 +1167,8 @@ mod test {
         fn all_dependencies(
             &self,
             _key: &str,
-        ) -> Result<Option<HashMap<String, String>>, turborepo_lockfiles::Error> {
+        ) -> Result<Option<std::borrow::Cow<'_, HashMap<String, String>>>, turborepo_lockfiles::Error>
+        {
             unreachable!()
         }
 

crates/turborepo-globwalk/src/lib.rs

@@ -460,7 +460,7 @@ pub fn globwalk_with_settings(
     walk_type: WalkType,
     settings: Settings,
 ) -> Result<HashSet<AbsoluteSystemPathBuf>, WalkError> {
-    globwalk_internal(base_path, include, exclude, walk_type, settings)
+    retry_on_emfile(|| globwalk_internal(base_path, include, exclude, walk_type, settings))
 }
 
 pub fn globwalk(
@@ -469,7 +469,58 @@ pub fn globwalk(
     exclude: &[ValidatedGlob],
     walk_type: WalkType,
 ) -> Result<HashSet<AbsoluteSystemPathBuf>, WalkError> {
-    globwalk_internal(base_path, include, exclude, walk_type, Default::default())
+    retry_on_emfile(|| {
+        globwalk_internal(base_path, include, exclude, walk_type, Default::default())
+    })
+}
+
+fn is_too_many_open_files(err: &WalkError) -> bool {
+    // visit_file converts all walkdir/wax errors into WalkError::IO,
+    // so this is the only variant we need to check.
+    let WalkError::IO(e) = err else {
+        return false;
+    };
+
+    #[cfg(unix)]
+    {
+        e.raw_os_error() == Some(24)
+    } // EMFILE
+    #[cfg(windows)]
+    {
+        e.raw_os_error() == Some(4)
+    } // ERROR_TOO_MANY_OPEN_FILES
+    #[cfg(not(any(unix, windows)))]
+    {
+        false
+    }
+}
+
+fn retry_on_emfile<F>(mut f: F) -> Result<HashSet<AbsoluteSystemPathBuf>, WalkError>
+where
+    F: FnMut() -> Result<HashSet<AbsoluteSystemPathBuf>, WalkError>,
+{
+    const MAX_RETRIES: u32 = 10;
+    const BASE_DELAY_MS: u64 = 10;
+    const MAX_DELAY_MS: u64 = 1000;
+
+    for attempt in 0..MAX_RETRIES {
+        match f() {
+            Ok(result) => return Ok(result),
+            Err(err) if is_too_many_open_files(&err) => {
+                let delay = std::cmp::min(BASE_DELAY_MS * 2u64.pow(attempt), MAX_DELAY_MS);
+                debug!(
+                    attempt = attempt + 1,
+                    delay_ms = delay,
+                    "too many open files, retrying globwalk"
+                );
+                std::thread::sleep(std::time::Duration::from_millis(delay));
+            }
+            Err(err) => return Err(err),
+        }
+    }
+
+    // Final attempt — propagate whatever happens.
+    f()
 }
 
 #[tracing::instrument(skip(include, exclude))]

crates/turborepo-lib/src/run/mod.rs

@@ -19,7 +19,6 @@ use std::{
 use chrono::{DateTime, Local};
 use futures::StreamExt;
 use itertools::Itertools;
-use rayon::iter::ParallelBridge;
 use shared_child::SharedChild;
 use tokio::{pin, select, task::JoinHandle};
 use tracing::{debug, error, info, instrument, warn};
@@ -589,7 +588,7 @@ impl Run {
         let repo_index = self.repo_index.as_ref().as_ref();
         let package_inputs_hashes = PackageInputsHashes::calculate_file_hashes(
             &self.scm,
-            self.engine.tasks().par_bridge(),
+            self.engine.tasks(),
             workspaces,
             self.engine.task_definitions(),
             &self.repo_root,

crates/turborepo-lockfiles/examples/berry_resolutions.rs

@@ -6,13 +6,10 @@ fn main() {
     let data = LockfileData::from_bytes(lockfile_bytes.as_slice()).unwrap();
     let lockfile = BerryLockfile::new(data, Some(manifest)).unwrap();
     let key = "debug@npm:3.2.7";
+    let deps = lockfile.all_dependencies(key).unwrap().unwrap();
     println!(
         "Dependencies of {key}: {}",
-        lockfile
-            .all_dependencies(key)
-            .unwrap()
-            .unwrap()
-            .into_iter()
+        deps.iter()
             .map(|(k, v)| format!("{k}@{v}"))
             .collect::<Vec<_>>()
             .join(", ")

crates/turborepo-lockfiles/src/berry/mod.rs

@@ -498,7 +498,8 @@ impl Lockfile for BerryLockfile {
     fn all_dependencies(
         &self,
         key: &str,
-    ) -> Result<Option<std::collections::HashMap<String, String>>, crate::Error> {
+    ) -> Result<Option<std::borrow::Cow<'_, std::collections::HashMap<String, String>>>, crate::Error>
+    {
         let locator = Locator::try_from(key).map_err(Error::from)?;
 
         let Some(package) = self.locator_package.get(&locator) else {
@@ -518,8 +519,7 @@ impl Lockfile for BerryLockfile {
             }
             map.insert(dependency.ident.to_string(), dependency.range.to_string());
         }
-        // For each dependency we need to check if there's an override
-        Ok(Some(map))
+        Ok(Some(std::borrow::Cow::Owned(map)))
     }
 
     fn subgraph(

crates/turborepo-lockfiles/src/bun/mod.rs

@@ -588,7 +588,8 @@ impl Lockfile for BunLockfile {
     fn all_dependencies(
         &self,
         key: &str,
-    ) -> Result<Option<std::collections::HashMap<String, String>>, crate::Error> {
+    ) -> Result<Option<std::borrow::Cow<'_, std::collections::HashMap<String, String>>>, crate::Error>
+    {
         let entry_key = self
             .key_to_entry
             .get(key)
@@ -602,18 +603,14 @@ impl Lockfile for BunLockfile {
         let mut deps = HashMap::new();
 
         let Some(info) = &entry.info else {
-            return Ok(Some(deps));
+            return Ok(Some(std::borrow::Cow::Owned(deps)));
         };
 
         for (dependency, version) in info.all_dependencies() {
             let is_optional = info.optional_dependencies.contains_key(dependency)
                 || info.optional_peers.contains(dependency);
 
             if is_optional {
-                // Optional peers without nested entries should be skipped (prevents pulling
-                // unrelated packages like "next" into @vercel/analytics). But declared
-                // optionalDependencies (platform-specific binaries) should include hoisted
-                // versions when no nested version exists.
                 let parent_key = format!("{entry_key}/{dependency}");
                 let has_nested = self.data.packages.contains_key(&parent_key);
 
@@ -631,7 +628,7 @@ impl Lockfile for BunLockfile {
             deps.insert(dependency.to_string(), version.to_string());
         }
 
-        Ok(Some(deps))
+        Ok(Some(std::borrow::Cow::Owned(deps)))
     }
 
     fn subgraph(

crates/turborepo-lockfiles/src/lib.rs

@@ -21,6 +21,7 @@ mod yarn1;
 
 use std::{
     any::Any,
+    borrow::Cow,
     collections::{HashMap, HashSet},
 };
 
@@ -35,7 +36,7 @@ use serde::Serialize;
 use turbopath::RelativeUnixPathBuf;
 pub use yarn1::{Yarn1Lockfile, yarn_subgraph};
 
-type ResolveCache = DashMap<(String, String, String), Option<Package>>;
+type ResolveCache = DashMap<String, Option<Package>>;
 
 #[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash, Serialize)]
 pub struct Package {
@@ -71,7 +72,10 @@ pub trait Lockfile: Send + Sync + Any + std::fmt::Debug {
 
     /// Given a lockfile key return all (prod/dev/optional) direct dependencies
     /// of that package.
-    fn all_dependencies(&self, key: &str) -> Result<Option<HashMap<String, String>>, Error>;
+    fn all_dependencies(
+        &self,
+        key: &str,
+    ) -> Result<Option<Cow<'_, HashMap<String, String>>>, Error>;
 
     /// Given a list of workspace packages and external packages that are
     /// dependencies of the workspace packages, produce a lockfile that only
@@ -169,71 +173,102 @@ fn transitive_closure_cached<L: Lockfile + ?Sized>(
     resolve_cache: &ResolveCache,
 ) -> Result<HashSet<Package>, Error> {
     let mut transitive_deps = HashSet::new();
+    let mut key_buf = String::new();
     transitive_closure_helper(
         lockfile,
         workspace_path,
-        unresolved_deps,
+        &unresolved_deps,
         &mut transitive_deps,
         ignore_missing_packages,
         resolve_cache,
+        &mut key_buf,
     )?;
 
     Ok(transitive_deps)
 }
 
-fn transitive_closure_helper<L: Lockfile + ?Sized>(
+fn make_cache_key(buf: &mut String, workspace_path: &str, name: &str, specifier: &str) {
+    buf.clear();
+    buf.reserve(workspace_path.len() + name.len() + specifier.len() + 2);
+    buf.push_str(workspace_path);
+    buf.push('\0');
+    buf.push_str(name);
+    buf.push('\0');
+    buf.push_str(specifier);
+}
+
+fn resolve_deps<L: Lockfile + ?Sized>(
     lockfile: &L,
     workspace_path: &str,
-    unresolved_deps: HashMap<String, impl AsRef<str>>,
-    resolved_deps: &mut HashSet<Package>,
+    unresolved_deps: &HashMap<String, String>,
     ignore_missing_packages: bool,
     resolve_cache: &ResolveCache,
-) -> Result<(), Error> {
+    key_buf: &mut String,
+) -> Result<Vec<Package>, Error> {
+    let mut newly_resolved = Vec::new();
+
     for (name, specifier) in unresolved_deps {
-        let specifier_ref = specifier.as_ref();
-        let cache_key = (
-            workspace_path.to_string(),
-            name.clone(),
-            specifier_ref.to_string(),
-        );
-
-        let pkg = match resolve_cache.get(&cache_key) {
+        make_cache_key(key_buf, workspace_path, name, specifier);
+
+        let pkg = match resolve_cache.get(key_buf.as_str()) {
             Some(cached) => cached.clone(),
             None => {
-                let result = match lockfile.resolve_package(workspace_path, &name, specifier_ref) {
+                let result = match lockfile.resolve_package(workspace_path, name, specifier) {
                     Ok(pkg) => pkg,
                     Err(Error::MissingWorkspace(_)) if ignore_missing_packages => {
-                        resolve_cache.insert(cache_key, None);
+                        resolve_cache.insert(key_buf.clone(), None);
                         continue;
                     }
                     Err(e) => return Err(e),
                 };
-                resolve_cache.insert(cache_key, result.clone());
+                resolve_cache.insert(key_buf.clone(), result.clone());
                 result
             }
         };
 
-        match pkg {
-            None => {
-                continue;
-            }
-            Some(pkg) if resolved_deps.contains(&pkg) => {
-                continue;
-            }
-            Some(pkg) => {
-                let all_deps = lockfile.all_dependencies(&pkg.key)?;
-                resolved_deps.insert(pkg);
-                if let Some(deps) = all_deps {
-                    transitive_closure_helper(
-                        lockfile,
-                        workspace_path,
-                        deps,
-                        resolved_deps,
-                        false,
-                        resolve_cache,
-                    )?;
-                }
-            }
+        if let Some(pkg) = pkg {
+            newly_resolved.push(pkg);
+        }
+    }
+
+    Ok(newly_resolved)
+}
+
+fn transitive_closure_helper<L: Lockfile + ?Sized>(
+    lockfile: &L,
+    workspace_path: &str,
+    unresolved_deps: &HashMap<String, String>,
+    resolved_deps: &mut HashSet<Package>,
+    ignore_missing_packages: bool,
+    resolve_cache: &ResolveCache,
+    key_buf: &mut String,
+) -> Result<(), Error> {
+    let newly_resolved = resolve_deps(
+        lockfile,
+        workspace_path,
+        unresolved_deps,
+        ignore_missing_packages,
+        resolve_cache,
+        key_buf,
+    )?;
+
+    for pkg in newly_resolved {
+        if resolved_deps.contains(&pkg) {
+            continue;
+        }
+
+        let all_deps = lockfile.all_dependencies(&pkg.key)?;
+        resolved_deps.insert(pkg);
+        if let Some(deps) = all_deps {
+            transitive_closure_helper(
+                lockfile,
+                workspace_path,
+                &deps,
+                resolved_deps,
+                false,
+                resolve_cache,
+                key_buf,
+            )?;
         }
     }
 

crates/turborepo-lockfiles/src/npm.rs

@@ -81,7 +81,10 @@ impl Lockfile for NpmLockfile {
     }
 
     #[tracing::instrument(skip(self))]
-    fn all_dependencies(&self, key: &str) -> Result<Option<HashMap<String, String>>, Error> {
+    fn all_dependencies(
+        &self,
+        key: &str,
+    ) -> Result<Option<std::borrow::Cow<'_, HashMap<String, String>>>, Error> {
         self.packages
             .get(key)
             .map(|pkg| {
@@ -98,7 +101,8 @@ impl Lockfile for NpmLockfile {
                                 }
                             })
                     })
-                    .collect()
+                    .collect::<Result<HashMap<_, _>, _>>()
+                    .map(std::borrow::Cow::Owned)
             })
             .transpose()
     }