perf: Reduce allocations in SCM hashing, glob preprocessing, and cache lookups

anthonyshew · anthonyshew · commit 9c1fedde76ea · 2026-02-19T12:10:42.000-07:00
Eliminate unnecessary heap allocations and reduce syscalls across the
hot paths of turbo run:

- ls_tree/status: Replace double allocation (to_vec + String::from_utf8)
  with single allocation (str::from_utf8 + to_owned). Increase BufReader
  buffer from 8KB to 64KB to reduce read syscalls.
- hash_object: Pre-allocate HashMap capacity from known input size.
- package_deps: Avoid cloning every input glob to String — iterate &amp;str
  references and reuse a single buffer for path joining. Skip re-hashing
  files already known from the git index in the inputs+index path.
- task-hash: Collect &amp;str refs under mutex for dependency hash dedup
  instead of cloning each String. Add HashSet capacity hint.
- globwalk: Add Vec capacity hints for include/exclude paths. Use
  Cow::into_owned instead of .to_string() to avoid allocation when
  collapse_path returns a borrowed reference.
- cache: Reuse a single String buffer across the 3 path suffixes in
  FSCache::exists instead of 3 format!() + join_component() calls.
diff --git a/crates/turborepo-cache/src/fs.rs b/crates/turborepo-cache/src/fs.rs
@@ -125,19 +125,29 @@ impl FSCache {
 
     #[tracing::instrument(skip_all)]
     pub(crate) fn exists(&self, hash: &str) -> Result<Option<CacheHitMetadata>, CacheError> {
-        let uncompressed_cache_path = self.cache_directory.join_component(&format!("{hash}.tar"));
-        let compressed_cache_path = self
-            .cache_directory
-            .join_component(&format!("{hash}.tar.zst"));
+        let cache_dir = self.cache_directory.as_str();
+        let mut buf = String::with_capacity(cache_dir.len() + 1 + hash.len() + "-meta.json".len());
+        buf.push_str(cache_dir);
+        buf.push(std::path::MAIN_SEPARATOR);
+        buf.push_str(hash);
+        let prefix_len = buf.len();
+
+        buf.push_str(".tar");
+        let uncompressed_exists = std::path::Path::new(&buf).exists();
 
-        if !uncompressed_cache_path.exists() && !compressed_cache_path.exists() {
+        buf.push_str(".zst");
+        let compressed_exists = std::path::Path::new(&buf).exists();
+
+        if !uncompressed_exists && !compressed_exists {
             return Ok(None);
         }
 
+        buf.truncate(prefix_len);
+        buf.push_str("-meta.json");
+
         let duration = CacheMetadata::read(
-            &self
-                .cache_directory
-                .join_component(&format!("{hash}-meta.json")),
+            &AbsoluteSystemPathBuf::try_from(buf.as_str())
+                .map_err(|_| CacheError::ConfigCacheInvalidBase)?,
         )
         .map(|meta| meta.duration)
         .unwrap_or(0);
diff --git a/crates/turborepo-globwalk/src/lib.rs b/crates/turborepo-globwalk/src/lib.rs
@@ -80,25 +80,22 @@ fn preprocess_paths_and_globs<S: AsRef<str>>(
         .ok_or(WalkError::InvalidPath)?;
     let base_path_slash = escape_glob_literals(&raw_slash);
 
-    let (include_paths, lowest_segment) = include
-        .iter()
-        .map(|s| fix_glob_pattern(s.as_ref()).into_owned())
-        .map(|mut s| {
+    let (include_paths, lowest_segment) = {
+        let mut paths = Vec::with_capacity(include.len());
+        let mut lowest = usize::MAX;
+        for s in include {
+            let mut fixed = fix_glob_pattern(s.as_ref()).into_owned();
             // We need to check inclusion globs before the join
             // as to_slash doesn't preserve Windows drive names.
-            add_doublestar_to_dir(base_path, &mut s);
-            s
-        })
-        .map(|s| join_unix_like_paths(&base_path_slash, &s))
-        .filter_map(|s| collapse_path(&s).map(|(s, v)| (s.to_string(), v)))
-        .fold(
-            (vec![], usize::MAX),
-            |(mut vec, lowest_segment), (path, lowest_segment_next)| {
-                let lowest_segment = std::cmp::min(lowest_segment, lowest_segment_next);
-                vec.push(path); // we stringify here due to lifetime issues
-                (vec, lowest_segment)
-            },
-        );
+            add_doublestar_to_dir(base_path, &mut fixed);
+            let joined = join_unix_like_paths(&base_path_slash, &fixed);
+            if let Some((collapsed, segment)) = collapse_path(&joined) {
+                lowest = std::cmp::min(lowest, segment);
+                paths.push(collapsed.into_owned());
+            }
+        }
+        (paths, lowest)
+    };
 
     let base_path = base_path
         .components()
@@ -108,16 +105,13 @@ fn preprocess_paths_and_globs<S: AsRef<str>>(
         )
         .collect::<PathBuf>();
 
-    let mut exclude_paths = vec![];
-    for split in exclude
-        .iter()
-        .map(|s| fix_glob_pattern(s.as_ref()))
-        .map(|s| join_unix_like_paths(&base_path_slash, s.as_ref()))
-        .filter_map(|g| collapse_path(&g).map(|(s, _)| s.to_string()))
-    {
-        // if the glob ends with a slash, then we need to add a double star,
-        // unless it already ends with a double star
-        add_trailing_double_star(&mut exclude_paths, &split);
+    let mut exclude_paths = Vec::with_capacity(exclude.len() * 2);
+    for s in exclude {
+        let fixed = fix_glob_pattern(s.as_ref());
+        let joined = join_unix_like_paths(&base_path_slash, fixed.as_ref());
+        if let Some((collapsed, _)) = collapse_path(&joined) {
+            add_trailing_double_star(&mut exclude_paths, &collapsed);
+        }
     }
 
     Ok((base_path, include_paths, exclude_paths))
diff --git a/crates/turborepo-scm/src/hash_object.rs b/crates/turborepo-scm/src/hash_object.rs
@@ -45,6 +45,7 @@ pub(crate) fn hash_objects(
 ) -> Result<(), Error> {
     let pkg_prefix = git_root.anchor(pkg_path).ok().map(|a| a.to_unix());
 
+    hashes.reserve(to_hash.len());
     let results: Vec<Result<Option<(RelativeUnixPathBuf, String)>, Error>> = to_hash
         .into_par_iter()
         .map(|filename| {
diff --git a/crates/turborepo-scm/src/ls_tree.rs b/crates/turborepo-scm/src/ls_tree.rs
@@ -71,26 +71,32 @@ impl GitRepo {
 }
 
 fn read_ls_tree<R: Read>(reader: R, hashes: &mut GitHashes) -> Result<(), Error> {
-    let mut reader = BufReader::new(reader);
+    let mut reader = BufReader::with_capacity(64 * 1024, reader);
     let mut buffer = Vec::new();
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_ls_tree(&buffer)?;
-        let hash = String::from_utf8(entry.hash.to_vec())?;
-        let path = RelativeUnixPathBuf::new(String::from_utf8(entry.filename.to_vec())?)?;
-        hashes.insert(path, hash);
+        let hash = std::str::from_utf8(entry.hash)
+            .map_err(|e| Error::git_error(format!("invalid utf8 in ls-tree hash: {e}")))?;
+        let filename = std::str::from_utf8(entry.filename)
+            .map_err(|e| Error::git_error(format!("invalid utf8 in ls-tree filename: {e}")))?;
+        let path = RelativeUnixPathBuf::new(filename)?;
+        hashes.insert(path, hash.to_owned());
         buffer.clear();
     }
     Ok(())
 }
 
 fn read_ls_tree_sorted<R: Read>(reader: R, hashes: &mut SortedGitHashes) -> Result<(), Error> {
-    let mut reader = BufReader::new(reader);
+    let mut reader = BufReader::with_capacity(64 * 1024, reader);
     let mut buffer = Vec::new();
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_ls_tree(&buffer)?;
-        let hash = String::from_utf8(entry.hash.to_vec())?;
-        let path = RelativeUnixPathBuf::new(String::from_utf8(entry.filename.to_vec())?)?;
-        hashes.insert(path, hash);
+        let hash = std::str::from_utf8(entry.hash)
+            .map_err(|e| Error::git_error(format!("invalid utf8 in ls-tree hash: {e}")))?;
+        let filename = std::str::from_utf8(entry.filename)
+            .map_err(|e| Error::git_error(format!("invalid utf8 in ls-tree filename: {e}")))?;
+        let path = RelativeUnixPathBuf::new(filename)?;
+        hashes.insert(path, hash.to_owned());
         buffer.clear();
     }
     Ok(())
diff --git a/crates/turborepo-scm/src/package_deps.rs b/crates/turborepo-scm/src/package_deps.rs
@@ -200,45 +200,33 @@ impl GitRepo {
         let package_unix_path_buf = package_path.to_unix();
         let package_unix_path = package_unix_path_buf.as_str();
 
-        let mut inputs = inputs
+        static CONFIG_FILES: &[&str] = &["package.json", "turbo.json", "turbo.jsonc"];
+        let extra_inputs = if include_configs { CONFIG_FILES } else { &[] };
+        let total_inputs = inputs.len() + extra_inputs.len();
+
+        // Build glob lists directly from &str references — no need to clone
+        // every input into an owned String. We only allocate for the joined
+        // "package_path/glob" strings that globwalk requires.
+        let mut inclusions = Vec::with_capacity(total_inputs);
+        let mut exclusions = Vec::with_capacity(total_inputs);
+        let mut glob_buf = String::with_capacity(package_unix_path.len() + 1 + 64);
+
+        let all_inputs = inputs
             .iter()
-            .map(|s| s.as_ref().to_string())
-            .collect::<Vec<String>>();
-
-        if include_configs {
-            // Add in package.json and turbo.json to input patterns. Both file paths are
-            // relative to pkgPath
-            //
-            // - package.json is an input because if the `scripts` in the package.json
-            //   change (i.e. the tasks that turbo executes), we want a cache miss, since
-            //   any existing cache could be invalid.
-            // - turbo.json because it's the definition of the tasks themselves. The root
-            //   turbo.json is similarly included in the global hash. This file may not
-            //   exist in the workspace, but that is ok, because it will get ignored
-            //   downstream.
-            inputs.push("package.json".to_string());
-            inputs.push("turbo.json".to_string());
-            inputs.push("turbo.jsonc".to_string());
-        }
-
-        // The input patterns are relative to the package.
-        // However, we need to change the globbing to be relative to the repo root.
-        // Prepend the package path to each of the input patterns.
-        //
-        // FIXME: we don't yet error on absolute unix paths being passed in as inputs,
-        // and instead tack them on as if they were relative paths. This should be an
-        // error further upstream, but since we haven't pulled the switch yet,
-        // we need to mimic the Go behavior here and trim leading `/`
-        // characters.
-        let mut inclusions = vec![];
-        let mut exclusions = vec![];
-        for raw_glob in inputs {
+            .map(|s| s.as_ref())
+            .chain(extra_inputs.iter().copied());
+        for raw_glob in all_inputs {
+            glob_buf.clear();
             if let Some(exclusion) = raw_glob.strip_prefix('!') {
-                let glob_str = [package_unix_path, exclusion.trim_start_matches('/')].join("/");
-                exclusions.push(ValidatedGlob::from_str(&glob_str)?);
+                glob_buf.push_str(package_unix_path);
+                glob_buf.push('/');
+                glob_buf.push_str(exclusion.trim_start_matches('/'));
+                exclusions.push(ValidatedGlob::from_str(&glob_buf)?);
             } else {
-                let glob_str = [package_unix_path, raw_glob.trim_start_matches('/')].join("/");
-                inclusions.push(ValidatedGlob::from_str(&glob_str)?);
+                glob_buf.push_str(package_unix_path);
+                glob_buf.push('/');
+                glob_buf.push_str(raw_glob.trim_start_matches('/'));
+                inclusions.push(ValidatedGlob::from_str(&glob_buf)?);
             }
         }
         let files = globwalk::globwalk(
@@ -247,14 +235,11 @@ impl GitRepo {
             &exclusions,
             globwalk::WalkType::Files,
         )?;
-        let to_hash = files
-            .iter()
-            .map(|entry| {
-                let path = self.root.anchor(entry)?.to_unix();
-                Ok(path)
-            })
-            .collect::<Result<Vec<_>, Error>>()?;
-        let mut hashes = GitHashes::new();
+        let mut to_hash = Vec::with_capacity(files.len());
+        for entry in &files {
+            to_hash.push(self.root.anchor(entry)?.to_unix());
+        }
+        let mut hashes = GitHashes::with_capacity(files.len());
         hash_objects(&self.root, &full_pkg_path, to_hash, &mut hashes)?;
         Ok(hashes)
     }
@@ -283,16 +268,66 @@ impl GitRepo {
         }
 
         // Include globs can find files not in the git index (e.g. gitignored files
-        // that a user explicitly wants to track). We still need globwalk for these
-        // but can skip re-hashing files already known from the index.
+        // that a user explicitly wants to track). Walk the filesystem for these
+        // files but skip re-hashing any already known from the index.
         if !includes.is_empty() {
-            let include_hashes = self.get_package_file_hashes_from_inputs(
+            let full_pkg_path = turbo_root.resolve(package_path);
+            let package_unix_path_buf = package_path.to_unix();
+            let package_unix_path = package_unix_path_buf.as_str();
+
+            static CONFIG_FILES: &[&str] = &["package.json", "turbo.json", "turbo.jsonc"];
+            let mut inclusions = Vec::with_capacity(includes.len() + CONFIG_FILES.len());
+            let mut exclusions = Vec::new();
+            let mut glob_buf = String::with_capacity(package_unix_path.len() + 1 + 64);
+
+            let all = includes.iter().copied().chain(CONFIG_FILES.iter().copied());
+            for raw_glob in all {
+                glob_buf.clear();
+                if let Some(exclusion) = raw_glob.strip_prefix('!') {
+                    glob_buf.push_str(package_unix_path);
+                    glob_buf.push('/');
+                    glob_buf.push_str(exclusion.trim_start_matches('/'));
+                    exclusions.push(ValidatedGlob::from_str(&glob_buf)?);
+                } else {
+                    glob_buf.push_str(package_unix_path);
+                    glob_buf.push('/');
+                    glob_buf.push_str(raw_glob.trim_start_matches('/'));
+                    inclusions.push(ValidatedGlob::from_str(&glob_buf)?);
+                }
+            }
+
+            let files = globwalk::globwalk(
                 turbo_root,
-                package_path,
-                &includes,
-                true,
+                &inclusions,
+                &exclusions,
+                globwalk::WalkType::Files,
             )?;
-            hashes.extend(include_hashes);
+
+            // Only hash files not already present from the git index
+            let mut to_hash = Vec::new();
+            for entry in &files {
+                let git_relative = self.root.anchor(entry)?.to_unix();
+                let pkg_relative = turbopath::RelativeUnixPath::strip_prefix(
+                    &git_relative,
+                    &package_unix_path_buf,
+                )
+                .ok()
+                .map(|s| s.to_owned());
+
+                let already_known = pkg_relative
+                    .as_ref()
+                    .is_some_and(|rel| hashes.contains_key(rel));
+
+                if !already_known {
+                    to_hash.push(git_relative);
+                }
+            }
+
+            if !to_hash.is_empty() {
+                let mut new_hashes = GitHashes::with_capacity(to_hash.len());
+                hash_objects(&self.root, &full_pkg_path, to_hash, &mut new_hashes)?;
+                hashes.extend(new_hashes);
+            }
         }
 
         // Apply excludes via in-memory matching — no filesystem walk needed since
diff --git a/crates/turborepo-scm/src/status.rs b/crates/turborepo-scm/src/status.rs
@@ -83,11 +83,13 @@ fn read_status<R: Read>(
     hashes: &mut GitHashes,
 ) -> Result<Vec<RelativeUnixPathBuf>, Error> {
     let mut to_hash = Vec::new();
-    let mut reader = BufReader::new(reader);
+    let mut reader = BufReader::with_capacity(64 * 1024, reader);
     let mut buffer = Vec::new();
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_status(&buffer)?;
-        let path = RelativeUnixPathBuf::new(String::from_utf8(entry.filename.to_owned())?)?;
+        let filename = std::str::from_utf8(entry.filename)
+            .map_err(|e| Error::git_error(format!("invalid utf8 in git status: {e}")))?;
+        let path = RelativeUnixPathBuf::new(filename)?;
         if entry.is_delete {
             let path = path.strip_prefix(pkg_prefix).map_err(|_| {
                 Error::git_error(format!(
@@ -106,11 +108,13 @@ fn read_status<R: Read>(
 
 fn read_status_raw<R: Read>(reader: R) -> Result<Vec<RepoStatusEntry>, Error> {
     let mut entries = Vec::new();
-    let mut reader = BufReader::new(reader);
+    let mut reader = BufReader::with_capacity(64 * 1024, reader);
     let mut buffer = Vec::new();
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_status(&buffer)?;
-        let path = RelativeUnixPathBuf::new(String::from_utf8(entry.filename.to_owned())?)?;
+        let filename = std::str::from_utf8(entry.filename)
+            .map_err(|e| Error::git_error(format!("invalid utf8 in git status: {e}")))?;
+        let path = RelativeUnixPathBuf::new(filename)?;
         entries.push(RepoStatusEntry {
             path,
             is_delete: entry.is_delete,
diff --git a/crates/turborepo-task-hash/src/lib.rs b/crates/turborepo-task-hash/src/lib.rs
@@ -419,7 +419,7 @@ impl<'a, R: RunOptsHashInfo> TaskHasher<'a, R> {
             .lock()
             .expect("hash tracker mutex poisoned");
 
-        let mut dependency_hash_set = HashSet::new();
+        let mut dependency_hash_set = HashSet::with_capacity(dependency_set.len());
         for dependency_task in dependency_set {
             let TaskNode::Task(dependency_task_id) = dependency_task else {
                 continue;
@@ -429,11 +429,14 @@ impl<'a, R: RunOptsHashInfo> TaskHasher<'a, R> {
                 .package_task_hashes
                 .get(dependency_task_id)
                 .ok_or_else(|| Error::MissingDependencyTaskHash(dependency_task.to_string()))?;
-            dependency_hash_set.insert(dependency_hash.clone());
+            dependency_hash_set.insert(dependency_hash.as_str());
         }
-        drop(state);
 
-        let mut dependency_hash_list = dependency_hash_set.into_iter().collect::<Vec<_>>();
+        let mut dependency_hash_list: Vec<String> = dependency_hash_set
+            .into_iter()
+            .map(|s| s.to_owned())
+            .collect();
+        drop(state);
         dependency_hash_list.sort_unstable();
 
         Ok(dependency_hash_list)
