perf: Optimize hotpath memory management and reduce allocations

- Replace hex::encode(to_be_bytes()) with direct format!("{:016x}") in
  TurboHash::hash(), eliminating intermediate byte array and the hex
  crate dependency entirely
- Pre-size capnp first segment for FileHashes, LockFilePackages, and
  LockFilePackagesRef based on entry count, avoiding repeated segment
  growth allocations during message construction
- Use as_str() instead of to_string() for package_dir in TaskHashable
  serialization, avoiding a per-task-hash String allocation
- Replace HashSet with sort_unstable+dedup in calculate_dependency_hashes,
  removing HashSet hashing overhead and reducing allocations when
  dependency fan-out is large
- Pre-allocate HashMap capacity in RepoGitIndex::get_package_hashes
  using exact count (full-repo) or BTreeMap range size_hint (per-package)
- Use String::new() instead of "".into() for zero-allocation empty
  string returns in get_external_deps_hash and get_internal_deps_hash

https://claude.ai/code/session_01TXs2FAMUjxXr25hLQwTMzF

Author: claude

Cargo.lock

@@ -10,7 +10,6 @@ workspace = true
 
 [dependencies]
 capnp = "0.24"
-hex = "0.4.3"
 turbopath = { workspace = true }
 turborepo-lockfiles = { workspace = true }
 turborepo-types = { workspace = true }

crates/turborepo-hash/Cargo.toml

@@ -153,10 +153,13 @@ impl From<HashableTaskOutputs> for Builder<HeapAllocator> {
 
 impl From<LockFilePackages> for Builder<HeapAllocator> {
     fn from(LockFilePackages(packages): LockFilePackages) -> Self {
+        // Pre-size: each package has key + version strings (~12 words each).
+        let estimated_words = packages.len() * 12 + 8;
+        let allocator = HeapAllocator::default().first_segment_words(estimated_words as u32);
         let mut message = ::capnp::message::TypedBuilder::<
             proto_capnp::lock_file_packages::Owned,
             HeapAllocator,
-        >::new_default();
+        >::new(Builder::new(allocator));
         let mut builder = message.init_root();
 
         {
@@ -189,10 +192,12 @@ impl From<LockFilePackages> for Builder<HeapAllocator> {
 
 impl<'a> From<LockFilePackagesRef<'a>> for Builder<HeapAllocator> {
     fn from(LockFilePackagesRef(packages): LockFilePackagesRef<'a>) -> Self {
+        let estimated_words = packages.len() * 12 + 8;
+        let allocator = HeapAllocator::default().first_segment_words(estimated_words as u32);
         let mut message = ::capnp::message::TypedBuilder::<
             proto_capnp::lock_file_packages::Owned,
             HeapAllocator,
-        >::new_default();
+        >::new(Builder::new(allocator));
         let mut builder = message.init_root();
 
         {
@@ -225,10 +230,16 @@ impl<'a> From<LockFilePackagesRef<'a>> for Builder<HeapAllocator> {
 
 impl From<FileHashes> for Builder<HeapAllocator> {
     fn from(FileHashes(file_hashes): FileHashes) -> Self {
+        // Pre-size the first segment to avoid repeated growth allocations.
+        // Each file-hash entry needs roughly 16 capnp words (struct pointer +
+        // two text fields with their data), plus fixed overhead for the message
+        // root and list header.
+        let estimated_words = file_hashes.len() * 16 + 8;
+        let allocator = HeapAllocator::default().first_segment_words(estimated_words as u32);
         let mut message = ::capnp::message::TypedBuilder::<
             proto_capnp::file_hashes::Owned,
             HeapAllocator,
-        >::new_default();
+        >::new(Builder::new(allocator));
         let mut builder = message.init_root();
 
         {
@@ -268,10 +279,12 @@ impl From<FileHashes> for Builder<HeapAllocator> {
 
 impl From<&FileHashes> for Builder<HeapAllocator> {
     fn from(FileHashes(file_hashes): &FileHashes) -> Self {
+        let estimated_words = file_hashes.len() * 16 + 8;
+        let allocator = HeapAllocator::default().first_segment_words(estimated_words as u32);
         let mut message = ::capnp::message::TypedBuilder::<
             proto_capnp::file_hashes::Owned,
             HeapAllocator,
-        >::new_default();
+        >::new(Builder::new(allocator));
         let mut builder = message.init_root();
 
         {
@@ -313,8 +326,8 @@ impl From<TaskHashable<'_>> for Builder<HeapAllocator> {
         let mut builder = message.init_root();
 
         builder.set_global_hash(task_hashable.global_hash);
-        if let Some(package_dir) = task_hashable.package_dir {
-            builder.set_package_dir(package_dir.to_string());
+        if let Some(ref package_dir) = task_hashable.package_dir {
+            builder.set_package_dir(package_dir.as_str());
         }
 
         builder.set_hash_of_files(task_hashable.hash_of_files);

crates/turborepo-hash/src/lib.rs

@@ -1,3 +1,5 @@
+use std::fmt::Write;
+
 use capnp::message::{Allocator, Builder};
 
 pub trait Sealed<A> {}
@@ -20,17 +22,17 @@ where
 {
     fn hash(self) -> String {
         let message = self.into();
+        let segments = message.get_segments_for_output();
 
-        debug_assert_eq!(
-            message.get_segments_for_output().len(),
-            1,
-            "message is not canonical"
-        );
-
-        let buf = message.get_segments_for_output()[0];
+        debug_assert_eq!(segments.len(), 1, "message is not canonical");
 
-        let out = xxhash_rust::xxh64::xxh64(buf, 0);
+        let out = xxhash_rust::xxh64::xxh64(segments[0], 0);
 
-        hex::encode(out.to_be_bytes())
+        // Format u64 directly as 16-char zero-padded lowercase hex.
+        // Avoids the intermediate to_be_bytes() + hex::encode() roundtrip
+        // which creates a temporary byte array and an extra Vec allocation.
+        let mut s = String::with_capacity(16);
+        write!(s, "{out:016x}").unwrap();
+        s
     }
 }

crates/turborepo-hash/src/traits.rs

@@ -59,20 +59,28 @@ impl RepoGitIndex {
         let prefix_str = pkg_prefix.as_str();
         let prefix_is_empty = prefix_str.is_empty();
 
-        let mut hashes = GitHashes::new();
-        if prefix_is_empty {
+        let mut hashes = if prefix_is_empty {
+            // Full-repo case: pre-allocate to the exact ls-tree count.
+            let mut h = GitHashes::with_capacity(self.ls_tree_hashes.len());
             for (path, hash) in &self.ls_tree_hashes {
-                hashes.insert(path.clone(), hash.clone());
+                h.insert(path.clone(), hash.clone());
             }
+            h
         } else {
             let range_start = RelativeUnixPathBuf::new(format!("{}/", prefix_str)).unwrap();
             let range_end = RelativeUnixPathBuf::new(format!("{}0", prefix_str)).unwrap();
-            for (path, hash) in self.ls_tree_hashes.range(range_start..range_end) {
+            let range = self.ls_tree_hashes.range(range_start..range_end);
+            // Use size_hint to pre-allocate; BTreeMap ranges provide a lower
+            // bound based on the tree structure.
+            let (lower, _) = range.size_hint();
+            let mut h = GitHashes::with_capacity(lower);
+            for (path, hash) in range {
                 if let Ok(stripped) = path.strip_prefix(pkg_prefix) {
-                    hashes.insert(stripped, hash.clone());
+                    h.insert(stripped, hash.clone());
                 }
             }
-        }
+            h
+        };
 
         let mut to_hash = Vec::new();
         for entry in &self.status_entries {

crates/turborepo-scm/src/repo_index.rs

@@ -430,7 +430,11 @@ impl<'a, R: RunOptsHashInfo> TaskHasher<'a, R> {
             .lock()
             .expect("hash tracker mutex poisoned");
 
-        let mut dependency_hash_set = HashSet::with_capacity(dependency_set.len());
+        // Collect borrowed hash strings, then sort+dedup instead of using an
+        // intermediate HashSet.  This avoids the HashSet's per-element hashing
+        // overhead and its allocation, which matters when the dependency fan-out
+        // is large.
+        let mut dependency_hashes: Vec<&str> = Vec::with_capacity(dependency_set.len());
         for dependency_task in dependency_set {
             let TaskNode::Task(dependency_task_id) = dependency_task else {
                 continue;
@@ -440,17 +444,18 @@ impl<'a, R: RunOptsHashInfo> TaskHasher<'a, R> {
                 .package_task_hashes
                 .get(dependency_task_id)
                 .ok_or_else(|| Error::MissingDependencyTaskHash(dependency_task.to_string()))?;
-            dependency_hash_set.insert(dependency_hash.as_str());
+            dependency_hashes.push(dependency_hash.as_str());
         }
 
-        let mut dependency_hash_list: Vec<String> = dependency_hash_set
-            .into_iter()
-            .map(|s| s.to_owned())
-            .collect();
+        dependency_hashes.sort_unstable();
+        dependency_hashes.dedup();
+
+        // Convert to owned Strings only after deduplication to minimize
+        // allocations.
+        let result: Vec<String> = dependency_hashes.into_iter().map(str::to_owned).collect();
         drop(state);
-        dependency_hash_list.sort_unstable();
 
-        Ok(dependency_hash_list)
+        Ok(result)
     }
 
     pub fn into_task_hash_tracker_state(self) -> TaskHashTrackerState {
@@ -497,7 +502,7 @@ pub fn get_external_deps_hash(
     transitive_dependencies: &Option<HashSet<turborepo_lockfiles::Package>>,
 ) -> String {
     let Some(transitive_dependencies) = transitive_dependencies else {
-        return "".into();
+        return String::new();
     };
 
     // Collect references instead of cloning each Package (which has two Strings).
@@ -519,7 +524,7 @@ pub fn get_internal_deps_hash(
     pre_built_index: Option<&RepoGitIndex>,
 ) -> Result<String, Error> {
     if package_dirs.is_empty() {
-        return Ok("".into());
+        return Ok(String::new());
     }
 
     let owned_index;