perf: Optimize hotpath memory management and reduce heap allocations

- Stream file hashing in manual.rs: use fixed 8KiB stack buffer instead
  of reading entire files into heap-allocated Vecs, reducing per-file
  memory from O(file_size) to O(1)
- Stack-allocated hex encoding: replace hex::encode() with inline
  hex encoding into stack buffers in both turborepo-hash (xxh64 → 16B)
  and turborepo-scm (SHA1 → 40B), eliminating intermediate Vec allocs
- Pre-sized Oid formatting in hash_object.rs: use String::with_capacity(40)
  + write!() instead of to_string() to avoid reallocation for every file
  hashed in parallel via rayon
- Pre-allocate parse buffers in ls_tree.rs and status.rs: initialize
  read buffers with 256B capacity instead of zero to avoid repeated
  growth during git output parsing
- Optimize repo_index.rs: pre-size HashMap for root package, reuse a
  single String buffer for BTreeMap range boundary construction instead
  of two separate format!() allocations per package lookup
- Remove hex crate dependency from turborepo-hash and turborepo-scm

https://claude.ai/code/session_0117j5aMJ8CCBs1uimoSxDaC

Author: claude

Cargo.lock

@@ -10,7 +10,6 @@ workspace = true
 
 [dependencies]
 capnp = "0.24"
-hex = "0.4.3"
 turbopath = { workspace = true }
 turborepo-lockfiles = { workspace = true }
 turborepo-types = { workspace = true }

crates/turborepo-hash/Cargo.toml

@@ -13,6 +13,20 @@ where
 {
 }
 
+/// Hex-encode a u64 into a fixed 16-byte stack buffer, returning a `&str`.
+/// Avoids the heap allocation that `hex::encode()` would perform.
+#[inline]
+fn hex_encode_u64(value: u64, buf: &mut [u8; 16]) -> &str {
+    const HEX_CHARS: &[u8; 16] = b"0123456789abcdef";
+    let bytes = value.to_be_bytes();
+    for (i, &b) in bytes.iter().enumerate() {
+        buf[i * 2] = HEX_CHARS[(b >> 4) as usize];
+        buf[i * 2 + 1] = HEX_CHARS[(b & 0x0f) as usize];
+    }
+    // SAFETY: buf is filled with ASCII hex characters only.
+    unsafe { std::str::from_utf8_unchecked(buf) }
+}
+
 impl<T, A> TurboHash<A> for T
 where
     T: Into<Builder<A>>,
@@ -31,6 +45,9 @@ where
 
         let out = xxhash_rust::xxh64::xxh64(buf, 0);
 
-        hex::encode(out.to_be_bytes())
+        // Encode into a stack buffer and create the String from that, avoiding
+        // the intermediate Vec allocation that hex::encode performs.
+        let mut hex_buf = [0u8; 16];
+        hex_encode_u64(out, &mut hex_buf).to_owned()
     }
 }

crates/turborepo-hash/src/traits.rs

@@ -13,7 +13,6 @@ workspace = true
 bstr = "1.4.0"
 git2 = { workspace = true, default-features = false, optional = true }
 globwalk = { path = "../turborepo-globwalk" }
-hex = { workspace = true }
 ignore = "0.4.20"
 
 nom = "7.1.3"

crates/turborepo-scm/Cargo.toml

@@ -1,4 +1,6 @@
 #![cfg(feature = "git2")]
+use std::fmt::Write;
+
 use rayon::prelude::*;
 use tracing::debug;
 use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf, RelativeUnixPath, RelativeUnixPathBuf};
@@ -63,7 +65,11 @@ pub(crate) fn hash_objects(
                             AnchoredSystemPathBuf::relative_path_between(pkg_path, &full_file_path)
                                 .to_unix()
                         });
-                    Ok(Some((package_relative_path, hash.to_string())))
+                    // Format the OID hex directly into a pre-sized String to
+                    // avoid the intermediate allocations of Display + to_string().
+                    let mut hex = String::with_capacity(40);
+                    write!(hex, "{hash}").expect("writing to String cannot fail");
+                    Ok(Some((package_relative_path, hex)))
                 }
                 Err(e) => {
                     if e.class() == git2::ErrorClass::Os

crates/turborepo-scm/src/hash_object.rs

@@ -72,7 +72,9 @@ impl GitRepo {
 
 fn read_ls_tree<R: Read>(reader: R, hashes: &mut GitHashes) -> Result<(), Error> {
     let mut reader = BufReader::with_capacity(64 * 1024, reader);
-    let mut buffer = Vec::new();
+    // Typical ls-tree entries are ~80-120 bytes; pre-allocate to avoid
+    // repeated growth from zero.
+    let mut buffer = Vec::with_capacity(256);
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_ls_tree(&buffer)?;
         let hash = std::str::from_utf8(entry.hash)
@@ -88,7 +90,7 @@ fn read_ls_tree<R: Read>(reader: R, hashes: &mut GitHashes) -> Result<(), Error>
 
 fn read_ls_tree_sorted<R: Read>(reader: R, hashes: &mut SortedGitHashes) -> Result<(), Error> {
     let mut reader = BufReader::with_capacity(64 * 1024, reader);
-    let mut buffer = Vec::new();
+    let mut buffer = Vec::with_capacity(256);
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_ls_tree(&buffer)?;
         let hash = std::str::from_utf8(entry.hash)

crates/turborepo-scm/src/ls_tree.rs

@@ -4,35 +4,61 @@
 use std::io::{ErrorKind, Read};
 
 use globwalk::fix_glob_pattern;
-use hex::ToHex;
 use ignore::WalkBuilder;
 use sha1::{Digest, Sha1};
 use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, IntoUnix};
 use wax::{Glob, Program, any};
 
 use crate::{Error, GitHashes};
 
+/// Hex-encode a 20-byte SHA1 digest into a stack-allocated buffer, returning
+/// a `&str`. Avoids the heap allocation that `encode_hex::<String>()` performs.
+#[inline]
+fn hex_encode_sha1(digest: &[u8; 20]) -> [u8; 40] {
+    const HEX: &[u8; 16] = b"0123456789abcdef";
+    let mut buf = [0u8; 40];
+    for (i, &b) in digest.iter().enumerate() {
+        buf[i * 2] = HEX[(b >> 4) as usize];
+        buf[i * 2 + 1] = HEX[(b & 0x0f) as usize];
+    }
+    buf
+}
+
 fn git_like_hash_file(path: &AbsoluteSystemPath) -> Result<String, Error> {
     let mut hasher = Sha1::new();
     let mut f = path.open()?;
-    // Pre-allocate the buffer based on file metadata to avoid repeated
-    // reallocations during read_to_end. The +1 accounts for read_to_end's
-    // probe read that confirms EOF.
-    let estimated_size = f.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
-    let mut buffer = Vec::with_capacity(estimated_size);
-    // Note that read_to_end reads the target if f is a symlink. Currently, this can
+    let metadata = f.metadata()?;
+    let file_size = metadata.len();
+
+    // Write the git blob header: "blob <size>\0"
+    // This must use the exact byte count that will follow.
+    hasher.update(b"blob ");
+    hasher.update(file_size.to_string().as_bytes());
+    hasher.update([b'\0']);
+
+    // Stream file contents through the hasher using a fixed stack buffer
+    // instead of reading the entire file into a heap-allocated Vec. This
+    // reduces per-file memory from O(file_size) to O(1) (8 KiB).
+    //
+    // Note that read reads the target if f is a symlink. Currently, this can
     // happen when we are hashing a specific set of files, which in turn only
     // happens for handling dotEnv files. It is likely that in the future we
     // will want to ensure that the target is better accounted for in the set of
     // inputs to the task. Manual hashing, as well as global deps and other
     // places that support globs all ignore symlinks.
-    let size = f.read_to_end(&mut buffer)?;
-    hasher.update("blob ".as_bytes());
-    hasher.update(size.to_string().as_bytes());
-    hasher.update([b'\0']);
-    hasher.update(buffer.as_slice());
-    let result = hasher.finalize();
-    Ok(result.encode_hex::<String>())
+    let mut buf = [0u8; 8192];
+    loop {
+        let n = f.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        hasher.update(&buf[..n]);
+    }
+
+    let digest: [u8; 20] = hasher.finalize().into();
+    let hex_buf = hex_encode_sha1(&digest);
+    // SAFETY: hex_buf contains only ASCII hex characters.
+    Ok(unsafe { String::from_utf8_unchecked(hex_buf.to_vec()) })
 }
 
 fn to_glob(input: &str) -> Result<Glob<'_>, Error> {

crates/turborepo-scm/src/manual.rs

@@ -59,14 +59,29 @@ impl RepoGitIndex {
         let prefix_str = pkg_prefix.as_str();
         let prefix_is_empty = prefix_str.is_empty();
 
-        let mut hashes = GitHashes::new();
+        let mut hashes = if prefix_is_empty {
+            GitHashes::with_capacity(self.ls_tree_hashes.len())
+        } else {
+            GitHashes::new()
+        };
         if prefix_is_empty {
             for (path, hash) in &self.ls_tree_hashes {
                 hashes.insert(path.clone(), hash.clone());
             }
         } else {
-            let range_start = RelativeUnixPathBuf::new(format!("{}/", prefix_str)).unwrap();
-            let range_end = RelativeUnixPathBuf::new(format!("{}0", prefix_str)).unwrap();
+            // Build range boundary strings in a reusable buffer to avoid two
+            // separate format!() heap allocations per call. The '/' and '0'
+            // characters are adjacent in ASCII, so `prefix/`..`prefix0`
+            // captures exactly the entries under this package prefix.
+            let mut key_buf = String::with_capacity(prefix_str.len() + 1);
+            key_buf.push_str(prefix_str);
+            key_buf.push('/');
+            let range_start = RelativeUnixPathBuf::new(&key_buf).unwrap();
+            // Replace trailing '/' with '0' (the next ASCII char after '/')
+            // to form the exclusive upper bound.
+            key_buf.pop();
+            key_buf.push('0');
+            let range_end = RelativeUnixPathBuf::new(&key_buf).unwrap();
             for (path, hash) in self.ls_tree_hashes.range(range_start..range_end) {
                 if let Ok(stripped) = path.strip_prefix(pkg_prefix) {
                     hashes.insert(stripped, hash.clone());

crates/turborepo-scm/src/repo_index.rs

@@ -84,7 +84,9 @@ fn read_status<R: Read>(
 ) -> Result<Vec<RelativeUnixPathBuf>, Error> {
     let mut to_hash = Vec::new();
     let mut reader = BufReader::with_capacity(64 * 1024, reader);
-    let mut buffer = Vec::new();
+    // Status entries are typically ~6 bytes overhead + filename; pre-allocate
+    // to avoid repeated growth from zero.
+    let mut buffer = Vec::with_capacity(256);
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_status(&buffer)?;
         let filename = std::str::from_utf8(entry.filename)
@@ -109,7 +111,7 @@ fn read_status<R: Read>(
 fn read_status_raw<R: Read>(reader: R) -> Result<Vec<RepoStatusEntry>, Error> {
     let mut entries = Vec::new();
     let mut reader = BufReader::with_capacity(64 * 1024, reader);
-    let mut buffer = Vec::new();
+    let mut buffer = Vec::with_capacity(256);
     while reader.read_until(b'\0', &mut buffer)? != 0 {
         let entry = parse_status(&buffer)?;
         let filename = std::str::from_utf8(entry.filename)