test: Add regression tests for hotpath performance optimizations

Author: anthonyshew

crates/turborepo-run-cache/Cargo.toml

@@ -19,7 +19,7 @@ turbopath = { workspace = true }
 turborepo-cache = { workspace = true }
 turborepo-hash = { workspace = true }
 turborepo-repository = { path = "../turborepo-repository" }
-turborepo-scm = { workspace = true }
+turborepo-scm = { workspace = true, features = ["git2"] }
 turborepo-task-id = { workspace = true }
 turborepo-telemetry = { path = "../turborepo-telemetry" }
 turborepo-types = { workspace = true }

crates/turborepo-scm/src/manual.rs

@@ -464,4 +464,104 @@ mod tests {
 
         assert_eq!(hashes, expected);
     }
+
+    #[test]
+    fn test_include_default_files_deduplicates_with_explicit_includes() {
+        // When include_default_files=true AND explicit includes are provided,
+        // the first walk collects files matching the includes, and the second
+        // walk collects gitignore-respecting defaults. Files appearing in both
+        // walks should not be hashed twice — verify that the result is correct
+        // and contains each file exactly once.
+        let (_tmp, turbo_root) = tmp_dir();
+        let pkg_path = AnchoredSystemPathBuf::from_raw("my-pkg").unwrap();
+        let pkg_dir = turbo_root.resolve(&pkg_path);
+        pkg_dir.create_dir_all().unwrap();
+
+        // Create files: one matched by the explicit include, one only in defaults
+        let shared_file = pkg_dir.join_component("shared.ts");
+        shared_file.create_with_contents("shared content").unwrap();
+
+        let default_only = pkg_dir.join_component("default-only.ts");
+        default_only
+            .create_with_contents("default only content")
+            .unwrap();
+
+        let package_json = pkg_dir.join_component("package.json");
+        package_json.create_with_contents("{}").unwrap();
+
+        let turbo_json = pkg_dir.join_component("turbo.json");
+        turbo_json.create_with_contents("{}").unwrap();
+
+        // "*.ts" matches both shared.ts and default-only.ts in the first walk
+        // (since git_ignore=false for explicit inputs). The second walk with
+        // git_ignore=true should not re-hash files already found.
+        let hashes = get_package_file_hashes_without_git(
+            &turbo_root,
+            &pkg_path,
+            &["*.ts"],
+            true,
+        )
+        .unwrap();
+
+        // All four files should appear exactly once
+        assert!(
+            hashes.contains_key(&RelativeUnixPathBuf::new("shared.ts").unwrap()),
+            "shared.ts should be present"
+        );
+        assert!(
+            hashes.contains_key(&RelativeUnixPathBuf::new("default-only.ts").unwrap()),
+            "default-only.ts should be present"
+        );
+        assert!(
+            hashes.contains_key(&RelativeUnixPathBuf::new("package.json").unwrap()),
+            "package.json should be present (added by include pattern augmentation)"
+        );
+        assert!(
+            hashes.contains_key(&RelativeUnixPathBuf::new("turbo.json").unwrap()),
+            "turbo.json should be present (added by include pattern augmentation)"
+        );
+
+        // Verify the hash values are deterministic (same content = same hash)
+        let hashes2 = get_package_file_hashes_without_git(
+            &turbo_root,
+            &pkg_path,
+            &["*.ts"],
+            true,
+        )
+        .unwrap();
+        assert_eq!(hashes, hashes2, "hashes should be deterministic");
+    }
+
+    #[test]
+    fn test_include_default_files_with_exclusion() {
+        // Verify that exclusions work correctly when include_default_files=true:
+        // excluded files should not appear even if the default walk finds them.
+        let (_tmp, turbo_root) = tmp_dir();
+        let pkg_path = AnchoredSystemPathBuf::from_raw("lib").unwrap();
+        let pkg_dir = turbo_root.resolve(&pkg_path);
+        pkg_dir.create_dir_all().unwrap();
+
+        let keep = pkg_dir.join_component("keep.ts");
+        keep.create_with_contents("keep").unwrap();
+
+        let excluded = pkg_dir.join_component("excluded.ts");
+        excluded.create_with_contents("excluded").unwrap();
+
+        let hashes = get_package_file_hashes_without_git(
+            &turbo_root,
+            &pkg_path,
+            &["*.ts", "!excluded.ts"],
+            true,
+        )
+        .unwrap();
+
+        assert!(
+            hashes.contains_key(&RelativeUnixPathBuf::new("keep.ts").unwrap()),
+            "keep.ts should be present"
+        );
+        assert!(
+            !hashes.contains_key(&RelativeUnixPathBuf::new("excluded.ts").unwrap()),
+            "excluded.ts should NOT be present"
+        );
+    }
 }

crates/turborepo-scm/src/repo_index.rs

@@ -3,7 +3,7 @@
 use tracing::{debug, trace};
 use turbopath::RelativeUnixPathBuf;
 
-use crate::{Error, GitHashes, GitRepo, ls_tree::SortedGitHashes, status::RepoStatusEntry};
+use crate::{ls_tree::SortedGitHashes, status::RepoStatusEntry, Error, GitHashes, GitRepo};
 
 /// Pre-computed repo-wide git index that caches the results of `git ls-tree`
 /// and `git status` so they can be filtered per-package without spawning
@@ -116,3 +116,133 @@ impl RepoGitIndex {
         Ok((hashes, to_hash))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeMap;
+
+    use turbopath::RelativeUnixPathBuf;
+
+    use super::*;
+
+    fn path(s: &str) -> RelativeUnixPathBuf {
+        RelativeUnixPathBuf::new(s).unwrap()
+    }
+
+    fn make_index(ls_tree: Vec<(&str, &str)>, status: Vec<(&str, bool)>) -> RepoGitIndex {
+        let ls_tree_hashes: SortedGitHashes = ls_tree
+            .into_iter()
+            .map(|(p, h)| (path(p), h.to_string()))
+            .collect::<BTreeMap<_, _>>();
+        let status_entries = status
+            .into_iter()
+            .map(|(p, is_delete)| RepoStatusEntry {
+                path: path(p),
+                is_delete,
+            })
+            .collect();
+        RepoGitIndex {
+            ls_tree_hashes,
+            status_entries,
+        }
+    }
+
+    #[test]
+    fn test_empty_prefix_returns_all_files() {
+        let index = make_index(
+            vec![
+                ("apps/web/src/index.ts", "aaa"),
+                ("packages/ui/button.tsx", "bbb"),
+                ("root-file.json", "ccc"),
+            ],
+            vec![],
+        );
+        let (hashes, to_hash) = index.get_package_hashes(&path("")).unwrap();
+        assert_eq!(hashes.len(), 3);
+        assert!(hashes.contains_key(&path("apps/web/src/index.ts")));
+        assert!(hashes.contains_key(&path("packages/ui/button.tsx")));
+        assert!(hashes.contains_key(&path("root-file.json")));
+        assert!(to_hash.is_empty());
+    }
+
+    #[test]
+    fn test_prefix_filters_to_package_and_strips_prefix() {
+        let index = make_index(
+            vec![
+                ("apps/web/src/index.ts", "aaa"),
+                ("apps/web/package.json", "bbb"),
+                ("apps/docs/README.md", "ccc"),
+                ("packages/ui/button.tsx", "ddd"),
+            ],
+            vec![],
+        );
+        let (hashes, to_hash) = index.get_package_hashes(&path("apps/web")).unwrap();
+        assert_eq!(hashes.len(), 2);
+        assert_eq!(hashes.get(&path("src/index.ts")).unwrap(), "aaa");
+        assert_eq!(hashes.get(&path("package.json")).unwrap(), "bbb");
+        assert!(to_hash.is_empty());
+    }
+
+    #[test]
+    fn test_prefix_does_not_match_sibling_with_shared_prefix() {
+        // "apps/web-admin" should NOT match when filtering for "apps/web"
+        let index = make_index(
+            vec![
+                ("apps/web/index.ts", "aaa"),
+                ("apps/web-admin/index.ts", "bbb"),
+            ],
+            vec![],
+        );
+        let (hashes, _) = index.get_package_hashes(&path("apps/web")).unwrap();
+        assert_eq!(hashes.len(), 1);
+        assert!(hashes.contains_key(&path("index.ts")));
+    }
+
+    #[test]
+    fn test_status_modified_file_added_to_to_hash() {
+        let index = make_index(
+            vec![("my-pkg/file.ts", "aaa")],
+            vec![("my-pkg/new-file.ts", false)],
+        );
+        let (hashes, to_hash) = index.get_package_hashes(&path("my-pkg")).unwrap();
+        assert_eq!(hashes.len(), 1);
+        assert_eq!(to_hash, vec![path("my-pkg/new-file.ts")]);
+    }
+
+    #[test]
+    fn test_status_deleted_file_removed_from_hashes() {
+        let index = make_index(
+            vec![("my-pkg/keep.ts", "aaa"), ("my-pkg/deleted.ts", "bbb")],
+            vec![("my-pkg/deleted.ts", true)],
+        );
+        let (hashes, to_hash) = index.get_package_hashes(&path("my-pkg")).unwrap();
+        assert_eq!(hashes.len(), 1);
+        assert!(hashes.contains_key(&path("keep.ts")));
+        assert!(!hashes.contains_key(&path("deleted.ts")));
+        assert!(to_hash.is_empty());
+    }
+
+    #[test]
+    fn test_status_entries_for_other_packages_ignored() {
+        let index = make_index(
+            vec![("pkg-a/file.ts", "aaa")],
+            vec![("pkg-b/new.ts", false), ("pkg-b/gone.ts", true)],
+        );
+        let (hashes, to_hash) = index.get_package_hashes(&path("pkg-a")).unwrap();
+        assert_eq!(hashes.len(), 1);
+        assert!(to_hash.is_empty());
+    }
+
+    #[test]
+    fn test_empty_prefix_with_status() {
+        let index = make_index(
+            vec![("file.ts", "aaa")],
+            vec![("new.ts", false), ("file.ts", true)],
+        );
+        let (hashes, to_hash) = index.get_package_hashes(&path("")).unwrap();
+        // file.ts was deleted via status
+        assert!(hashes.is_empty());
+        // new.ts is untracked/modified
+        assert_eq!(to_hash, vec![path("new.ts")]);
+    }
+}

crates/turborepo-task-hash/src/lib.rs

@@ -23,9 +23,9 @@ use turborepo_cache::CacheHitMetadata;
 // Re-export turborepo_engine::TaskNode for convenience
 pub use turborepo_engine::TaskNode;
 use turborepo_env::{
-    BUILTIN_PASS_THROUGH_ENV, BySource, CompiledWildcards, DetailedMap, EnvironmentVariableMap,
+    BySource, CompiledWildcards, DetailedMap, EnvironmentVariableMap, BUILTIN_PASS_THROUGH_ENV,
 };
-use turborepo_frameworks::{Slug as FrameworkSlug, infer_framework};
+use turborepo_frameworks::{infer_framework, Slug as FrameworkSlug};
 use turborepo_hash::{FileHashes, LockFilePackagesRef, TaskHashable, TurboHash};
 use turborepo_repository::package_graph::{PackageInfo, PackageName};
 use turborepo_scm::{RepoGitIndex, SCM};
@@ -433,8 +433,7 @@ impl<'a, R: RunOptsHashInfo> TaskHasher<'a, R> {
         // Collect owned strings directly to avoid borrow lifetime issues with
         // the RwLock guard. We sort + dedup instead of using a HashSet to avoid
         // the overhead of hashing the hash strings.
-        let mut dependency_hash_list: Vec<String> =
-            Vec::with_capacity(dependency_set.len());
+        let mut dependency_hash_list: Vec<String> = Vec::with_capacity(dependency_set.len());
         for dependency_task in &dependency_set {
             let TaskNode::Task(dependency_task_id) = dependency_task else {
                 continue;
@@ -701,4 +700,100 @@ mod test {
         assert_send::<TaskHashTracker>();
         assert_sync::<TaskHashTracker>();
     }
+
+    #[test]
+    fn test_hash_tracker_concurrent_reads() {
+        let tracker = TaskHashTracker::new(HashMap::new());
+        let task_id: TaskId<'static> = TaskId::new("pkg", "build");
+        tracker.insert_hash(
+            task_id.clone(),
+            DetailedMap::default(),
+            "abc123".to_string(),
+            None,
+        );
+
+        // Multiple concurrent reads should not deadlock or panic with RwLock
+        std::thread::scope(|s| {
+            for _ in 0..8 {
+                let tracker = &tracker;
+                let task_id = &task_id;
+                s.spawn(move || {
+                    for _ in 0..100 {
+                        let h = tracker.hash(task_id);
+                        assert_eq!(h.as_deref(), Some("abc123"));
+                    }
+                });
+            }
+        });
+    }
+
+    #[test]
+    fn test_hash_tracker_concurrent_read_write() {
+        let tracker = TaskHashTracker::new(HashMap::new());
+
+        // Pre-create owned task IDs to avoid lifetime issues with TaskId borrows
+        let task_ids: Vec<TaskId<'static>> = (0..50)
+            .map(|i| TaskId::new("pkg", &format!("task-{i}")).into_owned())
+            .collect();
+
+        // One writer, many readers — verifies RwLock allows concurrent reads
+        // while writes are exclusive, without deadlock.
+        std::thread::scope(|s| {
+            let tracker = &tracker;
+            let task_ids = &task_ids;
+
+            s.spawn(move || {
+                for (i, task_id) in task_ids.iter().enumerate() {
+                    tracker.insert_hash(
+                        task_id.clone(),
+                        DetailedMap::default(),
+                        format!("hash-{i}"),
+                        None,
+                    );
+                }
+            });
+
+            for _ in 0..4 {
+                s.spawn(move || {
+                    for task_id in task_ids {
+                        // May or may not find the hash depending on timing — that's fine,
+                        // we're testing for absence of panics/deadlocks.
+                        let _ = tracker.hash(task_id);
+                        let _ = tracker.env_vars(task_id);
+                        let _ = tracker.cache_status(task_id);
+                    }
+                });
+            }
+        });
+    }
+
+    // Validates that sort+dedup produces the same result as the previous
+    // HashSet→Vec→sort approach for dependency hash deduplication.
+    #[test]
+    fn test_sort_dedup_matches_hashset_behavior() {
+        let inputs: Vec<Vec<&str>> = vec![
+            vec!["abc", "def", "abc", "ghi", "def"],
+            vec!["zzz", "aaa", "mmm"],
+            vec!["same", "same", "same"],
+            vec![],
+            vec!["only-one"],
+        ];
+
+        for input in inputs {
+            // New approach: sort + dedup
+            let mut sort_dedup: Vec<String> = input.iter().map(|s| s.to_string()).collect();
+            sort_dedup.sort_unstable();
+            sort_dedup.dedup();
+
+            // Old approach: HashSet → Vec → sort
+            let hash_set: HashSet<String> = input.iter().map(|s| s.to_string()).collect();
+            let mut hashset_sorted: Vec<String> = hash_set.into_iter().collect();
+            hashset_sorted.sort();
+
+            assert_eq!(
+                sort_dedup, hashset_sorted,
+                "sort+dedup and hashset+sort should produce identical results for: {input:?}"
+            );
+        }
+    }
 }