Skip to content

Commit 9399811

Browse files
authored
perf: Parallelize lockfile parsing with workspace discovery (#11927)
## Summary - Start reading and parsing the lockfile on a background thread concurrently with workspace discovery and package.json parsing - The lockfile read only needs the package manager identity (cheap string parse from root package.json) and the root package.json — both available before the package graph pipeline begins - If the background read completes before `resolve_lockfile`, the result is injected into the state and the synchronous path becomes a no-op - Falls back gracefully: if package manager detection fails or the background read errors, the existing synchronous path runs as before The package graph build pipeline was: ``` parse_package_jsons → read_lockfile → connect_internal_deps → ... ``` Now: ``` parse_package_jsons ──┐ ├→ connect_internal_deps → ... read_lockfile ────────┘ ``` This saves `min(lockfile_parse, workspace_discovery)` off the critical path. ## Benchmarks `hyperfine` comparing `faster-and-faster` (Benchmark 1) vs `main` (Benchmark 2), 10 runs each with 5 warmup runs, `--skip-infer --dry` to isolate turbo overhead. **Large monorepo (~1000 packages):** | Run | faster-and-faster | main | Speedup | |-----|-------------------|------|---------| | 1 | 2.053s ± 0.117s | 2.157s ± 0.177s | **1.05x ± 0.10** | | 2 | 1.959s ± 0.083s | 2.170s ± 0.191s | **1.11x ± 0.11** | **Medium monorepo (~50 packages):** | Run | faster-and-faster | main | Speedup | |-----|-------------------|------|---------| | 1 | 1.470s ± 0.206s | 1.388s ± 0.064s | 1.00x ± 0.16 (noise) | | 2 | 1.428s ± 0.071s | 1.436s ± 0.129s | 1.01x ± 0.10 | **Small monorepo (~10 packages):** | Run | faster-and-faster | main | Speedup | |-----|-------------------|------|---------| | 1 | 1.029s ± 0.021s | 1.048s ± 0.022s | **1.02x ± 0.03** | | 2 | 1.040s ± 0.039s | 948.8ms ± 33.7ms | 1.00x ± 0.07 (noise) |
1 parent 6ef1582 commit 9399811

File tree

1 file changed

+46
-1
lines changed
  • crates/turborepo-repository/src/package_graph

1 file changed

+46
-1
lines changed

crates/turborepo-repository/src/package_graph/builder.rs

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ pub struct PackageGraphBuilder<'a, T> {
2929
package_jsons: Option<HashMap<AbsoluteSystemPathBuf, PackageJson>>,
3030
lockfile: Option<Box<dyn Lockfile>>,
3131
package_discovery: T,
32+
package_manager: Option<PackageManager>,
3233
}
3334

3435
#[derive(Debug, Diagnostic, thiserror::Error)]
@@ -88,6 +89,7 @@ impl<'a> PackageGraphBuilder<'a, LocalPackageDiscoveryBuilder> {
8889
is_single_package: false,
8990
package_jsons: None,
9091
lockfile: None,
92+
package_manager: None,
9193
}
9294
}
9395

@@ -98,6 +100,7 @@ impl<'a> PackageGraphBuilder<'a, LocalPackageDiscoveryBuilder> {
98100
}
99101

100102
pub fn with_package_manager(mut self, package_manager: PackageManager) -> Self {
103+
self.package_manager = Some(package_manager.clone());
101104
self.package_discovery
102105
.with_package_manager(Some(package_manager));
103106
self
@@ -137,6 +140,7 @@ impl<'a, P> PackageGraphBuilder<'a, P> {
137140
package_jsons: self.package_jsons,
138141
lockfile: self.lockfile,
139142
package_discovery: discovery,
143+
package_manager: self.package_manager,
140144
}
141145
}
142146
}
@@ -149,14 +153,49 @@ where
149153
{
150154
/// Build the `PackageGraph`.
151155
#[tracing::instrument(skip(self))]
152-
pub async fn build(self) -> Result<PackageGraph, Error> {
156+
pub async fn build(mut self) -> Result<PackageGraph, Error> {
153157
let is_single_package = self.is_single_package;
158+
159+
// If no pre-supplied lockfile, start reading it on a blocking thread
160+
// concurrently with package discovery + JSON parsing.
161+
let known_pm = self.package_manager.take().or_else(|| {
162+
PackageManager::get_package_manager(self.repo_root, &self.root_package_json).ok()
163+
});
164+
let lockfile_future = if !is_single_package && self.lockfile.is_none() {
165+
if let Some(pm) = known_pm {
166+
let repo_root = self.repo_root.to_owned();
167+
let root_package_json = self.root_package_json.clone();
168+
Some(tokio::task::spawn_blocking(
169+
move || -> Option<Box<dyn Lockfile>> {
170+
pm.read_lockfile(&repo_root, &root_package_json).ok()
171+
},
172+
))
173+
} else {
174+
None
175+
}
176+
} else {
177+
None
178+
};
179+
154180
let state = BuildState::new(self)?;
155181

156182
match is_single_package {
157183
true => Ok(state.build_single_package_graph().await?),
158184
false => {
159185
let state = state.parse_package_jsons().await?;
186+
187+
// If we started a lockfile read, collect the result before
188+
// entering resolve_lockfile so it becomes a cache hit.
189+
let state = if let Some(handle) = lockfile_future {
190+
if let Ok(Some(lockfile)) = handle.await {
191+
state.with_lockfile(lockfile)
192+
} else {
193+
state
194+
}
195+
} else {
196+
state
197+
};
198+
160199
let state = state.resolve_lockfile().await?;
161200
Ok(state.build_inner().await?)
162201
}
@@ -220,6 +259,7 @@ where
220259
package_jsons,
221260
lockfile,
222261
package_discovery,
262+
package_manager: _,
223263
} = builder;
224264
let mut workspaces = HashMap::new();
225265
workspaces.insert(
@@ -388,6 +428,11 @@ impl<'a, T: PackageDiscovery> BuildState<'a, ResolvedPackageManager, T> {
388428
}
389429

390430
impl<'a, T: PackageDiscovery> BuildState<'a, ResolvedWorkspaces, T> {
431+
fn with_lockfile(mut self, lockfile: Box<dyn Lockfile>) -> Self {
432+
self.lockfile = Some(lockfile);
433+
self
434+
}
435+
391436
#[tracing::instrument(skip(self))]
392437
fn connect_internal_dependencies(
393438
&mut self,

0 commit comments

Comments
 (0)