Skip to content

Commit dc51786

Browse files
anthonyshewanthonyshew
committed
perf: Parallelize lockfile parsing with workspace discovery
Start reading and parsing the lockfile on a blocking thread concurrently with workspace discovery and package.json parsing. The lockfile read only needs the package manager identity (cheap to resolve from root package.json) and the root package.json itself, both available before the package graph pipeline begins. The existing typestate pipeline was: parse_package_jsons → read_lockfile → connect_internal_deps → ... Now: parse_package_jsons ──┐ ├→ connect_internal_deps → ... read_lockfile ────────┘ This saves min(lockfile_parse, workspace_discovery) off the critical path.
1 parent 6ef1582 commit dc51786

File tree

4 files changed

+575
-1
lines changed

4 files changed

+575
-1
lines changed

‎crates/turborepo-repository/src/package_graph/builder.rs‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ pub struct PackageGraphBuilder<'a, T> {
2929
package_jsons: Option<HashMap<AbsoluteSystemPathBuf, PackageJson>>,
3030
lockfile: Option<Box<dyn Lockfile>>,
3131
package_discovery: T,
32+
package_manager: Option<PackageManager>,
3233
}
3334

3435
#[derive(Debug, Diagnostic, thiserror::Error)]
@@ -88,6 +89,7 @@ impl<'a> PackageGraphBuilder<'a, LocalPackageDiscoveryBuilder> {
8889
is_single_package: false,
8990
package_jsons: None,
9091
lockfile: None,
92+
package_manager: None,
9193
}
9294
}
9395

@@ -98,6 +100,7 @@ impl<'a> PackageGraphBuilder<'a, LocalPackageDiscoveryBuilder> {
98100
}
99101

100102
pub fn with_package_manager(mut self, package_manager: PackageManager) -> Self {
103+
self.package_manager = Some(package_manager.clone());
101104
self.package_discovery
102105
.with_package_manager(Some(package_manager));
103106
self
@@ -137,6 +140,7 @@ impl<'a, P> PackageGraphBuilder<'a, P> {
137140
package_jsons: self.package_jsons,
138141
lockfile: self.lockfile,
139142
package_discovery: discovery,
143+
package_manager: self.package_manager,
140144
}
141145
}
142146
}
@@ -149,14 +153,49 @@ where
149153
{
150154
/// Build the `PackageGraph`.
151155
#[tracing::instrument(skip(self))]
152-
pub async fn build(self) -> Result<PackageGraph, Error> {
156+
pub async fn build(mut self) -> Result<PackageGraph, Error> {
153157
let is_single_package = self.is_single_package;
158+
159+
// If no pre-supplied lockfile, start reading it on a blocking thread
160+
// concurrently with package discovery + JSON parsing.
161+
let known_pm = self.package_manager.take().or_else(|| {
162+
PackageManager::get_package_manager(self.repo_root, &self.root_package_json).ok()
163+
});
164+
let lockfile_future = if !is_single_package && self.lockfile.is_none() {
165+
if let Some(pm) = known_pm {
166+
let repo_root = self.repo_root.to_owned();
167+
let root_package_json = self.root_package_json.clone();
168+
Some(tokio::task::spawn_blocking(
169+
move || -> Option<Box<dyn Lockfile>> {
170+
pm.read_lockfile(&repo_root, &root_package_json).ok()
171+
},
172+
))
173+
} else {
174+
None
175+
}
176+
} else {
177+
None
178+
};
179+
154180
let state = BuildState::new(self)?;
155181

156182
match is_single_package {
157183
true => Ok(state.build_single_package_graph().await?),
158184
false => {
159185
let state = state.parse_package_jsons().await?;
186+
187+
// If we started a lockfile read, collect the result before
188+
// entering resolve_lockfile so it becomes a cache hit.
189+
let state = if let Some(handle) = lockfile_future {
190+
if let Ok(Some(lockfile)) = handle.await {
191+
state.with_lockfile(lockfile)
192+
} else {
193+
state
194+
}
195+
} else {
196+
state
197+
};
198+
160199
let state = state.resolve_lockfile().await?;
161200
Ok(state.build_inner().await?)
162201
}
@@ -220,6 +259,7 @@ where
220259
package_jsons,
221260
lockfile,
222261
package_discovery,
262+
package_manager: _,
223263
} = builder;
224264
let mut workspaces = HashMap::new();
225265
workspaces.insert(
@@ -388,6 +428,11 @@ impl<'a, T: PackageDiscovery> BuildState<'a, ResolvedPackageManager, T> {
388428
}
389429

390430
impl<'a, T: PackageDiscovery> BuildState<'a, ResolvedWorkspaces, T> {
431+
fn with_lockfile(mut self, lockfile: Box<dyn Lockfile>) -> Self {
432+
self.lockfile = Some(lockfile);
433+
self
434+
}
435+
391436
#[tracing::instrument(skip(self))]
392437
fn connect_internal_dependencies(
393438
&mut self,

‎profile.large.md‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
# CPU Profile
2+
3+
| Duration | Spans | Functions |
4+
| -------- | ----- | --------- |
5+
| 1.4s | 16359 | 33 |
6+
7+
**Top 10:** `walk_glob` 50.6%, `compile_globs` 24.3%, `new` 19.7%, `git_status_repo_root` 19.7%, `queue_task` 14.8%, `get_package_file_hashes_from_inputs_and_index` 12.3%, `parse_lockfile` 7.6%, `finish` 7.0%, `parse_package_jsons` 6.6%, `calculate_file_hashes` 6.4%
8+
9+
## Hot Functions (Self Time)
10+
11+
| Self% | Self | Total% | Total | Function | Location |
12+
| ----: | ------: | -----: | ------: | ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
13+
| 50.6% | 714.5ms | 50.6% | 714.5ms | `walk_glob` | `crates/turborepo-globwalk/src/lib.rs:601` |
14+
| 24.3% | 343.7ms | 26.0% | 367.6ms | `compile_globs` | `crates/turborepo-globwalk/src/lib.rs:519` |
15+
| 19.7% | 278.3ms | 19.7% | 278.3ms | `new` | `crates/turborepo-scm/src/repo_index.rs:20` |
16+
| 19.7% | 278.2ms | 19.7% | 278.2ms | `git_status_repo_root` | `crates/turborepo-scm/src/status.rs:56` |
17+
| 14.8% | 209.2ms | 21.1% | 298.5ms | `queue_task` | `crates/turborepo-lib/src/task_graph/visitor/mod.rs:205` |
18+
| 12.3% | 173.4ms | 82.9% | 1.2s | `get_package_file_hashes_from_inputs_and_index` | `crates/turborepo-scm/src/package_deps.rs:244` |
19+
| 7.6% | 107.0ms | 7.6% | 107.0ms | `parse_lockfile` | `crates/turborepo-repository/src/package_manager/mod.rs:479` |
20+
| 7.0% | 99.3ms | 7.0% | 99.3ms | `finish` | `crates/turborepo-run-summary/src/tracker.rs:307` |
21+
| 6.6% | 92.7ms | 6.6% | 93.7ms | `parse_package_jsons` | `crates/turborepo-repository/src/package_graph/builder.rs:289` |
22+
| 6.4% | 90.9ms | 6.4% | 90.9ms | `calculate_file_hashes` | `crates/turborepo-task-hash/src/lib.rs:79` |
23+
| 6.3% | 89.3ms | 6.3% | 89.3ms | `calculate_task_hash` | `crates/turborepo-task-hash/src/lib.rs:290` |
24+
| 6.2% | 88.2ms | 6.2% | 88.2ms | `to_summary` | `crates/turborepo-run-summary/src/tracker.rs:121` |
25+
| 4.6% | 64.5ms | 4.6% | 64.5ms | `git_ls_tree_repo_root_sorted` | `crates/turborepo-scm/src/ls_tree.rs:41` |
26+
| 4.4% | 62.2ms | 4.4% | 62.2ms | `hash_objects` | `crates/turborepo-scm/src/hash_object.rs:39` |
27+
| 3.6% | 50.3ms | 3.6% | 50.3ms | `connect_internal_dependencies` | `crates/turborepo-repository/src/package_graph/builder.rs:390` |
28+
| 2.4% | 33.4ms | 2.4% | 33.4ms | `parse` | `/Users/anthonyshew/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/biome_json_parser-0.5.7/src/lib.rs:32` |
29+
| 2.0% | 28.9ms | 2.0% | 28.9ms | `populate_transitive_dependencies` | `crates/turborepo-repository/src/package_graph/builder.rs:542` |
30+
| 1.7% | 24.4ms | 1.9% | 26.9ms | `get_package_file_hashes_from_index` | `crates/turborepo-scm/src/package_deps.rs:148` |
31+
| 1.7% | 24.0ms | 1.7% | 24.0ms | `preprocess_paths_and_globs` | `crates/turborepo-globwalk/src/lib.rs:71` |
32+
| 0.7% | 10.1ms | 0.7% | 10.1ms | `new` | `crates/turborepo-scm/src/lib.rs:286` |
33+
| 0.5% | 7.0ms | 21.6% | 305.5ms | `visit` | `crates/turborepo-lib/src/task_graph/visitor/mod.rs:180` |
34+
| 0.5% | 6.5ms | 0.5% | 6.5ms | `exists` | `crates/turborepo-cache/src/fs.rs:126` |
35+
36+
## Call Tree (Total Time)
37+
38+
| Total% | Total | Self% | Self | Function | Location |
39+
| -----: | ------: | ----: | ------: | ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
40+
| 84.2% | 1.2s | 0.3% | 4.6ms | `get_package_file_hashes` | `crates/turborepo-scm/src/package_deps.rs:27` |
41+
| 82.9% | 1.2s | 12.3% | 173.4ms | `get_package_file_hashes_from_inputs_and_index` | `crates/turborepo-scm/src/package_deps.rs:244` |
42+
| 50.6% | 714.5ms | 50.6% | 714.5ms | `walk_glob` | `crates/turborepo-globwalk/src/lib.rs:601` |
43+
| 26.0% | 367.6ms | 24.3% | 343.7ms | `compile_globs` | `crates/turborepo-globwalk/src/lib.rs:519` |
44+
| 21.6% | 305.5ms | 0.5% | 7.0ms | `visit` | `crates/turborepo-lib/src/task_graph/visitor/mod.rs:180` |
45+
| 21.1% | 298.5ms | 14.8% | 209.2ms | `queue_task` | `crates/turborepo-lib/src/task_graph/visitor/mod.rs:205` |
46+
| 19.9% | 281.1ms | 0.0% | 451us | `build` | `crates/turborepo-repository/src/package_graph/builder.rs:150` |
47+
| 19.7% | 278.3ms | 19.7% | 278.3ms | `new` | `crates/turborepo-scm/src/repo_index.rs:20` |
48+
| 19.7% | 278.2ms | 19.7% | 278.2ms | `git_status_repo_root` | `crates/turborepo-scm/src/status.rs:56` |
49+
| 11.2% | 158.0ms | 0.0% | 73us | `resolve_lockfile` | `crates/turborepo-repository/src/package_graph/builder.rs:467` |
50+
| 7.6% | 107.6ms | 0.0% | 61us | `populate_lockfile` | `crates/turborepo-repository/src/package_graph/builder.rs:443` |
51+
| 7.6% | 107.5ms | 0.0% | 538us | `read_lockfile` | `crates/turborepo-repository/src/package_manager/mod.rs:458` |
52+
| 7.6% | 107.0ms | 7.6% | 107.0ms | `parse_lockfile` | `crates/turborepo-repository/src/package_manager/mod.rs:479` |
53+
| 7.0% | 99.3ms | 7.0% | 99.3ms | `finish` | `crates/turborepo-run-summary/src/tracker.rs:307` |
54+
| 6.6% | 93.7ms | 6.6% | 92.7ms | `parse_package_jsons` | `crates/turborepo-repository/src/package_graph/builder.rs:289` |
55+
| 6.4% | 90.9ms | 6.4% | 90.9ms | `calculate_file_hashes` | `crates/turborepo-task-hash/src/lib.rs:79` |
56+
| 6.3% | 89.3ms | 6.3% | 89.3ms | `calculate_task_hash` | `crates/turborepo-task-hash/src/lib.rs:290` |
57+
| 6.2% | 88.2ms | 6.2% | 88.2ms | `to_summary` | `crates/turborepo-run-summary/src/tracker.rs:121` |
58+
| 4.6% | 64.5ms | 4.6% | 64.5ms | `git_ls_tree_repo_root_sorted` | `crates/turborepo-scm/src/ls_tree.rs:41` |
59+
| 4.4% | 62.2ms | 4.4% | 62.2ms | `hash_objects` | `crates/turborepo-scm/src/hash_object.rs:39` |
60+
| 3.6% | 50.3ms | 3.6% | 50.3ms | `connect_internal_dependencies` | `crates/turborepo-repository/src/package_graph/builder.rs:390` |
61+
| 2.4% | 33.4ms | 2.4% | 33.4ms | `parse` | `/Users/anthonyshew/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/biome_json_parser-0.5.7/src/lib.rs:32` |
62+
| 2.1% | 29.0ms | 0.0% | 65us | `build_inner` | `crates/turborepo-repository/src/package_graph/builder.rs:561` |
63+
| 2.0% | 28.9ms | 2.0% | 28.9ms | `populate_transitive_dependencies` | `crates/turborepo-repository/src/package_graph/builder.rs:542` |
64+
| 1.9% | 26.9ms | 1.7% | 24.4ms | `get_package_file_hashes_from_index` | `crates/turborepo-scm/src/package_deps.rs:148` |
65+
| 1.7% | 24.0ms | 1.7% | 24.0ms | `preprocess_paths_and_globs` | `crates/turborepo-globwalk/src/lib.rs:71` |
66+
| 0.7% | 10.1ms | 0.7% | 10.1ms | `new` | `crates/turborepo-scm/src/lib.rs:286` |
67+
| 0.5% | 6.5ms | 0.5% | 6.5ms | `exists` | `crates/turborepo-cache/src/fs.rs:126` |
68+
69+
## Function Details
70+
71+
### `walk_glob`
72+
73+
`crates/turborepo-globwalk/src/lib.rs:601` | Self: 50.6% (714.5ms) | Total: 50.6% (714.5ms) | Calls: 1009
74+
75+
**Called by:**
76+
77+
- `get_package_file_hashes_from_inputs_and_index` (992)
78+
79+
### `compile_globs`
80+
81+
`crates/turborepo-globwalk/src/lib.rs:519` | Self: 24.3% (343.7ms) | Total: 26.0% (367.6ms) | Calls: 994
82+
83+
**Called by:**
84+
85+
- `parse_package_jsons` (1)
86+
- `get_package_file_hashes_from_inputs_and_index` (992)
87+
88+
**Calls:**
89+
90+
- `preprocess_paths_and_globs` (994)
91+
92+
### `new`
93+
94+
`crates/turborepo-scm/src/repo_index.rs:20` | Self: 19.7% (278.3ms) | Total: 19.7% (278.3ms) | Calls: 1
95+
96+
### `git_status_repo_root`
97+
98+
`crates/turborepo-scm/src/status.rs:56` | Self: 19.7% (278.2ms) | Total: 19.7% (278.2ms) | Calls: 1
99+
100+
### `queue_task`
101+
102+
`crates/turborepo-lib/src/task_graph/visitor/mod.rs:205` | Self: 14.8% (209.2ms) | Total: 21.1% (298.5ms) | Calls: 1690
103+
104+
**Called by:**
105+
106+
- `visit` (1690)
107+
108+
**Calls:**
109+
110+
- `calculate_task_hash` (1690)
111+
112+
### `get_package_file_hashes_from_inputs_and_index`
113+
114+
`crates/turborepo-scm/src/package_deps.rs:244` | Self: 12.3% (173.4ms) | Total: 82.9% (1.2s) | Calls: 992
115+
116+
**Called by:**
117+
118+
- `get_package_file_hashes` (992)
119+
120+
**Calls:**
121+
122+
- `walk_glob` (992)
123+
- `hash_objects` (992)
124+
- `get_package_file_hashes_from_index` (992)
125+
- `compile_globs` (992)
126+
127+
### `parse_lockfile`
128+
129+
`crates/turborepo-repository/src/package_manager/mod.rs:479` | Self: 7.6% (107.0ms) | Total: 7.6% (107.0ms) | Calls: 1
130+
131+
**Called by:**
132+
133+
- `read_lockfile` (1)
134+
135+
### `finish`
136+
137+
`crates/turborepo-run-summary/src/tracker.rs:307` | Self: 7.0% (99.3ms) | Total: 7.0% (99.3ms) | Calls: 1
138+
139+
### `parse_package_jsons`
140+
141+
`crates/turborepo-repository/src/package_graph/builder.rs:289` | Self: 6.6% (92.7ms) | Total: 6.6% (93.7ms) | Calls: 1
142+
143+
**Called by:**
144+
145+
- `build` (1)
146+
147+
**Calls:**
148+
149+
- `compile_globs` (1)
150+
151+
### `calculate_file_hashes`
152+
153+
`crates/turborepo-task-hash/src/lib.rs:79` | Self: 6.4% (90.9ms) | Total: 6.4% (90.9ms) | Calls: 1
154+
155+
### `calculate_task_hash`
156+
157+
`crates/turborepo-task-hash/src/lib.rs:290` | Self: 6.3% (89.3ms) | Total: 6.3% (89.3ms) | Calls: 1690
158+
159+
**Called by:**
160+
161+
- `queue_task` (1690)
162+
163+
### `to_summary`
164+
165+
`crates/turborepo-run-summary/src/tracker.rs:121` | Self: 6.2% (88.2ms) | Total: 6.2% (88.2ms) | Calls: 1
166+
167+
### `git_ls_tree_repo_root_sorted`
168+
169+
`crates/turborepo-scm/src/ls_tree.rs:41` | Self: 4.6% (64.5ms) | Total: 4.6% (64.5ms) | Calls: 1
170+
171+
### `hash_objects`
172+
173+
`crates/turborepo-scm/src/hash_object.rs:39` | Self: 4.4% (62.2ms) | Total: 4.4% (62.2ms) | Calls: 2687
174+
175+
**Called by:**
176+
177+
- `get_package_file_hashes_from_index` (1688)
178+
- `get_package_file_hashes_from_inputs_and_index` (992)
179+
180+
### `connect_internal_dependencies`
181+
182+
`crates/turborepo-repository/src/package_graph/builder.rs:390` | Self: 3.6% (50.3ms) | Total: 3.6% (50.3ms) | Calls: 1
183+
184+
**Called by:**
185+
186+
- `resolve_lockfile` (1)
187+
188+
### `parse`
189+
190+
`/Users/anthonyshew/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/biome_json_parser-0.5.7/src/lib.rs:32` | Self: 2.4% (33.4ms) | Total: 2.4% (33.4ms) | Calls: 1215
191+
192+
### `populate_transitive_dependencies`
193+
194+
`crates/turborepo-repository/src/package_graph/builder.rs:542` | Self: 2.0% (28.9ms) | Total: 2.0% (28.9ms) | Calls: 1
195+
196+
**Called by:**
197+
198+
- `build_inner` (1)
199+
200+
### `get_package_file_hashes_from_index`
201+
202+
`crates/turborepo-scm/src/package_deps.rs:148` | Self: 1.7% (24.4ms) | Total: 1.9% (26.9ms) | Calls: 1688
203+
204+
**Called by:**
205+
206+
- `get_package_file_hashes` (690)
207+
- `get_package_file_hashes_from_inputs_and_index` (992)
208+
209+
**Calls:**
210+
211+
- `hash_objects` (1688)
212+
213+
### `preprocess_paths_and_globs`
214+
215+
`crates/turborepo-globwalk/src/lib.rs:71` | Self: 1.7% (24.0ms) | Total: 1.7% (24.0ms) | Calls: 1000
216+
217+
**Called by:**
218+
219+
- `compile_globs` (994)
220+
221+
### `new`
222+
223+
`crates/turborepo-scm/src/lib.rs:286` | Self: 0.7% (10.1ms) | Total: 0.7% (10.1ms) | Calls: 1

0 commit comments

Comments
 (0)