Skip to content

Commit 58ca58f

Browse files
mischnicmischnic
committed
wip
1 parent f384c21 commit 58ca58f

File tree

8 files changed

+151
-38
lines changed

8 files changed

+151
-38
lines changed

‎turbopack/crates/turbopack-core/src/chunk/availability_info.rs

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
use anyhow::Result;
22
use turbo_tasks::{ResolvedVc, Vc};
33

4-
use super::available_modules::{AvailableModuleInfoMap, AvailableModules};
4+
use crate::{
5+
chunk::available_chunk_groups::AvailableChunkGroups,
6+
module_graph::chunk_group_info::RoaringBitmapWrapperCell,
7+
};
58

69
#[turbo_tasks::value(serialization = "auto_for_input")]
710
#[derive(Hash, Clone, Copy, Debug)]
@@ -12,12 +15,12 @@ pub enum AvailabilityInfo {
1215
Root,
1316
/// There are modules already available.
1417
Complete {
15-
available_modules: ResolvedVc<AvailableModules>,
18+
available_modules: ResolvedVc<AvailableChunkGroups>,
1619
},
1720
}
1821

1922
impl AvailabilityInfo {
20-
pub fn available_modules(&self) -> Option<ResolvedVc<AvailableModules>> {
23+
pub fn available_chunk_groups(&self) -> Option<ResolvedVc<AvailableChunkGroups>> {
2124
match self {
2225
Self::Untracked => None,
2326
Self::Root => None,
@@ -27,15 +30,15 @@ impl AvailabilityInfo {
2730
}
2831
}
2932

30-
pub async fn with_modules(self, modules: Vc<AvailableModuleInfoMap>) -> Result<Self> {
33+
pub async fn with_modules(self, chunk_group: Vc<RoaringBitmapWrapperCell>) -> Result<Self> {
3134
Ok(match self {
3235
AvailabilityInfo::Untracked => AvailabilityInfo::Untracked,
3336
AvailabilityInfo::Root => AvailabilityInfo::Complete {
34-
available_modules: AvailableModules::new(modules).to_resolved().await?,
37+
available_modules: AvailableChunkGroups::new(chunk_group).to_resolved().await?,
3538
},
3639
AvailabilityInfo::Complete { available_modules } => AvailabilityInfo::Complete {
3740
available_modules: available_modules
38-
.with_modules(modules)
41+
.with_chunk_group(chunk_group)
3942
.to_resolved()
4043
.await?,
4144
},

‎turbopack/crates/turbopack-core/src/chunk/available_chunk_groups.rs

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
use std::hash::{Hash, Hasher};
2+
3+
use anyhow::Result;
4+
use rustc_hash::FxHasher;
5+
use turbo_tasks::Vc;
6+
7+
use crate::module_graph::chunk_group_info::{RoaringBitmapWrapper, RoaringBitmapWrapperCell};
8+
9+
/// Allows to gather information about which assets are already available.
10+
#[turbo_tasks::value]
11+
pub struct AvailableChunkGroups {
12+
chunk_groups: RoaringBitmapWrapper,
13+
}
14+
15+
#[turbo_tasks::value_impl]
16+
impl AvailableChunkGroups {
17+
#[turbo_tasks::function]
18+
pub async fn new(chunk_groups: Vc<RoaringBitmapWrapperCell>) -> Result<Vc<Self>> {
19+
Ok(AvailableChunkGroups {
20+
chunk_groups: chunk_groups.owned().await?,
21+
}
22+
.cell())
23+
}
24+
25+
#[turbo_tasks::function]
26+
pub async fn with_chunk_group(
27+
&self,
28+
chunk_group: Vc<RoaringBitmapWrapperCell>,
29+
) -> Result<Vc<Self>> {
30+
Ok(AvailableChunkGroups {
31+
chunk_groups: RoaringBitmapWrapper::new(&*self.chunk_groups | &**chunk_group.await?),
32+
}
33+
.cell())
34+
}
35+
36+
#[turbo_tasks::function]
37+
pub async fn hash(&self) -> Result<Vc<u64>> {
38+
let mut hasher = FxHasher::default();
39+
self.chunk_groups.hash(&mut hasher);
40+
// TODO a more deterministic hash? Previously, this hashed all `module.ident().to_string()`
41+
Ok(Vc::cell(hasher.finish()))
42+
}
43+
44+
#[turbo_tasks::function]
45+
pub async fn is_available(&self, module: Vc<RoaringBitmapWrapperCell>) -> Result<Vc<bool>> {
46+
Ok(Vc::cell(self.is_available_individual(&*module.await?)))
47+
}
48+
}
49+
50+
impl AvailableChunkGroups {
51+
pub fn is_available_individual(&self, module: &RoaringBitmapWrapper) -> bool {
52+
// `self.chunk_groups` is the union of all parent chunk groups (i.e. a single chunking path
53+
// leading to this module)
54+
//
55+
// `module` is the union of all chunk groups of the module (i.e. the union of all paths
56+
// leading to this module)
57+
self.chunk_groups.is_superset(module)
58+
}
59+
}

‎turbopack/crates/turbopack-core/src/chunk/chunk_group.rs

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pub async fn make_chunk_group(
4444
traced_modules,
4545
passthrough_modules,
4646
} = chunk_group_content(
47-
&*module_graph.await?,
47+
module_graph,
4848
chunk_group_entries,
4949
availability_info,
5050
can_split_async,
@@ -79,9 +79,11 @@ pub async fn make_chunk_group(
7979
.collect::<FxIndexMap<_, Option<ResolvedVc<AsyncModuleInfo>>>>();
8080

8181
// Compute new [AvailabilityInfo]
82-
let availability_info = availability_info
83-
.with_modules(Vc::cell(chunkable_modules))
84-
.await?;
82+
let own_chunk_group_info = module_graph
83+
.chunk_group_info()
84+
// TODO which module should actually be looked up here?
85+
.get(*ResolvedVc::upcast(*chunkable_modules.first().unwrap()));
86+
let availability_info = availability_info.with_modules(own_chunk_group_info).await?;
8587

8688
// Insert async chunk loaders for every referenced async module
8789
let async_loaders = async_modules
@@ -192,7 +194,7 @@ pub async fn references_to_output_assets(
192194
}
193195

194196
pub async fn chunk_group_content(
195-
module_graph: &ModuleGraph,
197+
module_graph: Vc<ModuleGraph>,
196198
chunk_group_entries: impl IntoIterator<Item = ResolvedVc<Box<dyn Module>>>,
197199
availability_info: AvailabilityInfo,
198200
can_split_async: bool,
@@ -216,8 +218,11 @@ pub async fn chunk_group_content(
216218
},
217219
};
218220

219-
let available_modules = match availability_info.available_modules() {
220-
Some(available_modules) => Some(available_modules.snapshot().await?),
221+
let chunk_group_info = module_graph.chunk_group_info().await?;
222+
let module_graph = module_graph.await?;
223+
224+
let available_chunk_groups = match availability_info.available_chunk_groups() {
225+
Some(available_chunk_groups) => Some(available_chunk_groups.await?),
221226
None => None,
222227
};
223228

@@ -244,9 +249,13 @@ pub async fn chunk_group_content(
244249
return Ok(GraphTraversalAction::Skip);
245250
};
246251

247-
let is_available = available_modules
248-
.as_ref()
249-
.is_some_and(|available_modules| available_modules.get(chunkable_module));
252+
let is_available = if let Some(available_modules) = &available_chunk_groups {
253+
available_modules.is_available_individual(
254+
chunk_group_info.get_individual(ResolvedVc::upcast(chunkable_module))?,
255+
)
256+
} else {
257+
false
258+
};
250259

251260
let Some((_, edge)) = parent_info else {
252261
return Ok(if is_available {

‎turbopack/crates/turbopack-core/src/chunk/chunking/production.rs

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,16 @@
11
use std::{collections::BinaryHeap, hash::BuildHasherDefault};
22

3-
use anyhow::{bail, Result};
3+
use anyhow::Result;
44
use rustc_hash::FxHasher;
55
use tracing::{field::Empty, Instrument};
66
use turbo_prehash::BuildHasherExt;
7-
use turbo_tasks::{FxIndexMap, ResolvedVc, ValueToString, Vc};
7+
use turbo_tasks::{FxIndexMap, ResolvedVc, Vc};
88

99
use crate::{
1010
chunk::{
1111
chunking::{make_chunk, ChunkItemWithInfo, SplitContext},
1212
ChunkingConfig,
1313
},
14-
module::Module,
1514
module_graph::ModuleGraph,
1615
};
1716

@@ -37,15 +36,7 @@ pub async fn make_production_chunks(
3736
for chunk_item in chunk_items {
3837
let ChunkItemWithInfo { module, .. } = chunk_item;
3938
let chunk_groups = if let Some(module) = module {
40-
match chunk_group_info.get(&ResolvedVc::upcast(module)) {
41-
Some(chunk_group) => Some(chunk_group),
42-
None => {
43-
bail!(
44-
"Module {:?} has no chunk group info",
45-
module.ident().to_string().await?,
46-
);
47-
}
48-
}
39+
Some(chunk_group_info.get_individual(ResolvedVc::upcast(module))?)
4940
} else {
5041
None
5142
};

‎turbopack/crates/turbopack-core/src/chunk/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
pub mod availability_info;
2+
pub mod available_chunk_groups;
23
pub mod available_modules;
34
pub mod chunk_group;
45
pub mod chunking;

‎turbopack/crates/turbopack-core/src/module_graph/chunk_group_info.rs

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ use serde::{Deserialize, Serialize};
1313
use tracing::Instrument;
1414
use turbo_rcstr::RcStr;
1515
use turbo_tasks::{
16-
debug::ValueDebugFormat, trace::TraceRawVcs, NonLocalValue, ResolvedVc, TryJoinIterExt, Vc,
16+
debug::ValueDebugFormat, trace::TraceRawVcs, NonLocalValue, ResolvedVc, TaskInput,
17+
TryJoinIterExt, Vc,
1718
};
1819

1920
use crate::{
@@ -24,13 +25,16 @@ use crate::{
2425
SingleModuleGraphNode,
2526
},
2627
};
27-
2828
#[derive(
2929
Clone, Debug, Default, PartialEq, Serialize, Deserialize, TraceRawVcs, ValueDebugFormat,
3030
)]
3131
pub struct RoaringBitmapWrapper(#[turbo_tasks(trace_ignore)] RoaringBitmap);
3232

3333
impl RoaringBitmapWrapper {
34+
pub fn new(value: RoaringBitmap) -> RoaringBitmapWrapper {
35+
RoaringBitmapWrapper(value)
36+
}
37+
3438
/// Whether `self` contains bits that are not in `other`
3539
///
3640
/// The existing `is_superset` method also returns true for equal sets
@@ -42,6 +46,11 @@ impl RoaringBitmapWrapper {
4246
self.0
4347
}
4448
}
49+
impl TaskInput for RoaringBitmapWrapper {
50+
fn is_transient(&self) -> bool {
51+
false
52+
}
53+
}
4554
unsafe impl NonLocalValue for RoaringBitmapWrapper {}
4655

4756
// RoaringBitmap doesn't impl Eq: https://github.com/RoaringBitmap/roaring-rs/issues/302
@@ -77,8 +86,32 @@ impl Hash for RoaringBitmapWrapper {
7786
}
7887

7988
#[turbo_tasks::value(transparent)]
89+
pub struct RoaringBitmapWrapperCell(RoaringBitmapWrapper);
90+
91+
#[turbo_tasks::value]
8092
pub struct ChunkGroupInfo(FxHashMap<ResolvedVc<Box<dyn Module>>, RoaringBitmapWrapper>);
8193

94+
#[turbo_tasks::value_impl]
95+
impl ChunkGroupInfo {
96+
#[turbo_tasks::function]
97+
pub fn get(&self, module: ResolvedVc<Box<dyn Module>>) -> Result<Vc<RoaringBitmapWrapperCell>> {
98+
Ok(Vc::cell(self.get_individual(module)?.clone()))
99+
}
100+
}
101+
102+
impl ChunkGroupInfo {
103+
pub fn get_individual(
104+
&self,
105+
module: ResolvedVc<Box<dyn Module>>,
106+
) -> Result<&RoaringBitmapWrapper> {
107+
if let Some(chunk_group) = self.0.get(&module) {
108+
Ok(chunk_group)
109+
} else {
110+
anyhow::bail!("Module has no chunk group info");
111+
}
112+
}
113+
}
114+
82115
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
83116
enum ChunkGroup {
84117
/// e.g. a page
@@ -349,7 +382,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
349382
span.record("visit_count", visit_count);
350383
span.record("chunk_group_count", next_chunk_group_id);
351384

352-
Ok(Vc::cell(module_chunk_groups))
385+
Ok(ChunkGroupInfo(module_chunk_groups).cell())
353386
}
354387
.instrument(span_outer)
355388
.await

‎turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,15 @@ impl AsyncLoaderChunkItem {
3535
#[turbo_tasks::function]
3636
pub(super) async fn chunks(&self) -> Result<Vc<OutputAssets>> {
3737
let module = self.module.await?;
38-
if let Some(chunk_items) = module.availability_info.available_modules() {
39-
if *chunk_items.get(*module.inner).await? {
38+
if let Some(available_chunk_groups) = module.availability_info.available_chunk_groups() {
39+
if *available_chunk_groups
40+
.is_available(
41+
self.module_graph
42+
.chunk_group_info()
43+
.get(*ResolvedVc::upcast(module.inner)),
44+
)
45+
.await?
46+
{
4047
return Ok(Vc::cell(vec![]));
4148
}
4249
}
@@ -154,8 +161,11 @@ impl ChunkItem for AsyncLoaderChunkItem {
154161
#[turbo_tasks::function]
155162
async fn content_ident(&self) -> Result<Vc<AssetIdent>> {
156163
let mut ident = self.module.ident();
157-
if let Some(available_chunk_items) =
158-
self.module.await?.availability_info.available_modules()
164+
if let Some(available_chunk_items) = self
165+
.module
166+
.await?
167+
.availability_info
168+
.available_chunk_groups()
159169
{
160170
ident = ident.with_modifier(Vc::cell(
161171
available_chunk_items.hash().await?.to_string().into(),

‎turbopack/crates/turbopack-ecmascript/src/manifest/chunk_asset.rs

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,15 @@ impl ManifestAsyncModule {
6969
#[turbo_tasks::function]
7070
pub async fn manifest_chunks(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
7171
let this = self.await?;
72-
if let Some(chunk_items) = this.availability_info.available_modules() {
73-
if *chunk_items.get(*this.inner).await? {
72+
if let Some(available_chunk_groups) = this.availability_info.available_chunk_groups() {
73+
if *available_chunk_groups
74+
.is_available(
75+
this.module_graph
76+
.chunk_group_info()
77+
.get(*ResolvedVc::upcast(this.inner)),
78+
)
79+
.await?
80+
{
7481
return Ok(Vc::cell(vec![]));
7582
}
7683
}
@@ -89,7 +96,7 @@ impl ManifestAsyncModule {
8996
#[turbo_tasks::function]
9097
pub async fn content_ident(&self) -> Result<Vc<AssetIdent>> {
9198
let mut ident = self.inner.ident();
92-
if let Some(available_modules) = self.availability_info.available_modules() {
99+
if let Some(available_modules) = self.availability_info.available_chunk_groups() {
93100
ident =
94101
ident.with_modifier(Vc::cell(available_modules.hash().await?.to_string().into()));
95102
}

0 commit comments

Comments
 (0)