Skip to content

Commit 18c7763

Browse files
cslinwangcslinwang
committed
Fixed bugs in chunkdict and added blobinfo support to chunkdict.
1 parent eae2b55 commit 18c7763

File tree

11 files changed

+306
-81
lines changed

11 files changed

+306
-81
lines changed

‎builder/src/chunkdict_generator.rs

Lines changed: 53 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use nydus_rafs::metadata::chunk::ChunkWrapper;
2323
use nydus_rafs::metadata::inode::InodeWrapper;
2424
use nydus_rafs::metadata::layout::RafsXAttrs;
2525
use nydus_storage::meta::BlobChunkInfoV1Ondisk;
26+
use nydus_utils::compress::Algorithm;
2627
use nydus_utils::digest::RafsDigest;
2728
use std::ffi::OsString;
2829
use std::mem::size_of;
@@ -41,6 +42,16 @@ pub struct ChunkdictChunkInfo {
4142
pub chunk_uncompressed_offset: u64,
4243
}
4344

45+
pub struct ChunkdictBlobInfo {
46+
pub blob_id: String,
47+
pub blob_compressed_size: u64,
48+
pub blob_uncompressed_size: u64,
49+
pub blob_compressor: String,
50+
pub blob_meta_ci_compressed_size: u64,
51+
pub blob_meta_ci_uncompressed_size: u64,
52+
pub blob_meta_ci_offset: u64,
53+
}
54+
4455
/// Struct to generate chunkdict RAFS bootstrap.
4556
pub struct Generator {}
4657

@@ -50,17 +61,17 @@ impl Generator {
5061
ctx: &mut BuildContext,
5162
bootstrap_mgr: &mut BootstrapManager,
5263
blob_mgr: &mut BlobManager,
53-
chunkdict_origin: Vec<ChunkdictChunkInfo>,
64+
chunkdict_chunks_origin: Vec<ChunkdictChunkInfo>,
65+
chunkdict_blobs: Vec<ChunkdictBlobInfo>,
5466
) -> Result<BuildOutput> {
5567
// Validate and remove chunks whose belonged blob sizes are smaller than a block.
56-
let mut chunkdict = chunkdict_origin.to_vec();
57-
Self::validate_and_remove_chunks(ctx, &mut chunkdict);
58-
68+
let mut chunkdict_chunks = chunkdict_chunks_origin.to_vec();
69+
Self::validate_and_remove_chunks(ctx, &mut chunkdict_chunks);
5970
// build root tree
6071
let mut tree = Self::build_root_tree(ctx)?;
6172

6273
// build child tree
63-
let child = Self::build_child_tree(ctx, blob_mgr, &chunkdict)?;
74+
let child = Self::build_child_tree(ctx, blob_mgr, &chunkdict_chunks, &chunkdict_blobs)?;
6475
let result = vec![child];
6576
tree.children = result;
6677

@@ -156,7 +167,8 @@ impl Generator {
156167
fn build_child_tree(
157168
ctx: &mut BuildContext,
158169
blob_mgr: &mut BlobManager,
159-
chunkdict: &[ChunkdictChunkInfo],
170+
chunkdict_chunks: &[ChunkdictChunkInfo],
171+
chunkdict_blobs: &[ChunkdictBlobInfo],
160172
) -> Result<Tree> {
161173
// node
162174
let mut inode = InodeWrapper::new(ctx.fs_version);
@@ -185,7 +197,7 @@ impl Generator {
185197
let mut node = Node::new(inode, node_info, 0);
186198

187199
// insert chunks
188-
Self::insert_chunks(ctx, blob_mgr, &mut node, chunkdict)?;
200+
Self::insert_chunks(ctx, blob_mgr, &mut node, chunkdict_chunks, chunkdict_blobs)?;
189201

190202
let node_size: u64 = node
191203
.chunks
@@ -209,16 +221,22 @@ impl Generator {
209221
ctx: &mut BuildContext,
210222
blob_mgr: &mut BlobManager,
211223
node: &mut Node,
212-
chunkdict: &[ChunkdictChunkInfo],
224+
chunkdict_chunks: &[ChunkdictChunkInfo],
225+
chunkdict_blobs: &[ChunkdictBlobInfo],
213226
) -> Result<()> {
214-
for (i, chunk_info) in chunkdict.iter().enumerate() {
227+
for (index, chunk_info) in chunkdict_chunks.iter().enumerate() {
215228
let chunk_size: u32 = chunk_info.chunk_compressed_size;
216-
let file_offset = i as u64 * chunk_size as u64;
229+
let file_offset = index as u64 * chunk_size as u64;
217230
let mut chunk = ChunkWrapper::new(ctx.fs_version);
218231

219232
// update blob context
220233
let (blob_index, blob_ctx) =
221234
blob_mgr.get_or_cerate_blob_for_chunkdict(ctx, &chunk_info.chunk_blob_id)?;
235+
if blob_ctx.blob_id.is_empty() {
236+
blob_ctx.blob_id = chunk_info.chunk_blob_id.clone();
237+
}
238+
239+
// blob_ctx.
222240
let chunk_uncompressed_size = chunk_info.chunk_uncompressed_size;
223241
let pre_d_offset = blob_ctx.current_uncompressed_offset;
224242
blob_ctx.uncompressed_blob_size = pre_d_offset + chunk_uncompressed_size as u64;
@@ -228,6 +246,31 @@ impl Generator {
228246
blob_ctx.blob_meta_header.ci_uncompressed_size()
229247
+ size_of::<BlobChunkInfoV1Ondisk>() as u64,
230248
);
249+
blob_ctx.blob_meta_header.set_ci_compressed_size(
250+
blob_ctx.blob_meta_header.ci_uncompressed_size()
251+
+ size_of::<BlobChunkInfoV1Ondisk>() as u64,
252+
);
253+
let chunkdict_blob_info = chunkdict_blobs
254+
.iter()
255+
.find(|blob| blob.blob_id == chunk_info.chunk_blob_id)
256+
.unwrap();
257+
blob_ctx.blob_compressor = match chunkdict_blob_info.blob_compressor.as_str() {
258+
"None" => Algorithm::None,
259+
"Lz4Block" => Algorithm::Lz4Block,
260+
"GZip" => Algorithm::GZip,
261+
"Zstd" => Algorithm::Zstd,
262+
_ => Algorithm::None,
263+
};
264+
blob_ctx
265+
.blob_meta_header
266+
.set_ci_uncompressed_size(chunkdict_blob_info.blob_meta_ci_uncompressed_size);
267+
blob_ctx
268+
.blob_meta_header
269+
.set_ci_compressed_size(chunkdict_blob_info.blob_meta_ci_compressed_size);
270+
blob_ctx
271+
.blob_meta_header
272+
.set_ci_compressed_offset(chunkdict_blob_info.blob_meta_ci_offset);
273+
blob_ctx.blob_meta_header.set_ci_compressor(Algorithm::Zstd);
231274

232275
// update chunk
233276
let chunk_index = blob_ctx.alloc_chunk_index()?;

‎builder/src/core/context.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,9 @@ impl BlobContext {
597597
blob_ctx
598598
.blob_meta_header
599599
.set_encrypted(features.contains(BlobFeatures::ENCRYPTED));
600+
blob_ctx
601+
.blob_meta_header
602+
.set_is_chunkdict_generated(features.contains(BlobFeatures::IS_CHUNKDICT_GENERATED));
600603

601604
blob_ctx
602605
}
@@ -1120,6 +1123,7 @@ impl BlobManager {
11201123
compressed_blob_size,
11211124
blob_features,
11221125
flags,
1126+
build_ctx.is_chunkdict_generated,
11231127
);
11241128
}
11251129
RafsBlobTable::V6(table) => {
@@ -1139,6 +1143,7 @@ impl BlobManager {
11391143
ctx.blob_toc_digest,
11401144
ctx.blob_meta_size,
11411145
ctx.blob_toc_size,
1146+
build_ctx.is_chunkdict_generated,
11421147
ctx.blob_meta_header,
11431148
ctx.cipher_object.clone(),
11441149
ctx.cipher_ctx.clone(),
@@ -1316,6 +1321,9 @@ pub struct BuildContext {
13161321
pub configuration: Arc<ConfigV2>,
13171322
/// Generate the blob cache and blob meta
13181323
pub blob_cache_generator: Option<BlobCacheGenerator>,
1324+
1325+
/// Whether is chunkdict.
1326+
pub is_chunkdict_generated: bool,
13191327
}
13201328

13211329
impl BuildContext {
@@ -1384,6 +1392,7 @@ impl BuildContext {
13841392
features,
13851393
configuration: Arc::new(ConfigV2::default()),
13861394
blob_cache_generator: None,
1395+
is_chunkdict_generated: false,
13871396
}
13881397
}
13891398

@@ -1402,6 +1411,10 @@ impl BuildContext {
14021411
pub fn set_configuration(&mut self, config: Arc<ConfigV2>) {
14031412
self.configuration = config;
14041413
}
1414+
1415+
pub fn set_is_chunkdict(&mut self, is_chunkdict: bool) {
1416+
self.is_chunkdict_generated = is_chunkdict;
1417+
}
14051418
}
14061419

14071420
impl Default for BuildContext {
@@ -1434,6 +1447,7 @@ impl Default for BuildContext {
14341447
features: Features::new(),
14351448
configuration: Arc::new(ConfigV2::default()),
14361449
blob_cache_generator: None,
1450+
is_chunkdict_generated: false,
14371451
}
14381452
}
14391453
}

‎builder/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use sha2::Digest;
2323

2424
use self::core::node::{Node, NodeInfo};
2525

26+
pub use self::chunkdict_generator::ChunkdictBlobInfo;
2627
pub use self::chunkdict_generator::ChunkdictChunkInfo;
2728
pub use self::chunkdict_generator::Generator;
2829
pub use self::compact::BlobCompactor;

‎rafs/src/metadata/cached_v5.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -990,6 +990,7 @@ mod cached_tests {
990990
0,
991991
BlobFeatures::_V5_NO_EXT_BLOB_TABLE,
992992
meta.flags,
993+
false,
993994
);
994995
let mut cached_inode = CachedInodeV5::new(blob_table, meta.clone());
995996
cached_inode.load(&meta, &mut reader).unwrap();

‎rafs/src/metadata/layout/v5.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,7 @@ impl RafsV5BlobTable {
563563
compressed_size: u64,
564564
blob_features: BlobFeatures,
565565
flags: RafsSuperFlags,
566+
is_chunkdict: bool,
566567
) -> u32 {
567568
let blob_index = self.entries.len() as u32;
568569
let mut blob_info = BlobInfo::new(
@@ -578,6 +579,9 @@ impl RafsV5BlobTable {
578579
blob_info.set_compressor(flags.into());
579580
blob_info.set_digester(flags.into());
580581
blob_info.set_prefetch_info(prefetch_offset as u64, prefetch_size as u64);
582+
if is_chunkdict {
583+
blob_info.set_chunkdict_generated(true);
584+
}
581585

582586
self.entries.push(Arc::new(blob_info));
583587
self.extended.add(

‎rafs/src/metadata/layout/v6.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1754,7 +1754,8 @@ impl RafsV6Blob {
17541754
blob_features.bits()
17551755
);
17561756
return false;
1757-
} else if !tarfs_mode
1757+
} else if !blob_features.contains(BlobFeatures::IS_CHUNKDICT_GENERATED)
1758+
&& !tarfs_mode
17581759
&& ci_uncompr_size != count * size_of::<BlobChunkInfoV1Ondisk>() as u64
17591760
{
17601761
error!(
@@ -1819,6 +1820,7 @@ impl RafsV6BlobTable {
18191820
blob_toc_digest: [u8; 32],
18201821
blob_meta_size: u64,
18211822
blob_toc_size: u32,
1823+
is_chunkdict: bool,
18221824
header: BlobCompressionContextHeader,
18231825
cipher_object: Arc<Cipher>,
18241826
cipher_context: Option<CipherContext>,
@@ -1851,6 +1853,8 @@ impl RafsV6BlobTable {
18511853
blob_info.set_blob_toc_size(blob_toc_size);
18521854
blob_info.set_cipher_info(flags.into(), cipher_object, cipher_context);
18531855

1856+
blob_info.set_chunkdict_generated(is_chunkdict);
1857+
18541858
self.entries.push(Arc::new(blob_info));
18551859

18561860
blob_index
@@ -2726,6 +2730,7 @@ mod tests {
27262730
[0; 32],
27272731
0,
27282732
0,
2733+
false,
27292734
BlobCompressionContextHeader::default(),
27302735
Arc::new(crypt::Algorithm::Aes128Xts.new_cipher().unwrap()),
27312736
Some(CipherContext::default()),
@@ -2768,6 +2773,7 @@ mod tests {
27682773
[0; 32],
27692774
0,
27702775
0,
2776+
false,
27712777
BlobCompressionContextHeader::default(),
27722778
Arc::new(crypt::Algorithm::Aes128Xts.new_cipher().unwrap()),
27732779
Some(CipherContext::default()),

0 commit comments

Comments
 (0)