feat: builder: add backend configure support for optimize subcommand

With backend configure supoorted, we can read chunk on demand by
specifying backend such as registry without downloadling whole image
during image building.

Signed-off-by: maxing.lan <[email protected]>

Author: maxing.lan

builder/src/optimize_prefetch.rs

@@ -20,14 +20,13 @@ use nydus_api::ConfigV2;
 use nydus_rafs::metadata::layout::RafsBlobTable;
 use nydus_rafs::metadata::RafsSuper;
 use nydus_rafs::metadata::RafsVersion;
+use nydus_storage::backend::BlobBackend;
 use nydus_storage::device::BlobInfo;
 use nydus_storage::meta::BatchContextGenerator;
 use nydus_storage::meta::BlobChunkInfoV2Ondisk;
 use nydus_utils::compress;
 use sha2::Digest;
 use std::cmp::{max, min};
-use std::fs::File;
-use std::io::{Read, Seek, Write};
 use std::mem::size_of;
 use std::sync::Arc;
 pub struct OptimizePrefetch {}
@@ -94,7 +93,7 @@ impl PrefetchFileInfo {
 }
 
 impl PrefetchBlobState {
-    fn new(ctx: &BuildContext, blob_layer_num: u32, blobs_dir_path: &Path) -> Result<Self> {
+    fn new(ctx: &BuildContext, blob_layer_num: u32, output_blob_dir_path: &Path) -> Result<Self> {
         let mut blob_info = BlobInfo::new(
             blob_layer_num,
             String::from("prefetch-blob"),
@@ -109,7 +108,7 @@ impl PrefetchBlobState {
         let mut blob_ctx = BlobContext::from(ctx, &blob_info, ChunkSource::Build)?;
         blob_ctx.blob_meta_info_enabled = true;
         let blob_writer = ArtifactWriter::new(crate::ArtifactStorage::FileDir(
-            blobs_dir_path.to_path_buf(),
+            output_blob_dir_path.to_path_buf(),
         ))
         .map(|writer| Box::new(writer) as Box<dyn Artifact>)?;
         Ok(Self {
@@ -127,16 +126,18 @@ impl OptimizePrefetch {
         ctx: &mut BuildContext,
         bootstrap_mgr: &mut BootstrapManager,
         blob_table: &mut RafsBlobTable,
-        blobs_dir_path: PathBuf,
+        output_blob_dir_path: PathBuf,
         prefetch_files: Vec<PrefetchFileInfo>,
+        backend: Arc<dyn BlobBackend + Send + Sync>,
     ) -> Result<BuildOutput> {
         // create a new blob for prefetch layer
 
         let blob_layer_num = match blob_table {
             RafsBlobTable::V5(table) => table.get_all().len(),
             RafsBlobTable::V6(table) => table.get_all().len(),
         };
-        let mut blob_state = PrefetchBlobState::new(&ctx, blob_layer_num as u32, &blobs_dir_path)?;
+        let mut blob_state =
+            PrefetchBlobState::new(&ctx, blob_layer_num as u32, &output_blob_dir_path)?;
         let mut batch = BatchContextGenerator::new(0)?;
         for node in prefetch_files.clone() {
             Self::process_prefetch_node(
@@ -145,7 +146,7 @@ impl OptimizePrefetch {
                 &mut blob_state,
                 &mut batch,
                 blob_table,
-                &blobs_dir_path,
+                backend.clone(),
             )?;
         }
 
@@ -286,7 +287,7 @@ impl OptimizePrefetch {
         prefetch_state: &mut PrefetchBlobState,
         batch: &mut BatchContextGenerator,
         blob_table: &RafsBlobTable,
-        blobs_dir_path: &Path,
+        backend: Arc<dyn BlobBackend + Send + Sync>,
     ) -> Result<()> {
         let file = prefetch_file_info.file.clone();
         if tree.get_node_mut(&file).is_none() {
@@ -329,13 +330,17 @@ impl OptimizePrefetch {
                 .get(chunk.inner.blob_index() as usize)
                 .map(|entry| entry.blob_id())
                 .ok_or(anyhow!("failed to get blob id"))?;
-            let mut blob_file = Arc::new(File::open(blobs_dir_path.join(blob_id))?);
 
             let inner = Arc::make_mut(&mut chunk.inner);
 
+            let reader = backend
+                .clone()
+                .get_reader(&blob_id.clone())
+                .expect("get blob err");
             let mut buf = vec![0u8; inner.compressed_size() as usize];
-            blob_file.seek(std::io::SeekFrom::Start(inner.compressed_offset()))?;
-            blob_file.read_exact(&mut buf)?;
+            reader
+                .read(&mut buf, inner.compressed_offset())
+                .expect("read blob err");
             prefetch_state.blob_writer.write_all(&buf)?;
             inner.set_blob_index(blob_info.blob_index());
             if blob_ctx.chunk_count == u32::MAX {

src/bin/nydus-image/main.rs

@@ -555,6 +555,32 @@ fn prepare_cmd_args(bti_string: &'static str) -> App {
                     .num_args(1),
             )
             .arg(arg_config.clone())
+            .arg(
+                Arg::new("backend-type")
+                    .long("backend-type")
+                    .help(format!(
+                        "Type of backend [possible values: {}]",
+                        BlobFactory::supported_backends()
+                            .into_iter()
+                            .filter(|x| x != "localfs")
+                            .collect::<Vec<_>>()
+                            .join(", ")
+                    ))
+                    .required(false)
+            )
+            .arg(
+                Arg::new("backend-config")
+                    .long("backend-config")
+                    .help("Config string of backend")
+                    .required(false),
+            )
+            .arg(
+                Arg::new("backend-config-file")
+                    .long("backend-config-file")
+                    .help("Config file of backend")
+                    .conflicts_with("backend-config")
+                    .required(false),
+            )
             .arg(
                 Arg::new("blob-dir")
                     .long("blob-dir")
@@ -570,6 +596,11 @@ fn prepare_cmd_args(bti_string: &'static str) -> App {
                     .short('O')
                     .help("Output path of optimized bootstrap"),
             )
+            .arg(
+                Arg::new("output-blob-dir")
+                    .long("output-blob-dir")
+                    .help("Directroy path for storing optimized blob"),
+            )
             .arg(
                 arg_output_json.clone(),
             )
@@ -1683,7 +1714,7 @@ impl Command {
     }
 
     fn optimize(matches: &ArgMatches, build_info: &BuildTimeInfo) -> Result<()> {
-        let blobs_dir_path = Self::get_blobs_dir(matches)?;
+        let output_blob_dir_path = Self::get_output_blob_dir(matches)?;
         let prefetch_file = Self::get_prefetch_files(matches)?;
         let bootstrap_path = Self::get_bootstrap(matches)?;
         let dst_bootstrap = match matches.get_one::<String>("output-bootstrap") {
@@ -1699,6 +1730,13 @@ impl Command {
             ..Default::default()
         };
 
+        let (_c, backend) = match Self::get_backend(matches, "optimizer") {
+            Ok((c, b)) => (c, b),
+            Err(e) => {
+                bail!("{}, --blob-dir or --backend-type must be specified", e);
+            }
+        };
+
         let sb = update_ctx_from_bootstrap(&mut build_ctx, config, bootstrap_path)?;
         let mut tree = Tree::from_bootstrap(&sb, &mut ())?;
         let mut bootstrap_mgr = BootstrapManager::new(Some(dst_bootstrap), None);
@@ -1718,8 +1756,9 @@ impl Command {
             &mut build_ctx,
             &mut bootstrap_mgr,
             &mut blob_table,
-            blobs_dir_path.to_path_buf(),
+            output_blob_dir_path.to_path_buf(),
             prefetch_nodes,
+            backend,
         )
         .with_context(|| "Failed to generate prefetch bootstrap")?;
 
@@ -1832,10 +1871,10 @@ impl Command {
         }
     }
 
-    fn get_blobs_dir(matches: &ArgMatches) -> Result<&Path> {
-        match matches.get_one::<String>("blob-dir") {
+    fn get_output_blob_dir(matches: &ArgMatches) -> Result<&Path> {
+        match matches.get_one::<String>("output-blob-dir") {
             Some(s) => Ok(Path::new(s)),
-            None => bail!("missing parameter `blob-dir`"),
+            None => bail!("missing parameter `output-blob-dir`"),
         }
     }