storage: support external config merge

Signed-off-by: Yan Song <[email protected]>

Author: imeoer

Cargo.lock

@@ -93,6 +93,7 @@ default = [
     "backend-s3",
     "backend-http-proxy",
     "backend-localdisk",
+    "backend-external",
     "dedup",
 ]
 virtiofs = [
@@ -114,6 +115,7 @@ backend-localdisk = [
 backend-oss = ["nydus-storage/backend-oss"]
 backend-registry = ["nydus-storage/backend-registry"]
 backend-s3 = ["nydus-storage/backend-s3"]
+backend-external = ["nydus-storage/backend-external"]
 
 dedup = ["nydus-storage/dedup"]
 

Cargo.toml

@@ -25,6 +25,9 @@ pub struct ConfigV2 {
     pub id: String,
     /// Configuration information for storage backend.
     pub backend: Option<BackendConfigV2>,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration information for local cache system.
     pub cache: Option<CacheConfigV2>,
     /// Configuration information for RAFS filesystem.
@@ -42,6 +45,7 @@ impl Default for ConfigV2 {
             version: 2,
             id: String::new(),
             backend: None,
+            external_backends: Vec::new(),
             cache: None,
             rafs: None,
             overlay: None,
@@ -57,6 +61,7 @@ impl ConfigV2 {
             version: 2,
             id: id.to_string(),
             backend: None,
+            external_backends: Vec::new(),
             cache: None,
             rafs: None,
             overlay: None,
@@ -971,6 +976,9 @@ pub struct BlobCacheEntryConfigV2 {
     /// Configuration information for storage backend.
     #[serde(default)]
     pub backend: BackendConfigV2,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration information for local cache system.
     #[serde(default)]
     pub cache: CacheConfigV2,
@@ -1034,6 +1042,7 @@ impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
             version: c.version,
             id: c.id.clone(),
             backend: Some(c.backend.clone()),
+            external_backends: c.external_backends.clone(),
             cache: Some(c.cache.clone()),
             rafs: None,
             overlay: None,
@@ -1309,6 +1318,19 @@ struct CacheConfig {
     pub prefetch_config: BlobPrefetchConfig,
 }
 
+/// Additional configuration information for external backend, its items
+/// will be merged to the configuration from image.
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct ExternalBackendConfig {
+    /// External backend identifier to merge.
+    pub patch: HashMap<String, String>,
+    /// External backend type.
+    #[serde(rename = "type")]
+    pub kind: String,
+    /// External backend config items to merge.
+    pub config: HashMap<String, String>,
+}
+
 impl TryFrom<&CacheConfig> for CacheConfigV2 {
     type Error = std::io::Error;
 
@@ -1350,6 +1372,9 @@ struct FactoryConfig {
     pub id: String,
     /// Configuration for storage backend.
     pub backend: BackendConfig,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration for blob cache manager.
     #[serde(default)]
     pub cache: CacheConfig,
@@ -1410,6 +1435,7 @@ impl TryFrom<RafsConfig> for ConfigV2 {
             version: 2,
             id: v.device.id,
             backend: Some(backend),
+            external_backends: v.device.external_backends,
             cache: Some(cache),
             rafs: Some(rafs),
             overlay: None,
@@ -1500,6 +1526,9 @@ pub(crate) struct BlobCacheEntryConfig {
     ///
     /// Possible value: `LocalFsConfig`, `RegistryConfig`, `OssConfig`, `LocalDiskConfig`.
     backend_config: Value,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    external_backends: Vec<ExternalBackendConfig>,
     /// Type of blob cache, corresponding to `FactoryConfig::CacheConfig::cache_type`.
     ///
     /// Possible value: "fscache", "filecache".
@@ -1535,6 +1564,7 @@ impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
             version: 2,
             id: v.id.clone(),
             backend: (&backend_config).try_into()?,
+            external_backends: v.external_backends.clone(),
             cache: (&cache_config).try_into()?,
             metadata_path: v.metadata_path.clone(),
         })

api/src/config.rs

@@ -1626,6 +1626,7 @@ mod tests {
                 registry: None,
                 http_proxy: None,
             }),
+            external_backends: Vec::new(),
             id: "id".to_owned(),
             cache: None,
             rafs: None,

builder/src/core/context.rs

@@ -26,3 +26,4 @@ backend-oss = ["nydus-storage/backend-oss"]
 backend-registry = ["nydus-storage/backend-registry"]
 backend-http-proxy = ["nydus-storage/backend-http-proxy"]
 backend-localdisk = ["nydus-storage/backend-localdisk"]
+backend-external = ["nydus-storage/backend-external"]

clib/Cargo.toml

@@ -44,6 +44,8 @@ url = { version = "2.1.1", optional = true }
 vm-memory = "0.14.1"
 fuse-backend-rs = "^0.12.0"
 gpt = { version = "3.1.0", optional = true }
+chrono = { version = "0.4", features = ["serde"] }
+serde_bytes = "0.11"
 
 nydus-api = { version = "0.3", path = "../api" }
 nydus-utils = { version = "0.4", path = "../utils", features = [
@@ -66,6 +68,7 @@ backend-oss = ["base64", "httpdate", "hmac", "sha1", "reqwest", "url"]
 backend-registry = ["base64", "reqwest", "url"]
 backend-s3 = ["base64", "hmac", "http", "reqwest", "sha2", "time", "url"]
 backend-http-proxy = ["hyper", "hyperlocal", "http", "reqwest", "url"]
+backend-external = ["base64", "reqwest", "url"]
 dedup = ["rusqlite", "r2d2", "r2d2_sqlite"]
 prefetch-rate-limit = ["leaky-bucket"]
 

storage/Cargo.toml

@@ -74,6 +74,7 @@ pub(crate) struct ConnectionConfig {
     pub timeout: u32,
     pub connect_timeout: u32,
     pub retry_limit: u8,
+    pub redirect: bool,
 }
 
 impl Default for ConnectionConfig {
@@ -85,6 +86,7 @@ impl Default for ConnectionConfig {
             timeout: 5,
             connect_timeout: 5,
             retry_limit: 0,
+            redirect: false,
         }
     }
 }
@@ -98,6 +100,7 @@ impl From<OssConfig> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -111,6 +114,7 @@ impl From<S3Config> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -124,6 +128,7 @@ impl From<RegistryConfig> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -137,6 +142,7 @@ impl From<HttpProxyConfig> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -651,8 +657,12 @@ impl Connection {
 
         let mut cb = Client::builder()
             .timeout(timeout)
-            .connect_timeout(connect_timeout)
-            .redirect(Policy::none());
+            .connect_timeout(connect_timeout);
+        if config.redirect {
+            cb = cb.redirect(Policy::default());
+        } else {
+            cb = cb.redirect(Policy::none());
+        }
 
         if config.skip_verify {
             cb = cb.danger_accept_invalid_certs(true);

storage/src/backend/connection.rs

@@ -3,9 +3,9 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::collections::HashMap;
-use std::io::Result;
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
+use std::result::Result;
 
 use crate::backend::external::{meta::MetaMap, ExternalBlobReader};
 use crate::device::BlobChunkInfo;
@@ -24,25 +24,33 @@ pub struct LocalBackend {
 }
 
 impl LocalBackend {
-    pub fn new(meta_path: PathBuf, config: &HashMap<String, String>) -> Result<Self> {
-        let meta_map = MetaMap::new(meta_path)?;
-        let root = PathBuf::from(config.get("root").unwrap());
+    pub fn new(meta_path: PathBuf, config: &HashMap<String, String>) -> Result<Self, String> {
+        let meta_map = MetaMap::new(meta_path).map_err(|e| e.to_string())?;
+        let root = PathBuf::from(
+            config
+                .get("root")
+                .ok_or_else(|| format!("root is not specified in local backend config"))?,
+        );
         Ok(Self { meta_map, root })
     }
 }
 
 impl ExternalBlobReader for LocalBackend {
-    fn read(&self, buf: &mut [u8], chunks: &[&dyn BlobChunkInfo]) -> Result<usize> {
+    fn read(&self, buf: &mut [u8], chunks: &[&dyn BlobChunkInfo]) -> Result<usize, String> {
         let chunk_index = chunks[0].id();
-        let (object_bytes, chunk) = self.meta_map.get_object(chunk_index)?;
+        let (object_bytes, chunk) = self
+            .meta_map
+            .get_object(chunk_index)
+            .map_err(|e| e.to_string())?;
 
         let object: Object = rmp_serde::from_slice(&object_bytes)
-            .map_err(|_e| einval!("failed to deserialize object"))?;
+            .map_err(|e| format!("failed to deserialize object: {}", e.to_string()))?;
 
         let path = self.root.join(&object.path);
 
-        let file = std::fs::File::open(path)?;
-        file.read_exact_at(buf, chunk.object_offset)?;
+        let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
+        file.read_exact_at(buf, chunk.object_offset)
+            .map_err(|e| e.to_string())?;
 
         Ok(buf.len())
     }

storage/src/backend/external/local.rs

@@ -7,8 +7,9 @@ use std::mem::size_of;
 use std::path::Path;
 use std::{fs::File, os::unix::fs::MetadataExt};
 
+use serde::{Serialize, Deserialize};
+
 use nydus_utils::filemap::FileMapState;
-use serde::{Deserialize, Serialize};
 
 // Layout
 //
@@ -92,7 +93,7 @@ impl MetaMap {
         let object_meta_offset = header.object_meta_offset;
 
         let chunk_meta = self.map.get_ref::<ChunkMeta>(chunk_meta_offset as usize)?;
-        let object_meta = self
+        let _object_meta = self
             .map
             .get_ref::<ObjectMeta>(object_meta_offset as usize)?;
 
@@ -102,16 +103,20 @@ impl MetaMap {
                 + chunk_index as usize * chunk_meta.entry_size as usize,
         )?;
         let object_index = chunk.object_index;
-        let object_offset = if object_meta.entry_size == 0 {
-            let object_offset_offset = object_meta_offset as usize
-                + size_of::<ObjectMeta>()
-                + object_index as usize * size_of::<ObjectOffset>();
-            *self.map.get_ref::<ObjectOffset>(object_offset_offset)? as usize
-        } else {
-            object_meta_offset as usize
-                + size_of::<ObjectMeta>()
-                + object_index as usize * object_meta.entry_size as usize
-        };
+        // let object_offset_offset = if object_meta.entry_size == 0 {
+        //     object_meta_offset as usize
+        //         + size_of::<ObjectMeta>()
+        //         + object_index as usize * size_of::<ObjectOffset>()
+        // } else {
+        //     object_meta_offset as usize
+        //         + size_of::<ObjectMeta>()
+        //         + object_index as usize
+        //             * (size_of::<u32>() as usize + object_meta.entry_size as usize) as usize
+        // };
+        let object_offset_offset = object_meta_offset as usize
+            + size_of::<ObjectMeta>()
+            + object_index as usize * size_of::<ObjectOffset>();
+        let object_offset = *self.map.get_ref::<ObjectOffset>(object_offset_offset)? as usize;
 
         let object_size = *self.map.get_ref::<u32>(object_offset)? as usize;