Revert "Remove mirrors configuration" (#17)

This reverts commit f7d5138.

Author: ecdatadog

.github/copilot-instructions.md

@@ -74,12 +74,12 @@ pub enum NydusError {
 ## Development Guidelines
 
 ### Storage Backend Development
-- When implementing new storage backends:
-- - Implement the `BlobBackend` trait
-- - Support timeout, retry, and connection management
-- - Add configuration in the backend config structure
-- - Consider proxy support for high availability
-- - Implement proper error handling and logging  
+When implementing new storage backends:
+- Implement the `BlobBackend` trait
+- Support timeout, retry, and connection management
+- Add configuration in the backend config structure
+- Consider mirror/proxy support for high availability
+- Implement proper error handling and logging
 
 ### Daemon Service Development
 - Use the `NydusDaemon` trait for service implementations

api/src/config.rs

@@ -519,6 +519,9 @@ pub struct OssConfig {
     /// Enable HTTP proxy for the read request.
     #[serde(default)]
     pub proxy: ProxyConfig,
+    /// Enable mirrors for the read request.
+    #[serde(default)]
+    pub mirrors: Vec<MirrorConfig>,
 }
 
 /// S3 configuration information to access blobs.
@@ -560,6 +563,9 @@ pub struct S3Config {
     /// Enable HTTP proxy for the read request.
     #[serde(default)]
     pub proxy: ProxyConfig,
+    /// Enable mirrors for the read request.
+    #[serde(default)]
+    pub mirrors: Vec<MirrorConfig>,
 }
 
 /// Http proxy configuration information to access blobs.
@@ -586,6 +592,9 @@ pub struct HttpProxyConfig {
     /// Enable HTTP proxy for the read request.
     #[serde(default)]
     pub proxy: ProxyConfig,
+    /// Enable mirrors for the read request.
+    #[serde(default)]
+    pub mirrors: Vec<MirrorConfig>,
 }
 
 /// Container registry configuration information to access blobs.
@@ -629,6 +638,9 @@ pub struct RegistryConfig {
     /// Disable background token refresh thread. Defaults to false.
     #[serde(skip_deserializing)]
     pub disable_token_refresh: bool,
+    /// Enable mirrors for the read request.
+    #[serde(default)]
+    pub mirrors: Vec<MirrorConfig>,
 }
 
 /// Configuration information for blob cache manager.
@@ -921,6 +933,41 @@ impl Default for ProxyConfig {
     }
 }
 
+/// Configuration for registry mirror.
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct MirrorConfig {
+    /// Mirror server URL, for example http://127.0.0.1:65001.
+    pub host: String,
+    /// Ping URL to check mirror server health.
+    #[serde(default)]
+    pub ping_url: String,
+    /// HTTP request headers to be passed to mirror server.
+    #[serde(default)]
+    pub headers: HashMap<String, String>,
+    /// Interval for mirror health checking, in seconds.
+    #[serde(default = "default_check_interval")]
+    pub health_check_interval: u64,
+    /// Maximum number of failures before marking a mirror as unusable.
+    #[serde(default = "default_failure_limit")]
+    pub failure_limit: u8,
+    /// Elapsed time to pause mirror health check when the request is inactive, in seconds.
+    #[serde(default = "default_check_pause_elapsed")]
+    pub health_check_pause_elapsed: u64,
+}
+
+impl Default for MirrorConfig {
+    fn default() -> Self {
+        Self {
+            host: String::new(),
+            headers: HashMap::new(),
+            health_check_interval: 5,
+            failure_limit: 5,
+            ping_url: String::new(),
+            health_check_pause_elapsed: 300,
+        }
+    }
+}
+
 /// Configuration information for a cached blob`.
 #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
 pub struct BlobCacheEntryConfigV2 {
@@ -1168,6 +1215,10 @@ fn default_check_pause_elapsed() -> u64 {
     300
 }
 
+fn default_failure_limit() -> u8 {
+    5
+}
+
 fn default_work_dir() -> String {
     ".".to_string()
 }
@@ -1854,6 +1905,11 @@ mod tests {
         fallback = true
         check_interval = 10
         use_http = true
+        [[backend.oss.mirrors]]
+        host = "http://127.0.0.1:65001"
+        ping_url = "http://127.0.0.1:65001/ping"
+        health_check_interval = 10
+        failure_limit = 10
         "#;
         let config: ConfigV2 = toml::from_str(content).unwrap();
         assert_eq!(config.version, 2);
@@ -1880,6 +1936,14 @@ mod tests {
         assert_eq!(oss.proxy.check_interval, 10);
         assert!(oss.proxy.fallback);
         assert!(oss.proxy.use_http);
+
+        assert_eq!(oss.mirrors.len(), 1);
+        let mirror = &oss.mirrors[0];
+        assert_eq!(mirror.host, "http://127.0.0.1:65001");
+        assert_eq!(mirror.ping_url, "http://127.0.0.1:65001/ping");
+        assert!(mirror.headers.is_empty());
+        assert_eq!(mirror.health_check_interval, 10);
+        assert_eq!(mirror.failure_limit, 10);
     }
 
     #[test]
@@ -1905,6 +1969,11 @@ mod tests {
         fallback = true
         check_interval = 10
         use_http = true
+        [[backend.registry.mirrors]]
+        host = "http://127.0.0.1:65001"
+        ping_url = "http://127.0.0.1:65001/ping"
+        health_check_interval = 10
+        failure_limit = 10
         "#;
         let config: ConfigV2 = toml::from_str(content).unwrap();
         assert_eq!(config.version, 2);
@@ -1933,6 +2002,14 @@ mod tests {
         assert_eq!(registry.proxy.check_interval, 10);
         assert!(registry.proxy.fallback);
         assert!(registry.proxy.use_http);
+
+        assert_eq!(registry.mirrors.len(), 1);
+        let mirror = &registry.mirrors[0];
+        assert_eq!(mirror.host, "http://127.0.0.1:65001");
+        assert_eq!(mirror.ping_url, "http://127.0.0.1:65001/ping");
+        assert!(mirror.headers.is_empty());
+        assert_eq!(mirror.health_check_interval, 10);
+        assert_eq!(mirror.failure_limit, 10);
     }
 
     #[test]
@@ -2348,6 +2425,15 @@ mod tests {
         assert!(res);
     }
 
+    #[test]
+    fn test_default_mirror_config() {
+        let cfg = MirrorConfig::default();
+        assert_eq!(cfg.host, "");
+        assert_eq!(cfg.health_check_interval, 5);
+        assert_eq!(cfg.failure_limit, 5);
+        assert_eq!(cfg.ping_url, "");
+    }
+
     #[test]
     fn test_config_v2_from_file() {
         let content = r#"version=2
@@ -2557,6 +2643,7 @@ mod tests {
     #[test]
     fn test_default_value() {
         assert!(default_true());
+        assert_eq!(default_failure_limit(), 5);
         assert_eq!(default_prefetch_batch_size(), 1024 * 1024);
         assert_eq!(default_prefetch_threads_count(), 8);
     }

docs/nydusd.md

@@ -319,8 +319,64 @@ or
   }
 }
 ```
--
-- The `HttpProxy` backend also supports the `Proxy` configuration for remote usage like the `Registry backend` described above.
+
+The `HttpProxy` backend also supports the `Proxy` and `Mirrors` configurations for remote usage like the `Registry backend` described above.
+
+##### Enable Mirrors for Storage Backend (Recommend)
+
+Nydus is deeply integrated with [Dragonfly](https://d7y.io/) P2P mirror mode, please refer the [doc](https://d7y.io/docs/next/operations/integrations/container-runtime/nydus/) to learn how configuring Nydus to use Dragonfly.
+
+Add `device.backend.config.mirrors` field to enable mirrors for storage backend. The mirror can be a P2P distribution server or registry. If the request to mirror server failed, it will fall back to the original registry.
+Currently, the mirror mode is only tested in the registry backend, and in theory, the OSS backend also supports it.
+
+<font color='red'>!!</font> The `mirrors` field conflicts with `proxy` field.
+
+```
+{
+  "device": {
+    "backend": {
+      "type": "registry",
+      "config": {
+        "mirrors": [
+          {
+            // Mirror server URL (including scheme), e.g. Dragonfly dfdaemon server URL
+            "host": "http://dragonfly1.io:65001",
+            // Headers for mirror server
+            "headers": {
+              // For Dragonfly dfdaemon server URL, we need to specify "X-Dragonfly-Registry" (including scheme).
+              // When Dragonfly does not cache data, nydusd will pull it from "X-Dragonfly-Registry".
+              // If not set "X-Dragonfly-Registry", Dragonfly will pull data from proxy.registryMirror.url.
+              "X-Dragonfly-Registry": "https://index.docker.io"
+            },
+            // This URL endpoint is used to check the health of mirror server, and if the mirror is unhealthy, 
+            // the request will fallback to the next mirror or the original registry server. 
+            // Use $host/v2 as default if left empty.
+            "ping_url": "http://127.0.0.1:40901/server/ping",
+            // Interval time (s) to check and recover unavailable mirror. Use 5 as default if left empty.
+            "health_check_interval": 5,
+            // Failure counts before disabling this mirror. Use 5 as default if left empty.
+            "failure_limit": 5,
+            // Elapsed time to pause mirror health check when the request is inactive, in seconds.
+            // Use 300 as default if left empty.
+            "health_check_pause_elapsed": 300,
+          },
+          {
+            "host": "http://dragonfly2.io:65001",
+            "headers": {
+              "X-Dragonfly-Registry": "https://index.docker.io"
+            },
+          }
+        ],
+        ...
+      }
+    },
+    ...
+  },
+  ...
+}
+```
+
+
 ##### Enable P2P Proxy for Storage Backend
 
 Add `device.backend.config.proxy` field to enable HTTP proxy for storage backend. For example, use P2P distribution service to reduce network workload and latency in large scale container cluster using [Dragonfly](https://d7y.io/) (enable centralized dfdaemon mode).

misc/configs/nydusd-blob-cache-entry-configuration-v2.toml

@@ -50,6 +50,18 @@ check_interval = 5
 # Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
 use_http = false
 
+[[backend.oss.mirrors]]
+# Mirror server URL, for example http://127.0.0.1:65001.
+host = "http://127.0.0.1:65001"
+# Ping URL to check mirror server health.
+ping_url = "http://127.0.0.1:65001/ping"
+# HTTP request headers to be passed to mirror server.
+# headers =
+# Interval for mirror health checking, in seconds.
+health_check_interval = 5
+# Maximum number of failures before marking a mirror as unusable.
+failure_limit = 5
+
 [backend.registy]
 # Registry http scheme, either 'http' or 'https'
 scheme = "https"
@@ -87,6 +99,18 @@ check_interval = 5
 # Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
 use_http = false
 
+[[backend.registry.mirrors]]
+# Mirror server URL, for example http://127.0.0.1:65001.
+host = "http://127.0.0.1:65001"
+# Ping URL to check mirror server health.
+ping_url = "http://127.0.0.1:65001/ping"
+# HTTP request headers to be passed to mirror server.
+# headers =
+# Interval for mirror health checking, in seconds.
+health_check_interval = 5
+# Maximum number of failures before marking a mirror as unusable.
+failure_limit = 5
+
 [cache]
 # Type of blob cache: "blobcache", "filecache", "fscache", "dummycache" or ""
 type = "filecache"

misc/configs/nydusd-blob-cache-entry.toml

@@ -55,6 +55,18 @@ check_interval = 5
 # Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
 use_http = false
 
+[[config_v2.backend.oss.mirrors]]
+# Mirror server URL, for example http://127.0.0.1:65001.
+host = "http://127.0.0.1:65001"
+# Ping URL to check mirror server health.
+ping_url = "http://127.0.0.1:65001/ping"
+# HTTP request headers to be passed to mirror server.
+# headers =
+# Interval for mirror health checking, in seconds.
+health_check_interval = 5
+# Maximum number of failures before marking a mirror as unusable.
+failure_limit = 5
+
 [config_v2.backend.registry]
 # Registry http scheme, either 'http' or 'https'
 scheme = "https"
@@ -92,6 +104,18 @@ check_interval = 5
 # Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
 use_http = false
 
+[[config_v2.backend.registry.mirrors]]
+# Mirror server URL, for example http://127.0.0.1:65001.
+host = "http://127.0.0.1:65001"
+# Ping URL to check mirror server health.
+ping_url = "http://127.0.0.1:65001/ping"
+# HTTP request headers to be passed to mirror server.
+# headers =
+# Interval for mirror health checking, in seconds.
+health_check_interval = 5
+# Maximum number of failures before marking a mirror as unusable.
+failure_limit = 5
+
 [config_v2.cache]
 # Type of blob cache: "blobcache", "filecache", "fscache", "dummycache" or ""
 type = "filecache"

misc/configs/nydusd-config-v2.toml

@@ -48,6 +48,18 @@ check_interval = 5
 # Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
 use_http = false
 
+[[backend.oss.mirrors]]
+# Mirror server URL, for example http://127.0.0.1:65001.
+host = "http://127.0.0.1:65001"
+# Ping URL to check mirror server health.
+ping_url = "http://127.0.0.1:65001/ping"
+# HTTP request headers to be passed to mirror server.
+# headers =
+# Interval for mirror health checking, in seconds.
+health_check_interval = 5
+# Maximum number of failures before marking a mirror as unusable.
+failure_limit = 5
+
 [backend.registy]
 # Registry http scheme, either 'http' or 'https'
 scheme = "https"
@@ -85,6 +97,17 @@ check_interval = 5
 # Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
 use_http = false
 
+[[backend.registry.mirrors]]
+# Mirror server URL, for example http://127.0.0.1:65001.
+host = "http://127.0.0.1:65001"
+# Ping URL to check mirror server health.
+ping_url = "http://127.0.0.1:65001/ping"
+# HTTP request headers to be passed to mirror server.
+# headers =
+# Interval for mirror health checking, in seconds.
+health_check_interval = 5
+# Maximum number of failures before marking a mirror as unusable.
+failure_limit = 5
 
 [cache]
 # Type of blob cache: "blobcache", "filecache", "fscache", "dummycache" or ""

misc/performance/snapshotter_config.toml

@@ -63,6 +63,12 @@ address = ":9110"
 [remote]
 convert_vpc_registry = false
 
+[remote.mirrors_config]
+# Snapshotter will overwrite daemon's mirrors configuration
+# if the values loaded from this driectory are not null before starting a daemon.
+# Set to "" or an empty directory to disable it.
+#dir = "/etc/nydus/certs.d"
+
 [remote.auth]
 # Fetch the private registry auth by listening to K8s API server
 enable_kubeconfig_keychain = false