feat: support for Tencent Cloud Object Storage

Cargo.toml

@@ -165,8 +165,10 @@ all = [
   "gha",
   "webdav",
   "oss",
+  "cos",
 ]
 azure = ["opendal/services-azblob", "reqsign", "reqwest"]
+cos = ["opendal/services-cos", "reqsign", "reqwest"]
 default = ["all"]
 gcs = ["opendal/services-gcs", "reqsign", "url", "reqwest"]
 gha = ["opendal/services-ghac", "reqwest"]

README.md

@@ -43,6 +43,7 @@ Table of Contents (ToC)
   * [GitHub Actions](docs/GHA.md)
   * [WebDAV (Ccache/Bazel/Gradle compatible)](docs/Webdav.md)
   * [Alibaba OSS](docs/OSS.md)
+  * [Tencent Cloud Object Storage](docs/COS.md)
 
 ---
 
@@ -238,7 +239,7 @@ Build
 If you are building sccache for non-development purposes make sure you use `cargo build --release` to get optimized binaries:
 
 ```bash
-cargo build --release [--no-default-features --features=s3|redis|gcs|memcached|azure|gha|webdav|oss]
+cargo build --release [--no-default-features --features=s3|redis|gcs|memcached|azure|gha|webdav|oss|cos]
 ```
 
 The list of features can be found in the `Cargo.toml` file, `[features]` section.
@@ -351,3 +352,4 @@ Storage Options
 * [GitHub Actions](docs/GHA.md)
 * [WebDAV (Ccache/Bazel/Gradle compatible)](docs/Webdav.md)
 * [Alibaba OSS](docs/OSS.md)
+* [Tencent Cloud Object Storage](docs/COS.md)

docs/COS.md

@@ -0,0 +1,11 @@
+# COS
+
+If you want to use _Tencent Cloud Object Storage_ (aka COS) for the sccache cache, you need to set the `SCCACHE_COS_BUCKET` environment variable to the name of the COS bucket to use.
+
+You **must** specify the endpoint URL using the `SCCACHE_COS_ENDPOINT` environment variable. More details are at [COS endpoints](https://www.tencentcloud.com/document/product/436/6224).
+
+You can also define a prefix that will be prepended to the keys of all cache objects created and read within the COS bucket, effectively creating a scope. To do that use the `SCCACHE_COS_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application.
+
+## Credentials
+
+Sccache is able to load credentials from environment variables: `TENCENTCLOUD_SECRET_ID` and `TENCENTCLOUD_SECRET_KEY`. More details about the access of COS bucket can be found at the [introduction page](https://www.tencentcloud.com/document/product/436/7751).

docs/Configuration.md

@@ -112,6 +112,11 @@ bucket = "name"
 endpoint = "oss-us-east-1.aliyuncs.com"
 key_prefix = "ossprefix"
 no_credentials = true
+
+[cache.cos]
+bucket = "name"
+endpoint = "cos.na-siliconvalley.myqcloud.com"
+key_prefix = "cosprefix"
 ```
 
 sccache looks for its configuration file at the path indicated by env variable `SCCACHE_CONF`.
@@ -234,3 +239,11 @@ The full url appears then as `redis://user:[email protected]:6379/?db=1`.
 * `ALIBABA_CLOUD_ACCESS_KEY_ID`
 * `ALIBABA_CLOUD_ACCESS_KEY_SECRET`
 * `SCCACHE_OSS_NO_CREDENTIALS`
+
+#### Tencent Cloud Object Storage (COS)
+
+* `SCCACHE_COS_BUCKET`
+* `SCCACHE_COS_ENDPOINT`
+* `SCCACHE_COS_KEY_PREFIX`
+* `TENCENTCLOUD_SECRET_ID`
+* `TENCENTCLOUD_SECRET_KEY`

src/cache/cache.rs

@@ -14,6 +14,8 @@
 
 #[cfg(feature = "azure")]
 use crate::cache::azure::AzureBlobCache;
+#[cfg(feature = "cos")]
+use crate::cache::cos::COSCache;
 use crate::cache::disk::DiskCache;
 #[cfg(feature = "gcs")]
 use crate::cache::gcs::GCSCache;
@@ -39,7 +41,8 @@ use crate::config::Config;
     feature = "redis",
     feature = "s3",
     feature = "webdav",
-    feature = "oss"
+    feature = "oss",
+    feature = "cos"
 ))]
 use crate::config::{self, CacheType};
 use async_trait::async_trait;
@@ -725,6 +728,18 @@ pub fn storage_from_config(
 
                 return Ok(Arc::new(storage));
             }
+            #[cfg(feature = "cos")]
+            CacheType::COS(c) => {
+                debug!(
+                    "Init cos cache with bucket {}, endpoint {:?}",
+                    c.bucket, c.endpoint
+                );
+
+                let storage = COSCache::build(&c.bucket, &c.key_prefix, c.endpoint.as_deref())
+                    .map_err(|err| anyhow!("create cos cache failed: {err:?}"))?;
+
+                return Ok(Arc::new(storage));
+            }
             #[allow(unreachable_patterns)]
             // if we build only with `cargo build --no-default-features`
             // we only want to use sccache with a local cache (no remote storage)

src/cache/cos.rs

@@ -0,0 +1,38 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use opendal::Operator;
+use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::services::Cos;
+
+use crate::errors::*;
+
+use super::http_client::set_user_agent;
+
+pub struct COSCache;
+
+// Implement for Tencent Cloud Object Storage
+impl COSCache {
+    pub fn build(bucket: &str, key_prefix: &str, endpoint: Option<&str>) -> Result<Operator> {
+        let mut builder = Cos::default().bucket(bucket).root(key_prefix);
+
+        if let Some(endpoint) = endpoint {
+            builder = builder.endpoint(endpoint);
+        }
+
+        let op = Operator::new(builder)?
+            .layer(HttpClientLayer::new(set_user_agent()))
+            .layer(LoggingLayer::default())
+            .finish();
+        Ok(op)
+    }
+}

src/cache/mod.rs

@@ -16,6 +16,8 @@
 pub mod azure;
 #[allow(clippy::module_inception)]
 pub mod cache;
+#[cfg(feature = "cos")]
+pub mod cos;
 pub mod disk;
 #[cfg(feature = "gcs")]
 pub mod gcs;
@@ -39,7 +41,8 @@ pub mod webdav;
     feature = "gha",
     feature = "s3",
     feature = "webdav",
-    feature = "oss"
+    feature = "oss",
+    feature = "cos"
 ))]
 pub(crate) mod http_client;
 

src/cmdline.rs

@@ -120,6 +120,9 @@ fn get_clap_command() -> clap::Command {
             "\n",
             "    OSS:       ",
             cfg!(feature = "oss"),
+            "\n",
+            "    COS:       ",
+            cfg!(feature = "cos"),
             "\n"
         ))
         .args(&[

src/config.rs

@@ -361,6 +361,15 @@ pub struct OSSCacheConfig {
     pub no_credentials: bool,
 }
 
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct COSCacheConfig {
+    pub bucket: String,
+    #[serde(default)]
+    pub key_prefix: String,
+    pub endpoint: Option<String>,
+}
+
 #[derive(Debug, PartialEq, Eq)]
 pub enum CacheType {
     Azure(AzureCacheConfig),
@@ -371,6 +380,7 @@ pub enum CacheType {
     S3(S3CacheConfig),
     Webdav(WebdavCacheConfig),
     OSS(OSSCacheConfig),
+    COS(COSCacheConfig),
 }
 
 #[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
@@ -385,6 +395,7 @@ pub struct CacheConfigs {
     pub s3: Option<S3CacheConfig>,
     pub webdav: Option<WebdavCacheConfig>,
     pub oss: Option<OSSCacheConfig>,
+    pub cos: Option<COSCacheConfig>,
 }
 
 impl CacheConfigs {
@@ -401,6 +412,7 @@ impl CacheConfigs {
             s3,
             webdav,
             oss,
+            cos,
         } = self;
 
         let cache_type = s3
@@ -411,7 +423,8 @@ impl CacheConfigs {
             .or_else(|| gha.map(CacheType::GHA))
             .or_else(|| azure.map(CacheType::Azure))
             .or_else(|| webdav.map(CacheType::Webdav))
-            .or_else(|| oss.map(CacheType::OSS));
+            .or_else(|| oss.map(CacheType::OSS))
+            .or_else(|| cos.map(CacheType::COS));
 
         let fallback = disk.unwrap_or_default();
 
@@ -430,6 +443,7 @@ impl CacheConfigs {
             s3,
             webdav,
             oss,
+            cos,
         } = other;
 
         if azure.is_some() {
@@ -456,10 +470,12 @@ impl CacheConfigs {
         if webdav.is_some() {
             self.webdav = webdav;
         }
-
         if oss.is_some() {
             self.oss = oss;
         }
+        if cos.is_some() {
+            self.cos = cos;
+        }
     }
 }
 
@@ -892,6 +908,20 @@ fn config_from_env() -> Result<EnvConfig> {
         bail!("If setting OSS credentials, SCCACHE_OSS_NO_CREDENTIALS must not be set.");
     }
 
+    // ======= COS =======
+    let cos = if let Ok(bucket) = env::var("SCCACHE_COS_BUCKET") {
+        let endpoint = env::var("SCCACHE_COS_ENDPOINT").ok();
+        let key_prefix = key_prefix_from_env_var("SCCACHE_COS_KEY_PREFIX");
+
+        Some(COSCacheConfig {
+            bucket,
+            endpoint,
+            key_prefix,
+        })
+    } else {
+        None
+    };
+
     // ======= Local =======
     let disk_dir = env::var_os("SCCACHE_DIR").map(PathBuf::from);
     let disk_sz = env::var("SCCACHE_CACHE_SIZE")
@@ -944,6 +974,7 @@ fn config_from_env() -> Result<EnvConfig> {
         s3,
         webdav,
         oss,
+        cos,
     };
 
     Ok(EnvConfig { cache })
@@ -1559,6 +1590,11 @@ bucket = "name"
 endpoint = "oss-us-east-1.aliyuncs.com"
 key_prefix = "ossprefix"
 no_credentials = true
+
+[cache.cos]
+bucket = "name"
+endpoint = "cos.na-siliconvalley.myqcloud.com"
+key_prefix = "cosprefix"
 "#;
 
     let file_config: FileConfig = toml::from_str(CONFIG_STR).expect("Is valid toml.");
@@ -1625,6 +1661,11 @@ no_credentials = true
                     key_prefix: "ossprefix".into(),
                     no_credentials: true,
                 }),
+                cos: Some(COSCacheConfig {
+                    bucket: "name".to_owned(),
+                    endpoint: Some("cos.na-siliconvalley.myqcloud.com".to_owned()),
+                    key_prefix: "cosprefix".into(),
+                }),
             },
             dist: DistConfig {
                 auth: DistAuth::Token {

tests/harness/mod.rs

@@ -180,6 +180,7 @@ pub fn sccache_client_cfg(
             s3: None,
             webdav: None,
             oss: None,
+            cos: None,
         },
         dist: sccache::config::DistConfig {
             auth: Default::default(), // dangerously_insecure