Skip to content

Commit a310952

Browse files
prakad1xroot on di2inpun4749lv0
authored and
root on di2inpun4749lv0
committed
Add ONTAP S3 Store with existence cache
Implements a NetApp ONTAP S3 compatible backend with an existence cache for better performance when checking if objects exist. This provides a specialized S3 implementation for NetApp ONTAP system with optimized handling for Content Addressable Storage. Signed-off-by: Kadam (EXT), Prajwal v08wha <[email protected]>
1 parent c72adee commit a310952

10 files changed

+2754
-0
lines changed

Cargo.lock

+3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

nativelink-config/src/stores.rs

+89
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,59 @@ pub enum StoreSpec {
7878
///
7979
experimental_s3_store(S3Spec),
8080

81+
/// `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend
82+
/// to store files. This store is specifically configured for ONTAP's S3 requirements
83+
/// including custom TLS configuration, credentials management, and proper vserver
84+
/// configuration.
85+
///
86+
/// This store uses AWS environment variables for credentials:
87+
/// - `AWS_ACCESS_KEY_ID`
88+
/// - `AWS_SECRET_ACCESS_KEY`
89+
/// - `AWS_DEFAULT_REGION`
90+
///
91+
/// Example JSON Config:
92+
/// ```json
93+
/// "ontap_s3_store": {
94+
/// "endpoint": "https://ontap-s3-endpoint:443",
95+
/// "vserver_name": "your-vserver",
96+
/// "bucket": "your-bucket",
97+
/// "root_certificates": "/path/to/certs.pem", // Optional
98+
/// "key_prefix": "test-prefix/", // Optional
99+
/// "retry": {
100+
/// "max_retries": 6,
101+
/// "delay": 0.3,
102+
/// "jitter": 0.5
103+
/// },
104+
/// "multipart_max_concurrent_uploads": 10
105+
/// }
106+
/// ```
107+
ontap_s3_store(OntapS3Spec),
108+
109+
/// ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store
110+
/// to optimize repeated existence checks. It maintains an in-memory cache of object
111+
/// digests and periodically syncs this cache to disk for persistence.
112+
///
113+
/// The cache helps reduce latency for repeated calls to check object existence,
114+
/// while still ensuring eventual consistency with the underlying ONTAP S3 store.
115+
///
116+
/// Example JSON Config:
117+
/// ```json
118+
/// "ontap_s3_existence_cache": {
119+
/// "index_path": "/path/to/cache/index.json",
120+
/// "sync_interval_seconds": 300,
121+
/// "backend": {
122+
/// "ontap_s3_store": {
123+
/// "endpoint": "https://ontap-s3-endpoint:443",
124+
/// "vserver_name": "your-vserver",
125+
/// "bucket": "your-bucket",
126+
/// "key_prefix": "test-prefix/"
127+
/// }
128+
/// }
129+
/// }
130+
/// ```
131+
///
132+
ontap_s3_existence_cache(Box<OntapS3ExistenceCacheSpec>),
133+
81134
/// Verify store is used to apply verifications to an underlying
82135
/// store implementation. It is strongly encouraged to validate
83136
/// as much data as you can before accepting data from a client,
@@ -512,6 +565,42 @@ pub struct FilesystemSpec {
512565
pub block_size: u64,
513566
}
514567

568+
// NetApp ONTAP S3 Spec
569+
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
570+
#[serde(deny_unknown_fields)]
571+
pub struct OntapS3Spec {
572+
#[serde(deserialize_with = "convert_string_with_shellexpand")]
573+
pub endpoint: String,
574+
#[serde(deserialize_with = "convert_string_with_shellexpand")]
575+
pub vserver_name: String,
576+
#[serde(deserialize_with = "convert_string_with_shellexpand")]
577+
pub bucket: String,
578+
#[serde(default)]
579+
pub root_certificates: Option<String>,
580+
#[serde(default)]
581+
pub key_prefix: Option<String>,
582+
#[serde(default)]
583+
pub retry: Retry,
584+
#[serde(default, deserialize_with = "convert_duration_with_shellexpand")]
585+
pub consider_expired_after_s: u32,
586+
pub max_retry_buffer_per_request: Option<usize>,
587+
pub multipart_max_concurrent_uploads: Option<usize>,
588+
#[serde(default)]
589+
pub insecure_allow_http: bool,
590+
#[serde(default)]
591+
pub disable_http2: bool,
592+
}
593+
594+
#[derive(Serialize, Deserialize, Debug, Clone)]
595+
#[serde(deny_unknown_fields)]
596+
pub struct OntapS3ExistenceCacheSpec {
597+
#[serde(deserialize_with = "convert_string_with_shellexpand")]
598+
pub index_path: String,
599+
#[serde(deserialize_with = "convert_numeric_with_shellexpand")]
600+
pub sync_interval_seconds: u32,
601+
pub backend: Box<StoreSpec>,
602+
}
603+
515604
#[derive(Serialize, Deserialize, Debug, Clone)]
516605
#[serde(deny_unknown_fields)]
517606
pub struct FastSlowSpec {

nativelink-store/BUILD.bazel

+10
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ rust_library(
2222
"src/lib.rs",
2323
"src/memory_store.rs",
2424
"src/noop_store.rs",
25+
"src/ontap_s3_existence_cache_store.rs",
26+
"src/ontap_s3_store.rs",
2527
"src/redis_store.rs",
2628
"src/redis_utils/ft_aggregate.rs",
2729
"src/redis_utils/mod.rs",
@@ -53,18 +55,23 @@ rust_library(
5355
"@crates//:bytes",
5456
"@crates//:bytes-utils",
5557
"@crates//:const_format",
58+
"@crates//:filetime",
5659
"@crates//:fred",
5760
"@crates//:futures",
5861
"@crates//:hex",
5962
"@crates//:http-body",
6063
"@crates//:hyper-0.14.32",
6164
"@crates//:hyper-rustls",
65+
"@crates//:hyper-util",
6266
"@crates//:lz4_flex",
6367
"@crates//:parking_lot",
6468
"@crates//:patricia_tree",
6569
"@crates//:prost",
6670
"@crates//:rand",
71+
"@crates//:rustls",
72+
"@crates//:rustls-pemfile",
6773
"@crates//:serde",
74+
"@crates//:serde_json",
6875
"@crates//:tokio",
6976
"@crates//:tokio-stream",
7077
"@crates//:tokio-util",
@@ -86,6 +93,8 @@ rust_test_suite(
8693
"tests/fast_slow_store_test.rs",
8794
"tests/filesystem_store_test.rs",
8895
"tests/memory_store_test.rs",
96+
"tests/ontap_s3_existence_cache_store_test.rs",
97+
"tests/ontap_s3_store_test.rs",
8998
"tests/redis_store_test.rs",
9099
"tests/ref_store_test.rs",
91100
"tests/s3_store_test.rs",
@@ -126,6 +135,7 @@ rust_test_suite(
126135
"@crates//:serde_json",
127136
"@crates//:serial_test",
128137
"@crates//:sha2",
138+
"@crates//:tempfile",
129139
"@crates//:tokio",
130140
"@crates//:tokio-stream",
131141
"@crates//:tracing",

nativelink-store/Cargo.toml

+4
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,10 @@ lz4_flex = { version = "0.11.3", default-features = false }
5454
parking_lot = "0.12.3"
5555
prost = { version = "0.13.5", default-features = false }
5656
rand = { version = "0.9.0", default-features = false, features = ["thread_rng"] }
57+
rustls = { version = "0.21.12", default-features = false, features = ["dangerous_configuration",] }
58+
rustls-pemfile = { version = "2.1.2", default-features = false }
5759
serde = { version = "1.0.218", default-features = false }
60+
serde_json = { version = "1.0.135", default-features = false }
5861
tokio = { version = "1.43.0", features = ["fs", "rt-multi-thread", "signal", "io-util"], default-features = false }
5962
tokio-stream = { version = "0.1.17", features = ["fs"], default-features = false }
6063
tokio-util = { version = "0.7.13" }
@@ -80,5 +83,6 @@ aws-sdk-s3 = { version = "=1.78.0", features = [
8083
aws-smithy-runtime-api = "=1.7.3"
8184
rand = { version = "0.9.0", default-features = false, features = ["thread_rng", "small_rng"] }
8285
serde_json = "1.0.139"
86+
tempfile = "3.8.1"
8387
fred = { version = "10.0.4", default-features = false, features = ["mocks"] }
8488
tracing-subscriber = { version = "0.3.19", default-features = false }

nativelink-store/src/default_store_factory.rs

+6
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ use crate::filesystem_store::FilesystemStore;
3232
use crate::grpc_store::GrpcStore;
3333
use crate::memory_store::MemoryStore;
3434
use crate::noop_store::NoopStore;
35+
use crate::ontap_s3_existence_cache_store::OntapS3ExistenceCache;
36+
use crate::ontap_s3_store::OntapS3Store;
3537
use crate::redis_store::RedisStore;
3638
use crate::ref_store::RefStore;
3739
use crate::s3_store::S3Store;
@@ -51,6 +53,10 @@ pub fn store_factory<'a>(
5153
let store: Arc<dyn StoreDriver> = match backend {
5254
StoreSpec::memory(spec) => MemoryStore::new(spec),
5355
StoreSpec::experimental_s3_store(spec) => S3Store::new(spec, SystemTime::now).await?,
56+
StoreSpec::ontap_s3_store(spec) => OntapS3Store::new(spec, SystemTime::now).await?,
57+
StoreSpec::ontap_s3_existence_cache(spec) => {
58+
OntapS3ExistenceCache::new(spec, SystemTime::now).await?
59+
}
5460
StoreSpec::redis_store(spec) => RedisStore::new(spec.clone())?,
5561
StoreSpec::verify(spec) => VerifyStore::new(
5662
spec,

nativelink-store/src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ pub mod filesystem_store;
2424
pub mod grpc_store;
2525
pub mod memory_store;
2626
pub mod noop_store;
27+
pub mod ontap_s3_existence_cache_store;
28+
pub mod ontap_s3_store;
2729
pub mod redis_store;
2830
mod redis_utils;
2931
pub mod ref_store;

0 commit comments

Comments
 (0)