Skip to content

Commit 5d625c2

Browse files
amankrxamankrx
authored
Implement Azure Blob Store (TraceMachina#1554)
* Add the implementation for Azure Blob Store * Add clippy fixes * Fix pre commit hook for Azure Blob Store
1 parent e71d738 commit 5d625c2

11 files changed

Lines changed: 2162 additions & 29 deletions

File tree

‎Cargo.lock‎

Lines changed: 275 additions & 13 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎nativelink-config/examples/azure_blob_backend.json5‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
{
2+
stores: [
3+
{
4+
name: "CAS_MAIN_STORE",
5+
verify: {
6+
backend: {
7+
dedup: {
8+
index_store: {
9+
fast_slow: {
10+
fast: {
11+
filesystem: {
12+
content_path: "/tmp/nativelink/data/content_path-index",
13+
temp_path: "/tmp/nativelink/data/tmp_path-index",
14+
eviction_policy: {
15+
max_bytes: 500000000, // 500mb
16+
},
17+
},
18+
},
19+
slow: {
20+
experimental_cloud_object_store: {
21+
provider: "azure",
22+
account_name: "rbetestnativelink",
23+
container: "storageacctcontainer",
24+
key_prefix: "test-prefix-index/",
25+
retry: {
26+
max_retries: 6,
27+
delay: 0.3,
28+
jitter: 0.5,
29+
},
30+
multipart_max_concurrent_uploads: 10,
31+
},
32+
},
33+
},
34+
},
35+
content_store: {
36+
compression: {
37+
compression_algorithm: {
38+
lz4: {},
39+
},
40+
backend: {
41+
fast_slow: {
42+
fast: {
43+
filesystem: {
44+
content_path: "/tmp/nativelink/data/content_path-content",
45+
temp_path: "/tmp/nativelink/data/tmp_path-content",
46+
eviction_policy: {
47+
max_bytes: 2000000000, // 2gb
48+
},
49+
},
50+
},
51+
slow: {
52+
experimental_cloud_object_store: {
53+
provider: "azure",
54+
account_name: "rbetestnativelink",
55+
container: "storageacctcontainer",
56+
key_prefix: "test-prefix-dedup-cas/",
57+
retry: {
58+
max_retries: 6,
59+
delay: 0.3,
60+
jitter: 0.5,
61+
},
62+
multipart_max_concurrent_uploads: 10,
63+
},
64+
},
65+
},
66+
},
67+
},
68+
},
69+
},
70+
},
71+
verify_size: true,
72+
},
73+
},
74+
{
75+
name: "AC_MAIN_STORE",
76+
fast_slow: {
77+
fast: {
78+
filesystem: {
79+
content_path: "/tmp/nativelink/data/content_path-ac",
80+
temp_path: "/tmp/nativelink/data/tmp_path-ac",
81+
eviction_policy: {
82+
max_bytes: 500000000, // 500mb
83+
},
84+
},
85+
},
86+
slow: {
87+
experimental_cloud_object_store: {
88+
provider: "azure",
89+
account_name: "rbetestnativelink",
90+
container: "storageacctcontainer",
91+
key_prefix: "test-prefix-ac/",
92+
retry: {
93+
max_retries: 6,
94+
delay: 0.3,
95+
jitter: 0.5,
96+
},
97+
multipart_max_concurrent_uploads: 10,
98+
},
99+
},
100+
},
101+
},
102+
],
103+
schedulers: [
104+
{
105+
name: "MAIN_SCHEDULER",
106+
simple: {
107+
supported_platform_properties: {
108+
cpu_count: "minimum",
109+
memory_kb: "minimum",
110+
network_kbps: "minimum",
111+
disk_read_iops: "minimum",
112+
disk_read_bps: "minimum",
113+
disk_write_iops: "minimum",
114+
disk_write_bps: "minimum",
115+
shm_size: "minimum",
116+
gpu_count: "minimum",
117+
gpu_model: "exact",
118+
cpu_vendor: "exact",
119+
cpu_arch: "exact",
120+
cpu_model: "exact",
121+
kernel_version: "exact",
122+
docker_image: "priority",
123+
"lre-rs": "priority",
124+
},
125+
},
126+
},
127+
],
128+
servers: [
129+
{
130+
listener: {
131+
http: {
132+
socket_address: "0.0.0.0:50051",
133+
},
134+
},
135+
services: {
136+
cas: [
137+
{
138+
instance_name: "main",
139+
cas_store: "CAS_MAIN_STORE",
140+
},
141+
],
142+
ac: [
143+
{
144+
instance_name: "main",
145+
ac_store: "AC_MAIN_STORE",
146+
},
147+
],
148+
execution: [
149+
{
150+
instance_name: "main",
151+
cas_store: "CAS_MAIN_STORE",
152+
scheduler: "MAIN_SCHEDULER",
153+
},
154+
],
155+
capabilities: [
156+
{
157+
instance_name: "main",
158+
remote_execution: {
159+
scheduler: "MAIN_SCHEDULER",
160+
},
161+
},
162+
],
163+
bytestream: [
164+
{
165+
instance_name: "main",
166+
cas_store: "CAS_MAIN_STORE",
167+
},
168+
],
169+
health: {},
170+
},
171+
},
172+
],
173+
}

‎nativelink-config/examples/stores-config.json5‎

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,21 @@
4141
},
4242
{
4343
name: "3",
44+
"experimental_cloud_object_store": {
45+
"provider": "azure",
46+
"account_name": "cloudshell1393657559",
47+
"container": "simple-test-container",
48+
"key_prefix": "folder/",
49+
"retry": {
50+
"max_retries": 6,
51+
"delay": 0.3,
52+
"jitter": 0.5
53+
},
54+
"multipart_max_concurrent_uploads": 10
55+
}
56+
},
57+
{
58+
name: "4",
4459
"experimental_cloud_object_store": {
4560
"provider": "ontap",
4661
"endpoint": "https://ontap-s3-endpoint:443",
@@ -57,7 +72,7 @@
5772
}
5873
},
5974
{
60-
name: "4",
75+
name: "5",
6176
"ontap_s3_existence_cache": {
6277
"index_path": "/path/to/cache/index.json",
6378
"sync_interval_seconds": 300,
@@ -70,7 +85,7 @@
7085
}
7186
},
7287
{
73-
name: "5",
88+
name: "6",
7489
"verify": {
7590
"backend": {
7691
"memory": {
@@ -84,7 +99,7 @@
8499
}
85100
},
86101
{
87-
name: "6",
102+
name: "7",
88103
"completeness_checking": {
89104
"backend": {
90105
"filesystem": {
@@ -103,7 +118,7 @@
103118
}
104119
},
105120
{
106-
name: "7",
121+
name: "8",
107122
"compression": {
108123
"compression_algorithm": {
109124
"lz4": {}
@@ -120,7 +135,7 @@
120135
}
121136
},
122137
{
123-
name: "8",
138+
name: "9",
124139
"dedup": {
125140
"index_store": {
126141
"memory": {
@@ -159,7 +174,7 @@
159174
}
160175
},
161176
{
162-
name: "9",
177+
name: "10",
163178
"existence_cache": {
164179
"backend": {
165180
"memory": {
@@ -175,7 +190,7 @@
175190
}
176191
},
177192
{
178-
name: "10",
193+
name: "11",
179194
"fast_slow": {
180195
"fast": {
181196
"filesystem": {
@@ -198,7 +213,7 @@
198213
}
199214
},
200215
{
201-
name: "11",
216+
name: "12",
202217
"shard": {
203218
"stores": [
204219
{
@@ -214,7 +229,7 @@
214229
}
215230
},
216231
{
217-
name: "12",
232+
name: "13",
218233
"filesystem": {
219234
"content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
220235
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
@@ -224,13 +239,13 @@
224239
}
225240
},
226241
{
227-
name: "13",
242+
name: "14",
228243
"ref_store": {
229244
"name": "FS_CONTENT_STORE"
230245
}
231246
},
232247
{
233-
name: "14",
248+
name: "15",
234249
"size_partitioning": {
235250
"size": "128mib",
236251
"lower_store": {
@@ -247,7 +262,7 @@
247262
}
248263
},
249264
{
250-
name: "15",
265+
name: "16",
251266
"grpc": {
252267
"instance_name": "main",
253268
"endpoints": [
@@ -259,7 +274,7 @@
259274
}
260275
},
261276
{
262-
name: "16",
277+
name: "17",
263278
"redis_store": {
264279
"addresses": [
265280
"redis://127.0.0.1:6379/",
@@ -268,11 +283,11 @@
268283
}
269284
},
270285
{
271-
name: "17",
286+
name: "18",
272287
"noop": {}
273288
},
274289
{
275-
name: "18",
290+
name: "19",
276291
"experimental_mongo": {
277292
"connection_string": "mongodb://localhost:27017",
278293
"database": "nativelink",

‎nativelink-config/src/stores.rs‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,28 @@ pub enum StoreSpec {
109109
/// }
110110
/// ```
111111
///
112-
/// 3. **`NetApp` ONTAP S3**
112+
/// 3. **Azure Blob Store:**
113+
/// Azure Blob store will use Microsoft's Azure Blob service as a
114+
/// backend to store the files. This configuration can be used to
115+
/// share files across multiple instances.
116+
///
117+
/// **Example JSON Config:**
118+
/// ```json
119+
/// "experimental_cloud_object_store": {
120+
/// "provider": "azure",
121+
/// "account_name": "cloudshell1393657559",
122+
/// "container": "simple-test-container",
123+
/// "key_prefix": "folder/",
124+
/// "retry": {
125+
/// "max_retries": 6,
126+
/// "delay": 0.3,
127+
/// "jitter": 0.5
128+
/// },
129+
/// "multipart_max_concurrent_uploads": 10
130+
/// }
131+
/// ```
132+
///
133+
/// 4. **`NetApp` ONTAP S3**
113134
/// `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend
114135
/// to store files. This store is specifically configured for ONTAP's S3 requirements
115136
/// including custom TLS configuration, credentials management, and proper vserver
@@ -922,6 +943,7 @@ pub struct EvictionPolicy {
922943
pub enum ExperimentalCloudObjectSpec {
923944
Aws(ExperimentalAwsSpec),
924945
Gcs(ExperimentalGcsSpec),
946+
Azure(ExperimentalAzureSpec),
925947
Ontap(ExperimentalOntapS3Spec),
926948
}
927949

@@ -984,6 +1006,33 @@ pub struct ExperimentalGcsSpec {
9841006
pub read_timeout_s: u64,
9851007
}
9861008

1009+
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
1010+
#[serde(deny_unknown_fields)]
1011+
#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
1012+
pub struct ExperimentalAzureSpec {
1013+
/// The Azure Storage account name.
1014+
#[serde(default, deserialize_with = "convert_string_with_shellexpand")]
1015+
pub account_name: String,
1016+
1017+
/// The container name to use as the backend.
1018+
#[serde(default, deserialize_with = "convert_string_with_shellexpand")]
1019+
pub container: String,
1020+
1021+
/// Common retry and upload configuration.
1022+
#[serde(flatten)]
1023+
pub common: CommonObjectSpec,
1024+
1025+
/// Connection timeout in milliseconds.
1026+
/// Default: 3000
1027+
#[serde(default, deserialize_with = "convert_duration_with_shellexpand")]
1028+
pub connection_timeout_s: u64,
1029+
1030+
/// Read timeout in milliseconds.
1031+
/// Default: 3000
1032+
#[serde(default, deserialize_with = "convert_duration_with_shellexpand")]
1033+
pub read_timeout_s: u64,
1034+
}
1035+
9871036
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
9881037
#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
9891038
pub struct CommonObjectSpec {

0 commit comments

Comments
 (0)