Skip to content

Commit 3d76991

Browse files
ashishsonammashishsonamm
authored
Add Cloudflare R2 support (#2319)
1 parent 2c04038 commit 3d76991

8 files changed

Lines changed: 410 additions & 0 deletions

File tree

‎nativelink-config/README.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,54 @@ stores, but in this example we'll store the raw files.
233233
}
234234
```
235235

236+
### R2 Store
237+
238+
[Cloudflare R2](https://developers.cloudflare.com/r2/) is an S3-compatible
239+
object store with no egress fees, which makes it attractive for read-heavy
240+
CAS workloads. The endpoint is derived from your Cloudflare `account_id`;
241+
credentials are read from env vars via `shellexpand`.
242+
243+
```js
244+
{
245+
"stores": [
246+
{
247+
"name": "CAS_MAIN_STORE",
248+
"experimental_cloud_object_store": {
249+
"provider": "r2",
250+
"account_id": "your-cloudflare-account-id",
251+
"bucket": "nativelink-cas",
252+
"access_key_id": "${R2_ACCESS_KEY_ID}",
253+
"secret_access_key": "${R2_SECRET_ACCESS_KEY}",
254+
"key_prefix": "cas/",
255+
"retry": {
256+
"max_retries": 6,
257+
"delay": 0.3,
258+
"jitter": 0.5,
259+
}
260+
}
261+
},
262+
{
263+
"name": "AC_MAIN_STORE",
264+
"experimental_cloud_object_store": {
265+
"provider": "r2",
266+
"account_id": "your-cloudflare-account-id",
267+
"bucket": "nativelink-cas",
268+
"access_key_id": "${R2_ACCESS_KEY_ID}",
269+
"secret_access_key": "${R2_SECRET_ACCESS_KEY}",
270+
"key_prefix": "ac/",
271+
}
272+
}
273+
],
274+
// Place rest of configuration here ...
275+
}
276+
```
277+
278+
A complete runnable example with CAS and AC is at
279+
[`nativelink-config/examples/r2_backend.json5`](https://github.com/TraceMachina/nativelink/blob/main/nativelink-config/examples/r2_backend.json5).
280+
If `access_key_id` and `secret_access_key` are omitted, NativeLink falls
281+
back to the standard AWS credential chain (`AWS_*` env vars,
282+
`~/.aws/credentials`, IMDS).
283+
236284
### Fast Slow Store
237285

238286
This store will first attempt to read from the `fast` store when reading and if

‎nativelink-config/examples/r2_backend.json5‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
// Replace the access_key_id / secret_access_key placeholders below with
2+
// real values, or with shellexpand refs like "${R2_ACCESS_KEY_ID}".
3+
// You can also choose to omit both placeholders to fall back to the AWS
4+
// default credential chain (but it only reads AWS_* env var names — your R2 keys would need to live there).
5+
{
6+
stores: [
7+
{
8+
name: "CAS_MAIN_STORE",
9+
verify: {
10+
verify_size: true,
11+
backend: {
12+
dedup: {
13+
index_store: {
14+
fast_slow: {
15+
fast: {
16+
memory: {
17+
eviction_policy: {
18+
max_bytes: "100mb",
19+
},
20+
},
21+
},
22+
slow: {
23+
experimental_cloud_object_store: {
24+
provider: "r2",
25+
account_id: "636b76d9ace55d0aacba7980073c7ca7",
26+
bucket: "nativelink-cas-test",
27+
access_key_id: "r2_access_key_id",
28+
secret_access_key: "r2_secret_access_key",
29+
key_prefix: "cas-index/",
30+
retry: {
31+
max_retries: 6,
32+
delay: 0.3,
33+
jitter: 0.5,
34+
},
35+
},
36+
},
37+
},
38+
},
39+
content_store: {
40+
compression: {
41+
compression_algorithm: {
42+
lz4: {},
43+
},
44+
backend: {
45+
fast_slow: {
46+
fast: {
47+
memory: {
48+
eviction_policy: {
49+
max_bytes: "100mb",
50+
},
51+
},
52+
},
53+
slow: {
54+
experimental_cloud_object_store: {
55+
provider: "r2",
56+
account_id: "636b76d9ace55d0aacba7980073c7ca7",
57+
bucket: "nativelink-cas-test",
58+
access_key_id: "r2_access_key_id",
59+
secret_access_key: "r2_secret_access_key",
60+
key_prefix: "cas/",
61+
retry: {
62+
max_retries: 6,
63+
delay: 0.3,
64+
jitter: 0.5,
65+
},
66+
},
67+
},
68+
},
69+
},
70+
},
71+
},
72+
},
73+
},
74+
},
75+
},
76+
{
77+
name: "AC_MAIN_STORE",
78+
fast_slow: {
79+
fast: {
80+
memory: {
81+
eviction_policy: {
82+
max_bytes: "100mb",
83+
},
84+
},
85+
},
86+
slow: {
87+
experimental_cloud_object_store: {
88+
provider: "r2",
89+
account_id: "636b76d9ace55d0aacba7980073c7ca7",
90+
bucket: "nativelink-cas-test",
91+
access_key_id: "r2_access_key_id",
92+
secret_access_key: "r2_secret_access_key",
93+
key_prefix: "ac/",
94+
retry: {
95+
max_retries: 6,
96+
delay: 0.3,
97+
jitter: 0.5,
98+
},
99+
},
100+
},
101+
},
102+
},
103+
],
104+
schedulers: [
105+
{
106+
name: "MAIN_SCHEDULER",
107+
simple: {
108+
supported_platform_properties: {
109+
cpu_count: "minimum",
110+
memory_kb: "minimum",
111+
network_kbps: "minimum",
112+
disk_read_iops: "minimum",
113+
disk_read_bps: "minimum",
114+
disk_write_iops: "minimum",
115+
disk_write_bps: "minimum",
116+
shm_size: "minimum",
117+
gpu_count: "minimum",
118+
gpu_model: "exact",
119+
cpu_vendor: "exact",
120+
cpu_arch: "exact",
121+
cpu_model: "exact",
122+
kernel_version: "exact",
123+
docker_image: "priority",
124+
"lre-rs": "priority",
125+
ISA: "exact",
126+
},
127+
},
128+
},
129+
],
130+
servers: [
131+
{
132+
listener: {
133+
http: {
134+
socket_address: "0.0.0.0:50051",
135+
},
136+
},
137+
services: {
138+
cas: [
139+
{
140+
instance_name: "main",
141+
cas_store: "CAS_MAIN_STORE",
142+
},
143+
],
144+
ac: [
145+
{
146+
instance_name: "main",
147+
ac_store: "AC_MAIN_STORE",
148+
},
149+
],
150+
execution: [
151+
{
152+
instance_name: "main",
153+
cas_store: "CAS_MAIN_STORE",
154+
scheduler: "MAIN_SCHEDULER",
155+
},
156+
],
157+
capabilities: [
158+
{
159+
instance_name: "main",
160+
remote_execution: {
161+
scheduler: "MAIN_SCHEDULER",
162+
},
163+
},
164+
],
165+
bytestream: [
166+
{
167+
instance_name: "main",
168+
cas_store: "CAS_MAIN_STORE",
169+
},
170+
],
171+
health: {},
172+
},
173+
},
174+
],
175+
}

‎nativelink-config/src/stores.rs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -686,6 +686,33 @@ pub struct ExperimentalOntapS3Spec {
686686
pub common: CommonObjectSpec,
687687
}
688688

689+
// Cloudflare R2 Spec
690+
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
691+
#[serde(deny_unknown_fields)]
692+
#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
693+
pub struct ExperimentalR2Spec {
694+
/// Cloudflare account ID. Endpoint is derived as
695+
/// `https://{account_id}.r2.cloudflarestorage.com`.
696+
#[serde(deserialize_with = "convert_string_with_shellexpand")]
697+
pub account_id: String,
698+
699+
/// Bucket name to use as the backend.
700+
#[serde(deserialize_with = "convert_string_with_shellexpand")]
701+
pub bucket: String,
702+
703+
/// Explicit R2 access key.
704+
#[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")]
705+
pub access_key_id: Option<String>,
706+
707+
/// Explicit R2 secret key.
708+
#[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")]
709+
pub secret_access_key: Option<String>,
710+
711+
/// Retry and upload settings.
712+
#[serde(flatten)]
713+
pub common: CommonObjectSpec,
714+
}
715+
689716
#[derive(Serialize, Deserialize, Debug, Clone)]
690717
#[serde(deny_unknown_fields)]
691718
#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
@@ -956,6 +983,7 @@ pub enum ExperimentalCloudObjectSpec {
956983
Gcs(ExperimentalGcsSpec),
957984
Azure(ExperimentalAzureSpec),
958985
Ontap(ExperimentalOntapS3Spec),
986+
R2(ExperimentalR2Spec),
959987
}
960988

961989
impl Default for ExperimentalCloudObjectSpec {

‎nativelink-store/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ rust_library(
3434
"src/noop_store.rs",
3535
"src/ontap_s3_existence_cache_store.rs",
3636
"src/ontap_s3_store.rs",
37+
"src/r2_store.rs",
3738
"src/redis_store.rs",
3839
"src/redis_utils/aggregate_types.rs",
3940
"src/redis_utils/ft_aggregate.rs",
@@ -129,6 +130,7 @@ rust_test_suite(
129130
"tests/mongo_store_test.rs",
130131
"tests/ontap_s3_existence_cache_store_test.rs",
131132
"tests/ontap_s3_store_test.rs",
133+
"tests/r2_store_test.rs",
132134
"tests/redis_store_test.rs",
133135
"tests/ref_store_test.rs",
134136
"tests/s3_store_test.rs",

‎nativelink-store/src/default_store_factory.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ use crate::mongo_store::ExperimentalMongoStore;
3737
use crate::noop_store::NoopStore;
3838
use crate::ontap_s3_existence_cache_store::OntapS3ExistenceCache;
3939
use crate::ontap_s3_store::OntapS3Store;
40+
use crate::r2_store::R2Store;
4041
use crate::redis_store::RedisStore;
4142
use crate::ref_store::RefStore;
4243
use crate::s3_store::S3Store;
@@ -68,6 +69,9 @@ pub fn store_factory<'a>(
6869
ExperimentalCloudObjectSpec::Azure(azure_config) => {
6970
AzureBlobStore::new(azure_config, SystemTime::now).await?
7071
}
72+
ExperimentalCloudObjectSpec::R2(r2_config) => {
73+
R2Store::new(r2_config, SystemTime::now).await?
74+
}
7175
},
7276
StoreSpec::RedisStore(spec) => {
7377
if spec.mode == RedisMode::Cluster {

‎nativelink-store/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ pub mod mongo_store;
3232
pub mod noop_store;
3333
pub mod ontap_s3_existence_cache_store;
3434
pub mod ontap_s3_store;
35+
pub mod r2_store;
3536
pub mod redis_store;
3637
mod redis_utils;
3738
pub mod ref_store;

0 commit comments

Comments
 (0)