Skip to content

Commit 59e9579

Browse files
vmagrovmagro
committed
configure buck2 for localhost nativelink for caching
1 parent 7f38eb3 commit 59e9579

File tree

5 files changed

+258
-0
lines changed

5 files changed

+258
-0
lines changed

‎.buckconfig

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,19 @@
2323

2424
[buck2]
2525
file_watcher = watchman
26+
materializations = deferred
27+
sqlite_materializer_state = true
28+
defer_write_actions = true
29+
hash_all_commands = true
30+
31+
[buck2_re_client]
32+
action_cache_address = grpcs://127.0.0.1:50051
33+
engine_address = grpcs://127.0.0.1:50051
34+
cas_address = grpcs://127.0.0.1:50051
35+
tls = false
36+
instance_name = main
37+
38+
[build]
39+
execution_platforms = antlir//platforms:exec-platforms
2640

2741
<file:.buckconfig.generated>

‎.github/workflows/ci.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,20 @@ jobs:
1111
strategy:
1212
matrix:
1313
shard: [1/3, 2/3, 3/3]
14+
15+
services:
16+
nativelink:
17+
image: ghcr.io/tracemachina/nativelink:v0.5.1
18+
options: /config
19+
volumes:
20+
- ${{ github.workspace }}/ci/nativelink.json:/config
21+
1422
steps:
1523
- uses: actions/checkout@v4
1624
with:
1725
submodules: true
26+
- name: Restart NativeLink
27+
run: docker restart nativelink
1828
- uses: facebook/install-dotslash@latest
1929
- uses: dtolnay/rust-toolchain@stable
2030
with:

‎ci/nativelink.json

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
{
2+
"stores": {
3+
"AC_MAIN_STORE": {
4+
"filesystem": {
5+
"content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
6+
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
7+
"eviction_policy": {
8+
// 1gb.
9+
"max_bytes": 1000000000,
10+
}
11+
}
12+
},
13+
"WORKER_FAST_SLOW_STORE": {
14+
"fast_slow": {
15+
// "fast" must be a "filesystem" store because the worker uses it to make
16+
// hardlinks on disk to a directory where the jobs are running.
17+
"fast": {
18+
"filesystem": {
19+
"content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
20+
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
21+
"eviction_policy": {
22+
// 10gb.
23+
"max_bytes": 10000000000,
24+
}
25+
}
26+
},
27+
"slow": {
28+
/// Discard data.
29+
/// This example usage has the CAS and the Worker live in the same place,
30+
/// so they share the same underlying CAS. Since workers require a fast_slow
31+
/// store, we use the fast store as our primary data store, and the slow store
32+
/// is just a noop, since there's no shared storage in this config.
33+
"noop": {}
34+
}
35+
}
36+
}
37+
},
38+
"schedulers": {
39+
"MAIN_SCHEDULER": {
40+
"simple": {
41+
"supported_platform_properties": {
42+
"cpu_count": "minimum",
43+
"memory_kb": "minimum",
44+
"network_kbps": "minimum",
45+
"disk_read_iops": "minimum",
46+
"disk_read_bps": "minimum",
47+
"disk_write_iops": "minimum",
48+
"disk_write_bps": "minimum",
49+
"shm_size": "minimum",
50+
"gpu_count": "minimum",
51+
"gpu_model": "exact",
52+
"cpu_vendor": "exact",
53+
"cpu_arch": "exact",
54+
"cpu_model": "exact",
55+
"kernel_version": "exact",
56+
"OSFamily": "priority",
57+
"container-image": "priority",
58+
// Example of how to set which docker images are available and set
59+
// them in the platform properties.
60+
// "docker_image": "priority",
61+
}
62+
}
63+
}
64+
},
65+
"workers": [{
66+
"local": {
67+
"worker_api_endpoint": {
68+
"uri": "grpc://127.0.0.1:50061",
69+
},
70+
"cas_fast_slow_store": "WORKER_FAST_SLOW_STORE",
71+
"upload_action_result": {
72+
"ac_store": "AC_MAIN_STORE",
73+
},
74+
"work_directory": "/tmp/nativelink/work",
75+
"platform_properties": {
76+
"cpu_count": {
77+
"values": ["16"],
78+
},
79+
"memory_kb": {
80+
"values": ["500000"],
81+
},
82+
"network_kbps": {
83+
"values": ["100000"],
84+
},
85+
"cpu_arch": {
86+
"values": ["x86_64"],
87+
},
88+
"OSFamily": {
89+
"values": [""]
90+
},
91+
"container-image": {
92+
"values": [""]
93+
},
94+
// Example of how to set which docker images are available and set
95+
// them in the platform properties.
96+
// "docker_image": {
97+
// "query_cmd": "docker images --format {{.Repository}}:{{.Tag}}",
98+
// }
99+
}
100+
}
101+
}],
102+
"servers": [{
103+
"name": "public",
104+
"listener": {
105+
"http": {
106+
"socket_address": "0.0.0.0:50051"
107+
}
108+
},
109+
"services": {
110+
"cas": {
111+
"main": {
112+
"cas_store": "WORKER_FAST_SLOW_STORE"
113+
}
114+
},
115+
"ac": {
116+
"main": {
117+
"ac_store": "AC_MAIN_STORE"
118+
}
119+
},
120+
"execution": {
121+
"main": {
122+
"cas_store": "WORKER_FAST_SLOW_STORE",
123+
"scheduler": "MAIN_SCHEDULER",
124+
}
125+
},
126+
"capabilities": {
127+
"main": {
128+
"remote_execution": {
129+
"scheduler": "MAIN_SCHEDULER",
130+
}
131+
}
132+
},
133+
"bytestream": {
134+
"cas_stores": {
135+
"main": "WORKER_FAST_SLOW_STORE",
136+
}
137+
}
138+
}
139+
}, {
140+
"name": "private_workers_servers",
141+
"listener": {
142+
"http": {
143+
"socket_address": "0.0.0.0:50061"
144+
}
145+
},
146+
"services": {
147+
"experimental_prometheus": {
148+
"path": "/metrics"
149+
},
150+
// Note: This should be served on a different port, because it has
151+
// a different permission set than the other services.
152+
// In other words, this service is a backend api. The ones above
153+
// are a frontend api.
154+
"worker_api": {
155+
"scheduler": "MAIN_SCHEDULER",
156+
},
157+
"admin": {},
158+
"health": {},
159+
}
160+
}],
161+
"global": {
162+
"max_open_files": 512
163+
}
164+
}

‎platforms/BUCK

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
load("@prelude//:rules.bzl", "platform")
2+
load("@prelude//platforms:defs.bzl", "execution_platform", "host_configuration")
3+
load(":defs.bzl", "nativelink_platform", "exec_platforms")
4+
5+
nativelink_platform(
6+
name = "nativelink",
7+
cpu_configuration = "config//cpu:x86_64",
8+
os_configuration = "config//os:linux",
9+
visibility = ["PUBLIC"],
10+
)
11+
12+
execution_platform(
13+
name = "host",
14+
cpu_configuration = host_configuration.cpu,
15+
os_configuration = host_configuration.os,
16+
use_windows_path_separators = host_info().os.is_windows,
17+
visibility = ["PUBLIC"],
18+
)
19+
20+
exec_platforms(
21+
name = "exec-platforms",
22+
platforms = [
23+
":nativelink",
24+
":host",
25+
]
26+
)

‎platforms/defs.bzl

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
def _nativelink_platform(ctx):
2+
constraints = {}
3+
constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints)
4+
constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints)
5+
6+
configuration = ConfigurationInfo(
7+
constraints = constraints,
8+
values = {},
9+
)
10+
11+
platform = ExecutionPlatformInfo(
12+
label = ctx.label.raw_target(),
13+
configuration = configuration,
14+
executor_config = CommandExecutorConfig(
15+
local_enabled = True,
16+
remote_enabled = True,
17+
use_limited_hybrid = True,
18+
remote_execution_properties = {
19+
# "arch": ctx.attrs.cpu_configuration.label.name,
20+
# "os": ctx.attrs.os_configuration.label.name,
21+
},
22+
remote_execution_use_case = "antlir",
23+
remote_output_paths = "output_paths",
24+
),
25+
)
26+
27+
return [DefaultInfo(), platform]
28+
29+
nativelink_platform = rule(attrs = {
30+
"cpu_configuration": attrs.dep(providers = [ConfigurationInfo]),
31+
"os_configuration": attrs.dep(providers = [ConfigurationInfo]),
32+
}, impl = _nativelink_platform)
33+
34+
def _exec_platforms(ctx):
35+
return [
36+
DefaultInfo(),
37+
ExecutionPlatformRegistrationInfo(
38+
platforms = [p[ExecutionPlatformInfo] for p in ctx.attrs.platforms],
39+
),
40+
]
41+
42+
exec_platforms = rule(attrs = {
43+
"platforms": attrs.list(attrs.dep(providers = [ExecutionPlatformInfo])),
44+
}, impl = _exec_platforms)

0 commit comments

Comments
 (0)