Skip to content

Commit 00680e7

Browse files
palfreypalfrey
committed
Build toolchain-examples
1 parent 5e487f3 commit 00680e7

10 files changed

Lines changed: 102 additions & 30 deletions

File tree

‎.github/workflows/nix.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797
name: ${{ matrix.test-name }}
9898
strategy:
9999
matrix:
100-
test-name: [buildstream, mongo]
100+
test-name: [buildstream, mongo, rbe-toolchain]
101101
runs-on: ubuntu-24.04
102102
timeout-minutes: 45
103103
steps:

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@ rust-project.json
2323
darwin.bazelrc
2424
nativelink.bazelrc
2525
integration_tests/**/*.log
26+
toolchain-examples/*.log

‎MODULE.bazel‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ module(
44
compatibility_level = 0,
55
)
66

7-
bazel_dep(name = "rules_cc", version = "0.1.1")
8-
bazel_dep(name = "platforms", version = "0.0.11")
7+
bazel_dep(name = "rules_cc", version = "0.1.5")
8+
bazel_dep(name = "platforms", version = "1.0.0")
99
bazel_dep(name = "bazel_skylib", version = "1.7.1")
1010
bazel_dep(name = "rules_python", version = "1.3.0") # TODO(palfrey): Bump.
1111
bazel_dep(name = "rules_shell", version = "0.4.1")

‎flake.nix‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,9 @@
379379
mongo-with-nativelink-test = pkgs.callPackage integration_tests/mongo/mongo-with-nativelink-test.nix {
380380
inherit nativelink mongodb wait4x bazelisk;
381381
};
382+
rbe-toolchain-with-nativelink-test = pkgs.callPackage toolchain-examples/rbe-toolchain-test.nix {
383+
inherit nativelink bazelisk;
384+
};
382385
}
383386
// (
384387
# It's not possible to crosscompile to darwin, not even between

‎local-remote-execution/MODULE.bazel‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ module(
77
compatibility_level = 0,
88
)
99

10-
bazel_dep(name = "platforms", version = "0.0.11")
10+
bazel_dep(name = "platforms", version = "1.0.0")
1111

1212
# Use the starlark implementation of C++ rules instead of the builtin ones.
13-
bazel_dep(name = "rules_cc", version = "0.1.1")
13+
bazel_dep(name = "rules_cc", version = "0.1.5")
1414

1515
# Use the starlark implementation of Java rules instead of the builtin ones.
1616
bazel_dep(name = "rules_java", version = "8.11.0")

‎nativelink-util/src/telemetry.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ fn otlp_filter() -> EnvFilter {
7171

7272
// Create a tracing layer intended for stdout printing.
7373
//
74-
// The output of this layer is configurable via the `NL_LOG_FMT` environment
74+
// The output of this layer is configurable via the `NL_LOG` environment
7575
// variable.
7676
fn tracing_stdout_layer() -> impl Layer<Registry> {
7777
let nl_log_fmt = env::var("NL_LOG").unwrap_or_else(|_| "pretty".to_string());

‎toolchain-examples/.bazelrc‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
# Don't use the host's default PATH and LD_LIBRARY_PATH.
2-
build --incompatible_strict_action_env
3-
41
# Use rules_python's builtin script to emulate a bootstrap python.
52
build --@rules_python//python/config_settings:bootstrap_impl=script
63

‎toolchain-examples/MODULE.bazel‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ module(
44
compatibility_level = 0,
55
)
66

7-
bazel_dep(name = "platforms", version = "0.0.11")
7+
bazel_dep(name = "platforms", version = "1.0.0")
88

99
# C++
10-
bazel_dep(name = "rules_cc", version = "0.1.1")
10+
bazel_dep(name = "rules_cc", version = "0.1.5")
1111

1212
# Java
1313
bazel_dep(name = "rules_java", version = "8.11.0")
@@ -34,7 +34,16 @@ python.toolchain(
3434
use_repo(pip, "pip")
3535

3636
# Go
37-
bazel_dep(name = "rules_go", version = "0.53.0")
37+
bazel_dep(name = "rules_go", version = "0.57.0")
38+
39+
# Adds https://github.com/bazel-contrib/rules_go/commit/74199c92e20399b6ef46684b2c6fdd94b50a7892
40+
# to fix bash issues with Nix
41+
archive_override(
42+
module_name = "rules_go",
43+
integrity = "sha256-ukyyC80j4VhRCD7DOaenkk41Vvnmsp7uAfHr4lxdXtQ=",
44+
strip_prefix = "rules_go-74199c92e20399b6ef46684b2c6fdd94b50a7892",
45+
urls = ["https://github.com/bazel-contrib/rules_go/archive/74199c92e20399b6ef46684b2c6fdd94b50a7892.zip"],
46+
)
3847

3948
# Rust
4049
bazel_dep(name = "rules_rust", version = "0.61.0")
@@ -89,8 +98,5 @@ bazel_dep(name = "abseil-cpp", version = "20250127.0")
8998
# Abseil for python
9099
bazel_dep(name = "abseil-py", version = "2.1.0")
91100

92-
# GRPC
93-
bazel_dep(name = "grpc", version = "1.71.0")
94-
95101
# Circl (Go, C++)
96102
bazel_dep(name = "circl", version = "1.3.8")

‎toolchain-examples/nativelink-config.json5‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
{
2-
stores: {
3-
AC_MAIN_STORE: {
2+
stores: [
3+
{
4+
name: "AC_MAIN_STORE",
45
filesystem: {
56
content_path: "/tmp/nativelink/data-worker-test/content_path-ac",
67
temp_path: "/tmp/nativelink/data-worker-test/tmp_path-ac",
78
eviction_policy: {
8-
// 1gb.
9-
max_bytes: 1000000000,
9+
max_bytes: "1gb",
1010
},
1111
},
1212
},
13-
WORKER_FAST_SLOW_STORE: {
13+
{
14+
name: "WORKER_FAST_SLOW_STORE",
1415
fast_slow: {
1516
// "fast" must be a "filesystem" store because the worker uses it to make
1617
// hardlinks on disk to a directory where the jobs are running.
@@ -19,8 +20,7 @@
1920
content_path: "/tmp/nativelink/data-worker-test/content_path-cas",
2021
temp_path: "/tmp/nativelink/data-worker-test/tmp_path-cas",
2122
eviction_policy: {
22-
// 10gb.
23-
max_bytes: 10000000000,
23+
max_bytes: "10gb",
2424
},
2525
},
2626
},
@@ -34,9 +34,10 @@
3434
},
3535
},
3636
},
37-
},
38-
schedulers: {
39-
MAIN_SCHEDULER: {
37+
],
38+
schedulers: [
39+
{
40+
name: "MAIN_SCHEDULER",
4041
simple: {
4142
supported_platform_properties: {
4243
cpu_count: "minimum",
@@ -48,7 +49,7 @@
4849
},
4950
},
5051
},
51-
},
52+
],
5253
workers: [
5354
{
5455
local: {
@@ -127,11 +128,11 @@
127128
},
128129
},
129130
],
130-
bytestream: {
131-
cas_stores: {
132-
"": "WORKER_FAST_SLOW_STORE",
131+
bytestream: [
132+
{
133+
cas_store: "WORKER_FAST_SLOW_STORE",
133134
},
134-
},
135+
],
135136
},
136137
},
137138
{

‎toolchain-examples/rbe-toolchain-test.nix‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{
2+
nativelink,
3+
writeShellScriptBin,
4+
bazelisk,
5+
}:
6+
writeShellScriptBin "rbe-toolchain-test" ''
7+
set -uo pipefail
8+
9+
cleanup() {
10+
local pids=$(jobs -pr)
11+
[ -n "$pids" ] && kill $pids
12+
}
13+
trap "cleanup" INT QUIT TERM EXIT
14+
15+
NO_COLOR=true ${nativelink}/bin/nativelink -- toolchain-examples/nativelink-config.json5 | tee -i toolchain-examples/nativelink.log &
16+
17+
CORE_BAZEL_ARGS="${bazelisk}/bin/bazelisk build --check_direct_dependencies=error --remote_cache=grpc://localhost:50051 --remote_executor=grpc://localhost:50051"
18+
19+
# As per https://nativelink.com/docs/rbe/remote-execution-examples#minimal-example-targets
20+
COMMANDS=("//cpp --config=zig-cc"
21+
"//cpp --config=llvm"
22+
"//python"
23+
"//go --config=zig-cc"
24+
# "//rust --config=zig-cc" # rules_rust isn't RBE-compatible
25+
"//java:HelloWorld --config=java"
26+
"@curl//... --config=zig-cc"
27+
"@zstd//... --config=zig-cc"
28+
# "@abseil-cpp//... --config=zig-cc" # Buggy build due to google_benchmark errors
29+
"@abseil-py//..."
30+
"@circl//... --config=zig-cc"
31+
)
32+
33+
echo "" > toolchain-examples/cmd.log
34+
for cmd in "''${COMMANDS[@]}"
35+
do
36+
FULL_CMD="$CORE_BAZEL_ARGS $cmd"
37+
echo $FULL_CMD
38+
echo -e \\n$FULL_CMD\\n >> toolchain-examples/cmd.log
39+
cmd_output=$(cd toolchain-examples && eval "$FULL_CMD" 2>&1 | tee -ai cmd.log)
40+
cmd_exit_code=$?
41+
case $cmd_exit_code in
42+
0 )
43+
echo "Saw a successful $cmd build"
44+
;;
45+
*)
46+
echo "Failed $cmd build:"
47+
echo $cmd_output
48+
exit 1
49+
;;
50+
esac
51+
done
52+
53+
nativelink_output=$(cat toolchain-examples/nativelink.log)
54+
55+
case $nativelink_output in
56+
*"ERROR "* )
57+
echo "Error in nativelink build"
58+
exit 1
59+
;;
60+
*)
61+
echo 'Successful nativelink build'
62+
;;
63+
esac
64+
''

0 commit comments

Comments
 (0)