Skip to content

Commit e649222

Browse files
committed
new: Support Depot for remote caching. (#1792)
* Add settings. * More testing. * Get things working. * Add new action state. * Update impls. * Fix output. * Fix tests. * Fixes. * Change types. * Fix target state. * Improve errors. * Add backtrace. * Try from_str. * Fix tests. * More debugging. * Add docs. * Polish. * Bump versions.
1 parent fa43ab1 commit e649222

28 files changed

+951
-611
lines changed

.github/workflows/moon.yml

+2
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,11 @@ jobs:
3737
cache-base: '^(master|develop-)'
3838
- run: cargo run -- --color --log trace ci --base ${{ github.base_ref || 'master' }}
3939
env:
40+
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
4041
MOON_NODE_VERSION: ${{ matrix.node-version }}
4142
MOONBASE_SECRET_KEY: ${{ secrets.MOONBASE_SECRET_KEY }}
4243
MOONBASE_ACCESS_KEY: ${{ secrets.MOONBASE_ACCESS_KEY }}
44+
RUST_BACKTRACE: '1'
4345
- uses: moonrepo/run-report-action@v1
4446
if: success() || failure()
4547
with:

.moon/workspace.yml

+19-14
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,22 @@ docker:
3535
include:
3636
- '*.config.js'
3737
- '*.json'
38-
# unstable_remote:
39-
# host: 'http://0.0.0.0:8080'
40-
# # host: 'grpc://0.0.0.0:9092'
41-
# cache:
42-
# compression: 'zstd'
43-
# mtls:
44-
# caCert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem'
45-
# clientCert: 'crates/remote/tests/__fixtures__/certs-local/client.pem'
46-
# clientKey: 'crates/remote/tests/__fixtures__/certs-local/client.key'
47-
# domain: 'localhost'
48-
# tls:
49-
# # assumeHttp2: true
50-
# cert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem'
51-
# # domain: 'localhost'
38+
39+
unstable_remote:
40+
host: 'grpcs://cache.depot.dev'
41+
auth:
42+
token: 'DEPOT_TOKEN'
43+
headers:
44+
'X-Depot-Org': '1xtpjd084j'
45+
'X-Depot-Project': '90xxfkst9n'
46+
# cache:
47+
# compression: 'zstd'
48+
# mtls:
49+
# caCert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem'
50+
# clientCert: 'crates/remote/tests/__fixtures__/certs-local/client.pem'
51+
# clientKey: 'crates/remote/tests/__fixtures__/certs-local/client.key'
52+
# domain: 'localhost'
53+
# tls:
54+
# # assumeHttp2: true
55+
# cert: 'crates/remote/tests/__fixtures__/certs-local/client.pem'
56+
# domain: 'localhost'

.yarn/versions/c89cc34c.yml

+15-8
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
releases:
2-
"@moonrepo/cli": minor
3-
"@moonrepo/core-linux-arm64-gnu": minor
4-
"@moonrepo/core-linux-arm64-musl": minor
5-
"@moonrepo/core-linux-x64-gnu": minor
6-
"@moonrepo/core-linux-x64-musl": minor
7-
"@moonrepo/core-macos-arm64": minor
8-
"@moonrepo/core-macos-x64": minor
9-
"@moonrepo/core-windows-x64-msvc": minor
2+
'@moonrepo/cli': minor
3+
'@moonrepo/core-linux-arm64-gnu': minor
4+
'@moonrepo/core-linux-arm64-musl': minor
5+
'@moonrepo/core-linux-x64-gnu': minor
6+
'@moonrepo/core-linux-x64-musl': minor
7+
'@moonrepo/core-macos-arm64': minor
8+
'@moonrepo/core-macos-x64': minor
9+
'@moonrepo/core-windows-x64-msvc': minor
10+
'@moonrepo/types': minor
11+
12+
declined:
13+
- '@moonrepo/nx-compat'
14+
- '@moonrepo/report'
15+
- '@moonrepo/runtime'
16+
- website

CHANGELOG.md

+12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
11
# Changelog
22

3+
## Unreleased
4+
5+
#### 🚀 Updates
6+
7+
- Updated our unstable remote service (Bazel RE API) with new functionality:
8+
- You can now use `http(s)` protocols for gRPC servers, instead of just `grpc(s)`.
9+
- Added an `unstable_remote.api` setting, which can be used to inform the server's API format.
10+
Defaults to `grpc`.
11+
- Added an `unstable_remote.auth` setting, which can be used for HTTP Bearer/token Authorization
12+
based endpoints. Can also be used to set headers for all requests.
13+
- Added support for Depot cloud-based caching: https://depot.dev/docs/cache/overview
14+
315
## 1.31.3
416

517
#### 🐞 Fixes

Cargo.lock

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/config/src/workspace/remote_config.rs

+38-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use crate::portable_path::FilePath;
2+
use rustc_hash::FxHashMap;
23
use schematic::{derive_enum, validate, Config, ConfigEnum, ValidateError, ValidateResult};
34

45
fn path_is_required<D, C>(
@@ -15,6 +16,27 @@ fn path_is_required<D, C>(
1516
}
1617

1718
derive_enum!(
19+
/// The API format of the remote service.
20+
#[derive(Copy, ConfigEnum, Default)]
21+
pub enum RemoteApi {
22+
/// gRPC endpoints.
23+
#[default]
24+
Grpc,
25+
}
26+
);
27+
28+
/// Configures basic HTTP authentication.
29+
#[derive(Clone, Config, Debug)]
30+
pub struct RemoteAuthConfig {
31+
/// HTTP headers to inject into every request.
32+
pub headers: FxHashMap<String, String>,
33+
34+
/// The name of an environment variable to use as a bearer token.
35+
pub token: Option<String>,
36+
}
37+
38+
derive_enum!(
39+
/// Supported blob compression levels.
1840
#[derive(Copy, ConfigEnum, Default)]
1941
pub enum RemoteCompression {
2042
/// No compression.
@@ -81,6 +103,13 @@ pub struct RemoteMtlsConfig {
81103
/// Configures the remote service, powered by the Bazel Remote Execution API.
82104
#[derive(Clone, Config, Debug)]
83105
pub struct RemoteConfig {
106+
/// The API format of the remote service.
107+
pub api: RemoteApi,
108+
109+
/// Connect to the host using basic HTTP authentication.
110+
#[setting(nested)]
111+
pub auth: Option<RemoteAuthConfig>,
112+
84113
/// Configures the action cache (AC) and content addressable cache (CAS).
85114
#[setting(nested)]
86115
pub cache: RemoteCacheConfig,
@@ -101,11 +130,19 @@ pub struct RemoteConfig {
101130
}
102131

103132
impl RemoteConfig {
133+
pub fn is_bearer_auth(&self) -> bool {
134+
self.auth.as_ref().is_some_and(|auth| auth.token.is_some())
135+
}
136+
104137
pub fn is_localhost(&self) -> bool {
105138
self.host.contains("localhost") || self.host.contains("0.0.0.0")
106139
}
107140

108141
pub fn is_secure(&self) -> bool {
109-
self.tls.is_some() || self.mtls.is_some()
142+
self.is_bearer_auth() || self.tls.is_some() || self.mtls.is_some()
143+
}
144+
145+
pub fn is_secure_protocol(&self) -> bool {
146+
self.host.starts_with("https") || self.host.starts_with("grpcs")
110147
}
111148
}

crates/remote/Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ publish = false
88
moon_action = { path = "../action" }
99
moon_common = { path = "../common" }
1010
moon_config = { path = "../config" }
11+
moon_task = { path = "../task" }
1112
async-trait = { workspace = true }
1213
bazel-remote-apis = { version = "0.12.0", features = ["serde"] }
14+
bincode = "1.3.3"
1315
chrono = { workspace = true }
1416
miette = { workspace = true }
1517
reqwest = { workspace = true, features = ["json"] }

crates/remote/src/action_state.rs

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
use crate::fs_digest::*;
2+
use bazel_remote_apis::build::bazel::remote::execution::v2::{
3+
command, platform, Action, ActionResult, Command, Digest, ExecutedActionMetadata,
4+
};
5+
use miette::IntoDiagnostic;
6+
use moon_action::Operation;
7+
use moon_task::Task;
8+
use std::collections::BTreeMap;
9+
use std::path::Path;
10+
11+
pub struct ActionState<'task> {
12+
task: &'task Task,
13+
14+
// RE API
15+
pub action: Option<Action>,
16+
pub action_result: Option<ActionResult>,
17+
pub command: Option<Command>,
18+
pub digest: Digest,
19+
20+
// To upload
21+
pub blobs: Vec<Blob>,
22+
}
23+
24+
impl ActionState<'_> {
25+
pub fn new(digest: Digest, task: &Task) -> ActionState<'_> {
26+
ActionState {
27+
task,
28+
action: None,
29+
action_result: None,
30+
command: None,
31+
digest,
32+
blobs: vec![],
33+
}
34+
}
35+
36+
pub fn create_action_from_task(&mut self) {
37+
let mut action = Action {
38+
command_digest: Some(self.digest.clone()),
39+
do_not_cache: !self.task.options.cache,
40+
input_root_digest: None, // TODO?
41+
..Default::default()
42+
};
43+
44+
// https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/platform.md
45+
if let Some(os_list) = &self.task.options.os {
46+
let platform = action.platform.get_or_insert_default();
47+
48+
for os in os_list {
49+
platform.properties.push(platform::Property {
50+
name: "OSFamily".into(),
51+
value: os.to_string(),
52+
});
53+
}
54+
}
55+
56+
// Since we don't support (or plan to) remote execution,
57+
// then we can ignore all the working directory logic
58+
let mut command = Command {
59+
arguments: vec![self.task.command.clone()],
60+
output_paths: vec![], // TODO
61+
..Default::default()
62+
};
63+
64+
command.arguments.extend(self.task.args.clone());
65+
66+
for (name, value) in BTreeMap::from_iter(self.task.env.clone()) {
67+
command
68+
.environment_variables
69+
.push(command::EnvironmentVariable { name, value });
70+
}
71+
72+
self.action = Some(action);
73+
self.command = Some(command);
74+
}
75+
76+
pub fn create_action_result_from_operation(
77+
&mut self,
78+
operation: &Operation,
79+
) -> miette::Result<()> {
80+
let mut result = ActionResult {
81+
execution_metadata: Some(ExecutedActionMetadata {
82+
worker: "moon".into(),
83+
execution_start_timestamp: create_timestamp_from_naive(operation.started_at),
84+
execution_completed_timestamp: operation
85+
.finished_at
86+
.and_then(create_timestamp_from_naive),
87+
..Default::default()
88+
}),
89+
..Default::default()
90+
};
91+
92+
if let Some(exec) = operation.get_output() {
93+
result.exit_code = exec.exit_code.unwrap_or_default();
94+
95+
if let Some(stderr) = &exec.stderr {
96+
let blob = Blob::new(stderr.as_bytes().to_owned());
97+
98+
result.stderr_digest = Some(blob.digest.clone());
99+
self.blobs.push(blob);
100+
}
101+
102+
if let Some(stdout) = &exec.stdout {
103+
let blob = Blob::new(stdout.as_bytes().to_owned());
104+
105+
result.stdout_digest = Some(blob.digest.clone());
106+
self.blobs.push(blob);
107+
}
108+
}
109+
110+
self.action_result = Some(result);
111+
112+
Ok(())
113+
}
114+
115+
pub fn set_action_result(&mut self, result: ActionResult) {
116+
self.action_result = Some(result);
117+
}
118+
119+
pub fn compute_outputs(&mut self, workspace_root: &Path) -> miette::Result<()> {
120+
let mut outputs = OutputDigests::default();
121+
122+
for path in self.task.get_output_files(workspace_root, true)? {
123+
outputs.insert_relative_path(path, workspace_root)?;
124+
}
125+
126+
if let Some(result) = &mut self.action_result {
127+
result.output_files = outputs.files;
128+
result.output_symlinks = outputs.symlinks;
129+
result.output_directories = outputs.dirs;
130+
self.blobs.extend(outputs.blobs);
131+
}
132+
133+
Ok(())
134+
}
135+
136+
pub fn get_command_as_bytes(&self) -> miette::Result<Vec<u8>> {
137+
bincode::serialize(&self.command).into_diagnostic()
138+
}
139+
140+
pub fn prepare_for_upload(&mut self) -> Option<(ActionResult, Vec<Blob>)> {
141+
self.action_result
142+
.take()
143+
.map(|result| (result, self.blobs.drain(0..).collect::<Vec<_>>()))
144+
}
145+
}

crates/remote/src/compression.rs

+15
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,21 @@ pub fn get_compressor(compression: RemoteCompression) -> i32 {
2323
}
2424
}
2525

26+
#[allow(dead_code)]
27+
pub fn get_compression_from_value(compressor: compressor::Value) -> RemoteCompression {
28+
match compressor {
29+
compressor::Value::Zstd => RemoteCompression::Zstd,
30+
_ => RemoteCompression::None,
31+
}
32+
}
33+
34+
pub fn get_compression_from_code(compressor: i32) -> RemoteCompression {
35+
match compressor {
36+
zstd if zstd == compressor::Value::Zstd as i32 => RemoteCompression::Zstd,
37+
_ => RemoteCompression::None,
38+
}
39+
}
40+
2641
pub fn compress_blob(compression: RemoteCompression, bytes: Vec<u8>) -> miette::Result<Vec<u8>> {
2742
let result = match compression {
2843
RemoteCompression::None => Ok(bytes),

crates/remote/src/fs_digest.rs

+2-16
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use bazel_remote_apis::build::bazel::remote::execution::v2::{
44
Digest, NodeProperties, OutputDirectory, OutputFile, OutputSymlink,
55
};
6-
use bazel_remote_apis::google::protobuf::{Timestamp, UInt32Value};
6+
use bazel_remote_apis::google::protobuf::Timestamp;
77
use chrono::NaiveDateTime;
88
use moon_common::path::{PathExt, WorkspaceRelativePathBuf};
99
use sha2::{Digest as Sha256Digest, Sha256};
@@ -14,7 +14,6 @@ use std::{
1414
path::{Path, PathBuf},
1515
time::{Duration, SystemTime, UNIX_EPOCH},
1616
};
17-
use tracing::instrument;
1817

1918
pub struct Blob {
2019
pub bytes: Vec<u8>,
@@ -77,6 +76,7 @@ pub fn compute_node_properties(metadata: &Metadata) -> NodeProperties {
7776

7877
#[cfg(unix)]
7978
{
79+
use bazel_remote_apis::google::protobuf::UInt32Value;
8080
use std::os::unix::fs::PermissionsExt;
8181

8282
props.unix_mode = Some(UInt32Value {
@@ -157,20 +157,6 @@ impl OutputDigests {
157157
}
158158
}
159159

160-
#[instrument]
161-
pub fn compute_digests_for_outputs(
162-
paths: Vec<WorkspaceRelativePathBuf>,
163-
workspace_root: &Path,
164-
) -> miette::Result<OutputDigests> {
165-
let mut result = OutputDigests::default();
166-
167-
for path in paths {
168-
result.insert_relative_path(path, workspace_root)?;
169-
}
170-
171-
Ok(result)
172-
}
173-
174160
fn apply_node_properties(path: &Path, props: &NodeProperties) -> miette::Result<()> {
175161
if let Some(mtime) = &props.mtime {
176162
let modified = Duration::new(mtime.seconds as u64, mtime.nanos as u32);

0 commit comments

Comments
 (0)