Skip to content

Commit 33ffd57

Browse files
committed
feat(compute): add standalone first-party drivers
Build Docker, Podman, Kubernetes, and VM drivers as external binaries and exercise each through the public compute-driver API. Keep the external E2E setup complete at introduction, including VM image selection, Kubernetes post-renderer isolation, supervisor reuse, and scoped Podman coverage. Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent 64c20da commit 33ffd57

33 files changed

Lines changed: 1054 additions & 239 deletions

File tree

.github/workflows/branch-e2e.yml

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,20 @@ jobs:
196196
e2e-task: e2e:kubernetes:workspace-managed
197197
cli-artifact-prefix: rust-binary-cli
198198

199+
kubernetes-external-driver-e2e:
200+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
201+
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
202+
permissions:
203+
actions: read
204+
contents: read
205+
packages: read
206+
uses: ./.github/workflows/e2e-kubernetes-test.yml
207+
with:
208+
image-tag: ${{ github.sha }}
209+
job-name: Kubernetes E2E (external compute driver)
210+
e2e-task: e2e:kubernetes:external-driver
211+
cli-artifact-prefix: rust-binary-cli
212+
199213
kubernetes-workspace-operator-e2e:
200214
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
201215
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
@@ -240,7 +254,7 @@ jobs:
240254

241255
core-e2e-result:
242256
name: Core E2E result
243-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e]
257+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e]
244258
if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
245259
runs-on: ubuntu-latest
246260
steps:
@@ -252,6 +266,7 @@ jobs:
252266
BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }}
253267
E2E_RESULT: ${{ needs.e2e.result }}
254268
KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }}
269+
KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT: ${{ needs.kubernetes-external-driver-e2e.result }}
255270
KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }}
256271
KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }}
257272
run: |
@@ -264,6 +279,7 @@ jobs:
264279
"build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \
265280
"e2e:$E2E_RESULT" \
266281
"kubernetes-e2e:$KUBERNETES_E2E_RESULT" \
282+
"kubernetes-external-driver-e2e:$KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT" \
267283
"kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \
268284
"kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do
269285
name="${item%%:*}"

.github/workflows/e2e-test.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ jobs:
5959
- suite: rust-docker
6060
cmd: "mise run --no-deps --skip-deps e2e:rust"
6161
apt_packages: "openssh-client"
62+
- suite: rust-docker-external-driver
63+
cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver"
64+
apt_packages: "openssh-client"
6265
- suite: mcp
6366
cmd: "mise run --no-deps --skip-deps e2e:mcp"
6467
apt_packages: ""
@@ -264,6 +267,9 @@ jobs:
264267
- name: Run rootless Podman E2E
265268
run: mise run --no-deps --skip-deps e2e:podman:rootless
266269

270+
- name: Run external Podman driver E2E
271+
run: env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:podman:external-driver
272+
267273
- name: Print AppArmor denials
268274
if: always()
269275
run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true
@@ -362,4 +368,4 @@ jobs:
362368
cache-on-failure: "true"
363369

364370
- name: Run VM E2E
365-
run: mise run --no-deps --skip-deps e2e:vm
371+
run: env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:vm:external-driver

Cargo.lock

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

architecture/compute-runtimes.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,12 @@ to `run_cli_with_compute_drivers`; factories receive merged driver config and
108108
finish through the same in-process runtime adapter. A configured UDS endpoint
109109
still takes precedence over a compiled registration with the same name.
110110

111+
The standard server crate groups first-party registrations behind the
112+
`in-tree-compute-drivers` feature. Protocol-only gateway builds disable that
113+
feature and link no compute-driver crates. E2E lanes compose that gateway with
114+
Docker, Podman, Kubernetes, and VM driver executables over the public UDS gRPC
115+
contract so an in-tree driver cannot silently depend on a server-only API.
116+
111117
## Stop and Start Lifecycle
112118

113119
The gateway persists lifecycle intent before mutating compute:

crates/openshell-core/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ tempfile = { version = "3", optional = true }
3535

3636
[target.'cfg(unix)'.dependencies]
3737
nix = { workspace = true }
38+
rustix = { workspace = true }
3839

3940
[features]
4041
default = ["telemetry"]
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Public Unix-socket transport helpers for out-of-process drivers.
5+
6+
use std::io;
7+
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
8+
use std::path::{Path, PathBuf};
9+
use std::pin::Pin;
10+
use std::task::{Context, Poll};
11+
12+
use tokio::net::{UnixListener, UnixStream};
13+
use tokio_stream::Stream;
14+
15+
/// Prepare and bind a private Unix socket owned by the current effective UID.
16+
pub fn bind_private(path: &Path) -> Result<UnixListener, String> {
17+
let parent = path
18+
.parent()
19+
.ok_or_else(|| format!("driver socket path '{}' has no parent", path.display()))?;
20+
let expected_uid = rustix::process::geteuid().as_raw();
21+
std::fs::create_dir_all(parent)
22+
.map_err(|err| format!("create socket directory {}: {err}", parent.display()))?;
23+
let parent_metadata = std::fs::symlink_metadata(parent)
24+
.map_err(|err| format!("stat socket directory {}: {err}", parent.display()))?;
25+
if parent_metadata.file_type().is_symlink() || !parent_metadata.file_type().is_dir() {
26+
return Err(format!(
27+
"driver socket parent '{}' must be a directory, not a symlink",
28+
parent.display()
29+
));
30+
}
31+
if parent_metadata.uid() != expected_uid {
32+
return Err(format!(
33+
"driver socket parent '{}' is owned by uid {}, expected {}",
34+
parent.display(),
35+
parent_metadata.uid(),
36+
expected_uid
37+
));
38+
}
39+
std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))
40+
.map_err(|err| format!("chmod socket directory {}: {err}", parent.display()))?;
41+
42+
match std::fs::symlink_metadata(path) {
43+
Ok(metadata)
44+
if metadata.file_type().is_socket()
45+
&& !metadata.file_type().is_symlink()
46+
&& metadata.uid() == expected_uid =>
47+
{
48+
std::fs::remove_file(path)
49+
.map_err(|err| format!("remove stale socket {}: {err}", path.display()))?;
50+
}
51+
Ok(_) => {
52+
return Err(format!(
53+
"driver socket path '{}' exists but is not an owned Unix socket",
54+
path.display()
55+
));
56+
}
57+
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
58+
Err(err) => return Err(format!("stat driver socket {}: {err}", path.display())),
59+
}
60+
61+
let listener = UnixListener::bind(path)
62+
.map_err(|err| format!("bind driver socket {}: {err}", path.display()))?;
63+
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))
64+
.map_err(|err| format!("chmod driver socket {}: {err}", path.display()))?;
65+
Ok(listener)
66+
}
67+
68+
/// Remove a socket created by [`bind_private`].
69+
pub struct SocketCleanup(PathBuf);
70+
71+
impl SocketCleanup {
72+
#[must_use]
73+
pub fn new(path: PathBuf) -> Self {
74+
Self(path)
75+
}
76+
}
77+
78+
impl Drop for SocketCleanup {
79+
fn drop(&mut self) {
80+
let _ = std::fs::remove_file(&self.0);
81+
}
82+
}
83+
84+
/// Incoming UDS connections restricted to the driver's effective UID.
85+
pub struct SameUidUnixIncoming {
86+
listener: UnixListener,
87+
expected_uid: u32,
88+
}
89+
90+
impl SameUidUnixIncoming {
91+
#[must_use]
92+
pub fn new(listener: UnixListener) -> Self {
93+
Self {
94+
listener,
95+
expected_uid: rustix::process::geteuid().as_raw(),
96+
}
97+
}
98+
}
99+
100+
impl Stream for SameUidUnixIncoming {
101+
type Item = io::Result<UnixStream>;
102+
103+
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
104+
let this = self.get_mut();
105+
loop {
106+
match this.listener.poll_accept(cx) {
107+
Poll::Ready(Ok((stream, _))) => match stream.peer_cred() {
108+
Ok(credentials) if credentials.uid() == this.expected_uid => {
109+
return Poll::Ready(Some(Ok(stream)));
110+
}
111+
Ok(credentials) => tracing::warn!(
112+
peer_uid = credentials.uid(),
113+
expected_uid = this.expected_uid,
114+
"rejected external driver socket client"
115+
),
116+
Err(err) => {
117+
tracing::warn!(error = %err, "failed to authenticate driver socket client");
118+
}
119+
},
120+
Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))),
121+
Poll::Pending => return Poll::Pending,
122+
}
123+
}
124+
}
125+
}

crates/openshell-core/src/lib.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ pub mod driver_mounts;
1818
pub mod driver_utils;
1919
pub mod endpoint_path;
2020
pub mod error;
21+
#[cfg(unix)]
22+
pub mod external_driver_socket;
2123
pub mod forward;
2224
pub mod google_cloud;
2325
pub mod gpu;
@@ -29,6 +31,7 @@ pub mod jwt;
2931
pub mod metadata;
3032
pub mod middleware;
3133
pub mod net;
34+
pub mod operator_namespace_allowlist;
3235
pub mod paths;
3336
pub mod policy;
3437
pub mod progress;
@@ -53,6 +56,7 @@ pub use error::{ComputeDriverError, Error, Result};
5356
pub use metadata::{
5457
GetResourceVersion, ObjectId, ObjectLabels, ObjectName, ObjectWorkspace, SetResourceVersion,
5558
};
59+
pub use operator_namespace_allowlist::OperatorNamespaceAllowlist;
5660

5761
/// Build version string derived from git metadata.
5862
///
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
use std::collections::BTreeSet;
5+
use std::sync::{Arc, RwLock};
6+
7+
/// Thread-safe dynamic allowlist of Kubernetes operator-mode namespaces.
8+
///
9+
/// This type lives in the public core API because both the Kubernetes driver
10+
/// and gateway authentication boundary consume it.
11+
#[derive(Debug, Clone)]
12+
pub struct OperatorNamespaceAllowlist {
13+
inner: Arc<RwLock<BTreeSet<String>>>,
14+
}
15+
16+
impl OperatorNamespaceAllowlist {
17+
fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet<String>> {
18+
self.inner
19+
.read()
20+
.unwrap_or_else(std::sync::PoisonError::into_inner)
21+
}
22+
23+
fn write_guard(&self) -> std::sync::RwLockWriteGuard<'_, BTreeSet<String>> {
24+
self.inner
25+
.write()
26+
.unwrap_or_else(std::sync::PoisonError::into_inner)
27+
}
28+
29+
#[must_use]
30+
pub fn new() -> Self {
31+
Self {
32+
inner: Arc::new(RwLock::new(BTreeSet::new())),
33+
}
34+
}
35+
36+
#[must_use]
37+
pub fn from_set(set: BTreeSet<String>) -> Self {
38+
Self {
39+
inner: Arc::new(RwLock::new(set)),
40+
}
41+
}
42+
43+
pub fn replace(&self, new_set: BTreeSet<String>) {
44+
*self.write_guard() = new_set;
45+
}
46+
47+
pub fn merge(&self, additional: &BTreeSet<String>) {
48+
self.write_guard().extend(additional.iter().cloned());
49+
}
50+
51+
pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet<String>> {
52+
self.read_guard()
53+
}
54+
55+
#[must_use]
56+
pub fn contains(&self, namespace: &str) -> bool {
57+
self.read_guard().contains(namespace)
58+
}
59+
60+
pub fn insert(&self, name: String) -> bool {
61+
self.write_guard().insert(name)
62+
}
63+
64+
pub fn remove(&self, name: &str) -> bool {
65+
self.write_guard().remove(name)
66+
}
67+
68+
#[must_use]
69+
pub fn shared(&self) -> Arc<RwLock<BTreeSet<String>>> {
70+
Arc::clone(&self.inner)
71+
}
72+
}
73+
74+
impl Default for OperatorNamespaceAllowlist {
75+
fn default() -> Self {
76+
Self::new()
77+
}
78+
}

crates/openshell-driver-docker/Cargo.toml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,15 @@ rust-version.workspace = true
1010
license.workspace = true
1111
repository.workspace = true
1212

13+
[[bin]]
14+
name = "openshell-driver-docker"
15+
path = "src/main.rs"
16+
1317
[dependencies]
1418
openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] }
1519

1620
tokio = { workspace = true }
17-
tonic = { workspace = true }
21+
tonic = { workspace = true, features = ["transport"] }
1822
futures = { workspace = true }
1923
tokio-stream = { workspace = true }
2024
tracing = { workspace = true }
@@ -24,6 +28,10 @@ serde_json = { workspace = true }
2428
prost-types = { workspace = true }
2529
bollard = { version = "0.20" }
2630
url = { workspace = true }
31+
clap = { workspace = true }
32+
miette = { workspace = true }
33+
toml = { workspace = true }
34+
tracing-subscriber = { workspace = true }
2735

2836
[dev-dependencies]
2937
prost-types = { workspace = true }

0 commit comments

Comments
 (0)