fix(attest): derive simulated trust anchors inside the guest

Author: kvinwang

dstack/Cargo.lock

@@ -62,9 +62,22 @@ into verifier/KMS/gateway. The independently running host collateral service
 reconstructs the same hierarchy from the seed. Configure it under
 `[attestation.urls]`: TDX uses `pccs`, and SEV-SNP uses `amd_kds`.
 
-Every verifier process must also explicitly set
+The guest needs those roots too, to verify the KMS and the gateway it talks to.
+They do not travel from the host: `dstack-tee-simulator` derives them from the
+same seed and writes them to `/run/dstack/attestation`, guest tmpfs the host
+cannot reach, before `dstack-prepare` starts. `dstack-util` reads that one
+directory and nothing else, so a host can never nominate the trust anchor that
+authenticates its guest's key provider. Only the development image ships the
+simulator, and image contents are measured, so on a production image the
+directory never exists and vendor production roots are the only outcome.
+
+Every service configured with a mock root through its own TOML — KMS, gateway,
+`dstack-verifier` — must also explicitly set
 `attestation.insecure_allow_external_trust_anchors = true`. Merely mounting and
 configuring a mock root is rejected at startup while this flag remains false.
+The flag exists to make an operator acknowledge a hand-written non-production
+root, so it has no counterpart in the guest handoff above, where one program
+writes the roots and the next reads them out of a directory it authenticates.
 
 The seed adds only 64 hex bytes (the simulator config is well below 1 KiB).
 

dstack/crates/mock-attestation/README.md

@@ -18,7 +18,7 @@ dstack-types.workspace = true
 ez-hash.workspace = true
 fs-err.workspace = true
 safe-write.workspace = true
-rustix.workspace = true
+rustix = { workspace = true, features = ["process"] }
 hex.workspace = true
 hex_fmt.workspace = true
 or-panic.workspace = true
@@ -62,3 +62,5 @@ quote = [
 futures = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 dstack-mr = { workspace = true }
+rcgen = { workspace = true }
+tempfile = { workspace = true }

dstack/dstack-attest/Cargo.toml

@@ -61,7 +61,6 @@ pub struct AttestationVerifier {
     aws_nitro_tpm: nsm_qvl::QuoteVerifier,
     sev_snp: sev_snp_qvl::QuoteVerifier,
     amd_kds: AmdKdsClient,
-    external_trust_anchors: bool,
 }
 
 impl AttestationVerifier {
@@ -149,7 +148,6 @@ impl AttestationVerifier {
             aws_nitro_tpm: nsm(aws_nitro_tpm.as_deref(), "AWS NitroTPM")?,
             sev_snp,
             amd_kds: AmdKdsClient::with_base_url(amd_kds)?,
-            external_trust_anchors: external_requested,
         })
     }
 
@@ -175,33 +173,9 @@ impl AttestationVerifier {
                     .filter(|url| !url.trim().is_empty())
                     .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL),
             )?,
-            external_trust_anchors: false,
         })
     }
 
-    /// Construct a verifier with a development-only external TDX trust root.
-    ///
-    /// Callers must require an explicit insecure opt-in and surface the result
-    /// as simulated evidence; production verification must use `new_prod`.
-    pub fn new_with_tdx_root(
-        collateral_urls: Option<&CollateralUrls>,
-        root_ca: &[u8],
-    ) -> Result<Self> {
-        validate_x509_certificate(root_ca, "TDX")?;
-        let mut verifier = Self::new_prod(collateral_urls)?;
-        verifier.tdx = dcap_qvl::verify::QuoteVerifier::new(tdx_root_der(root_ca.to_vec())?);
-        verifier.external_trust_anchors = true;
-        Ok(verifier)
-    }
-
-    /// Whether this verifier accepts development-only external trust roots.
-    ///
-    /// A true value must be surfaced as simulated evidence by every caller;
-    /// production roots never set this flag.
-    pub fn is_simulated(&self) -> bool {
-        self.external_trust_anchors
-    }
-
     async fn verify_tdx_quote(&self, quote: &[u8]) -> Result<TdxVerifiedReport> {
         let collateral = self.tdx_collateral.fetch(quote).await?;
         let now = SystemTime::now()

dstack/dstack-attest/src/attestation.rs

@@ -17,12 +17,48 @@ pub mod attestation;
 mod aws_nitro_tpm;
 #[cfg(feature = "quote")]
 mod sev_snp;
+pub mod trust_anchors;
 mod v1;
 
 const RUNTIME_EVENT_DIR: &str = "/run/log/dstack";
 const RUNTIME_EVENT_VERSION_FILE: &str = "/run/log/dstack/runtime_event_version";
 const RUNTIME_EVENT_LOCK_FILE: &str = "/run/log/dstack/runtime_event.lock";
 
+/// Build the verifier a guest authenticates the KMS and the gateway with.
+///
+/// Trust anchors are taken from [`trust_anchors::ANCHOR_DIR`] when that
+/// directory holds a set published inside this guest. When it does not — the
+/// only outcome on a production image — the vendor production roots apply.
+///
+/// `collateral_urls` selects where signed collateral is fetched from; the trust
+/// anchor still has to sign it.
+pub fn default_verifier(
+    collateral_urls: &attestation::CollateralUrls,
+) -> anyhow::Result<attestation::AttestationVerifier> {
+    use attestation::{AttestationVerifier, AttestationVerifierConfig};
+
+    let Some(root_ca) =
+        trust_anchors::load_anchors(std::path::Path::new(trust_anchors::ANCHOR_DIR))
+            .context("failed to load local attestation anchors")?
+    else {
+        return AttestationVerifier::new_prod(Some(collateral_urls));
+    };
+    tracing::warn!(
+        dir = trust_anchors::ANCHOR_DIR,
+        "verifying attestation against external trust anchors published by the in-guest TEE \
+         simulator; this guest cannot verify production evidence"
+    );
+    AttestationVerifier::load(&AttestationVerifierConfig {
+        // The opt-in exists to make an operator acknowledge a non-production
+        // root in a hand-written service config. Nothing here is hand-written:
+        // the roots came from a guest-local directory `load_anchors` already
+        // authenticated, so the flag has no one left to warn.
+        insecure_allow_external_trust_anchors: true,
+        urls: collateral_urls.clone(),
+        root_ca,
+    })
+}
+
 /// Acquire the system-wide runtime event lock, blocking until it is available.
 ///
 /// The wait is deliberately unbounded. The lock serializes the event-log append

dstack/dstack-attest/src/lib.rs

@@ -0,0 +1,207 @@
+// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! Trust anchors published inside a guest, and the checks that make them
+//! trustworthy to read.
+//!
+//! A CVM must never let its host pick the trust anchor that verifies remote
+//! attestation. The host sits outside the trust boundary, so a host-supplied
+//! root would let it stand up a fake key provider and hand the guest keys it
+//! never earned.
+//!
+//! An image that must verify non-production evidence still needs external
+//! roots, so that handoff runs entirely inside the guest: `dstack-tee-simulator`
+//! derives them from its seed and writes them to [`ANCHOR_DIR`], a tmpfs
+//! directory the host cannot reach. Only the development image ships the
+//! simulator, and image contents are measured, so on a production image nothing
+//! ever creates that directory and the vendor production roots are the only
+//! reachable outcome.
+//!
+//! [`crate::default_verifier`] is the only thing that should act on what
+//! [`load_anchors`] returns.
+
+use std::{
+    os::unix::fs::MetadataExt as _,
+    path::{Path, PathBuf},
+};
+
+use anyhow::{bail, Context, Result};
+
+use crate::attestation::RootCaPaths;
+
+/// Guest tmpfs directory carrying locally published trust anchors.
+pub const ANCHOR_DIR: &str = "/run/dstack/attestation";
+
+const ROOTS_FILE: &str = "roots.json";
+
+/// Path of the published [`RootCaPaths`] within a trust anchor directory.
+///
+/// The publisher writes it; [`load_anchors`] is the only reader.
+pub fn roots_path(dir: &Path) -> PathBuf {
+    dir.join(ROOTS_FILE)
+}
+
+/// Load trust anchors published inside this guest, if any.
+///
+/// Returns `Ok(None)` when nothing published anchors, which is the only outcome
+/// on a production image.
+pub fn load_anchors(dir: &Path) -> Result<Option<RootCaPaths>> {
+    let path = roots_path(dir);
+    if !path.exists() {
+        return Ok(None);
+    }
+    ensure_owned_and_unwritable(dir, "trust anchor directory")?;
+    let meta = ensure_owned_and_unwritable(&path, "published roots")?;
+    if !meta.is_file() {
+        bail!("published roots is not a regular file");
+    }
+    let root_ca: RootCaPaths =
+        serde_json::from_slice(&fs_err::read(&path).context("failed to read published roots")?)
+            .context("failed to parse published roots")?;
+    for root in [
+        &root_ca.tdx,
+        &root_ca.gcp_tpm,
+        &root_ca.aws_nitro_enclave,
+        &root_ca.aws_nitro_tpm,
+        &root_ca.sev_snp_milan,
+        &root_ca.sev_snp_genoa,
+        &root_ca.sev_snp_turin,
+    ]
+    .into_iter()
+    .flatten()
+    {
+        // Confining every root to the published directory keeps a stale or
+        // tampered file from redirecting the verifier at a host-shared root.
+        if root.parent() != Some(dir) || root.file_name().is_none() {
+            bail!(
+                "trust anchor {} is outside {}",
+                root.display(),
+                dir.display()
+            );
+        }
+        ensure_owned_and_unwritable(root, "trust anchor")?;
+    }
+    Ok(Some(root_ca))
+}
+
+/// Reject anything this process does not own or that others could rewrite.
+///
+/// Symlink metadata, not the followed target: a symlink planted by another user
+/// would otherwise pass the check while resolving somewhere unowned.
+fn ensure_owned_and_unwritable(path: &Path, what: &str) -> Result<std::fs::Metadata> {
+    let meta = fs_err::symlink_metadata(path)
+        .with_context(|| format!("failed to stat {what} {}", path.display()))?;
+    let euid = rustix::process::geteuid().as_raw();
+    if meta.uid() != euid {
+        bail!(
+            "{what} {} is owned by uid {} instead of {euid}",
+            path.display(),
+            meta.uid()
+        );
+    }
+    if meta.mode() & 0o022 != 0 {
+        bail!(
+            "{what} {} is writable by group or others (mode {:o})",
+            path.display(),
+            meta.mode() & 0o7777
+        );
+    }
+    Ok(meta)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    // Mirrors what `crate::default_verifier` does with a loaded set, so the
+    // published roots are proven usable by the real verifier.
+    use crate::attestation::{AttestationVerifier, AttestationVerifierConfig};
+    use std::os::unix::fs::PermissionsExt as _;
+
+    fn sample_root() -> String {
+        let key = rcgen::KeyPair::generate_for(&rcgen::PKCS_ECDSA_P256_SHA256).unwrap();
+        let mut params = rcgen::CertificateParams::new(vec![]).unwrap();
+        params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
+        params.self_signed(&key).unwrap().pem()
+    }
+
+    /// Stand in for the publisher, which lives in `dstack-tee-simulator`.
+    fn publish(dir: &Path, tdx_root: &str) -> RootCaPaths {
+        fs_err::create_dir_all(dir).unwrap();
+        fs_err::set_permissions(dir, std::fs::Permissions::from_mode(0o700)).unwrap();
+        let root = dir.join("tdx-root-ca.pem");
+        safe_write::safe_write_with_mode(&root, tdx_root.as_bytes(), 0o600).unwrap();
+        let root_ca = RootCaPaths {
+            tdx: Some(root),
+            ..Default::default()
+        };
+        write_roots(dir, &root_ca);
+        root_ca
+    }
+
+    fn write_roots(dir: &Path, root_ca: &RootCaPaths) {
+        safe_write::safe_write_with_mode(
+            roots_path(dir),
+            serde_json::to_vec(root_ca).unwrap(),
+            0o600,
+        )
+        .unwrap();
+    }
+
+    fn verifier_for(root_ca: RootCaPaths) -> Result<AttestationVerifier> {
+        AttestationVerifier::load(&AttestationVerifierConfig {
+            insecure_allow_external_trust_anchors: true,
+            urls: Default::default(),
+            root_ca,
+        })
+    }
+
+    #[test]
+    fn absent_directory_selects_production_roots() {
+        let dir = tempfile::tempdir().unwrap();
+        assert!(load_anchors(&dir.path().join("missing")).unwrap().is_none());
+    }
+
+    #[test]
+    fn published_roots_round_trip() {
+        let dir = tempfile::tempdir().unwrap();
+        let dir = dir.path().join("attestation");
+        let published = publish(&dir, &sample_root());
+
+        let root_ca = load_anchors(&dir).unwrap().expect("anchors should load");
+        assert_eq!(root_ca.tdx, published.tdx);
+        assert_eq!(root_ca.gcp_tpm, None);
+        // What was published must be loadable by the real verifier.
+        verifier_for(root_ca).unwrap();
+    }
+
+    #[test]
+    fn a_malformed_root_fails_verifier_construction() {
+        let dir = tempfile::tempdir().unwrap();
+        let dir = dir.path().join("attestation");
+        publish(&dir, "not a certificate");
+        let root_ca = load_anchors(&dir).unwrap().unwrap();
+        assert!(verifier_for(root_ca).is_err());
+    }
+
+    #[test]
+    fn world_writable_roots_are_rejected() {
+        let dir = tempfile::tempdir().unwrap();
+        let dir = dir.path().join("attestation");
+        publish(&dir, &sample_root());
+        fs_err::set_permissions(roots_path(&dir), std::fs::Permissions::from_mode(0o666)).unwrap();
+        let error = load_anchors(&dir).unwrap_err().to_string();
+        assert!(error.contains("writable by group or others"), "{error}");
+    }
+
+    #[test]
+    fn trust_anchor_outside_the_directory_is_rejected() {
+        let dir = tempfile::tempdir().unwrap();
+        let anchors = dir.path().join("attestation");
+        let mut root_ca = publish(&anchors, &sample_root());
+        root_ca.tdx = Some(dir.path().join("host-shared-root.pem"));
+        write_roots(&anchors, &root_ca);
+        let error = load_anchors(&anchors).unwrap_err().to_string();
+        assert!(error.contains("is outside"), "{error}");
+    }
+}