feat(attest): accept several collateral endpoints instead of one

Attestation verification depends on collateral only the vendor publishes,
and until now each source was configured as exactly one URL. AMD KDS is the
sharper case: a single global endpoint with no official mirror, rate limited
to roughly one identical request per ten seconds, and unreachable for hours
at a time -- during which nothing on the host can verify an SEV-SNP quote,
so no AMD guest can obtain keys.

`pccs` and `amd_kds` now take either a string or a list. Endpoints are tried
in order and the first answer wins. Existing configs and already-serialized
`SysConfig` blobs parse unchanged, and a one-element list serializes back as
a bare string, because guests hash the sys-config they were given rather
than what a newer binary would have written.

Failing over relaxes nothing. Collateral is vendor-signed and checked
against roots compiled into the binary, so an endpoint can be slow, stale or
absent but cannot forge an answer; which endpoint replied has no bearing on
whether the signature checks pass. That is what makes a cache or mirror a
legitimate entry in the list rather than a hole in the trust model.

The two clients fail over on different rules, because they can see different
things:

- AMD KDS is in-tree, so it classifies. Transport failures, 408, 429 and 5xx
  say something about the endpoint and move on; a 404 is an answer about the
  chip that every mirror repeats, so it surfaces as-is rather than being
  retried into a pile of identical errors.
- PCCS goes through `dcap-qvl`, which returns `anyhow::Error` with no status
  to classify on. It fails over on any error. Matching on error strings to
  do better would be worse than the coarse rule it replaced.

Tests drive real sockets rather than asserting on the shape of the code: a
429 and a refused connection each hand off, a 404 stops with the second
endpoint never contacted, and an all-fail run names every endpoint it tried.

Author: kvinwang

dstack/dstack-attest/src/attestation.rs

@@ -11,10 +11,10 @@ pub const TDX_QUOTE_REPORT_DATA_RANGE: std::ops::Range<usize> = 568..632;
 
 use std::{borrow::Cow, time::SystemTime};
 
+use crate::collateral::{amd_kds_client, PccsClient};
 use anyhow::{anyhow, bail, Context, Result};
 use cc_eventlog::{EventLogVersion, RuntimeEvent, TdxEvent};
 use dcap_qvl::{
-    collateral::CollateralClient,
     quote::{EnclaveReport, Quote, Report, TDReport10, TDReport15},
     verify::VerifiedReport as TdxVerifiedReport,
 };
@@ -57,7 +57,7 @@ pub struct AttestationVerifierConfig {
 
 pub struct AttestationVerifier {
     tdx: dcap_qvl::verify::QuoteVerifier,
-    tdx_collateral: CollateralClient,
+    tdx_collateral: PccsClient,
     gcp_tpm: tpm_qvl::QuoteVerifier,
     aws_nitro_enclave: nsm_qvl::QuoteVerifier,
     aws_nitro_tpm: nsm_qvl::QuoteVerifier,
@@ -130,51 +130,27 @@ impl AttestationVerifier {
                 sev_snp = sev_snp.with_root(product, root);
             }
         }
-        let pccs = config
-            .urls
-            .pccs
-            .as_deref()
-            .filter(|v| !v.trim().is_empty())
-            .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL);
-        let amd_kds = config
-            .urls
-            .amd_kds
-            .as_deref()
-            .filter(|v| !v.trim().is_empty())
-            .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL);
         Ok(Self {
             tdx,
-            tdx_collateral: CollateralClient::with_default_http(pccs)?,
+            tdx_collateral: PccsClient::new(&config.urls.pccs)?,
             gcp_tpm,
             aws_nitro_enclave: nsm(aws_nitro_enclave.as_deref(), "AWS Nitro Enclave")?,
             aws_nitro_tpm: nsm(aws_nitro_tpm.as_deref(), "AWS NitroTPM")?,
             sev_snp,
-            amd_kds: AmdKdsClient::with_base_url(amd_kds)?,
+            amd_kds: amd_kds_client(&config.urls.amd_kds)?,
         })
     }
 
     pub fn new_prod(collateral_urls: Option<&CollateralUrls>) -> Result<Self> {
         let collateral_urls = collateral_urls.cloned().unwrap_or_default();
         Ok(Self {
             tdx: dcap_qvl::verify::QuoteVerifier::new_prod(),
-            tdx_collateral: CollateralClient::with_default_http(
-                collateral_urls
-                    .pccs
-                    .as_deref()
-                    .filter(|url| !url.trim().is_empty())
-                    .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL),
-            )?,
+            tdx_collateral: PccsClient::new(&collateral_urls.pccs)?,
             gcp_tpm: tpm_qvl::QuoteVerifier::new_prod(Platform::Gcp)?,
             aws_nitro_enclave: nsm_qvl::QuoteVerifier::new_prod(),
             aws_nitro_tpm: nsm_qvl::QuoteVerifier::new_prod(),
             sev_snp: sev_snp_qvl::QuoteVerifier::new_prod(),
-            amd_kds: AmdKdsClient::with_base_url(
-                collateral_urls
-                    .amd_kds
-                    .as_deref()
-                    .filter(|url| !url.trim().is_empty())
-                    .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL),
-            )?,
+            amd_kds: amd_kds_client(&collateral_urls.amd_kds)?,
         })
     }
 

dstack/dstack-attest/src/collateral.rs

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! Failover across interchangeable collateral endpoints.
+//!
+//! Attestation verification depends on collateral that only the vendor
+//! publishes: Intel's PCCS and AMD's KDS. Both were single points of failure
+//! for verification until now -- one URL each, and no answer meant no
+//! verification. AMD KDS in particular is a single global endpoint with no
+//! official mirror, and it has been unreachable for hours at a time.
+//!
+//! Nothing here relaxes verification. Collateral is signed by the vendor and
+//! checked against roots compiled into the binary, so an endpoint can serve a
+//! stale or absent answer but cannot serve a forged one; which endpoint
+//! answered has no bearing on whether the signature checks pass.
+
+use anyhow::{bail, Context, Result};
+use dcap_qvl::{
+    collateral::{CollateralClient, PHALA_PCCS_URL},
+    verify::VerifiedReport,
+    QuoteCollateralV3,
+};
+use dstack_types::UrlList;
+use sev_snp_qvl::{AmdKdsClient, AMD_KDS_DEFAULT_BASE_URL};
+
+/// Build an AMD KDS client over the configured endpoints, or the vendor
+/// default when none are configured.
+pub(crate) fn amd_kds_client(urls: &UrlList) -> Result<AmdKdsClient> {
+    if urls.is_empty() {
+        return AmdKdsClient::with_base_url(AMD_KDS_DEFAULT_BASE_URL);
+    }
+    AmdKdsClient::with_base_urls(urls.as_slice())
+}
+
+/// Intel PCCS access over one or more interchangeable endpoints.
+///
+/// Endpoints are tried in order and the first success wins.
+///
+/// Unlike the AMD KDS client, this fails over on *any* error rather than only
+/// on ones that look transient. `dcap-qvl` returns `anyhow::Error`, so there is
+/// no status code to classify on, and inventing a classifier by matching error
+/// strings would be worse than the thing it replaces. The cost of the coarser
+/// rule is bounded: a genuinely absent FMSPC costs one extra request per
+/// configured endpoint, and configuring an endpoint list is opt-in.
+#[derive(Clone)]
+pub struct PccsClient {
+    clients: Vec<(String, CollateralClient)>,
+}
+
+impl PccsClient {
+    pub fn new(urls: &UrlList) -> Result<Self> {
+        let urls: Vec<&str> = if urls.is_empty() {
+            vec![PHALA_PCCS_URL]
+        } else {
+            urls.as_slice().iter().map(String::as_str).collect()
+        };
+        let clients = urls
+            .into_iter()
+            .map(|url| {
+                CollateralClient::with_default_http(url)
+                    .with_context(|| format!("failed to create PCCS client for {url}"))
+                    .map(|client| (url.to_string(), client))
+            })
+            .collect::<Result<Vec<_>>>()?;
+        if clients.is_empty() {
+            bail!("PCCS endpoint list is empty");
+        }
+        Ok(Self { clients })
+    }
+
+    pub async fn fetch(&self, quote: &[u8]) -> Result<QuoteCollateralV3> {
+        self.failover("fetch", |client| client.fetch(quote)).await
+    }
+
+    pub async fn fetch_and_verify(&self, quote: &[u8]) -> Result<VerifiedReport> {
+        self.failover("fetch_and_verify", |client| client.fetch_and_verify(quote))
+            .await
+    }
+
+    async fn failover<'a, T, F, Fut>(&'a self, label: &str, mut call: F) -> Result<T>
+    where
+        F: FnMut(&'a CollateralClient) -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        let mut errors: Vec<String> = Vec::new();
+        for (url, client) in &self.clients {
+            match call(client).await {
+                Ok(value) => return Ok(value),
+                Err(err) => errors.push(format!("{url}: {err:#}")),
+            }
+        }
+        match errors.len() {
+            0 => bail!("PCCS {label} had no endpoint to try"),
+            1 => bail!("PCCS {label} failed: {}", errors[0]),
+            n => bail!(
+                "PCCS {label} failed on all {n} endpoints: {}",
+                errors.join("; ")
+            ),
+        }
+    }
+}

dstack/dstack-attest/src/lib.rs

@@ -15,6 +15,7 @@ pub mod amd_sev_snp;
 pub mod attestation;
 #[cfg(feature = "quote")]
 mod aws_nitro_tpm;
+pub mod collateral;
 #[cfg(feature = "quote")]
 mod sev_snp;
 pub mod trust_anchors;

dstack/dstack-types/src/lib.rs

@@ -1238,20 +1238,114 @@ pub struct GatewayClusterConfig {
     pub urls: Vec<String>,
 }
 
+/// One or more interchangeable endpoints for the same collateral service.
+///
+/// Deserializes from either a bare string or a list, so every config file and
+/// every already-serialized `SysConfig` written by an older host keeps parsing
+/// unchanged. Serializes back as a bare string when there is exactly one entry,
+/// for the same reason.
+///
+/// Order is meaningful: callers try entries front to back and keep the first
+/// answer. Blank entries are dropped on the way in, because a config that says
+/// `pccs = ["", "https://..."]` means the second one.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct UrlList(Vec<String>);
+
+impl UrlList {
+    pub fn new(urls: impl IntoIterator<Item = impl Into<String>>) -> Self {
+        Self(
+            urls.into_iter()
+                .map(Into::into)
+                .map(|url| url.trim().to_string())
+                .filter(|url| !url.is_empty())
+                .collect(),
+        )
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    /// The endpoints in the order they should be tried.
+    pub fn as_slice(&self) -> &[String] {
+        &self.0
+    }
+
+    pub fn iter(&self) -> std::slice::Iter<'_, String> {
+        self.0.iter()
+    }
+
+    /// The endpoint a single-URL caller should use. `None` when the list is
+    /// empty, which callers read as "use the platform default".
+    pub fn first(&self) -> Option<&str> {
+        self.0.first().map(String::as_str)
+    }
+}
+
+impl From<String> for UrlList {
+    fn from(url: String) -> Self {
+        Self::new([url])
+    }
+}
+
+impl From<&str> for UrlList {
+    fn from(url: &str) -> Self {
+        Self::new([url])
+    }
+}
+
+impl From<Vec<String>> for UrlList {
+    fn from(urls: Vec<String>) -> Self {
+        Self::new(urls)
+    }
+}
+
+impl Serialize for UrlList {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        match self.0.as_slice() {
+            [single] => serializer.serialize_str(single),
+            many => many.serialize(serializer),
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for UrlList {
+    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum OneOrMany {
+            One(String),
+            Many(Vec<String>),
+        }
+        Ok(match OneOrMany::deserialize(deserializer)? {
+            OneOrMany::One(url) => Self::new([url]),
+            OneOrMany::Many(urls) => Self::new(urls),
+        })
+    }
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub struct CollateralUrls {
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub pccs: Option<String>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub amd_kds: Option<String>,
+    /// Intel PCCS endpoints. Empty means the built-in default.
+    #[serde(default, skip_serializing_if = "UrlList::is_empty")]
+    pub pccs: UrlList,
+    /// AMD KDS endpoints, or mirrors of it. Empty means the built-in default.
+    #[serde(default, skip_serializing_if = "UrlList::is_empty")]
+    pub amd_kds: UrlList,
 }
 
 impl SysConfig {
     pub fn collateral_urls(&self) -> CollateralUrls {
         let mut urls = self.collateral_urls.clone().unwrap_or_default();
-        if urls.pccs.is_none() {
-            urls.pccs.clone_from(&self.legacy_pccs_url);
+        if urls.pccs.is_empty() {
+            if let Some(legacy) = &self.legacy_pccs_url {
+                urls.pccs = UrlList::from(legacy.clone());
+            }
         }
         urls
     }
@@ -2682,3 +2776,88 @@ mod appcompose_sdk_parity {
         );
     }
 }
+
+#[cfg(test)]
+mod collateral_url_tests {
+    use super::*;
+
+    #[test]
+    fn a_bare_string_and_a_list_both_parse() {
+        let single: CollateralUrls =
+            serde_json::from_str(r#"{"pccs":"https://pccs.example"}"#).unwrap();
+        assert_eq!(single.pccs.as_slice(), ["https://pccs.example"]);
+
+        let many: CollateralUrls =
+            serde_json::from_str(r#"{"amd_kds":["https://a/vcek/v1","https://b/vcek/v1"]}"#)
+                .unwrap();
+        assert_eq!(
+            many.amd_kds.as_slice(),
+            ["https://a/vcek/v1", "https://b/vcek/v1"]
+        );
+    }
+
+    /// A `SysConfig` written by a host that predates lists must still round
+    /// trip through a new binary unchanged, because the guest hashes what it
+    /// is given rather than what it would have written itself.
+    #[test]
+    fn a_single_url_serializes_back_as_a_bare_string() {
+        let urls = CollateralUrls {
+            pccs: UrlList::from("https://pccs.example"),
+            amd_kds: UrlList::default(),
+        };
+        assert_eq!(
+            serde_json::to_string(&urls).unwrap(),
+            r#"{"pccs":"https://pccs.example"}"#
+        );
+    }
+
+    #[test]
+    fn several_urls_serialize_as_a_list() {
+        let urls = CollateralUrls {
+            pccs: UrlList::default(),
+            amd_kds: UrlList::new(["https://a", "https://b"]),
+        };
+        assert_eq!(
+            serde_json::to_string(&urls).unwrap(),
+            r#"{"amd_kds":["https://a","https://b"]}"#
+        );
+    }
+
+    #[test]
+    fn blank_entries_are_dropped_on_the_way_in() {
+        let urls: CollateralUrls =
+            serde_json::from_str(r#"{"pccs":["", "  ", "https://real"]}"#).unwrap();
+        assert_eq!(urls.pccs.as_slice(), ["https://real"]);
+
+        let blank: CollateralUrls = serde_json::from_str(r#"{"pccs":"   "}"#).unwrap();
+        assert!(
+            blank.pccs.is_empty(),
+            "a blank string means 'unset', not 'an endpoint named blank'"
+        );
+    }
+
+    #[test]
+    fn the_legacy_pccs_url_field_still_wins_when_no_list_is_configured() {
+        let sys_config: SysConfig = serde_json::from_str(
+            r#"{"kms_urls":[],"gateway_urls":[],"pccs_url":"https://legacy.example","vm_config":"{}"}"#,
+        )
+        .unwrap();
+        assert_eq!(
+            sys_config.collateral_urls().pccs.as_slice(),
+            ["https://legacy.example"]
+        );
+    }
+
+    #[test]
+    fn an_explicit_list_takes_precedence_over_the_legacy_field() {
+        let sys_config: SysConfig = serde_json::from_str(
+            r#"{"kms_urls":[],"gateway_urls":[],"pccs_url":"https://legacy.example",
+                "collateral_urls":{"pccs":["https://new.example"]},"vm_config":"{}"}"#,
+        )
+        .unwrap();
+        assert_eq!(
+            sys_config.collateral_urls().pccs.as_slice(),
+            ["https://new.example"]
+        );
+    }
+}