From 50de9b52ab5602703805314395c750fb035420d6 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 11:28:47 -0400 Subject: [PATCH 01/56] feat(space-host): add dynamic multi-space authority registry --- Cargo.lock | 6 +- rsky-pds/Cargo.toml | 3 +- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/authority.rs | 127 ++++++++++++++++++++++++++++++- rsky-space-host/src/error.rs | 2 + rsky-space-host/src/http.rs | 5 ++ 6 files changed, 137 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b6aade8..30529afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8192,7 +8192,7 @@ dependencies = [ [[package]] name = "rsky-oauth" -version = "0.3.1" +version = "0.3.2" dependencies = [ "async-trait", "base64 0.22.1", @@ -8211,7 +8211,7 @@ dependencies = [ [[package]] name = "rsky-pds" -version = "0.13.16" +version = "0.13.17" dependencies = [ "anyhow", "argon2", @@ -8421,7 +8421,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.5.1" +version = "0.6.0" dependencies = [ "async-trait", "axum", diff --git a/rsky-pds/Cargo.toml b/rsky-pds/Cargo.toml index 866802b3..5615672c 100644 --- a/rsky-pds/Cargo.toml +++ b/rsky-pds/Cargo.toml @@ -54,7 +54,7 @@ rsky-identity = { workspace = true } rsky-lexicon = { workspace = true } rsky-repo = { workspace = true } rsky-space = { path = "../rsky-space", version = "0.4.0" } -rsky-space-host = { path = "../rsky-space-host", version = "0.5.1" } +rsky-space-host = { path = "../rsky-space-host", version = "0.6.0" } rsky-syntax = { workspace = true } hickory-resolver = "0.24.1" secp256k1 = { workspace = true } @@ -80,4 +80,3 @@ ws = { package = "rocket_ws", version = "0.1.1" } tempfile = "3.10" http-auth-basic = { version = "0.3.5" } - diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index bdfdaede..c6b0bf1d 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.5.1" +version = "0.6.0" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/authority.rs b/rsky-space-host/src/authority.rs index 58acc276..06aa12f9 100644 --- a/rsky-space-host/src/authority.rs +++ b/rsky-space-host/src/authority.rs @@ -8,6 +8,8 @@ use rsky_space::credential::{ self, Confirmation, JwtHeader, SpaceClaims, CREDENTIAL_TTL_SECS, CREDENTIAL_TYP, }; use rsky_space::space_id::SpaceId; +use std::collections::BTreeSet; +use std::sync::RwLock; use crate::appaccess::AppAccess; use crate::attestation::{verify_client_attestation, JtiStore, MetadataFetcher}; @@ -22,17 +24,44 @@ pub trait KeyResolver: Send + Sync { async fn signing_key(&self, did: &str) -> Result; } -/// A space authority for a single space. +/// A space authority for one or more spaces of a single type. pub struct Authority { pub space: SpaceId, + hosted: RwLock>, + registered: RwLock>, pub signer: Signer, pub app_access: AppAccess, } impl Authority { pub fn new(space: SpaceId, signer: Signer, app_access: AppAccess) -> Self { + Self::new_many( + space.authority.clone(), + space.space_type.clone(), + [space.skey.clone()], + signer, + app_access, + ) + } + + pub fn new_many( + authority_did: impl Into, + space_type: impl Into, + skeys: impl IntoIterator, + signer: Signer, + app_access: AppAccess, + ) -> Self { + let authority_did = authority_did.into(); + let space_type = space_type.into(); + let hosted: BTreeSet = skeys.into_iter().collect(); + let first_skey = hosted + .first() + .expect("space authority needs at least one hosted space") + .clone(); Self { - space, + space: SpaceId::new(authority_did, space_type, first_skey), + registered: RwLock::new(hosted.clone()), + hosted: RwLock::new(hosted), signer, app_access, } @@ -46,6 +75,90 @@ impl Authority { self.space.uri() } + pub fn space(&self, skey: &str) -> SpaceId { + SpaceId::new(&self.space.authority, &self.space.space_type, skey) + } + + pub fn spaces(&self) -> Vec { + self.hosted + .read() + .expect("hosted spaces") + .iter() + .map(|skey| self.space(skey).uri()) + .collect() + } + + pub fn resolve(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority == self.space.authority + && space.space_type == self.space.space_type + && self + .hosted + .read() + .expect("hosted spaces") + .contains(&space.skey) + { + Ok(space) + } else { + Err(HostError::SpaceNotFound(space_uri.to_string())) + } + } + + pub fn register(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority != self.space.authority || space.space_type != self.space.space_type { + return Err(HostError::SpaceNotFound(space_uri.to_string())); + } + let newly_registered = self + .registered + .write() + .expect("registered spaces") + .insert(space.skey.clone()); + self.hosted + .write() + .expect("hosted spaces") + .insert(space.skey); + Ok(newly_registered) + } + + pub fn resolve_registered(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority == self.space.authority + && space.space_type == self.space.space_type + && self + .registered + .read() + .expect("registered spaces") + .contains(&space.skey) + { + Ok(space) + } else { + Err(HostError::SpaceNotFound(space_uri.to_string())) + } + } + + pub fn unregister(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority != self.space.authority || space.space_type != self.space.space_type { + return Err(HostError::SpaceNotFound(space_uri.to_string())); + } + let registered = self + .registered + .write() + .expect("registered spaces") + .remove(&space.skey); + let hosted = self + .hosted + .write() + .expect("hosted spaces") + .remove(&space.skey); + Ok(registered || hosted) + } + /// The simplespace config surfaced by `getSpace`. pub fn space_config(&self, policy: &Policy) -> SimplespaceConfig { let app_access = match &self.app_access { @@ -172,6 +285,16 @@ mod tests { Authority::new(space, test_signer(), AppAccess::Open) } + #[test] + fn registering_a_space_makes_it_immediately_serveable() { + let authority = authority(); + let space = "at://did:plc:communityauthority/space/community.blacksky.feed/new"; + + assert!(authority.register(space).unwrap()); + assert_eq!(authority.resolve(space).unwrap().skey, "new"); + assert!(authority.spaces().contains(&space.to_string())); + } + fn member_policy(dids: &[&str]) -> Policy { Policy::MemberList(Arc::new(InMemoryMembership::new( dids.iter().map(|d| d.to_string()), diff --git a/rsky-space-host/src/error.rs b/rsky-space-host/src/error.rs index 3a6b0f60..639abfe9 100644 --- a/rsky-space-host/src/error.rs +++ b/rsky-space-host/src/error.rs @@ -22,6 +22,8 @@ pub enum HostError { Resolution(String), #[error("store error: {0}")] Store(String), + #[error("space not hosted here: {0}")] + SpaceNotFound(String), #[error(transparent)] Space(#[from] rsky_space::SpaceError), } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 9c19a76a..a4d9ce15 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -128,6 +128,11 @@ impl From for ApiError { "ClientNotAuthorized", "client not authorized for space", ), + HostError::SpaceNotFound(_) => Self::new( + StatusCode::NOT_FOUND, + "SpaceNotFound", + "space not hosted here", + ), HostError::Key(_) | HostError::Membership(_) | HostError::ManagingApp(_) From 59bfbaadedbd84b1e63daeec388be0a970f13dbc Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 11:33:32 -0400 Subject: [PATCH 02/56] feat(space-host): add persistent permissioned repo storage --- Cargo.lock | 1 + rsky-space-host/src/commits.rs | 138 ++++ rsky-space-host/src/error.rs | 8 + rsky-space-host/src/http.rs | 14 + rsky-space-host/src/lib.rs | 2 + rsky-space-host/src/repo.rs | 1258 ++++++++++++++++++++++++++++++++ rsky-space/Cargo.toml | 1 + rsky-space/src/error.rs | 2 + rsky-space/src/lib.rs | 3 + rsky-space/src/record.rs | 210 ++++++ 10 files changed, 1637 insertions(+) create mode 100644 rsky-space-host/src/commits.rs create mode 100644 rsky-space-host/src/repo.rs create mode 100644 rsky-space/src/record.rs diff --git a/Cargo.lock b/Cargo.lock index 30529afe..259ae855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8403,6 +8403,7 @@ dependencies = [ "hex", "hkdf", "hmac", + "ipld-core", "iroh-car", "p256 0.13.2", "rsky-common", diff --git a/rsky-space-host/src/commits.rs b/rsky-space-host/src/commits.rs new file mode 100644 index 00000000..77a4215b --- /dev/null +++ b/rsky-space-host/src/commits.rs @@ -0,0 +1,138 @@ +//! Minting signed commits for a hosted repo (spec §Commit signature). +//! +//! `ikm` is fresh per reader, so a commit is produced at serve time from the +//! repo's persisted `(rev, state)` rather than stored alongside them. +//! +//! The spec has the account's own signing key sign the commit context. A host +//! holding repos on behalf of accounts whose PDS does not implement +//! permissioned data has no access to those keys, so [`CommitSigner`] names the +//! signer explicitly instead of assuming it. + +use rsky_lexicon::com::atproto::space::SignedCommit; +use rsky_space::commit::{build_ctx, compute_mac}; + +use crate::error::{HostError, Result}; +use crate::signing::Signer; + +pub const COMMIT_VERSION: i64 = 1; +pub const IKM_BYTES: usize = 32; + +pub trait CommitSigner: Send + Sync { + /// The `did:key` a reader verifies commits against. + fn did_key(&self) -> &str; + fn sign(&self, message: &[u8]) -> Result>; +} + +impl CommitSigner for Signer { + fn did_key(&self) -> &str { + Signer::did_key(self) + } + + fn sign(&self, message: &[u8]) -> Result> { + Signer::sign(self, message).map_err(HostError::Key) + } +} + +/// Build a signed commit over a repo's current digest. +pub fn mint_commit( + signer: &dyn CommitSigner, + space_uri: &str, + author_did: &str, + rev: &str, + hash: &[u8; 32], + ikm: [u8; IKM_BYTES], +) -> Result { + let ctx = build_ctx(space_uri, author_did, rev, &ikm); + let sig = signer.sign(&ctx)?; + let mac = compute_mac(&ikm, &ctx, hash)?; + Ok(SignedCommit { + ver: COMMIT_VERSION, + hash: hash.to_vec(), + ikm: ikm.to_vec(), + sig, + mac: mac.to_vec(), + rev: rev.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::signing::test_signer; + use rsky_space::commit::verify_commit; + + const SPACE: &str = "at://did:plc:auth/space/community.blacksky.feed/main"; + const AUTHOR: &str = "did:plc:member"; + + fn minted(hash: [u8; 32], ikm: [u8; 32]) -> (Signer, SignedCommit) { + let signer = test_signer(); + let commit = mint_commit(&signer, SPACE, AUTHOR, "3rev1", &hash, ikm).unwrap(); + (signer, commit) + } + + fn verify(signer: &Signer, commit: &SignedCommit, hash: &[u8]) -> rsky_space::Result<()> { + verify_commit( + CommitSigner::did_key(signer), + SPACE, + AUTHOR, + &commit.rev, + &commit.ikm, + &commit.sig, + &commit.mac, + hash, + ) + } + + #[test] + fn a_minted_commit_verifies() { + let hash = [7u8; 32]; + let (signer, commit) = minted(hash, [3u8; 32]); + assert_eq!(commit.ver, COMMIT_VERSION); + assert_eq!(commit.hash, hash.to_vec()); + verify(&signer, &commit, &hash).unwrap(); + } + + #[test] + fn a_tampered_hash_fails_the_mac() { + let (signer, commit) = minted([7u8; 32], [3u8; 32]); + assert!(verify(&signer, &commit, &[8u8; 32]).is_err()); + } + + #[test] + fn a_different_ikm_yields_a_different_signature() { + let hash = [7u8; 32]; + let (_, a) = minted(hash, [3u8; 32]); + let (signer, b) = minted(hash, [4u8; 32]); + assert_ne!(a.sig, b.sig); + assert_ne!(a.mac, b.mac); + verify(&signer, &b, &hash).unwrap(); + } + + #[test] + fn a_commit_does_not_verify_under_another_space_or_author() { + let hash = [7u8; 32]; + let (signer, commit) = minted(hash, [3u8; 32]); + assert!(verify_commit( + CommitSigner::did_key(&signer), + "at://did:plc:auth/space/community.blacksky.feed/other", + AUTHOR, + &commit.rev, + &commit.ikm, + &commit.sig, + &commit.mac, + &hash, + ) + .is_err()); + assert!(verify_commit( + CommitSigner::did_key(&signer), + SPACE, + "did:plc:someoneelse", + &commit.rev, + &commit.ikm, + &commit.sig, + &commit.mac, + &hash, + ) + .is_err()); + } +} diff --git a/rsky-space-host/src/error.rs b/rsky-space-host/src/error.rs index 639abfe9..9a93a829 100644 --- a/rsky-space-host/src/error.rs +++ b/rsky-space-host/src/error.rs @@ -22,8 +22,16 @@ pub enum HostError { Resolution(String), #[error("store error: {0}")] Store(String), + #[error("invalid request: {0}")] + InvalidRequest(String), #[error("space not hosted here: {0}")] SpaceNotFound(String), + #[error("repo not found")] + RepoNotFound, + #[error("swap cid did not match")] + InvalidSwap, + #[error("requested history is no longer available")] + HistoryUnavailable, #[error(transparent)] Space(#[from] rsky_space::SpaceError), } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index a4d9ce15..1106314a 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -133,6 +133,20 @@ impl From for ApiError { "SpaceNotFound", "space not hosted here", ), + HostError::RepoNotFound => { + Self::new(StatusCode::NOT_FOUND, "RepoNotFound", "repo not found") + } + HostError::InvalidRequest(message) => Self::invalid_request(message.clone()), + HostError::InvalidSwap => Self::new( + StatusCode::CONFLICT, + "InvalidSwap", + "swap cid did not match", + ), + HostError::HistoryUnavailable => Self::new( + StatusCode::GONE, + "HistoryUnavailable", + "requested history is no longer available", + ), HostError::Key(_) | HostError::Membership(_) | HostError::ManagingApp(_) diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index 499d54f7..4065a30a 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -21,6 +21,7 @@ pub mod appaccess; pub mod attestation; pub mod authority; +pub mod commits; pub mod config; pub mod error; pub mod http; @@ -29,6 +30,7 @@ pub mod managing_app; pub mod membership; pub mod notify; pub mod policy; +pub mod repo; pub mod service_jwt; pub mod signing; pub mod store; diff --git a/rsky-space-host/src/repo.rs b/rsky-space-host/src/repo.rs new file mode 100644 index 00000000..9e8d5ae7 --- /dev/null +++ b/rsky-space-host/src/repo.rs @@ -0,0 +1,1258 @@ +//! Permissioned repo storage (spec §Permissioned repos, §Incremental sync). +//! +//! One repo is one account's records within one space. A repo carries three +//! pieces of state, all advanced atomically by [`RepoStore::apply_writes`]: +//! +//! - the records themselves, keyed `(collection, rkey)`; +//! - the LtHash state, from which the commit `hash` is derived; +//! - an operation log, the transport optimization syncers page through. +//! +//! Commits are not stored. `ikm` is fresh per reader, so a commit is minted at +//! serve time from the persisted `(rev, state)` pair. + +use async_trait::async_trait; +use rsky_space::lthash::{element, LtHash}; +use rsky_space::record::dag_cbor_cid; +use rusqlite::{Connection, OptionalExtension}; +use std::collections::BTreeMap; +use std::sync::Mutex; + +use crate::error::{HostError, Result}; + +/// Size cap on a single stored record. Storage is shape-agnostic, so this and +/// DAG-CBOR well-formedness are the only limits the host applies to a value. +pub const MAX_RECORD_BYTES: usize = 64 * 1024; + +const STATE_BYTES: usize = 2048; + +/// One mutation in an atomic batch. Values are already-encoded DAG-CBOR. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RepoWrite { + Create { + collection: String, + rkey: String, + value: Vec, + }, + Update { + collection: String, + rkey: String, + value: Vec, + swap_record: Option, + }, + Delete { + collection: String, + rkey: String, + swap_record: Option, + }, +} + +impl RepoWrite { + pub fn collection(&self) -> &str { + match self { + Self::Create { collection, .. } + | Self::Update { collection, .. } + | Self::Delete { collection, .. } => collection, + } + } + + pub fn rkey(&self) -> &str { + match self { + Self::Create { rkey, .. } | Self::Update { rkey, .. } | Self::Delete { rkey, .. } => { + rkey + } + } + } +} + +/// What a write did. A delete of an absent record is [`WriteOutcome::Noop`]: +/// it produces no oplog entry and leaves the digest untouched. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteOutcome { + Created { cid: String }, + Updated { cid: String }, + Deleted, + Noop, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredRecord { + pub collection: String, + pub rkey: String, + pub cid: String, + pub value: Vec, +} + +impl StoredRecord { + /// The `{collection}/{rkey}` path used as the list cursor and CAR index key. + pub fn path(&self) -> String { + record_path(&self.collection, &self.rkey) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredOp { + pub seq: i64, + pub rev: String, + pub collection: String, + pub rkey: String, + pub cid: Option, + pub prev: Option, +} + +/// A repo's current position: the latest revision and the LtHash state behind +/// its commit digest. +#[derive(Clone)] +pub struct RepoHead { + pub rev: String, + pub state: [u8; STATE_BYTES], +} + +impl RepoHead { + pub fn hash(&self) -> [u8; 32] { + LtHash::from_state_bytes(&self.state).hash() + } +} + +/// A page of the operation log. `complete` is true when the page reaches the +/// repo's head, which is when a caller may attach the current commit. +pub struct OpPage { + pub ops: Vec, + pub cursor: Option, + pub complete: bool, +} + +/// The result of an atomic batch: the new head plus one outcome per write. +pub struct Applied { + pub rev: String, + pub hash: [u8; 32], + pub outcomes: Vec, +} + +pub fn record_path(collection: &str, rkey: &str) -> String { + format!("{collection}/{rkey}") +} + +#[async_trait] +pub trait RepoStore: Send + Sync { + /// Apply a batch atomically at revision `rev`. Every write in the batch + /// shares the revision, which is how syncers see them as one mutation. + async fn apply_writes( + &self, + space_uri: &str, + did: &str, + rev: &str, + writes: &[RepoWrite], + ) -> Result; + + async fn head(&self, space_uri: &str, did: &str) -> Result>; + + async fn get_record( + &self, + space_uri: &str, + did: &str, + collection: &str, + rkey: &str, + ) -> Result>; + + /// Records ordered by `{collection}/{rkey}`; `cursor` is the last path of + /// the previous page. + async fn list_records( + &self, + space_uri: &str, + did: &str, + collection: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result<(Vec, Option)>; + + /// Operations after `since` (a revision), ordered by insertion. Returns + /// [`HostError::HistoryUnavailable`] when `since` predates the retained + /// window, which is a syncer's signal to fall back to full-state recovery. + async fn list_ops( + &self, + space_uri: &str, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result; + + /// Drop a repo entirely (account deletion, space deletion). + async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()>; +} + +/// Fold one batch into an existing record set + digest. Shared by both +/// backings so their semantics cannot drift. +fn plan_batch( + existing: &BTreeMap, + lt: &mut LtHash, + writes: &[RepoWrite], +) -> Result> { + let mut planned = Vec::with_capacity(writes.len()); + let mut seen: BTreeMap> = BTreeMap::new(); + + for write in writes { + let path = record_path(write.collection(), write.rkey()); + let current = match seen.get(&path) { + Some(cid) => cid.clone(), + None => existing.get(&path).map(|r| r.cid.clone()), + }; + + let planned_write = match write { + RepoWrite::Create { + collection, + rkey, + value, + } => { + if current.is_some() { + return Err(HostError::InvalidRequest(format!( + "record already exists: {path}" + ))); + } + let cid = dag_cbor_cid(value).to_string(); + lt.add(&element(collection, rkey, &cid)); + PlannedWrite { + collection: collection.clone(), + rkey: rkey.clone(), + cid: Some(cid.clone()), + prev: None, + value: Some(value.clone()), + outcome: WriteOutcome::Created { cid }, + } + } + RepoWrite::Update { + collection, + rkey, + value, + swap_record, + } => { + check_swap(swap_record.as_deref(), current.as_deref())?; + let cid = dag_cbor_cid(value).to_string(); + if let Some(prev) = ¤t { + lt.remove(&element(collection, rkey, prev)); + } + lt.add(&element(collection, rkey, &cid)); + PlannedWrite { + collection: collection.clone(), + rkey: rkey.clone(), + cid: Some(cid.clone()), + prev: current.clone(), + value: Some(value.clone()), + outcome: WriteOutcome::Updated { cid }, + } + } + RepoWrite::Delete { + collection, + rkey, + swap_record, + } => { + check_swap(swap_record.as_deref(), current.as_deref())?; + let Some(prev) = current.clone() else { + planned.push(PlannedWrite { + collection: collection.clone(), + rkey: rkey.clone(), + cid: None, + prev: None, + value: None, + outcome: WriteOutcome::Noop, + }); + continue; + }; + lt.remove(&element(collection, rkey, &prev)); + PlannedWrite { + collection: collection.clone(), + rkey: rkey.clone(), + cid: None, + prev: Some(prev), + value: None, + outcome: WriteOutcome::Deleted, + } + } + }; + + seen.insert(path, planned_write.cid.clone()); + planned.push(planned_write); + } + Ok(planned) +} + +fn check_swap(swap: Option<&str>, current: Option<&str>) -> Result<()> { + match swap { + Some(expected) if current != Some(expected) => Err(HostError::InvalidSwap), + _ => Ok(()), + } +} + +struct PlannedWrite { + collection: String, + rkey: String, + cid: Option, + prev: Option, + value: Option>, + outcome: WriteOutcome, +} + +impl PlannedWrite { + fn is_noop(&self) -> bool { + self.outcome == WriteOutcome::Noop + } +} + +fn page_cursor(page: &[T], limit: u32, key: impl Fn(&T) -> String) -> Option { + match page.last() { + Some(last) if page.len() == limit as usize => Some(key(last)), + _ => None, + } +} + +// ---------------------------------------------------------------- in memory + +struct MemRepo { + records: BTreeMap, + ops: Vec, + rev: String, + state: [u8; STATE_BYTES], +} + +impl Default for MemRepo { + fn default() -> Self { + Self { + records: BTreeMap::new(), + ops: Vec::new(), + rev: String::new(), + state: [0u8; STATE_BYTES], + } + } +} + +#[derive(Default)] +pub struct InMemoryRepos { + repos: Mutex>, + next_seq: Mutex, +} + +#[async_trait] +impl RepoStore for InMemoryRepos { + async fn apply_writes( + &self, + space_uri: &str, + did: &str, + rev: &str, + writes: &[RepoWrite], + ) -> Result { + let mut repos = self.repos.lock().unwrap(); + let repo = repos + .entry((space_uri.to_string(), did.to_string())) + .or_default(); + let mut lt = LtHash::from_state_bytes(&repo.state); + let planned = plan_batch(&repo.records, &mut lt, writes)?; + + let mut seq = self.next_seq.lock().unwrap(); + for p in &planned { + if p.is_noop() { + continue; + } + let path = record_path(&p.collection, &p.rkey); + match &p.cid { + Some(cid) => { + repo.records.insert( + path, + StoredRecord { + collection: p.collection.clone(), + rkey: p.rkey.clone(), + cid: cid.clone(), + value: p.value.clone().unwrap_or_default(), + }, + ); + } + None => { + repo.records.remove(&path); + } + } + *seq += 1; + repo.ops.push(StoredOp { + seq: *seq, + rev: rev.to_string(), + collection: p.collection.clone(), + rkey: p.rkey.clone(), + cid: p.cid.clone(), + prev: p.prev.clone(), + }); + } + + repo.state = lt.state_bytes(); + if planned.iter().any(|p| !p.is_noop()) { + repo.rev = rev.to_string(); + } + Ok(Applied { + rev: repo.rev.clone(), + hash: lt.hash(), + outcomes: planned.into_iter().map(|p| p.outcome).collect(), + }) + } + + async fn head(&self, space_uri: &str, did: &str) -> Result> { + Ok(self + .repos + .lock() + .unwrap() + .get(&(space_uri.to_string(), did.to_string())) + .map(|r| RepoHead { + rev: r.rev.clone(), + state: r.state, + })) + } + + async fn get_record( + &self, + space_uri: &str, + did: &str, + collection: &str, + rkey: &str, + ) -> Result> { + Ok(self + .repos + .lock() + .unwrap() + .get(&(space_uri.to_string(), did.to_string())) + .and_then(|r| r.records.get(&record_path(collection, rkey)).cloned())) + } + + async fn list_records( + &self, + space_uri: &str, + did: &str, + collection: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result<(Vec, Option)> { + let repos = self.repos.lock().unwrap(); + let Some(repo) = repos.get(&(space_uri.to_string(), did.to_string())) else { + return Err(HostError::RepoNotFound); + }; + let page: Vec = repo + .records + .iter() + .filter(|(path, record)| { + collection.is_none_or(|c| record.collection == c) + && cursor.is_none_or(|c| path.as_str() > c) + }) + .take(limit as usize) + .map(|(_, record)| record.clone()) + .collect(); + let cursor = page_cursor(&page, limit, |r| r.path()); + Ok((page, cursor)) + } + + async fn list_ops( + &self, + space_uri: &str, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result { + let repos = self.repos.lock().unwrap(); + let Some(repo) = repos.get(&(space_uri.to_string(), did.to_string())) else { + return Err(HostError::RepoNotFound); + }; + ensure_history(since, repo.ops.first().map(|o| o.rev.as_str()))?; + let after = parse_cursor(cursor)?; + let page: Vec = repo + .ops + .iter() + .filter(|op| { + since.is_none_or(|s| op.rev.as_str() > s) && after.is_none_or(|c| op.seq > c) + }) + .take(limit as usize) + .cloned() + .collect(); + Ok(finish_op_page(page, limit, repo.ops.last().map(|o| o.seq))) + } + + async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()> { + self.repos + .lock() + .unwrap() + .remove(&(space_uri.to_string(), did.to_string())); + Ok(()) + } +} + +fn ensure_history(since: Option<&str>, earliest_retained: Option<&str>) -> Result<()> { + match (since, earliest_retained) { + // Every retained operation is already newer than `since`, so the + // caller's revision fell out of the window (or never existed here). + (Some(since), Some(earliest)) if earliest > since => Err(HostError::HistoryUnavailable), + (Some(_), None) => Err(HostError::HistoryUnavailable), + _ => Ok(()), + } +} + +fn parse_cursor(cursor: Option<&str>) -> Result> { + cursor + .map(|c| { + c.parse::() + .map_err(|_| HostError::InvalidRequest(format!("invalid cursor: {c}"))) + }) + .transpose() +} + +fn finish_op_page(ops: Vec, limit: u32, last_seq: Option) -> OpPage { + let reached_head = ops.last().map(|o| o.seq) == last_seq; + let cursor = page_cursor(&ops, limit, |o| o.seq.to_string()); + OpPage { + complete: reached_head, + cursor: if reached_head { None } else { cursor }, + ops, + } +} + +// ------------------------------------------------------------------- sqlite + +/// SQLite-backed repo storage. Volume per host is modest and every batch is a +/// single transaction, so one connection behind a mutex is sufficient. +pub struct SqliteRepos { + conn: Mutex, +} + +impl SqliteRepos { + pub fn open_in_memory() -> Result { + Self::init(Connection::open_in_memory().map_err(sql_err)?) + } + + pub fn open(path: impl AsRef) -> Result { + Self::init(Connection::open(path).map_err(sql_err)?) + } + + pub fn init(conn: Connection) -> Result { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS repo ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL DEFAULT '', + state BLOB NOT NULL, + PRIMARY KEY (space_uri, did) + ); + CREATE TABLE IF NOT EXISTS record ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + path TEXT NOT NULL, + collection TEXT NOT NULL, + rkey TEXT NOT NULL, + cid TEXT NOT NULL, + value BLOB NOT NULL, + PRIMARY KEY (space_uri, did, path) + ); + CREATE TABLE IF NOT EXISTS repo_op ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + collection TEXT NOT NULL, + rkey TEXT NOT NULL, + cid TEXT, + prev TEXT + ); + CREATE INDEX IF NOT EXISTS repo_op_repo_seq ON repo_op (space_uri, did, seq);", + ) + .map_err(sql_err)?; + Ok(Self { + conn: Mutex::new(conn), + }) + } +} + +fn sql_err(e: rusqlite::Error) -> HostError { + HostError::Store(e.to_string()) +} + +fn state_from_blob(blob: Vec) -> Result<[u8; STATE_BYTES]> { + blob.try_into() + .map_err(|_| HostError::Store("corrupt lthash state".into())) +} + +fn row_to_record(row: &rusqlite::Row) -> rusqlite::Result { + Ok(StoredRecord { + collection: row.get("collection")?, + rkey: row.get("rkey")?, + cid: row.get("cid")?, + value: row.get("value")?, + }) +} + +fn row_to_op(row: &rusqlite::Row) -> rusqlite::Result { + Ok(StoredOp { + seq: row.get("seq")?, + rev: row.get("rev")?, + collection: row.get("collection")?, + rkey: row.get("rkey")?, + cid: row.get("cid")?, + prev: row.get("prev")?, + }) +} + +#[async_trait] +impl RepoStore for SqliteRepos { + async fn apply_writes( + &self, + space_uri: &str, + did: &str, + rev: &str, + writes: &[RepoWrite], + ) -> Result { + let mut conn = self.conn.lock().unwrap(); + let tx = conn.transaction().map_err(sql_err)?; + + let existing_state: Option> = tx + .query_row( + "SELECT state FROM repo WHERE space_uri = ?1 AND did = ?2", + rusqlite::params![space_uri, did], + |row| row.get(0), + ) + .optional() + .map_err(sql_err)?; + let mut current_rev: String = tx + .query_row( + "SELECT rev FROM repo WHERE space_uri = ?1 AND did = ?2", + rusqlite::params![space_uri, did], + |row| row.get(0), + ) + .optional() + .map_err(sql_err)? + .unwrap_or_default(); + + let mut lt = match existing_state { + Some(blob) => LtHash::from_state_bytes(&state_from_blob(blob)?), + None => LtHash::new(), + }; + + // Only the paths this batch touches are needed to plan it. + let mut existing = BTreeMap::new(); + for write in writes { + let path = record_path(write.collection(), write.rkey()); + if let Some(record) = tx + .query_row( + "SELECT collection, rkey, cid, value FROM record + WHERE space_uri = ?1 AND did = ?2 AND path = ?3", + rusqlite::params![space_uri, did, path], + row_to_record, + ) + .optional() + .map_err(sql_err)? + { + existing.insert(path, record); + } + } + + let planned = plan_batch(&existing, &mut lt, writes)?; + for p in &planned { + if p.is_noop() { + continue; + } + let path = record_path(&p.collection, &p.rkey); + match (&p.cid, &p.value) { + (Some(cid), Some(value)) => { + tx.execute( + "INSERT INTO record (space_uri, did, path, collection, rkey, cid, value) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT (space_uri, did, path) + DO UPDATE SET cid = ?6, value = ?7", + rusqlite::params![space_uri, did, path, p.collection, p.rkey, cid, value], + ) + .map_err(sql_err)?; + } + _ => { + tx.execute( + "DELETE FROM record WHERE space_uri = ?1 AND did = ?2 AND path = ?3", + rusqlite::params![space_uri, did, path], + ) + .map_err(sql_err)?; + } + } + tx.execute( + "INSERT INTO repo_op (space_uri, did, rev, collection, rkey, cid, prev) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + rusqlite::params![space_uri, did, rev, p.collection, p.rkey, p.cid, p.prev], + ) + .map_err(sql_err)?; + current_rev = rev.to_string(); + } + + tx.execute( + "INSERT INTO repo (space_uri, did, rev, state) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (space_uri, did) DO UPDATE SET rev = ?3, state = ?4", + rusqlite::params![space_uri, did, current_rev, lt.state_bytes().to_vec()], + ) + .map_err(sql_err)?; + tx.commit().map_err(sql_err)?; + + Ok(Applied { + rev: current_rev, + hash: lt.hash(), + outcomes: planned.into_iter().map(|p| p.outcome).collect(), + }) + } + + async fn head(&self, space_uri: &str, did: &str) -> Result> { + let conn = self.conn.lock().unwrap(); + let row: Option<(String, Vec)> = conn + .query_row( + "SELECT rev, state FROM repo WHERE space_uri = ?1 AND did = ?2", + rusqlite::params![space_uri, did], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(sql_err)?; + row.map(|(rev, state)| { + Ok(RepoHead { + rev, + state: state_from_blob(state)?, + }) + }) + .transpose() + } + + async fn get_record( + &self, + space_uri: &str, + did: &str, + collection: &str, + rkey: &str, + ) -> Result> { + self.conn + .lock() + .unwrap() + .query_row( + "SELECT collection, rkey, cid, value FROM record + WHERE space_uri = ?1 AND did = ?2 AND path = ?3", + rusqlite::params![space_uri, did, record_path(collection, rkey)], + row_to_record, + ) + .optional() + .map_err(sql_err) + } + + async fn list_records( + &self, + space_uri: &str, + did: &str, + collection: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result<(Vec, Option)> { + let conn = self.conn.lock().unwrap(); + require_repo(&conn, space_uri, did)?; + let mut stmt = conn + .prepare( + "SELECT collection, rkey, cid, value FROM record + WHERE space_uri = ?1 AND did = ?2 AND path > ?3 + AND (?4 IS NULL OR collection = ?4) + ORDER BY path ASC LIMIT ?5", + ) + .map_err(sql_err)?; + let page = stmt + .query_map( + rusqlite::params![space_uri, did, cursor.unwrap_or(""), collection, limit], + row_to_record, + ) + .map_err(sql_err)? + .collect::>>() + .map_err(sql_err)?; + let cursor = page_cursor(&page, limit, |r| r.path()); + Ok((page, cursor)) + } + + async fn list_ops( + &self, + space_uri: &str, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result { + let conn = self.conn.lock().unwrap(); + require_repo(&conn, space_uri, did)?; + let bounds: (Option, Option) = conn + .query_row( + "SELECT (SELECT rev FROM repo_op WHERE space_uri = ?1 AND did = ?2 + ORDER BY seq ASC LIMIT 1), + (SELECT seq FROM repo_op WHERE space_uri = ?1 AND did = ?2 + ORDER BY seq DESC LIMIT 1)", + rusqlite::params![space_uri, did], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(sql_err)?; + ensure_history(since, bounds.0.as_deref())?; + let after = parse_cursor(cursor)?; + + let mut stmt = conn + .prepare( + "SELECT seq, rev, collection, rkey, cid, prev FROM repo_op + WHERE space_uri = ?1 AND did = ?2 + AND (?3 IS NULL OR rev > ?3) + AND (?4 IS NULL OR seq > ?4) + ORDER BY seq ASC LIMIT ?5", + ) + .map_err(sql_err)?; + let ops = stmt + .query_map( + rusqlite::params![space_uri, did, since, after, limit], + row_to_op, + ) + .map_err(sql_err)? + .collect::>>() + .map_err(sql_err)?; + Ok(finish_op_page(ops, limit, bounds.1)) + } + + async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()> { + let mut conn = self.conn.lock().unwrap(); + let tx = conn.transaction().map_err(sql_err)?; + for table in ["record", "repo_op", "repo"] { + tx.execute( + &format!("DELETE FROM {table} WHERE space_uri = ?1 AND did = ?2"), + rusqlite::params![space_uri, did], + ) + .map_err(sql_err)?; + } + tx.commit().map_err(sql_err) + } +} + +fn require_repo(conn: &Connection, space_uri: &str, did: &str) -> Result<()> { + let exists: Option = conn + .query_row( + "SELECT 1 FROM repo WHERE space_uri = ?1 AND did = ?2", + rusqlite::params![space_uri, did], + |row| row.get(0), + ) + .optional() + .map_err(sql_err)?; + exists.map(|_| ()).ok_or(HostError::RepoNotFound) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const SPACE: &str = "at://did:plc:auth/space/community.blacksky.feed/main"; + const DID: &str = "did:plc:member"; + const POST: &str = "app.bsky.feed.post"; + const LIKE: &str = "app.bsky.feed.like"; + + fn value(text: &str) -> Vec { + rsky_space::record::encode_record(&json!({"text": text}), MAX_RECORD_BYTES).unwrap() + } + + fn create(rkey: &str, text: &str) -> RepoWrite { + RepoWrite::Create { + collection: POST.to_string(), + rkey: rkey.to_string(), + value: value(text), + } + } + + fn delete(rkey: &str) -> RepoWrite { + RepoWrite::Delete { + collection: POST.to_string(), + rkey: rkey.to_string(), + swap_record: None, + } + } + + async fn exercise_write_read_cycle(store: &dyn RepoStore) { + let applied = store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + assert_eq!(applied.rev, "3rev1"); + assert!(matches!(applied.outcomes[0], WriteOutcome::Created { .. })); + + let got = store + .get_record(SPACE, DID, POST, "a") + .await + .unwrap() + .unwrap(); + assert_eq!( + rsky_space::record::decode_record(&got.value).unwrap(), + json!({"text": "one"}) + ); + assert_eq!(got.cid, dag_cbor_cid(&value("one")).to_string()); + + // The digest matches an independent fold over the record set. + let head = store.head(SPACE, DID).await.unwrap().unwrap(); + let mut expected = LtHash::new(); + expected.add(&element(POST, "a", &got.cid)); + assert_eq!(head.hash(), expected.hash()); + + // Update swaps the element, delete removes it, digest returns to empty. + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Update { + collection: POST.to_string(), + rkey: "a".to_string(), + value: value("two"), + swap_record: Some(got.cid.clone()), + }], + ) + .await + .unwrap(); + let updated = store + .get_record(SPACE, DID, POST, "a") + .await + .unwrap() + .unwrap(); + assert_eq!(updated.cid, dag_cbor_cid(&value("two")).to_string()); + + store + .apply_writes(SPACE, DID, "3rev3", &[delete("a")]) + .await + .unwrap(); + assert!(store + .get_record(SPACE, DID, POST, "a") + .await + .unwrap() + .is_none()); + assert_eq!( + store.head(SPACE, DID).await.unwrap().unwrap().hash(), + LtHash::new().hash() + ); + } + + async fn exercise_swap_and_conflict(store: &dyn RepoStore) { + store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + + assert!(matches!( + store + .apply_writes(SPACE, DID, "3rev2", &[create("a", "again")]) + .await, + Err(HostError::InvalidRequest(_)) + )); + assert!(matches!( + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Delete { + collection: POST.to_string(), + rkey: "a".to_string(), + swap_record: Some("bafyreiwrong".to_string()), + }] + ) + .await, + Err(HostError::InvalidSwap) + )); + // A swap against an absent record is also a swap failure. + assert!(matches!( + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Update { + collection: POST.to_string(), + rkey: "missing".to_string(), + value: value("x"), + swap_record: Some("bafyreiwrong".to_string()), + }] + ) + .await, + Err(HostError::InvalidSwap) + )); + + // A rejected batch leaves nothing behind. + let head = store.head(SPACE, DID).await.unwrap().unwrap(); + assert_eq!(head.rev, "3rev1"); + let (records, _) = store + .list_records(SPACE, DID, None, None, 10) + .await + .unwrap(); + assert_eq!(records.len(), 1); + } + + async fn exercise_noop_delete(store: &dyn RepoStore) { + store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + let applied = store + .apply_writes(SPACE, DID, "3rev2", &[delete("ghost")]) + .await + .unwrap(); + assert_eq!(applied.outcomes, vec![WriteOutcome::Noop]); + // A no-op neither advances the revision nor writes an oplog entry. + assert_eq!(applied.rev, "3rev1"); + let page = store.list_ops(SPACE, DID, None, None, 10).await.unwrap(); + assert_eq!(page.ops.len(), 1); + } + + async fn exercise_atomic_batch(store: &dyn RepoStore) { + store + .apply_writes( + SPACE, + DID, + "3rev1", + &[ + create("a", "one"), + create("b", "two"), + RepoWrite::Create { + collection: LIKE.to_string(), + rkey: "l1".to_string(), + value: value("like"), + }, + ], + ) + .await + .unwrap(); + let page = store.list_ops(SPACE, DID, None, None, 10).await.unwrap(); + assert_eq!(page.ops.len(), 3); + // Operations mutated atomically share a revision. + assert!(page.ops.iter().all(|o| o.rev == "3rev1")); + assert!(page.complete); + + // A create-then-delete of the same path within one batch nets to nothing. + store + .apply_writes(SPACE, DID, "3rev2", &[create("c", "three"), delete("c")]) + .await + .unwrap(); + assert!(store + .get_record(SPACE, DID, POST, "c") + .await + .unwrap() + .is_none()); + } + + async fn exercise_listing(store: &dyn RepoStore) { + for (i, rkey) in ["a", "b", "c"].iter().enumerate() { + store + .apply_writes(SPACE, DID, &format!("3rev{i}"), &[create(rkey, rkey)]) + .await + .unwrap(); + } + store + .apply_writes( + SPACE, + DID, + "3rev9", + &[RepoWrite::Create { + collection: LIKE.to_string(), + rkey: "l1".to_string(), + value: value("like"), + }], + ) + .await + .unwrap(); + + let (page, cursor) = store.list_records(SPACE, DID, None, None, 2).await.unwrap(); + assert_eq!(page.len(), 2); + assert_eq!(page[0].collection, LIKE); + let (page, cursor2) = store + .list_records(SPACE, DID, None, cursor.as_deref(), 2) + .await + .unwrap(); + assert_eq!(page.len(), 2); + // A full final page still yields a cursor; the empty page ends paging. + let (page, cursor3) = store + .list_records(SPACE, DID, None, cursor2.as_deref(), 2) + .await + .unwrap(); + assert!(page.is_empty()); + assert!(cursor3.is_none()); + + let (posts, _) = store + .list_records(SPACE, DID, Some(POST), None, 10) + .await + .unwrap(); + assert_eq!(posts.len(), 3); + assert!(posts.iter().all(|r| r.collection == POST)); + + assert!(matches!( + store + .list_records(SPACE, "did:plc:nobody", None, None, 10) + .await, + Err(HostError::RepoNotFound) + )); + } + + async fn exercise_oplog_paging(store: &dyn RepoStore) { + for i in 0..5 { + store + .apply_writes( + SPACE, + DID, + &format!("3rev{i}"), + &[create(&format!("r{i}"), "x")], + ) + .await + .unwrap(); + } + + let page = store.list_ops(SPACE, DID, None, None, 2).await.unwrap(); + assert_eq!(page.ops.len(), 2); + assert!(!page.complete); + let page2 = store + .list_ops(SPACE, DID, None, page.cursor.as_deref(), 10) + .await + .unwrap(); + assert_eq!(page2.ops.len(), 3); + assert!(page2.complete); + assert!(page2.cursor.is_none()); + + // `since` yields strictly later revisions. + let page = store + .list_ops(SPACE, DID, Some("3rev2"), None, 10) + .await + .unwrap(); + assert_eq!( + page.ops.iter().map(|o| o.rev.as_str()).collect::>(), + vec!["3rev3", "3rev4"] + ); + + // A revision at or past the head yields an empty, complete page. + let page = store + .list_ops(SPACE, DID, Some("3rev4"), None, 10) + .await + .unwrap(); + assert!(page.ops.is_empty()); + + assert!(matches!( + store.list_ops(SPACE, DID, None, Some("abc"), 10).await, + Err(HostError::InvalidRequest(_)) + )); + assert!(matches!( + store + .list_ops(SPACE, "did:plc:nobody", None, None, 10) + .await, + Err(HostError::RepoNotFound) + )); + } + + async fn exercise_history_unavailable(store: &dyn RepoStore) { + store + .apply_writes(SPACE, DID, "3rev5", &[create("a", "one")]) + .await + .unwrap(); + // `since` predates every retained operation. + assert!(matches!( + store.list_ops(SPACE, DID, Some("3rev1"), None, 10).await, + Err(HostError::HistoryUnavailable) + )); + } + + async fn exercise_isolation_and_deletion(store: &dyn RepoStore) { + let other_space = "at://did:plc:auth/space/community.blacksky.feed/other"; + store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + store + .apply_writes(other_space, DID, "3rev1", &[create("a", "other")]) + .await + .unwrap(); + store + .apply_writes(SPACE, "did:plc:other", "3rev1", &[create("a", "theirs")]) + .await + .unwrap(); + + store.delete_repo(SPACE, DID).await.unwrap(); + assert!(store.head(SPACE, DID).await.unwrap().is_none()); + assert!(store.head(other_space, DID).await.unwrap().is_some()); + assert!(store.head(SPACE, "did:plc:other").await.unwrap().is_some()); + // Deleting an absent repo is not an error. + store.delete_repo(SPACE, DID).await.unwrap(); + } + + macro_rules! both_backings { + ($name:ident, $exercise:ident) => { + #[tokio::test] + async fn $name() { + $exercise(&InMemoryRepos::default()).await; + $exercise(&SqliteRepos::open_in_memory().unwrap()).await; + } + }; + } + + both_backings!(write_read_cycle, exercise_write_read_cycle); + both_backings!(swap_and_conflict, exercise_swap_and_conflict); + both_backings!(noop_delete, exercise_noop_delete); + both_backings!(atomic_batch, exercise_atomic_batch); + both_backings!(listing, exercise_listing); + both_backings!(oplog_paging, exercise_oplog_paging); + both_backings!(history_unavailable, exercise_history_unavailable); + both_backings!(isolation_and_deletion, exercise_isolation_and_deletion); + + #[tokio::test] + async fn digest_is_order_independent_across_repos() { + let a = InMemoryRepos::default(); + a.apply_writes(SPACE, DID, "3rev1", &[create("x", "one")]) + .await + .unwrap(); + a.apply_writes(SPACE, DID, "3rev2", &[create("y", "two")]) + .await + .unwrap(); + + let b = SqliteRepos::open_in_memory().unwrap(); + b.apply_writes(SPACE, DID, "3rev1", &[create("y", "two")]) + .await + .unwrap(); + b.apply_writes(SPACE, DID, "3rev2", &[create("x", "one")]) + .await + .unwrap(); + + assert_eq!( + a.head(SPACE, DID).await.unwrap().unwrap().hash(), + b.head(SPACE, DID).await.unwrap().unwrap().hash() + ); + } + + #[tokio::test] + async fn sqlite_persists_across_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("repos.db"); + let cid = { + let store = SqliteRepos::open(&path).unwrap(); + let applied = store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + match &applied.outcomes[0] { + WriteOutcome::Created { cid } => cid.clone(), + other => panic!("unexpected outcome {other:?}"), + } + }; + let store = SqliteRepos::open(&path).unwrap(); + let got = store + .get_record(SPACE, DID, POST, "a") + .await + .unwrap() + .unwrap(); + assert_eq!(got.cid, cid); + assert_eq!(store.head(SPACE, DID).await.unwrap().unwrap().rev, "3rev1"); + + assert!(matches!( + SqliteRepos::open(dir.path().join("missing/nested.db")), + Err(HostError::Store(_)) + )); + } + + #[test] + fn history_window_and_cursor_edges() { + assert!(ensure_history(None, None).is_ok()); + assert!(ensure_history(Some("3rev1"), Some("3rev1")).is_ok()); + assert!(matches!( + ensure_history(Some("3rev1"), None), + Err(HostError::HistoryUnavailable) + )); + assert_eq!(parse_cursor(None).unwrap(), None); + assert_eq!(parse_cursor(Some("7")).unwrap(), Some(7)); + } +} diff --git a/rsky-space/Cargo.toml b/rsky-space/Cargo.toml index 88862d6e..a639209b 100644 --- a/rsky-space/Cargo.toml +++ b/rsky-space/Cargo.toml @@ -16,6 +16,7 @@ serde_json = { workspace = true } serde_bytes = { workspace = true } serde_ipld_dagcbor = { workspace = true } lexicon_cid = { workspace = true } +ipld-core = { workspace = true } iroh-car = "0.5.1" tokio = { workspace = true } sha2 = { workspace = true } diff --git a/rsky-space/src/error.rs b/rsky-space/src/error.rs index eb1ab649..ba2f07f6 100644 --- a/rsky-space/src/error.rs +++ b/rsky-space/src/error.rs @@ -35,6 +35,8 @@ pub enum SpaceError { Car(String), #[error("decode error: {0}")] Decode(String), + #[error("record is {size} bytes, over the {max}-byte limit")] + RecordTooLarge { size: usize, max: usize }, #[error("invalid jwk: {0}")] InvalidJwk(String), #[error("crypto error: {0}")] diff --git a/rsky-space/src/lib.rs b/rsky-space/src/lib.rs index 6e73f18c..7cb4d953 100644 --- a/rsky-space/src/lib.rs +++ b/rsky-space/src/lib.rs @@ -13,6 +13,7 @@ //! signature/MAC verification. //! - [`credential`] — delegation tokens, space credentials, and client //! attestations (the JWT envelope + verification). +//! - [`record`] — shape-agnostic permissioned-record encoding (DAG-CBOR). //! - [`jwk`] — minimal EC JWK (P-256) verification for ES256 client //! attestations. //! - [`space_id`] — space and permissioned-record `at://.../space/...` @@ -29,6 +30,7 @@ pub mod credential; pub mod error; pub mod jwk; pub mod lthash; +pub mod record; pub mod space_id; pub mod types; @@ -36,5 +38,6 @@ pub use car::{repo_car_bytes, write_repo_car, RepoCarValidator}; pub use error::{Result, SpaceError}; pub use jwk::{verify_es256, EcJwk, JwkSet}; pub use lthash::LtHash; +pub use record::{dag_cbor_cid, decode_record, encode_record}; pub use space_id::{is_space_uri, RecordId, SpaceId}; pub use types::{RepoOp, RepoRef, SignedCommit}; diff --git a/rsky-space/src/record.rs b/rsky-space/src/record.rs new file mode 100644 index 00000000..3a08fd58 --- /dev/null +++ b/rsky-space/src/record.rs @@ -0,0 +1,210 @@ +//! Permissioned-record bytes: the atproto data model over DAG-CBOR. +//! +//! A repo host stores permissioned records shape-agnostically — it checks that +//! the value is a well-formed atproto data-model object under a size cap and +//! nothing more. Lexicon validation of permissioned records happens at the +//! consuming boundary, not here, because records in a space legitimately carry +//! space URIs in fields a public lexicon declares as `at-uri`. +//! +//! JSON ↔ DAG-CBOR follows the atproto data model: `{"$link": "…"}` is a CID +//! link, `{"$bytes": "…"}` is base64 bytes, and floats are rejected. + +use base64::Engine; +use ipld_core::ipld::Ipld; +use lexicon_cid::multihash::Multihash; +use lexicon_cid::Cid; +use serde_json::{Map, Number, Value}; +use sha2::{Digest, Sha256}; + +use crate::error::{Result, SpaceError}; + +const SHA2_256: u64 = 0x12; +/// DAG-CBOR multicodec, the codec of every permissioned record block. +pub const DAG_CBOR: u64 = 0x71; + +/// The CID of an already-encoded DAG-CBOR block. +pub fn dag_cbor_cid(bytes: &[u8]) -> Cid { + let digest = Sha256::digest(bytes); + let multihash = Multihash::wrap(SHA2_256, &digest).expect("sha256 digest fits in multihash"); + Cid::new_v1(DAG_CBOR, multihash) +} + +fn b64() -> base64::engine::general_purpose::GeneralPurpose { + base64::engine::general_purpose::STANDARD +} + +fn decode_err(msg: impl Into) -> SpaceError { + SpaceError::Decode(msg.into()) +} + +fn json_to_ipld(value: &Value) -> Result { + Ok(match value { + Value::Null => Ipld::Null, + Value::Bool(b) => Ipld::Bool(*b), + Value::String(s) => Ipld::String(s.clone()), + Value::Number(n) => Ipld::Integer( + n.as_i64() + .map(i128::from) + .ok_or_else(|| decode_err("only integer numbers are representable"))?, + ), + Value::Array(items) => Ipld::List(items.iter().map(json_to_ipld).collect::>()?), + Value::Object(map) => match tagged(map) { + Some(("$link", Value::String(s))) => Ipld::Link( + s.parse::() + .map_err(|e| decode_err(format!("invalid $link: {e}")))?, + ), + Some(("$bytes", Value::String(s))) => Ipld::Bytes( + b64() + .decode(s) + .map_err(|e| decode_err(format!("invalid $bytes: {e}")))?, + ), + _ => Ipld::Map( + map.iter() + .map(|(k, v)| Ok((k.clone(), json_to_ipld(v)?))) + .collect::>()?, + ), + }, + }) +} + +/// A single-entry map carrying one of the data model's reserved `$` keys. +fn tagged(map: &Map) -> Option<(&str, &Value)> { + if map.len() != 1 { + return None; + } + let (key, value) = map.iter().next()?; + matches!(key.as_str(), "$link" | "$bytes").then_some((key.as_str(), value)) +} + +fn ipld_to_json(value: &Ipld) -> Result { + Ok(match value { + Ipld::Null => Value::Null, + Ipld::Bool(b) => Value::Bool(*b), + Ipld::String(s) => Value::String(s.clone()), + Ipld::Integer(i) => Value::Number(Number::from( + i64::try_from(*i).map_err(|_| decode_err("integer out of range"))?, + )), + Ipld::Float(_) => return Err(decode_err("floats are not part of the atproto data model")), + Ipld::Bytes(b) => serde_json::json!({ "$bytes": b64().encode(b) }), + Ipld::Link(cid) => serde_json::json!({ "$link": cid.to_string() }), + Ipld::List(items) => Value::Array(items.iter().map(ipld_to_json).collect::>()?), + Ipld::Map(map) => Value::Object( + map.iter() + .map(|(k, v)| Ok((k.clone(), ipld_to_json(v)?))) + .collect::>()?, + ), + }) +} + +/// Encode a record value to canonical DAG-CBOR. The value must be an object; +/// no other structural or lexicon constraint is applied. +pub fn encode_record(value: &Value, max_bytes: usize) -> Result> { + if !value.is_object() { + return Err(decode_err("record must be an object")); + } + let ipld = json_to_ipld(value)?; + let bytes = serde_ipld_dagcbor::to_vec(&ipld).map_err(|e| decode_err(e.to_string()))?; + if bytes.len() > max_bytes { + return Err(SpaceError::RecordTooLarge { + size: bytes.len(), + max: max_bytes, + }); + } + Ok(bytes) +} + +/// Decode stored DAG-CBOR record bytes back to their JSON representation. +pub fn decode_record(bytes: &[u8]) -> Result { + let ipld: Ipld = + serde_ipld_dagcbor::from_slice(bytes).map_err(|e| decode_err(e.to_string()))?; + ipld_to_json(&ipld) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const MAX: usize = 64 * 1024; + + #[test] + fn roundtrips_the_data_model() { + let value = json!({ + "$type": "app.bsky.feed.post", + "text": "hi", + "count": 3, + "flag": true, + "nothing": null, + "tags": ["a", "b"], + "embed": { + "image": {"$link": "bafkreiabc2hs4mmvxbtmvj47xrslw6ijbcbtoz2jyxbjyqmjkgu3f4z2wq"}, + "raw": {"$bytes": "AQID"} + } + }); + let bytes = encode_record(&value, MAX).unwrap(); + assert_eq!(decode_record(&bytes).unwrap(), value); + } + + #[test] + fn encoding_is_canonical_and_order_independent() { + let a = encode_record(&json!({"a": 1, "b": 2}), MAX).unwrap(); + let b = encode_record(&json!({"b": 2, "a": 1}), MAX).unwrap(); + assert_eq!(a, b); + assert_eq!(dag_cbor_cid(&a), dag_cbor_cid(&b)); + } + + #[test] + fn cid_is_a_dag_cbor_sha256_link() { + let cid = dag_cbor_cid(&encode_record(&json!({"a": 1}), MAX).unwrap()); + assert_eq!(cid.codec(), DAG_CBOR); + assert_eq!(cid.hash().code(), SHA2_256); + assert!(cid.to_string().starts_with("bafyrei")); + } + + #[test] + fn a_space_uri_is_stored_verbatim() { + let uri = + "at://did:plc:auth/space/community.blacksky.feed/main/did:plc:a/app.bsky.feed.post/3k"; + let value = json!({"subject": {"uri": uri, "cid": "bafyreia"}}); + let bytes = encode_record(&value, MAX).unwrap(); + assert_eq!(decode_record(&bytes).unwrap(), value); + } + + #[test] + fn rejects_non_objects_floats_and_oversize() { + assert!(matches!( + encode_record(&json!([1, 2]), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"n": 1.5}), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"$link": "not-a-cid"}), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"$bytes": "!!!"}), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"text": "x".repeat(100)}), 16), + Err(SpaceError::RecordTooLarge { .. }) + )); + } + + #[test] + fn multi_key_dollar_maps_stay_maps() { + let value = json!({"m": {"$link": "bafyreia", "extra": 1}}); + let bytes = encode_record(&value, MAX).unwrap(); + assert_eq!(decode_record(&bytes).unwrap(), value); + } + + #[test] + fn decode_rejects_floats_and_malformed_cbor() { + let float = serde_ipld_dagcbor::to_vec(&Ipld::Float(1.5)).unwrap(); + assert!(matches!(decode_record(&float), Err(SpaceError::Decode(_)))); + assert!(matches!(decode_record(&[0xff]), Err(SpaceError::Decode(_)))); + } +} From 2de3b575cec69c76d0eb75efc91675d042fc3f09 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 11:41:48 -0400 Subject: [PATCH 03/56] feat(space-host): register spaces through lifecycle handshake --- rsky-space-host/src/authority.rs | 55 +++++++-- rsky-space-host/src/config.rs | 22 ++++ rsky-space-host/src/http.rs | 183 +++++++++++++++++++++++++--- rsky-space-host/src/lib.rs | 1 + rsky-space-host/src/main.rs | 11 ++ rsky-space-host/src/registration.rs | 92 ++++++++++++++ rsky-space-host/src/service_jwt.rs | 24 ++++ 7 files changed, 359 insertions(+), 29 deletions(-) create mode 100644 rsky-space-host/src/registration.rs diff --git a/rsky-space-host/src/authority.rs b/rsky-space-host/src/authority.rs index 06aa12f9..54371e51 100644 --- a/rsky-space-host/src/authority.rs +++ b/rsky-space-host/src/authority.rs @@ -181,7 +181,13 @@ impl Authority { /// `dpop_jkt` must come from a DPoP proof this server verified, never from /// a request field: a field is an assertion anyone holding a delegation /// token can make about a key someone else controls. - pub fn mint_credential(&self, now: u64, jti: String, dpop_jkt: &str) -> Result { + pub fn mint_credential_for( + &self, + space: &SpaceId, + now: u64, + jti: String, + dpop_jkt: &str, + ) -> Result { let header = JwtHeader { typ: CREDENTIAL_TYP.to_string(), alg: rsky_crypto::constants::SECP256K1_JWT_ALG.to_string(), @@ -189,7 +195,7 @@ impl Authority { }; let claims = SpaceClaims { iss: self.authority_did().to_string(), - sub: self.space_uri(), + sub: space.uri(), aud: None, iat: now, exp: now + CREDENTIAL_TTL_SECS, @@ -202,12 +208,17 @@ impl Authority { Ok(jwt) } + pub fn mint_credential(&self, now: u64, jti: String, dpop_jkt: &str) -> Result { + self.mint_credential_for(&self.space, now, jti, dpop_jkt) + } + /// The full `getSpaceCredential` flow: verify the client attestation (when /// required or presented), apply appAccess, verify the delegation token, /// consult the policy, then mint. #[allow(clippy::too_many_arguments)] - pub async fn get_space_credential( + pub async fn get_space_credential_for( &self, + space: &SpaceId, delegation_jwt: &str, attestation_jwt: Option<&str>, policy: &Policy, @@ -241,7 +252,7 @@ impl Authority { let user_key = keys.signing_key(&user_did).await?; let verified_user = credential::verify_delegation_token( delegation_jwt, - &self.space_uri(), + &space.uri(), self.authority_did(), &user_key, now, @@ -249,16 +260,40 @@ impl Authority { .map_err(|e| HostError::Delegation(e.to_string()))?; // User axis: the policy decision (member list, public, or managing app). if !policy - .authorizes( - &self.space_uri(), - &verified_user, - attested_client_id.as_deref(), - ) + .authorizes(&space.uri(), &verified_user, attested_client_id.as_deref()) .await? { return Err(HostError::NotAuthorized); } - self.mint_credential(now, jti, dpop_jkt) + self.mint_credential_for(space, now, jti, dpop_jkt) + } + + #[allow(clippy::too_many_arguments)] + pub async fn get_space_credential( + &self, + delegation_jwt: &str, + attestation_jwt: Option<&str>, + policy: &Policy, + keys: &dyn KeyResolver, + metadata: &dyn MetadataFetcher, + jti_store: &dyn JtiStore, + now: u64, + jti: String, + dpop_jkt: &str, + ) -> Result { + self.get_space_credential_for( + &self.space, + delegation_jwt, + attestation_jwt, + policy, + keys, + metadata, + jti_store, + now, + jti, + dpop_jkt, + ) + .await } } diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index c903afc7..bd2701a5 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -49,6 +49,14 @@ pub struct Config { #[arg(long, env = "SPACEHOST_MEMBERSHIP_DB_URL", default_value = "")] pub membership_db_url: String, + /// Feeds base URL that receives host-registration acknowledgements. + #[arg(long, env = "SPACEHOST_LIFECYCLE_URL", default_value = "")] + pub lifecycle_url: String, + + /// Feeds service DID, used as the acknowledgement JWT audience. + #[arg(long, env = "SPACEHOST_LIFECYCLE_SERVICE_DID", default_value = "")] + pub lifecycle_service_did: String, + /// SQLite path for host state (writer set, registrations, used nonces). #[arg(long, env = "SPACEHOST_DB_PATH", default_value = "./space_host.db")] pub db_path: String, @@ -98,6 +106,14 @@ impl Config { "managing-app policy requires SPACEHOST_MANAGING_APP (did#fragment)".to_string(), ); } + if self.policy == PolicyMode::ManagingApp + && (self.lifecycle_url.is_empty() || self.lifecycle_service_did.is_empty()) + { + return Err( + "managing-app policy requires SPACEHOST_LIFECYCLE_URL and SPACEHOST_LIFECYCLE_SERVICE_DID" + .to_string(), + ); + } if self.public_url.trim_end_matches('/').is_empty() { return Err("SPACEHOST_PUBLIC_URL must be an absolute origin".to_string()); } @@ -157,6 +173,8 @@ mod tests { std::env::set_var("SPACEHOST_MANAGING_APP", "did:web:app#svc"); std::env::set_var("SPACEHOST_MEMBERS", "did:plc:aaa, did:plc:bbb,"); std::env::set_var("SPACEHOST_MEMBERSHIP_DB_URL", "postgres://env"); + std::env::set_var("SPACEHOST_LIFECYCLE_URL", "https://feeds.example"); + std::env::set_var("SPACEHOST_LIFECYCLE_SERVICE_DID", "did:web:feeds.example"); std::env::set_var("SPACEHOST_DB_PATH", "/tmp/space.db"); std::env::set_var("SPACEHOST_PLC_URL", "https://plc.example"); std::env::set_var("SPACEHOST_BIND", "127.0.0.1:1234"); @@ -168,6 +186,8 @@ mod tests { "SPACEHOST_MANAGING_APP", "SPACEHOST_MEMBERS", "SPACEHOST_MEMBERSHIP_DB_URL", + "SPACEHOST_LIFECYCLE_URL", + "SPACEHOST_LIFECYCLE_SERVICE_DID", "SPACEHOST_DB_PATH", "SPACEHOST_PLC_URL", "SPACEHOST_BIND", @@ -182,6 +202,8 @@ mod tests { vec!["did:plc:aaa".to_string(), "did:plc:bbb".to_string()] ); assert_eq!(cfg.db_path, "/tmp/space.db"); + assert_eq!(cfg.lifecycle_url, "https://feeds.example"); + assert_eq!(cfg.lifecycle_service_did, "did:web:feeds.example"); assert_eq!(cfg.plc_url, "https://plc.example"); assert_eq!(cfg.bind, "127.0.0.1:1234"); assert!(cfg.validate().is_ok()); diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 1106314a..d1df81eb 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -24,6 +24,7 @@ use crate::keys::DocSource; use crate::managing_app::require_https; use crate::notify::{fan_out_write, Notifier, NOTIFY_WRITE_LXM}; use crate::policy::Policy; +use crate::registration::{LifecycleAcker, REGISTER_SPACE_LXM}; use crate::service_jwt; use crate::store::{RegistrationStore, Subscriber, WriterSetStore}; @@ -40,6 +41,7 @@ pub struct AppState { pub jti_store: Arc, pub writers: Arc, pub registrations: Arc, + pub lifecycle_acker: Option>, /// Resolves a subscriber's service identifier to its delivery endpoint. pub docs: Arc, pub notifier: Arc, @@ -68,6 +70,10 @@ pub fn router(state: AppState) -> Router { post(register_notify), ) .route("/xrpc/com.atproto.space.notifyWrite", post(notify_write)) + .route( + "/xrpc/community.blacksky.space.register", + post(register_space), + ) .with_state(state) } @@ -95,6 +101,10 @@ impl ApiError { fn auth_required(message: impl Into) -> Self { Self::new(StatusCode::UNAUTHORIZED, "AuthenticationRequired", message) } + + fn forbidden(message: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, "Forbidden", message) + } } impl IntoResponse for ApiError { @@ -230,11 +240,13 @@ fn require_space_credential( headers: &HeaderMap, method: &str, nsid: &str, -) -> Result<(), ApiError> { + space_uri: &str, +) -> Result { + let space = require_this_space(state, space_uri)?; let jwt = dpop_credential(headers)?; let bound_jkt = credential::verify_space_credential( jwt, - &state.authority.space_uri(), + &space.uri(), state.authority.authority_did(), state.authority.signer.did_key(), (state.now)(), @@ -248,7 +260,7 @@ fn require_space_credential( "DPoP key thumbprint does not match the credential binding", )); } - Ok(()) + Ok(space) } /// Resolve a `did:...#fragment` subscriber to its delivery endpoint. The @@ -277,13 +289,14 @@ async fn resolve_service_endpoint(docs: &dyn DocSource, service: &str) -> Result }) } -fn require_this_space(state: &AppState, space: &str) -> Result<(), ApiError> { - if space != state.authority.space_uri() { - return Err(ApiError::invalid_request(format!( - "space not hosted here: {space}" - ))); - } - Ok(()) +fn require_this_space( + state: &AppState, + space: &str, +) -> Result { + state + .authority + .resolve(space) + .map_err(|_| ApiError::invalid_request(format!("space not hosted here: {space}"))) } async fn health() -> Json { @@ -295,20 +308,45 @@ async fn get_space( headers: HeaderMap, Query(params): Query, ) -> Result, ApiError> { - require_space_credential(&state, &headers, "GET", "com.atproto.space.getSpace")?; - require_this_space(&state, ¶ms.space)?; + let space = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.getSpace", + ¶ms.space, + )?; Ok(Json(GetSpaceOutput { - space: state.authority.space_uri(), + space: space.uri(), config: SpaceConfig::Simplespace(state.authority.space_config(&state.policy)), })) } +async fn require_service_auth( + state: &AppState, + headers: &HeaderMap, + expected_lxm: &str, +) -> Result { + let jwt = bearer(headers)?; + let claims = service_jwt::claims(jwt)?; + let issuer_key = state.keys.signing_key(&claims.iss).await?; + let authority_did = state.authority.authority_did(); + let space_host_aud = format!("{authority_did}#atproto_space_host"); + service_jwt::verify( + jwt, + &[authority_did, space_host_aud.as_str()], + expected_lxm, + &issuer_key, + (state.now)(), + ) + .map_err(|e| ApiError::new(StatusCode::UNAUTHORIZED, "InvalidToken", e.to_string())) +} + async fn get_space_credential( State(state): State, headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - require_this_space(&state, &input.space)?; + let space = require_this_space(&state, &input.space)?; let delegation_token = bearer(&headers)?; // Before the delegation token, so a caller with a bad proof does not burn // its single-use grant finding out. @@ -321,7 +359,8 @@ async fn get_space_credential( )?; let credential = state .authority - .get_space_credential( + .get_space_credential_for( + &space, delegation_token, input.client_attestation.as_deref(), &state.policy, @@ -341,8 +380,13 @@ async fn list_repos( headers: HeaderMap, Query(params): Query, ) -> Result, ApiError> { - require_space_credential(&state, &headers, "GET", "com.atproto.space.listRepos")?; - require_this_space(&state, ¶ms.space)?; + require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.listRepos", + ¶ms.space, + )?; let limit = params .limit .unwrap_or(DEFAULT_LIST_LIMIT) @@ -359,8 +403,13 @@ async fn register_notify( headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - require_space_credential(&state, &headers, "POST", "com.atproto.space.registerNotify")?; - require_this_space(&state, &input.space)?; + require_space_credential( + &state, + &headers, + "POST", + "com.atproto.space.registerNotify", + &input.space, + )?; // `service` names the subscriber, which is both where to deliver and who // the delivery is addressed to (proposals#100); `endpoint` is the // pre-amendment shape and loses when both are sent. @@ -393,6 +442,43 @@ async fn register_notify( })) } +#[derive(serde::Deserialize)] +struct RegisterSpaceInput { + space: String, + generation: i64, +} + +async fn register_space( + State(state): State, + headers: HeaderMap, + Json(input): Json, +) -> Result, ApiError> { + if input.generation < 1 { + return Err(ApiError::invalid_request("generation must be positive")); + } + let claims = require_service_auth(&state, &headers, REGISTER_SPACE_LXM).await?; + let expected_issuer = state + .policy + .managing_app() + .and_then(|service| service.split_once('#').map(|(did, _)| did)) + .ok_or_else(|| ApiError::forbidden("space has no managing app"))?; + if claims.iss != expected_issuer { + return Err(ApiError::forbidden("issuer is not the managing app")); + } + let acker = state.lifecycle_acker.as_ref().ok_or_else(|| { + ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "LifecycleUnavailable", + "lifecycle acknowledgement is not configured", + ) + })?; + state.authority.register(&input.space)?; + acker + .ack_host_registered(&input.space, input.generation) + .await?; + Ok(Json(serde_json::json!({}))) +} + async fn notify_write( State(state): State, headers: HeaderMap, @@ -546,6 +632,26 @@ mod tests { writes: tokio::sync::mpsc::UnboundedReceiver<(String, NotifyWriteInput)>, } + #[derive(Default)] + struct RecordingAcker(std::sync::Mutex>); + + #[async_trait] + impl LifecycleAcker for RecordingAcker { + async fn ack_host_registered(&self, space: &str, generation: i64) -> HostResult<()> { + self.0.lock().unwrap().push((space.to_string(), generation)); + Ok(()) + } + } + + struct UnusedManagingApp; + + #[async_trait] + impl crate::managing_app::ManagingAppClient for UnusedManagingApp { + async fn check_user_access(&self, _: &str, _: &str, _: Option<&str>) -> HostResult { + Ok(false) + } + } + fn fixture(app_access: AppAccess, members: &[&str]) -> Fixture { let space = SpaceId::new( "did:plc:communityauthority", @@ -564,6 +670,7 @@ mod tests { jti_store: Arc::new(InMemoryJtiStore::default()), writers: Arc::new(InMemoryWriterSet::default()), registrations: Arc::new(InMemoryRegistrations::default()), + lifecycle_acker: None, docs: Arc::new(NoDocs), notifier: Arc::new(RecordingNotifier { tx }), dpop: Arc::new(rsky_oauth::dpop::DpopManager::new( @@ -1092,6 +1199,44 @@ mod tests { assert_eq!(out["error"], "InternalError"); } + #[tokio::test] + async fn registration_authenticates_the_managing_app_and_acks_before_activation() { + let mut f = fixture(AppAccess::Open, &[]); + f.state.policy = Arc::new(Policy::ManagingApp { + service_id: format!("{MEMBER}#bsky_fg"), + client: Arc::new(UnusedManagingApp), + }); + let acker = Arc::new(RecordingAcker::default()); + f.state.lifecycle_acker = Some(acker.clone()); + let space = "at://did:plc:communityauthority/space/community.blacksky.feed/new"; + let audience = format!("{}#atproto_space_host", f.state.authority.authority_did()); + let token = service_jwt::mint( + &user_signer(), + MEMBER, + &audience, + REGISTER_SPACE_LXM, + NOW, + "register-1".to_string(), + ) + .unwrap(); + let (status, body) = send( + &f.state, + post_req( + &format!("/xrpc/{REGISTER_SPACE_LXM}"), + Some(&token), + serde_json::json!({"space": space, "generation": 7}), + ), + ) + .await; + + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(f.state.authority.resolve_registered(space).is_ok()); + assert_eq!( + acker.0.lock().unwrap().as_slice(), + &[(space.to_string(), 7)] + ); + } + #[tokio::test] async fn error_mapping_covers_every_lexicon_error_name() { for (err, status, name) in [ diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index 4065a30a..f61b921f 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -30,6 +30,7 @@ pub mod managing_app; pub mod membership; pub mod notify; pub mod policy; +pub mod registration; pub mod repo; pub mod service_jwt; pub mod signing; diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index 11192e65..723c5546 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -16,6 +16,7 @@ use rsky_space_host::managing_app::HttpManagingApp; use rsky_space_host::membership::InMemoryMembership; use rsky_space_host::notify::HttpNotifier; use rsky_space_host::policy::Policy; +use rsky_space_host::registration::HttpLifecycleAcker; use rsky_space_host::signing::Signer; use rsky_space_host::store::SqliteStore; use std::sync::Arc; @@ -89,6 +90,16 @@ async fn main() -> Result<(), Box> { jti_store: store.clone(), writers: store.clone(), registrations: store, + lifecycle_acker: (cfg.policy == PolicyMode::ManagingApp).then(|| { + Arc::new(HttpLifecycleAcker::new( + cfg.lifecycle_url.clone(), + cfg.lifecycle_service_did.clone(), + cfg.authority_did.clone(), + signer.clone(), + now.clone(), + jti.clone(), + )) as Arc + }), notifier: Arc::new(HttpNotifier::new( cfg.authority_did.clone(), signer, diff --git a/rsky-space-host/src/registration.rs b/rsky-space-host/src/registration.rs new file mode 100644 index 00000000..1d88c0e5 --- /dev/null +++ b/rsky-space-host/src/registration.rs @@ -0,0 +1,92 @@ +use async_trait::async_trait; +use serde::Serialize; + +use crate::error::{HostError, Result}; +use crate::service_jwt::{ServiceJwtIssuer, SignerIssuer}; +use crate::signing::Signer; + +pub const REGISTER_SPACE_LXM: &str = "community.blacksky.space.register"; +pub const ACK_HOST_REGISTERED_LXM: &str = "community.blacksky.space.ackHostRegistered"; + +#[async_trait] +pub trait LifecycleAcker: Send + Sync { + async fn ack_host_registered(&self, space: &str, generation: i64) -> Result<()>; +} + +pub struct HttpLifecycleAcker { + base_url: String, + audience: String, + issuer: std::sync::Arc, + now: std::sync::Arc u64 + Send + Sync>, + jti: std::sync::Arc String + Send + Sync>, + http: reqwest::Client, +} + +impl HttpLifecycleAcker { + pub fn new( + base_url: impl Into, + audience: impl Into, + issuer: impl Into, + signer: Signer, + now: std::sync::Arc u64 + Send + Sync>, + jti: std::sync::Arc String + Send + Sync>, + ) -> Self { + Self::with_issuer( + base_url, + audience, + std::sync::Arc::new(SignerIssuer::new(issuer, signer)), + now, + jti, + ) + } + + pub fn with_issuer( + base_url: impl Into, + audience: impl Into, + issuer: std::sync::Arc, + now: std::sync::Arc u64 + Send + Sync>, + jti: std::sync::Arc String + Send + Sync>, + ) -> Self { + Self { + base_url: base_url.into().trim_end_matches('/').to_string(), + audience: audience.into(), + issuer, + now, + jti, + http: reqwest::Client::new(), + } + } +} + +#[derive(Serialize)] +struct Ack<'a> { + space: &'a str, + generation: i64, +} + +#[async_trait] +impl LifecycleAcker for HttpLifecycleAcker { + async fn ack_host_registered(&self, space: &str, generation: i64) -> Result<()> { + let token = self.issuer.mint( + &self.audience, + ACK_HOST_REGISTERED_LXM, + (self.now)(), + (self.jti)(), + )?; + let response = self + .http + .post(format!("{}/xrpc/{ACK_HOST_REGISTERED_LXM}", self.base_url)) + .bearer_auth(token) + .json(&Ack { space, generation }) + .send() + .await + .map_err(|e| HostError::ManagingApp(e.to_string()))?; + if !response.status().is_success() { + return Err(HostError::ManagingApp(format!( + "lifecycle acknowledgement returned {}", + response.status() + ))); + } + Ok(()) + } +} diff --git a/rsky-space-host/src/service_jwt.rs b/rsky-space-host/src/service_jwt.rs index c5393ac3..172ac3ce 100644 --- a/rsky-space-host/src/service_jwt.rs +++ b/rsky-space-host/src/service_jwt.rs @@ -33,6 +33,30 @@ pub struct ServiceClaims { pub const SERVICE_JWT_TTL_SECS: u64 = 60; +pub trait ServiceJwtIssuer: Send + Sync { + fn mint(&self, aud: &str, lxm: &str, now: u64, jti: String) -> Result; +} + +pub struct SignerIssuer { + issuer: String, + signer: Signer, +} + +impl SignerIssuer { + pub fn new(issuer: impl Into, signer: Signer) -> Self { + Self { + issuer: issuer.into(), + signer, + } + } +} + +impl ServiceJwtIssuer for SignerIssuer { + fn mint(&self, aud: &str, lxm: &str, now: u64, jti: String) -> Result { + mint(&self.signer, &self.issuer, aud, lxm, now, jti) + } +} + /// Mint a service-auth JWT signed by `signer` (sha256 then ECDSA, compact r||s). pub fn mint( signer: &Signer, From a1bd9631ea739be80f8c680c7c6f88d1fb4cf4b4 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 11:43:07 -0400 Subject: [PATCH 04/56] feat(space-host): sign hosted commits with PDS account keys --- Cargo.lock | 2 + rsky-space-host/Cargo.toml | 2 + rsky-space-host/src/commits.rs | 25 ++- rsky-space-host/src/error.rs | 2 + rsky-space-host/src/http.rs | 5 + rsky-space-host/src/lib.rs | 1 + rsky-space-host/src/pds_seam.rs | 307 ++++++++++++++++++++++++++++++++ 7 files changed, 335 insertions(+), 9 deletions(-) create mode 100644 rsky-space-host/src/pds_seam.rs diff --git a/Cargo.lock b/Cargo.lock index 259ae855..1c761014 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8430,6 +8430,7 @@ dependencies = [ "chrono", "clap", "hex", + "hmac", "http-body-util", "p256 0.13.2", "rand 0.8.5", @@ -8453,6 +8454,7 @@ dependencies = [ "tracing", "tracing-subscriber", "wiremock", + "zeroize", ] [[package]] diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index c6b0bf1d..630f0edb 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -21,6 +21,7 @@ rsky-oauth = { path = "../rsky-oauth", version = "0.3.0" } rsky-syntax = { workspace = true } secp256k1 = { workspace = true } sha2 = { workspace = true } +hmac = "0.12" serde = { workspace = true } serde_json = { workspace = true } rusqlite = { workspace = true } @@ -29,6 +30,7 @@ async-trait = "0.1" hex = "0.4" base64 = "0.22" rand = { workspace = true } +zeroize = "1" chrono = { version = "0.4.24", features = ["serde"] } axum = "0.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] } diff --git a/rsky-space-host/src/commits.rs b/rsky-space-host/src/commits.rs index 77a4215b..2af8579f 100644 --- a/rsky-space-host/src/commits.rs +++ b/rsky-space-host/src/commits.rs @@ -19,16 +19,23 @@ pub const IKM_BYTES: usize = 32; pub trait CommitSigner: Send + Sync { /// The `did:key` a reader verifies commits against. - fn did_key(&self) -> &str; - fn sign(&self, message: &[u8]) -> Result>; + fn did_key(&self, author_did: &str) -> Result; + fn sign(&self, author_did: &str, space_uri: &str, rev: &str, message: &[u8]) + -> Result>; } impl CommitSigner for Signer { - fn did_key(&self) -> &str { - Signer::did_key(self) + fn did_key(&self, _author_did: &str) -> Result { + Ok(Signer::did_key(self).to_string()) } - fn sign(&self, message: &[u8]) -> Result> { + fn sign( + &self, + _author_did: &str, + _space_uri: &str, + _rev: &str, + message: &[u8], + ) -> Result> { Signer::sign(self, message).map_err(HostError::Key) } } @@ -43,7 +50,7 @@ pub fn mint_commit( ikm: [u8; IKM_BYTES], ) -> Result { let ctx = build_ctx(space_uri, author_did, rev, &ikm); - let sig = signer.sign(&ctx)?; + let sig = signer.sign(author_did, space_uri, rev, &ctx)?; let mac = compute_mac(&ikm, &ctx, hash)?; Ok(SignedCommit { ver: COMMIT_VERSION, @@ -72,7 +79,7 @@ mod tests { fn verify(signer: &Signer, commit: &SignedCommit, hash: &[u8]) -> rsky_space::Result<()> { verify_commit( - CommitSigner::did_key(signer), + &CommitSigner::did_key(signer, AUTHOR).unwrap(), SPACE, AUTHOR, &commit.rev, @@ -113,7 +120,7 @@ mod tests { let hash = [7u8; 32]; let (signer, commit) = minted(hash, [3u8; 32]); assert!(verify_commit( - CommitSigner::did_key(&signer), + &CommitSigner::did_key(&signer, AUTHOR).unwrap(), "at://did:plc:auth/space/community.blacksky.feed/other", AUTHOR, &commit.rev, @@ -124,7 +131,7 @@ mod tests { ) .is_err()); assert!(verify_commit( - CommitSigner::did_key(&signer), + &CommitSigner::did_key(&signer, AUTHOR).unwrap(), SPACE, "did:plc:someoneelse", &commit.rev, diff --git a/rsky-space-host/src/error.rs b/rsky-space-host/src/error.rs index 9a93a829..00c393cd 100644 --- a/rsky-space-host/src/error.rs +++ b/rsky-space-host/src/error.rs @@ -26,6 +26,8 @@ pub enum HostError { InvalidRequest(String), #[error("space not hosted here: {0}")] SpaceNotFound(String), + #[error("account not hosted here: {0}")] + AccountNotHosted(String), #[error("repo not found")] RepoNotFound, #[error("swap cid did not match")] diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index d1df81eb..49b0a94d 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -146,6 +146,11 @@ impl From for ApiError { HostError::RepoNotFound => { Self::new(StatusCode::NOT_FOUND, "RepoNotFound", "repo not found") } + HostError::AccountNotHosted(_) => Self::new( + StatusCode::NOT_FOUND, + "RepoNotFound", + "repo not hosted here", + ), HostError::InvalidRequest(message) => Self::invalid_request(message.clone()), HostError::InvalidSwap => Self::new( StatusCode::CONFLICT, diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index f61b921f..c58e7e22 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -29,6 +29,7 @@ pub mod keys; pub mod managing_app; pub mod membership; pub mod notify; +pub mod pds_seam; pub mod policy; pub mod registration; pub mod repo; diff --git a/rsky-space-host/src/pds_seam.rs b/rsky-space-host/src/pds_seam.rs new file mode 100644 index 00000000..be51ae76 --- /dev/null +++ b/rsky-space-host/src/pds_seam.rs @@ -0,0 +1,307 @@ +use hmac::{Hmac, Mac}; +use secp256k1::SecretKey; +use sha2::{Digest, Sha256}; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use zeroize::{Zeroize, Zeroizing}; + +use crate::commits::CommitSigner; +use crate::error::{HostError, Result}; +use crate::service_jwt::{self, ServiceJwtIssuer}; +use crate::signing::Signer; + +const SECRET_KEY_BYTES: usize = 32; +pub const COMMIT_SIGN_AUDIT_EVENT: &str = "space_host_commit_signed"; + +#[derive(Clone)] +pub struct VerifyOnlyHs256Secret(Arc>>); + +impl VerifyOnlyHs256Secret { + pub fn new(secret: impl Into>) -> Option { + let secret = secret.into(); + (!secret.is_empty()).then(|| Self(Arc::new(Zeroizing::new(secret)))) + } + + pub(crate) fn verify(&self, signing_input: &[u8], signature: &[u8]) -> bool { + let Ok(mut mac) = as Mac>::new_from_slice(self.0.as_slice()) else { + return false; + }; + mac.update(signing_input); + mac.verify_slice(signature).is_ok() + } +} + +impl fmt::Debug for VerifyOnlyHs256Secret { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("VerifyOnlyHs256Secret([REDACTED])") + } +} + +pub trait SigningAudit: Send + Sync { + fn commit_signed(&self, did: &str, space: &str, rev: &str); +} + +struct TracingSigningAudit; + +impl SigningAudit for TracingSigningAudit { + fn commit_signed(&self, did: &str, space: &str, rev: &str) { + tracing::info!( + event = COMMIT_SIGN_AUDIT_EVENT, + did, + space, + rev, + "space repo commit signed" + ); + } +} + +pub struct PdsSeam { + directory: PathBuf, + audit: Arc, +} + +pub struct PdsServiceJwtIssuer { + seam: Arc, + issuer: String, +} + +impl PdsServiceJwtIssuer { + pub fn new(seam: Arc, issuer: impl Into) -> Self { + Self { + seam, + issuer: issuer.into(), + } + } +} + +impl ServiceJwtIssuer for PdsServiceJwtIssuer { + fn mint(&self, aud: &str, lxm: &str, now: u64, jti: String) -> Result { + let signer = self.seam.require_signer(&self.issuer)?; + service_jwt::mint(&signer, &self.issuer, aud, lxm, now, jti) + } +} + +impl PdsSeam { + pub fn open(directory: impl Into) -> Result { + let directory = directory.into(); + validate_actor_store_layout(&directory)?; + Ok(Self { + directory, + audit: Arc::new(TracingSigningAudit), + }) + } + + #[cfg(test)] + pub(crate) fn with_audit( + directory: impl Into, + audit: Arc, + ) -> Result { + let directory = directory.into(); + validate_actor_store_layout(&directory)?; + Ok(Self { directory, audit }) + } + + pub fn key_path(&self, author_did: &str) -> Option { + if author_did.is_empty() + || !author_did.starts_with("did:") + || author_did.contains('/') + || author_did.contains('\\') + || author_did.contains("..") + { + return None; + } + let digest = hex::encode(Sha256::digest(author_did.as_bytes())); + Some( + self.directory + .join(&digest[..2]) + .join(author_did) + .join("key"), + ) + } + + fn require_signer(&self, author_did: &str) -> Result { + let Some(path) = self.key_path(author_did) else { + return Err(HostError::InvalidRequest(format!( + "unusable did: {author_did}" + ))); + }; + let mut bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(HostError::AccountNotHosted(author_did.to_string())) + } + Err(error) => return Err(HostError::Key(error.to_string())), + }; + signer_from_bytes(&mut bytes) + } +} + +fn signer_from_bytes(bytes: &mut [u8]) -> Result { + let parsed = if bytes.len() == SECRET_KEY_BYTES { + SecretKey::from_slice(bytes).map_err(|error| HostError::Key(error.to_string())) + } else { + Err(HostError::Key(format!( + "actor key is {} bytes, expected {SECRET_KEY_BYTES}", + bytes.len() + ))) + }; + bytes.zeroize(); + parsed.map(Signer::from_secret) +} + +impl CommitSigner for PdsSeam { + fn did_key(&self, author_did: &str) -> Result { + Ok(self.require_signer(author_did)?.did_key().to_string()) + } + + fn sign( + &self, + author_did: &str, + space_uri: &str, + rev: &str, + message: &[u8], + ) -> Result> { + let signature = self + .require_signer(author_did)? + .sign(message) + .map_err(HostError::Key)?; + self.audit.commit_signed(author_did, space_uri, rev); + Ok(signature) + } +} + +fn validate_actor_store_layout(root: &Path) -> Result<()> { + if !root.is_dir() { + return Err(HostError::Store(format!( + "actor store is not a directory: {}", + root.display() + ))); + } + for prefix in std::fs::read_dir(root).map_err(|error| HostError::Store(error.to_string()))? { + let prefix = prefix.map_err(|error| HostError::Store(error.to_string()))?; + let name = prefix.file_name(); + let name = name.to_string_lossy(); + if name == "reserved_keys" { + continue; + } + if name.len() != 2 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(HostError::Store(format!( + "unrecognized actor-store layout entry: {name}" + ))); + } + if !prefix.path().is_dir() { + return Err(HostError::Store(format!( + "actor-store shard is not a directory: {name}" + ))); + } + for actor in + std::fs::read_dir(prefix.path()).map_err(|error| HostError::Store(error.to_string()))? + { + let actor = actor.map_err(|error| HostError::Store(error.to_string()))?; + let actor_name = actor.file_name().to_string_lossy().to_string(); + if !actor_name.starts_with("did:") + || !actor.path().join("store.sqlite").is_file() + || !actor.path().join("key").is_file() + { + return Err(HostError::Store(format!( + "unrecognized actor-store account layout: {actor_name}" + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commits::mint_commit; + use std::sync::Mutex; + + const DID: &str = "did:plc:member"; + const SPACE: &str = "at://did:plc:authority/space/community.blacksky.feed/main"; + + fn actor_store(secret: [u8; 32]) -> tempfile::TempDir { + let directory = tempfile::tempdir().unwrap(); + let digest = hex::encode(Sha256::digest(DID.as_bytes())); + let actor = directory.path().join(&digest[..2]).join(DID); + std::fs::create_dir_all(&actor).unwrap(); + std::fs::write(actor.join("key"), secret).unwrap(); + std::fs::write(actor.join("store.sqlite"), []).unwrap(); + directory + } + + #[test] + fn actor_key_buffer_is_zeroized_after_parsing() { + let mut bytes = [7u8; SECRET_KEY_BYTES]; + signer_from_bytes(&mut bytes).unwrap(); + assert_eq!(bytes, [0u8; SECRET_KEY_BYTES]); + } + + #[test] + fn service_auth_uses_the_authority_account_key() { + let directory = actor_store([7u8; SECRET_KEY_BYTES]); + let seam = Arc::new(PdsSeam::open(directory.path()).unwrap()); + let issuer = PdsServiceJwtIssuer::new(seam, DID); + let token = issuer + .mint("did:web:feeds.test", "test.method", 1_000, "jti-1".into()) + .unwrap(); + let account_key = Signer::from_secret(SecretKey::from_slice(&[7u8; 32]).unwrap()); + let claims = service_jwt::verify( + &token, + &["did:web:feeds.test"], + "test.method", + account_key.did_key(), + 1_001, + ) + .unwrap(); + assert_eq!(claims.iss, DID); + } + + #[test] + fn layout_drift_refuses_startup() { + let directory = tempfile::tempdir().unwrap(); + std::fs::create_dir(directory.path().join("unexpected-layout")).unwrap(); + assert!(matches!( + PdsSeam::open(directory.path()), + Err(HostError::Store(message)) if message.contains("layout") + )); + } + + #[derive(Default)] + struct RecordingAudit(Mutex>); + + impl SigningAudit for RecordingAudit { + fn commit_signed(&self, did: &str, space: &str, rev: &str) { + self.0 + .lock() + .unwrap() + .push((did.to_string(), space.to_string(), rev.to_string())); + } + } + + #[test] + fn signing_emits_the_stable_audit_event_payload() { + let directory = actor_store([9u8; 32]); + let audit = Arc::new(RecordingAudit::default()); + let seam = PdsSeam::with_audit(directory.path(), audit.clone()).unwrap(); + mint_commit(&seam, SPACE, DID, "3rev1", &[1u8; 32], [2u8; 32]).unwrap(); + assert_eq!( + *audit.0.lock().unwrap(), + vec![(DID.to_string(), SPACE.to_string(), "3rev1".to_string())] + ); + assert_eq!(COMMIT_SIGN_AUDIT_EVENT, "space_host_commit_signed"); + } + + #[test] + fn hs256_secret_exposes_verification_without_exposing_key_material() { + let secret = VerifyOnlyHs256Secret::new(b"verify-only".to_vec()).unwrap(); + let mut mac = as Mac>::new_from_slice(b"verify-only").unwrap(); + mac.update(b"header.payload"); + let signature = mac.finalize().into_bytes(); + assert!(secret.verify(b"header.payload", signature.as_slice())); + assert!(!secret.verify(b"other", signature.as_slice())); + assert_eq!(format!("{secret:?}"), "VerifyOnlyHs256Secret([REDACTED])"); + } +} From 0bdd4fe25526c987d7f9f8d27a54ec4149ad4227 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 11:47:05 -0400 Subject: [PATCH 05/56] feat(space-host): verify existing PDS write sessions --- rsky-space-host/src/lib.rs | 1 + rsky-space-host/src/oauth.rs | 995 +++++++++++++++++++++++++++++++++++ 2 files changed, 996 insertions(+) create mode 100644 rsky-space-host/src/oauth.rs diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index c58e7e22..0c70867d 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -29,6 +29,7 @@ pub mod keys; pub mod managing_app; pub mod membership; pub mod notify; +pub mod oauth; pub mod pds_seam; pub mod policy; pub mod registration; diff --git a/rsky-space-host/src/oauth.rs b/rsky-space-host/src/oauth.rs new file mode 100644 index 00000000..537800a8 --- /dev/null +++ b/rsky-space-host/src/oauth.rs @@ -0,0 +1,995 @@ +//! Write-path authentication: DPoP-bound OAuth access tokens (D24). +//! +//! Space writes arrive with the same access token the account's PDS issued for +//! ordinary XRPC, so this verifies a token it did not mint: +//! +//! - the authorization server's signature over the token, against its JWKS; +//! - `iss` is the trusted authorization server; +//! - `aud` is the PDS service DID — a `did:` string, never an origin URL; +//! - the header `typ` is `at+jwt` and `exp` has not passed; +//! - `cnf.jkt` matches the thumbprint of the presented DPoP proof's key; +//! - `sub` is the account the caller claims to be writing as; +//! - `client_id` is on the first-party allowlist. +//! +//! Tokens carry no `scope` claim, so no scope check is possible and +//! `client_id` stands in for one. A token revoked before it expires still +//! verifies until `exp`. + +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use rsky_space::jwk::{verify_es256, EcJwk, JwkSet}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use crate::attestation::{JtiStore, MetadataFetcher, MAX_IAT_SKEW_SECS}; +use crate::error::{HostError, Result}; +use crate::pds_seam::VerifyOnlyHs256Secret; + +pub const ACCESS_TOKEN_TYP: &str = "at+jwt"; +pub const DPOP_TYP: &str = "dpop+jwt"; +pub const SUPPORTED_ALG: &str = "ES256"; +pub const HS256: &str = "HS256"; +/// What an access token may be signed with. `HS256` is not a weakening: it is +/// what a standalone PDS actually issues (see `verify_as_signature`). +pub const SUPPORTED_TOKEN_ALGS: [&str; 2] = ["ES256", "HS256"]; +pub const SUPPORTED_PROOF_ALGS: [&str; 1] = ["ES256"]; + +/// How long a DPoP proof stays acceptable after its `iat`. +pub const MAX_DPOP_AGE_SECS: u64 = 300; + +/// The trust anchors the shim checks a token against. +#[derive(Debug, Clone)] +pub struct AuthConfig { + /// The authorization server's issuer identifier. + pub issuer: String, + /// The JWKS document the authorization server signs tokens with. + pub jwks_uri: String, + /// The PDS service DID tokens are audienced to (`did:web:…`). + pub audience: String, + /// First-party `client_id`s permitted to write into a space. + pub client_ids: Vec, + /// The authorization server's symmetric signing secret, present when the + /// PDS is its own authorization server (see `verify_as_signature`). + /// `None` selects the JWKS path. + pub hs256_secret: Option, +} + +impl AuthConfig { + pub fn validate(&self) -> std::result::Result<(), String> { + if !self.audience.starts_with("did:") { + return Err(format!( + "audience must be a service DID, got {}", + self.audience + )); + } + if self.client_ids.is_empty() { + return Err("client allowlist is empty".to_string()); + } + Ok(()) + } +} + +/// The parts of an inbound request the shim needs. +pub struct RequestAuth<'a> { + pub authorization: Option<&'a str>, + pub dpop: Option<&'a str>, + pub method: &'a str, + /// Absolute request URI as the client saw it, query and fragment stripped. + pub url: &'a str, +} + +/// A verified caller. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccessContext { + pub did: String, + pub client_id: String, + pub jkt: String, +} + +impl AccessContext { + /// A token authorizes writes only as its own subject, so a request naming + /// another author is rejected rather than silently rewritten. + pub fn require_author(&self, author_did: &str) -> Result<()> { + if self.did == author_did { + Ok(()) + } else { + Err(auth_err("token subject is not the named author")) + } + } +} + +#[derive(Debug, Deserialize)] +struct JwtHeader { + #[serde(default)] + typ: String, + #[serde(default)] + alg: String, + #[serde(default)] + kid: Option, + #[serde(default)] + jwk: Option, +} + +#[derive(Debug, Deserialize)] +struct AccessClaims { + iss: String, + aud: Audience, + sub: String, + exp: u64, + #[serde(default)] + client_id: Option, + #[serde(default)] + cnf: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum Audience { + One(String), + Many(Vec), +} + +impl Audience { + fn contains(&self, want: &str) -> bool { + match self { + Self::One(aud) => aud == want, + Self::Many(auds) => auds.iter().any(|a| a == want), + } + } +} + +#[derive(Debug, Deserialize)] +struct Confirmation { + #[serde(default)] + jkt: Option, +} + +#[derive(Debug, Deserialize)] +struct DpopClaims { + jti: String, + htm: String, + htu: String, + iat: u64, + #[serde(default)] + ath: Option, +} + +struct DecodedJwt { + header: JwtHeader, + claims: C, + signing_input: Vec, + signature: Vec, +} + +fn auth_err(msg: impl Into) -> HostError { + HostError::Delegation(msg.into()) +} + +fn decode_part Deserialize<'de>>(part: &str, what: &str) -> Result { + let bytes = URL_SAFE_NO_PAD + .decode(part) + .map_err(|e| auth_err(format!("{what} is not base64url: {e}")))?; + serde_json::from_slice(&bytes).map_err(|e| auth_err(format!("malformed {what}: {e}"))) +} + +fn decode_jwt Deserialize<'de>>(jwt: &str) -> Result> { + let parts: Vec<&str> = jwt.split('.').collect(); + if parts.len() != 3 { + return Err(auth_err("jwt must have three parts")); + } + Ok(DecodedJwt { + header: decode_part(parts[0], "jwt header")?, + claims: decode_part(parts[1], "jwt claims")?, + signing_input: format!("{}.{}", parts[0], parts[1]).into_bytes(), + signature: URL_SAFE_NO_PAD + .decode(parts[2]) + .map_err(|e| auth_err(format!("signature is not base64url: {e}")))?, + }) +} + +/// RFC 7638 JWK thumbprint: SHA-256 over the required members in +/// lexicographic order, base64url-encoded. +pub fn jwk_thumbprint(jwk: &EcJwk) -> String { + let canonical = format!( + r#"{{"crv":"{}","kty":"{}","x":"{}","y":"{}"}}"#, + jwk.crv, jwk.kty, jwk.x, jwk.y + ); + URL_SAFE_NO_PAD.encode(Sha256::digest(canonical.as_bytes())) +} + +fn access_token_hash(token: &str) -> String { + URL_SAFE_NO_PAD.encode(Sha256::digest(token.as_bytes())) +} + +fn split_scheme(header: &str) -> Result<(&str, &str)> { + header + .split_once(' ') + .map(|(scheme, value)| (scheme, value.trim())) + .ok_or_else(|| auth_err("malformed authorization header")) +} + +/// Strip query and fragment; `htu` compares only scheme, host and path. +fn htu_of(url: &str) -> &str { + let end = url.find(['?', '#']).unwrap_or(url.len()); + &url[..end] +} + +/// Verify a DPoP-bound access token on an inbound write and return the caller. +pub async fn verify_access( + request: &RequestAuth<'_>, + config: &AuthConfig, + fetcher: &dyn MetadataFetcher, + jti_store: &dyn JtiStore, + now: u64, +) -> Result { + let authorization = request + .authorization + .ok_or_else(|| auth_err("missing authorization header"))?; + let (scheme, token) = split_scheme(authorization)?; + if !scheme.eq_ignore_ascii_case("DPoP") { + return Err(auth_err(format!("unsupported auth scheme {scheme}"))); + } + let proof = request + .dpop + .ok_or_else(|| auth_err("missing DPoP proof header"))?; + + let jkt = verify_dpop_proof(proof, token, request, jti_store, now).await?; + let decoded: DecodedJwt = decode_jwt(token)?; + + if decoded.header.typ != ACCESS_TOKEN_TYP { + return Err(auth_err(format!( + "token typ {} != {ACCESS_TOKEN_TYP}", + decoded.header.typ + ))); + } + if !SUPPORTED_TOKEN_ALGS.contains(&decoded.header.alg.as_str()) { + return Err(auth_err(format!( + "unsupported token alg {}", + decoded.header.alg + ))); + } + let claims = &decoded.claims; + if claims.iss != config.issuer { + return Err(auth_err( + "token issuer is not the trusted authorization server", + )); + } + if !claims.aud.contains(&config.audience) { + return Err(auth_err("token audience is not this pds service did")); + } + if now >= claims.exp { + return Err(auth_err("token expired")); + } + let bound = claims + .cnf + .as_ref() + .and_then(|c| c.jkt.as_deref()) + .ok_or_else(|| auth_err("token is not dpop-bound"))?; + if bound != jkt { + return Err(auth_err("token is bound to a different key")); + } + let client_id = claims + .client_id + .clone() + .ok_or_else(|| auth_err("token carries no client_id"))?; + if !config.client_ids.iter().any(|c| *c == client_id) { + return Err(auth_err("client_id is not first-party")); + } + + verify_as_signature(&decoded, config, fetcher).await?; + + Ok(AccessContext { + did: claims.sub.clone(), + client_id, + jkt, + }) +} + +/// Verify the token's signature against the authorization server. +/// +/// **A PDS that is its own authorization server signs access tokens with +/// HS256**, using the same symmetric secret it uses for legacy session JWTs +/// (`pds/src/context.ts`, the `keyset` given to `OAuthProvider`). It therefore +/// publishes an *empty* JWKS: a symmetric key has no public half. The +/// asymmetric path applies only when an entryway issues the tokens instead. +/// +/// So a co-located verifier has two options, and only two: hold the same +/// secret, or ask the PDS. This holds the secret — the host already reads +/// every local account's signing key from the actor store, which is strictly +/// more sensitive, and asking would put a network call in every write. When +/// the secret is not configured the asymmetric path is used unchanged, so an +/// entryway deployment needs no different build. +async fn verify_as_signature( + decoded: &DecodedJwt, + config: &AuthConfig, + fetcher: &dyn MetadataFetcher, +) -> Result<()> { + if decoded.header.alg == HS256 { + let secret = config + .hs256_secret + .as_ref() + .ok_or_else(|| auth_err("token is HS256 but no shared secret is configured"))?; + return secret + .verify(&decoded.signing_input, &decoded.signature) + .then_some(()) + .ok_or_else(|| auth_err("token signature: hs256 verification failed")); + } + let jwks: JwkSet = fetcher.jwks(&config.jwks_uri).await?; + let jwk = match decoded.header.kid.as_deref() { + Some(kid) => jwks + .find(kid) + .ok_or_else(|| auth_err(format!("authorization server has no key {kid}")))?, + None => jwks + .keys + .first() + .ok_or_else(|| auth_err("authorization server jwks is empty"))?, + }; + verify_es256(jwk, &decoded.signing_input, &decoded.signature) + .map_err(|e| auth_err(format!("token signature: {e}"))) +} + +async fn verify_dpop_proof( + proof: &str, + token: &str, + request: &RequestAuth<'_>, + jti_store: &dyn JtiStore, + now: u64, +) -> Result { + let decoded: DecodedJwt = decode_jwt(proof)?; + if decoded.header.typ != DPOP_TYP { + return Err(auth_err(format!( + "proof typ {} != {DPOP_TYP}", + decoded.header.typ + ))); + } + if !SUPPORTED_PROOF_ALGS.contains(&decoded.header.alg.as_str()) { + return Err(auth_err(format!( + "unsupported proof alg {}", + decoded.header.alg + ))); + } + let jwk = decoded + .header + .jwk + .as_ref() + .ok_or_else(|| auth_err("proof carries no jwk"))?; + verify_es256(jwk, &decoded.signing_input, &decoded.signature) + .map_err(|e| auth_err(format!("proof signature: {e}")))?; + + let claims = &decoded.claims; + if !claims.htm.eq_ignore_ascii_case(request.method) { + return Err(auth_err("proof htm does not match the request method")); + } + if htu_of(&claims.htu) != htu_of(request.url) { + return Err(auth_err("proof htu does not match the request url")); + } + if claims.ath.as_deref() != Some(access_token_hash(token).as_str()) { + return Err(auth_err("proof ath does not match the access token")); + } + if claims.iat > now + MAX_IAT_SKEW_SECS { + return Err(auth_err("proof iat is in the future")); + } + let expires = claims.iat.saturating_add(MAX_DPOP_AGE_SECS); + if now >= expires { + return Err(auth_err("proof is too old")); + } + if !jti_store.consume(&claims.jti, expires).await? { + return Err(auth_err("proof jti replayed")); + } + Ok(jwk_thumbprint(jwk)) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::attestation::{ClientMetadata, InMemoryJtiStore}; + use async_trait::async_trait; + use p256::ecdsa::signature::hazmat::PrehashSigner; + use p256::ecdsa::{Signature, SigningKey}; + use serde_json::json; + + pub(crate) const NOW: u64 = 1_700_000_000; + pub(crate) const ISSUER: &str = "https://pds.example.com"; + pub(crate) const PDS_DID: &str = "did:web:pds.example.com"; + pub(crate) const CLIENT: &str = "https://blacksky.community/oauth-client-metadata.json"; + pub(crate) const AUTHOR: &str = "did:plc:member"; + pub(crate) const URL: &str = "https://pds.example.com/xrpc/com.atproto.space.createRecord"; + + fn as_key() -> SigningKey { + SigningKey::from_slice(&[0x31u8; 32]).unwrap() + } + + pub(crate) fn client_key() -> SigningKey { + SigningKey::from_slice(&[0x32u8; 32]).unwrap() + } + + pub(crate) fn jwk_of(key: &SigningKey, kid: Option<&str>) -> EcJwk { + let point = key.verifying_key().to_encoded_point(false); + let bytes = point.as_bytes(); + EcJwk { + kty: "EC".to_string(), + crv: "P-256".to_string(), + x: URL_SAFE_NO_PAD.encode(&bytes[1..33]), + y: URL_SAFE_NO_PAD.encode(&bytes[33..65]), + kid: kid.map(str::to_string), + } + } + + pub(crate) fn sign_jwt( + key: &SigningKey, + header: serde_json::Value, + claims: serde_json::Value, + ) -> String { + let header = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).unwrap()); + let claims = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let input = format!("{header}.{claims}"); + let digest = Sha256::digest(input.as_bytes()); + let sig: Signature = key.sign_prehash(&digest).unwrap(); + let sig = sig.normalize_s().unwrap_or(sig); + format!("{input}.{}", URL_SAFE_NO_PAD.encode(sig.to_vec())) + } + + pub(crate) struct AsJwks; + #[async_trait] + impl MetadataFetcher for AsJwks { + async fn client_metadata(&self, _client_id: &str) -> Result { + Err(auth_err("not used")) + } + async fn jwks(&self, _url: &str) -> Result { + Ok(JwkSet { + keys: vec![jwk_of(&as_key(), Some("as-key-1"))], + }) + } + } + + pub(crate) fn config() -> AuthConfig { + AuthConfig { + issuer: ISSUER.to_string(), + jwks_uri: format!("{ISSUER}/oauth/jwks"), + audience: PDS_DID.to_string(), + client_ids: vec![CLIENT.to_string()], + hs256_secret: None, + } + } + + /// The claim set of a real stateful-mode token: no `scope`, DID `aud`. + pub(crate) fn token_claims() -> serde_json::Value { + json!({ + "iss": ISSUER, + "aud": PDS_DID, + "sub": AUTHOR, + "exp": NOW + 3600, + "iat": NOW, + "jti": "tok-1", + "client_id": CLIENT, + "cnf": {"jkt": jwk_thumbprint(&jwk_of(&client_key(), None))}, + }) + } + + pub(crate) fn token_with(claims: serde_json::Value) -> String { + sign_jwt( + &as_key(), + json!({"typ": ACCESS_TOKEN_TYP, "alg": "ES256", "kid": "as-key-1"}), + claims, + ) + } + + pub(crate) fn token() -> String { + token_with(token_claims()) + } + + pub(crate) fn proof_claims(token: &str) -> serde_json::Value { + json!({ + "jti": "proof-1", + "htm": "POST", + "htu": URL, + "iat": NOW, + "ath": access_token_hash(token), + }) + } + + pub(crate) fn proof_with(claims: serde_json::Value) -> String { + sign_jwt( + &client_key(), + json!({ + "typ": DPOP_TYP, + "alg": "ES256", + "jwk": jwk_of(&client_key(), None), + }), + claims, + ) + } + + async fn check_at( + token: &str, + proof: &str, + method: &str, + url: &str, + now: u64, + jti: &InMemoryJtiStore, + ) -> Result { + let authorization = format!("DPoP {token}"); + verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(proof), + method, + url, + }, + &config(), + &AsJwks, + jti, + now, + ) + .await + } + + async fn check(token: &str, proof: &str) -> Result { + check_at(token, proof, "POST", URL, NOW, &InMemoryJtiStore::default()).await + } + + const HS_SECRET: &[u8] = b"a-pds-jwt-secret"; + + fn hs256_token(claims: serde_json::Value) -> String { + use hmac::{Hmac, Mac}; + let header = json!({"typ": ACCESS_TOKEN_TYP, "alg": "HS256"}); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).unwrap()), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()) + ); + let mut mac = as Mac>::new_from_slice(HS_SECRET).unwrap(); + mac.update(input.as_bytes()); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) + } + + fn hs_config() -> AuthConfig { + AuthConfig { + hs256_secret: VerifyOnlyHs256Secret::new(HS_SECRET.to_vec()), + ..config() + } + } + + /// A PDS that is its own authorization server signs with HS256 and + /// publishes no public key, so this is the only path that can verify a + /// real token from one. + #[tokio::test] + async fn accepts_an_hs256_token_from_a_standalone_pds() { + let token = hs256_token(token_claims()); + let proof = proof_with(proof_claims(&token)); + let authorization = format!("DPoP {token}"); + let context = verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &hs_config(), + &AsJwks, + &InMemoryJtiStore::default(), + NOW, + ) + .await + .expect("hs256 token rejected"); + assert_eq!(context.did, AUTHOR); + } + + #[tokio::test] + async fn an_hs256_token_is_refused_without_the_secret_or_with_the_wrong_one() { + let token = hs256_token(token_claims()); + let proof = proof_with(proof_claims(&token)); + let authorization = format!("DPoP {token}"); + let attempt = |config: AuthConfig| { + let authorization = authorization.clone(); + let proof = proof.clone(); + async move { + verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config, + &AsJwks, + &InMemoryJtiStore::default(), + NOW, + ) + .await + } + }; + // Unconfigured: an HS256 token must not fall through to the JWKS path. + assert!(attempt(config()).await.is_err()); + assert!(attempt(AuthConfig { + hs256_secret: VerifyOnlyHs256Secret::new(b"not-the-secret".to_vec()), + ..config() + }) + .await + .is_err()); + } + + #[tokio::test] + async fn a_tampered_hs256_token_is_refused() { + let mut claims = token_claims(); + claims["sub"] = json!("did:plc:someone-else"); + let token = hs256_token(token_claims()); + let forged = format!( + "{}.{}.{}", + token.split('.').next().unwrap(), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()), + token.split('.').nth(2).unwrap() + ); + let proof = proof_with(proof_claims(&forged)); + let authorization = format!("DPoP {forged}"); + assert!(verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &hs_config(), + &AsJwks, + &InMemoryJtiStore::default(), + NOW, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn accepts_a_real_shaped_token() { + let token = token(); + let context = check(&token, &proof_with(proof_claims(&token))) + .await + .unwrap(); + assert_eq!(context.did, AUTHOR); + assert_eq!(context.client_id, CLIENT); + assert_eq!(context.jkt, jwk_thumbprint(&jwk_of(&client_key(), None))); + } + + #[tokio::test] + async fn accepts_a_token_with_no_scope_claim() { + // Stateful-mode tokens omit `scope` entirely; that must not be fatal. + let claims = token_claims(); + assert!(claims.get("scope").is_none()); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_ok()); + } + + #[tokio::test] + async fn rejects_an_origin_url_audience() { + // The regression this suite exists for: `aud` is the service DID, and a + // vector built from the origin would pass a lenient check but 401 in + // production. + let mut claims = token_claims(); + claims["aud"] = json!(ISSUER); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn accepts_a_did_audience_in_a_list() { + let mut claims = token_claims(); + claims["aud"] = json!(["did:web:other.example", PDS_DID]); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_ok()); + } + + #[tokio::test] + async fn rejects_bad_token_claims() { + for (name, mutate) in [ + ("wrong iss", json!("https://evil.example")), + ("wrong aud", json!("did:web:other.example")), + ] { + let mut claims = token_claims(); + let field = if name == "wrong iss" { "iss" } else { "aud" }; + claims[field] = mutate; + let token = token_with(claims); + assert!( + check(&token, &proof_with(proof_claims(&token))) + .await + .is_err(), + "{name} was accepted" + ); + } + + // Expired. + let mut claims = token_claims(); + claims["exp"] = json!(NOW - 1); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // Not dpop-bound. + let mut claims = token_claims(); + claims["cnf"] = json!({}); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // Bound to another key. + let mut claims = token_claims(); + claims["cnf"] = json!({"jkt": "someone-elses-thumbprint"}); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // No client_id. + let mut claims = token_claims(); + claims.as_object_mut().unwrap().remove("client_id"); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // Non-allowlisted client_id. + let mut claims = token_claims(); + claims["client_id"] = json!("https://third-party.example/client-metadata.json"); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_a_wrong_token_typ() { + let token = sign_jwt( + &as_key(), + json!({"typ": "JWT", "alg": "ES256", "kid": "as-key-1"}), + token_claims(), + ); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_a_token_signed_by_another_key() { + let token = sign_jwt( + &client_key(), + json!({"typ": ACCESS_TOKEN_TYP, "alg": "ES256", "kid": "as-key-1"}), + token_claims(), + ); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_a_tampered_token_signature() { + let token = token(); + let mut tampered: Vec<&str> = token.split('.').collect(); + let flipped = format!("{}A", &tampered[2][..tampered[2].len() - 1]); + tampered[2] = &flipped; + let tampered = tampered.join("."); + assert!(check(&tampered, &proof_with(proof_claims(&tampered))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_an_unknown_signing_key_id() { + let token = sign_jwt( + &as_key(), + json!({"typ": ACCESS_TOKEN_TYP, "alg": "ES256", "kid": "rotated-away"}), + token_claims(), + ); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_bad_dpop_proofs() { + let token = token(); + + // Wrong htu. + let mut claims = proof_claims(&token); + claims["htu"] = json!("https://pds.example.com/xrpc/com.atproto.space.deleteRecord"); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Wrong htm. + let mut claims = proof_claims(&token); + claims["htm"] = json!("GET"); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // ath bound to a different token. + let mut claims = proof_claims(&token); + claims["ath"] = json!(access_token_hash("some.other.token")); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Stale proof. + let mut claims = proof_claims(&token); + claims["iat"] = json!(NOW - MAX_DPOP_AGE_SECS - 1); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Proof from the future. + let mut claims = proof_claims(&token); + claims["iat"] = json!(NOW + MAX_IAT_SKEW_SECS + 10); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Proof signed by a key other than the one it embeds. + let forged = sign_jwt( + &as_key(), + json!({"typ": DPOP_TYP, "alg": "ES256", "jwk": jwk_of(&client_key(), None)}), + proof_claims(&token), + ); + assert!(check(&token, &forged).await.is_err()); + + // Proof with no embedded key. + let no_jwk = sign_jwt( + &client_key(), + json!({"typ": DPOP_TYP, "alg": "ES256"}), + proof_claims(&token), + ); + assert!(check(&token, &no_jwk).await.is_err()); + + // Wrong proof typ. + let wrong_typ = sign_jwt( + &client_key(), + json!({"typ": "JWT", "alg": "ES256", "jwk": jwk_of(&client_key(), None)}), + proof_claims(&token), + ); + assert!(check(&token, &wrong_typ).await.is_err()); + } + + #[tokio::test] + async fn rejects_a_replayed_proof() { + let jti = InMemoryJtiStore::default(); + let token = token(); + let proof = proof_with(proof_claims(&token)); + assert!(check_at(&token, &proof, "POST", URL, NOW, &jti) + .await + .is_ok()); + assert!(check_at(&token, &proof, "POST", URL, NOW, &jti) + .await + .is_err()); + } + + #[tokio::test] + async fn htu_ignores_query_and_fragment() { + let token = token(); + let mut claims = proof_claims(&token); + claims["htu"] = json!(format!("{URL}?x=1")); + let jti = InMemoryJtiStore::default(); + assert!(check_at( + &token, + &proof_with(claims), + "POST", + &format!("{URL}#frag"), + NOW, + &jti + ) + .await + .is_ok()); + } + + #[tokio::test] + async fn rejects_malformed_and_missing_headers() { + let token = token(); + let proof = proof_with(proof_claims(&token)); + let jti = InMemoryJtiStore::default(); + + // Bearer instead of DPoP. + let bearer = format!("Bearer {token}"); + assert!(verify_access( + &RequestAuth { + authorization: Some(&bearer), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // No authorization header. + assert!(verify_access( + &RequestAuth { + authorization: None, + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // No DPoP proof header. + let authorization = format!("DPoP {token}"); + assert!(verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: None, + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // Header with no scheme separator. + assert!(verify_access( + &RequestAuth { + authorization: Some("DPoPnospace"), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // Structurally broken JWTs. + assert!(check("not.a.jwt", &proof).await.is_err()); + assert!(check(&token, "only.two").await.is_err()); + } + + #[tokio::test] + async fn a_token_may_only_write_as_its_own_subject() { + let token = token(); + let context = check(&token, &proof_with(proof_claims(&token))) + .await + .unwrap(); + assert!(context.require_author(AUTHOR).is_ok()); + assert!(context.require_author("did:plc:someoneelse").is_err()); + } + + #[test] + fn config_requires_a_did_audience_and_a_client_allowlist() { + assert!(config().validate().is_ok()); + + let mut origin_aud = config(); + origin_aud.audience = ISSUER.to_string(); + assert!(origin_aud.validate().is_err()); + + let mut no_clients = config(); + no_clients.client_ids.clear(); + assert!(no_clients.validate().is_err()); + } + + #[test] + fn thumbprint_uses_the_canonical_member_order() { + let jwk = EcJwk { + kty: "EC".to_string(), + crv: "P-256".to_string(), + x: "eA".to_string(), + y: "eQ".to_string(), + kid: Some("ignored".to_string()), + }; + let expected = URL_SAFE_NO_PAD.encode(Sha256::digest( + r#"{"crv":"P-256","kty":"EC","x":"eA","y":"eQ"}"#, + )); + assert_eq!(jwk_thumbprint(&jwk), expected); + } +} From 230ffdb17d0deaf132a9c3e402479c3b5b20b27c Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 11:50:13 -0400 Subject: [PATCH 06/56] fix(daemon): use repo parameters for space reads --- rsky-daemon/src/repohost.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rsky-daemon/src/repohost.rs b/rsky-daemon/src/repohost.rs index 55180ca1..f7271dba 100644 --- a/rsky-daemon/src/repohost.rs +++ b/rsky-daemon/src/repohost.rs @@ -122,7 +122,7 @@ impl RepoHostClient for HttpRepoHost { since: Option<&str>, cursor: Option<&str>, ) -> Result { - let mut query = vec![("space", space), ("did", did)]; + let mut query = vec![("space", space), ("repo", did)]; if let Some(since) = since { query.push(("since", since)); } @@ -146,7 +146,7 @@ impl RepoHostClient for HttpRepoHost { let resp = self .get( "com.atproto.space.getRepo", - &[("space", space), ("did", did)], + &[("space", space), ("repo", did)], ) .await?; Ok(resp.bytes().await.map_err(net_err)?.to_vec()) @@ -156,7 +156,7 @@ impl RepoHostClient for HttpRepoHost { let out: wire::GetLatestCommitOutput = self .get( "com.atproto.space.getLatestCommit", - &[("space", space), ("did", did)], + &[("space", space), ("repo", did)], ) .await? .json() @@ -208,7 +208,7 @@ mod tests { Mock::given(method("GET")) .and(path("/xrpc/com.atproto.space.listRepoOps")) .and(query_param("space", SPACE)) - .and(query_param("did", AUTHOR)) + .and(query_param("repo", AUTHOR)) .and(query_param("since", "3ka")) .and(query_param("cursor", "c1")) .and(header("authorization", "DPoP sc.jwt")) @@ -311,7 +311,7 @@ mod tests { Mock::given(method("GET")) .and(path("/xrpc/com.atproto.space.getRepo")) .and(query_param("space", SPACE)) - .and(query_param("did", AUTHOR)) + .and(query_param("repo", AUTHOR)) .and(header("authorization", "DPoP sc.jwt")) .and(header_exists("dpop")) .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![0xCAu8, 0x11])) @@ -329,7 +329,7 @@ mod tests { Mock::given(method("GET")) .and(path("/xrpc/com.atproto.space.getLatestCommit")) .and(query_param("space", SPACE)) - .and(query_param("did", AUTHOR)) + .and(query_param("repo", AUTHOR)) .and(header("authorization", "DPoP sc.jwt")) .and(header_exists("dpop")) .respond_with( From 534261842a4448f62500430b4138b10f1cfffc09 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 14:33:03 -0400 Subject: [PATCH 07/56] feat(space-host): serve direct space writes --- rsky-space-host/src/config.rs | 60 ++++++++ rsky-space-host/src/http.rs | 274 +++++++++++++++++++++++++++++++++- rsky-space-host/src/main.rs | 9 ++ rsky-space-host/src/oauth.rs | 4 +- 4 files changed, 344 insertions(+), 3 deletions(-) diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index bd2701a5..3e5dbb26 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -1,5 +1,7 @@ //! Configuration for the space-host service (env prefix `SPACEHOST_`). +use crate::oauth::AuthConfig; +use crate::pds_seam::VerifyOnlyHs256Secret; use clap::Parser; /// The Blacksky community space (v1: a single typed space under the authority). @@ -81,6 +83,24 @@ pub struct Config { default_value = "http://localhost:3600" )] pub public_url: String, + + #[arg(long, env = "SPACEHOST_OAUTH_ISSUER", default_value = "")] + pub oauth_issuer: String, + #[arg(long, env = "SPACEHOST_OAUTH_JWKS_URI", default_value = "")] + pub oauth_jwks_uri: String, + #[arg(long, env = "SPACEHOST_OAUTH_AUDIENCE", default_value = "")] + pub oauth_audience: String, + #[arg(long, env = "SPACEHOST_OAUTH_CLIENT_IDS", default_value = "")] + pub oauth_client_ids: String, + #[arg( + long, + env = "SPACEHOST_OAUTH_HS256_SECRET", + default_value = "", + hide_env_values = true + )] + pub oauth_hs256_secret: String, + #[arg(long, env = "SPACEHOST_ACTOR_STORE_DIR", default_value = "")] + pub actor_store_dir: String, } impl Config { @@ -100,6 +120,22 @@ impl Config { .collect() } + pub fn auth_config(&self) -> AuthConfig { + AuthConfig { + issuer: self.oauth_issuer.clone(), + jwks_uri: self.oauth_jwks_uri.clone(), + audience: self.oauth_audience.clone(), + client_ids: self + .oauth_client_ids + .split(',') + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(str::to_string) + .collect(), + hs256_secret: VerifyOnlyHs256Secret::new(self.oauth_hs256_secret.as_bytes().to_vec()), + } + } + pub fn validate(&self) -> Result<(), String> { if self.policy == PolicyMode::ManagingApp && !self.managing_app.contains('#') { return Err( @@ -117,6 +153,10 @@ impl Config { if self.public_url.trim_end_matches('/').is_empty() { return Err("SPACEHOST_PUBLIC_URL must be an absolute origin".to_string()); } + self.auth_config().validate()?; + if self.actor_store_dir.is_empty() { + return Err("SPACEHOST_ACTOR_STORE_DIR is required".to_string()); + } Ok(()) } } @@ -137,6 +177,16 @@ mod tests { "did:plc:authority", "--signing-key-hex", "aa".repeat(32).as_str(), + "--oauth-issuer", + "https://pds.example", + "--oauth-jwks-uri", + "https://pds.example/jwks", + "--oauth-audience", + "did:web:pds.example", + "--oauth-client-ids", + "https://client.example", + "--actor-store-dir", + "/actors", ]) .unwrap(); assert_eq!(cfg.authority_did, "did:plc:authority"); @@ -178,6 +228,11 @@ mod tests { std::env::set_var("SPACEHOST_DB_PATH", "/tmp/space.db"); std::env::set_var("SPACEHOST_PLC_URL", "https://plc.example"); std::env::set_var("SPACEHOST_BIND", "127.0.0.1:1234"); + std::env::set_var("SPACEHOST_OAUTH_ISSUER", "https://pds.example"); + std::env::set_var("SPACEHOST_OAUTH_JWKS_URI", "https://pds.example/jwks"); + std::env::set_var("SPACEHOST_OAUTH_AUDIENCE", "did:web:pds.example"); + std::env::set_var("SPACEHOST_OAUTH_CLIENT_IDS", "https://client.example"); + std::env::set_var("SPACEHOST_ACTOR_STORE_DIR", "/actors"); let cfg = Config::try_parse_from(["rsky-space-host"]).unwrap(); for k in [ "SPACEHOST_AUTHORITY_DID", @@ -191,6 +246,11 @@ mod tests { "SPACEHOST_DB_PATH", "SPACEHOST_PLC_URL", "SPACEHOST_BIND", + "SPACEHOST_OAUTH_ISSUER", + "SPACEHOST_OAUTH_JWKS_URI", + "SPACEHOST_OAUTH_AUDIENCE", + "SPACEHOST_OAUTH_CLIENT_IDS", + "SPACEHOST_ACTOR_STORE_DIR", ] { std::env::remove_var(k); } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 49b0a94d..21381b9c 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -15,16 +15,20 @@ use rsky_lexicon::com::atproto::space::{ }; use rsky_oauth::dpop::{DpopManager, DpopProof, DpopRequest}; use rsky_space::credential; +use serde_json::Value; use std::sync::Arc; use crate::attestation::{JtiStore, MetadataFetcher}; use crate::authority::{Authority, KeyResolver}; +use crate::commits::CommitSigner; use crate::error::HostError; use crate::keys::DocSource; use crate::managing_app::require_https; use crate::notify::{fan_out_write, Notifier, NOTIFY_WRITE_LXM}; +use crate::oauth::{verify_access, AuthConfig, RequestAuth}; use crate::policy::Policy; use crate::registration::{LifecycleAcker, REGISTER_SPACE_LXM}; +use crate::repo::{RepoStore, RepoWrite, WriteOutcome, MAX_RECORD_BYTES}; use crate::service_jwt; use crate::store::{RegistrationStore, Subscriber, WriterSetStore}; @@ -54,6 +58,10 @@ pub struct AppState { pub now: Arc u64 + Send + Sync>, pub jti: Arc String + Send + Sync>, pub registration_ttl_secs: u64, + pub repos: Arc, + pub commit_signer: Arc, + pub auth: AuthConfig, + pub rev: Arc String + Send + Sync>, } pub fn router(state: AppState) -> Router { @@ -74,6 +82,8 @@ pub fn router(state: AppState) -> Router { "/xrpc/community.blacksky.space.register", post(register_space), ) + .route("/xrpc/com.atproto.space.createRecord", post(create_record)) + .route("/xrpc/com.atproto.space.deleteRecord", post(delete_record)) .with_state(state) } @@ -484,6 +494,162 @@ async fn register_space( Ok(Json(serde_json::json!({}))) } +async fn write_actor( + state: &AppState, + headers: &HeaderMap, + path: &str, + space: &str, + repo: &str, +) -> Result { + let space = require_this_space(state, space)?; + let url = format!("{}{}", state.public_url.trim_end_matches('/'), path); + let access = verify_access( + &RequestAuth { + authorization: headers.get("authorization").and_then(|v| v.to_str().ok()), + dpop: headers.get("dpop").and_then(|v| v.to_str().ok()), + method: "POST", + url: &url, + }, + &state.auth, + state.metadata.as_ref(), + state.jti_store.as_ref(), + (state.now)(), + ) + .await?; + if access.did != repo { + return Err(ApiError::auth_required( + "session subject does not match repo", + )); + } + Ok(space) +} + +async fn create_record( + State(state): State, + headers: HeaderMap, + Json(input): Json, +) -> Result, ApiError> { + let space = write_actor( + &state, + &headers, + "/xrpc/com.atproto.space.createRecord", + &input.space, + &input.repo, + ) + .await?; + if contains_blob_ref(&input.record) { + return Err(ApiError::invalid_request( + "blob references are not supported", + )); + } + let value = rsky_space::record::encode_record(&input.record, MAX_RECORD_BYTES) + .map_err(HostError::from)?; + let rev = (state.rev)(); + let rkey = input.rkey.unwrap_or_else(|| rev.clone()); + let applied = state + .repos + .apply_writes( + &space.uri(), + &input.repo, + &rev, + &[RepoWrite::Create { + collection: input.collection.clone(), + rkey: rkey.clone(), + value, + }], + ) + .await?; + let cid = match &applied.outcomes[0] { + WriteOutcome::Created { cid } => cid.clone(), + _ => return Err(HostError::Store("create did not create".into()).into()), + }; + record_write(&state, &space.uri(), &input.repo, &applied.rev).await?; + Ok(Json( + rsky_lexicon::com::atproto::space::CreateRecordOutput { + uri: space.record_uri(&input.repo, &input.collection, &rkey), + cid, + commit: Some(rsky_lexicon::com::atproto::space::CommitMeta { + rev: applied.rev, + hash: hex::encode(applied.hash), + }), + }, + )) +} + +fn contains_blob_ref(value: &Value) -> bool { + match value { + Value::Array(values) => values.iter().any(contains_blob_ref), + Value::Object(values) => { + values.get("$type").and_then(Value::as_str) == Some("blob") + || values.values().any(contains_blob_ref) + } + _ => false, + } +} + +async fn delete_record( + State(state): State, + headers: HeaderMap, + Json(input): Json, +) -> Result, ApiError> { + let space = write_actor( + &state, + &headers, + "/xrpc/com.atproto.space.deleteRecord", + &input.space, + &input.repo, + ) + .await?; + let applied = state + .repos + .apply_writes( + &space.uri(), + &input.repo, + &(state.rev)(), + &[RepoWrite::Delete { + collection: input.collection, + rkey: input.rkey, + swap_record: input.swap_record, + }], + ) + .await?; + if !matches!(applied.outcomes[0], WriteOutcome::Noop) { + record_write(&state, &space.uri(), &input.repo, &applied.rev).await?; + } + Ok(Json( + rsky_lexicon::com::atproto::space::DeleteRecordOutput { + commit: Some(rsky_lexicon::com::atproto::space::CommitMeta { + rev: applied.rev, + hash: hex::encode(applied.hash), + }), + }, + )) +} + +async fn record_write( + state: &AppState, + space: &str, + repo: &str, + rev: &str, +) -> Result<(), ApiError> { + let now = (state.now)(); + state + .writers + .upsert_writer(space, repo, rev, None, now) + .await?; + let endpoints = state.registrations.endpoints(space, now).await?; + fan_out_write( + state.notifier.clone(), + endpoints, + NotifyWriteInput { + space: space.to_string(), + repo: repo.to_string(), + rev: rev.to_string(), + }, + ); + Ok(()) +} + async fn notify_write( State(state): State, headers: HeaderMap, @@ -671,7 +837,7 @@ mod tests { members.iter().map(|m| m.to_string()), )))), keys: Arc::new(UserKeys), - metadata: Arc::new(NoFetch), + metadata: Arc::new(crate::oauth::tests::AsJwks), jti_store: Arc::new(InMemoryJtiStore::default()), writers: Arc::new(InMemoryWriterSet::default()), registrations: Arc::new(InMemoryRegistrations::default()), @@ -686,6 +852,10 @@ mod tests { now: Arc::new(|| NOW), jti: Arc::new(|| "jti-fixed".to_string()), registration_ttl_secs: DEFAULT_REGISTRATION_TTL_SECS, + repos: Arc::new(crate::repo::InMemoryRepos::default()), + commit_signer: Arc::new(test_signer()), + auth: crate::oauth::tests::config(), + rev: Arc::new(|| "3jzfcijpj2z2c".to_string()), }; Fixture { state, writes } } @@ -816,6 +986,108 @@ mod tests { .unwrap() } + fn pds_write_req(path: &str, body: serde_json::Value) -> Request { + let token = crate::oauth::tests::token(); + let mut claims = crate::oauth::tests::proof_claims(&token); + claims["htu"] = serde_json::json!(format!("{PUBLIC_URL}{path}")); + claims["jti"] = serde_json::json!(format!( + "write-proof-{}", + PROOF_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + )); + Request::builder() + .method("POST") + .uri(path) + .header("content-type", "application/json") + .header("authorization", format!("DPoP {token}")) + .header("dpop", crate::oauth::tests::proof_with(claims)) + .body(Body::from(body.to_string())) + .unwrap() + } + + #[tokio::test] + async fn create_and_delete_records_verify_the_pds_session() { + let f = fixture(AppAccess::Open, &[]); + let mut state = f.state.clone(); + state.now = Arc::new(|| crate::oauth::tests::NOW); + let create_path = "/xrpc/com.atproto.space.createRecord"; + let create = serde_json::json!({ + "space": space_uri(), + "repo": MEMBER, + "collection": "app.bsky.feed.post", + "rkey": "3jzfcijpj2z2c", + "record": {"$type": "app.bsky.feed.post", "text": "private", "createdAt": "2026-08-19T00:00:00Z"}, + }); + let (status, out) = send(&state, pds_write_req(create_path, create)).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + out["uri"], + format!( + "{}/did:plc:member/app.bsky.feed.post/3jzfcijpj2z2c", + space_uri() + ) + ); + assert!(out["cid"].as_str().is_some()); + assert_eq!(out["commit"]["rev"], "3jzfcijpj2z2c"); + + let stored = f + .state + .repos + .get_record(&space_uri(), MEMBER, "app.bsky.feed.post", "3jzfcijpj2z2c") + .await + .unwrap(); + assert!(stored.is_some()); + + let delete = serde_json::json!({ + "space": space_uri(), + "repo": MEMBER, + "collection": "app.bsky.feed.post", + "rkey": "3jzfcijpj2z2c", + }); + let (status, out) = send( + &state, + pds_write_req("/xrpc/com.atproto.space.deleteRecord", delete), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(out["commit"]["rev"], "3jzfcijpj2z2c"); + assert!(f + .state + .repos + .get_record(&space_uri(), MEMBER, "app.bsky.feed.post", "3jzfcijpj2z2c") + .await + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn create_record_rejects_blobs_and_subject_mismatch() { + let f = fixture(AppAccess::Open, &[]); + let mut state = f.state.clone(); + state.now = Arc::new(|| crate::oauth::tests::NOW); + let blob = serde_json::json!({ + "space": space_uri(), "repo": MEMBER, "collection": "app.bsky.feed.post", "rkey": "x", + "record": {"$type": "app.bsky.feed.post", "embed": {"image": {"$type": "blob", "ref": {"$link": "bafk"}}}}, + }); + let (status, out) = send( + &state, + pds_write_req("/xrpc/com.atproto.space.createRecord", blob), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(out["error"], "InvalidRequest"); + + let mismatch = serde_json::json!({ + "space": space_uri(), "repo": "did:plc:other", "collection": "app.bsky.feed.post", "rkey": "x", "record": {"$type": "app.bsky.feed.post"}, + }); + let (status, out) = send( + &state, + pds_write_req("/xrpc/com.atproto.space.createRecord", mismatch), + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(out["error"], "AuthenticationRequired"); + } + #[tokio::test] async fn health_reports_version() { let f = fixture(AppAccess::Open, &[]); diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index 723c5546..dfff12e5 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -15,8 +15,10 @@ use rsky_space_host::keys::{DocKeyResolver, ResolverDocSource}; use rsky_space_host::managing_app::HttpManagingApp; use rsky_space_host::membership::InMemoryMembership; use rsky_space_host::notify::HttpNotifier; +use rsky_space_host::pds_seam::PdsSeam; use rsky_space_host::policy::Policy; use rsky_space_host::registration::HttpLifecycleAcker; +use rsky_space_host::repo::SqliteRepos; use rsky_space_host::signing::Signer; use rsky_space_host::store::SqliteStore; use std::sync::Arc; @@ -81,6 +83,9 @@ async fn main() -> Result<(), Box> { }, }; let store = Arc::new(SqliteStore::open(&cfg.db_path)?); + let repos = Arc::new(SqliteRepos::open(&cfg.db_path)?); + let commit_signer = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); + let ticker = std::sync::Mutex::new(rsky_common::tid::Ticker::new()); let state = AppState { authority: Arc::new(authority), policy: Arc::new(policy), @@ -114,6 +119,10 @@ async fn main() -> Result<(), Box> { now, jti, registration_ttl_secs: DEFAULT_REGISTRATION_TTL_SECS, + repos, + commit_signer, + auth: cfg.auth_config(), + rev: Arc::new(move || ticker.lock().expect("ticker").next(None).to_string()), }; let listener = tokio::net::TcpListener::bind(&cfg.bind).await?; diff --git a/rsky-space-host/src/oauth.rs b/rsky-space-host/src/oauth.rs index 537800a8..ed846e35 100644 --- a/rsky-space-host/src/oauth.rs +++ b/rsky-space-host/src/oauth.rs @@ -325,7 +325,7 @@ async fn verify_as_signature( .ok_or_else(|| auth_err("authorization server jwks is empty"))?, }; verify_es256(jwk, &decoded.signing_input, &decoded.signature) - .map_err(|e| auth_err(format!("token signature: {e}"))) + .map_err(|e| auth_err(format!("token signature: {e}"))) } async fn verify_dpop_proof( @@ -354,7 +354,7 @@ async fn verify_dpop_proof( .as_ref() .ok_or_else(|| auth_err("proof carries no jwk"))?; verify_es256(jwk, &decoded.signing_input, &decoded.signature) - .map_err(|e| auth_err(format!("proof signature: {e}")))?; + .map_err(|e| auth_err(format!("proof signature: {e}")))?; let claims = &decoded.claims; if !claims.htm.eq_ignore_ascii_case(request.method) { From 9abde1d9f2cf9539d7c10b7a20b2e50ebe5511d0 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 14:34:46 -0400 Subject: [PATCH 08/56] feat(space-host): add bound service credential minting --- rsky-daemon/src/config.rs | 10 +++++ rsky-daemon/src/dpop.rs | 25 ++++++++++++ rsky-daemon/src/main.rs | 6 ++- rsky-space-host/src/config.rs | 29 ++++++++++++++ rsky-space-host/src/http.rs | 72 +++++++++++++++++++++++++++++++++++ rsky-space-host/src/main.rs | 5 +++ 6 files changed, 146 insertions(+), 1 deletion(-) diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index f30193b3..e1560765 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -48,6 +48,16 @@ pub struct Config { )] pub static_credential: String, + #[arg( + long, + env = "DAEMON_SPACE_HOST_MINT_TOKEN", + default_value = "", + hide_env_values = true + )] + pub space_host_mint_token: String, + #[arg(long, env = "DAEMON_DPOP_KEY_PATH", default_value = "")] + pub dpop_key_path: String, + /// Bind address for the notify listener. #[arg(long, env = "DAEMON_NOTIFY_BIND", default_value = "127.0.0.1:8055")] pub notify_bind: String, diff --git a/rsky-daemon/src/dpop.rs b/rsky-daemon/src/dpop.rs index c649bff9..792d07aa 100644 --- a/rsky-daemon/src/dpop.rs +++ b/rsky-daemon/src/dpop.rs @@ -16,6 +16,7 @@ use base64::Engine as _; use rsky_oauth::jwk::{EcCurve, Jwk}; use rsky_oauth::jwt::{sign, JwtClaims, JwtHeader}; use sha2::{Digest, Sha256}; +use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -47,6 +48,30 @@ impl DpopSigner { )) } + /// Load the daemon's stable proof key, creating it once when absent. + pub fn load_or_generate(path: impl AsRef) -> Result { + let path = path.as_ref(); + if path.exists() { + let bytes = std::fs::read(path).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + return Jwk::from_private_key_bytes(EcCurve::P256, &bytes) + .map(|key| Self { + key, + counter: AtomicU64::new(0), + }) + .map_err(|e| DaemonError::Xrpc(e.to_string())); + } + let signer = Self::generate()?; + let bytes = signer + .key + .private_key_bytes() + .map_err(|e| DaemonError::Xrpc(e.to_string()))?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + } + std::fs::write(path, bytes).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + Ok(signer) + } + /// RFC 7638 thumbprint — what a credential minted for this signer carries /// in `cnf.jkt`. pub fn thumbprint(&self) -> String { diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 70107da6..aa9b3c81 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -75,7 +75,11 @@ async fn main() -> std::result::Result<(), Box> { let space = SpaceId::parse(&cfg.space_uri)?; // One proof-of-possession key for the process: the credential it mints is // bound to it, and every host it is presented to checks that binding. - let dpop = Arc::new(rsky_daemon::dpop::DpopSigner::generate()?); + let dpop = Arc::new(if cfg.dpop_key_path.is_empty() { + rsky_daemon::dpop::DpopSigner::generate()? + } else { + rsky_daemon::dpop::DpopSigner::load_or_generate(&cfg.dpop_key_path)? + }); let host = Arc::new(HttpSpaceHost::new(&cfg.space_host_url, dpop.clone())); let keys: Arc = Arc::new(DidKeyResolver::new()); diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index 3e5dbb26..b6ba4276 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -101,6 +101,17 @@ pub struct Config { pub oauth_hs256_secret: String, #[arg(long, env = "SPACEHOST_ACTOR_STORE_DIR", default_value = "")] pub actor_store_dir: String, + #[arg( + long, + env = "SPACEHOST_MINT_TOKEN", + default_value = "", + hide_env_values = true + )] + pub mint_token: String, + #[arg(long, env = "SPACEHOST_DAEMON_SERVICE_DID", default_value = "")] + pub daemon_service_did: String, + #[arg(long, env = "SPACEHOST_APPVIEW_SERVICE_DID", default_value = "")] + pub appview_service_did: String, } impl Config { @@ -157,6 +168,12 @@ impl Config { if self.actor_store_dir.is_empty() { return Err("SPACEHOST_ACTOR_STORE_DIR is required".to_string()); } + if self.mint_token.is_empty() + || self.daemon_service_did.is_empty() + || self.appview_service_did.is_empty() + { + return Err("SPACEHOST_MINT_TOKEN, SPACEHOST_DAEMON_SERVICE_DID, and SPACEHOST_APPVIEW_SERVICE_DID are required".to_string()); + } Ok(()) } } @@ -187,6 +204,12 @@ mod tests { "https://client.example", "--actor-store-dir", "/actors", + "--mint-token", + "token", + "--daemon-service-did", + "did:plc:daemon", + "--appview-service-did", + "did:plc:appview", ]) .unwrap(); assert_eq!(cfg.authority_did, "did:plc:authority"); @@ -233,6 +256,9 @@ mod tests { std::env::set_var("SPACEHOST_OAUTH_AUDIENCE", "did:web:pds.example"); std::env::set_var("SPACEHOST_OAUTH_CLIENT_IDS", "https://client.example"); std::env::set_var("SPACEHOST_ACTOR_STORE_DIR", "/actors"); + std::env::set_var("SPACEHOST_MINT_TOKEN", "token"); + std::env::set_var("SPACEHOST_DAEMON_SERVICE_DID", "did:plc:daemon"); + std::env::set_var("SPACEHOST_APPVIEW_SERVICE_DID", "did:plc:appview"); let cfg = Config::try_parse_from(["rsky-space-host"]).unwrap(); for k in [ "SPACEHOST_AUTHORITY_DID", @@ -251,6 +277,9 @@ mod tests { "SPACEHOST_OAUTH_AUDIENCE", "SPACEHOST_OAUTH_CLIENT_IDS", "SPACEHOST_ACTOR_STORE_DIR", + "SPACEHOST_MINT_TOKEN", + "SPACEHOST_DAEMON_SERVICE_DID", + "SPACEHOST_APPVIEW_SERVICE_DID", ] { std::env::remove_var(k); } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 21381b9c..378433a5 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -62,6 +62,8 @@ pub struct AppState { pub commit_signer: Arc, pub auth: AuthConfig, pub rev: Arc String + Send + Sync>, + pub mint_token: String, + pub credential_mint_services: [String; 2], } pub fn router(state: AppState) -> Router { @@ -84,6 +86,7 @@ pub fn router(state: AppState) -> Router { ) .route("/xrpc/com.atproto.space.createRecord", post(create_record)) .route("/xrpc/com.atproto.space.deleteRecord", post(delete_record)) + .route("/admin/mintCredential", post(mint_credential)) .with_state(state) } @@ -216,6 +219,16 @@ fn check_proof( access_token: Option<&str>, ) -> Result { let uri = format!("{}/xrpc/{nsid}", state.public_url.trim_end_matches('/')); + check_proof_uri(state, headers, method, &uri, access_token) +} + +fn check_proof_uri( + state: &AppState, + headers: &HeaderMap, + method: &str, + uri: &str, + access_token: Option<&str>, +) -> Result { let proofs: Vec<&str> = headers .get_all("dpop") .iter() @@ -242,6 +255,63 @@ fn check_proof( }) } +#[derive(serde::Deserialize)] +struct MintCredentialParams { + space: String, +} + +async fn mint_credential( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result, ApiError> { + let supplied = headers + .get("x-spacehost-mint-token") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + if !same_secret(supplied, &state.mint_token) { + return Err(ApiError::forbidden("invalid mint token")); + } + let jwt = bearer(&headers)?; + let claims = service_jwt::claims(jwt)?; + if !state + .credential_mint_services + .iter() + .any(|did| did == &claims.iss) + { + return Err(ApiError::forbidden( + "service is not allowed to mint credentials", + )); + } + let key = state.keys.signing_key(&claims.iss).await?; + service_jwt::verify( + jwt, + &[state.authority.authority_did()], + "community.blacksky.space.mintCredential", + &key, + (state.now)(), + )?; + let space = require_this_space(&state, ¶ms.space)?; + let uri = format!( + "{}/admin/mintCredential", + state.public_url.trim_end_matches('/') + ); + let proof = check_proof_uri(&state, &headers, "POST", &uri, None)?; + let credential = + state + .authority + .mint_credential_for(&space, (state.now)(), (state.jti)(), &proof.jkt)?; + Ok(Json(serde_json::json!({"credential": credential}))) +} + +fn same_secret(left: &str, right: &str) -> bool { + let mut diff = left.len() ^ right.len(); + for (a, b) in left.bytes().zip(right.bytes()) { + diff |= usize::from(a ^ b); + } + diff == 0 +} + /// Space-credential auth: verify the presented credential against this /// authority's own space key, then confirm the presenter holds the key it is /// bound to. @@ -856,6 +926,8 @@ mod tests { commit_signer: Arc::new(test_signer()), auth: crate::oauth::tests::config(), rev: Arc::new(|| "3jzfcijpj2z2c".to_string()), + mint_token: "test-mint-token".to_string(), + credential_mint_services: ["did:plc:daemon".to_string(), "did:plc:appview".to_string()], }; Fixture { state, writes } } diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index dfff12e5..7ffbc373 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -123,6 +123,11 @@ async fn main() -> Result<(), Box> { commit_signer, auth: cfg.auth_config(), rev: Arc::new(move || ticker.lock().expect("ticker").next(None).to_string()), + mint_token: cfg.mint_token.clone(), + credential_mint_services: [ + cfg.daemon_service_did.clone(), + cfg.appview_service_did.clone(), + ], }; let listener = tokio::net::TcpListener::bind(&cfg.bind).await?; From 86fc44544341cf683932b4d32adf83d028629b12 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 14:37:06 -0400 Subject: [PATCH 09/56] feat(space-host): serve permissioned repo reads --- Cargo.lock | 1 + rsky-space-host/Cargo.toml | 1 + rsky-space-host/src/http.rs | 247 +++++++++++++++++++++++++++++++++++- 3 files changed, 248 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1c761014..38a9971e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8428,6 +8428,7 @@ dependencies = [ "axum", "base64 0.22.1", "chrono", + "cid", "clap", "hex", "hmac", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 630f0edb..21cc1431 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -19,6 +19,7 @@ rsky-identity = { workspace = true } rsky-lexicon = { workspace = true } rsky-oauth = { path = "../rsky-oauth", version = "0.3.0" } rsky-syntax = { workspace = true } +lexicon_cid = { workspace = true } secp256k1 = { workspace = true } sha2 = { workspace = true } hmac = "0.12" diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 378433a5..8d10d458 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -9,13 +9,15 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use rsky_lexicon::com::atproto::space::{ - GetSpaceCredentialInput, GetSpaceCredentialOutput, GetSpaceOutput, GetSpaceParams, + GetLatestCommitOutput, GetLatestCommitParams, GetRepoParams, GetSpaceCredentialInput, + GetSpaceCredentialOutput, GetSpaceOutput, GetSpaceParams, ListRepoOpsOutput, ListRepoOpsParams, ListReposOutput, ListReposParams, NotifyWriteInput, RegisterNotifyInput, RegisterNotifyOutput, SpaceConfig, }; use rsky_oauth::dpop::{DpopManager, DpopProof, DpopRequest}; use rsky_space::credential; use serde_json::Value; +use std::collections::BTreeMap; use std::sync::Arc; use crate::attestation::{JtiStore, MetadataFetcher}; @@ -75,6 +77,12 @@ pub fn router(state: AppState) -> Router { post(get_space_credential), ) .route("/xrpc/com.atproto.space.listRepos", get(list_repos)) + .route("/xrpc/com.atproto.space.getRepo", get(get_repo)) + .route("/xrpc/com.atproto.space.listRepoOps", get(list_repo_ops)) + .route( + "/xrpc/com.atproto.space.getLatestCommit", + get(get_latest_commit), + ) .route( "/xrpc/com.atproto.space.registerNotify", post(register_notify), @@ -483,6 +491,159 @@ async fn list_repos( Ok(Json(ListReposOutput { cursor, repos })) } +async fn signed_head( + state: &AppState, + space: &rsky_space::space_id::SpaceId, + repo: &str, +) -> Result { + let head = state + .repos + .head(&space.uri(), repo) + .await? + .ok_or(HostError::RepoNotFound)?; + crate::commits::mint_commit( + state.commit_signer.as_ref(), + &space.uri(), + repo, + &head.rev, + &head.hash(), + rand::random(), + ) + .map_err(Into::into) +} + +async fn get_latest_commit( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result, ApiError> { + let space = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.getLatestCommit", + ¶ms.space, + )?; + Ok(Json(GetLatestCommitOutput { + commit: signed_head(&state, &space, ¶ms.repo).await?, + })) +} + +async fn list_repo_ops( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result, ApiError> { + let space = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.listRepoOps", + ¶ms.space, + )?; + let limit = params + .limit + .unwrap_or(DEFAULT_LIST_LIMIT) + .clamp(1, MAX_LIST_LIMIT) as u32; + let page = state + .repos + .list_ops( + &space.uri(), + ¶ms.repo, + params.since.as_deref(), + params.cursor.as_deref(), + limit, + ) + .await?; + let commit = if page.complete { + Some(signed_head(&state, &space, ¶ms.repo).await?) + } else { + None + }; + let mut ops = Vec::with_capacity(page.ops.len()); + for op in page.ops { + let value = if params.exclude_values.unwrap_or(false) || op.cid.is_none() { + None + } else { + state + .repos + .get_record(&space.uri(), ¶ms.repo, &op.collection, &op.rkey) + .await? + .map(|record| rsky_space::record::decode_record(&record.value)) + .transpose() + .map_err(HostError::from)? + }; + ops.push(rsky_lexicon::com::atproto::space::RepoOp { + rev: op.rev, + collection: op.collection, + rkey: op.rkey, + cid: op.cid, + prev: op.prev, + value, + }); + } + Ok(Json(ListRepoOpsOutput { + cursor: page.cursor, + ops, + commit, + })) +} + +async fn get_repo( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result { + let space = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.getRepo", + ¶ms.space, + )?; + let commit = signed_head(&state, &space, ¶ms.repo).await?; + let records = state + .repos + .list_records( + &space.uri(), + ¶ms.repo, + None, + None, + MAX_LIST_LIMIT as u32, + ) + .await? + .0; + let mut entries = BTreeMap::new(); + let mut blocks = BTreeMap::new(); + for record in records { + let cid: lexicon_cid::Cid = record.cid.parse().map_err(|_| { + ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "InternalError", + "stored record has invalid cid", + ) + })?; + entries.insert(record.path(), cid); + blocks.insert(cid, record.value); + } + let internal = rsky_space::types::SignedCommit { + ver: commit.ver as u8, + hash: commit.hash.into(), + ikm: commit.ikm.into(), + sig: commit.sig.into(), + mac: commit.mac.into(), + rev: commit.rev, + }; + let car = rsky_space::repo_car_bytes(&internal, &entries, |cid| blocks.get(cid).cloned()) + .await + .map_err(HostError::from)?; + Ok(( + [(axum::http::header::CONTENT_TYPE, "application/vnd.ipld.car")], + car, + ) + .into_response()) +} + async fn register_notify( State(state): State, headers: HeaderMap, @@ -1160,6 +1321,90 @@ mod tests { assert_eq!(out["error"], "AuthenticationRequired"); } + #[tokio::test] + async fn repo_reads_require_credentials_and_serve_sync_shapes() { + let f = fixture(AppAccess::Open, &[]); + let value = rsky_space::record::encode_record( + &serde_json::json!({"$type":"app.bsky.feed.post","text":"sync"}), + MAX_RECORD_BYTES, + ) + .unwrap(); + f.state + .repos + .apply_writes( + &space_uri(), + MEMBER, + "3jzfcijpj2z2c", + &[RepoWrite::Create { + collection: "app.bsky.feed.post".to_string(), + rkey: "3jzfcijpj2z2c".to_string(), + value, + }], + ) + .await + .unwrap(); + let credential = credential_for(&f.state); + let query = format!( + "space={}&repo={}", + urlencode(&space_uri()), + urlencode(MEMBER) + ); + + let (status, body) = send( + &f.state, + get_req( + &format!("/xrpc/com.atproto.space.getLatestCommit?{query}"), + None, + ), + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(body["error"], "AuthenticationRequired"); + + let (status, body) = send( + &f.state, + get_req( + &format!("/xrpc/com.atproto.space.listRepoOps?{query}"), + Some(&credential), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["ops"].as_array().unwrap().len(), 1); + assert!(body["commit"].is_object()); + + let (status, body) = send( + &f.state, + get_req( + &format!("/xrpc/com.atproto.space.getLatestCommit?{query}"), + Some(&credential), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["commit"]["rev"], "3jzfcijpj2z2c"); + + let response = router(f.state.clone()) + .oneshot(get_req( + &format!("/xrpc/com.atproto.space.getRepo?{query}"), + Some(&credential), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[axum::http::header::CONTENT_TYPE], + "application/vnd.ipld.car" + ); + assert!(!response + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .is_empty()); + } + #[tokio::test] async fn health_reports_version() { let f = fixture(AppAccess::Open, &[]); From 26d8b2bd29ec7bd29ebb2a50313660296106f561 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 14:39:09 -0400 Subject: [PATCH 10/56] feat(daemon): mint bound credentials internally --- rsky-daemon/Cargo.toml | 1 + rsky-daemon/src/config.rs | 7 +++ rsky-daemon/src/credentials.rs | 49 ++++++++++++++++++ rsky-daemon/src/lib.rs | 5 +- rsky-daemon/src/main.rs | 35 ++++++++----- rsky-daemon/src/service_jwt.rs | 90 ++++++++++++++++++++++++++++++++++ rsky-daemon/src/xrpc.rs | 25 ++++++++++ 7 files changed, 197 insertions(+), 15 deletions(-) create mode 100644 rsky-daemon/src/service_jwt.rs diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index 64428658..a45f389d 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -36,6 +36,7 @@ chrono = { version = "0.4", features = ["serde"] } axum = "0.7" base64 = "0.22" sha2 = { workspace = true } +secp256k1 = { workspace = true } [dev-dependencies] secp256k1 = { workspace = true } diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index e1560765..bb43c233 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -57,6 +57,13 @@ pub struct Config { pub space_host_mint_token: String, #[arg(long, env = "DAEMON_DPOP_KEY_PATH", default_value = "")] pub dpop_key_path: String, + #[arg( + long, + env = "DAEMON_SERVICE_SIGNING_KEY_HEX", + default_value = "", + hide_env_values = true + )] + pub service_signing_key_hex: String, /// Bind address for the notify listener. #[arg(long, env = "DAEMON_NOTIFY_BIND", default_value = "127.0.0.1:8055")] diff --git a/rsky-daemon/src/credentials.rs b/rsky-daemon/src/credentials.rs index 255ff33f..ca5a1260 100644 --- a/rsky-daemon/src/credentials.rs +++ b/rsky-daemon/src/credentials.rs @@ -10,6 +10,7 @@ use tokio::sync::Mutex; use crate::error::Result; use crate::xrpc::{check, http_client, net_err, SpaceHostClient}; +use crate::{service_jwt::ServiceJwtIssuer, HttpSpaceHost}; /// Seconds since the Unix epoch; the injectable-`now` boundary for tests. pub fn unix_now() -> u64 { @@ -77,6 +78,54 @@ impl CredentialSource for StaticCredential { } } +pub struct InternalCredentialProvider { + space: String, + authority_did: String, + mint_token: String, + issuer: ServiceJwtIssuer, + host: Arc, + cached: Mutex>, +} +impl InternalCredentialProvider { + pub fn new( + space: impl Into, + authority_did: impl Into, + mint_token: impl Into, + issuer: ServiceJwtIssuer, + host: Arc, + ) -> Self { + Self { + space: space.into(), + authority_did: authority_did.into(), + mint_token: mint_token.into(), + issuer, + host, + cached: Mutex::new(None), + } + } +} +#[async_trait] +impl CredentialSource for InternalCredentialProvider { + async fn credential(&self, now: u64) -> Result { + let mut cached = self.cached.lock().await; + if let Some((jwt, exp)) = cached.as_ref() { + if now < exp.saturating_sub(CREDENTIAL_TTL_SECS / 5) { + return Ok(jwt.clone()); + } + } + let service_jwt = self + .issuer + .mint(&self.authority_did, now, &format!("mint-{now}"))?; + let jwt = self + .host + .mint_internal_credential(&self.space, &service_jwt, &self.mint_token) + .await?; + let exp = decode(&jwt)?.claims.exp; + *cached = Some((jwt.clone(), exp)); + Ok(jwt) + } +} + /// Mints and caches a space credential, re-minting once 80% of the credential /// TTL has elapsed so the daemon never presents one near expiry. pub struct CredentialProvider { diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index da741e1d..2c5b0774 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -28,12 +28,13 @@ pub mod notify; pub mod recovery; pub mod repohost; pub mod runner; +pub mod service_jwt; pub mod sqlite_index; pub mod xrpc; pub use credentials::{ - unix_now, CredentialProvider, CredentialSource, DelegationSource, PdsDelegationSource, - StaticCredential, + unix_now, CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, + PdsDelegationSource, StaticCredential, }; pub use engine::{sync_repo, CommitKeyResolver, SyncOutcome}; pub use error::{DaemonError, Result}; diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index aa9b3c81..1b29b37f 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -5,9 +5,9 @@ use clap::Parser; use rsky_daemon::config::Config; use rsky_daemon::engine::CommitKeyResolver; use rsky_daemon::{ - notify_router, run, CredentialProvider, CredentialSource, DaemonError, HttpRepoHost, - HttpSpaceHost, InMemoryIndex, NotifyState, PdsDelegationSource, Result, RunnerOptions, - SpaceIndex, SqliteIndex, StaticCredential, + notify_router, run, CredentialSource, DaemonError, HttpRepoHost, HttpSpaceHost, InMemoryIndex, + InternalCredentialProvider, NotifyState, Result, RunnerOptions, SpaceIndex, SqliteIndex, + StaticCredential, }; use rsky_identity::did::atproto_data::{get_did_key_from_multibase, VerificationMaterial}; use rsky_identity::types::{IdentityResolverOpts, MemoryCache}; @@ -75,11 +75,12 @@ async fn main() -> std::result::Result<(), Box> { let space = SpaceId::parse(&cfg.space_uri)?; // One proof-of-possession key for the process: the credential it mints is // bound to it, and every host it is presented to checks that binding. - let dpop = Arc::new(if cfg.dpop_key_path.is_empty() { - rsky_daemon::dpop::DpopSigner::generate()? - } else { - rsky_daemon::dpop::DpopSigner::load_or_generate(&cfg.dpop_key_path)? - }); + if cfg.dpop_key_path.is_empty() { + return Err("DAEMON_DPOP_KEY_PATH is required".into()); + } + let dpop = Arc::new(rsky_daemon::dpop::DpopSigner::load_or_generate( + &cfg.dpop_key_path, + )?); let host = Arc::new(HttpSpaceHost::new(&cfg.space_host_url, dpop.clone())); let keys: Arc = Arc::new(DidKeyResolver::new()); @@ -91,12 +92,20 @@ async fn main() -> std::result::Result<(), Box> { }; let creds: Arc = if cfg.static_credential.is_empty() { - Arc::new(CredentialProvider::new( + if cfg.space_host_mint_token.is_empty() || cfg.service_signing_key_hex.is_empty() { + return Err( + "DAEMON_SPACE_HOST_MINT_TOKEN and DAEMON_SERVICE_SIGNING_KEY_HEX are required" + .into(), + ); + } + Arc::new(InternalCredentialProvider::new( &cfg.space_uri, - Box::new(PdsDelegationSource::new( - &cfg.pds_url, - &cfg.pds_access_token, - )), + &space.authority, + &cfg.space_host_mint_token, + rsky_daemon::service_jwt::ServiceJwtIssuer::from_hex( + &cfg.service_identity, + &cfg.service_signing_key_hex, + )?, host.clone(), )) } else { diff --git a/rsky-daemon/src/service_jwt.rs b/rsky-daemon/src/service_jwt.rs new file mode 100644 index 00000000..a9c19379 --- /dev/null +++ b/rsky-daemon/src/service_jwt.rs @@ -0,0 +1,90 @@ +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine as _; +use secp256k1::{Message, Secp256k1, SecretKey}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::error::{DaemonError, Result}; + +pub const MINT_LXM: &str = "community.blacksky.space.mintCredential"; + +pub struct ServiceJwtIssuer { + did: String, + secret: SecretKey, +} + +impl ServiceJwtIssuer { + pub fn from_hex(did: impl Into, key: &str) -> Result { + let bytes = hex::decode(key.trim()).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + let secret = SecretKey::from_slice(&bytes).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + Ok(Self { + did: did.into(), + secret, + }) + } + + pub fn mint(&self, audience: &str, now: u64, jti: &str) -> Result { + #[derive(Serialize)] + struct Header<'a> { + typ: &'a str, + alg: &'a str, + } + #[derive(Serialize)] + struct Claims<'a> { + iss: &'a str, + aud: &'a str, + exp: u64, + lxm: &'a str, + jti: &'a str, + iat: u64, + } + let h = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&Header { + typ: "JWT", + alg: "ES256K", + }) + .unwrap(), + ); + let c = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&Claims { + iss: &self.did, + aud: audience, + exp: now + 60, + lxm: MINT_LXM, + jti, + iat: now, + }) + .unwrap(), + ); + let input = format!("{h}.{c}"); + let digest = Sha256::digest(input.as_bytes()); + let message = + Message::from_digest_slice(&digest).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + let mut sig = Secp256k1::new().sign_ecdsa(&message, &self.secret); + sig.normalize_s(); + Ok(format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(sig.serialize_compact()) + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn mints_a_short_lived_method_bound_token() { + let issuer = ServiceJwtIssuer::from_hex("did:plc:daemon", &hex::encode([7u8; 32])).unwrap(); + let jwt = issuer.mint("did:plc:authority", 1000, "jti").unwrap(); + let claims: serde_json::Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(jwt.split('.').nth(1).unwrap()) + .unwrap(), + ) + .unwrap(); + assert_eq!(claims["iss"], "did:plc:daemon"); + assert_eq!(claims["aud"], "did:plc:authority"); + assert_eq!(claims["lxm"], MINT_LXM); + assert_eq!(claims["exp"], 1060); + } +} diff --git a/rsky-daemon/src/xrpc.rs b/rsky-daemon/src/xrpc.rs index 77c954ab..b5f2af4d 100644 --- a/rsky-daemon/src/xrpc.rs +++ b/rsky-daemon/src/xrpc.rs @@ -100,6 +100,31 @@ impl HttpSpaceHost { fn url(&self, nsid: &str) -> String { format!("{}/xrpc/{nsid}", self.base_url) } + + pub async fn mint_internal_credential( + &self, + space: &str, + service_jwt: &str, + mint_token: &str, + ) -> Result { + let url = format!("{}/admin/mintCredential", self.base_url); + let out: GetSpaceCredentialOutput = check( + self.http + .post(&url) + .header("Authorization", format!("Bearer {service_jwt}")) + .header("X-Spacehost-Mint-Token", mint_token) + .header("DPoP", self.dpop.proof("POST", &url, None)?) + .query(&[("space", space)]) + .send() + .await + .map_err(net_err)?, + ) + .await? + .json() + .await + .map_err(net_err)?; + Ok(out.credential) + } } #[async_trait] From 7f01f2e6902d817fe4a3f644630ebb9d45b94fb7 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 14:44:11 -0400 Subject: [PATCH 11/56] test(space-host): cover shared spaces wire contract --- rsky-space-host/src/http.rs | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 8d10d458..0de75f9e 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -1405,6 +1405,45 @@ mod tests { .is_empty()); } + #[tokio::test] + async fn shared_read_surface_rejects_wrong_key_credentials() { + let f = fixture(AppAccess::Open, &[]); + let value = rsky_space::record::encode_record( + &serde_json::json!({"$type":"app.bsky.feed.post"}), + MAX_RECORD_BYTES, + ) + .unwrap(); + f.state + .repos + .apply_writes( + &space_uri(), + MEMBER, + "3jzfcijpj2z2c", + &[RepoWrite::Create { + collection: "app.bsky.feed.post".to_string(), + rkey: "3jzfcijpj2z2c".to_string(), + value, + }], + ) + .await + .unwrap(); + let path = format!( + "/xrpc/com.atproto.space.getLatestCommit?space={}&repo={}", + urlencode(&space_uri()), + urlencode(MEMBER) + ); + + let wrong_key = f + .state + .authority + .mint_credential(NOW, "wrong-key".to_string(), "another-thumbprint") + .unwrap(); + let (status, body) = send(&f.state, get_req(&path, Some(&wrong_key))).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(body["error"], "InvalidDpopProof"); + assert!(body.get("message").and_then(Value::as_str).is_some()); + } + #[tokio::test] async fn health_reports_version() { let f = fixture(AppAccess::Open, &[]); From 7717d9b52916e7813b0b860d357d876f54a50aa1 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 15:16:19 -0400 Subject: [PATCH 12/56] feat(daemon): add dynamic space discovery primitives --- rsky-daemon/src/credentials.rs | 66 +++++++-- rsky-daemon/src/lib.rs | 14 +- rsky-daemon/src/spaces.rs | 242 +++++++++++++++++++++++++++++++++ 3 files changed, 302 insertions(+), 20 deletions(-) create mode 100644 rsky-daemon/src/spaces.rs diff --git a/rsky-daemon/src/credentials.rs b/rsky-daemon/src/credentials.rs index ca5a1260..d33777e5 100644 --- a/rsky-daemon/src/credentials.rs +++ b/rsky-daemon/src/credentials.rs @@ -4,13 +4,14 @@ use async_trait::async_trait; use rsky_lexicon::com::atproto::space::GetDelegationTokenOutput; -use rsky_space::credential::{decode, CREDENTIAL_TTL_SECS}; +use rsky_space::credential::{CREDENTIAL_TTL_SECS, decode}; +use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Mutex; use crate::error::Result; -use crate::xrpc::{check, http_client, net_err, SpaceHostClient}; -use crate::{service_jwt::ServiceJwtIssuer, HttpSpaceHost}; +use crate::xrpc::{SpaceHostClient, check, http_client, net_err}; +use crate::{HttpSpaceHost, service_jwt::ServiceJwtIssuer}; /// Seconds since the Unix epoch; the injectable-`now` boundary for tests. pub fn unix_now() -> u64 { @@ -79,12 +80,12 @@ impl CredentialSource for StaticCredential { } pub struct InternalCredentialProvider { - space: String, + default_space: String, authority_did: String, mint_token: String, issuer: ServiceJwtIssuer, host: Arc, - cached: Mutex>, + cached: Mutex>, } impl InternalCredentialProvider { pub fn new( @@ -95,20 +96,21 @@ impl InternalCredentialProvider { host: Arc, ) -> Self { Self { - space: space.into(), + default_space: space.into(), authority_did: authority_did.into(), mint_token: mint_token.into(), issuer, host, - cached: Mutex::new(None), + cached: Mutex::new(HashMap::new()), } } -} -#[async_trait] -impl CredentialSource for InternalCredentialProvider { - async fn credential(&self, now: u64) -> Result { + + /// Get a credential bound to this daemon's DPoP key for any discovered + /// space. The service identity is shared, but credentials never cross a + /// space boundary in the cache. + pub async fn credential_for(&self, space: &str, now: u64) -> Result { let mut cached = self.cached.lock().await; - if let Some((jwt, exp)) = cached.as_ref() { + if let Some((jwt, exp)) = cached.get(space) { if now < exp.saturating_sub(CREDENTIAL_TTL_SECS / 5) { return Ok(jwt.clone()); } @@ -118,13 +120,19 @@ impl CredentialSource for InternalCredentialProvider { .mint(&self.authority_did, now, &format!("mint-{now}"))?; let jwt = self .host - .mint_internal_credential(&self.space, &service_jwt, &self.mint_token) + .mint_internal_credential(space, &service_jwt, &self.mint_token) .await?; let exp = decode(&jwt)?.claims.exp; - *cached = Some((jwt.clone(), exp)); + cached.insert(space.to_string(), (jwt.clone(), exp)); Ok(jwt) } } +#[async_trait] +impl CredentialSource for InternalCredentialProvider { + async fn credential(&self, now: u64) -> Result { + self.credential_for(&self.default_space, now).await + } +} /// Mints and caches a space credential, re-minting once 80% of the credential /// TTL has elapsed so the daemon never presents one near expiry. @@ -178,7 +186,7 @@ mod tests { use super::*; use chrono::{DateTime, Utc}; use rsky_lexicon::com::atproto::space::ListReposOutput; - use rsky_space::credential::{encode, JwtHeader, SpaceClaims, CREDENTIAL_TYP}; + use rsky_space::credential::{CREDENTIAL_TYP, JwtHeader, SpaceClaims, encode}; use std::sync::atomic::{AtomicUsize, Ordering}; use wiremock::matchers::{header, method, path, query_param}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -273,6 +281,34 @@ mod tests { assert_eq!(host.mints.load(Ordering::SeqCst), 2); } + #[tokio::test] + async fn internal_credentials_are_cached_per_space() { + let a = SPACE; + let b = "at://did:plc:authority/space/community.blacksky.feed/other"; + let provider = InternalCredentialProvider::new( + a, + "did:plc:authority", + "mint-token", + ServiceJwtIssuer::from_hex("did:plc:daemon", &"11".repeat(32)).unwrap(), + Arc::new(HttpSpaceHost::new( + "http://127.0.0.1:9", + Arc::new(crate::dpop::DpopSigner::generate().unwrap()), + )), + ); + provider.cached.lock().await.extend([ + (a.into(), (credential_jwt(1000), 8200)), + (b.into(), (credential_jwt(2000), 9200)), + ]); + assert_eq!( + provider.credential_for(a, 3000).await.unwrap(), + credential_jwt(1000) + ); + assert_eq!( + provider.credential_for(b, 3000).await.unwrap(), + credential_jwt(2000) + ); + } + struct GarbageHost; #[async_trait] impl SpaceHostClient for GarbageHost { diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index 2c5b0774..93641851 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -29,19 +29,23 @@ pub mod recovery; pub mod repohost; pub mod runner; pub mod service_jwt; +pub mod spaces; pub mod sqlite_index; pub mod xrpc; pub use credentials::{ - unix_now, CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, - PdsDelegationSource, StaticCredential, + CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, + PdsDelegationSource, StaticCredential, unix_now, }; -pub use engine::{sync_repo, CommitKeyResolver, SyncOutcome}; +pub use engine::{CommitKeyResolver, SyncOutcome, sync_repo}; pub use error::{DaemonError, Result}; pub use index::{InMemoryIndex, SpaceIndex}; -pub use notify::{router as notify_router, NotifyState, WriteNotice}; +pub use notify::{NotifyState, WriteNotice, router as notify_router}; pub use recovery::recover_repo; pub use repohost::{HttpRepoHost, OplogPage, RepoHostClient}; -pub use runner::{run, sync_repo_healing, sync_space_once, RunnerOptions, SweepReport}; +pub use runner::{RunnerOptions, SweepReport, run, sync_repo_healing, sync_space_once}; +pub use spaces::{ + CombinedSource, HttpSpaceSource, SpaceRegistry, SpaceSource, SpaceTarget, StaticSpaces, +}; pub use sqlite_index::{SpaceScopedIndex, SqliteIndex}; pub use xrpc::{HttpSpaceHost, SpaceHostClient}; diff --git a/rsky-daemon/src/spaces.rs b/rsky-daemon/src/spaces.rs new file mode 100644 index 00000000..fa63d656 --- /dev/null +++ b/rsky-daemon/src/spaces.rs @@ -0,0 +1,242 @@ +//! Runtime discovery of the spaces a daemon should sync. + +use async_trait::async_trait; +use rsky_space::space_id::SpaceId; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, RwLock}; + +use crate::error::{DaemonError, Result}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SpaceTarget { + pub generation: i64, + pub state: String, +} + +#[async_trait] +pub trait SpaceSource: Send + Sync { + async fn spaces(&self) -> Result>; +} + +pub struct StaticSpaces(pub BTreeSet); + +impl StaticSpaces { + pub fn new, S: Into>(spaces: I) -> Self { + Self(spaces.into_iter().map(Into::into).collect()) + } +} + +#[async_trait] +impl SpaceSource for StaticSpaces { + async fn spaces(&self) -> Result> { + Ok(self + .0 + .iter() + .cloned() + .map(|space| { + ( + space, + SpaceTarget { + generation: 1, + state: "active".into(), + }, + ) + }) + .collect()) + } +} + +pub struct HttpSpaceSource { + url: String, + api_key: String, + authority_did: String, + space_type: String, + http: reqwest::Client, +} + +impl HttpSpaceSource { + pub fn new( + url: impl Into, + api_key: impl Into, + authority_did: impl Into, + space_type: impl Into, + ) -> Self { + Self { + url: url.into().trim_end_matches('/').into(), + api_key: api_key.into(), + authority_did: authority_did.into(), + space_type: space_type.into(), + http: reqwest::Client::new(), + } + } +} + +#[derive(serde::Deserialize)] +struct SpacesResponse { + spaces: Vec, +} +#[derive(serde::Deserialize)] +struct SyncableSpace { + space: String, + generation: i64, + state: String, +} + +#[async_trait] +impl SpaceSource for HttpSpaceSource { + async fn spaces(&self) -> Result> { + let response = self + .http + .get(format!("{}/admin/sync-spaces", self.url)) + .query(&[("authority", &self.authority_did)]) + .header("X-RSKY-KEY", &self.api_key) + .send() + .await + .map_err(|e| DaemonError::Xrpc(e.to_string()))?; + if !response.status().is_success() { + return Err(DaemonError::Xrpc(format!( + "space list returned {}", + response.status() + ))); + } + let body: SpacesResponse = response + .json() + .await + .map_err(|e| DaemonError::Xrpc(e.to_string()))?; + Ok(body + .spaces + .into_iter() + .filter_map(|entry| { + let space = SpaceId::parse(&entry.space).ok()?; + (entry.generation > 0 + && matches!( + entry.state.as_str(), + "host_registered" | "active" | "deleting" + ) + && space.authority == self.authority_did + && space.space_type == self.space_type) + .then_some(( + space.uri(), + SpaceTarget { + generation: entry.generation, + state: entry.state, + }, + )) + }) + .collect()) + } +} + +pub struct CombinedSource(pub Vec>); + +#[async_trait] +impl SpaceSource for CombinedSource { + async fn spaces(&self) -> Result> { + let mut all = BTreeMap::new(); + let mut last_error = None; + for source in &self.0 { + match source.spaces().await { + Ok(spaces) => { + for (space, target) in spaces { + all.entry(space) + .and_modify(|current: &mut SpaceTarget| { + if target.generation >= current.generation { + *current = target.clone(); + } + }) + .or_insert(target); + } + } + Err(error) => { + tracing::warn!(error = %error, "a space source failed"); + last_error = Some(error); + } + } + } + match last_error { + Some(error) if all.is_empty() => Err(error), + _ => Ok(all), + } + } +} + +#[derive(Clone, Default)] +pub struct SpaceRegistry(Arc>>); +impl SpaceRegistry { + pub fn new() -> Self { + Self::default() + } + pub fn contains(&self, space: &str) -> bool { + self.0.read().expect("space registry").contains(space) + } + pub fn snapshot(&self) -> BTreeSet { + self.0.read().expect("space registry").clone() + } + pub fn replace(&self, spaces: BTreeSet) { + *self.0.write().expect("space registry") = spaces; + } + pub fn insert(&self, space: impl Into) { + self.0.write().expect("space registry").insert(space.into()); + } + pub fn remove(&self, space: &str) { + self.0.write().expect("space registry").remove(space); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use wiremock::matchers::{header, method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + const A: &str = "at://did:plc:c/space/community.blacksky.feed/a"; + const B: &str = "at://did:plc:c/space/community.blacksky.feed/b"; + fn source(url: String) -> HttpSpaceSource { + HttpSpaceSource::new(url, "key", "did:plc:c", "community.blacksky.feed") + } + #[tokio::test] + async fn filters_and_reads_managing_app_spaces() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/admin/sync-spaces")).and(query_param("authority", "did:plc:c")).and(header("X-RSKY-KEY", "key")).respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"spaces":[{"space":A,"generation":2,"state":"active"},{"space":B,"generation":3,"state":"deleting"},{"space":"at://did:plc:other/space/community.blacksky.feed/x","generation":1,"state":"active"}]}))).mount(&server).await; + assert_eq!(source(server.uri()).spaces().await.unwrap().len(), 2); + } + #[tokio::test] + async fn pin_survives_source_outage() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let spaces = CombinedSource(vec![ + Box::new(StaticSpaces::new([A])), + Box::new(source(server.uri())), + ]) + .spaces() + .await + .unwrap(); + assert!(spaces.contains_key(A)); + } + #[tokio::test] + async fn every_source_failure_errors() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + assert!( + CombinedSource(vec![Box::new(source(server.uri()))]) + .spaces() + .await + .is_err() + ); + } + #[test] + fn registry_tracks_spaces() { + let registry = SpaceRegistry::new(); + registry.insert(A); + assert!(registry.contains(A)); + registry.replace(BTreeSet::from([B.into()])); + assert!(!registry.contains(A) && registry.contains(B)); + registry.remove(B); + assert!(registry.snapshot().is_empty()); + } +} From d9cc30b854106d9fb75aa588e18567597775feae Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 15:19:39 -0400 Subject: [PATCH 13/56] feat(daemon): add discovery configuration --- rsky-daemon/src/config.rs | 42 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index bb43c233..fa9a54e6 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -9,9 +9,22 @@ use clap::Parser; )] pub struct Config { /// The space to sync: `at://{authority}/space/{type}/{skey}`. - #[arg(long, env = "DAEMON_SPACE_URI")] + #[arg(long, env = "DAEMON_SPACE_URI", default_value = "")] pub space_uri: String, + /// Managing-app base URL for dynamic space discovery. + #[arg(long, env = "DAEMON_SPACES_URL", default_value = "")] + pub spaces_url: String, + /// Managing-app API key for dynamic space discovery. + #[arg(long, env = "DAEMON_SPACES_API_KEY", default_value = "", hide_env_values = true)] + pub spaces_api_key: String, + /// Authority whose spaces this daemon discovers. + #[arg(long, env = "DAEMON_AUTHORITY_DID", default_value = "")] + pub authority_did: String, + /// Space type accepted from the managing app. + #[arg(long, env = "DAEMON_SPACE_TYPE", default_value = "community.blacksky.feed")] + pub space_type: String, + /// The space host (authority) base URL, for listRepos + credential mint. #[arg(long, env = "DAEMON_SPACE_HOST_URL")] pub space_host_url: String, @@ -84,6 +97,20 @@ pub struct Config { } impl Config { + pub fn validate(&self) -> std::result::Result<(), String> { + if self.space_uri.is_empty() && self.spaces_url.is_empty() { + return Err("DAEMON_SPACE_URI or DAEMON_SPACES_URL is required".into()); + } + if !self.spaces_url.is_empty() + && (self.spaces_api_key.is_empty() || self.authority_did.is_empty()) + { + return Err( + "DAEMON_SPACES_API_KEY and DAEMON_AUTHORITY_DID are required with DAEMON_SPACES_URL" + .into(), + ); + } + Ok(()) + } pub fn repo_host_url(&self) -> &str { if self.repo_host_url.is_empty() { &self.space_host_url @@ -174,5 +201,18 @@ mod tests { assert_eq!(cfg.notify_endpoint(), "http://0.0.0.0:9000"); assert_eq!(cfg.index_db_path, "/data/space.sqlite"); assert_eq!(cfg.sweep_interval_secs, 60); + assert!(cfg.validate().is_ok()); + + let discovery_only = Config::try_parse_from([ + "rsky-daemon", "--space-host-url", "https://host.example", + "--service-identity", "did:web:syncer.example", "--spaces-url", "https://feeds.example", + "--spaces-api-key", "key", "--authority-did", "did:plc:authority", + ]).unwrap(); + assert!(discovery_only.validate().is_ok()); + let neither = Config::try_parse_from([ + "rsky-daemon", "--space-host-url", "https://host.example", + "--service-identity", "did:web:syncer.example", + ]).unwrap(); + assert!(neither.validate().is_err()); } } From 54fd0063865a60caaabf6972990baabd82ab397c Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 15:21:10 -0400 Subject: [PATCH 14/56] feat(daemon): add multi-space worker supervisor --- rsky-daemon/src/credentials.rs | 15 +++++++++++ rsky-daemon/src/lib.rs | 2 +- rsky-daemon/src/runner.rs | 49 ++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/rsky-daemon/src/credentials.rs b/rsky-daemon/src/credentials.rs index d33777e5..053890ea 100644 --- a/rsky-daemon/src/credentials.rs +++ b/rsky-daemon/src/credentials.rs @@ -87,6 +87,21 @@ pub struct InternalCredentialProvider { host: Arc, cached: Mutex>, } + +/// Adapts the shared internal provider to one worker's `CredentialSource`. +pub struct SpaceCredentialSource { + provider: Arc, + space: String, +} +impl SpaceCredentialSource { + pub fn new(provider: Arc, space: impl Into) -> Self { + Self { provider, space: space.into() } + } +} +#[async_trait] +impl CredentialSource for SpaceCredentialSource { + async fn credential(&self, now: u64) -> Result { self.provider.credential_for(&self.space, now).await } +} impl InternalCredentialProvider { pub fn new( space: impl Into, diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index 93641851..47c77491 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -35,7 +35,7 @@ pub mod xrpc; pub use credentials::{ CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, - PdsDelegationSource, StaticCredential, unix_now, + PdsDelegationSource, SpaceCredentialSource, StaticCredential, unix_now, }; pub use engine::{CommitKeyResolver, SyncOutcome, sync_repo}; pub use error::{DaemonError, Result}; diff --git a/rsky-daemon/src/runner.rs b/rsky-daemon/src/runner.rs index 30410035..d22a9ecf 100644 --- a/rsky-daemon/src/runner.rs +++ b/rsky-daemon/src/runner.rs @@ -5,6 +5,7 @@ use rsky_lexicon::com::atproto::space::RepoRef; use std::sync::Arc; use std::time::Duration; +use std::collections::HashMap; use tokio::sync::{mpsc, watch}; use tokio::time::Instant; @@ -16,12 +17,60 @@ use crate::notify::WriteNotice; use crate::recovery::recover_repo; use crate::repohost::RepoHostClient; use crate::xrpc::SpaceHostClient; +use crate::spaces::{SpaceRegistry, SpaceSource}; const REGISTER_RETRY_SECS: u64 = 30; const MIN_REREGISTER_SECS: u64 = 30; /// Builds a repo-host client bound to the current space credential. pub type RepoHostFactory = Box Arc + Send + Sync>; +pub type MultiSpaceFactory = Arc Result<(Arc, RepoHostFactory, Arc)> + Send + Sync>; + +pub struct MultiRunnerOptions { + pub refresh_interval_secs: u64, + pub sweep_interval_secs: u64, + pub notify_endpoint: String, + pub service_identity: String, + pub now_fn: fn() -> u64, +} + +/// Supervises one existing `run` worker per discovered space. Source errors +/// retain the current worker set; a transient managing-app outage must not +/// silently stop private-feed projection. +#[allow(clippy::too_many_arguments)] +pub async fn run_multi( + opts: MultiRunnerOptions, + source: Arc, + registry: SpaceRegistry, + factory: MultiSpaceFactory, + host: Arc, + keys: Arc, + mut notices: mpsc::Receiver, + mut shutdown: watch::Receiver, +) { + struct Worker { generation: i64, stop: watch::Sender, notices: mpsc::Sender, handle: tokio::task::JoinHandle<()> } + let mut workers: HashMap = HashMap::new(); + let mut refresh = tokio::time::interval(Duration::from_secs(opts.refresh_interval_secs.max(1))); + loop { tokio::select! { + _ = shutdown.changed() => break, + _ = refresh.tick() => { + let desired = match source.spaces().await { Ok(value) => value, Err(error) => { tracing::warn!(error = %error, "space source unavailable; keeping current workers"); continue; } }; + let stale: Vec<_> = workers.iter().filter(|(space, worker)| desired.get(*space).is_none_or(|target| target.generation != worker.generation)).map(|(space, _)| space.clone()).collect(); + for space in stale { if let Some(worker) = workers.remove(&space) { let _ = worker.stop.send(true); let _ = worker.handle.await; } } + for (space, target) in &desired { if workers.contains_key(space) { continue; } + let (creds, repo, index) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; + let (tx, rx) = mpsc::channel(256); let (stop, stop_rx) = watch::channel(false); + let worker_opts = RunnerOptions { space_uri: space.clone(), sweep_interval_secs: opts.sweep_interval_secs, notify_endpoint: opts.notify_endpoint.clone(), service_identity: opts.service_identity.clone(), now_fn: opts.now_fn }; + let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), rx, stop_rx)); + workers.insert(space.clone(), Worker { generation: target.generation, stop, notices: tx, handle }); + } + registry.replace(workers.keys().cloned().collect()); + } + Some(notice) = notices.recv() => { if let Some(worker) = workers.get(¬ice.0) { let _ = worker.notices.send(notice).await; } else { tracing::warn!(space = %notice.0, "notice for a space we do not sync"); } } + }} + for (space, worker) in workers { let _ = worker.stop.send(true); let _ = worker.handle.await; tracing::info!(%space, "stopped"); } + registry.replace(Default::default()); +} #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct SweepReport { From 35d9f2c472f8c95c60e69edfb27a84e3b71d206f Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 15:22:41 -0400 Subject: [PATCH 15/56] feat(daemon): enable dynamic space discovery --- rsky-daemon/src/lib.rs | 2 +- rsky-daemon/src/main.rs | 70 +++++++++++++++++---------------------- rsky-daemon/src/notify.rs | 7 ++-- 3 files changed, 37 insertions(+), 42 deletions(-) diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index 47c77491..5c4fa44b 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -43,7 +43,7 @@ pub use index::{InMemoryIndex, SpaceIndex}; pub use notify::{NotifyState, WriteNotice, router as notify_router}; pub use recovery::recover_repo; pub use repohost::{HttpRepoHost, OplogPage, RepoHostClient}; -pub use runner::{RunnerOptions, SweepReport, run, sync_repo_healing, sync_space_once}; +pub use runner::{MultiRunnerOptions, RunnerOptions, SweepReport, run, run_multi, sync_repo_healing, sync_space_once}; pub use spaces::{ CombinedSource, HttpSpaceSource, SpaceRegistry, SpaceSource, SpaceTarget, StaticSpaces, }; diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 1b29b37f..d1204d54 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -5,9 +5,10 @@ use clap::Parser; use rsky_daemon::config::Config; use rsky_daemon::engine::CommitKeyResolver; use rsky_daemon::{ - notify_router, run, CredentialSource, DaemonError, HttpRepoHost, HttpSpaceHost, InMemoryIndex, - InternalCredentialProvider, NotifyState, Result, RunnerOptions, SpaceIndex, SqliteIndex, - StaticCredential, + notify_router, CombinedSource, CredentialSource, DaemonError, HttpRepoHost, HttpSpaceHost, + HttpSpaceSource, InMemoryIndex, InternalCredentialProvider, MultiRunnerOptions, NotifyState, + Result, SpaceCredentialSource, SpaceIndex, SpaceRegistry, SqliteIndex, StaticCredential, + StaticSpaces, run_multi, }; use rsky_identity::did::atproto_data::{get_did_key_from_multibase, VerificationMaterial}; use rsky_identity::types::{IdentityResolverOpts, MemoryCache}; @@ -72,7 +73,8 @@ async fn main() -> std::result::Result<(), Box> { .init(); let cfg = Config::parse(); - let space = SpaceId::parse(&cfg.space_uri)?; + cfg.validate()?; + let authority_did = if cfg.authority_did.is_empty() { SpaceId::parse(&cfg.space_uri)?.authority } else { cfg.authority_did.clone() }; // One proof-of-possession key for the process: the credential it mints is // bound to it, and every host it is presented to checks that binding. if cfg.dpop_key_path.is_empty() { @@ -84,49 +86,50 @@ async fn main() -> std::result::Result<(), Box> { let host = Arc::new(HttpSpaceHost::new(&cfg.space_host_url, dpop.clone())); let keys: Arc = Arc::new(DidKeyResolver::new()); - let index: Arc = if cfg.index_db_path.is_empty() { - tracing::warn!("no DAEMON_INDEX_DB_PATH set; using a non-persistent in-memory index"); - Arc::new(InMemoryIndex::new()) - } else { - Arc::new(Arc::new(SqliteIndex::open(&cfg.index_db_path)?).for_space(&cfg.space_uri)) - }; + let db = if cfg.index_db_path.is_empty() { None } else { Some(Arc::new(SqliteIndex::open(&cfg.index_db_path)?)) }; - let creds: Arc = if cfg.static_credential.is_empty() { + let shared_creds = if cfg.static_credential.is_empty() { if cfg.space_host_mint_token.is_empty() || cfg.service_signing_key_hex.is_empty() { return Err( "DAEMON_SPACE_HOST_MINT_TOKEN and DAEMON_SERVICE_SIGNING_KEY_HEX are required" .into(), ); } - Arc::new(InternalCredentialProvider::new( + Some(Arc::new(InternalCredentialProvider::new( &cfg.space_uri, - &space.authority, + &authority_did, &cfg.space_host_mint_token, rsky_daemon::service_jwt::ServiceJwtIssuer::from_hex( &cfg.service_identity, &cfg.service_signing_key_hex, )?, host.clone(), - )) + ))) } else { - tracing::warn!("using a static space credential (dev mode)"); - Arc::new(StaticCredential(cfg.static_credential.clone())) + None }; + let mut sources: Vec> = Vec::new(); + if !cfg.space_uri.is_empty() { sources.push(Box::new(StaticSpaces::new([cfg.space_uri.clone()]))); } + if !cfg.spaces_url.is_empty() { sources.push(Box::new(HttpSpaceSource::new(&cfg.spaces_url, &cfg.spaces_api_key, &authority_did, &cfg.space_type))); } + let source = Arc::new(CombinedSource(sources)); + let registry = SpaceRegistry::new(); + let (notify_tx, notify_rx) = mpsc::channel(1024); let (shutdown_tx, shutdown_rx) = watch::channel(false); let notify_state = NotifyState { space_uri: cfg.space_uri.clone(), - authority_did: space.authority.clone(), + registry: registry.clone(), + authority_did: authority_did.clone(), service_identity: cfg.service_identity.clone(), resolver: keys.clone(), - index: index.clone(), + index: Arc::new(InMemoryIndex::new()), tx: notify_tx, now_fn: rsky_daemon::unix_now, }; let listener = tokio::net::TcpListener::bind(&cfg.notify_bind).await?; tracing::info!( - space = %cfg.space_uri, + authority = %authority_did, host = %cfg.space_host_url, notify_bind = %cfg.notify_bind, sweep_secs = cfg.sweep_interval_secs, @@ -142,30 +145,19 @@ async fn main() -> std::result::Result<(), Box> { .await }); - let repo_host_base = cfg.repo_host_url().to_string(); - let opts = RunnerOptions { - space_uri: cfg.space_uri.clone(), - sweep_interval_secs: cfg.sweep_interval_secs, + let repo_host_base = cfg.repo_host_url().to_string(); let static_credential = cfg.static_credential.clone(); let db_for_factory = db.clone(); let dpop_for_factory = dpop.clone(); let shared_for_factory = shared_creds.clone(); + let factory = Arc::new(move |space: &str| -> Result<(Arc, rsky_daemon::runner::RepoHostFactory, Arc)> { + let creds: Arc = match &shared_for_factory { Some(provider) => Arc::new(SpaceCredentialSource::new(provider.clone(), space)), None => Arc::new(StaticCredential(static_credential.clone()) )}; + let index: Arc = match &db_for_factory { Some(db) => Arc::new(db.for_space(space)), None => Arc::new(InMemoryIndex::new()) }; + let base = repo_host_base.clone(); let proof = dpop_for_factory.clone(); + Ok((creds, Box::new(move |credential| Arc::new(HttpRepoHost::new(base.clone(), credential, proof.clone()))), index)) + }); + let opts = MultiRunnerOptions { refresh_interval_secs: cfg.sweep_interval_secs, sweep_interval_secs: cfg.sweep_interval_secs, notify_endpoint: cfg.notify_endpoint(), service_identity: cfg.service_identity.clone(), now_fn: rsky_daemon::unix_now, }; - let runner = tokio::spawn(run( - opts, - host, - creds, - Box::new(move |credential| { - Arc::new(HttpRepoHost::new( - repo_host_base.clone(), - credential, - dpop.clone(), - )) - }), - index, - keys, - notify_rx, - shutdown_rx, - )); + let runner = tokio::spawn(run_multi(opts, source, registry, factory, host, keys, notify_rx, shutdown_rx)); tokio::signal::ctrl_c().await?; tracing::info!("ctrl-c received; shutting down"); diff --git a/rsky-daemon/src/notify.rs b/rsky-daemon/src/notify.rs index 033090dc..56cd3f05 100644 --- a/rsky-daemon/src/notify.rs +++ b/rsky-daemon/src/notify.rs @@ -19,6 +19,7 @@ use tokio::sync::mpsc; use crate::engine::CommitKeyResolver; use crate::error::{DaemonError, Result}; use crate::index::SpaceIndex; +use crate::spaces::SpaceRegistry; /// A queued `(space, did)` write notice for the runner to pull. pub type WriteNotice = (String, String); @@ -88,6 +89,7 @@ pub fn decode_claims(jwt: &str) -> Result { #[derive(Clone)] pub struct NotifyState { pub space_uri: String, + pub registry: SpaceRegistry, /// The authority (space host) DID whose key signs inbound notifications. pub authority_did: String, /// This syncer's service identity: the required `aud` on inbound tokens. @@ -143,7 +145,7 @@ async fn notify_write( error_body("AuthenticationRequired", e), ); } - if input.space != state.space_uri { + if !state.registry.contains(&input.space) { return ( StatusCode::BAD_REQUEST, error_body("InvalidRequest", "space is not synced by this daemon"), @@ -170,7 +172,7 @@ async fn notify_space_deleted( error_body("AuthenticationRequired", e), ); } - if input.space != state.space_uri { + if !state.registry.contains(&input.space) { return ( StatusCode::BAD_REQUEST, error_body("InvalidRequest", "space is not synced by this daemon"), @@ -243,6 +245,7 @@ mod tests { ) -> NotifyState { NotifyState { space_uri: SPACE.to_string(), + registry: { let registry = SpaceRegistry::new(); registry.insert(SPACE); registry }, authority_did: AUTHORITY.to_string(), service_identity: SYNCER.to_string(), resolver: Arc::new(FixedKey(did_key.to_string())), From 4a7a9f271af0f7633cd187a7606a712ab2d89e9f Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 15:51:33 -0400 Subject: [PATCH 16/56] refactor(space-host): resolve authority context per request via registry --- rsky-space-host/src/authority.rs | 65 ++++++++++++- rsky-space-host/src/http.rs | 152 +++++++++++++++++-------------- rsky-space-host/src/lib.rs | 2 +- rsky-space-host/src/main.rs | 39 ++++---- 4 files changed, 172 insertions(+), 86 deletions(-) diff --git a/rsky-space-host/src/authority.rs b/rsky-space-host/src/authority.rs index 54371e51..de449a5c 100644 --- a/rsky-space-host/src/authority.rs +++ b/rsky-space-host/src/authority.rs @@ -8,15 +8,76 @@ use rsky_space::credential::{ self, Confirmation, JwtHeader, SpaceClaims, CREDENTIAL_TTL_SECS, CREDENTIAL_TYP, }; use rsky_space::space_id::SpaceId; -use std::collections::BTreeSet; -use std::sync::RwLock; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, RwLock}; use crate::appaccess::AppAccess; use crate::attestation::{verify_client_attestation, JtiStore, MetadataFetcher}; use crate::error::{HostError, Result}; +use crate::notify::Notifier; use crate::policy::Policy; +use crate::registration::LifecycleAcker; use crate::signing::Signer; +/// Everything derived from one space authority: the authority itself plus the +/// collaborators that mint, verify, or sign in its name. +pub struct AuthorityContext { + pub authority: Arc, + pub policy: Arc, + pub notifier: Arc, + pub lifecycle_acker: Option>, +} + +impl AuthorityContext { + pub fn authority_did(&self) -> &str { + self.authority.authority_did() + } +} + +/// The authorities this host answers for, keyed by authority DID. +#[derive(Default)] +pub struct AuthorityRegistry { + contexts: RwLock>>, +} + +impl AuthorityRegistry { + pub fn new() -> Self { + Self::default() + } + + pub fn insert(&self, context: Arc) { + self.contexts + .write() + .expect("authority registry") + .insert(context.authority_did().to_string(), context); + } + + pub fn authority(&self, authority_did: &str) -> Result> { + self.contexts + .read() + .expect("authority registry") + .get(authority_did) + .cloned() + .ok_or_else(|| HostError::SpaceNotFound(authority_did.to_string())) + } + + pub fn for_space(&self, space_uri: &str) -> Result> { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + self.authority(&space.authority) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string())) + } + + pub fn contexts(&self) -> Vec> { + self.contexts + .read() + .expect("authority registry") + .values() + .cloned() + .collect() + } +} + /// Resolves an account's atproto signing `did:key` (from its DID document), used /// to verify a delegation token minted by that user's PDS. #[async_trait] diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 0de75f9e..ed23000b 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -21,15 +21,14 @@ use std::collections::BTreeMap; use std::sync::Arc; use crate::attestation::{JtiStore, MetadataFetcher}; -use crate::authority::{Authority, KeyResolver}; +use crate::authority::{AuthorityContext, AuthorityRegistry, KeyResolver}; use crate::commits::CommitSigner; use crate::error::HostError; use crate::keys::DocSource; use crate::managing_app::require_https; -use crate::notify::{fan_out_write, Notifier, NOTIFY_WRITE_LXM}; +use crate::notify::{fan_out_write, NOTIFY_WRITE_LXM}; use crate::oauth::{verify_access, AuthConfig, RequestAuth}; -use crate::policy::Policy; -use crate::registration::{LifecycleAcker, REGISTER_SPACE_LXM}; +use crate::registration::REGISTER_SPACE_LXM; use crate::repo::{RepoStore, RepoWrite, WriteOutcome, MAX_RECORD_BYTES}; use crate::service_jwt; use crate::store::{RegistrationStore, Subscriber, WriterSetStore}; @@ -40,17 +39,14 @@ const MAX_LIST_LIMIT: i64 = 1000; #[derive(Clone)] pub struct AppState { - pub authority: Arc, - pub policy: Arc, + pub registry: Arc, pub keys: Arc, pub metadata: Arc, pub jti_store: Arc, pub writers: Arc, pub registrations: Arc, - pub lifecycle_acker: Option>, /// Resolves a subscriber's service identifier to its delivery endpoint. pub docs: Arc, - pub notifier: Arc, /// Verifies DPoP proofs on the credential-issuance and credential-presenting /// paths. Space issuance does not challenge with nonces, so this manager /// carries none; the replay store is what makes a proof single-use. @@ -291,22 +287,22 @@ async fn mint_credential( "service is not allowed to mint credentials", )); } + let (context, space) = require_this_space(&state, ¶ms.space)?; let key = state.keys.signing_key(&claims.iss).await?; service_jwt::verify( jwt, - &[state.authority.authority_did()], + &[context.authority_did()], "community.blacksky.space.mintCredential", &key, (state.now)(), )?; - let space = require_this_space(&state, ¶ms.space)?; let uri = format!( "{}/admin/mintCredential", state.public_url.trim_end_matches('/') ); let proof = check_proof_uri(&state, &headers, "POST", &uri, None)?; let credential = - state + context .authority .mint_credential_for(&space, (state.now)(), (state.jti)(), &proof.jkt)?; Ok(Json(serde_json::json!({"credential": credential}))) @@ -334,14 +330,14 @@ fn require_space_credential( method: &str, nsid: &str, space_uri: &str, -) -> Result { - let space = require_this_space(state, space_uri)?; +) -> Result<(Arc, rsky_space::space_id::SpaceId), ApiError> { + let (context, space) = require_this_space(state, space_uri)?; let jwt = dpop_credential(headers)?; let bound_jkt = credential::verify_space_credential( jwt, &space.uri(), - state.authority.authority_did(), - state.authority.signer.did_key(), + context.authority_did(), + context.authority.signer.did_key(), (state.now)(), ) .map_err(|e| ApiError::new(StatusCode::UNAUTHORIZED, "InvalidToken", e.to_string()))?; @@ -353,7 +349,7 @@ fn require_space_credential( "DPoP key thumbprint does not match the credential binding", )); } - Ok(space) + Ok((context, space)) } /// Resolve a `did:...#fragment` subscriber to its delivery endpoint. The @@ -385,11 +381,16 @@ async fn resolve_service_endpoint(docs: &dyn DocSource, service: &str) -> Result fn require_this_space( state: &AppState, space: &str, -) -> Result { - state +) -> Result<(Arc, rsky_space::space_id::SpaceId), ApiError> { + let context = state + .registry + .for_space(space) + .map_err(|_| ApiError::invalid_request(format!("space not hosted here: {space}")))?; + let space = context .authority .resolve(space) - .map_err(|_| ApiError::invalid_request(format!("space not hosted here: {space}"))) + .map_err(|_| ApiError::invalid_request(format!("space not hosted here: {space}")))?; + Ok((context, space)) } async fn health() -> Json { @@ -401,7 +402,7 @@ async fn get_space( headers: HeaderMap, Query(params): Query, ) -> Result, ApiError> { - let space = require_space_credential( + let (context, space) = require_space_credential( &state, &headers, "GET", @@ -410,19 +411,20 @@ async fn get_space( )?; Ok(Json(GetSpaceOutput { space: space.uri(), - config: SpaceConfig::Simplespace(state.authority.space_config(&state.policy)), + config: SpaceConfig::Simplespace(context.authority.space_config(&context.policy)), })) } async fn require_service_auth( state: &AppState, + context: &AuthorityContext, headers: &HeaderMap, expected_lxm: &str, ) -> Result { let jwt = bearer(headers)?; let claims = service_jwt::claims(jwt)?; let issuer_key = state.keys.signing_key(&claims.iss).await?; - let authority_did = state.authority.authority_did(); + let authority_did = context.authority_did(); let space_host_aud = format!("{authority_did}#atproto_space_host"); service_jwt::verify( jwt, @@ -439,7 +441,7 @@ async fn get_space_credential( headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - let space = require_this_space(&state, &input.space)?; + let (context, space) = require_this_space(&state, &input.space)?; let delegation_token = bearer(&headers)?; // Before the delegation token, so a caller with a bad proof does not burn // its single-use grant finding out. @@ -450,13 +452,13 @@ async fn get_space_credential( "com.atproto.space.getSpaceCredential", None, )?; - let credential = state + let credential = context .authority .get_space_credential_for( &space, delegation_token, input.client_attestation.as_deref(), - &state.policy, + &context.policy, state.keys.as_ref(), state.metadata.as_ref(), state.jti_store.as_ref(), @@ -517,7 +519,7 @@ async fn get_latest_commit( headers: HeaderMap, Query(params): Query, ) -> Result, ApiError> { - let space = require_space_credential( + let (_, space) = require_space_credential( &state, &headers, "GET", @@ -534,7 +536,7 @@ async fn list_repo_ops( headers: HeaderMap, Query(params): Query, ) -> Result, ApiError> { - let space = require_space_credential( + let (_, space) = require_space_credential( &state, &headers, "GET", @@ -594,7 +596,7 @@ async fn get_repo( headers: HeaderMap, Query(params): Query, ) -> Result { - let space = require_space_credential( + let (_, space) = require_space_credential( &state, &headers, "GET", @@ -702,8 +704,9 @@ async fn register_space( if input.generation < 1 { return Err(ApiError::invalid_request("generation must be positive")); } - let claims = require_service_auth(&state, &headers, REGISTER_SPACE_LXM).await?; - let expected_issuer = state + let context = state.registry.for_space(&input.space)?; + let claims = require_service_auth(&state, &context, &headers, REGISTER_SPACE_LXM).await?; + let expected_issuer = context .policy .managing_app() .and_then(|service| service.split_once('#').map(|(did, _)| did)) @@ -711,14 +714,14 @@ async fn register_space( if claims.iss != expected_issuer { return Err(ApiError::forbidden("issuer is not the managing app")); } - let acker = state.lifecycle_acker.as_ref().ok_or_else(|| { + let acker = context.lifecycle_acker.as_ref().ok_or_else(|| { ApiError::new( StatusCode::SERVICE_UNAVAILABLE, "LifecycleUnavailable", "lifecycle acknowledgement is not configured", ) })?; - state.authority.register(&input.space)?; + context.authority.register(&input.space)?; acker .ack_host_registered(&input.space, input.generation) .await?; @@ -731,8 +734,8 @@ async fn write_actor( path: &str, space: &str, repo: &str, -) -> Result { - let space = require_this_space(state, space)?; +) -> Result<(Arc, rsky_space::space_id::SpaceId), ApiError> { + let (context, space) = require_this_space(state, space)?; let url = format!("{}{}", state.public_url.trim_end_matches('/'), path); let access = verify_access( &RequestAuth { @@ -752,7 +755,7 @@ async fn write_actor( "session subject does not match repo", )); } - Ok(space) + Ok((context, space)) } async fn create_record( @@ -760,7 +763,7 @@ async fn create_record( headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - let space = write_actor( + let (context, space) = write_actor( &state, &headers, "/xrpc/com.atproto.space.createRecord", @@ -794,7 +797,7 @@ async fn create_record( WriteOutcome::Created { cid } => cid.clone(), _ => return Err(HostError::Store("create did not create".into()).into()), }; - record_write(&state, &space.uri(), &input.repo, &applied.rev).await?; + record_write(&state, &context, &space.uri(), &input.repo, &applied.rev).await?; Ok(Json( rsky_lexicon::com::atproto::space::CreateRecordOutput { uri: space.record_uri(&input.repo, &input.collection, &rkey), @@ -823,7 +826,7 @@ async fn delete_record( headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - let space = write_actor( + let (context, space) = write_actor( &state, &headers, "/xrpc/com.atproto.space.deleteRecord", @@ -845,7 +848,7 @@ async fn delete_record( ) .await?; if !matches!(applied.outcomes[0], WriteOutcome::Noop) { - record_write(&state, &space.uri(), &input.repo, &applied.rev).await?; + record_write(&state, &context, &space.uri(), &input.repo, &applied.rev).await?; } Ok(Json( rsky_lexicon::com::atproto::space::DeleteRecordOutput { @@ -859,6 +862,7 @@ async fn delete_record( async fn record_write( state: &AppState, + context: &AuthorityContext, space: &str, repo: &str, rev: &str, @@ -870,7 +874,7 @@ async fn record_write( .await?; let endpoints = state.registrations.endpoints(space, now).await?; fan_out_write( - state.notifier.clone(), + context.notifier.clone(), endpoints, NotifyWriteInput { space: space.to_string(), @@ -886,7 +890,7 @@ async fn notify_write( headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - require_this_space(&state, &input.space)?; + let (context, _) = require_this_space(&state, &input.space)?; let jwt = bearer(&headers)?; let claims = service_jwt::claims(jwt)?; // The repo host signs with the member's own key, so a notification may only @@ -899,7 +903,7 @@ async fn notify_write( )); } let issuer_key = state.keys.signing_key(&claims.iss).await?; - let authority_did = state.authority.authority_did(); + let authority_did = context.authority_did(); let space_host_aud = format!("{authority_did}#atproto_space_host"); service_jwt::verify( jwt, @@ -915,7 +919,7 @@ async fn notify_write( .upsert_writer(&input.space, &input.repo, &input.rev, None, now) .await?; let endpoints = state.registrations.endpoints(&input.space, now).await?; - fan_out_write(state.notifier.clone(), endpoints, input); + fan_out_write(context.notifier.clone(), endpoints, input); Ok(Json(serde_json::json!({}))) } @@ -924,8 +928,12 @@ mod tests { use super::*; use crate::appaccess::AppAccess; use crate::attestation::{ClientMetadata, InMemoryJtiStore}; + use crate::authority::Authority; use crate::error::Result as HostResult; use crate::membership::InMemoryMembership; + use crate::notify::Notifier; + use crate::policy::Policy; + use crate::registration::LifecycleAcker; use crate::signing::{test_signer, Signer}; use crate::store::{InMemoryRegistrations, InMemoryWriterSet}; use async_trait::async_trait; @@ -1054,6 +1062,10 @@ mod tests { } } + fn ctx(state: &AppState) -> Arc { + state.registry.for_space(&space_uri()).unwrap() + } + fn fixture(app_access: AppAccess, members: &[&str]) -> Fixture { let space = SpaceId::new( "did:plc:communityauthority", @@ -1062,19 +1074,23 @@ mod tests { ); let authority = Authority::new(space, test_signer(), app_access); let (tx, writes) = tokio::sync::mpsc::unbounded_channel(); - let state = AppState { + let registry = Arc::new(AuthorityRegistry::new()); + registry.insert(Arc::new(AuthorityContext { authority: Arc::new(authority), policy: Arc::new(Policy::MemberList(Arc::new(InMemoryMembership::new( members.iter().map(|m| m.to_string()), )))), + notifier: Arc::new(RecordingNotifier { tx }), + lifecycle_acker: None, + })); + let state = AppState { + registry, keys: Arc::new(UserKeys), metadata: Arc::new(crate::oauth::tests::AsJwks), jti_store: Arc::new(InMemoryJtiStore::default()), writers: Arc::new(InMemoryWriterSet::default()), registrations: Arc::new(InMemoryRegistrations::default()), - lifecycle_acker: None, docs: Arc::new(NoDocs), - notifier: Arc::new(RecordingNotifier { tx }), dpop: Arc::new(rsky_oauth::dpop::DpopManager::new( None, Box::new(rsky_oauth::dpop::InMemoryReplayStore::default()), @@ -1132,7 +1148,7 @@ mod tests { } fn credential_for(state: &AppState) -> String { - state + ctx(state) .authority .mint_credential(NOW, "cred-jti".to_string(), &dpop_key().thumbprint()) .unwrap() @@ -1144,12 +1160,13 @@ mod tests { alg: rsky_crypto::constants::SECP256K1_JWT_ALG.to_string(), kid: Some("#atproto".to_string()), }; + let context = ctx(state); let claims = SpaceClaims { iss: user.to_string(), - sub: state.authority.space_uri(), + sub: context.authority.space_uri(), aud: Some(format!( "{}#atproto_space_host", - state.authority.authority_did() + context.authority_did() )), iat: NOW, exp: NOW + 60, @@ -1433,8 +1450,7 @@ mod tests { urlencode(MEMBER) ); - let wrong_key = f - .state + let wrong_key = ctx(&f.state) .authority .mint_credential(NOW, "wrong-key".to_string(), "another-thumbprint") .unwrap(); @@ -1488,8 +1504,7 @@ mod tests { assert_eq!(body["error"], "AuthenticationRequired"); // A credential bound to someone else's key is refused as well. - let other_binding = f - .state + let other_binding = ctx(&f.state) .authority .mint_credential(NOW, "other-jti".to_string(), "some-other-thumbprint") .unwrap(); @@ -1527,8 +1542,8 @@ mod tests { credential::verify_space_credential( out["credential"].as_str().unwrap(), &space_uri(), - f.state.authority.authority_did(), - f.state.authority.signer.did_key(), + ctx(&f.state).authority_did(), + ctx(&f.state).authority.signer.did_key(), NOW, ) .unwrap(); @@ -1716,7 +1731,7 @@ mod tests { ) .await .unwrap(); - let aud = format!("{}#atproto_space_host", f.state.authority.authority_did()); + let aud = format!("{}#atproto_space_host", ctx(&f.state).authority_did()); let token = member_service_jwt(&aud, NOTIFY_WRITE_LXM); let path = "/xrpc/com.atproto.space.notifyWrite"; let body = serde_json::json!({ @@ -1747,7 +1762,7 @@ mod tests { async fn notify_write_auth_failures() { let f = fixture(AppAccess::Open, &[]); let path = "/xrpc/com.atproto.space.notifyWrite"; - let authority_did = f.state.authority.authority_did().to_string(); + let authority_did = ctx(&f.state).authority_did().to_string(); let body = serde_json::json!({ "space": space_uri(), "repo": MEMBER, @@ -1815,7 +1830,7 @@ mod tests { let mut broken = f.state.clone(); broken.writers = Arc::new(BrokenWriters); let token = member_service_jwt( - &format!("{}#atproto_space_host", broken.authority.authority_did()), + &format!("{}#atproto_space_host", ctx(&broken).authority_did()), NOTIFY_WRITE_LXM, ); let body = serde_json::json!({ @@ -1834,15 +1849,20 @@ mod tests { #[tokio::test] async fn registration_authenticates_the_managing_app_and_acks_before_activation() { - let mut f = fixture(AppAccess::Open, &[]); - f.state.policy = Arc::new(Policy::ManagingApp { - service_id: format!("{MEMBER}#bsky_fg"), - client: Arc::new(UnusedManagingApp), - }); + let f = fixture(AppAccess::Open, &[]); let acker = Arc::new(RecordingAcker::default()); - f.state.lifecycle_acker = Some(acker.clone()); + let existing = ctx(&f.state); + f.state.registry.insert(Arc::new(AuthorityContext { + authority: existing.authority.clone(), + policy: Arc::new(Policy::ManagingApp { + service_id: format!("{MEMBER}#bsky_fg"), + client: Arc::new(UnusedManagingApp), + }), + notifier: existing.notifier.clone(), + lifecycle_acker: Some(acker.clone()), + })); let space = "at://did:plc:communityauthority/space/community.blacksky.feed/new"; - let audience = format!("{}#atproto_space_host", f.state.authority.authority_did()); + let audience = format!("{}#atproto_space_host", ctx(&f.state).authority_did()); let token = service_jwt::mint( &user_signer(), MEMBER, @@ -1863,7 +1883,7 @@ mod tests { .await; assert_eq!(status, StatusCode::OK, "{body}"); - assert!(f.state.authority.resolve_registered(space).is_ok()); + assert!(ctx(&f.state).authority.resolve_registered(space).is_ok()); assert_eq!( acker.0.lock().unwrap().as_slice(), &[(space.to_string(), 7)] diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index 0c70867d..57b4fcb0 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -38,6 +38,6 @@ pub mod service_jwt; pub mod signing; pub mod store; -pub use authority::{Authority, KeyResolver}; +pub use authority::{Authority, AuthorityContext, AuthorityRegistry, KeyResolver}; pub use error::{HostError, Result}; pub use policy::Policy; diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index 7ffbc373..b7877bcd 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -8,7 +8,7 @@ use rsky_oauth::dpop::{DpopManager, InMemoryReplayStore}; use rsky_space::space_id::SpaceId; use rsky_space_host::appaccess::AppAccess; use rsky_space_host::attestation::HttpMetadataFetcher; -use rsky_space_host::authority::Authority; +use rsky_space_host::authority::{Authority, AuthorityContext, AuthorityRegistry}; use rsky_space_host::config::{Config, PolicyMode}; use rsky_space_host::http::{router, AppState, DEFAULT_REGISTRATION_TTL_SECS}; use rsky_space_host::keys::{DocKeyResolver, ResolverDocSource}; @@ -86,31 +86,35 @@ async fn main() -> Result<(), Box> { let repos = Arc::new(SqliteRepos::open(&cfg.db_path)?); let commit_signer = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); let ticker = std::sync::Mutex::new(rsky_common::tid::Ticker::new()); - let state = AppState { + let registry = Arc::new(AuthorityRegistry::new()); + registry.insert(Arc::new(AuthorityContext { authority: Arc::new(authority), policy: Arc::new(policy), - keys: Arc::new(DocKeyResolver::new(docs.clone())), - docs, - metadata: Arc::new(HttpMetadataFetcher::new()), - jti_store: store.clone(), - writers: store.clone(), - registrations: store, + notifier: Arc::new(HttpNotifier::new( + cfg.authority_did.clone(), + signer.clone(), + now.clone(), + jti.clone(), + )), lifecycle_acker: (cfg.policy == PolicyMode::ManagingApp).then(|| { Arc::new(HttpLifecycleAcker::new( cfg.lifecycle_url.clone(), cfg.lifecycle_service_did.clone(), cfg.authority_did.clone(), - signer.clone(), + signer, now.clone(), jti.clone(), )) as Arc }), - notifier: Arc::new(HttpNotifier::new( - cfg.authority_did.clone(), - signer, - now.clone(), - jti.clone(), - )), + })); + let state = AppState { + registry, + keys: Arc::new(DocKeyResolver::new(docs.clone())), + docs, + metadata: Arc::new(HttpMetadataFetcher::new()), + jti_store: store.clone(), + writers: store.clone(), + registrations: store, dpop: Arc::new(DpopManager::new( None, Box::new(InMemoryReplayStore::default()), @@ -131,9 +135,10 @@ async fn main() -> Result<(), Box> { }; let listener = tokio::net::TcpListener::bind(&cfg.bind).await?; + let bootstrap = state.registry.authority(&cfg.authority_did)?; tracing::info!( - space = %state.authority.space_uri(), - authority_key = %state.authority.signer.did_key(), + space = %bootstrap.authority.space_uri(), + authority_key = %bootstrap.authority.signer.did_key(), policy = ?cfg.policy, bind = %cfg.bind, db = %cfg.db_path, From 5f8e4046c0024468b937642133ab7a1a3b899efa Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 15:58:59 -0400 Subject: [PATCH 17/56] feat(space-host): acquire authorities dynamically at registration --- Cargo.lock | 2 +- rsky-pds/Cargo.toml | 2 +- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/authority.rs | 16 +++ rsky-space-host/src/config.rs | 93 +++++++++++- rsky-space-host/src/http.rs | 236 ++++++++++++++++++++++++++++++- rsky-space-host/src/main.rs | 177 +++++++++++++++-------- rsky-space-host/src/pds_seam.rs | 4 + rsky-space-host/src/store.rs | 93 ++++++++++++ 9 files changed, 556 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38a9971e..80af2458 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8422,7 +8422,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.6.0" +version = "0.7.0" dependencies = [ "async-trait", "axum", diff --git a/rsky-pds/Cargo.toml b/rsky-pds/Cargo.toml index 5615672c..2ae66164 100644 --- a/rsky-pds/Cargo.toml +++ b/rsky-pds/Cargo.toml @@ -54,7 +54,7 @@ rsky-identity = { workspace = true } rsky-lexicon = { workspace = true } rsky-repo = { workspace = true } rsky-space = { path = "../rsky-space", version = "0.4.0" } -rsky-space-host = { path = "../rsky-space-host", version = "0.6.0" } +rsky-space-host = { path = "../rsky-space-host", version = "0.7.0" } rsky-syntax = { workspace = true } hickory-resolver = "0.24.1" secp256k1 = { workspace = true } diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 21cc1431..e833f581 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.6.0" +version = "0.7.0" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/authority.rs b/rsky-space-host/src/authority.rs index de449a5c..21d9ce6a 100644 --- a/rsky-space-host/src/authority.rs +++ b/rsky-space-host/src/authority.rs @@ -34,6 +34,11 @@ impl AuthorityContext { } } +/// Builds the [`AuthorityContext`] for an authority first seen at +/// registration time; fails when the authority's signing key is unavailable. +pub type AuthorityFactory = + Arc Result> + Send + Sync>; + /// The authorities this host answers for, keyed by authority DID. #[derive(Default)] pub struct AuthorityRegistry { @@ -52,6 +57,17 @@ impl AuthorityRegistry { .insert(context.authority_did().to_string(), context); } + /// Insert unless the authority is already present; returns the context + /// that ends up registered either way. + pub fn insert_if_absent(&self, context: Arc) -> Arc { + self.contexts + .write() + .expect("authority registry") + .entry(context.authority_did().to_string()) + .or_insert(context) + .clone() + } + pub fn authority(&self, authority_did: &str) -> Result> { self.contexts .read() diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index b6ba4276..ac701762 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -21,12 +21,16 @@ pub enum PolicyMode { about = "atproto permissioned-data space authority/host" )] pub struct Config { - /// The space authority DID (dedicated community DID). - #[arg(long, env = "SPACEHOST_AUTHORITY_DID")] + /// Optional bootstrap authority pin: a space authority DID served from + /// startup with an explicit signing key. Set together with + /// `SPACEHOST_SIGNING_KEY_HEX`, or leave both unset and let authorities + /// arrive via registration. + #[arg(long, env = "SPACEHOST_AUTHORITY_DID", default_value = "")] pub authority_did: String, - /// Hex-encoded secp256k1 space signing key (`#atproto_space`). - #[arg(long, env = "SPACEHOST_SIGNING_KEY_HEX")] + /// Hex-encoded secp256k1 space signing key (`#atproto_space`) for the + /// pinned bootstrap authority. + #[arg(long, env = "SPACEHOST_SIGNING_KEY_HEX", default_value = "")] pub signing_key_hex: String, /// How the authority authorizes users at credential-mint time. @@ -147,7 +151,24 @@ impl Config { } } + pub fn bootstrap_pin(&self) -> Option<(&str, &str)> { + (!self.authority_did.is_empty() && !self.signing_key_hex.is_empty()) + .then_some((self.authority_did.as_str(), self.signing_key_hex.as_str())) + } + pub fn validate(&self) -> Result<(), String> { + if self.authority_did.is_empty() != self.signing_key_hex.is_empty() { + return Err( + "SPACEHOST_AUTHORITY_DID and SPACEHOST_SIGNING_KEY_HEX must be set together (bootstrap pin) or both left unset" + .to_string(), + ); + } + if self.bootstrap_pin().is_none() && self.actor_store_dir.is_empty() { + return Err( + "no space authority available: set SPACEHOST_ACTOR_STORE_DIR (authorities register with actor-store keys) or pin one with SPACEHOST_AUTHORITY_DID + SPACEHOST_SIGNING_KEY_HEX" + .to_string(), + ); + } if self.policy == PolicyMode::ManagingApp && !self.managing_app.contains('#') { return Err( "managing-app policy requires SPACEHOST_MANAGING_APP (did#fragment)".to_string(), @@ -186,7 +207,9 @@ mod tests { // which would race sibling tests run in parallel. #[test] fn parses_args_env_and_requirements() { - assert!(Config::try_parse_from(["rsky-space-host"]).is_err()); + let bare = Config::try_parse_from(["rsky-space-host"]).unwrap(); + assert!(bare.bootstrap_pin().is_none()); + assert!(bare.validate().is_err()); let cfg = Config::try_parse_from([ "rsky-space-host", @@ -224,7 +247,13 @@ mod tests { assert!(format!("{cfg:?}").contains("did:plc:authority")); let mut cfg = cfg; - cfg.update_from(["rsky-space-host", "--bind", "127.0.0.1:9"]); + cfg.update_from([ + "rsky-space-host", + "--bind", + "127.0.0.1:9", + "--authority-did", + "did:plc:authority", + ]); assert_eq!(cfg.bind, "127.0.0.1:9"); assert_eq!(cfg.authority_did, "did:plc:authority"); @@ -302,4 +331,56 @@ mod tests { invalid.managing_app = String::new(); assert!(invalid.validate().is_err()); } + + fn valid_unpinned() -> Config { + Config::try_parse_from([ + "rsky-space-host", + "--oauth-issuer", + "https://pds.example", + "--oauth-jwks-uri", + "https://pds.example/jwks", + "--oauth-audience", + "did:web:pds.example", + "--oauth-client-ids", + "https://client.example", + "--actor-store-dir", + "/actors", + "--mint-token", + "token", + "--daemon-service-did", + "did:plc:daemon", + "--appview-service-did", + "did:plc:appview", + ]) + .unwrap() + } + + #[test] + fn bootstrap_pin_is_optional_but_all_or_nothing() { + let cfg = valid_unpinned(); + assert!(cfg.bootstrap_pin().is_none()); + assert!(cfg.validate().is_ok()); + + let mut half = valid_unpinned(); + half.authority_did = "did:plc:authority".to_string(); + assert!(half.validate().is_err()); + + let mut half = valid_unpinned(); + half.signing_key_hex = "aa".repeat(32); + assert!(half.validate().is_err()); + + let mut pinned = valid_unpinned(); + pinned.authority_did = "did:plc:authority".to_string(); + pinned.signing_key_hex = "aa".repeat(32); + assert_eq!( + pinned.bootstrap_pin(), + Some(("did:plc:authority", pinned.signing_key_hex.as_str())) + ); + assert!(pinned.validate().is_ok()); + + let mut keyless = valid_unpinned(); + keyless.actor_store_dir = String::new(); + let message = keyless.validate().unwrap_err(); + assert!(message.contains("no space authority available"), "{message}"); + } } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index ed23000b..8ae13383 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -21,7 +21,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use crate::attestation::{JtiStore, MetadataFetcher}; -use crate::authority::{AuthorityContext, AuthorityRegistry, KeyResolver}; +use crate::authority::{AuthorityContext, AuthorityFactory, AuthorityRegistry, KeyResolver}; use crate::commits::CommitSigner; use crate::error::HostError; use crate::keys::DocSource; @@ -31,7 +31,7 @@ use crate::oauth::{verify_access, AuthConfig, RequestAuth}; use crate::registration::REGISTER_SPACE_LXM; use crate::repo::{RepoStore, RepoWrite, WriteOutcome, MAX_RECORD_BYTES}; use crate::service_jwt; -use crate::store::{RegistrationStore, Subscriber, WriterSetStore}; +use crate::store::{HostedSpaceStore, RegistrationStore, Subscriber, WriterSetStore}; pub const DEFAULT_REGISTRATION_TTL_SECS: u64 = 24 * 60 * 60; const DEFAULT_LIST_LIMIT: i64 = 100; @@ -40,6 +40,9 @@ const MAX_LIST_LIMIT: i64 = 1000; #[derive(Clone)] pub struct AppState { pub registry: Arc, + /// Builds the context for an authority first seen at registration time. + pub authority_factory: AuthorityFactory, + pub hosted_spaces: Arc, pub keys: Arc, pub metadata: Arc, pub jti_store: Arc, @@ -704,7 +707,24 @@ async fn register_space( if input.generation < 1 { return Err(ApiError::invalid_request("generation must be positive")); } - let context = state.registry.for_space(&input.space)?; + let (context, adopted) = match state.registry.for_space(&input.space) { + Ok(context) => (context, false), + Err(_) => { + let space = rsky_space::space_id::SpaceId::parse(&input.space).map_err(|_| { + ApiError::invalid_request(format!("invalid space uri: {}", input.space)) + })?; + let context = (state.authority_factory)(&space).map_err(|e| match e { + HostError::AccountNotHosted(did) => ApiError::invalid_request(format!( + "authority signing key does not resolve: {did}" + )), + HostError::SpaceNotFound(space) => ApiError::invalid_request(format!( + "space not hosted here: {space}" + )), + other => ApiError::from(other), + })?; + (context, true) + } + }; let claims = require_service_auth(&state, &context, &headers, REGISTER_SPACE_LXM).await?; let expected_issuer = context .policy @@ -714,7 +734,7 @@ async fn register_space( if claims.iss != expected_issuer { return Err(ApiError::forbidden("issuer is not the managing app")); } - let acker = context.lifecycle_acker.as_ref().ok_or_else(|| { + let acker = context.lifecycle_acker.clone().ok_or_else(|| { ApiError::new( StatusCode::SERVICE_UNAVAILABLE, "LifecycleUnavailable", @@ -722,6 +742,19 @@ async fn register_space( ) })?; context.authority.register(&input.space)?; + let context = if adopted { + let winner = state.registry.insert_if_absent(context.clone()); + if !Arc::ptr_eq(&winner, &context) { + winner.authority.register(&input.space)?; + } + winner + } else { + context + }; + state + .hosted_spaces + .record_space(context.authority_did(), &input.space) + .await?; acker .ack_host_registered(&input.space, input.generation) .await?; @@ -1085,6 +1118,10 @@ mod tests { })); let state = AppState { registry, + authority_factory: Arc::new(|space: &SpaceId| { + Err(HostError::AccountNotHosted(space.authority.clone())) + }), + hosted_spaces: Arc::new(crate::store::InMemoryHostedSpaces::default()), keys: Arc::new(UserKeys), metadata: Arc::new(crate::oauth::tests::AsJwks), jti_store: Arc::new(InMemoryJtiStore::default()), @@ -1890,6 +1927,197 @@ mod tests { ); } + #[tokio::test] + async fn each_authority_mints_and_verifies_with_its_own_key() { + let f = fixture(AppAccess::Open, &[]); + let other_space_uri = "at://did:plc:otherauthority/space/community.blacksky.feed/main"; + let other_signer = + Signer::from_secret(secp256k1::SecretKey::from_slice(&[0x66u8; 32]).unwrap()); + assert_ne!(other_signer.did_key(), test_signer().did_key()); + let (tx, _writes) = tokio::sync::mpsc::unbounded_channel(); + f.state.registry.insert(Arc::new(AuthorityContext { + authority: Arc::new(Authority::new( + SpaceId::parse(other_space_uri).unwrap(), + other_signer, + AppAccess::Open, + )), + policy: Arc::new(Policy::Public), + notifier: Arc::new(RecordingNotifier { tx }), + lifecycle_acker: None, + })); + let other = f.state.registry.for_space(other_space_uri).unwrap(); + let path = |space: &str| { + format!( + "/xrpc/com.atproto.space.getSpace?space={}", + urlencode(space) + ) + }; + + // Each authority's credential serves its own space. + let own = ctx(&f.state) + .authority + .mint_credential(NOW, "cred-a".to_string(), &dpop_key().thumbprint()) + .unwrap(); + let (status, body) = send(&f.state, get_req(&path(&space_uri()), Some(&own))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["space"], space_uri()); + + let other_cred = other + .authority + .mint_credential(NOW, "cred-b".to_string(), &dpop_key().thumbprint()) + .unwrap(); + let (status, body) = send(&f.state, get_req(&path(other_space_uri), Some(&other_cred))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["space"], other_space_uri); + assert_eq!(body["config"]["policy"], "public"); + + // A credential signed with authority A's key never opens authority B's + // space, even with matching claims. + let cross = ctx(&f.state) + .authority + .mint_credential_for( + &SpaceId::parse(other_space_uri).unwrap(), + NOW, + "cred-cross".to_string(), + &dpop_key().thumbprint(), + ) + .unwrap(); + let (status, body) = send(&f.state, get_req(&path(other_space_uri), Some(&cross))).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(body["error"], "InvalidToken"); + } + + const DYNAMIC_AUTHORITY: &str = "did:plc:dynauthority"; + const DYNAMIC_AUTHORITY_KEY: [u8; 32] = [0x33u8; 32]; + + fn dynamic_actor_store() -> tempfile::TempDir { + use sha2::Digest; + let directory = tempfile::tempdir().unwrap(); + let digest = hex::encode(sha2::Sha256::digest(DYNAMIC_AUTHORITY.as_bytes())); + let actor = directory.path().join(&digest[..2]).join(DYNAMIC_AUTHORITY); + std::fs::create_dir_all(&actor).unwrap(); + std::fs::write(actor.join("key"), DYNAMIC_AUTHORITY_KEY).unwrap(); + std::fs::write(actor.join("store.sqlite"), []).unwrap(); + directory + } + + fn seam_factory( + seam: Arc, + acker: Arc, + ) -> crate::authority::AuthorityFactory { + Arc::new(move |space: &SpaceId| { + let signer = seam.signer(&space.authority)?; + let (tx, _writes) = tokio::sync::mpsc::unbounded_channel(); + Ok(Arc::new(AuthorityContext { + authority: Arc::new(Authority::new( + space.clone(), + signer, + AppAccess::Open, + )), + policy: Arc::new(Policy::ManagingApp { + service_id: format!("{MEMBER}#bsky_fg"), + client: Arc::new(UnusedManagingApp), + }), + notifier: Arc::new(RecordingNotifier { tx }), + lifecycle_acker: Some(acker.clone()), + })) + }) + } + + #[tokio::test] + async fn registration_creates_an_unknown_authority_from_the_actor_store() { + let directory = dynamic_actor_store(); + let seam = Arc::new(crate::pds_seam::PdsSeam::open(directory.path()).unwrap()); + let acker = Arc::new(RecordingAcker::default()); + let mut f = fixture(AppAccess::Open, &[]); + f.state.authority_factory = seam_factory(seam, acker.clone()); + let space = format!("at://{DYNAMIC_AUTHORITY}/space/community.blacksky.feed/main"); + let token = service_jwt::mint( + &user_signer(), + MEMBER, + &format!("{DYNAMIC_AUTHORITY}#atproto_space_host"), + REGISTER_SPACE_LXM, + NOW, + "register-dyn".to_string(), + ) + .unwrap(); + + let (status, body) = send( + &f.state, + post_req( + &format!("/xrpc/{REGISTER_SPACE_LXM}"), + Some(&token), + serde_json::json!({"space": space, "generation": 1}), + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let context = f.state.registry.authority(DYNAMIC_AUTHORITY).unwrap(); + assert!(context.authority.resolve_registered(&space).is_ok()); + // The new authority signs with the key resolved from the actor store. + let expected = Signer::from_secret( + secp256k1::SecretKey::from_slice(&DYNAMIC_AUTHORITY_KEY).unwrap(), + ); + assert_eq!(context.authority.signer.did_key(), expected.did_key()); + let credential = context + .authority + .mint_credential(NOW, "dyn-jti".to_string(), "jkt") + .unwrap(); + credential::verify_space_credential( + &credential, + &space, + DYNAMIC_AUTHORITY, + expected.did_key(), + NOW, + ) + .unwrap(); + assert_eq!( + acker.0.lock().unwrap().as_slice(), + &[(space.clone(), 1)] + ); + assert_eq!( + f.state.hosted_spaces.hosted_spaces().await.unwrap(), + vec![(DYNAMIC_AUTHORITY.to_string(), space)] + ); + } + + #[tokio::test] + async fn registration_for_an_unresolvable_authority_fails_clean() { + let directory = dynamic_actor_store(); + let seam = Arc::new(crate::pds_seam::PdsSeam::open(directory.path()).unwrap()); + let mut f = fixture(AppAccess::Open, &[]); + f.state.authority_factory = seam_factory(seam, Arc::new(RecordingAcker::default())); + let space = "at://did:plc:keyless/space/community.blacksky.feed/main"; + let token = service_jwt::mint( + &user_signer(), + MEMBER, + "did:plc:keyless#atproto_space_host", + REGISTER_SPACE_LXM, + NOW, + "register-keyless".to_string(), + ) + .unwrap(); + + let (status, body) = send( + &f.state, + post_req( + &format!("/xrpc/{REGISTER_SPACE_LXM}"), + Some(&token), + serde_json::json!({"space": space, "generation": 1}), + ), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], "InvalidRequest"); + assert!(body["message"] + .as_str() + .unwrap() + .contains("did:plc:keyless")); + assert!(f.state.registry.authority("did:plc:keyless").is_err()); + assert!(f.state.hosted_spaces.hosted_spaces().await.unwrap().is_empty()); + } + #[tokio::test] async fn error_mapping_covers_every_lexicon_error_name() { for (err, status, name) in [ diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index b7877bcd..9a797066 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -1,4 +1,4 @@ -//! Space-host service entrypoint: parse config, wire the authority, policy, +//! Space-host service entrypoint: parse config, wire the authority registry, //! stores, and HTTP surface, and serve until shutdown. use clap::Parser; @@ -8,19 +8,21 @@ use rsky_oauth::dpop::{DpopManager, InMemoryReplayStore}; use rsky_space::space_id::SpaceId; use rsky_space_host::appaccess::AppAccess; use rsky_space_host::attestation::HttpMetadataFetcher; -use rsky_space_host::authority::{Authority, AuthorityContext, AuthorityRegistry}; +use rsky_space_host::authority::{ + Authority, AuthorityContext, AuthorityFactory, AuthorityRegistry, +}; use rsky_space_host::config::{Config, PolicyMode}; use rsky_space_host::http::{router, AppState, DEFAULT_REGISTRATION_TTL_SECS}; -use rsky_space_host::keys::{DocKeyResolver, ResolverDocSource}; +use rsky_space_host::keys::{DocKeyResolver, DocSource, ResolverDocSource}; use rsky_space_host::managing_app::HttpManagingApp; use rsky_space_host::membership::InMemoryMembership; use rsky_space_host::notify::HttpNotifier; use rsky_space_host::pds_seam::PdsSeam; use rsky_space_host::policy::Policy; -use rsky_space_host::registration::HttpLifecycleAcker; +use rsky_space_host::registration::{HttpLifecycleAcker, LifecycleAcker}; use rsky_space_host::repo::SqliteRepos; use rsky_space_host::signing::Signer; -use rsky_space_host::store::SqliteStore; +use rsky_space_host::store::{HostedSpaceStore, SqliteStore}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; @@ -40,6 +42,61 @@ async fn shutdown_signal() { tracing::info!("shutdown signal received"); } +/// Builds one authority's context from the shared host configuration. +struct ContextBuilder { + policy: PolicyMode, + managing_app: String, + members: Vec, + lifecycle_url: String, + lifecycle_service_did: String, + docs: Arc, + now: Arc u64 + Send + Sync>, + jti: Arc String + Send + Sync>, +} + +impl ContextBuilder { + fn context(&self, space: SpaceId, signer: Signer) -> AuthorityContext { + let authority_did = space.authority.clone(); + let policy = match self.policy { + PolicyMode::Public => Policy::Public, + PolicyMode::MemberList => { + Policy::MemberList(Arc::new(InMemoryMembership::new(self.members.clone()))) + } + PolicyMode::ManagingApp => Policy::ManagingApp { + service_id: self.managing_app.clone(), + client: Arc::new(HttpManagingApp::new( + self.managing_app.clone(), + authority_did.clone(), + signer.clone(), + self.docs.clone(), + self.now.clone(), + self.jti.clone(), + )), + }, + }; + AuthorityContext { + authority: Arc::new(Authority::new(space, signer.clone(), AppAccess::Open)), + policy: Arc::new(policy), + notifier: Arc::new(HttpNotifier::new( + authority_did.clone(), + signer.clone(), + self.now.clone(), + self.jti.clone(), + )), + lifecycle_acker: (self.policy == PolicyMode::ManagingApp).then(|| { + Arc::new(HttpLifecycleAcker::new( + self.lifecycle_url.clone(), + self.lifecycle_service_did.clone(), + authority_did, + signer, + self.now.clone(), + self.jti.clone(), + )) as Arc + }), + } + } +} + #[tokio::main] async fn main() -> Result<(), Box> { tracing_subscriber::fmt() @@ -50,13 +107,6 @@ async fn main() -> Result<(), Box> { let cfg = Config::parse(); cfg.validate()?; - let signer = Signer::from_hex(&cfg.signing_key_hex)?; - let space = SpaceId::new( - cfg.authority_did.clone(), - cfg.space_type().to_string(), - cfg.space_skey().to_string(), - ); - let authority = Authority::new(space, signer.clone(), AppAccess::Open); let now: Arc u64 + Send + Sync> = Arc::new(unix_now); let jti: Arc String + Send + Sync> = Arc::new(random_jti); @@ -65,50 +115,66 @@ async fn main() -> Result<(), Box> { plc_url: Some(cfg.plc_url.clone()), did_cache: std::sync::Arc::new(MemoryCache::new(None, None)), }))); - let policy = match cfg.policy { - PolicyMode::Public => Policy::Public, - PolicyMode::MemberList => { - Policy::MemberList(Arc::new(InMemoryMembership::new(cfg.member_dids()))) - } - PolicyMode::ManagingApp => Policy::ManagingApp { - service_id: cfg.managing_app.clone(), - client: Arc::new(HttpManagingApp::new( - cfg.managing_app.clone(), - cfg.authority_did.clone(), - signer.clone(), - docs.clone(), - now.clone(), - jti.clone(), - )), - }, - }; let store = Arc::new(SqliteStore::open(&cfg.db_path)?); let repos = Arc::new(SqliteRepos::open(&cfg.db_path)?); - let commit_signer = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); - let ticker = std::sync::Mutex::new(rsky_common::tid::Ticker::new()); + let seam = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); + + let builder = Arc::new(ContextBuilder { + policy: cfg.policy, + managing_app: cfg.managing_app.clone(), + members: cfg.member_dids(), + lifecycle_url: cfg.lifecycle_url.clone(), + lifecycle_service_did: cfg.lifecycle_service_did.clone(), + docs: docs.clone(), + now: now.clone(), + jti: jti.clone(), + }); let registry = Arc::new(AuthorityRegistry::new()); - registry.insert(Arc::new(AuthorityContext { - authority: Arc::new(authority), - policy: Arc::new(policy), - notifier: Arc::new(HttpNotifier::new( - cfg.authority_did.clone(), - signer.clone(), - now.clone(), - jti.clone(), - )), - lifecycle_acker: (cfg.policy == PolicyMode::ManagingApp).then(|| { - Arc::new(HttpLifecycleAcker::new( - cfg.lifecycle_url.clone(), - cfg.lifecycle_service_did.clone(), - cfg.authority_did.clone(), - signer, - now.clone(), - jti.clone(), - )) as Arc - }), - })); + if let Some((authority_did, signing_key_hex)) = cfg.bootstrap_pin() { + let signer = Signer::from_hex(signing_key_hex)?; + let space = SpaceId::new( + authority_did.to_string(), + cfg.space_type().to_string(), + cfg.space_skey().to_string(), + ); + registry.insert(Arc::new(builder.context(space, signer))); + } + let factory: AuthorityFactory = { + let builder = builder.clone(); + let seam = seam.clone(); + Arc::new(move |space: &SpaceId| { + let signer = seam.signer(&space.authority)?; + Ok(Arc::new(builder.context(space.clone(), signer))) + }) + }; + for (authority_did, space_uri) in store.hosted_spaces().await? { + let context = match registry.authority(&authority_did) { + Ok(context) => context, + Err(_) => { + let built = SpaceId::parse(&space_uri) + .map_err(|e| e.to_string()) + .and_then(|space| factory(&space).map_err(|e| e.to_string())); + match built { + Ok(context) => registry.insert_if_absent(context), + Err(error) => { + tracing::warn!(authority = %authority_did, space = %space_uri, error = %error, + "cannot re-serve persisted space"); + continue; + } + } + } + }; + if let Err(error) = context.authority.register(&space_uri) { + tracing::warn!(space = %space_uri, error = %error, + "cannot re-register persisted space"); + } + } + + let ticker = std::sync::Mutex::new(rsky_common::tid::Ticker::new()); let state = AppState { - registry, + registry: registry.clone(), + authority_factory: factory, + hosted_spaces: store.clone(), keys: Arc::new(DocKeyResolver::new(docs.clone())), docs, metadata: Arc::new(HttpMetadataFetcher::new()), @@ -124,7 +190,7 @@ async fn main() -> Result<(), Box> { jti, registration_ttl_secs: DEFAULT_REGISTRATION_TTL_SECS, repos, - commit_signer, + commit_signer: seam, auth: cfg.auth_config(), rev: Arc::new(move || ticker.lock().expect("ticker").next(None).to_string()), mint_token: cfg.mint_token.clone(), @@ -135,10 +201,9 @@ async fn main() -> Result<(), Box> { }; let listener = tokio::net::TcpListener::bind(&cfg.bind).await?; - let bootstrap = state.registry.authority(&cfg.authority_did)?; tracing::info!( - space = %bootstrap.authority.space_uri(), - authority_key = %bootstrap.authority.signer.did_key(), + authorities = registry.contexts().len(), + bootstrap = %cfg.authority_did, policy = ?cfg.policy, bind = %cfg.bind, db = %cfg.db_path, diff --git a/rsky-space-host/src/pds_seam.rs b/rsky-space-host/src/pds_seam.rs index be51ae76..79e8a39a 100644 --- a/rsky-space-host/src/pds_seam.rs +++ b/rsky-space-host/src/pds_seam.rs @@ -120,6 +120,10 @@ impl PdsSeam { ) } + pub fn signer(&self, author_did: &str) -> Result { + self.require_signer(author_did) + } + fn require_signer(&self, author_did: &str) -> Result { let Some(path) = self.key_path(author_did) else { return Err(HostError::InvalidRequest(format!( diff --git a/rsky-space-host/src/store.rs b/rsky-space-host/src/store.rs index e050b66a..5afdc802 100644 --- a/rsky-space-host/src/store.rs +++ b/rsky-space-host/src/store.rs @@ -68,6 +68,34 @@ pub trait RegistrationStore: Send + Sync { async fn endpoints(&self, space_uri: &str, now: u64) -> Result>; } +/// The spaces registered with this host (via `community.blacksky.space.register`), +/// grouped by authority, so a restart re-serves them. +#[async_trait] +pub trait HostedSpaceStore: Send + Sync { + async fn record_space(&self, authority_did: &str, space_uri: &str) -> Result<()>; + async fn hosted_spaces(&self) -> Result>; +} + +#[derive(Default)] +pub struct InMemoryHostedSpaces { + spaces: Mutex>, +} + +#[async_trait] +impl HostedSpaceStore for InMemoryHostedSpaces { + async fn record_space(&self, authority_did: &str, space_uri: &str) -> Result<()> { + self.spaces + .lock() + .unwrap() + .insert((authority_did.to_string(), space_uri.to_string()), ()); + Ok(()) + } + + async fn hosted_spaces(&self) -> Result> { + Ok(self.spaces.lock().unwrap().keys().cloned().collect()) + } +} + fn next_cursor(page: &[RepoRef], limit: u32) -> Option { if page.len() == limit as usize { page.last().map(|r| r.did.clone()) @@ -194,6 +222,11 @@ impl SqliteStore { CREATE TABLE IF NOT EXISTS used_jti ( jti TEXT PRIMARY KEY, exp INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS hosted_space ( + authority_did TEXT NOT NULL, + space_uri TEXT NOT NULL, + PRIMARY KEY (authority_did, space_uri) );", ) .map_err(sql_err)?; @@ -313,6 +346,36 @@ impl RegistrationStore for SqliteStore { } } +#[async_trait] +impl HostedSpaceStore for SqliteStore { + async fn record_space(&self, authority_did: &str, space_uri: &str) -> Result<()> { + self.conn + .lock() + .unwrap() + .execute( + "INSERT OR IGNORE INTO hosted_space (authority_did, space_uri) VALUES (?1, ?2)", + rusqlite::params![authority_did, space_uri], + ) + .map_err(sql_err)?; + Ok(()) + } + + async fn hosted_spaces(&self) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn + .prepare( + "SELECT authority_did, space_uri FROM hosted_space + ORDER BY authority_did ASC, space_uri ASC", + ) + .map_err(sql_err)?; + let rows = stmt + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .map_err(sql_err)?; + rows.collect::, _>>() + .map_err(sql_err) + } +} + #[async_trait] impl JtiStore for SqliteStore { async fn consume(&self, jti: &str, exp: u64) -> Result { @@ -451,6 +514,36 @@ mod tests { .unwrap()); } + async fn exercise_hosted_spaces(store: &dyn HostedSpaceStore) { + store.record_space("did:plc:auth", SPACE).await.unwrap(); + store.record_space("did:plc:auth", SPACE).await.unwrap(); + store + .record_space("did:plc:auth", OTHER_SPACE) + .await + .unwrap(); + store + .record_space("did:plc:other", "at://did:plc:other/space/t/main") + .await + .unwrap(); + assert_eq!( + store.hosted_spaces().await.unwrap(), + vec![ + ("did:plc:auth".to_string(), SPACE.to_string()), + ("did:plc:auth".to_string(), OTHER_SPACE.to_string()), + ( + "did:plc:other".to_string(), + "at://did:plc:other/space/t/main".to_string() + ), + ] + ); + } + + #[tokio::test] + async fn hosted_spaces_round_trip() { + exercise_hosted_spaces(&InMemoryHostedSpaces::default()).await; + exercise_hosted_spaces(&SqliteStore::open_in_memory().unwrap()).await; + } + #[tokio::test] async fn sqlite_persists_across_reopen() { let dir = tempfile::tempdir().unwrap(); From 20e44144eb95b847fafdea625f91d2b066daf4d4 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 16:02:20 -0400 Subject: [PATCH 18/56] feat(daemon): make the discovery authority filter optional --- Cargo.lock | 2 +- rsky-daemon/Cargo.toml | 2 +- rsky-daemon/src/config.rs | 32 ++++++++--- rsky-daemon/src/credentials.rs | 9 +-- rsky-daemon/src/main.rs | 9 +-- rsky-daemon/src/notify.rs | 101 +++++++++++++++++++++++++++------ rsky-daemon/src/spaces.rs | 55 +++++++++++++++--- 7 files changed, 161 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80af2458..1754ac52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8005,7 +8005,7 @@ dependencies = [ [[package]] name = "rsky-daemon" -version = "0.4.0" +version = "0.5.0" dependencies = [ "async-trait", "axum", diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index a45f389d..9409ef91 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-daemon" -version = "0.4.0" +version = "0.5.0" authors = ["Rudy Fraser "] description = "atproto permissioned-data syncer daemon: pulls, verifies, and indexes permissioned repos from members' PDSes" edition = "2021" diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index fa9a54e6..814b5d81 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -18,7 +18,8 @@ pub struct Config { /// Managing-app API key for dynamic space discovery. #[arg(long, env = "DAEMON_SPACES_API_KEY", default_value = "", hide_env_values = true)] pub spaces_api_key: String, - /// Authority whose spaces this daemon discovers. + /// Optional authority filter for dynamic discovery; unset, the daemon + /// accepts spaces from every authority the managing app serves. #[arg(long, env = "DAEMON_AUTHORITY_DID", default_value = "")] pub authority_did: String, /// Space type accepted from the managing app. @@ -101,16 +102,15 @@ impl Config { if self.space_uri.is_empty() && self.spaces_url.is_empty() { return Err("DAEMON_SPACE_URI or DAEMON_SPACES_URL is required".into()); } - if !self.spaces_url.is_empty() - && (self.spaces_api_key.is_empty() || self.authority_did.is_empty()) - { - return Err( - "DAEMON_SPACES_API_KEY and DAEMON_AUTHORITY_DID are required with DAEMON_SPACES_URL" - .into(), - ); + if !self.spaces_url.is_empty() && self.spaces_api_key.is_empty() { + return Err("DAEMON_SPACES_API_KEY is required with DAEMON_SPACES_URL".into()); } Ok(()) } + + pub fn authority_filter(&self) -> Option { + (!self.authority_did.is_empty()).then(|| self.authority_did.clone()) + } pub fn repo_host_url(&self) -> &str { if self.repo_host_url.is_empty() { &self.space_host_url @@ -209,6 +209,22 @@ mod tests { "--spaces-api-key", "key", "--authority-did", "did:plc:authority", ]).unwrap(); assert!(discovery_only.validate().is_ok()); + assert_eq!( + discovery_only.authority_filter().as_deref(), + Some("did:plc:authority") + ); + let all_authorities = Config::try_parse_from([ + "rsky-daemon", "--space-host-url", "https://host.example", + "--service-identity", "did:web:syncer.example", "--spaces-url", "https://feeds.example", + "--spaces-api-key", "key", + ]).unwrap(); + assert!(all_authorities.validate().is_ok()); + assert!(all_authorities.authority_filter().is_none()); + let keyless_discovery = Config::try_parse_from([ + "rsky-daemon", "--space-host-url", "https://host.example", + "--service-identity", "did:web:syncer.example", "--spaces-url", "https://feeds.example", + ]).unwrap(); + assert!(keyless_discovery.validate().is_err()); let neither = Config::try_parse_from([ "rsky-daemon", "--space-host-url", "https://host.example", "--service-identity", "did:web:syncer.example", diff --git a/rsky-daemon/src/credentials.rs b/rsky-daemon/src/credentials.rs index 053890ea..5b48ca75 100644 --- a/rsky-daemon/src/credentials.rs +++ b/rsky-daemon/src/credentials.rs @@ -81,7 +81,6 @@ impl CredentialSource for StaticCredential { pub struct InternalCredentialProvider { default_space: String, - authority_did: String, mint_token: String, issuer: ServiceJwtIssuer, host: Arc, @@ -105,14 +104,12 @@ impl CredentialSource for SpaceCredentialSource { impl InternalCredentialProvider { pub fn new( space: impl Into, - authority_did: impl Into, mint_token: impl Into, issuer: ServiceJwtIssuer, host: Arc, ) -> Self { Self { default_space: space.into(), - authority_did: authority_did.into(), mint_token: mint_token.into(), issuer, host, @@ -130,9 +127,8 @@ impl InternalCredentialProvider { return Ok(jwt.clone()); } } - let service_jwt = self - .issuer - .mint(&self.authority_did, now, &format!("mint-{now}"))?; + let authority = rsky_space::space_id::SpaceId::parse(space)?.authority; + let service_jwt = self.issuer.mint(&authority, now, &format!("mint-{now}"))?; let jwt = self .host .mint_internal_credential(space, &service_jwt, &self.mint_token) @@ -302,7 +298,6 @@ mod tests { let b = "at://did:plc:authority/space/community.blacksky.feed/other"; let provider = InternalCredentialProvider::new( a, - "did:plc:authority", "mint-token", ServiceJwtIssuer::from_hex("did:plc:daemon", &"11".repeat(32)).unwrap(), Arc::new(HttpSpaceHost::new( diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index d1204d54..39086352 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -13,7 +13,6 @@ use rsky_daemon::{ use rsky_identity::did::atproto_data::{get_did_key_from_multibase, VerificationMaterial}; use rsky_identity::types::{IdentityResolverOpts, MemoryCache}; use rsky_identity::IdResolver; -use rsky_space::space_id::SpaceId; use std::sync::Arc; use tokio::sync::{mpsc, watch}; @@ -74,7 +73,7 @@ async fn main() -> std::result::Result<(), Box> { let cfg = Config::parse(); cfg.validate()?; - let authority_did = if cfg.authority_did.is_empty() { SpaceId::parse(&cfg.space_uri)?.authority } else { cfg.authority_did.clone() }; + let authority_filter = cfg.authority_filter(); // One proof-of-possession key for the process: the credential it mints is // bound to it, and every host it is presented to checks that binding. if cfg.dpop_key_path.is_empty() { @@ -97,7 +96,6 @@ async fn main() -> std::result::Result<(), Box> { } Some(Arc::new(InternalCredentialProvider::new( &cfg.space_uri, - &authority_did, &cfg.space_host_mint_token, rsky_daemon::service_jwt::ServiceJwtIssuer::from_hex( &cfg.service_identity, @@ -111,7 +109,7 @@ async fn main() -> std::result::Result<(), Box> { let mut sources: Vec> = Vec::new(); if !cfg.space_uri.is_empty() { sources.push(Box::new(StaticSpaces::new([cfg.space_uri.clone()]))); } - if !cfg.spaces_url.is_empty() { sources.push(Box::new(HttpSpaceSource::new(&cfg.spaces_url, &cfg.spaces_api_key, &authority_did, &cfg.space_type))); } + if !cfg.spaces_url.is_empty() { sources.push(Box::new(HttpSpaceSource::new(&cfg.spaces_url, &cfg.spaces_api_key, authority_filter.clone(), &cfg.space_type))); } let source = Arc::new(CombinedSource(sources)); let registry = SpaceRegistry::new(); @@ -120,7 +118,6 @@ async fn main() -> std::result::Result<(), Box> { let notify_state = NotifyState { space_uri: cfg.space_uri.clone(), registry: registry.clone(), - authority_did: authority_did.clone(), service_identity: cfg.service_identity.clone(), resolver: keys.clone(), index: Arc::new(InMemoryIndex::new()), @@ -129,7 +126,7 @@ async fn main() -> std::result::Result<(), Box> { }; let listener = tokio::net::TcpListener::bind(&cfg.notify_bind).await?; tracing::info!( - authority = %authority_did, + authority = %authority_filter.as_deref().unwrap_or("(any)"), host = %cfg.space_host_url, notify_bind = %cfg.notify_bind, sweep_secs = cfg.sweep_interval_secs, diff --git a/rsky-daemon/src/notify.rs b/rsky-daemon/src/notify.rs index 56cd3f05..3aed0465 100644 --- a/rsky-daemon/src/notify.rs +++ b/rsky-daemon/src/notify.rs @@ -90,8 +90,6 @@ pub fn decode_claims(jwt: &str) -> Result { pub struct NotifyState { pub space_uri: String, pub registry: SpaceRegistry, - /// The authority (space host) DID whose key signs inbound notifications. - pub authority_did: String, /// This syncer's service identity: the required `aud` on inbound tokens. pub service_identity: String, pub resolver: Arc, @@ -114,7 +112,8 @@ fn error_body(error: &str, message: impl std::fmt::Display) -> Json { Json(json!({ "error": error, "message": message.to_string() })) } -async fn authenticate(headers: &HeaderMap, state: &NotifyState) -> Result<()> { +async fn authenticate(headers: &HeaderMap, state: &NotifyState, space_uri: &str) -> Result<()> { + let authority = rsky_space::space_id::SpaceId::parse(space_uri)?.authority; let jwt = headers .get(header::AUTHORIZATION) .and_then(|v| v.to_str().ok()) @@ -124,10 +123,10 @@ async fn authenticate(headers: &HeaderMap, state: &NotifyState) -> Result<()> { "missing bearer token".into(), )) })?; - let did_key = state.resolver.signing_key(&state.authority_did).await?; + let did_key = state.resolver.signing_key(&authority).await?; verify_service_auth( jwt, - &state.authority_did, + &authority, &state.service_identity, &did_key, (state.now_fn)(), @@ -139,18 +138,18 @@ async fn notify_write( headers: HeaderMap, Json(input): Json, ) -> (StatusCode, Json) { - if let Err(e) = authenticate(&headers, &state).await { - return ( - StatusCode::UNAUTHORIZED, - error_body("AuthenticationRequired", e), - ); - } if !state.registry.contains(&input.space) { return ( StatusCode::BAD_REQUEST, error_body("InvalidRequest", "space is not synced by this daemon"), ); } + if let Err(e) = authenticate(&headers, &state, &input.space).await { + return ( + StatusCode::UNAUTHORIZED, + error_body("AuthenticationRequired", e), + ); + } tracing::debug!(space = %input.space, repo = %input.repo, rev = %input.rev, "write notice"); if state.tx.send((input.space, input.repo)).await.is_err() { return ( @@ -166,18 +165,18 @@ async fn notify_space_deleted( headers: HeaderMap, Json(input): Json, ) -> (StatusCode, Json) { - if let Err(e) = authenticate(&headers, &state).await { - return ( - StatusCode::UNAUTHORIZED, - error_body("AuthenticationRequired", e), - ); - } if !state.registry.contains(&input.space) { return ( StatusCode::BAD_REQUEST, error_body("InvalidRequest", "space is not synced by this daemon"), ); } + if let Err(e) = authenticate(&headers, &state, &input.space).await { + return ( + StatusCode::UNAUTHORIZED, + error_body("AuthenticationRequired", e), + ); + } tracing::warn!(space = %input.space, "space deleted; purging all synced data"); if let Err(e) = state.index.purge_space().await { return ( @@ -246,7 +245,6 @@ mod tests { NotifyState { space_uri: SPACE.to_string(), registry: { let registry = SpaceRegistry::new(); registry.insert(SPACE); registry }, - authority_did: AUTHORITY.to_string(), service_identity: SYNCER.to_string(), resolver: Arc::new(FixedKey(did_key.to_string())), index, @@ -491,6 +489,73 @@ mod tests { idx.list_paths("d").await.unwrap(); } + #[tokio::test] + async fn notifications_verify_against_each_space_authority() { + use std::collections::BTreeMap; + struct KeyMap(BTreeMap); + #[async_trait] + impl CommitKeyResolver for KeyMap { + async fn signing_key(&self, did: &str) -> Result { + self.0 + .get(did) + .cloned() + .ok_or_else(|| DaemonError::KeyResolution(format!("unknown did {did}"))) + } + } + + let other_space = "at://did:plc:authority2/space/community.blacksky.feed/main"; + let (secret_a, key_a) = host_key(); + let secret_b = SecretKey::from_slice(&[0x66u8; 32]).unwrap(); + let key_b = rsky_crypto::utils::encode_did_key(&PublicKey::from_secret_key( + &Secp256k1::new(), + &secret_b, + )); + let registry = SpaceRegistry::new(); + registry.insert(SPACE); + registry.insert(other_space); + let (tx, mut rx) = mpsc::channel(4); + let app = router(NotifyState { + space_uri: SPACE.to_string(), + registry, + service_identity: SYNCER.to_string(), + resolver: Arc::new(KeyMap(BTreeMap::from([ + (AUTHORITY.to_string(), key_a), + ("did:plc:authority2".to_string(), key_b), + ]))), + index: Arc::new(InMemoryIndex::new()), + tx, + now_fn: fixed_now, + }); + + // Each authority's key opens only its own space's notifications. + let jwt_b = service_jwt(&secret_b, "did:plc:authority2", SYNCER, NOW + 60); + let resp = app + .clone() + .oneshot(request( + "/xrpc/com.atproto.space.notifyWrite", + Some(&jwt_b), + json!({ "space": other_space, "repo": "did:plc:w2", "rev": "3krev" }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + rx.recv().await.unwrap(), + (other_space.to_string(), "did:plc:w2".to_string()) + ); + + let jwt_a_for_b = service_jwt(&secret_a, AUTHORITY, SYNCER, NOW + 60); + let resp = app + .oneshot(request( + "/xrpc/com.atproto.space.notifyWrite", + Some(&jwt_a_for_b), + json!({ "space": other_space, "repo": "did:plc:w2", "rev": "3krev" }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + #[test] fn decode_claims_rejects_malformed_tokens() { assert!(decode_claims("one.two").is_err()); diff --git a/rsky-daemon/src/spaces.rs b/rsky-daemon/src/spaces.rs index fa63d656..cd99ebc5 100644 --- a/rsky-daemon/src/spaces.rs +++ b/rsky-daemon/src/spaces.rs @@ -49,7 +49,7 @@ impl SpaceSource for StaticSpaces { pub struct HttpSpaceSource { url: String, api_key: String, - authority_did: String, + authority_did: Option, space_type: String, http: reqwest::Client, } @@ -58,13 +58,13 @@ impl HttpSpaceSource { pub fn new( url: impl Into, api_key: impl Into, - authority_did: impl Into, + authority_did: Option, space_type: impl Into, ) -> Self { Self { url: url.into().trim_end_matches('/').into(), api_key: api_key.into(), - authority_did: authority_did.into(), + authority_did, space_type: space_type.into(), http: reqwest::Client::new(), } @@ -85,11 +85,14 @@ struct SyncableSpace { #[async_trait] impl SpaceSource for HttpSpaceSource { async fn spaces(&self) -> Result> { - let response = self + let mut request = self .http .get(format!("{}/admin/sync-spaces", self.url)) - .query(&[("authority", &self.authority_did)]) - .header("X-RSKY-KEY", &self.api_key) + .header("X-RSKY-KEY", &self.api_key); + if let Some(authority) = &self.authority_did { + request = request.query(&[("authority", authority)]); + } + let response = request .send() .await .map_err(|e| DaemonError::Xrpc(e.to_string()))?; @@ -113,7 +116,10 @@ impl SpaceSource for HttpSpaceSource { entry.state.as_str(), "host_registered" | "active" | "deleting" ) - && space.authority == self.authority_did + && self + .authority_did + .as_deref() + .is_none_or(|authority| space.authority == authority) && space.space_type == self.space_type) .then_some(( space.uri(), @@ -191,7 +197,12 @@ mod tests { const A: &str = "at://did:plc:c/space/community.blacksky.feed/a"; const B: &str = "at://did:plc:c/space/community.blacksky.feed/b"; fn source(url: String) -> HttpSpaceSource { - HttpSpaceSource::new(url, "key", "did:plc:c", "community.blacksky.feed") + HttpSpaceSource::new( + url, + "key", + Some("did:plc:c".to_string()), + "community.blacksky.feed", + ) } #[tokio::test] async fn filters_and_reads_managing_app_spaces() { @@ -199,6 +210,34 @@ mod tests { Mock::given(method("GET")).and(path("/admin/sync-spaces")).and(query_param("authority", "did:plc:c")).and(header("X-RSKY-KEY", "key")).respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"spaces":[{"space":A,"generation":2,"state":"active"},{"space":B,"generation":3,"state":"deleting"},{"space":"at://did:plc:other/space/community.blacksky.feed/x","generation":1,"state":"active"}]}))).mount(&server).await; assert_eq!(source(server.uri()).spaces().await.unwrap().len(), 2); } + #[tokio::test] + async fn without_an_authority_filter_all_authorities_are_discovered() { + use wiremock::matchers::query_param_is_missing; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/admin/sync-spaces")) + .and(query_param_is_missing("authority")) + .and(header("X-RSKY-KEY", "key")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "spaces": [ + {"space": A, "generation": 2, "state": "active"}, + {"space": "at://did:plc:other/space/community.blacksky.feed/x", "generation": 1, "state": "active"}, + {"space": "at://did:plc:other/space/wrong.type/y", "generation": 1, "state": "active"}, + {"space": "at://did:plc:third/space/community.blacksky.feed/z", "generation": 1, "state": "retired"}, + ] + }))) + .mount(&server) + .await; + let spaces = HttpSpaceSource::new(server.uri(), "key", None, "community.blacksky.feed") + .spaces() + .await + .unwrap(); + assert_eq!( + spaces.keys().collect::>(), + vec![A, "at://did:plc:other/space/community.blacksky.feed/x"] + ); + } + #[tokio::test] async fn pin_survives_source_outage() { let server = MockServer::start().await; From d142998a23d159455eed2e0789ec288da5c25f77 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 16:58:53 -0400 Subject: [PATCH 19/56] fix(space-host): accept ES256K DPoP proofs --- Cargo.lock | 4 +- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/oauth.rs | 131 +++++++++++++++++++++++++++++++++-- rsky-space/Cargo.toml | 2 +- rsky-space/src/jwk.rs | 124 +++++++++++++++++++++++++++++++-- 5 files changed, 249 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1754ac52..efb80d50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8395,7 +8395,7 @@ dependencies = [ [[package]] name = "rsky-space" -version = "0.4.1" +version = "0.4.2" dependencies = [ "base64 0.22.1", "blake3", @@ -8422,7 +8422,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.0" +version = "0.7.1" dependencies = [ "async-trait", "axum", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index e833f581..ad24a37e 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.0" +version = "0.7.1" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/oauth.rs b/rsky-space-host/src/oauth.rs index ed846e35..ff141dd9 100644 --- a/rsky-space-host/src/oauth.rs +++ b/rsky-space-host/src/oauth.rs @@ -17,7 +17,7 @@ use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; -use rsky_space::jwk::{verify_es256, EcJwk, JwkSet}; +use rsky_space::jwk::{verify_es256, verify_es256k, EcJwk, JwkSet}; use serde::Deserialize; use sha2::{Digest, Sha256}; @@ -32,7 +32,8 @@ pub const HS256: &str = "HS256"; /// What an access token may be signed with. `HS256` is not a weakening: it is /// what a standalone PDS actually issues (see `verify_as_signature`). pub const SUPPORTED_TOKEN_ALGS: [&str; 2] = ["ES256", "HS256"]; -pub const SUPPORTED_PROOF_ALGS: [&str; 1] = ["ES256"]; +pub const ES256K: &str = "ES256K"; +pub const SUPPORTED_PROOF_ALGS: [&str; 2] = ["ES256", ES256K]; /// How long a DPoP proof stays acceptable after its `iat`. pub const MAX_DPOP_AGE_SECS: u64 = 300; @@ -353,8 +354,11 @@ async fn verify_dpop_proof( .jwk .as_ref() .ok_or_else(|| auth_err("proof carries no jwk"))?; - verify_es256(jwk, &decoded.signing_input, &decoded.signature) - .map_err(|e| auth_err(format!("proof signature: {e}")))?; + let verified = match decoded.header.alg.as_str() { + ES256K => verify_es256k(jwk, &decoded.signing_input, &decoded.signature), + _ => verify_es256(jwk, &decoded.signing_input, &decoded.signature), + }; + verified.map_err(|e| auth_err(format!("proof signature: {e}")))?; let claims = &decoded.claims; if !claims.htm.eq_ignore_ascii_case(request.method) { @@ -528,6 +532,125 @@ pub(crate) mod tests { check_at(token, proof, "POST", URL, NOW, &InMemoryJtiStore::default()).await } + pub(crate) fn k1_client_key() -> secp256k1::SecretKey { + secp256k1::SecretKey::from_slice(&[0x33u8; 32]).unwrap() + } + + pub(crate) fn k1_jwk_of(key: &secp256k1::SecretKey) -> EcJwk { + let point = key + .public_key(secp256k1::SECP256K1) + .serialize_uncompressed(); + EcJwk { + kty: "EC".to_string(), + crv: "secp256k1".to_string(), + x: URL_SAFE_NO_PAD.encode(&point[1..33]), + y: URL_SAFE_NO_PAD.encode(&point[33..65]), + kid: None, + } + } + + pub(crate) fn k1_sign_jwt( + key: &secp256k1::SecretKey, + header: serde_json::Value, + claims: serde_json::Value, + ) -> String { + let header = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).unwrap()); + let claims = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let input = format!("{header}.{claims}"); + let digest = Sha256::digest(input.as_bytes()); + let message = secp256k1::Message::from_digest_slice(&digest).unwrap(); + let sig = secp256k1::SECP256K1.sign_ecdsa(&message, key); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(sig.serialize_compact()) + ) + } + + /// A proof whose header names `jwk_key` but whose signature is made by + /// `signing_key`; equal keys give an ordinary well-formed proof. + fn k1_proof( + jwk_key: &secp256k1::SecretKey, + signing_key: &secp256k1::SecretKey, + claims: serde_json::Value, + ) -> String { + k1_sign_jwt( + signing_key, + json!({"typ": DPOP_TYP, "alg": ES256K, "jwk": k1_jwk_of(jwk_key)}), + claims, + ) + } + + fn token_bound_to(jkt: String) -> String { + let mut claims = token_claims(); + claims["cnf"] = json!({ "jkt": jkt }); + token_with(claims) + } + + #[tokio::test] + async fn accepts_an_es256k_dpop_proof() { + let key = k1_client_key(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_proof(&key, &key, proof_claims(&token)); + let context = check(&token, &proof).await.unwrap(); + assert_eq!(context.did, AUTHOR); + assert_eq!(context.jkt, jwk_thumbprint(&k1_jwk_of(&key))); + } + + #[tokio::test] + async fn rejects_an_es256k_proof_signed_by_another_key() { + let key = k1_client_key(); + let impostor = secp256k1::SecretKey::from_slice(&[0x34u8; 32]).unwrap(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_proof(&key, &impostor, proof_claims(&token)); + assert!(check(&token, &proof).await.is_err()); + } + + /// The bound key is the one the token names, not whichever key presents a + /// self-consistent proof. + #[tokio::test] + async fn rejects_an_es256k_proof_for_an_unbound_key() { + let bound = k1_client_key(); + let other = secp256k1::SecretKey::from_slice(&[0x35u8; 32]).unwrap(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&bound))); + let proof = k1_proof(&other, &other, proof_claims(&token)); + assert!(check(&token, &proof).await.is_err()); + } + + #[tokio::test] + async fn rejects_an_es256k_proof_carrying_a_p256_key() { + let token = token(); + let proof = sign_jwt( + &client_key(), + json!({"typ": DPOP_TYP, "alg": ES256K, "jwk": jwk_of(&client_key(), None)}), + proof_claims(&token), + ); + assert!(check(&token, &proof).await.is_err()); + } + + #[tokio::test] + async fn rejects_an_es256_proof_carrying_a_secp256k1_key() { + let key = k1_client_key(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_sign_jwt( + &key, + json!({"typ": DPOP_TYP, "alg": "ES256", "jwk": k1_jwk_of(&key)}), + proof_claims(&token), + ); + assert!(check(&token, &proof).await.is_err()); + } + + #[tokio::test] + async fn rejects_an_unsupported_proof_alg() { + let key = k1_client_key(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_sign_jwt( + &key, + json!({"typ": DPOP_TYP, "alg": "ES512", "jwk": k1_jwk_of(&key)}), + proof_claims(&token), + ); + assert!(check(&token, &proof).await.is_err()); + } + const HS_SECRET: &[u8] = b"a-pds-jwt-secret"; fn hs256_token(claims: serde_json::Value) -> String { diff --git a/rsky-space/Cargo.toml b/rsky-space/Cargo.toml index a639209b..921a8c5c 100644 --- a/rsky-space/Cargo.toml +++ b/rsky-space/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space" -version = "0.4.1" +version = "0.4.2" authors = ["Rudy Fraser "] description = "atproto permissioned-data (spaces) primitives: LtHash commits, credentials, space repos" edition = "2021" diff --git a/rsky-space/src/jwk.rs b/rsky-space/src/jwk.rs index 13858e2f..6df49335 100644 --- a/rsky-space/src/jwk.rs +++ b/rsky-space/src/jwk.rs @@ -1,20 +1,27 @@ -//! Minimal EC JWK (P-256) support for verifying ES256-signed JWTs +//! Minimal EC JWK support for verifying ES256- and ES256K-signed JWTs //! (proposal §Client attestation). //! //! A space authority verifies a client attestation by resolving the client's //! published JWKS and checking the JWT signature against the key named by the -//! attestation's `kid`. Only `kty: "EC"` / `crv: "P-256"` keys are supported, -//! matching the attestation's required `alg: "ES256"`. +//! attestation's `kid`. Only `kty: "EC"` keys on the `P-256` and `secp256k1` +//! curves are supported. Each verifier requires its own curve, so a key can +//! never be verified under an algorithm it was not issued for. use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use crate::error::{Result, SpaceError}; const COORD_LEN: usize = 32; -/// An EC public JWK restricted to the P-256 curve. +/// The JWK `crv` of an ES256 key. +pub const CRV_P256: &str = "P-256"; +/// The JWK `crv` of an ES256K key. +pub const CRV_SECP256K1: &str = "secp256k1"; + +/// An EC public JWK on a supported curve. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct EcJwk { pub kty: String, @@ -42,7 +49,7 @@ impl EcJwk { self.kty ))); } - if self.crv != "P-256" { + if self.crv != CRV_P256 && self.crv != CRV_SECP256K1 { return Err(SpaceError::InvalidJwk(format!( "unsupported crv: {}", self.crv @@ -51,6 +58,18 @@ impl EcJwk { Ok(()) } + /// Validate the key and require one specific curve. + pub fn require_crv(&self, crv: &str) -> Result<()> { + self.validate()?; + if self.crv != crv { + return Err(SpaceError::InvalidJwk(format!( + "expected crv {crv}, got {}", + self.crv + ))); + } + Ok(()) + } + /// The SEC1 uncompressed point: `0x04 || x || y`. pub fn sec1_point(&self) -> Result> { let x = decode_coord(&self.x, "x")?; @@ -82,7 +101,7 @@ fn decode_coord(b64: &str, name: &str) -> Result> { /// `sig` must be the compact 64-byte low-S `r || s` encoding; anything else is /// rejected as a bad signature. pub fn verify_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { - jwk.validate()?; + jwk.require_crv(CRV_P256)?; let point = jwk.sec1_point()?; let ok = rsky_crypto::p256::operations::verify_sig(&point, signing_input, sig, None) .map_err(|e| SpaceError::Crypto(e.to_string()))?; @@ -93,6 +112,24 @@ pub fn verify_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> } } +/// Verify an ES256K signature over a JWT signing input +/// (`header_b64.payload_b64` bytes) against a secp256k1 JWK. +/// +/// `sig` must be the compact 64-byte low-S `r || s` encoding; anything else is +/// rejected as a bad signature. +pub fn verify_es256k(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { + jwk.require_crv(CRV_SECP256K1)?; + let point = jwk.sec1_point()?; + let digest = Sha256::digest(signing_input); + let ok = rsky_crypto::secp256k1::operations::verify_sig(&point, &digest, sig, None) + .map_err(|e| SpaceError::Crypto(e.to_string()))?; + if ok { + Ok(()) + } else { + Err(SpaceError::BadSignature) + } +} + /// A JWKS document (`jwks` / `jwks_uri` in client metadata). #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct JwkSet { @@ -255,6 +292,81 @@ mod tests { )); } + fn k1_key() -> secp256k1::SecretKey { + secp256k1::SecretKey::from_slice(&[0x63u8; 32]).unwrap() + } + + fn k1_jwk_for(key: &secp256k1::SecretKey) -> EcJwk { + let secp = secp256k1::Secp256k1::new(); + let point = key.public_key(&secp).serialize_uncompressed(); + EcJwk { + kty: "EC".to_string(), + crv: CRV_SECP256K1.to_string(), + x: URL_SAFE_NO_PAD.encode(&point[1..33]), + y: URL_SAFE_NO_PAD.encode(&point[33..65]), + kid: None, + } + } + + fn k1_sign(key: &secp256k1::SecretKey, input: &[u8]) -> Vec { + let secp = secp256k1::Secp256k1::new(); + let digest = Sha256::digest(input); + let message = secp256k1::Message::from_digest_slice(&digest).unwrap(); + secp.sign_ecdsa(&message, key).serialize_compact().to_vec() + } + + #[test] + fn es256k_verify_roundtrip() { + let key = k1_key(); + verify_es256k(&k1_jwk_for(&key), INPUT, &k1_sign(&key, INPUT)).unwrap(); + } + + #[test] + fn es256k_wrong_key_rejected() { + let sig = k1_sign(&k1_key(), INPUT); + let other = secp256k1::SecretKey::from_slice(&[0x64u8; 32]).unwrap(); + assert!(matches!( + verify_es256k(&k1_jwk_for(&other), INPUT, &sig), + Err(SpaceError::BadSignature) + )); + } + + #[test] + fn es256k_tampered_input_rejected() { + let key = k1_key(); + let sig = k1_sign(&key, INPUT); + let mut tampered = INPUT.to_vec(); + tampered[0] ^= 0xFF; + assert!(matches!( + verify_es256k(&k1_jwk_for(&key), &tampered, &sig), + Err(SpaceError::BadSignature) + )); + } + + #[test] + fn curve_and_algorithm_must_agree() { + let k1 = k1_key(); + let k1_sig = k1_sign(&k1, INPUT); + assert!(matches!( + verify_es256(&k1_jwk_for(&k1), INPUT, &k1_sig), + Err(SpaceError::InvalidJwk(msg)) if msg.contains("crv") + )); + + let p = signing_key(); + let p_sig = sign(&p, INPUT); + assert!(matches!( + verify_es256k(&jwk_for(&p, None), INPUT, &p_sig), + Err(SpaceError::InvalidJwk(msg)) if msg.contains("crv") + )); + } + + #[test] + fn es256k_jwk_survives_json_roundtrip() { + let jwk = k1_jwk_for(&k1_key()); + let json = serde_json::to_string(&jwk).unwrap(); + assert_eq!(EcJwk::from_json(&json).unwrap(), jwk); + } + #[test] fn jwk_set_find_by_kid() { let key = signing_key(); From b4b8ec159033e429b4f98d335e6f28f55e2c4feb Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:09:53 -0400 Subject: [PATCH 20/56] fix(daemon): resolve commit keys against the configured plc --- Cargo.lock | 2 +- rsky-daemon/Cargo.toml | 2 +- rsky-daemon/src/config.rs | 12 ++++++++++++ rsky-daemon/src/main.rs | 6 +++--- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efb80d50..3edc04ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8005,7 +8005,7 @@ dependencies = [ [[package]] name = "rsky-daemon" -version = "0.5.0" +version = "0.5.1" dependencies = [ "async-trait", "axum", diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index 9409ef91..817ea083 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-daemon" -version = "0.5.0" +version = "0.5.1" authors = ["Rudy Fraser "] description = "atproto permissioned-data syncer daemon: pulls, verifies, and indexes permissioned repos from members' PDSes" edition = "2021" diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index 814b5d81..4b679f1f 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -95,6 +95,11 @@ pub struct Config { /// Seconds between writer-set sweeps (self-healing when notifications drop). #[arg(long, env = "DAEMON_SWEEP_INTERVAL_SECS", default_value_t = 300)] pub sweep_interval_secs: u64, + + /// PLC directory to resolve DIDs against. Empty uses the public one, + /// which cannot know about a local or staging network. + #[arg(long, env = "DAEMON_PLC_URL", default_value = "")] + pub plc_url: String, } impl Config { @@ -111,6 +116,10 @@ impl Config { pub fn authority_filter(&self) -> Option { (!self.authority_did.is_empty()).then(|| self.authority_did.clone()) } + + pub fn plc_url(&self) -> Option { + (!self.plc_url.is_empty()).then(|| self.plc_url.clone()) + } pub fn repo_host_url(&self) -> &str { if self.repo_host_url.is_empty() { &self.space_host_url @@ -159,6 +168,7 @@ mod tests { assert_eq!(cfg.pds_url, ""); assert_eq!(cfg.pds_access_token, ""); assert_eq!(cfg.static_credential, ""); + assert_eq!(cfg.plc_url(), None); let mut cfg = Config::try_parse_from(REQUIRED.into_iter().chain([ "--repo-host-url", @@ -184,6 +194,7 @@ mod tests { ("DAEMON_NOTIFY_BIND", "0.0.0.0:9000"), ("DAEMON_INDEX_DB_PATH", "/data/space.sqlite"), ("DAEMON_SWEEP_INTERVAL_SECS", "60"), + ("DAEMON_PLC_URL", "http://localhost:2582"), ]; for (k, v) in env { std::env::set_var(k, v); @@ -198,6 +209,7 @@ mod tests { assert_eq!(cfg.pds_access_token, "access.jwt"); assert_eq!(cfg.static_credential, "sc.jwt"); assert_eq!(cfg.notify_bind, "0.0.0.0:9000"); + assert_eq!(cfg.plc_url(), Some("http://localhost:2582".to_string())); assert_eq!(cfg.notify_endpoint(), "http://0.0.0.0:9000"); assert_eq!(cfg.index_db_path, "/data/space.sqlite"); assert_eq!(cfg.sweep_interval_secs, 60); diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 39086352..e48561ba 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -22,11 +22,11 @@ struct DidKeyResolver { } impl DidKeyResolver { - fn new() -> Self { + fn new(plc_url: Option) -> Self { Self { resolver: tokio::sync::Mutex::new(IdResolver::new(IdentityResolverOpts { timeout: None, - plc_url: None, + plc_url, did_cache: Some(std::sync::Arc::new(MemoryCache::new(None, None))), backup_nameservers: None, })), @@ -83,7 +83,7 @@ async fn main() -> std::result::Result<(), Box> { &cfg.dpop_key_path, )?); let host = Arc::new(HttpSpaceHost::new(&cfg.space_host_url, dpop.clone())); - let keys: Arc = Arc::new(DidKeyResolver::new()); + let keys: Arc = Arc::new(DidKeyResolver::new(cfg.plc_url())); let db = if cfg.index_db_path.is_empty() { None } else { Some(Arc::new(SqliteIndex::open(&cfg.index_db_path)?)) }; From 7ed7f7b5507df20df2d6d540e667a8fe3eea3482 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:34:22 -0400 Subject: [PATCH 21/56] feat(daemon): journal synced batches for per-projector delivery Adds an IndexMutation journal written alongside every synced batch, independent per-projector cursors with dead-lettering, the Projector trait, and typed routing from index mutations to sync events. --- rsky-daemon/src/engine.rs | 29 ++- rsky-daemon/src/error.rs | 11 ++ rsky-daemon/src/index.rs | 183 +++++++++++++++++ rsky-daemon/src/journal.rs | 285 ++++++++++++++++++++++++++ rsky-daemon/src/lib.rs | 8 +- rsky-daemon/src/projection.rs | 17 ++ rsky-daemon/src/recovery.rs | 12 +- rsky-daemon/src/router.rs | 341 ++++++++++++++++++++++++++++++++ rsky-daemon/src/sqlite_index.rs | 247 ++++++++++++++++++++++- 9 files changed, 1121 insertions(+), 12 deletions(-) create mode 100644 rsky-daemon/src/journal.rs create mode 100644 rsky-daemon/src/projection.rs create mode 100644 rsky-daemon/src/router.rs diff --git a/rsky-daemon/src/engine.rs b/rsky-daemon/src/engine.rs index fce0e37e..3960aa8d 100644 --- a/rsky-daemon/src/engine.rs +++ b/rsky-daemon/src/engine.rs @@ -11,7 +11,7 @@ use rsky_space::commit::verify_commit; use rsky_space::lthash::element; use crate::error::{DaemonError, Result}; -use crate::index::SpaceIndex; +use crate::index::{IndexMutation, SpaceIndex}; use crate::repohost::RepoHostClient; /// Resolves an author's atproto signing `did:key` to verify their commit. @@ -49,6 +49,7 @@ pub async fn sync_repo( let mut prev_mismatches = 0usize; let mut cursor: Option = None; let mut last_rev: Option = None; + let mut mutations: Vec = Vec::new(); loop { let page = client @@ -70,20 +71,25 @@ pub async fn sync_repo( } match &op.cid { Some(cid) => { + let value = op.value.as_ref().map(|v| v.to_vec()); index - .upsert( - did, - &op.collection, - &op.rkey, - cid, - &op.rev, - op.value.as_ref().map(|v| v.to_vec()), - ) + .upsert(did, &op.collection, &op.rkey, cid, &op.rev, value.clone()) .await?; lth.add(&element(&op.collection, &op.rkey, cid)); + mutations.push(IndexMutation::Upsert { + collection: op.collection.clone(), + rkey: op.rkey.clone(), + cid: cid.clone(), + rev: op.rev.clone(), + value, + }); } None => { index.delete(did, &op.collection, &op.rkey).await?; + mutations.push(IndexMutation::Delete { + collection: op.collection.clone(), + rkey: op.rkey.clone(), + }); } } ops_applied += 1; @@ -109,6 +115,7 @@ pub async fn sync_repo( if lth.hash().as_slice() != commit.hash.as_slice() { return Err(DaemonError::Diverged(did.to_string())); } + index.journal_batch(did, &commit.rev, &mutations).await?; index.save_head(did, &commit.rev, <h).await?; return Ok(SyncOutcome { ops_applied, @@ -126,6 +133,10 @@ pub async fn sync_repo( // Exhausted the oplog without a terminal commit: advance to the last op's // rev without a hash check; a later sync carries the commit. if let Some(rev) = &last_rev { + // The head advances here without a commit to check it against, so a + // later sync will never replay these ops: journal them now or the + // projection loses them outright. + index.journal_batch(did, rev, &mutations).await?; index.save_head(did, rev, <h).await?; } Ok(SyncOutcome { diff --git a/rsky-daemon/src/error.rs b/rsky-daemon/src/error.rs index 608772c3..b09f4f62 100644 --- a/rsky-daemon/src/error.rs +++ b/rsky-daemon/src/error.rs @@ -16,8 +16,19 @@ pub enum DaemonError { /// full-state recovery (`getRepo`). #[error("history unavailable: {0}")] HistoryUnavailable(String), + /// A projection destination was unreachable or overloaded. Distinct from + /// [`DaemonError::Xrpc`] because it must not consume the batch's failure + /// budget: an outage is not a bad batch. + #[error("projection destination unavailable: {0}")] + RetryableProjection(String), #[error(transparent)] Space(#[from] rsky_space::SpaceError), } +impl DaemonError { + pub fn is_retryable_projection(&self) -> bool { + matches!(self, Self::RetryableProjection(_)) + } +} + pub type Result = std::result::Result; diff --git a/rsky-daemon/src/index.rs b/rsky-daemon/src/index.rs index eac02a41..2b1543bd 100644 --- a/rsky-daemon/src/index.rs +++ b/rsky-daemon/src/index.rs @@ -17,6 +17,43 @@ pub struct IndexedRecord { pub value: Option>, } +/// One record change from a synced batch, as journaled for projection. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum IndexMutation { + Upsert { + collection: String, + rkey: String, + cid: String, + rev: String, + value: Option>, + }, + Delete { + collection: String, + rkey: String, + }, +} + +impl IndexMutation { + pub fn collection(&self) -> &str { + match self { + Self::Upsert { collection, .. } | Self::Delete { collection, .. } => collection, + } + } + pub fn rkey(&self) -> &str { + match self { + Self::Upsert { rkey, .. } | Self::Delete { rkey, .. } => rkey, + } + } +} + +/// A journaled batch a projector has not yet delivered. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JournaledBatch { + pub author: String, + pub rev: String, + pub mutations: Vec, +} + /// Per-author sync state + records the daemon holds for a space. #[async_trait] pub trait SpaceIndex: Send + Sync { @@ -40,6 +77,47 @@ pub trait SpaceIndex: Send + Sync { async fn delete(&self, did: &str, collection: &str, rkey: &str) -> Result<()>; /// Persist the author's new head (rev + accumulator) after a synced batch. async fn save_head(&self, did: &str, rev: &str, lthash: &LtHash) -> Result<()>; + /// Record a synced batch for later projection, keyed by `(did, rev)`. + /// Journaling precedes the head write, so a crash between the two replays + /// the batch instead of losing it; the key makes that replay a no-op. + async fn journal_batch( + &self, + _did: &str, + _rev: &str, + _mutations: &[IndexMutation], + ) -> Result<()> { + Ok(()) + } + /// Batches past this projector's independent `(author, rev)` cursor. + async fn pending_batches(&self, _projector: &str) -> Result> { + Ok(Vec::new()) + } + async fn advance_projector_cursor( + &self, + _projector: &str, + _did: &str, + _rev: &str, + ) -> Result<()> { + Ok(()) + } + /// Returns the durable failure count for this batch, marking it + /// dead-lettered once the count reaches `dead_letter_after`. + async fn record_projection_failure( + &self, + _projector: &str, + _did: &str, + _rev: &str, + _error: &str, + _dead_letter_after: u32, + ) -> Result { + Ok(0) + } + /// Drop journal rows every one of `projectors` has advanced past, along + /// with their retryable failure rows. Dead-lettered batches are retained + /// until explicitly cleared. + async fn prune_journal(&self, _projectors: &[&str]) -> Result { + Ok(0) + } /// Enumerate an author's indexed records as `(collection, rkey, cid)`, /// used to diff against a recovered full-state CAR. async fn list_paths(&self, did: &str) -> Result>; @@ -59,10 +137,18 @@ struct AuthorState { records: HashMap, } +#[derive(Default)] +struct JournalState { + batches: Vec, + cursors: HashMap<(String, String), String>, + failures: HashMap<(String, String, String), (u32, bool)>, +} + /// In-memory [`SpaceIndex`] for tests and local runs. #[derive(Default)] pub struct InMemoryIndex { authors: RwLock>, + journal: RwLock, } impl InMemoryIndex { @@ -156,6 +242,102 @@ impl SpaceIndex for InMemoryIndex { Ok(()) } + async fn journal_batch(&self, did: &str, rev: &str, mutations: &[IndexMutation]) -> Result<()> { + let mut journal = self.journal.write().unwrap(); + if journal + .batches + .iter() + .any(|b| b.author == did && b.rev == rev) + { + return Ok(()); + } + journal.batches.push(JournaledBatch { + author: did.to_string(), + rev: rev.to_string(), + mutations: mutations.to_vec(), + }); + Ok(()) + } + + async fn pending_batches(&self, projector: &str) -> Result> { + let journal = self.journal.read().unwrap(); + let mut pending: Vec = journal + .batches + .iter() + .filter(|b| { + journal + .cursors + .get(&(projector.to_string(), b.author.clone())) + .is_none_or(|cursor| b.rev > *cursor) + && !journal + .failures + .get(&(projector.to_string(), b.author.clone(), b.rev.clone())) + .is_some_and(|(_, dead)| *dead) + }) + .cloned() + .collect(); + pending.sort_by(|a, b| (&a.author, &a.rev).cmp(&(&b.author, &b.rev))); + Ok(pending) + } + + async fn advance_projector_cursor(&self, projector: &str, did: &str, rev: &str) -> Result<()> { + self.journal + .write() + .unwrap() + .cursors + .insert((projector.to_string(), did.to_string()), rev.to_string()); + Ok(()) + } + + async fn record_projection_failure( + &self, + projector: &str, + did: &str, + rev: &str, + _error: &str, + dead_letter_after: u32, + ) -> Result { + let mut journal = self.journal.write().unwrap(); + let entry = journal + .failures + .entry((projector.to_string(), did.to_string(), rev.to_string())) + .or_insert((0, false)); + entry.0 += 1; + entry.1 = entry.0 >= dead_letter_after; + Ok(entry.0) + } + + async fn prune_journal(&self, projectors: &[&str]) -> Result { + if projectors.is_empty() { + return Ok(0); + } + let mut journal = self.journal.write().unwrap(); + let JournalState { + batches, + cursors, + failures, + } = &mut *journal; + let before = batches.len(); + batches.retain(|b| { + let all_passed = projectors.iter().all(|projector| { + cursors + .get(&(projector.to_string(), b.author.clone())) + .is_some_and(|cursor| *cursor >= b.rev) + }); + let dead_lettered = projectors.iter().any(|projector| { + failures + .get(&(projector.to_string(), b.author.clone(), b.rev.clone())) + .is_some_and(|(_, dead)| *dead) + }); + !all_passed || dead_lettered + }); + let pruned = before - batches.len(); + failures.retain(|(_, author, rev), (_, dead)| { + *dead || batches.iter().any(|b| b.author == *author && b.rev == *rev) + }); + Ok(pruned) + } + async fn list_paths(&self, did: &str) -> Result> { Ok(self .authors @@ -176,6 +358,7 @@ impl SpaceIndex for InMemoryIndex { async fn purge_space(&self) -> Result<()> { self.authors.write().unwrap().clear(); + *self.journal.write().unwrap() = JournalState::default(); Ok(()) } } diff --git a/rsky-daemon/src/journal.rs b/rsky-daemon/src/journal.rs new file mode 100644 index 00000000..e66d8fba --- /dev/null +++ b/rsky-daemon/src/journal.rs @@ -0,0 +1,285 @@ +//! Journal-driven projection delivery. + +use std::sync::Arc; + +use crate::error::Result; +use crate::index::SpaceIndex; +use crate::projection::Projector; +use crate::router::Router; + +pub const DEAD_LETTER_AFTER: u32 = 3; + +/// Reads the batch journal for one projector and advances only after its +/// destination has accepted the batch. Each projector gets an independent +/// cursor, so a failed destination cannot hold the index or another +/// destination hostage. +pub struct JournalConsumer { + name: &'static str, + router: Router, + projector: Box, + dead_letter_after: u32, +} + +impl JournalConsumer { + pub fn new(router: Router, projector: Box) -> Self { + Self { + name: projector.name(), + router, + projector, + dead_letter_after: DEAD_LETTER_AFTER, + } + } + + pub fn with_dead_letter_after(mut self, attempts: u32) -> Self { + self.dead_letter_after = attempts; + self + } + + pub fn name(&self) -> &'static str { + self.name + } + + async fn drain_with_status(&self, index: &dyn SpaceIndex) -> Result<(usize, bool)> { + let mut delivered = 0; + let mut succeeded = true; + for batch in index.pending_batches(self.name).await? { + let events = self.router.route_batch(&batch.author, &batch.mutations); + let result = if events.is_empty() { + Ok(()) + } else { + self.projector + .project(&batch.author, &batch.rev, &events) + .await + }; + match result { + Ok(()) => { + index + .advance_projector_cursor(self.name, &batch.author, &batch.rev) + .await?; + delivered += 1; + } + Err(error) => { + succeeded = false; + if error.is_retryable_projection() { + tracing::warn!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, error = %error, "projection destination unavailable; batch remains pending without consuming its failure budget"); + continue; + } + let attempts = index + .record_projection_failure( + self.name, + &batch.author, + &batch.rev, + &error.to_string(), + self.dead_letter_after, + ) + .await?; + if attempts >= self.dead_letter_after { + tracing::error!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, attempts, error = %error, "projection batch dead-lettered"); + } else { + tracing::warn!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, attempts, error = %error, "projection batch will retry"); + } + } + } + } + Ok((delivered, succeeded)) + } + + pub async fn drain(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index) + .await + .map(|(delivered, _)| delivered) + } + + pub async fn drain_succeeded(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index) + .await + .map(|(_, succeeded)| succeeded) + } +} + +pub type SharedJournalConsumer = Arc; + +/// Drain every projector, then drop the journal rows all of them have passed. +pub async fn drain_all(index: &dyn SpaceIndex, consumers: &[SharedJournalConsumer]) { + for consumer in consumers { + if let Err(error) = consumer.drain(index).await { + tracing::warn!(projector = consumer.name(), error = %error, "projection drain failed"); + } + } + if consumers.is_empty() { + return; + } + let names: Vec<&str> = consumers.iter().map(|c| c.name()).collect(); + if let Err(error) = index.prune_journal(&names).await { + tracing::warn!(error = %error, "journal prune failed"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::DaemonError; + use crate::index::{InMemoryIndex, IndexMutation}; + use crate::router::{SyncEvent, POST_COLLECTION}; + use async_trait::async_trait; + use rsky_space::record::encode_record; + use rsky_space::space_id::SpaceId; + use serde_json::json; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Mutex; + + const AUTHORITY: &str = "did:plc:community"; + const AUTHOR: &str = "did:plc:alice"; + + fn router() -> Router { + Router::new( + SpaceId::new(AUTHORITY, "community.blacksky.feed", "private"), + AUTHORITY, + ) + } + + fn post_mutation(rkey: &str) -> IndexMutation { + IndexMutation::Upsert { + collection: POST_COLLECTION.to_string(), + rkey: rkey.to_string(), + cid: "bafypost".to_string(), + rev: "3krev".to_string(), + value: Some( + encode_record( + &json!({"$type": POST_COLLECTION, "text": "hi", "createdAt": "2026-08-19T00:00:00Z"}), + 64 * 1024, + ) + .unwrap(), + ), + } + } + + #[derive(Default)] + struct Recorder { + delivered: Mutex>, + fail_next: AtomicUsize, + retryable: bool, + } + + #[async_trait] + impl Projector for Recorder { + fn name(&self) -> &'static str { + "recorder" + } + async fn project(&self, _did: &str, rev: &str, events: &[SyncEvent]) -> Result<()> { + if self.fail_next.load(Ordering::SeqCst) > 0 { + self.fail_next.fetch_sub(1, Ordering::SeqCst); + return Err(if self.retryable { + DaemonError::RetryableProjection("destination down".to_string()) + } else { + DaemonError::Xrpc("rejected".to_string()) + }); + } + self.delivered + .lock() + .unwrap() + .push((rev.to_string(), events.len())); + Ok(()) + } + } + + #[tokio::test] + async fn a_delivered_batch_advances_its_cursor_once() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + let consumer = JournalConsumer::new(router(), Box::::default()); + + assert_eq!(consumer.drain(&index).await.unwrap(), 1); + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + } + + #[tokio::test] + async fn a_retryable_failure_keeps_the_batch_without_spending_its_budget() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + let projector = Recorder { + retryable: true, + ..Default::default() + }; + projector.fail_next.store(1, Ordering::SeqCst); + let consumer = + JournalConsumer::new(router(), Box::new(projector)).with_dead_letter_after(1); + + assert!(!consumer.drain_succeeded(&index).await.unwrap()); + assert_eq!(index.pending_batches("recorder").await.unwrap().len(), 1); + assert_eq!(consumer.drain(&index).await.unwrap(), 1); + } + + #[tokio::test] + async fn a_rejected_batch_dead_letters_after_its_budget() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + let projector = Recorder::default(); + projector.fail_next.store(5, Ordering::SeqCst); + let consumer = + JournalConsumer::new(router(), Box::new(projector)).with_dead_letter_after(2); + + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + assert_eq!(index.pending_batches("recorder").await.unwrap().len(), 1); + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_batch_with_nothing_to_project_still_advances() { + let index = InMemoryIndex::new(); + index + .journal_batch( + AUTHOR, + "3krev", + &[IndexMutation::Delete { + collection: "app.bsky.graph.follow".to_string(), + rkey: "3ka".to_string(), + }], + ) + .await + .unwrap(); + let consumer = JournalConsumer::new(router(), Box::::default()); + + assert_eq!(consumer.drain(&index).await.unwrap(), 1); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn one_stalled_projector_does_not_hold_back_another() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + + struct Stalled; + #[async_trait] + impl Projector for Stalled { + fn name(&self) -> &'static str { + "stalled" + } + async fn project(&self, _did: &str, _rev: &str, _events: &[SyncEvent]) -> Result<()> { + Err(DaemonError::RetryableProjection("down".to_string())) + } + } + + let healthy: SharedJournalConsumer = + Arc::new(JournalConsumer::new(router(), Box::::default())); + let stalled: SharedJournalConsumer = + Arc::new(JournalConsumer::new(router(), Box::new(Stalled))); + drain_all(&index, &[healthy.clone(), stalled.clone()]).await; + + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + assert_eq!(index.pending_batches("stalled").await.unwrap().len(), 1); + } +} diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index 5c4fa44b..cad2dfe0 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -24,9 +24,12 @@ pub mod dpop; pub mod engine; pub mod error; pub mod index; +pub mod journal; pub mod notify; +pub mod projection; pub mod recovery; pub mod repohost; +pub mod router; pub mod runner; pub mod service_jwt; pub mod spaces; @@ -39,7 +42,10 @@ pub use credentials::{ }; pub use engine::{CommitKeyResolver, SyncOutcome, sync_repo}; pub use error::{DaemonError, Result}; -pub use index::{InMemoryIndex, SpaceIndex}; +pub use index::{IndexMutation, InMemoryIndex, JournaledBatch, SpaceIndex}; +pub use journal::{drain_all, JournalConsumer, SharedJournalConsumer}; +pub use projection::Projector; +pub use router::{Router, SyncEvent}; pub use notify::{NotifyState, WriteNotice, router as notify_router}; pub use recovery::recover_repo; pub use repohost::{HttpRepoHost, OplogPage, RepoHostClient}; diff --git a/rsky-daemon/src/projection.rs b/rsky-daemon/src/projection.rs new file mode 100644 index 00000000..5250c383 --- /dev/null +++ b/rsky-daemon/src/projection.rs @@ -0,0 +1,17 @@ +//! Projection adapter contract. Delivery is driven by the durable journal, +//! never by writing straight through from the sync path. + +use async_trait::async_trait; + +use crate::error::Result; +use crate::router::SyncEvent; + +/// Where a synced batch is sent. +#[async_trait] +pub trait Projector: Send + Sync { + fn name(&self) -> &'static str; + + /// Deliver the events of one batch. Must be idempotent: a retry after a + /// crash replays the same events. + async fn project(&self, did: &str, rev: &str, events: &[SyncEvent]) -> Result<()>; +} diff --git a/rsky-daemon/src/recovery.rs b/rsky-daemon/src/recovery.rs index abd316cf..16b8f8da 100644 --- a/rsky-daemon/src/recovery.rs +++ b/rsky-daemon/src/recovery.rs @@ -10,7 +10,7 @@ use rsky_space::lthash::{element, LtHash}; use crate::engine::{CommitKeyResolver, SyncOutcome}; use crate::error::{DaemonError, Result}; -use crate::index::SpaceIndex; +use crate::index::{IndexMutation, SpaceIndex}; use crate::repohost::RepoHostClient; /// Recover an author's repo from a full-state CAR: verify the commit with the @@ -43,6 +43,7 @@ pub async fn recover_repo( let mut lth = LtHash::new(); let mut keep: HashSet = HashSet::with_capacity(records.len()); let mut changed = 0usize; + let mut mutations: Vec = Vec::new(); for (path, cid, bytes) in &records { let (collection, rkey) = path .split_once('/') @@ -60,6 +61,13 @@ pub async fn recover_repo( ) .await?; changed += 1; + mutations.push(IndexMutation::Upsert { + collection: collection.to_string(), + rkey: rkey.to_string(), + cid: cid.clone(), + rev: commit.rev.clone(), + value: Some(bytes.clone()), + }); } lth.add(&element(collection, rkey, &cid)); keep.insert(path.clone()); @@ -68,8 +76,10 @@ pub async fn recover_repo( if !keep.contains(&format!("{collection}/{rkey}")) { index.delete(did, &collection, &rkey).await?; changed += 1; + mutations.push(IndexMutation::Delete { collection, rkey }); } } + index.journal_batch(did, &commit.rev, &mutations).await?; index.save_head(did, &commit.rev, <h).await?; tracing::info!(did, rev = %commit.rev, changed, "recovered repo from full-state CAR"); Ok(SyncOutcome { diff --git a/rsky-daemon/src/router.rs b/rsky-daemon/src/router.rs new file mode 100644 index 00000000..c67c8348 --- /dev/null +++ b/rsky-daemon/src/router.rs @@ -0,0 +1,341 @@ +//! Typed routing over an authenticated batch. +//! +//! A projection cares about *what happened*, not about record paths, so the +//! router turns index mutations into events before anything downstream sees +//! them. Routing runs on verified batches only — an unroutable or unknown +//! record is dropped and logged rather than raised, since one record a build +//! has never heard of must not stall a whole repo. + +use rsky_space::record::decode_record; +use rsky_space::space_id::{RecordId, SpaceId}; +use serde_json::Value; +use std::sync::atomic::{AtomicU64, Ordering}; +use tracing::debug; + +use crate::index::IndexMutation; + +/// Records on collections this build projects that were lost anyway because +/// their bytes would not decode. Dropping unknown collections is policy; +/// losing a known collection is data loss and must alarm. +pub static KNOWN_COLLECTION_DECODE_FAILURES: AtomicU64 = AtomicU64::new(0); + +/// Stable event name for the loss alarm, for log-based alerting. +pub const LOSS_EVENT: &str = "space_known_collection_decode_failure"; + +pub const POST_COLLECTION: &str = "app.bsky.feed.post"; +pub const LIKE_COLLECTION: &str = "app.bsky.feed.like"; +pub const MODERATION_ACTION_COLLECTION: &str = "community.blacksky.moderation.action"; + +/// One routed change, in the terms a projection acts on. +#[derive(Debug, Clone, PartialEq)] +pub enum SyncEvent { + PostCreated { + uri: String, + author: String, + cid: String, + rev: String, + record: Value, + }, + PostDeleted { + uri: String, + author: String, + }, + LikeCreated { + uri: String, + author: String, + cid: String, + rev: String, + record: Value, + }, + LikeDeleted { + uri: String, + author: String, + }, + /// A moderation action from the space authority's own repo. The router + /// emits this only for the authority (D25), so a projection never has to + /// re-check who wrote it. A lift is its own signed record (`neg: true`), + /// not deletion of the action it reverses. + ModerationAction { + uri: String, + rev: String, + record: Option, + }, +} + +impl SyncEvent { + pub fn uri(&self) -> &str { + match self { + Self::PostCreated { uri, .. } + | Self::PostDeleted { uri, .. } + | Self::LikeCreated { uri, .. } + | Self::LikeDeleted { uri, .. } + | Self::ModerationAction { uri, .. } => uri, + } + } +} + +/// Turns the mutations of one author's verified batch into events. +pub struct Router { + space: SpaceId, + authority: String, +} + +impl Router { + pub fn new(space: SpaceId, authority: impl Into) -> Self { + Self { + space, + authority: authority.into(), + } + } + + pub fn space(&self) -> &SpaceId { + &self.space + } + + pub fn route_batch(&self, did: &str, mutations: &[IndexMutation]) -> Vec { + mutations + .iter() + .filter_map(|m| self.route(did, m)) + .collect() + } + + pub fn route(&self, did: &str, mutation: &IndexMutation) -> Option { + let uri = RecordId { + space: self.space.clone(), + author: did.to_string(), + collection: mutation.collection().to_string(), + rkey: mutation.rkey().to_string(), + } + .uri(); + + match (mutation.collection(), mutation) { + (POST_COLLECTION, IndexMutation::Delete { .. }) => Some(SyncEvent::PostDeleted { + uri, + author: did.to_string(), + }), + (LIKE_COLLECTION, IndexMutation::Delete { .. }) => Some(SyncEvent::LikeDeleted { + uri, + author: did.to_string(), + }), + ( + POST_COLLECTION | LIKE_COLLECTION, + IndexMutation::Upsert { + cid, rev, value, .. + }, + ) => { + let record = decode_value(&uri, value.as_deref())?; + let is_post = mutation.collection() == POST_COLLECTION; + Some(if is_post { + SyncEvent::PostCreated { + uri, + author: did.to_string(), + cid: cid.clone(), + rev: rev.clone(), + record, + } + } else { + SyncEvent::LikeCreated { + uri, + author: did.to_string(), + cid: cid.clone(), + rev: rev.clone(), + record, + } + }) + } + (MODERATION_ACTION_COLLECTION, _) if did != self.authority => { + debug!(%uri, %did, "dropping moderation action from a repo that is not the authority's"); + None + } + (MODERATION_ACTION_COLLECTION, IndexMutation::Delete { .. }) => { + Some(SyncEvent::ModerationAction { + uri, + rev: String::new(), + record: None, + }) + } + (MODERATION_ACTION_COLLECTION, IndexMutation::Upsert { rev, value, .. }) => { + let record = decode_value(&uri, value.as_deref())?; + Some(SyncEvent::ModerationAction { + uri, + rev: rev.clone(), + record: Some(record), + }) + } + (other, _) => { + debug!(%uri, collection = %other, "dropping a collection this build does not project"); + None + } + } + } +} + +fn decode_value(uri: &str, value: Option<&[u8]>) -> Option { + match value { + Some(bytes) => match decode_record(bytes) { + Ok(value) => Some(value), + Err(err) => { + let total = KNOWN_COLLECTION_DECODE_FAILURES.fetch_add(1, Ordering::Relaxed) + 1; + tracing::error!( + event = LOSS_EVENT, + %uri, + %err, + total, + "a record on a projected collection will not decode and is lost" + ); + None + } + }, + None => { + // The host inlines a record's value with its op; without one there + // is nothing to project, and the sweep will bring it back. + debug!(%uri, "dropping a write carrying no record value"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rsky_space::record::encode_record; + use serde_json::json; + + const AUTHORITY: &str = "did:plc:community"; + const MEMBER: &str = "did:plc:alice"; + + fn router() -> Router { + Router::new( + SpaceId::new(AUTHORITY, "community.blacksky.feed", "private"), + AUTHORITY, + ) + } + + fn upsert(collection: &str, value: Value) -> IndexMutation { + IndexMutation::Upsert { + collection: collection.to_string(), + rkey: "3krkey".to_string(), + cid: "bafyrecord".to_string(), + rev: "3krev".to_string(), + value: Some(encode_record(&value, 64 * 1024).unwrap()), + } + } + + fn delete(collection: &str) -> IndexMutation { + IndexMutation::Delete { + collection: collection.to_string(), + rkey: "3krkey".to_string(), + } + } + + fn post() -> Value { + json!({"$type": POST_COLLECTION, "text": "hello", "createdAt": "2026-08-09T00:00:00.000Z"}) + } + + #[test] + fn a_post_routes_to_its_space_uri() { + match router().route(MEMBER, &upsert(POST_COLLECTION, post())) { + Some(SyncEvent::PostCreated { uri, record, .. }) => { + assert_eq!( + uri, + format!("at://{AUTHORITY}/space/community.blacksky.feed/private/{MEMBER}/{POST_COLLECTION}/3krkey") + ); + assert_eq!(record["text"], "hello"); + } + other => panic!("expected a post create, got {other:?}"), + } + } + + #[test] + fn a_like_over_a_space_subject_survives_routing() { + // The record standard lexicon validation would reject: its subject is + // a space URI, which is not an at-uri (D29). + let like = json!({ + "$type": LIKE_COLLECTION, + "subject": {"uri": format!("at://{AUTHORITY}/space/x/y/{MEMBER}/{POST_COLLECTION}/3kz"), "cid": "bafy"}, + "createdAt": "2026-08-09T00:00:00.000Z", + }); + match router().route(MEMBER, &upsert(LIKE_COLLECTION, like)) { + Some(SyncEvent::LikeCreated { record, .. }) => { + assert!(record["subject"]["uri"] + .as_str() + .unwrap() + .contains("/space/")); + } + other => panic!("expected a like create, got {other:?}"), + } + } + + #[test] + fn deletes_route_by_kind() { + let r = router(); + assert!(matches!( + r.route(MEMBER, &delete(POST_COLLECTION)), + Some(SyncEvent::PostDeleted { .. }) + )); + assert!(matches!( + r.route(MEMBER, &delete(LIKE_COLLECTION)), + Some(SyncEvent::LikeDeleted { .. }) + )); + } + + #[test] + fn moderation_actions_come_only_from_the_authority() { + let action = json!({ + "$type": MODERATION_ACTION_COLLECTION, + "subject": {"uri": "at://a/space/t/s/did:plc:alice/app.bsky.feed.post/3k"}, + "event": {"$type": "community.blacksky.moderation.action#delete"}, + "createdAt": "2026-08-09T00:00:00.000Z", + }); + let r = router(); + assert!(matches!( + r.route( + AUTHORITY, + &upsert(MODERATION_ACTION_COLLECTION, action.clone()) + ), + Some(SyncEvent::ModerationAction { + record: Some(_), + .. + }) + )); + // A member's own copy of the same record is a forgery attempt. + assert!(r + .route(MEMBER, &upsert(MODERATION_ACTION_COLLECTION, action)) + .is_none()); + // Deletes do not lift an effect: only a signed negation may do that. + assert!(matches!( + r.route(AUTHORITY, &delete(MODERATION_ACTION_COLLECTION)), + Some(SyncEvent::ModerationAction { record: None, .. }) + )); + } + + #[test] + fn only_unknown_collections_may_drop_silently() { + let r = router(); + + // Unknown collection: drop-and-log is policy, no alarm. + let before = KNOWN_COLLECTION_DECODE_FAILURES.load(Ordering::SeqCst); + assert!(r + .route( + MEMBER, + &upsert("app.bsky.graph.follow", json!({"$type": "x"})) + ) + .is_none()); + assert_eq!( + KNOWN_COLLECTION_DECODE_FAILURES.load(Ordering::SeqCst), + before + ); + + // A known collection failing to decode is data loss: it must raise + // the alarmed counter, never vanish silently. + let mut broken = upsert(POST_COLLECTION, post()); + if let IndexMutation::Upsert { value, .. } = &mut broken { + *value = Some(vec![0xff, 0xff]); + } + assert!(r.route(MEMBER, &broken).is_none()); + assert_eq!( + KNOWN_COLLECTION_DECODE_FAILURES.load(Ordering::SeqCst), + before + 1 + ); + } +} diff --git a/rsky-daemon/src/sqlite_index.rs b/rsky-daemon/src/sqlite_index.rs index 1ea8afc5..5aadc8b7 100644 --- a/rsky-daemon/src/sqlite_index.rs +++ b/rsky-daemon/src/sqlite_index.rs @@ -9,7 +9,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use crate::error::{DaemonError, Result}; -use crate::index::SpaceIndex; +use crate::index::{IndexMutation, JournaledBatch, SpaceIndex}; const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS sync_state ( @@ -29,6 +29,30 @@ CREATE TABLE IF NOT EXISTS record ( value BLOB, PRIMARY KEY (space_uri, did, collection, rkey) ); +CREATE TABLE IF NOT EXISTS projection_journal ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + mutations BLOB NOT NULL, + PRIMARY KEY (space_uri, did, rev) +); +CREATE TABLE IF NOT EXISTS projector_cursor ( + projector TEXT NOT NULL, + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + PRIMARY KEY (projector, space_uri, did) +); +CREATE TABLE IF NOT EXISTS projection_failure ( + projector TEXT NOT NULL, + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + attempts INTEGER NOT NULL, + last_error TEXT NOT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (projector, space_uri, did, rev) +); "; fn db_err(e: rusqlite::Error) -> DaemonError { @@ -157,6 +181,139 @@ impl SpaceIndex for SpaceScopedIndex { Ok(()) } + async fn journal_batch(&self, did: &str, rev: &str, mutations: &[IndexMutation]) -> Result<()> { + let encoded = serde_json::to_vec(mutations) + .map_err(|error| DaemonError::Index(format!("journal encode: {error}")))?; + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projection_journal (space_uri, did, rev, mutations) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (space_uri, did, rev) DO NOTHING", + params![self.space_uri, did, rev, encoded], + ) + .map_err(db_err)?; + Ok(()) + } + + async fn pending_batches(&self, projector: &str) -> Result> { + let conn = self.db.conn.lock().unwrap(); + let mut stmt = conn + .prepare( + "SELECT j.did, j.rev, j.mutations FROM projection_journal j + LEFT JOIN projector_cursor c + ON c.projector = ?1 AND c.space_uri = j.space_uri AND c.did = j.did + LEFT JOIN projection_failure f + ON f.projector = ?1 AND f.space_uri = j.space_uri AND f.did = j.did AND f.rev = j.rev + WHERE j.space_uri = ?2 + AND (c.rev IS NULL OR j.rev > c.rev) + AND COALESCE(f.dead_lettered, 0) = 0 + ORDER BY j.did, j.rev", + ) + .map_err(db_err)?; + let rows = stmt + .query_map(params![projector, self.space_uri], |row| { + let mutations: Vec = row.get(2)?; + let mutations = serde_json::from_slice(&mutations) + .map_err(|_| rusqlite::Error::InvalidQuery)?; + Ok(JournaledBatch { + author: row.get(0)?, + rev: row.get(1)?, + mutations, + }) + }) + .map_err(db_err)?; + rows.collect::>>().map_err(db_err) + } + + async fn advance_projector_cursor(&self, projector: &str, did: &str, rev: &str) -> Result<()> { + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projector_cursor (projector, space_uri, did, rev) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (projector, space_uri, did) DO UPDATE SET rev = ?4", + params![projector, self.space_uri, did, rev], + ) + .map_err(db_err)?; + Ok(()) + } + + async fn record_projection_failure( + &self, + projector: &str, + did: &str, + rev: &str, + error: &str, + dead_letter_after: u32, + ) -> Result { + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projection_failure + (projector, space_uri, did, rev, attempts, last_error, dead_lettered) + VALUES (?1, ?2, ?3, ?4, 1, ?5, CASE WHEN 1 >= ?6 THEN 1 ELSE 0 END) + ON CONFLICT (projector, space_uri, did, rev) DO UPDATE + SET attempts = attempts + 1, + last_error = ?5, + dead_lettered = CASE WHEN attempts + 1 >= ?6 THEN 1 ELSE 0 END", + params![ + projector, + self.space_uri, + did, + rev, + error, + dead_letter_after + ], + ) + .map_err(db_err)?; + conn.query_row( + "SELECT attempts FROM projection_failure + WHERE projector = ?1 AND space_uri = ?2 AND did = ?3 AND rev = ?4", + params![projector, self.space_uri, did, rev], + |row| row.get(0), + ) + .map_err(db_err) + } + + async fn prune_journal(&self, projectors: &[&str]) -> Result { + if projectors.is_empty() { + return Ok(0); + } + let mut conn = self.db.conn.lock().unwrap(); + let tx = conn.transaction().map_err(db_err)?; + let placeholders = (0..projectors.len()) + .map(|i| format!("?{}", i + 3)) + .collect::>() + .join(", "); + let sql = format!( + "DELETE FROM projection_journal WHERE space_uri = ?1 + AND (SELECT COUNT(*) FROM projector_cursor c + WHERE c.space_uri = projection_journal.space_uri + AND c.did = projection_journal.did + AND c.rev >= projection_journal.rev + AND c.projector IN ({placeholders})) = ?2 + AND NOT EXISTS (SELECT 1 FROM projection_failure f + WHERE f.space_uri = projection_journal.space_uri + AND f.did = projection_journal.did + AND f.rev = projection_journal.rev + AND f.dead_lettered = 1)" + ); + let count = projectors.len() as i64; + let mut values: Vec<&dyn rusqlite::ToSql> = vec![&self.space_uri, &count]; + for projector in projectors { + values.push(projector); + } + let pruned = tx.execute(&sql, &values[..]).map_err(db_err)?; + tx.execute( + "DELETE FROM projection_failure WHERE space_uri = ?1 AND dead_lettered = 0 + AND NOT EXISTS (SELECT 1 FROM projection_journal j + WHERE j.space_uri = projection_failure.space_uri + AND j.did = projection_failure.did + AND j.rev = projection_failure.rev)", + params![self.space_uri], + ) + .map_err(db_err)?; + tx.commit().map_err(db_err)?; + Ok(pruned) + } + async fn list_paths(&self, did: &str) -> Result> { let conn = self.db.conn.lock().unwrap(); let mut stmt = conn @@ -185,6 +342,17 @@ impl SpaceIndex for SpaceScopedIndex { params![self.space_uri], ) .map_err(db_err)?; + for table in [ + "projection_journal", + "projector_cursor", + "projection_failure", + ] { + conn.execute( + &format!("DELETE FROM {table} WHERE space_uri = ?1"), + params![self.space_uri], + ) + .map_err(db_err)?; + } Ok(()) } } @@ -381,6 +549,83 @@ mod tests { assert_eq!(other.list_paths(AUTHOR).await.unwrap().len(), 1); } + #[tokio::test] + async fn projector_cursors_are_independent_and_survive_dead_letters() { + let dir = tempfile::tempdir().unwrap(); + let db = open_at(&dir); + let index = db.for_space(SPACE); + let mutation = |rkey: &str| IndexMutation::Delete { + collection: "app.bsky.feed.post".to_string(), + rkey: rkey.to_string(), + }; + + index + .journal_batch(AUTHOR, "3rev1", &[mutation("3ka")]) + .await + .unwrap(); + index + .journal_batch(AUTHOR, "3rev2", &[mutation("3kb")]) + .await + .unwrap(); + // A replayed batch must not duplicate its journal row. + index + .journal_batch(AUTHOR, "3rev1", &[mutation("3ka")]) + .await + .unwrap(); + assert_eq!(index.pending_batches("feeds").await.unwrap().len(), 2); + + index + .advance_projector_cursor("feeds", AUTHOR, "3rev1") + .await + .unwrap(); + let pending = index.pending_batches("feeds").await.unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].rev, "3rev2"); + assert_eq!(pending[0].mutations, vec![mutation("3kb")]); + assert_eq!(index.pending_batches("appview").await.unwrap().len(), 2); + + assert_eq!( + index + .record_projection_failure("feeds", AUTHOR, "3rev2", "boom", 2) + .await + .unwrap(), + 1 + ); + assert_eq!(index.pending_batches("feeds").await.unwrap().len(), 1); + assert_eq!( + index + .record_projection_failure("feeds", AUTHOR, "3rev2", "boom", 2) + .await + .unwrap(), + 2 + ); + assert!(index.pending_batches("feeds").await.unwrap().is_empty()); + + // Nothing prunes while a dead-lettered batch is still on the journal. + index + .advance_projector_cursor("appview", AUTHOR, "3rev2") + .await + .unwrap(); + index + .advance_projector_cursor("feeds", AUTHOR, "3rev2") + .await + .unwrap(); + assert_eq!(index.prune_journal(&["feeds", "appview"]).await.unwrap(), 1); + let remaining: i64 = { + let conn = db.conn.lock().unwrap(); + conn.query_row( + "SELECT COUNT(*) FROM projection_journal WHERE space_uri = ?1", + params![SPACE], + |row| row.get(0), + ) + .unwrap() + }; + assert_eq!(remaining, 1); + + index.purge_space().await.unwrap(); + assert!(index.pending_batches("appview").await.unwrap().is_empty()); + } + #[tokio::test] async fn corrupt_lthash_state_is_an_error() { let dir = tempfile::tempdir().unwrap(); From 76a1d53a511aff965b5cb5271f74b37aaab8d560 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:36:19 -0400 Subject: [PATCH 22/56] feat(daemon): project synced posts to the feed service ingress --- rsky-daemon/src/config.rs | 67 +++-- rsky-daemon/src/credentials.rs | 17 +- rsky-daemon/src/feeds.rs | 449 +++++++++++++++++++++++++++++++++ rsky-daemon/src/lib.rs | 22 +- rsky-daemon/src/notify.rs | 6 +- rsky-daemon/src/runner.rs | 60 +++-- rsky-daemon/src/service_jwt.rs | 15 +- rsky-daemon/src/spaces.rs | 10 +- 8 files changed, 588 insertions(+), 58 deletions(-) create mode 100644 rsky-daemon/src/feeds.rs diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index 4b679f1f..c14f55b5 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -16,14 +16,23 @@ pub struct Config { #[arg(long, env = "DAEMON_SPACES_URL", default_value = "")] pub spaces_url: String, /// Managing-app API key for dynamic space discovery. - #[arg(long, env = "DAEMON_SPACES_API_KEY", default_value = "", hide_env_values = true)] + #[arg( + long, + env = "DAEMON_SPACES_API_KEY", + default_value = "", + hide_env_values = true + )] pub spaces_api_key: String, /// Optional authority filter for dynamic discovery; unset, the daemon /// accepts spaces from every authority the managing app serves. #[arg(long, env = "DAEMON_AUTHORITY_DID", default_value = "")] pub authority_did: String, /// Space type accepted from the managing app. - #[arg(long, env = "DAEMON_SPACE_TYPE", default_value = "community.blacksky.feed")] + #[arg( + long, + env = "DAEMON_SPACE_TYPE", + default_value = "community.blacksky.feed" + )] pub space_type: String, /// The space host (authority) base URL, for listRepos + credential mint. @@ -216,31 +225,57 @@ mod tests { assert!(cfg.validate().is_ok()); let discovery_only = Config::try_parse_from([ - "rsky-daemon", "--space-host-url", "https://host.example", - "--service-identity", "did:web:syncer.example", "--spaces-url", "https://feeds.example", - "--spaces-api-key", "key", "--authority-did", "did:plc:authority", - ]).unwrap(); + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + "--spaces-url", + "https://feeds.example", + "--spaces-api-key", + "key", + "--authority-did", + "did:plc:authority", + ]) + .unwrap(); assert!(discovery_only.validate().is_ok()); assert_eq!( discovery_only.authority_filter().as_deref(), Some("did:plc:authority") ); let all_authorities = Config::try_parse_from([ - "rsky-daemon", "--space-host-url", "https://host.example", - "--service-identity", "did:web:syncer.example", "--spaces-url", "https://feeds.example", - "--spaces-api-key", "key", - ]).unwrap(); + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + "--spaces-url", + "https://feeds.example", + "--spaces-api-key", + "key", + ]) + .unwrap(); assert!(all_authorities.validate().is_ok()); assert!(all_authorities.authority_filter().is_none()); let keyless_discovery = Config::try_parse_from([ - "rsky-daemon", "--space-host-url", "https://host.example", - "--service-identity", "did:web:syncer.example", "--spaces-url", "https://feeds.example", - ]).unwrap(); + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + "--spaces-url", + "https://feeds.example", + ]) + .unwrap(); assert!(keyless_discovery.validate().is_err()); let neither = Config::try_parse_from([ - "rsky-daemon", "--space-host-url", "https://host.example", - "--service-identity", "did:web:syncer.example", - ]).unwrap(); + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + ]) + .unwrap(); assert!(neither.validate().is_err()); } } diff --git a/rsky-daemon/src/credentials.rs b/rsky-daemon/src/credentials.rs index 5b48ca75..23ce9455 100644 --- a/rsky-daemon/src/credentials.rs +++ b/rsky-daemon/src/credentials.rs @@ -4,14 +4,14 @@ use async_trait::async_trait; use rsky_lexicon::com::atproto::space::GetDelegationTokenOutput; -use rsky_space::credential::{CREDENTIAL_TTL_SECS, decode}; +use rsky_space::credential::{decode, CREDENTIAL_TTL_SECS}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Mutex; use crate::error::Result; -use crate::xrpc::{SpaceHostClient, check, http_client, net_err}; -use crate::{HttpSpaceHost, service_jwt::ServiceJwtIssuer}; +use crate::xrpc::{check, http_client, net_err, SpaceHostClient}; +use crate::{service_jwt::ServiceJwtIssuer, HttpSpaceHost}; /// Seconds since the Unix epoch; the injectable-`now` boundary for tests. pub fn unix_now() -> u64 { @@ -94,12 +94,17 @@ pub struct SpaceCredentialSource { } impl SpaceCredentialSource { pub fn new(provider: Arc, space: impl Into) -> Self { - Self { provider, space: space.into() } + Self { + provider, + space: space.into(), + } } } #[async_trait] impl CredentialSource for SpaceCredentialSource { - async fn credential(&self, now: u64) -> Result { self.provider.credential_for(&self.space, now).await } + async fn credential(&self, now: u64) -> Result { + self.provider.credential_for(&self.space, now).await + } } impl InternalCredentialProvider { pub fn new( @@ -197,7 +202,7 @@ mod tests { use super::*; use chrono::{DateTime, Utc}; use rsky_lexicon::com::atproto::space::ListReposOutput; - use rsky_space::credential::{CREDENTIAL_TYP, JwtHeader, SpaceClaims, encode}; + use rsky_space::credential::{encode, JwtHeader, SpaceClaims, CREDENTIAL_TYP}; use std::sync::atomic::{AtomicUsize, Ordering}; use wiremock::matchers::{header, method, path, query_param}; use wiremock::{Mock, MockServer, ResponseTemplate}; diff --git a/rsky-daemon/src/feeds.rs b/rsky-daemon/src/feeds.rs new file mode 100644 index 00000000..2d740a71 --- /dev/null +++ b/rsky-daemon/src/feeds.rs @@ -0,0 +1,449 @@ +//! Projection to the feed service's record ingress. + +use async_trait::async_trait; +use serde::Serialize; +use serde_json::Value; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use tracing::debug; + +use crate::error::{DaemonError, Result}; +use crate::projection::Projector; +use crate::router::{SyncEvent, POST_COLLECTION}; +use crate::service_jwt::ServiceJwtIssuer; +use crate::unix_now; + +pub const PROJECT_RECORDS_LXM: &str = "community.blacksky.space.projectRecords"; +/// The receiving side rejects a token whose lifetime exceeds five minutes; +/// the margin absorbs clock skew between the two services. +const TOKEN_TTL_SECS: u64 = 240; + +#[derive(Clone, Debug, Serialize, PartialEq)] +pub struct ProjectRecordsRequest { + pub ops: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ProjectionOperation { + Create, + Delete, + Flag, + Unflag, +} + +#[derive(Clone, Debug, Serialize, PartialEq)] +pub struct ProjectRecord { + pub space: String, + pub author: String, + pub uri: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub cid: Option, + pub revision: String, + pub operation: ProjectionOperation, + pub collection: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub record: Option, + #[serde(rename = "actionUri", skip_serializing_if = "Option::is_none")] + pub action_uri: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub neg: Option, +} + +#[async_trait] +pub trait ProjectionIngress: Send + Sync { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()>; +} + +/// Posts batches to a service that accepts `projectRecords`, authenticated +/// with this daemon's own service identity. +pub struct HttpProjectionIngress { + target: &'static str, + base_url: String, + audience: String, + issuer: ServiceJwtIssuer, + http: reqwest::Client, +} + +impl HttpProjectionIngress { + pub fn new( + target: &'static str, + base_url: impl Into, + service_identity: impl Into, + audience: impl Into, + signing_key_hex: &str, + ) -> Result { + Ok(Self { + target, + base_url: base_url.into().trim_end_matches('/').to_string(), + audience: audience.into(), + issuer: ServiceJwtIssuer::from_hex(service_identity, signing_key_hex)?, + http: reqwest::Client::new(), + }) + } + + fn service_jwt(&self) -> Result { + static JTI: AtomicU64 = AtomicU64::new(1); + let now = unix_now(); + let jti = format!("{now}-{}", JTI.fetch_add(1, Ordering::Relaxed)); + self.issuer.mint_for( + &self.audience, + PROJECT_RECORDS_LXM, + now, + TOKEN_TTL_SECS, + &jti, + ) + } +} + +#[async_trait] +impl ProjectionIngress for HttpProjectionIngress { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()> { + let response = self + .http + .post(format!("{}/xrpc/{PROJECT_RECORDS_LXM}", self.base_url)) + .bearer_auth(self.service_jwt()?) + .json(request) + .send() + .await + .map_err(|error| { + DaemonError::RetryableProjection(format!("{} unreachable: {error}", self.target)) + })?; + let status = response.status(); + if status.is_success() { + return Ok(()); + } + let message = format!("{} {PROJECT_RECORDS_LXM} returned {status}", self.target); + if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { + return Err(DaemonError::RetryableProjection(message)); + } + Err(DaemonError::Xrpc(message)) + } +} + +/// Projects a space's posts and their moderation state to the feed service. +pub struct FeedsProjector { + ingress: I, + space: String, +} + +impl FeedsProjector { + pub fn new(ingress: I, space: impl Into) -> Self { + Self { + ingress, + space: space.into(), + } + } +} + +#[async_trait] +impl Projector for FeedsProjector { + fn name(&self) -> &'static str { + "feeds" + } + + async fn project(&self, author: &str, revision: &str, events: &[SyncEvent]) -> Result<()> { + // One URI may change more than once in a batch; only its final state + // is worth sending. + let mut final_posts: BTreeMap<&str, &SyncEvent> = BTreeMap::new(); + let mut ops = Vec::new(); + for event in events { + match event { + SyncEvent::PostCreated { .. } | SyncEvent::PostDeleted { .. } => { + final_posts.insert(event.uri(), event); + } + SyncEvent::ModerationAction { uri, record, .. } => { + if let Some(op) = self.moderation_op(author, revision, uri, record.as_ref()) { + ops.push(op); + } + } + other => debug!(uri = other.uri(), "feeds projection ignores this event"), + } + } + for event in final_posts.into_values() { + match event { + SyncEvent::PostCreated { + uri, cid, record, .. + } => ops.push(ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.clone(), + cid: Some(cid.clone()), + revision: revision.to_string(), + operation: ProjectionOperation::Create, + collection: POST_COLLECTION.to_string(), + record: Some(record.clone()), + action_uri: None, + neg: None, + }), + SyncEvent::PostDeleted { uri, .. } => ops.push(ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.clone(), + cid: None, + revision: revision.to_string(), + operation: ProjectionOperation::Delete, + collection: POST_COLLECTION.to_string(), + record: None, + action_uri: None, + neg: None, + }), + _ => unreachable!("only post events were retained"), + } + } + if ops.is_empty() { + return Ok(()); + } + self.ingress + .project_records(&ProjectRecordsRequest { ops }) + .await + } +} + +impl FeedsProjector { + fn moderation_op( + &self, + author: &str, + revision: &str, + uri: &str, + record: Option<&Value>, + ) -> Option { + let record = record?; + if record["val"].as_str() != Some("remove") { + return None; + } + // A lift is its own signed record carrying the action it reverses, + // so it names that action's URI rather than its own. + let neg = record["neg"].as_bool() == Some(true); + let (action_uri, cid) = if neg { + (record["action"]["uri"].as_str(), None) + } else { + (Some(uri), record["subject"]["cid"].as_str()) + }; + let Some(action_uri) = action_uri else { + debug!(%uri, "moderation action has no action URI"); + return None; + }; + if !neg && cid.is_none() { + debug!(%uri, "moderation action has no strong post reference"); + return None; + } + Some(ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.to_string(), + cid: cid.map(str::to_string), + revision: revision.to_string(), + operation: if neg { + ProjectionOperation::Unflag + } else { + ProjectionOperation::Flag + }, + collection: POST_COLLECTION.to_string(), + record: None, + action_uri: Some(action_uri.to_string()), + neg: Some(neg), + }) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use serde_json::json; + use std::sync::Mutex; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + pub(crate) const SPACE: &str = "at://did:plc:community/space/community.blacksky.feed/private"; + pub(crate) const AUTHOR: &str = "did:plc:alice"; + + #[derive(Default)] + pub(crate) struct CapturingIngress { + pub sent: Mutex>, + } + + #[async_trait] + impl ProjectionIngress for CapturingIngress { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()> { + self.sent.lock().unwrap().push(request.clone()); + Ok(()) + } + } + + pub(crate) fn post_created(rkey: &str, text: &str) -> SyncEvent { + SyncEvent::PostCreated { + uri: format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/{rkey}"), + author: AUTHOR.to_string(), + cid: "bafypost".to_string(), + rev: "3krev".to_string(), + record: json!({"$type": POST_COLLECTION, "text": text, "createdAt": "2026-08-19T00:00:00Z"}), + } + } + + fn ingress(server: &MockServer) -> HttpProjectionIngress { + HttpProjectionIngress::new( + "feeds", + server.uri(), + "did:web:syncer.example", + "did:web:feeds.example", + &hex::encode([1_u8; 32]), + ) + .unwrap() + } + + #[tokio::test] + async fn a_batch_projects_final_post_state_once() { + let projector = FeedsProjector::new(CapturingIngress::default(), SPACE); + let deleted = SyncEvent::PostDeleted { + uri: format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), + author: AUTHOR.to_string(), + }; + projector + .project( + AUTHOR, + "3krev", + &[ + post_created("3ka", "first"), + deleted, + post_created("3kb", "second"), + ], + ) + .await + .unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + assert_eq!(sent.len(), 1); + let ops = &sent[0].ops; + assert_eq!(ops.len(), 2); + assert_eq!(ops[0].operation, ProjectionOperation::Delete); + assert_eq!( + ops[0].uri, + format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka") + ); + assert_eq!(ops[1].operation, ProjectionOperation::Create); + assert_eq!(ops[1].record.as_ref().unwrap()["text"], "second"); + assert_eq!(ops[1].space, SPACE); + assert_eq!(ops[1].revision, "3krev"); + } + + #[tokio::test] + async fn moderation_removals_and_lifts_become_flags() { + let projector = FeedsProjector::new(CapturingIngress::default(), SPACE); + let action_uri = + format!("{SPACE}/did:plc:community/community.blacksky.moderation.action/3m"); + let events = vec![ + SyncEvent::ModerationAction { + uri: action_uri.clone(), + rev: "3krev".to_string(), + record: Some(json!({ + "val": "remove", + "subject": {"uri": format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), "cid": "bafypost"}, + })), + }, + SyncEvent::ModerationAction { + uri: format!("{SPACE}/did:plc:community/community.blacksky.moderation.action/3n"), + rev: "3krev".to_string(), + record: Some(json!({ + "val": "remove", + "neg": true, + "action": {"uri": action_uri}, + })), + }, + // Neither a removal nor decodable as one: dropped, not sent. + SyncEvent::ModerationAction { + uri: format!("{SPACE}/did:plc:community/community.blacksky.moderation.action/3o"), + rev: "3krev".to_string(), + record: Some(json!({"val": "spam"})), + }, + ]; + projector.project(AUTHOR, "3krev", &events).await.unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + let ops = &sent[0].ops; + assert_eq!(ops.len(), 2); + assert_eq!(ops[0].operation, ProjectionOperation::Flag); + assert_eq!(ops[0].neg, Some(false)); + assert_eq!(ops[0].cid.as_deref(), Some("bafypost")); + assert_eq!(ops[1].operation, ProjectionOperation::Unflag); + assert_eq!(ops[1].neg, Some(true)); + assert_eq!(ops[1].action_uri.as_deref(), Some(action_uri.as_str())); + } + + #[tokio::test] + async fn a_batch_with_nothing_to_send_makes_no_request() { + let projector = FeedsProjector::new(CapturingIngress::default(), SPACE); + projector + .project( + AUTHOR, + "3krev", + &[SyncEvent::LikeDeleted { + uri: format!("{SPACE}/{AUTHOR}/app.bsky.feed.like/3ka"), + author: AUTHOR.to_string(), + }], + ) + .await + .unwrap(); + assert!(projector.ingress.sent.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_batch_reaches_the_ingress_with_a_method_bound_service_token() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/xrpc/{PROJECT_RECORDS_LXM}"))) + .and(wiremock::matchers::header_regex( + "authorization", + "^Bearer ", + )) + .and(wiremock::matchers::body_string_contains("\"create\"")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({}))) + .mount(&server) + .await; + + let projector = FeedsProjector::new(ingress(&server), SPACE); + projector + .project(AUTHOR, "3krev", &[post_created("3ka", "hello")]) + .await + .unwrap(); + } + + #[tokio::test] + async fn destination_outages_are_retryable_and_rejections_are_not() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let error = ingress(&server) + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err(); + assert!(error.is_retryable_projection()); + + let rejecting = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400)) + .mount(&rejecting) + .await; + let error = ingress(&rejecting) + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err(); + assert!(!error.is_retryable_projection()); + + let unreachable = HttpProjectionIngress::new( + "feeds", + "http://127.0.0.1:1", + "did:web:syncer.example", + "did:web:feeds.example", + &hex::encode([1_u8; 32]), + ) + .unwrap(); + assert!(unreachable + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err() + .is_retryable_projection()); + } +} diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index cad2dfe0..c91a424d 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -23,6 +23,7 @@ pub mod credentials; pub mod dpop; pub mod engine; pub mod error; +pub mod feeds; pub mod index; pub mod journal; pub mod notify; @@ -37,19 +38,26 @@ pub mod sqlite_index; pub mod xrpc; pub use credentials::{ - CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, - PdsDelegationSource, SpaceCredentialSource, StaticCredential, unix_now, + unix_now, CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, + PdsDelegationSource, SpaceCredentialSource, StaticCredential, }; -pub use engine::{CommitKeyResolver, SyncOutcome, sync_repo}; +pub use engine::{sync_repo, CommitKeyResolver, SyncOutcome}; pub use error::{DaemonError, Result}; -pub use index::{IndexMutation, InMemoryIndex, JournaledBatch, SpaceIndex}; +pub use feeds::{ + FeedsProjector, HttpProjectionIngress, ProjectRecord, ProjectRecordsRequest, ProjectionIngress, + ProjectionOperation, +}; +pub use index::{InMemoryIndex, IndexMutation, JournaledBatch, SpaceIndex}; pub use journal::{drain_all, JournalConsumer, SharedJournalConsumer}; +pub use notify::{router as notify_router, NotifyState, WriteNotice}; pub use projection::Projector; -pub use router::{Router, SyncEvent}; -pub use notify::{NotifyState, WriteNotice, router as notify_router}; pub use recovery::recover_repo; pub use repohost::{HttpRepoHost, OplogPage, RepoHostClient}; -pub use runner::{MultiRunnerOptions, RunnerOptions, SweepReport, run, run_multi, sync_repo_healing, sync_space_once}; +pub use router::{Router, SyncEvent}; +pub use runner::{ + run, run_multi, sync_repo_healing, sync_space_once, MultiRunnerOptions, RunnerOptions, + SweepReport, +}; pub use spaces::{ CombinedSource, HttpSpaceSource, SpaceRegistry, SpaceSource, SpaceTarget, StaticSpaces, }; diff --git a/rsky-daemon/src/notify.rs b/rsky-daemon/src/notify.rs index 3aed0465..3fc7968c 100644 --- a/rsky-daemon/src/notify.rs +++ b/rsky-daemon/src/notify.rs @@ -244,7 +244,11 @@ mod tests { ) -> NotifyState { NotifyState { space_uri: SPACE.to_string(), - registry: { let registry = SpaceRegistry::new(); registry.insert(SPACE); registry }, + registry: { + let registry = SpaceRegistry::new(); + registry.insert(SPACE); + registry + }, service_identity: SYNCER.to_string(), resolver: Arc::new(FixedKey(did_key.to_string())), index, diff --git a/rsky-daemon/src/runner.rs b/rsky-daemon/src/runner.rs index d22a9ecf..acd33bfc 100644 --- a/rsky-daemon/src/runner.rs +++ b/rsky-daemon/src/runner.rs @@ -3,9 +3,9 @@ //! to full-state recovery when incremental sync cannot proceed. use rsky_lexicon::com::atproto::space::RepoRef; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use std::collections::HashMap; use tokio::sync::{mpsc, watch}; use tokio::time::Instant; @@ -16,15 +16,24 @@ use crate::index::SpaceIndex; use crate::notify::WriteNotice; use crate::recovery::recover_repo; use crate::repohost::RepoHostClient; -use crate::xrpc::SpaceHostClient; use crate::spaces::{SpaceRegistry, SpaceSource}; +use crate::xrpc::SpaceHostClient; const REGISTER_RETRY_SECS: u64 = 30; const MIN_REREGISTER_SECS: u64 = 30; /// Builds a repo-host client bound to the current space credential. pub type RepoHostFactory = Box Arc + Send + Sync>; -pub type MultiSpaceFactory = Arc Result<(Arc, RepoHostFactory, Arc)> + Send + Sync>; +pub type MultiSpaceFactory = Arc< + dyn Fn( + &str, + ) -> Result<( + Arc, + RepoHostFactory, + Arc, + )> + Send + + Sync, +>; pub struct MultiRunnerOptions { pub refresh_interval_secs: u64, @@ -48,27 +57,38 @@ pub async fn run_multi( mut notices: mpsc::Receiver, mut shutdown: watch::Receiver, ) { - struct Worker { generation: i64, stop: watch::Sender, notices: mpsc::Sender, handle: tokio::task::JoinHandle<()> } + struct Worker { + generation: i64, + stop: watch::Sender, + notices: mpsc::Sender, + handle: tokio::task::JoinHandle<()>, + } let mut workers: HashMap = HashMap::new(); let mut refresh = tokio::time::interval(Duration::from_secs(opts.refresh_interval_secs.max(1))); - loop { tokio::select! { - _ = shutdown.changed() => break, - _ = refresh.tick() => { - let desired = match source.spaces().await { Ok(value) => value, Err(error) => { tracing::warn!(error = %error, "space source unavailable; keeping current workers"); continue; } }; - let stale: Vec<_> = workers.iter().filter(|(space, worker)| desired.get(*space).is_none_or(|target| target.generation != worker.generation)).map(|(space, _)| space.clone()).collect(); - for space in stale { if let Some(worker) = workers.remove(&space) { let _ = worker.stop.send(true); let _ = worker.handle.await; } } - for (space, target) in &desired { if workers.contains_key(space) { continue; } - let (creds, repo, index) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; - let (tx, rx) = mpsc::channel(256); let (stop, stop_rx) = watch::channel(false); - let worker_opts = RunnerOptions { space_uri: space.clone(), sweep_interval_secs: opts.sweep_interval_secs, notify_endpoint: opts.notify_endpoint.clone(), service_identity: opts.service_identity.clone(), now_fn: opts.now_fn }; - let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), rx, stop_rx)); - workers.insert(space.clone(), Worker { generation: target.generation, stop, notices: tx, handle }); + loop { + tokio::select! { + _ = shutdown.changed() => break, + _ = refresh.tick() => { + let desired = match source.spaces().await { Ok(value) => value, Err(error) => { tracing::warn!(error = %error, "space source unavailable; keeping current workers"); continue; } }; + let stale: Vec<_> = workers.iter().filter(|(space, worker)| desired.get(*space).is_none_or(|target| target.generation != worker.generation)).map(|(space, _)| space.clone()).collect(); + for space in stale { if let Some(worker) = workers.remove(&space) { let _ = worker.stop.send(true); let _ = worker.handle.await; } } + for (space, target) in &desired { if workers.contains_key(space) { continue; } + let (creds, repo, index) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; + let (tx, rx) = mpsc::channel(256); let (stop, stop_rx) = watch::channel(false); + let worker_opts = RunnerOptions { space_uri: space.clone(), sweep_interval_secs: opts.sweep_interval_secs, notify_endpoint: opts.notify_endpoint.clone(), service_identity: opts.service_identity.clone(), now_fn: opts.now_fn }; + let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), rx, stop_rx)); + workers.insert(space.clone(), Worker { generation: target.generation, stop, notices: tx, handle }); + } + registry.replace(workers.keys().cloned().collect()); } - registry.replace(workers.keys().cloned().collect()); + Some(notice) = notices.recv() => { if let Some(worker) = workers.get(¬ice.0) { let _ = worker.notices.send(notice).await; } else { tracing::warn!(space = %notice.0, "notice for a space we do not sync"); } } } - Some(notice) = notices.recv() => { if let Some(worker) = workers.get(¬ice.0) { let _ = worker.notices.send(notice).await; } else { tracing::warn!(space = %notice.0, "notice for a space we do not sync"); } } - }} - for (space, worker) in workers { let _ = worker.stop.send(true); let _ = worker.handle.await; tracing::info!(%space, "stopped"); } + } + for (space, worker) in workers { + let _ = worker.stop.send(true); + let _ = worker.handle.await; + tracing::info!(%space, "stopped"); + } registry.replace(Default::default()); } diff --git a/rsky-daemon/src/service_jwt.rs b/rsky-daemon/src/service_jwt.rs index a9c19379..0ee0ab2a 100644 --- a/rsky-daemon/src/service_jwt.rs +++ b/rsky-daemon/src/service_jwt.rs @@ -24,6 +24,17 @@ impl ServiceJwtIssuer { } pub fn mint(&self, audience: &str, now: u64, jti: &str) -> Result { + self.mint_for(audience, MINT_LXM, now, 60, jti) + } + + pub fn mint_for( + &self, + audience: &str, + lxm: &str, + now: u64, + ttl_secs: u64, + jti: &str, + ) -> Result { #[derive(Serialize)] struct Header<'a> { typ: &'a str, @@ -49,8 +60,8 @@ impl ServiceJwtIssuer { serde_json::to_vec(&Claims { iss: &self.did, aud: audience, - exp: now + 60, - lxm: MINT_LXM, + exp: now + ttl_secs, + lxm, jti, iat: now, }) diff --git a/rsky-daemon/src/spaces.rs b/rsky-daemon/src/spaces.rs index cd99ebc5..3ec6c915 100644 --- a/rsky-daemon/src/spaces.rs +++ b/rsky-daemon/src/spaces.rs @@ -261,12 +261,10 @@ mod tests { .respond_with(ResponseTemplate::new(503)) .mount(&server) .await; - assert!( - CombinedSource(vec![Box::new(source(server.uri()))]) - .spaces() - .await - .is_err() - ); + assert!(CombinedSource(vec![Box::new(source(server.uri()))]) + .spaces() + .await + .is_err()); } #[test] fn registry_tracks_spaces() { From ac78e9a5cc3b9b5e17e33e261fa9bc5feaf7b457 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:37:08 -0400 Subject: [PATCH 23/56] feat(daemon): project synced posts, likes, and removals to the appview --- rsky-daemon/src/appview.rs | 270 +++++++++++++++++++++++++++++++++++++ rsky-daemon/src/lib.rs | 2 + 2 files changed, 272 insertions(+) create mode 100644 rsky-daemon/src/appview.rs diff --git a/rsky-daemon/src/appview.rs b/rsky-daemon/src/appview.rs new file mode 100644 index 00000000..7f7514b8 --- /dev/null +++ b/rsky-daemon/src/appview.rs @@ -0,0 +1,270 @@ +//! Projection to the appview's record ingress. + +use async_trait::async_trait; +use serde_json::Value; +use std::collections::BTreeMap; + +use crate::error::Result; +use crate::feeds::{ProjectRecord, ProjectRecordsRequest, ProjectionIngress, ProjectionOperation}; +use crate::projection::Projector; +use crate::router::{SyncEvent, LIKE_COLLECTION, MODERATION_ACTION_COLLECTION, POST_COLLECTION}; + +/// Projects a space's posts, likes and moderation state to the appview. +pub struct AppviewProjector { + ingress: I, + space: String, +} + +impl AppviewProjector { + pub fn new(ingress: I, space: impl Into) -> Self { + Self { + ingress, + space: space.into(), + } + } + + fn op( + &self, + author: &str, + uri: &str, + cid: Option<&str>, + revision: &str, + operation: ProjectionOperation, + collection: &str, + record: Option, + ) -> ProjectRecord { + ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.to_string(), + cid: cid.map(str::to_string), + revision: revision.to_string(), + operation, + collection: collection.to_string(), + record, + action_uri: None, + neg: None, + } + } + + fn moderation_op( + &self, + author: &str, + revision: &str, + uri: &str, + record: &Value, + ) -> Option { + if record["val"].as_str() != Some("remove") { + return None; + } + // A lift is its own signed record carrying the action it reverses, + // so it names that action's URI rather than its own. + let neg = record["neg"].as_bool() == Some(true); + let (action_uri, cid) = if neg { + (record["action"]["uri"].as_str(), None) + } else { + (Some(uri), record["subject"]["cid"].as_str()) + }; + Some(ProjectRecord { + action_uri: Some(action_uri?.to_string()), + neg: Some(neg), + ..self.op( + author, + uri, + cid, + revision, + if neg { + ProjectionOperation::Unflag + } else { + ProjectionOperation::Flag + }, + MODERATION_ACTION_COLLECTION, + None, + ) + }) + } +} + +#[async_trait] +impl Projector for AppviewProjector { + fn name(&self) -> &'static str { + "appview" + } + + async fn project(&self, author: &str, revision: &str, events: &[SyncEvent]) -> Result<()> { + // One URI may change more than once in a batch; only its final state + // is worth sending. + let mut final_records: BTreeMap<&str, &SyncEvent> = BTreeMap::new(); + for event in events { + final_records.insert(event.uri(), event); + } + let ops: Vec = final_records + .into_values() + .filter_map(|event| match event { + SyncEvent::PostCreated { + uri, cid, record, .. + } => Some(self.op( + author, + uri, + Some(cid), + revision, + ProjectionOperation::Create, + POST_COLLECTION, + Some(record.clone()), + )), + SyncEvent::PostDeleted { uri, .. } => Some(self.op( + author, + uri, + None, + revision, + ProjectionOperation::Delete, + POST_COLLECTION, + None, + )), + SyncEvent::LikeCreated { + uri, cid, record, .. + } => Some(self.op( + author, + uri, + Some(cid), + revision, + ProjectionOperation::Create, + LIKE_COLLECTION, + Some(record.clone()), + )), + SyncEvent::LikeDeleted { uri, .. } => Some(self.op( + author, + uri, + None, + revision, + ProjectionOperation::Delete, + LIKE_COLLECTION, + None, + )), + SyncEvent::ModerationAction { + uri, + record: Some(record), + .. + } => self.moderation_op(author, revision, uri, record), + SyncEvent::ModerationAction { record: None, .. } => None, + }) + .collect(); + if ops.is_empty() { + return Ok(()); + } + self.ingress + .project_records(&ProjectRecordsRequest { ops }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::feeds::tests::{post_created, CapturingIngress, AUTHOR, SPACE}; + use serde_json::json; + + fn like_created(rkey: &str) -> SyncEvent { + SyncEvent::LikeCreated { + uri: format!("{SPACE}/{AUTHOR}/{LIKE_COLLECTION}/{rkey}"), + author: AUTHOR.to_string(), + cid: "bafylike".to_string(), + rev: "3krev".to_string(), + record: json!({ + "$type": LIKE_COLLECTION, + "subject": {"uri": format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), "cid": "bafypost"}, + "createdAt": "2026-08-19T00:00:00Z", + }), + } + } + + #[tokio::test] + async fn posts_likes_and_removals_all_project() { + let projector = AppviewProjector::new(CapturingIngress::default(), SPACE); + let action_uri = format!("{SPACE}/did:plc:community/{MODERATION_ACTION_COLLECTION}/3m"); + projector + .project( + AUTHOR, + "3krev", + &[ + post_created("3ka", "hello"), + like_created("3kl"), + SyncEvent::ModerationAction { + uri: action_uri.clone(), + rev: "3krev".to_string(), + record: Some(json!({ + "val": "remove", + "subject": {"uri": format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), "cid": "bafypost"}, + })), + }, + ], + ) + .await + .unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + assert_eq!(sent.len(), 1); + let ops = &sent[0].ops; + assert_eq!(ops.len(), 3); + let collections: Vec<&str> = ops.iter().map(|op| op.collection.as_str()).collect(); + assert!(collections.contains(&POST_COLLECTION)); + assert!(collections.contains(&LIKE_COLLECTION)); + assert!(collections.contains(&MODERATION_ACTION_COLLECTION)); + let flagged = ops + .iter() + .find(|op| op.collection == MODERATION_ACTION_COLLECTION) + .unwrap(); + assert_eq!(flagged.operation, ProjectionOperation::Flag); + assert_eq!(flagged.action_uri.as_deref(), Some(action_uri.as_str())); + let like = ops + .iter() + .find(|op| op.collection == LIKE_COLLECTION) + .unwrap(); + assert!(like.record.as_ref().unwrap()["subject"]["uri"] + .as_str() + .unwrap() + .contains("/space/")); + } + + #[tokio::test] + async fn only_the_final_state_of_a_uri_is_sent() { + let projector = AppviewProjector::new(CapturingIngress::default(), SPACE); + let uri = format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"); + projector + .project( + AUTHOR, + "3krev", + &[ + post_created("3ka", "first"), + SyncEvent::PostDeleted { + uri: uri.clone(), + author: AUTHOR.to_string(), + }, + ], + ) + .await + .unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + assert_eq!(sent[0].ops.len(), 1); + assert_eq!(sent[0].ops[0].operation, ProjectionOperation::Delete); + } + + #[tokio::test] + async fn a_batch_with_nothing_to_send_makes_no_request() { + let projector = AppviewProjector::new(CapturingIngress::default(), SPACE); + projector + .project( + AUTHOR, + "3krev", + &[SyncEvent::ModerationAction { + uri: format!("{SPACE}/did:plc:community/{MODERATION_ACTION_COLLECTION}/3m"), + rev: "3krev".to_string(), + record: None, + }], + ) + .await + .unwrap(); + assert!(projector.ingress.sent.lock().unwrap().is_empty()); + } +} diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index c91a424d..8e7a1843 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -18,6 +18,7 @@ //! - [`runner`] — the loop composing all of the above. //! - [`index`] / [`sqlite_index`] — the synced record index. +pub mod appview; pub mod config; pub mod credentials; pub mod dpop; @@ -37,6 +38,7 @@ pub mod spaces; pub mod sqlite_index; pub mod xrpc; +pub use appview::AppviewProjector; pub use credentials::{ unix_now, CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, PdsDelegationSource, SpaceCredentialSource, StaticCredential, From beeda09f33d31510fb8f9a611b1ac1de123a3778 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:40:41 -0400 Subject: [PATCH 24/56] feat(daemon): configure and wire projection destinations per space DAEMON_FEEDS_URL / DAEMON_APPVIEW_URL (with their service DIDs) give every discovered space a worker that drains each destination on its own cursor; an empty URL leaves the daemon index-only. --- rsky-daemon/Cargo.toml | 2 +- rsky-daemon/src/config.rs | 62 ++++++++++++++++++ rsky-daemon/src/main.rs | 130 +++++++++++++++++++++++++++++++++----- rsky-daemon/src/runner.rs | 124 ++++++++++++++++++++++++++++++++---- 4 files changed, 290 insertions(+), 28 deletions(-) diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index 817ea083..e9a6ba42 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-daemon" -version = "0.5.1" +version = "0.6.0" authors = ["Rudy Fraser "] description = "atproto permissioned-data syncer daemon: pulls, verifies, and indexes permissioned repos from members' PDSes" edition = "2021" diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index c14f55b5..95a5ee67 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -88,6 +88,22 @@ pub struct Config { )] pub service_signing_key_hex: String, + /// Feed service base URL for record projection; empty leaves the daemon + /// index-only. + #[arg(long, env = "DAEMON_FEEDS_URL", default_value = "")] + pub feeds_url: String, + /// The `aud` the feed service requires on projection calls. + #[arg(long, env = "DAEMON_FEEDS_SERVICE_DID", default_value = "")] + pub feeds_service_did: String, + + /// Appview base URL for record projection; empty leaves the daemon + /// index-only. + #[arg(long, env = "DAEMON_APPVIEW_URL", default_value = "")] + pub appview_url: String, + /// The `aud` the appview requires on projection calls. + #[arg(long, env = "DAEMON_APPVIEW_SERVICE_DID", default_value = "")] + pub appview_service_did: String, + /// Bind address for the notify listener. #[arg(long, env = "DAEMON_NOTIFY_BIND", default_value = "127.0.0.1:8055")] pub notify_bind: String, @@ -119,9 +135,29 @@ impl Config { if !self.spaces_url.is_empty() && self.spaces_api_key.is_empty() { return Err("DAEMON_SPACES_API_KEY is required with DAEMON_SPACES_URL".into()); } + if !self.feeds_url.is_empty() && self.feeds_service_did.is_empty() { + return Err("DAEMON_FEEDS_SERVICE_DID is required with DAEMON_FEEDS_URL".into()); + } + if !self.appview_url.is_empty() && self.appview_service_did.is_empty() { + return Err("DAEMON_APPVIEW_SERVICE_DID is required with DAEMON_APPVIEW_URL".into()); + } Ok(()) } + /// `(base url, audience)` for the feed service, or `None` when the daemon + /// is configured index-only. + pub fn feeds_projection(&self) -> Option<(&str, &str)> { + (!self.feeds_url.is_empty()) + .then_some((self.feeds_url.as_str(), self.feeds_service_did.as_str())) + } + + /// `(base url, audience)` for the appview, or `None` when the daemon is + /// configured index-only. + pub fn appview_projection(&self) -> Option<(&str, &str)> { + (!self.appview_url.is_empty()) + .then_some((self.appview_url.as_str(), self.appview_service_did.as_str())) + } + pub fn authority_filter(&self) -> Option { (!self.authority_did.is_empty()).then(|| self.authority_did.clone()) } @@ -178,6 +214,8 @@ mod tests { assert_eq!(cfg.pds_access_token, ""); assert_eq!(cfg.static_credential, ""); assert_eq!(cfg.plc_url(), None); + assert_eq!(cfg.feeds_projection(), None); + assert_eq!(cfg.appview_projection(), None); let mut cfg = Config::try_parse_from(REQUIRED.into_iter().chain([ "--repo-host-url", @@ -204,6 +242,10 @@ mod tests { ("DAEMON_INDEX_DB_PATH", "/data/space.sqlite"), ("DAEMON_SWEEP_INTERVAL_SECS", "60"), ("DAEMON_PLC_URL", "http://localhost:2582"), + ("DAEMON_FEEDS_URL", "http://localhost:8080"), + ("DAEMON_FEEDS_SERVICE_DID", "did:web:feeds.example"), + ("DAEMON_APPVIEW_URL", "http://localhost:2584"), + ("DAEMON_APPVIEW_SERVICE_DID", "did:web:appview.example"), ]; for (k, v) in env { std::env::set_var(k, v); @@ -222,8 +264,28 @@ mod tests { assert_eq!(cfg.notify_endpoint(), "http://0.0.0.0:9000"); assert_eq!(cfg.index_db_path, "/data/space.sqlite"); assert_eq!(cfg.sweep_interval_secs, 60); + assert_eq!( + cfg.feeds_projection(), + Some(("http://localhost:8080", "did:web:feeds.example")) + ); + assert_eq!( + cfg.appview_projection(), + Some(("http://localhost:2584", "did:web:appview.example")) + ); assert!(cfg.validate().is_ok()); + let mut audienceless = Config::try_parse_from( + REQUIRED + .into_iter() + .chain(["--feeds-url", "http://localhost:8080"]), + ) + .unwrap(); + assert!(audienceless.validate().is_err()); + audienceless + .try_update_from(["rsky-daemon", "--appview-url", "http://localhost:2584"]) + .unwrap(); + assert!(audienceless.validate().is_err()); + let discovery_only = Config::try_parse_from([ "rsky-daemon", "--space-host-url", diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index e48561ba..293b40e3 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -4,15 +4,18 @@ use clap::Parser; use rsky_daemon::config::Config; use rsky_daemon::engine::CommitKeyResolver; +use rsky_daemon::runner::SpaceWorkerParts; use rsky_daemon::{ - notify_router, CombinedSource, CredentialSource, DaemonError, HttpRepoHost, HttpSpaceHost, - HttpSpaceSource, InMemoryIndex, InternalCredentialProvider, MultiRunnerOptions, NotifyState, - Result, SpaceCredentialSource, SpaceIndex, SpaceRegistry, SqliteIndex, StaticCredential, - StaticSpaces, run_multi, + notify_router, run_multi, AppviewProjector, CombinedSource, CredentialSource, DaemonError, + FeedsProjector, HttpProjectionIngress, HttpRepoHost, HttpSpaceHost, HttpSpaceSource, + InMemoryIndex, InternalCredentialProvider, JournalConsumer, MultiRunnerOptions, NotifyState, + Result, Router, SharedJournalConsumer, SpaceCredentialSource, SpaceIndex, SpaceRegistry, + SqliteIndex, StaticCredential, StaticSpaces, }; use rsky_identity::did::atproto_data::{get_did_key_from_multibase, VerificationMaterial}; use rsky_identity::types::{IdentityResolverOpts, MemoryCache}; use rsky_identity::IdResolver; +use rsky_space::space_id::SpaceId; use std::sync::Arc; use tokio::sync::{mpsc, watch}; @@ -63,6 +66,51 @@ impl CommitKeyResolver for DidKeyResolver { } } +/// The projection destinations this process was configured with, if any. +struct ProjectionConfig { + service_identity: String, + signing_key_hex: String, + feeds: Option<(String, String)>, + appview: Option<(String, String)>, +} + +impl ProjectionConfig { + fn consumers(&self, space: &str) -> Result> { + if self.feeds.is_none() && self.appview.is_none() { + return Ok(Vec::new()); + } + let space_id = SpaceId::parse(space)?; + let mut consumers: Vec = Vec::new(); + if let Some((url, audience)) = &self.feeds { + let ingress = HttpProjectionIngress::new( + "feeds", + url, + &self.service_identity, + audience, + &self.signing_key_hex, + )?; + consumers.push(Arc::new(JournalConsumer::new( + Router::new(space_id.clone(), space_id.authority.clone()), + Box::new(FeedsProjector::new(ingress, space)), + ))); + } + if let Some((url, audience)) = &self.appview { + let ingress = HttpProjectionIngress::new( + "appview", + url, + &self.service_identity, + audience, + &self.signing_key_hex, + )?; + consumers.push(Arc::new(JournalConsumer::new( + Router::new(space_id.clone(), space_id.authority.clone()), + Box::new(AppviewProjector::new(ingress, space)), + ))); + } + Ok(consumers) + } +} + #[tokio::main] async fn main() -> std::result::Result<(), Box> { tracing_subscriber::fmt() @@ -85,7 +133,11 @@ async fn main() -> std::result::Result<(), Box> { let host = Arc::new(HttpSpaceHost::new(&cfg.space_host_url, dpop.clone())); let keys: Arc = Arc::new(DidKeyResolver::new(cfg.plc_url())); - let db = if cfg.index_db_path.is_empty() { None } else { Some(Arc::new(SqliteIndex::open(&cfg.index_db_path)?)) }; + let db = if cfg.index_db_path.is_empty() { + None + } else { + Some(Arc::new(SqliteIndex::open(&cfg.index_db_path)?)) + }; let shared_creds = if cfg.static_credential.is_empty() { if cfg.space_host_mint_token.is_empty() || cfg.service_signing_key_hex.is_empty() { @@ -108,8 +160,17 @@ async fn main() -> std::result::Result<(), Box> { }; let mut sources: Vec> = Vec::new(); - if !cfg.space_uri.is_empty() { sources.push(Box::new(StaticSpaces::new([cfg.space_uri.clone()]))); } - if !cfg.spaces_url.is_empty() { sources.push(Box::new(HttpSpaceSource::new(&cfg.spaces_url, &cfg.spaces_api_key, authority_filter.clone(), &cfg.space_type))); } + if !cfg.space_uri.is_empty() { + sources.push(Box::new(StaticSpaces::new([cfg.space_uri.clone()]))); + } + if !cfg.spaces_url.is_empty() { + sources.push(Box::new(HttpSpaceSource::new( + &cfg.spaces_url, + &cfg.spaces_api_key, + authority_filter.clone(), + &cfg.space_type, + ))); + } let source = Arc::new(CombinedSource(sources)); let registry = SpaceRegistry::new(); @@ -142,19 +203,58 @@ async fn main() -> std::result::Result<(), Box> { .await }); - let repo_host_base = cfg.repo_host_url().to_string(); let static_credential = cfg.static_credential.clone(); let db_for_factory = db.clone(); let dpop_for_factory = dpop.clone(); let shared_for_factory = shared_creds.clone(); - let factory = Arc::new(move |space: &str| -> Result<(Arc, rsky_daemon::runner::RepoHostFactory, Arc)> { - let creds: Arc = match &shared_for_factory { Some(provider) => Arc::new(SpaceCredentialSource::new(provider.clone(), space)), None => Arc::new(StaticCredential(static_credential.clone()) )}; - let index: Arc = match &db_for_factory { Some(db) => Arc::new(db.for_space(space)), None => Arc::new(InMemoryIndex::new()) }; - let base = repo_host_base.clone(); let proof = dpop_for_factory.clone(); - Ok((creds, Box::new(move |credential| Arc::new(HttpRepoHost::new(base.clone(), credential, proof.clone()))), index)) + let repo_host_base = cfg.repo_host_url().to_string(); + let static_credential = cfg.static_credential.clone(); + let db_for_factory = db.clone(); + let dpop_for_factory = dpop.clone(); + let shared_for_factory = shared_creds.clone(); + let projection = ProjectionConfig { + service_identity: cfg.service_identity.clone(), + signing_key_hex: cfg.service_signing_key_hex.clone(), + feeds: cfg + .feeds_projection() + .map(|(url, aud)| (url.to_string(), aud.to_string())), + appview: cfg + .appview_projection() + .map(|(url, aud)| (url.to_string(), aud.to_string())), + }; + let factory = Arc::new(move |space: &str| -> Result { + let creds: Arc = match &shared_for_factory { + Some(provider) => Arc::new(SpaceCredentialSource::new(provider.clone(), space)), + None => Arc::new(StaticCredential(static_credential.clone())), + }; + let index: Arc = match &db_for_factory { + Some(db) => Arc::new(db.for_space(space)), + None => Arc::new(InMemoryIndex::new()), + }; + let base = repo_host_base.clone(); + let proof = dpop_for_factory.clone(); + Ok(( + creds, + Box::new(move |credential| { + Arc::new(HttpRepoHost::new(base.clone(), credential, proof.clone())) + }), + index, + projection.consumers(space)?, + )) }); - let opts = MultiRunnerOptions { refresh_interval_secs: cfg.sweep_interval_secs, sweep_interval_secs: cfg.sweep_interval_secs, + let opts = MultiRunnerOptions { + refresh_interval_secs: cfg.sweep_interval_secs, + sweep_interval_secs: cfg.sweep_interval_secs, notify_endpoint: cfg.notify_endpoint(), service_identity: cfg.service_identity.clone(), now_fn: rsky_daemon::unix_now, }; - let runner = tokio::spawn(run_multi(opts, source, registry, factory, host, keys, notify_rx, shutdown_rx)); + let runner = tokio::spawn(run_multi( + opts, + source, + registry, + factory, + host, + keys, + notify_rx, + shutdown_rx, + )); tokio::signal::ctrl_c().await?; tracing::info!("ctrl-c received; shutting down"); diff --git a/rsky-daemon/src/runner.rs b/rsky-daemon/src/runner.rs index acd33bfc..80d6d7e4 100644 --- a/rsky-daemon/src/runner.rs +++ b/rsky-daemon/src/runner.rs @@ -13,6 +13,7 @@ use crate::credentials::CredentialSource; use crate::engine::{sync_repo, CommitKeyResolver, SyncOutcome}; use crate::error::{DaemonError, Result}; use crate::index::SpaceIndex; +use crate::journal::{drain_all, SharedJournalConsumer}; use crate::notify::WriteNotice; use crate::recovery::recover_repo; use crate::repohost::RepoHostClient; @@ -21,19 +22,19 @@ use crate::xrpc::SpaceHostClient; const REGISTER_RETRY_SECS: u64 = 30; const MIN_REREGISTER_SECS: u64 = 30; +/// How often pending batches are retried when no sync has happened, so a +/// destination that was down recovers without waiting for the next write. +const PROJECTION_DRAIN_SECS: u64 = 30; /// Builds a repo-host client bound to the current space credential. pub type RepoHostFactory = Box Arc + Send + Sync>; -pub type MultiSpaceFactory = Arc< - dyn Fn( - &str, - ) -> Result<( - Arc, - RepoHostFactory, - Arc, - )> + Send - + Sync, ->; +pub type SpaceWorkerParts = ( + Arc, + RepoHostFactory, + Arc, + Vec, +); +pub type MultiSpaceFactory = Arc Result + Send + Sync>; pub struct MultiRunnerOptions { pub refresh_interval_secs: u64, @@ -73,10 +74,10 @@ pub async fn run_multi( let stale: Vec<_> = workers.iter().filter(|(space, worker)| desired.get(*space).is_none_or(|target| target.generation != worker.generation)).map(|(space, _)| space.clone()).collect(); for space in stale { if let Some(worker) = workers.remove(&space) { let _ = worker.stop.send(true); let _ = worker.handle.await; } } for (space, target) in &desired { if workers.contains_key(space) { continue; } - let (creds, repo, index) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; + let (creds, repo, index, projectors) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; let (tx, rx) = mpsc::channel(256); let (stop, stop_rx) = watch::channel(false); let worker_opts = RunnerOptions { space_uri: space.clone(), sweep_interval_secs: opts.sweep_interval_secs, notify_endpoint: opts.notify_endpoint.clone(), service_identity: opts.service_identity.clone(), now_fn: opts.now_fn }; - let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), rx, stop_rx)); + let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), projectors, rx, stop_rx)); workers.insert(space.clone(), Worker { generation: target.generation, stop, notices: tx, handle }); } registry.replace(workers.keys().cloned().collect()); @@ -275,11 +276,13 @@ pub async fn run( make_repo_host: RepoHostFactory, index: Arc, keys: Arc, + projectors: Vec, mut notify_rx: mpsc::Receiver, mut shutdown: watch::Receiver, ) { let mut sweep_timer = tokio::time::interval(Duration::from_secs(opts.sweep_interval_secs.max(1))); + let mut drain_timer = tokio::time::interval(Duration::from_secs(PROJECTION_DRAIN_SECS)); let mut register_at = Instant::now(); loop { tokio::select! { @@ -301,6 +304,10 @@ pub async fn run( keys.as_ref(), ) .await; + drain_all(index.as_ref(), &projectors).await; + } + _ = drain_timer.tick() => { + drain_all(index.as_ref(), &projectors).await; } Some(notice) = notify_rx.recv() => { handle_notice( @@ -312,6 +319,7 @@ pub async fn run( notice, ) .await; + drain_all(index.as_ref(), &projectors).await; } } } @@ -834,6 +842,7 @@ mod tests { make_repo_host, index.clone(), keys, + Vec::new(), rx, shutdown_rx, )); @@ -863,6 +872,95 @@ mod tests { handle.await.unwrap(); } + #[tokio::test(start_paused = true)] + async fn a_worker_drains_each_projector_independently() { + use crate::feeds::tests::{post_created, CapturingIngress}; + use crate::feeds::FeedsProjector; + use crate::index::IndexMutation; + use crate::journal::JournalConsumer; + use crate::projection::Projector; + use crate::router::{Router, SyncEvent, POST_COLLECTION}; + use rsky_space::record::encode_record; + use rsky_space::space_id::SpaceId; + + struct Stalled; + #[async_trait] + impl Projector for Stalled { + fn name(&self) -> &'static str { + "appview" + } + async fn project(&self, _did: &str, _rev: &str, _events: &[SyncEvent]) -> Result<()> { + Err(DaemonError::RetryableProjection("down".to_string())) + } + } + + let space = SpaceId::parse(SPACE).unwrap(); + let router = || Router::new(space.clone(), space.authority.clone()); + let feeds = Arc::new(JournalConsumer::new( + router(), + Box::new(FeedsProjector::new(CapturingIngress::default(), SPACE)), + )); + let appview = Arc::new(JournalConsumer::new(router(), Box::new(Stalled))); + + let index = Arc::new(InMemoryIndex::new()); + index + .journal_batch( + AUTHOR, + "3krev", + &[IndexMutation::Upsert { + collection: POST_COLLECTION.to_string(), + rkey: "3ka".to_string(), + cid: "bafypost".to_string(), + rev: "3krev".to_string(), + value: Some( + encode_record( + &match post_created("3ka", "hello") { + SyncEvent::PostCreated { record, .. } => record, + _ => unreachable!("fixture is a post create"), + }, + 64 * 1024, + ) + .unwrap(), + ), + }], + ) + .await + .unwrap(); + + let a = author(); + let host = Arc::new(PagedSpaceHost::new(vec![ListReposOutput { + cursor: None, + repos: vec![], + }])); + let client: Arc = Arc::new(ScriptedRepoHost(HashMap::new())); + let make_repo_host: RepoHostFactory = Box::new(move |_| client.clone()); + let (_tx, rx) = mpsc::channel(8); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let handle = tokio::spawn(run( + options(3600), + host, + Arc::new(StaticCredential("sc.jwt".to_string())), + make_repo_host, + index.clone(), + Arc::new(FixedKey(a.did_key.clone())), + vec![feeds.clone(), appview.clone()], + rx, + shutdown_rx, + )); + + for _ in 0..200 { + if index.pending_batches("feeds").await.unwrap().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(index.pending_batches("feeds").await.unwrap().is_empty()); + assert_eq!(index.pending_batches("appview").await.unwrap().len(), 1); + + shutdown_tx.send(true).unwrap(); + handle.await.unwrap(); + } + struct FailingSpaceHost; #[async_trait] impl SpaceHostClient for FailingSpaceHost { @@ -917,6 +1015,7 @@ mod tests { make_repo_host, index, keys, + Vec::new(), rx, shutdown_rx, )); @@ -964,6 +1063,7 @@ mod tests { make_repo_host, index, keys, + Vec::new(), rx, shutdown_rx, )); From 1bf8a7c53039133728b9909ec313714be83f5fa5 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:44:34 -0400 Subject: [PATCH 25/56] feat(daemon): acknowledge observed sync so a space accepts projections --- rsky-daemon/src/feeds.rs | 50 +++++++++++++++---- rsky-daemon/src/journal.rs | 15 ++++-- rsky-daemon/src/lib.rs | 2 +- rsky-daemon/src/main.rs | 25 ++++++---- rsky-daemon/src/runner.rs | 98 ++++++++++++++++++++++++++++++++++---- 5 files changed, 160 insertions(+), 30 deletions(-) diff --git a/rsky-daemon/src/feeds.rs b/rsky-daemon/src/feeds.rs index 2d740a71..1a2bb664 100644 --- a/rsky-daemon/src/feeds.rs +++ b/rsky-daemon/src/feeds.rs @@ -14,6 +14,7 @@ use crate::service_jwt::ServiceJwtIssuer; use crate::unix_now; pub const PROJECT_RECORDS_LXM: &str = "community.blacksky.space.projectRecords"; +pub const ACK_SYNCERS_OBSERVED_LXM: &str = "community.blacksky.space.ackSyncersObserved"; /// The receiving side rejects a token whose lifetime exceeds five minutes; /// the margin absorbs clock skew between the two services. const TOKEN_TTL_SECS: u64 = 240; @@ -55,6 +56,20 @@ pub trait ProjectionIngress: Send + Sync { async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()>; } +#[async_trait] +impl ProjectionIngress for std::sync::Arc { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()> { + self.as_ref().project_records(request).await + } +} + +/// Tells the feed service this space now has a syncer keeping it current. +/// Until it hears that, it refuses the space's projections. +#[async_trait] +pub trait SpaceLifecycleAcker: Send + Sync { + async fn acknowledge_sync(&self, space: &str, generation: i64) -> Result<()>; +} + /// Posts batches to a service that accepts `projectRecords`, authenticated /// with this daemon's own service identity. pub struct HttpProjectionIngress { @@ -82,17 +97,34 @@ impl HttpProjectionIngress { }) } - fn service_jwt(&self) -> Result { + fn service_jwt(&self, lxm: &str) -> Result { static JTI: AtomicU64 = AtomicU64::new(1); let now = unix_now(); let jti = format!("{now}-{}", JTI.fetch_add(1, Ordering::Relaxed)); - self.issuer.mint_for( - &self.audience, - PROJECT_RECORDS_LXM, - now, - TOKEN_TTL_SECS, - &jti, - ) + self.issuer + .mint_for(&self.audience, lxm, now, TOKEN_TTL_SECS, &jti) + } +} + +#[async_trait] +impl SpaceLifecycleAcker for HttpProjectionIngress { + async fn acknowledge_sync(&self, space: &str, generation: i64) -> Result<()> { + let response = self + .http + .post(format!("{}/xrpc/{ACK_SYNCERS_OBSERVED_LXM}", self.base_url)) + .bearer_auth(self.service_jwt(ACK_SYNCERS_OBSERVED_LXM)?) + .json(&serde_json::json!({"space": space, "generation": generation})) + .send() + .await + .map_err(|error| DaemonError::Xrpc(error.to_string()))?; + if !response.status().is_success() { + return Err(DaemonError::Xrpc(format!( + "{} {ACK_SYNCERS_OBSERVED_LXM} returned {}", + self.target, + response.status() + ))); + } + Ok(()) } } @@ -102,7 +134,7 @@ impl ProjectionIngress for HttpProjectionIngress { let response = self .http .post(format!("{}/xrpc/{PROJECT_RECORDS_LXM}", self.base_url)) - .bearer_auth(self.service_jwt()?) + .bearer_auth(self.service_jwt(PROJECT_RECORDS_LXM)?) .json(request) .send() .await diff --git a/rsky-daemon/src/journal.rs b/rsky-daemon/src/journal.rs index e66d8fba..bacba71c 100644 --- a/rsky-daemon/src/journal.rs +++ b/rsky-daemon/src/journal.rs @@ -100,19 +100,26 @@ impl JournalConsumer { pub type SharedJournalConsumer = Arc; /// Drain every projector, then drop the journal rows all of them have passed. -pub async fn drain_all(index: &dyn SpaceIndex, consumers: &[SharedJournalConsumer]) { +/// Returns whether every destination accepted everything pending for it. +pub async fn drain_all(index: &dyn SpaceIndex, consumers: &[SharedJournalConsumer]) -> bool { + let mut succeeded = true; for consumer in consumers { - if let Err(error) = consumer.drain(index).await { - tracing::warn!(projector = consumer.name(), error = %error, "projection drain failed"); + match consumer.drain_succeeded(index).await { + Ok(clean) => succeeded &= clean, + Err(error) => { + succeeded = false; + tracing::warn!(projector = consumer.name(), error = %error, "projection drain failed"); + } } } if consumers.is_empty() { - return; + return succeeded; } let names: Vec<&str> = consumers.iter().map(|c| c.name()).collect(); if let Err(error) = index.prune_journal(&names).await { tracing::warn!(error = %error, "journal prune failed"); } + succeeded } #[cfg(test)] diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index 8e7a1843..6181924c 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -47,7 +47,7 @@ pub use engine::{sync_repo, CommitKeyResolver, SyncOutcome}; pub use error::{DaemonError, Result}; pub use feeds::{ FeedsProjector, HttpProjectionIngress, ProjectRecord, ProjectRecordsRequest, ProjectionIngress, - ProjectionOperation, + ProjectionOperation, SpaceLifecycleAcker, }; pub use index::{InMemoryIndex, IndexMutation, JournaledBatch, SpaceIndex}; pub use journal::{drain_all, JournalConsumer, SharedJournalConsumer}; diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 293b40e3..262af6bc 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -9,8 +9,8 @@ use rsky_daemon::{ notify_router, run_multi, AppviewProjector, CombinedSource, CredentialSource, DaemonError, FeedsProjector, HttpProjectionIngress, HttpRepoHost, HttpSpaceHost, HttpSpaceSource, InMemoryIndex, InternalCredentialProvider, JournalConsumer, MultiRunnerOptions, NotifyState, - Result, Router, SharedJournalConsumer, SpaceCredentialSource, SpaceIndex, SpaceRegistry, - SqliteIndex, StaticCredential, StaticSpaces, + Result, Router, SharedJournalConsumer, SpaceCredentialSource, SpaceIndex, SpaceLifecycleAcker, + SpaceRegistry, SqliteIndex, StaticCredential, StaticSpaces, }; use rsky_identity::did::atproto_data::{get_did_key_from_multibase, VerificationMaterial}; use rsky_identity::types::{IdentityResolverOpts, MemoryCache}; @@ -74,21 +74,28 @@ struct ProjectionConfig { appview: Option<(String, String)>, } +type ProjectionParts = ( + Vec, + Option>, +); + impl ProjectionConfig { - fn consumers(&self, space: &str) -> Result> { + fn consumers(&self, space: &str) -> Result { if self.feeds.is_none() && self.appview.is_none() { - return Ok(Vec::new()); + return Ok((Vec::new(), None)); } let space_id = SpaceId::parse(space)?; let mut consumers: Vec = Vec::new(); + let mut acker: Option> = None; if let Some((url, audience)) = &self.feeds { - let ingress = HttpProjectionIngress::new( + let ingress = Arc::new(HttpProjectionIngress::new( "feeds", url, &self.service_identity, audience, &self.signing_key_hex, - )?; + )?); + acker = Some(ingress.clone()); consumers.push(Arc::new(JournalConsumer::new( Router::new(space_id.clone(), space_id.authority.clone()), Box::new(FeedsProjector::new(ingress, space)), @@ -107,7 +114,7 @@ impl ProjectionConfig { Box::new(AppviewProjector::new(ingress, space)), ))); } - Ok(consumers) + Ok((consumers, acker)) } } @@ -229,13 +236,15 @@ async fn main() -> std::result::Result<(), Box> { }; let base = repo_host_base.clone(); let proof = dpop_for_factory.clone(); + let (projectors, acker) = projection.consumers(space)?; Ok(( creds, Box::new(move |credential| { Arc::new(HttpRepoHost::new(base.clone(), credential, proof.clone())) }), index, - projection.consumers(space)?, + projectors, + acker, )) }); let opts = MultiRunnerOptions { diff --git a/rsky-daemon/src/runner.rs b/rsky-daemon/src/runner.rs index 80d6d7e4..f1fd736b 100644 --- a/rsky-daemon/src/runner.rs +++ b/rsky-daemon/src/runner.rs @@ -12,6 +12,7 @@ use tokio::time::Instant; use crate::credentials::CredentialSource; use crate::engine::{sync_repo, CommitKeyResolver, SyncOutcome}; use crate::error::{DaemonError, Result}; +use crate::feeds::SpaceLifecycleAcker; use crate::index::SpaceIndex; use crate::journal::{drain_all, SharedJournalConsumer}; use crate::notify::WriteNotice; @@ -33,6 +34,7 @@ pub type SpaceWorkerParts = ( RepoHostFactory, Arc, Vec, + Option>, ); pub type MultiSpaceFactory = Arc Result + Send + Sync>; @@ -74,10 +76,10 @@ pub async fn run_multi( let stale: Vec<_> = workers.iter().filter(|(space, worker)| desired.get(*space).is_none_or(|target| target.generation != worker.generation)).map(|(space, _)| space.clone()).collect(); for space in stale { if let Some(worker) = workers.remove(&space) { let _ = worker.stop.send(true); let _ = worker.handle.await; } } for (space, target) in &desired { if workers.contains_key(space) { continue; } - let (creds, repo, index, projectors) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; + let (creds, repo, index, projectors, acker) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; let (tx, rx) = mpsc::channel(256); let (stop, stop_rx) = watch::channel(false); - let worker_opts = RunnerOptions { space_uri: space.clone(), sweep_interval_secs: opts.sweep_interval_secs, notify_endpoint: opts.notify_endpoint.clone(), service_identity: opts.service_identity.clone(), now_fn: opts.now_fn }; - let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), projectors, rx, stop_rx)); + let worker_opts = RunnerOptions { space_uri: space.clone(), sweep_interval_secs: opts.sweep_interval_secs, notify_endpoint: opts.notify_endpoint.clone(), service_identity: opts.service_identity.clone(), generation: target.generation, now_fn: opts.now_fn }; + let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), projectors, acker, rx, stop_rx)); workers.insert(space.clone(), Worker { generation: target.generation, stop, notices: tx, handle }); } registry.replace(workers.keys().cloned().collect()); @@ -180,6 +182,9 @@ pub struct RunnerOptions { /// This syncer's own service identifier, so the host can address its /// deliveries to it. pub service_identity: String, + /// The space's generation as the managing app reported it; an + /// acknowledgement names the generation it observed. + pub generation: i64, pub now_fn: fn() -> u64, } @@ -221,7 +226,7 @@ async fn sweep( make_repo_host: &RepoHostFactory, index: &dyn SpaceIndex, keys: &dyn CommitKeyResolver, -) { +) -> bool { let attempt = async { let credential = creds.credential((opts.now_fn)()).await?; let client = make_repo_host(credential.clone()); @@ -236,8 +241,14 @@ async fn sweep( .await }; match attempt.await { - Ok(r) => tracing::info!(synced = %r.synced, recovered = %r.recovered, "sweep complete"), - Err(e) => tracing::warn!(error = %e, "sweep failed"), + Ok(r) => { + tracing::info!(synced = %r.synced, recovered = %r.recovered, "sweep complete"); + true + } + Err(e) => { + tracing::warn!(error = %e, "sweep failed"); + false + } } } @@ -277,6 +288,7 @@ pub async fn run( index: Arc, keys: Arc, projectors: Vec, + lifecycle_acker: Option>, mut notify_rx: mpsc::Receiver, mut shutdown: watch::Receiver, ) { @@ -284,6 +296,7 @@ pub async fn run( tokio::time::interval(Duration::from_secs(opts.sweep_interval_secs.max(1))); let mut drain_timer = tokio::time::interval(Duration::from_secs(PROJECTION_DRAIN_SECS)); let mut register_at = Instant::now(); + let mut acknowledged = false; loop { tokio::select! { _ = shutdown.changed() => { @@ -295,7 +308,7 @@ pub async fn run( + register(&opts, host.as_ref(), creds.as_ref()).await; } _ = sweep_timer.tick() => { - sweep( + let swept = sweep( &opts, host.as_ref(), creds.as_ref(), @@ -304,7 +317,15 @@ pub async fn run( keys.as_ref(), ) .await; - drain_all(index.as_ref(), &projectors).await; + let projected = drain_all(index.as_ref(), &projectors).await; + if swept && projected && !acknowledged { + if let Some(acker) = &lifecycle_acker { + match acker.acknowledge_sync(&opts.space_uri, opts.generation).await { + Ok(()) => acknowledged = true, + Err(error) => tracing::warn!(error = %error, "space lifecycle acknowledgement failed"), + } + } + } } _ = drain_timer.tick() => { drain_all(index.as_ref(), &projectors).await; @@ -805,6 +826,7 @@ mod tests { sweep_interval_secs: sweep_secs, notify_endpoint: "https://syncer.example/notify".to_string(), service_identity: "did:web:syncer.example".to_string(), + generation: 1, now_fn: fixed_now, } } @@ -843,6 +865,7 @@ mod tests { index.clone(), keys, Vec::new(), + None, rx, shutdown_rx, )); @@ -944,6 +967,7 @@ mod tests { index.clone(), Arc::new(FixedKey(a.did_key.clone())), vec![feeds.clone(), appview.clone()], + None, rx, shutdown_rx, )); @@ -961,6 +985,62 @@ mod tests { handle.await.unwrap(); } + #[tokio::test(start_paused = true)] + async fn a_clean_sweep_acknowledges_the_space_generation_once() { + #[derive(Default)] + struct RecordingAcker(std::sync::Mutex>); + #[async_trait] + impl SpaceLifecycleAcker for RecordingAcker { + async fn acknowledge_sync(&self, space: &str, generation: i64) -> Result<()> { + self.0.lock().unwrap().push((space.to_string(), generation)); + Ok(()) + } + } + + let a = author(); + let host = Arc::new(PagedSpaceHost::new(vec![ListReposOutput { + cursor: None, + repos: vec![], + }])); + let client: Arc = Arc::new(ScriptedRepoHost(HashMap::new())); + let make_repo_host: RepoHostFactory = Box::new(move |_| client.clone()); + let acker = Arc::new(RecordingAcker::default()); + let (_tx, rx) = mpsc::channel(8); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let handle = tokio::spawn(run( + RunnerOptions { + sweep_interval_secs: 1, + generation: 7, + ..options(1) + }, + host, + Arc::new(StaticCredential("sc.jwt".to_string())), + make_repo_host, + Arc::new(InMemoryIndex::new()), + Arc::new(FixedKey(a.did_key.clone())), + Vec::new(), + Some(acker.clone()), + rx, + shutdown_rx, + )); + + for _ in 0..200 { + if !acker.0.lock().unwrap().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + tokio::time::sleep(Duration::from_secs(5)).await; + shutdown_tx.send(true).unwrap(); + handle.await.unwrap(); + + assert_eq!( + *acker.0.lock().unwrap(), + vec![(SPACE.to_string(), 7_i64)], + "the acknowledgement is sent once, not on every sweep" + ); + } + struct FailingSpaceHost; #[async_trait] impl SpaceHostClient for FailingSpaceHost { @@ -1016,6 +1096,7 @@ mod tests { index, keys, Vec::new(), + None, rx, shutdown_rx, )); @@ -1064,6 +1145,7 @@ mod tests { index, keys, Vec::new(), + None, rx, shutdown_rx, )); From b8dc4f5d2e6a84865102dac8b0bc2984a95864ce Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:48:00 -0400 Subject: [PATCH 26/56] fix(daemon): accept oplog values inlined as json when routing --- rsky-daemon/src/router.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/rsky-daemon/src/router.rs b/rsky-daemon/src/router.rs index c67c8348..c27b3a86 100644 --- a/rsky-daemon/src/router.rs +++ b/rsky-daemon/src/router.rs @@ -170,9 +170,30 @@ impl Router { } } +/// The two sync paths inline a record's value in different encodings: the +/// oplog carries it as JSON, a full-state CAR as DAG-CBOR. Either may be what +/// the index holds for a given record, so both are accepted here. +fn decode_inlined(bytes: &[u8]) -> std::result::Result { + match decode_record(bytes) { + Ok(value) if value.is_object() => Ok(value), + cbor => serde_json::from_slice::(bytes) + .map_err(|json_err| match cbor { + Ok(_) => format!("dag-cbor value is not a record; json: {json_err}"), + Err(cbor_err) => format!("dag-cbor: {cbor_err}; json: {json_err}"), + }) + .and_then(|value| { + if value.is_object() { + Ok(value) + } else { + Err("value is not a record".to_string()) + } + }), + } +} + fn decode_value(uri: &str, value: Option<&[u8]>) -> Option { match value { - Some(bytes) => match decode_record(bytes) { + Some(bytes) => match decode_inlined(bytes) { Ok(value) => Some(value), Err(err) => { let total = KNOWN_COLLECTION_DECODE_FAILURES.fetch_add(1, Ordering::Relaxed) + 1; @@ -309,6 +330,18 @@ mod tests { )); } + #[test] + fn a_value_inlined_as_json_routes_like_one_inlined_as_dag_cbor() { + let mut json_valued = upsert(POST_COLLECTION, post()); + if let IndexMutation::Upsert { value, .. } = &mut json_valued { + *value = Some(serde_json::to_vec(&post()).unwrap()); + } + match router().route(MEMBER, &json_valued) { + Some(SyncEvent::PostCreated { record, .. }) => assert_eq!(record["text"], "hello"), + other => panic!("expected a post create, got {other:?}"), + } + } + #[test] fn only_unknown_collections_may_drop_silently() { let r = router(); From f1958bf894847ab7b2e9aa23f79f5b91ddaa1372 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 17:50:07 -0400 Subject: [PATCH 27/56] chore(daemon): silence the arity lint on the projection op builder --- Cargo.lock | 2 +- rsky-daemon/src/appview.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3edc04ee..fa3b5954 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8005,7 +8005,7 @@ dependencies = [ [[package]] name = "rsky-daemon" -version = "0.5.1" +version = "0.6.0" dependencies = [ "async-trait", "axum", diff --git a/rsky-daemon/src/appview.rs b/rsky-daemon/src/appview.rs index 7f7514b8..70f079ae 100644 --- a/rsky-daemon/src/appview.rs +++ b/rsky-daemon/src/appview.rs @@ -23,6 +23,7 @@ impl AppviewProjector { } } + #[allow(clippy::too_many_arguments)] fn op( &self, author: &str, From f15043eea686cdea105bba53c86de6a0da66b6cc Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 18:17:25 -0400 Subject: [PATCH 28/56] chore(daemon): add container build --- rsky-daemon/Dockerfile | 80 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 rsky-daemon/Dockerfile diff --git a/rsky-daemon/Dockerfile b/rsky-daemon/Dockerfile new file mode 100644 index 00000000..2687940f --- /dev/null +++ b/rsky-daemon/Dockerfile @@ -0,0 +1,80 @@ +# Use the official Rust image. +# https://hub.docker.com/_/rust +FROM rust AS builder + +WORKDIR /usr/src/rsky + +# Copy workspace and all crate manifests for dependency resolution +COPY Cargo.toml Cargo.lock rust-toolchain ./ +COPY palomar-sync/Cargo.toml palomar-sync/Cargo.toml +COPY rsky-common/Cargo.toml rsky-common/Cargo.toml +COPY rsky-crypto/Cargo.toml rsky-crypto/Cargo.toml +COPY rsky-daemon/Cargo.toml rsky-daemon/Cargo.toml +COPY rsky-feedgen/Cargo.toml rsky-feedgen/Cargo.toml +COPY rsky-firehose/Cargo.toml rsky-firehose/Cargo.toml +COPY rsky-identity/Cargo.toml rsky-identity/Cargo.toml +COPY rsky-jetstream-subscriber/Cargo.toml rsky-jetstream-subscriber/Cargo.toml +COPY rsky-labeler/Cargo.toml rsky-labeler/Cargo.toml +COPY rsky-lexicon/Cargo.toml rsky-lexicon/Cargo.toml +COPY rsky-oauth/Cargo.toml rsky-oauth/Cargo.toml +COPY rsky-pds/Cargo.toml rsky-pds/Cargo.toml +COPY rsky-relay/Cargo.toml rsky-relay/Cargo.toml +COPY rsky-repo/Cargo.toml rsky-repo/Cargo.toml +COPY rsky-satnav/Cargo.toml rsky-satnav/Cargo.toml +COPY rsky-space/Cargo.toml rsky-space/Cargo.toml +COPY rsky-space-host/Cargo.toml rsky-space-host/Cargo.toml +COPY rsky-syntax/Cargo.toml rsky-syntax/Cargo.toml +COPY rsky-video/Cargo.toml rsky-video/Cargo.toml +COPY rsky-wintermute/Cargo.toml rsky-wintermute/Cargo.toml + +# Copy real source for library crates that rsky-daemon depends on +COPY rsky-common/src rsky-common/src +COPY rsky-crypto/src rsky-crypto/src +COPY rsky-identity/src rsky-identity/src +COPY rsky-lexicon/src rsky-lexicon/src +COPY rsky-oauth/src rsky-oauth/src +COPY rsky-space/src rsky-space/src +COPY rsky-syntax/src rsky-syntax/src + +# Stub out the remaining workspace members so cargo can resolve the workspace +RUN mkdir -p \ + palomar-sync/src rsky-feedgen/src rsky-firehose/src \ + rsky-jetstream-subscriber/src rsky-labeler/src rsky-pds/src \ + rsky-relay/src rsky-repo/src rsky-satnav/src rsky-space-host/src \ + rsky-video/src rsky-wintermute/src && \ + for crate in palomar-sync rsky-feedgen rsky-firehose \ + rsky-jetstream-subscriber rsky-labeler rsky-pds rsky-relay \ + rsky-satnav rsky-space-host rsky-video rsky-wintermute; do \ + echo 'fn main() {}' > $crate/src/main.rs; \ + done && \ + touch rsky-pds/src/lib.rs rsky-repo/src/lib.rs && \ + mkdir -p rsky-wintermute/src/bin && \ + for bin in queue_backfill fix_blob_refs plc_import label_sync car_loader \ + cleanup_stale_deactivated; do \ + echo 'fn main() {}' > rsky-wintermute/src/bin/$bin.rs; \ + done + +# Create an empty src directory to trick Cargo into thinking it's a valid Rust project +RUN mkdir -p rsky-daemon/src && echo "fn main() {}" > rsky-daemon/src/main.rs + +# Install production dependencies and build a release artifact. +RUN cargo build --release --package rsky-daemon + +# Now copy the real source code and build the final binary +COPY rsky-daemon/src rsky-daemon/src + +RUN cargo build --release --package rsky-daemon + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /usr/src/rsky +COPY --from=builder /usr/src/rsky/target/release/rsky-daemon rsky-daemon +LABEL org.opencontainers.image.source=https://github.com/blacksky-algorithms/rsky +# State is sqlite + a persisted key: mount a volume covering DAEMON_INDEX_DB_PATH +# and DAEMON_DPOP_KEY_PATH. Required env: DAEMON_SPACE_HOST_URL, +# DAEMON_SERVICE_IDENTITY, DAEMON_SERVICE_SIGNING_KEY_HEX, +# DAEMON_SPACE_HOST_MINT_TOKEN, and DAEMON_SPACE_URI and/or DAEMON_SPACES_URL +# (+ DAEMON_SPACES_API_KEY). See rsky-daemon/src/config.rs for the full reference. +CMD ["sh", "-c", "DAEMON_NOTIFY_BIND=${DAEMON_NOTIFY_BIND:-0.0.0.0:8055} ./rsky-daemon"] From 6da61ceeffb9a78100345815e0207afa8eab411f Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 19 Aug 2026 18:17:25 -0400 Subject: [PATCH 29/56] chore(space-host): add container build --- rsky-space-host/Dockerfile | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 rsky-space-host/Dockerfile diff --git a/rsky-space-host/Dockerfile b/rsky-space-host/Dockerfile new file mode 100644 index 00000000..acf71996 --- /dev/null +++ b/rsky-space-host/Dockerfile @@ -0,0 +1,81 @@ +# Use the official Rust image. +# https://hub.docker.com/_/rust +FROM rust AS builder + +WORKDIR /usr/src/rsky + +# Copy workspace and all crate manifests for dependency resolution +COPY Cargo.toml Cargo.lock rust-toolchain ./ +COPY palomar-sync/Cargo.toml palomar-sync/Cargo.toml +COPY rsky-common/Cargo.toml rsky-common/Cargo.toml +COPY rsky-crypto/Cargo.toml rsky-crypto/Cargo.toml +COPY rsky-daemon/Cargo.toml rsky-daemon/Cargo.toml +COPY rsky-feedgen/Cargo.toml rsky-feedgen/Cargo.toml +COPY rsky-firehose/Cargo.toml rsky-firehose/Cargo.toml +COPY rsky-identity/Cargo.toml rsky-identity/Cargo.toml +COPY rsky-jetstream-subscriber/Cargo.toml rsky-jetstream-subscriber/Cargo.toml +COPY rsky-labeler/Cargo.toml rsky-labeler/Cargo.toml +COPY rsky-lexicon/Cargo.toml rsky-lexicon/Cargo.toml +COPY rsky-oauth/Cargo.toml rsky-oauth/Cargo.toml +COPY rsky-pds/Cargo.toml rsky-pds/Cargo.toml +COPY rsky-relay/Cargo.toml rsky-relay/Cargo.toml +COPY rsky-repo/Cargo.toml rsky-repo/Cargo.toml +COPY rsky-satnav/Cargo.toml rsky-satnav/Cargo.toml +COPY rsky-space/Cargo.toml rsky-space/Cargo.toml +COPY rsky-space-host/Cargo.toml rsky-space-host/Cargo.toml +COPY rsky-syntax/Cargo.toml rsky-syntax/Cargo.toml +COPY rsky-video/Cargo.toml rsky-video/Cargo.toml +COPY rsky-wintermute/Cargo.toml rsky-wintermute/Cargo.toml + +# Copy real source for library crates that rsky-space-host depends on +COPY rsky-common/src rsky-common/src +COPY rsky-crypto/src rsky-crypto/src +COPY rsky-identity/src rsky-identity/src +COPY rsky-lexicon/src rsky-lexicon/src +COPY rsky-oauth/src rsky-oauth/src +COPY rsky-space/src rsky-space/src +COPY rsky-syntax/src rsky-syntax/src + +# Stub out the remaining workspace members so cargo can resolve the workspace +RUN mkdir -p \ + palomar-sync/src rsky-daemon/src rsky-feedgen/src rsky-firehose/src \ + rsky-jetstream-subscriber/src rsky-labeler/src rsky-pds/src \ + rsky-relay/src rsky-repo/src rsky-satnav/src rsky-video/src \ + rsky-wintermute/src && \ + for crate in palomar-sync rsky-daemon rsky-feedgen rsky-firehose \ + rsky-jetstream-subscriber rsky-labeler rsky-pds rsky-relay \ + rsky-satnav rsky-video rsky-wintermute; do \ + echo 'fn main() {}' > $crate/src/main.rs; \ + done && \ + touch rsky-pds/src/lib.rs rsky-repo/src/lib.rs && \ + mkdir -p rsky-wintermute/src/bin && \ + for bin in queue_backfill fix_blob_refs plc_import label_sync car_loader \ + cleanup_stale_deactivated; do \ + echo 'fn main() {}' > rsky-wintermute/src/bin/$bin.rs; \ + done + +# Create an empty src directory to trick Cargo into thinking it's a valid Rust project +RUN mkdir -p rsky-space-host/src && echo "fn main() {}" > rsky-space-host/src/main.rs + +# Install production dependencies and build a release artifact. +RUN cargo build --release --package rsky-space-host + +# Now copy the real source code and build the final binary +COPY rsky-space-host/src rsky-space-host/src + +RUN cargo build --release --package rsky-space-host + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /usr/src/rsky +COPY --from=builder /usr/src/rsky/target/release/rsky-space-host rsky-space-host +LABEL org.opencontainers.image.source=https://github.com/blacksky-algorithms/rsky +# State is sqlite: mount a volume covering SPACEHOST_DB_PATH, and mount the PDS +# actor store read-only at SPACEHOST_ACTOR_STORE_DIR. Required env: +# SPACEHOST_PUBLIC_URL, SPACEHOST_MINT_TOKEN, and either the +# SPACEHOST_AUTHORITY_DID/SPACEHOST_SIGNING_KEY_HEX bootstrap pin or an actor +# store to acquire authorities from. See rsky-space-host/src/config.rs for the +# full reference. +CMD ["sh", "-c", "SPACEHOST_BIND=${SPACEHOST_BIND:-0.0.0.0:3600} ./rsky-space-host"] From ab1c8ef4a006b4017c7087835b5feb55d9346cd1 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 13:10:40 -0400 Subject: [PATCH 30/56] test(spaces-parity): add differential parity harness, S1-S5 red --- Cargo.lock | 380 +++++++++++++++++++++++++---- Cargo.toml | 1 + rsky-spaces-parity/Cargo.toml | 16 ++ rsky-spaces-parity/src/lib.rs | 83 +++++++ rsky-spaces-parity/tests/parity.rs | 206 ++++++++++++++++ 5 files changed, 638 insertions(+), 48 deletions(-) create mode 100644 rsky-spaces-parity/Cargo.toml create mode 100644 rsky-spaces-parity/src/lib.rs create mode 100644 rsky-spaces-parity/tests/parity.rs diff --git a/Cargo.lock b/Cargo.lock index fa3b5954..9fd0621a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7978,7 +7978,7 @@ dependencies = [ "rand 0.8.5", "rand_core 0.6.4", "regex", - "rsky-identity", + "rsky-identity 0.2.0", "secp256k1", "serde", "serde_ipld_dagcbor", @@ -7991,6 +7991,34 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "rsky-common" +version = "0.1.3" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "base64ct", + "chrono", + "cid", + "futures", + "indexmap 1.9.3", + "multihash", + "multihash-codetable", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "rsky-identity 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "secp256k1", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.16", + "tracing", + "url", + "urlencoding", +] + [[package]] name = "rsky-crypto" version = "0.2.0" @@ -8003,6 +8031,18 @@ dependencies = [ "unsigned-varint 0.8.0", ] +[[package]] +name = "rsky-crypto" +version = "0.2.0" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "multibase", + "p256 0.13.2", + "secp256k1", + "unsigned-varint 0.8.0", +] + [[package]] name = "rsky-daemon" version = "0.6.0" @@ -8016,13 +8056,13 @@ dependencies = [ "hex", "rand 0.8.5", "reqwest 0.12.23", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-oauth", - "rsky-space", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-oauth 0.3.2", + "rsky-space 0.4.2", + "rsky-syntax 0.1.0", "rusqlite", "secp256k1", "serde", @@ -8055,8 +8095,8 @@ dependencies = [ "reqwest 0.11.27", "rocket", "rocket_sync_db_pools", - "rsky-common", - "rsky-lexicon", + "rsky-common 0.1.3", + "rsky-lexicon 0.10.2", "serde", "serde_bytes", "serde_cbor", @@ -8083,7 +8123,7 @@ dependencies = [ "parking_lot", "reqwest 0.11.27", "retry", - "rsky-lexicon", + "rsky-lexicon 0.10.2", "serde", "serde_bytes", "serde_cbor", @@ -8106,7 +8146,7 @@ dependencies = [ "hickory-resolver 0.24.4", "multibase", "reqwest 0.12.23", - "rsky-crypto", + "rsky-crypto 0.2.0", "secp256k1", "serde", "serde_json", @@ -8116,6 +8156,23 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "rsky-identity" +version = "0.2.0" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "async-trait", + "hickory-resolver 0.24.4", + "reqwest 0.12.23", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "serde", + "serde_json", + "thiserror 1.0.69", + "url", + "urlencoding", +] + [[package]] name = "rsky-jetstream-subscriber" version = "0.1.0" @@ -8125,7 +8182,7 @@ dependencies = [ "dotenvy", "futures", "reqwest 0.11.27", - "rsky-lexicon", + "rsky-lexicon 0.10.2", "serde", "serde_derive", "serde_json", @@ -8155,8 +8212,8 @@ dependencies = [ "parking_lot", "reqwest 0.12.23", "retry", - "rsky-common", - "rsky-lexicon", + "rsky-common 0.1.3", + "rsky-lexicon 0.10.2", "serde", "serde_bytes", "serde_cbor", @@ -8190,6 +8247,27 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "rsky-lexicon" +version = "0.10.2" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "base64 0.22.1", + "chrono", + "cid", + "derive_builder 0.12.0", + "miette 5.10.0", + "parking_lot", + "secp256k1", + "serde", + "serde_bytes", + "serde_cbor", + "serde_derive", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "rsky-oauth" version = "0.3.2" @@ -8199,7 +8277,7 @@ dependencies = [ "hex", "hmac", "p256 0.13.2", - "rsky-crypto", + "rsky-crypto 0.2.0", "secp256k1", "serde", "serde_json", @@ -8209,6 +8287,25 @@ dependencies = [ "url", ] +[[package]] +name = "rsky-oauth" +version = "0.3.2" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "async-trait", + "base64 0.22.1", + "hex", + "hmac", + "p256 0.13.2", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "secp256k1", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.16", + "url", +] + [[package]] name = "rsky-pds" version = "0.13.17" @@ -8250,15 +8347,15 @@ dependencies = [ "reqwest 0.12.23", "rocket", "rocket_ws", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-oauth", - "rsky-repo", - "rsky-space", - "rsky-space-host", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-oauth 0.3.2", + "rsky-repo 0.0.6", + "rsky-space 0.4.2", + "rsky-space-host 0.7.1", + "rsky-syntax 0.1.0", "rusqlite", "secp256k1", "serde", @@ -8279,6 +8376,75 @@ dependencies = [ "url", ] +[[package]] +name = "rsky-pds" +version = "0.13.17" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "argon2", + "askama", + "async-event-emitter", + "async-trait", + "atrium-api 0.24.10", + "atrium-xrpc-client", + "aws-config", + "aws-sdk-s3", + "base64 0.22.1", + "base64-url", + "base64ct", + "chrono", + "cid", + "data-encoding", + "dotenvy", + "email_address", + "event-emitter-rs", + "futures", + "hex", + "hickory-resolver 0.24.4", + "image", + "indexmap 1.9.3", + "infer 0.15.0", + "ipld-core", + "jwt-simple", + "lazy_static", + "lru 0.14.0", + "mailchecker", + "mailgun-rs", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "reqwest 0.12.23", + "rocket", + "rocket_ws", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-identity 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-lexicon 0.10.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-oauth 0.3.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-repo 0.0.6 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-space 0.4.1", + "rsky-space-host 0.5.1", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rusqlite", + "secp256k1", + "serde", + "serde_bytes", + "serde_cbor", + "serde_derive", + "serde_ipld_dagcbor", + "serde_json", + "serde_repr", + "sha2 0.10.9", + "thiserror 1.0.69", + "time", + "tokio", + "toml 0.8.23", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "rsky-relay" version = "0.1.2" @@ -8310,8 +8476,8 @@ dependencies = [ "p256 0.13.2", "reqwest 0.12.23", "rs-car-sync", - "rsky-common", - "rsky-identity", + "rsky-common 0.1.3", + "rsky-identity 0.2.0", "rtrb", "rusqlite", "rustls 0.23.31", @@ -8356,10 +8522,44 @@ dependencies = [ "rand 0.8.5", "rand_core 0.6.4", "regex", - "rsky-common", - "rsky-crypto", - "rsky-lexicon", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-syntax 0.1.0", + "secp256k1", + "serde", + "serde_bytes", + "serde_cbor", + "serde_derive", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.10.9", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "rsky-repo" +version = "0.0.6" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "async-recursion", + "async-stream", + "async-trait", + "cid", + "futures", + "integer-encoding", + "ipld-core", + "iroh-car", + "lazy_static", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-lexicon 0.10.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "secp256k1", "serde", "serde_bytes", @@ -8393,6 +8593,31 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rsky-space" +version = "0.4.1" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "base64 0.22.1", + "blake3", + "cid", + "hex", + "hkdf", + "hmac", + "iroh-car", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.10.9", + "subtle", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "rsky-space" version = "0.4.2" @@ -8406,9 +8631,9 @@ dependencies = [ "ipld-core", "iroh-car", "p256 0.13.2", - "rsky-common", - "rsky-crypto", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-syntax 0.1.0", "secp256k1", "serde", "serde_bytes", @@ -8420,6 +8645,37 @@ dependencies = [ "tokio", ] +[[package]] +name = "rsky-space-host" +version = "0.5.1" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "chrono", + "clap", + "hex", + "rand 0.8.5", + "reqwest 0.12.23", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-identity 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-lexicon 0.10.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-oauth 0.3.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-space 0.4.1", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rusqlite", + "secp256k1", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "rsky-space-host" version = "0.7.1" @@ -8436,13 +8692,13 @@ dependencies = [ "p256 0.13.2", "rand 0.8.5", "reqwest 0.12.23", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-oauth", - "rsky-space", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-oauth 0.3.2", + "rsky-space 0.4.2", + "rsky-syntax 0.1.0", "rusqlite", "secp256k1", "serde", @@ -8458,9 +8714,37 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rsky-spaces-parity" +version = "0.1.0" +dependencies = [ + "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-space 0.4.1", + "rsky-space 0.4.2", + "rsky-space-host 0.7.1", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "rsky-syntax" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "lazy_static", + "regex", + "serde", + "serde_derive", + "thiserror 1.0.69", + "url", +] + [[package]] name = "rsky-syntax" version = "0.1.0" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" dependencies = [ "anyhow", "chrono", @@ -8495,7 +8779,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "reqwest 0.12.23", - "rsky-syntax", + "rsky-syntax 0.1.0", "rustls 0.23.31", "sec1 0.7.3", "serde", @@ -8542,12 +8826,12 @@ dependencies = [ "prometheus", "rand 0.8.5", "reqwest 0.12.23", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-repo", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-repo 0.0.6", + "rsky-syntax 0.1.0", "rtrb", "rusqlite", "rustls 0.23.31", diff --git a/Cargo.toml b/Cargo.toml index b9fb3f6d..ebdb0533 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "rsky-satnav", "rsky-space", "rsky-space-host", + "rsky-spaces-parity", "rsky-syntax", "rsky-video", "rsky-wintermute", diff --git a/rsky-spaces-parity/Cargo.toml b/rsky-spaces-parity/Cargo.toml new file mode 100644 index 00000000..94b15b9c --- /dev/null +++ b/rsky-spaces-parity/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rsky-spaces-parity" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +rsky-space-host = { path = "../rsky-space-host" } +rsky-space = { path = "../rsky-space" } +oracle-rsky-space = { package = "rsky-space", git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } +rsky-pds = { git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } +serde_json = { workspace = true } +tokio = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/rsky-spaces-parity/src/lib.rs b/rsky-spaces-parity/src/lib.rs new file mode 100644 index 00000000..a65fc2d3 --- /dev/null +++ b/rsky-spaces-parity/src/lib.rs @@ -0,0 +1,83 @@ +use rsky_pds::actor_store::space::SpaceStore; +use rsky_space_host::repo::RepoStore; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoDump { + pub records: Vec<(String, String, String, Vec)>, + pub lthash_state: Vec, + pub ops: Vec<(String, String, Option, Option)>, +} + +pub async fn dump_shim(store: &dyn RepoStore, space_uri: &str, did: &str) -> RepoDump { + let (records, _) = store + .list_records(space_uri, did, None, None, u32::MAX) + .await + .expect("shim records"); + let head = store + .head(space_uri, did) + .await + .expect("shim head") + .expect("shim repo"); + let ops = store + .list_ops(space_uri, did, None, None, u32::MAX) + .await + .expect("shim ops"); + RepoDump { + records: records + .into_iter() + .map(|r| (r.collection, r.rkey, r.cid, r.value)) + .collect(), + lthash_state: head.state.to_vec(), + ops: ops + .ops + .into_iter() + .map(|o| (o.collection, o.rkey, o.cid, o.prev)) + .collect(), + } +} + +pub async fn dump_pds(store: &SpaceStore, space_uri: &str) -> RepoDump { + let records = store.all_records(space_uri).await.expect("pds records"); + let state = store + .live_repo_state(space_uri) + .await + .expect("pds repo state"); + let (ops, _) = store + .list_repo_ops(space_uri, None, None, usize::MAX >> 1) + .await + .expect("pds ops"); + RepoDump { + records: records + .into_iter() + .map(|r| (r.collection, r.rkey, r.cid, r.value)) + .collect(), + lthash_state: state.lthash_state, + ops: ops + .into_iter() + .map(|o| (o.collection, o.rkey, o.cid, o.prev)) + .collect(), + } +} + +pub fn assert_parity(name: &str, shim: &RepoDump, pds: &RepoDump) -> bool { + let mut equal = true; + for (field, left, right) in [ + ( + "records", + format!("{:?}", shim.records), + format!("{:?}", pds.records), + ), + ( + "lthash_state", + format!("{:?}", shim.lthash_state), + format!("{:?}", pds.lthash_state), + ), + ("ops", format!("{:?}", shim.ops), format!("{:?}", pds.ops)), + ] { + if left != right { + eprintln!("{name}: {field} differs\n shim: {left}\n pds: {right}"); + equal = false; + } + } + equal +} diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs new file mode 100644 index 00000000..e96d4cc1 --- /dev/null +++ b/rsky-spaces-parity/tests/parity.rs @@ -0,0 +1,206 @@ +use oracle_rsky_space::space_id::SpaceId; +use rsky_pds::actor_store::db::get_migrated_db; +use rsky_pds::actor_store::space::{encode_record, oplog_window, SpaceStore, SpaceWrite}; +use rsky_space_host::repo::{RepoStore, RepoWrite, SqliteRepos}; +use rsky_spaces_parity::{assert_parity, dump_pds, dump_shim}; +use serde_json::{json, Value}; + +const DID: &str = "did:plc:parityauthor"; +const AUTHORITY: &str = "did:plc:parityauthority"; +const COLLECTION: &str = "app.bsky.feed.post"; + +#[derive(Clone)] +enum ScriptWrite { + Create { + rkey: &'static str, + value: Value, + }, + Update { + rkey: &'static str, + value: Value, + swap: Option, + }, + Delete { + rkey: &'static str, + swap: Option, + }, +} + +impl ScriptWrite { + fn shim(&self) -> RepoWrite { + match self { + Self::Create { rkey, value } => RepoWrite::Create { + collection: COLLECTION.into(), + rkey: (*rkey).into(), + value: encode_record(value).expect("record encoding").1, + }, + Self::Update { rkey, value, swap } => RepoWrite::Update { + collection: COLLECTION.into(), + rkey: (*rkey).into(), + value: encode_record(value).expect("record encoding").1, + swap_record: swap.clone(), + }, + Self::Delete { rkey, swap } => RepoWrite::Delete { + collection: COLLECTION.into(), + rkey: (*rkey).into(), + swap_record: swap.clone(), + }, + } + } + + fn pds(&self) -> SpaceWrite { + match self { + Self::Create { rkey, value } => SpaceWrite::Create { + collection: COLLECTION.into(), + rkey: (*rkey).into(), + value: value.clone(), + }, + Self::Update { rkey, value, swap } => SpaceWrite::Update { + collection: COLLECTION.into(), + rkey: (*rkey).into(), + value: value.clone(), + swap_cid: swap.clone(), + }, + Self::Delete { rkey, swap } => SpaceWrite::Delete { + collection: COLLECTION.into(), + rkey: (*rkey).into(), + swap_cid: swap.clone(), + }, + } + } +} + +async fn run(name: &str, script: Vec) -> bool { + let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); + let space_uri = space.uri(); + let temp = tempfile::tempdir().expect("tempdir"); + let shim_path = temp.path().join("shim.sqlite"); + let shim = SqliteRepos::open(&shim_path).expect("shim store"); + let pds = SpaceStore::new( + DID.into(), + get_migrated_db(temp.path().join("store.sqlite")) + .await + .expect("pds db"), + ); + let shim_result = shim + .apply_writes( + &space_uri, + DID, + "3shimrev", + &script.iter().map(ScriptWrite::shim).collect::>(), + ) + .await; + let pds_result = pds + .apply_writes( + &space, + script.iter().map(ScriptWrite::pds).collect(), + oplog_window(), + ) + .await; + let equal_outcome = match (shim_result, pds_result) { + (Ok(_), Ok(_)) => assert_parity( + name, + &dump_shim(&shim, &space_uri, DID).await, + &dump_pds(&pds, &space_uri).await, + ), + (Err(_), Err(_)) => true, + (left, right) => { + eprintln!( + "{name}: outcomes differ: shim_ok={}, pds_ok={}", + left.is_ok(), + right.is_ok() + ); + false + } + }; + let cross_open = match get_migrated_db(&shim_path).await { + Ok(db) => SpaceStore::new(DID.into(), db) + .live_repo_state(&space_uri) + .await + .is_ok(), + Err(_) => false, + }; + if !cross_open { + eprintln!("{name}: pinned pds cannot open the shim-written repo"); + } + equal_outcome && cross_open +} + +#[tokio::test] +async fn scoreboard() { + let first = json!({"text": "first"}); + let first_cid = encode_record(&first).expect("record encoding").0; + let scenarios = [ + ( + "S1 create", + vec![ScriptWrite::Create { + rkey: "one", + value: first.clone(), + }], + ), + ( + "S2 batch create", + vec![ + ScriptWrite::Create { + rkey: "one", + value: json!({"text":"one"}), + }, + ScriptWrite::Create { + rkey: "two", + value: json!({"text":"two"}), + }, + ScriptWrite::Create { + rkey: "three", + value: json!({"text":"three"}), + }, + ], + ), + ( + "S3 update swap success", + vec![ + ScriptWrite::Create { + rkey: "one", + value: first.clone(), + }, + ScriptWrite::Update { + rkey: "one", + value: json!({"text":"second"}), + swap: Some(first_cid.clone()), + }, + ], + ), + ( + "S4 swap conflict", + vec![ + ScriptWrite::Create { + rkey: "one", + value: first, + }, + ScriptWrite::Update { + rkey: "one", + value: json!({"text":"second"}), + swap: Some("bafyreinvalid".into()), + }, + ], + ), + ( + "S5 delete", + vec![ + ScriptWrite::Create { + rkey: "one", + value: json!({"text":"first"}), + }, + ScriptWrite::Delete { + rkey: "one", + swap: None, + }, + ], + ), + ]; + let mut equal = 0; + for (name, script) in scenarios { + equal += usize::from(run(name, script).await); + } + println!("parity: {equal}/5 scenarios byte-equal"); + assert_eq!(equal, 5, "parity harness must be red before convergence"); +} From f7c9e335958809eff5bc27d74cb9c73ef1df09cb Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 13:29:45 -0400 Subject: [PATCH 31/56] refactor(space-host): remove bespoke sqlite storage ahead of actor-store convergence --- rsky-space-host/src/error.rs | 2 + rsky-space-host/src/http.rs | 39 ++- rsky-space-host/src/main.rs | 4 +- rsky-space-host/src/repo.rs | 419 +++++------------------------ rsky-spaces-parity/tests/parity.rs | 19 +- 5 files changed, 88 insertions(+), 395 deletions(-) diff --git a/rsky-space-host/src/error.rs b/rsky-space-host/src/error.rs index 00c393cd..d68350a3 100644 --- a/rsky-space-host/src/error.rs +++ b/rsky-space-host/src/error.rs @@ -34,6 +34,8 @@ pub enum HostError { InvalidSwap, #[error("requested history is no longer available")] HistoryUnavailable, + #[error("not implemented")] + Unimplemented, #[error(transparent)] Space(#[from] rsky_space::SpaceError), } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 8ae13383..a74324cf 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -187,6 +187,7 @@ impl From for ApiError { | HostError::ManagingApp(_) | HostError::Resolution(_) | HostError::Store(_) + | HostError::Unimplemented | HostError::Space(_) => { tracing::error!(error = %e, "internal error"); Self::new( @@ -717,9 +718,9 @@ async fn register_space( HostError::AccountNotHosted(did) => ApiError::invalid_request(format!( "authority signing key does not resolve: {did}" )), - HostError::SpaceNotFound(space) => ApiError::invalid_request(format!( - "space not hosted here: {space}" - )), + HostError::SpaceNotFound(space) => { + ApiError::invalid_request(format!("space not hosted here: {space}")) + } other => ApiError::from(other), })?; (context, true) @@ -1201,10 +1202,7 @@ mod tests { let claims = SpaceClaims { iss: user.to_string(), sub: context.authority.space_uri(), - aud: Some(format!( - "{}#atproto_space_host", - context.authority_did() - )), + aud: Some(format!("{}#atproto_space_host", context.authority_did())), iat: NOW, exp: NOW + 60, jti: "delegation-jti".to_string(), @@ -1966,7 +1964,8 @@ mod tests { .authority .mint_credential(NOW, "cred-b".to_string(), &dpop_key().thumbprint()) .unwrap(); - let (status, body) = send(&f.state, get_req(&path(other_space_uri), Some(&other_cred))).await; + let (status, body) = + send(&f.state, get_req(&path(other_space_uri), Some(&other_cred))).await; assert_eq!(status, StatusCode::OK); assert_eq!(body["space"], other_space_uri); assert_eq!(body["config"]["policy"], "public"); @@ -2009,11 +2008,7 @@ mod tests { let signer = seam.signer(&space.authority)?; let (tx, _writes) = tokio::sync::mpsc::unbounded_channel(); Ok(Arc::new(AuthorityContext { - authority: Arc::new(Authority::new( - space.clone(), - signer, - AppAccess::Open, - )), + authority: Arc::new(Authority::new(space.clone(), signer, AppAccess::Open)), policy: Arc::new(Policy::ManagingApp { service_id: format!("{MEMBER}#bsky_fg"), client: Arc::new(UnusedManagingApp), @@ -2056,9 +2051,8 @@ mod tests { let context = f.state.registry.authority(DYNAMIC_AUTHORITY).unwrap(); assert!(context.authority.resolve_registered(&space).is_ok()); // The new authority signs with the key resolved from the actor store. - let expected = Signer::from_secret( - secp256k1::SecretKey::from_slice(&DYNAMIC_AUTHORITY_KEY).unwrap(), - ); + let expected = + Signer::from_secret(secp256k1::SecretKey::from_slice(&DYNAMIC_AUTHORITY_KEY).unwrap()); assert_eq!(context.authority.signer.did_key(), expected.did_key()); let credential = context .authority @@ -2072,10 +2066,7 @@ mod tests { NOW, ) .unwrap(); - assert_eq!( - acker.0.lock().unwrap().as_slice(), - &[(space.clone(), 1)] - ); + assert_eq!(acker.0.lock().unwrap().as_slice(), &[(space.clone(), 1)]); assert_eq!( f.state.hosted_spaces.hosted_spaces().await.unwrap(), vec![(DYNAMIC_AUTHORITY.to_string(), space)] @@ -2115,7 +2106,13 @@ mod tests { .unwrap() .contains("did:plc:keyless")); assert!(f.state.registry.authority("did:plc:keyless").is_err()); - assert!(f.state.hosted_spaces.hosted_spaces().await.unwrap().is_empty()); + assert!(f + .state + .hosted_spaces + .hosted_spaces() + .await + .unwrap() + .is_empty()); } #[tokio::test] diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index 9a797066..59493242 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -20,7 +20,7 @@ use rsky_space_host::notify::HttpNotifier; use rsky_space_host::pds_seam::PdsSeam; use rsky_space_host::policy::Policy; use rsky_space_host::registration::{HttpLifecycleAcker, LifecycleAcker}; -use rsky_space_host::repo::SqliteRepos; +use rsky_space_host::repo::ActorStoreRepos; use rsky_space_host::signing::Signer; use rsky_space_host::store::{HostedSpaceStore, SqliteStore}; use std::sync::Arc; @@ -116,7 +116,7 @@ async fn main() -> Result<(), Box> { did_cache: std::sync::Arc::new(MemoryCache::new(None, None)), }))); let store = Arc::new(SqliteStore::open(&cfg.db_path)?); - let repos = Arc::new(SqliteRepos::open(&cfg.db_path)?); + let repos = Arc::new(ActorStoreRepos::open(&cfg.actor_store_dir)?); let seam = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); let builder = Arc::new(ContextBuilder { diff --git a/rsky-space-host/src/repo.rs b/rsky-space-host/src/repo.rs index 9e8d5ae7..e1152910 100644 --- a/rsky-space-host/src/repo.rs +++ b/rsky-space-host/src/repo.rs @@ -13,7 +13,6 @@ use async_trait::async_trait; use rsky_space::lthash::{element, LtHash}; use rsky_space::record::dag_cbor_cid; -use rusqlite::{Connection, OptionalExtension}; use std::collections::BTreeMap; use std::sync::Mutex; @@ -181,6 +180,67 @@ pub trait RepoStore: Send + Sync { async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()>; } +pub struct ActorStoreRepos; + +impl ActorStoreRepos { + pub fn open(_directory: impl AsRef) -> Result { + Ok(Self) + } +} + +#[async_trait] +impl RepoStore for ActorStoreRepos { + async fn apply_writes( + &self, + _space_uri: &str, + _did: &str, + _rev: &str, + _writes: &[RepoWrite], + ) -> Result { + Err(HostError::Unimplemented) + } + + async fn head(&self, _space_uri: &str, _did: &str) -> Result> { + Err(HostError::Unimplemented) + } + + async fn get_record( + &self, + _space_uri: &str, + _did: &str, + _collection: &str, + _rkey: &str, + ) -> Result> { + Err(HostError::Unimplemented) + } + + async fn list_records( + &self, + _space_uri: &str, + _did: &str, + _collection: Option<&str>, + _cursor: Option<&str>, + _limit: u32, + ) -> Result<(Vec, Option)> { + Err(HostError::Unimplemented) + } + + async fn list_ops( + &self, + _space_uri: &str, + _did: &str, + _since: Option<&str>, + _cursor: Option<&str>, + _limit: u32, + ) -> Result { + Err(HostError::Unimplemented) + } + + async fn delete_repo(&self, _space_uri: &str, _did: &str) -> Result<()> { + Err(HostError::Unimplemented) + } +} + /// Fold one batch into an existing record set + digest. Shared by both /// backings so their semantics cannot drift. fn plan_batch( @@ -508,330 +568,6 @@ fn finish_op_page(ops: Vec, limit: u32, last_seq: Option) -> OpPa } } -// ------------------------------------------------------------------- sqlite - -/// SQLite-backed repo storage. Volume per host is modest and every batch is a -/// single transaction, so one connection behind a mutex is sufficient. -pub struct SqliteRepos { - conn: Mutex, -} - -impl SqliteRepos { - pub fn open_in_memory() -> Result { - Self::init(Connection::open_in_memory().map_err(sql_err)?) - } - - pub fn open(path: impl AsRef) -> Result { - Self::init(Connection::open(path).map_err(sql_err)?) - } - - pub fn init(conn: Connection) -> Result { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS repo ( - space_uri TEXT NOT NULL, - did TEXT NOT NULL, - rev TEXT NOT NULL DEFAULT '', - state BLOB NOT NULL, - PRIMARY KEY (space_uri, did) - ); - CREATE TABLE IF NOT EXISTS record ( - space_uri TEXT NOT NULL, - did TEXT NOT NULL, - path TEXT NOT NULL, - collection TEXT NOT NULL, - rkey TEXT NOT NULL, - cid TEXT NOT NULL, - value BLOB NOT NULL, - PRIMARY KEY (space_uri, did, path) - ); - CREATE TABLE IF NOT EXISTS repo_op ( - seq INTEGER PRIMARY KEY AUTOINCREMENT, - space_uri TEXT NOT NULL, - did TEXT NOT NULL, - rev TEXT NOT NULL, - collection TEXT NOT NULL, - rkey TEXT NOT NULL, - cid TEXT, - prev TEXT - ); - CREATE INDEX IF NOT EXISTS repo_op_repo_seq ON repo_op (space_uri, did, seq);", - ) - .map_err(sql_err)?; - Ok(Self { - conn: Mutex::new(conn), - }) - } -} - -fn sql_err(e: rusqlite::Error) -> HostError { - HostError::Store(e.to_string()) -} - -fn state_from_blob(blob: Vec) -> Result<[u8; STATE_BYTES]> { - blob.try_into() - .map_err(|_| HostError::Store("corrupt lthash state".into())) -} - -fn row_to_record(row: &rusqlite::Row) -> rusqlite::Result { - Ok(StoredRecord { - collection: row.get("collection")?, - rkey: row.get("rkey")?, - cid: row.get("cid")?, - value: row.get("value")?, - }) -} - -fn row_to_op(row: &rusqlite::Row) -> rusqlite::Result { - Ok(StoredOp { - seq: row.get("seq")?, - rev: row.get("rev")?, - collection: row.get("collection")?, - rkey: row.get("rkey")?, - cid: row.get("cid")?, - prev: row.get("prev")?, - }) -} - -#[async_trait] -impl RepoStore for SqliteRepos { - async fn apply_writes( - &self, - space_uri: &str, - did: &str, - rev: &str, - writes: &[RepoWrite], - ) -> Result { - let mut conn = self.conn.lock().unwrap(); - let tx = conn.transaction().map_err(sql_err)?; - - let existing_state: Option> = tx - .query_row( - "SELECT state FROM repo WHERE space_uri = ?1 AND did = ?2", - rusqlite::params![space_uri, did], - |row| row.get(0), - ) - .optional() - .map_err(sql_err)?; - let mut current_rev: String = tx - .query_row( - "SELECT rev FROM repo WHERE space_uri = ?1 AND did = ?2", - rusqlite::params![space_uri, did], - |row| row.get(0), - ) - .optional() - .map_err(sql_err)? - .unwrap_or_default(); - - let mut lt = match existing_state { - Some(blob) => LtHash::from_state_bytes(&state_from_blob(blob)?), - None => LtHash::new(), - }; - - // Only the paths this batch touches are needed to plan it. - let mut existing = BTreeMap::new(); - for write in writes { - let path = record_path(write.collection(), write.rkey()); - if let Some(record) = tx - .query_row( - "SELECT collection, rkey, cid, value FROM record - WHERE space_uri = ?1 AND did = ?2 AND path = ?3", - rusqlite::params![space_uri, did, path], - row_to_record, - ) - .optional() - .map_err(sql_err)? - { - existing.insert(path, record); - } - } - - let planned = plan_batch(&existing, &mut lt, writes)?; - for p in &planned { - if p.is_noop() { - continue; - } - let path = record_path(&p.collection, &p.rkey); - match (&p.cid, &p.value) { - (Some(cid), Some(value)) => { - tx.execute( - "INSERT INTO record (space_uri, did, path, collection, rkey, cid, value) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) - ON CONFLICT (space_uri, did, path) - DO UPDATE SET cid = ?6, value = ?7", - rusqlite::params![space_uri, did, path, p.collection, p.rkey, cid, value], - ) - .map_err(sql_err)?; - } - _ => { - tx.execute( - "DELETE FROM record WHERE space_uri = ?1 AND did = ?2 AND path = ?3", - rusqlite::params![space_uri, did, path], - ) - .map_err(sql_err)?; - } - } - tx.execute( - "INSERT INTO repo_op (space_uri, did, rev, collection, rkey, cid, prev) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - rusqlite::params![space_uri, did, rev, p.collection, p.rkey, p.cid, p.prev], - ) - .map_err(sql_err)?; - current_rev = rev.to_string(); - } - - tx.execute( - "INSERT INTO repo (space_uri, did, rev, state) VALUES (?1, ?2, ?3, ?4) - ON CONFLICT (space_uri, did) DO UPDATE SET rev = ?3, state = ?4", - rusqlite::params![space_uri, did, current_rev, lt.state_bytes().to_vec()], - ) - .map_err(sql_err)?; - tx.commit().map_err(sql_err)?; - - Ok(Applied { - rev: current_rev, - hash: lt.hash(), - outcomes: planned.into_iter().map(|p| p.outcome).collect(), - }) - } - - async fn head(&self, space_uri: &str, did: &str) -> Result> { - let conn = self.conn.lock().unwrap(); - let row: Option<(String, Vec)> = conn - .query_row( - "SELECT rev, state FROM repo WHERE space_uri = ?1 AND did = ?2", - rusqlite::params![space_uri, did], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .optional() - .map_err(sql_err)?; - row.map(|(rev, state)| { - Ok(RepoHead { - rev, - state: state_from_blob(state)?, - }) - }) - .transpose() - } - - async fn get_record( - &self, - space_uri: &str, - did: &str, - collection: &str, - rkey: &str, - ) -> Result> { - self.conn - .lock() - .unwrap() - .query_row( - "SELECT collection, rkey, cid, value FROM record - WHERE space_uri = ?1 AND did = ?2 AND path = ?3", - rusqlite::params![space_uri, did, record_path(collection, rkey)], - row_to_record, - ) - .optional() - .map_err(sql_err) - } - - async fn list_records( - &self, - space_uri: &str, - did: &str, - collection: Option<&str>, - cursor: Option<&str>, - limit: u32, - ) -> Result<(Vec, Option)> { - let conn = self.conn.lock().unwrap(); - require_repo(&conn, space_uri, did)?; - let mut stmt = conn - .prepare( - "SELECT collection, rkey, cid, value FROM record - WHERE space_uri = ?1 AND did = ?2 AND path > ?3 - AND (?4 IS NULL OR collection = ?4) - ORDER BY path ASC LIMIT ?5", - ) - .map_err(sql_err)?; - let page = stmt - .query_map( - rusqlite::params![space_uri, did, cursor.unwrap_or(""), collection, limit], - row_to_record, - ) - .map_err(sql_err)? - .collect::>>() - .map_err(sql_err)?; - let cursor = page_cursor(&page, limit, |r| r.path()); - Ok((page, cursor)) - } - - async fn list_ops( - &self, - space_uri: &str, - did: &str, - since: Option<&str>, - cursor: Option<&str>, - limit: u32, - ) -> Result { - let conn = self.conn.lock().unwrap(); - require_repo(&conn, space_uri, did)?; - let bounds: (Option, Option) = conn - .query_row( - "SELECT (SELECT rev FROM repo_op WHERE space_uri = ?1 AND did = ?2 - ORDER BY seq ASC LIMIT 1), - (SELECT seq FROM repo_op WHERE space_uri = ?1 AND did = ?2 - ORDER BY seq DESC LIMIT 1)", - rusqlite::params![space_uri, did], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .map_err(sql_err)?; - ensure_history(since, bounds.0.as_deref())?; - let after = parse_cursor(cursor)?; - - let mut stmt = conn - .prepare( - "SELECT seq, rev, collection, rkey, cid, prev FROM repo_op - WHERE space_uri = ?1 AND did = ?2 - AND (?3 IS NULL OR rev > ?3) - AND (?4 IS NULL OR seq > ?4) - ORDER BY seq ASC LIMIT ?5", - ) - .map_err(sql_err)?; - let ops = stmt - .query_map( - rusqlite::params![space_uri, did, since, after, limit], - row_to_op, - ) - .map_err(sql_err)? - .collect::>>() - .map_err(sql_err)?; - Ok(finish_op_page(ops, limit, bounds.1)) - } - - async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()> { - let mut conn = self.conn.lock().unwrap(); - let tx = conn.transaction().map_err(sql_err)?; - for table in ["record", "repo_op", "repo"] { - tx.execute( - &format!("DELETE FROM {table} WHERE space_uri = ?1 AND did = ?2"), - rusqlite::params![space_uri, did], - ) - .map_err(sql_err)?; - } - tx.commit().map_err(sql_err) - } -} - -fn require_repo(conn: &Connection, space_uri: &str, did: &str) -> Result<()> { - let exists: Option = conn - .query_row( - "SELECT 1 FROM repo WHERE space_uri = ?1 AND did = ?2", - rusqlite::params![space_uri, did], - |row| row.get(0), - ) - .optional() - .map_err(sql_err)?; - exists.map(|_| ()).ok_or(HostError::RepoNotFound) -} - #[cfg(test)] mod tests { use super::*; @@ -1176,7 +912,6 @@ mod tests { #[tokio::test] async fn $name() { $exercise(&InMemoryRepos::default()).await; - $exercise(&SqliteRepos::open_in_memory().unwrap()).await; } }; } @@ -1200,7 +935,7 @@ mod tests { .await .unwrap(); - let b = SqliteRepos::open_in_memory().unwrap(); + let b = InMemoryRepos::default(); b.apply_writes(SPACE, DID, "3rev1", &[create("y", "two")]) .await .unwrap(); @@ -1214,36 +949,6 @@ mod tests { ); } - #[tokio::test] - async fn sqlite_persists_across_reopen() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("repos.db"); - let cid = { - let store = SqliteRepos::open(&path).unwrap(); - let applied = store - .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) - .await - .unwrap(); - match &applied.outcomes[0] { - WriteOutcome::Created { cid } => cid.clone(), - other => panic!("unexpected outcome {other:?}"), - } - }; - let store = SqliteRepos::open(&path).unwrap(); - let got = store - .get_record(SPACE, DID, POST, "a") - .await - .unwrap() - .unwrap(); - assert_eq!(got.cid, cid); - assert_eq!(store.head(SPACE, DID).await.unwrap().unwrap().rev, "3rev1"); - - assert!(matches!( - SqliteRepos::open(dir.path().join("missing/nested.db")), - Err(HostError::Store(_)) - )); - } - #[test] fn history_window_and_cursor_edges() { assert!(ensure_history(None, None).is_ok()); diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs index e96d4cc1..f5218cce 100644 --- a/rsky-spaces-parity/tests/parity.rs +++ b/rsky-spaces-parity/tests/parity.rs @@ -1,7 +1,7 @@ use oracle_rsky_space::space_id::SpaceId; use rsky_pds::actor_store::db::get_migrated_db; use rsky_pds::actor_store::space::{encode_record, oplog_window, SpaceStore, SpaceWrite}; -use rsky_space_host::repo::{RepoStore, RepoWrite, SqliteRepos}; +use rsky_space_host::repo::{ActorStoreRepos, RepoStore, RepoWrite}; use rsky_spaces_parity::{assert_parity, dump_pds, dump_shim}; use serde_json::{json, Value}; @@ -74,8 +74,7 @@ async fn run(name: &str, script: Vec) -> bool { let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); let space_uri = space.uri(); let temp = tempfile::tempdir().expect("tempdir"); - let shim_path = temp.path().join("shim.sqlite"); - let shim = SqliteRepos::open(&shim_path).expect("shim store"); + let shim = ActorStoreRepos::open(temp.path()).expect("shim store"); let pds = SpaceStore::new( DID.into(), get_migrated_db(temp.path().join("store.sqlite")) @@ -103,7 +102,7 @@ async fn run(name: &str, script: Vec) -> bool { &dump_shim(&shim, &space_uri, DID).await, &dump_pds(&pds, &space_uri).await, ), - (Err(_), Err(_)) => true, + (Err(_), Err(_)) => false, (left, right) => { eprintln!( "{name}: outcomes differ: shim_ok={}, pds_ok={}", @@ -113,17 +112,7 @@ async fn run(name: &str, script: Vec) -> bool { false } }; - let cross_open = match get_migrated_db(&shim_path).await { - Ok(db) => SpaceStore::new(DID.into(), db) - .live_repo_state(&space_uri) - .await - .is_ok(), - Err(_) => false, - }; - if !cross_open { - eprintln!("{name}: pinned pds cannot open the shim-written repo"); - } - equal_outcome && cross_open + equal_outcome } #[tokio::test] From 8fb3443711c717ef0e1928913372577aa5694cd2 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 13:32:55 -0400 Subject: [PATCH 32/56] feat(space-host): pds-identical actor-store schema --- Cargo.lock | 1 + rsky-space-host/src/actor_schema.rs | 203 ++++++++++++++++++++++++++++ rsky-space-host/src/lib.rs | 1 + rsky-spaces-parity/Cargo.toml | 1 + rsky-spaces-parity/tests/parity.rs | 22 +++ 5 files changed, 228 insertions(+) create mode 100644 rsky-space-host/src/actor_schema.rs diff --git a/Cargo.lock b/Cargo.lock index 9fd0621a..ede225c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8722,6 +8722,7 @@ dependencies = [ "rsky-space 0.4.1", "rsky-space 0.4.2", "rsky-space-host 0.7.1", + "rusqlite", "serde_json", "tempfile", "tokio", diff --git a/rsky-space-host/src/actor_schema.rs b/rsky-space-host/src/actor_schema.rs new file mode 100644 index 00000000..4d4af5d9 --- /dev/null +++ b/rsky-space-host/src/actor_schema.rs @@ -0,0 +1,203 @@ +use rusqlite::Connection; +use std::collections::HashSet; +use std::path::Path; + +use crate::error::{HostError, Result}; + +const MIGRATIONS: &[(&str, &str)] = &[ + ( + "001", + "\ + CREATE TABLE repo_root (\ + did TEXT PRIMARY KEY, \ + cid TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + \"indexedAt\" TEXT NOT NULL\ + );\ + CREATE TABLE repo_block (\ + cid TEXT PRIMARY KEY, \ + \"repoRev\" TEXT NOT NULL, \ + size INTEGER NOT NULL, \ + content BLOB NOT NULL\ + );\ + CREATE INDEX repo_block_repo_rev_idx ON repo_block (\"repoRev\", cid);\ + CREATE TABLE record (\ + uri TEXT PRIMARY KEY, \ + cid TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + \"repoRev\" TEXT NOT NULL, \ + \"indexedAt\" TEXT NOT NULL, \ + \"takedownRef\" TEXT\ + );\ + CREATE INDEX record_cid_idx ON record (cid);\ + CREATE INDEX record_collection_idx ON record (collection);\ + CREATE INDEX record_repo_rev_idx ON record (\"repoRev\");\ + CREATE TABLE blob (\ + cid TEXT PRIMARY KEY, \ + \"mimeType\" TEXT NOT NULL, \ + size INTEGER NOT NULL, \ + \"tempKey\" TEXT, \ + width INTEGER, \ + height INTEGER, \ + \"createdAt\" TEXT NOT NULL, \ + \"takedownRef\" TEXT\ + );\ + CREATE INDEX blob_tempkey_idx ON blob (\"tempKey\");\ + CREATE TABLE record_blob (\ + \"blobCid\" TEXT NOT NULL, \ + \"recordUri\" TEXT NOT NULL, \ + PRIMARY KEY (\"blobCid\", \"recordUri\")\ + );\ + CREATE TABLE backlink (\ + uri TEXT NOT NULL, \ + path TEXT NOT NULL, \ + \"linkTo\" TEXT NOT NULL, \ + PRIMARY KEY (uri, path)\ + );\ + CREATE INDEX backlink_link_to_idx ON backlink (path, \"linkTo\");\ + CREATE TABLE account_pref (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + name TEXT NOT NULL, \ + \"valueJson\" TEXT NOT NULL\ + );", + ), + ( + "002", + "\ + CREATE TABLE space_repo (\ + space_uri TEXT PRIMARY KEY, \ + authority TEXT NOT NULL, \ + space_type TEXT NOT NULL, \ + skey TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + lthash_state BLOB NOT NULL, \ + oplog_floor_rev TEXT, \ + deleted INTEGER NOT NULL DEFAULT 0, \ + created_at TEXT NOT NULL\ + );\ + CREATE TABLE space_record (\ + space_uri TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + cid TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + value BLOB NOT NULL, \ + PRIMARY KEY (space_uri, collection, rkey)\ + );\ + CREATE TABLE space_oplog (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + space_uri TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + cid TEXT, \ + prev TEXT\ + );\ + CREATE INDEX space_oplog_space_idx ON space_oplog (space_uri, id);\ + CREATE TABLE space_blob_ref (\ + space_uri TEXT NOT NULL, \ + blob_cid TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + PRIMARY KEY (space_uri, blob_cid, collection, rkey)\ + );\ + CREATE TABLE space_repo_notify (\ + space_uri TEXT NOT NULL, \ + endpoint TEXT NOT NULL, \ + expires_at TEXT NOT NULL, \ + PRIMARY KEY (space_uri, endpoint)\ + );\ + CREATE TABLE space_def (\ + space_uri TEXT PRIMARY KEY, \ + space_type TEXT NOT NULL, \ + skey TEXT NOT NULL, \ + policy TEXT NOT NULL DEFAULT 'member-list', \ + app_access TEXT NOT NULL DEFAULT 'open', \ + allowed_clients TEXT, \ + managing_app TEXT, \ + deleted INTEGER NOT NULL DEFAULT 0, \ + created_at TEXT NOT NULL\ + );\ + CREATE TABLE space_member (\ + space_uri TEXT NOT NULL, \ + did TEXT NOT NULL, \ + PRIMARY KEY (space_uri, did)\ + );\ + CREATE TABLE space_writer (\ + space_uri TEXT NOT NULL, \ + did TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + hash TEXT, \ + PRIMARY KEY (space_uri, did)\ + );\ + CREATE TABLE space_host_reg (\ + space_uri TEXT NOT NULL, \ + endpoint TEXT NOT NULL, \ + expires_at TEXT NOT NULL, \ + PRIMARY KEY (space_uri, endpoint)\ + );\ + CREATE TABLE space_used_jti (\ + jti TEXT PRIMARY KEY, \ + exp INTEGER NOT NULL\ + );", + ), + // A notification is delivered with service auth addressed to the + // subscriber, so a registration has to remember who the subscriber is. + // Rows written before this carry no service identifier and keep the + // pre-amendment behaviour (proposals#100). + ( + "003", + "\ + ALTER TABLE space_repo_notify ADD COLUMN service TEXT;\ + ALTER TABLE space_host_reg ADD COLUMN service TEXT;", + ), + // Member rows carry when and at what rev they were added, which clients + // surface; and each account keeps a local index of spaces it was enrolled + // in, because listSpaces is how a member discovers a shared space at all -- + // a repo row only exists after the member's first write. + ( + "004", + "\ + ALTER TABLE space_member ADD COLUMN member_rev TEXT NOT NULL DEFAULT '';\ + ALTER TABLE space_member ADD COLUMN added_at TEXT NOT NULL DEFAULT '';\ + CREATE TABLE space_joined (\ + space_uri TEXT PRIMARY KEY, \ + authority TEXT NOT NULL, \ + space_type TEXT NOT NULL, \ + created_at TEXT NOT NULL\ + );", + ), +]; + +pub fn get_migrated_db(path: impl AsRef) -> Result { + let mut connection = Connection::open(path).map_err(sql_err)?; + let transaction = connection.transaction().map_err(sql_err)?; + transaction + .execute_batch("CREATE TABLE IF NOT EXISTS migrations (name TEXT PRIMARY KEY, \"appliedAt\" TEXT NOT NULL)") + .map_err(sql_err)?; + let applied = transaction + .prepare("SELECT name FROM migrations") + .map_err(sql_err)? + .query_map([], |row| row.get::<_, String>(0)) + .map_err(sql_err)? + .collect::>>() + .map_err(sql_err)?; + for (name, sql) in MIGRATIONS { + if !applied.contains(*name) { + transaction.execute_batch(sql).map_err(sql_err)?; + transaction + .execute( + "INSERT INTO migrations (name, \"appliedAt\") VALUES (?1, datetime('now'))", + [name], + ) + .map_err(sql_err)?; + } + } + transaction.commit().map_err(sql_err)?; + Ok(connection) +} + +fn sql_err(error: rusqlite::Error) -> HostError { + HostError::Store(error.to_string()) +} diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index 57b4fcb0..4d42afb4 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -18,6 +18,7 @@ //! `com.atproto.space.*` DTOs from rsky-lexicon, backed by in-memory or SQLite //! [stores](store). +pub mod actor_schema; pub mod appaccess; pub mod attestation; pub mod authority; diff --git a/rsky-spaces-parity/Cargo.toml b/rsky-spaces-parity/Cargo.toml index 94b15b9c..964a7956 100644 --- a/rsky-spaces-parity/Cargo.toml +++ b/rsky-spaces-parity/Cargo.toml @@ -11,6 +11,7 @@ oracle-rsky-space = { package = "rsky-space", git = "https://github.com/blacksky rsky-pds = { git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } serde_json = { workspace = true } tokio = { workspace = true } +rusqlite = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs index f5218cce..835c9a2f 100644 --- a/rsky-spaces-parity/tests/parity.rs +++ b/rsky-spaces-parity/tests/parity.rs @@ -5,6 +5,28 @@ use rsky_space_host::repo::{ActorStoreRepos, RepoStore, RepoWrite}; use rsky_spaces_parity::{assert_parity, dump_pds, dump_shim}; use serde_json::{json, Value}; +fn sqlite_master(path: &std::path::Path) -> Vec<(String, String, String)> { + let connection = rusqlite::Connection::open(path).expect("schema connection"); + let mut statement = connection + .prepare("SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name") + .expect("schema statement"); + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) + .expect("schema rows") + .collect::>>() + .expect("schema values") +} + +#[tokio::test] +async fn actor_schema_matches_pinned_oracle() { + let temp = tempfile::tempdir().expect("tempdir"); + let shim_path = temp.path().join("shim.sqlite"); + let pds_path = temp.path().join("pds.sqlite"); + rsky_space_host::actor_schema::get_migrated_db(&shim_path).expect("shim migration"); + get_migrated_db(&pds_path).await.expect("pds migration"); + assert_eq!(sqlite_master(&shim_path), sqlite_master(&pds_path)); +} + const DID: &str = "did:plc:parityauthor"; const AUTHORITY: &str = "did:plc:parityauthority"; const COLLECTION: &str = "app.bsky.feed.post"; From aa7fce63a13dc8913996439e863e6f081b5ace5a Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 13:55:13 -0400 Subject: [PATCH 33/56] feat(space-host): actor-store repos passes S1-S6 Implements RepoStore over per-account store.sqlite files carrying the space_* tables, transcribing the pinned oracle's apply-writes semantics: server-minted revisions, per-write swap checks, record-exists and record-not-found refusals, oplog append and compaction. The harness now drives both sides through per-batch scripts, compares refusal reasons as well as contents, and adds S6. --- Cargo.lock | 1 + rsky-space-host/src/actor_repos.rs | 454 +++++++++++++++++++++++++++++ rsky-space-host/src/error.rs | 4 + rsky-space-host/src/http.rs | 4 +- rsky-space-host/src/lib.rs | 1 + rsky-space-host/src/main.rs | 2 +- rsky-space-host/src/repo.rs | 69 +---- rsky-spaces-parity/Cargo.toml | 1 + rsky-spaces-parity/src/lib.rs | 87 +++++- rsky-spaces-parity/tests/parity.rs | 181 ++++++------ 10 files changed, 638 insertions(+), 166 deletions(-) create mode 100644 rsky-space-host/src/actor_repos.rs diff --git a/Cargo.lock b/Cargo.lock index ede225c4..a6bca5cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8718,6 +8718,7 @@ dependencies = [ name = "rsky-spaces-parity" version = "0.1.0" dependencies = [ + "anyhow", "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", diff --git a/rsky-space-host/src/actor_repos.rs b/rsky-space-host/src/actor_repos.rs new file mode 100644 index 00000000..ef1322ba --- /dev/null +++ b/rsky-space-host/src/actor_repos.rs @@ -0,0 +1,454 @@ +//! [`RepoStore`] over a directory of per-account actor stores. +//! +//! One `store.sqlite` per author DID, laid out as the PDS lays out its actor +//! files, carrying the `space_*` tables of [`crate::actor_schema`]. Revisions +//! are minted here rather than supplied by the caller, so a repo's `rev` is +//! always a TID monotonic in that repo's own history. + +use async_trait::async_trait; +use rsky_common::tid::TID; +use rsky_space::lthash::{element, LtHash}; +use rsky_space::record::dag_cbor_cid; +use rsky_space::space_id::SpaceId; +use rusqlite::{params, Connection, OptionalExtension, Transaction}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; + +use crate::error::{HostError, Result}; +use crate::repo::{ + page_cursor, parse_cursor, record_path, Applied, OpPage, RepoHead, RepoStore, RepoWrite, + StoredOp, StoredRecord, WriteOutcome, STATE_BYTES, +}; + +/// Oplog rows retained per repo before the oldest revisions are dropped. +pub const DEFAULT_OPLOG_WINDOW: usize = 10_000; + +pub struct ActorStoreRepos { + directory: PathBuf, + oplog_window: usize, + connections: Mutex>, +} + +impl ActorStoreRepos { + pub fn open(directory: impl Into) -> Result { + Self::with_oplog_window(directory, DEFAULT_OPLOG_WINDOW) + } + + pub fn with_oplog_window(directory: impl Into, oplog_window: usize) -> Result { + Ok(Self { + directory: directory.into(), + oplog_window: oplog_window.max(1), + connections: Mutex::new(HashMap::new()), + }) + } + + /// `{root}/{sha256(did)[..2]}/{did}/store.sqlite`, the PDS actor layout. + pub fn store_path(&self, did: &str) -> Result { + if did.is_empty() + || !did.starts_with("did:") + || did.contains('/') + || did.contains('\\') + || did.contains("..") + { + return Err(HostError::InvalidRequest(format!("unusable did: {did}"))); + } + let digest = hex::encode(Sha256::digest(did.as_bytes())); + Ok(self + .directory + .join(&digest[..2]) + .join(did) + .join("store.sqlite")) + } + + fn with_store( + &self, + did: &str, + act: impl FnOnce(&mut Connection) -> Result, + ) -> Result { + let mut connections = self.connections.lock().unwrap(); + if !connections.contains_key(did) { + let path = self.store_path(did)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| store(error.to_string()))?; + } + connections.insert(did.to_string(), crate::actor_schema::get_migrated_db(path)?); + } + act(connections.get_mut(did).expect("store just opened")) + } +} + +#[async_trait] +impl RepoStore for ActorStoreRepos { + /// `rev` is ignored: the store mints the revision every write in the batch + /// shares, and returns it on [`Applied`]. + async fn apply_writes( + &self, + space_uri: &str, + did: &str, + _rev: &str, + writes: &[RepoWrite], + ) -> Result { + let space = SpaceId::parse(space_uri)?; + let window = self.oplog_window; + self.with_store(did, move |conn| { + let tx = conn.transaction().map_err(sql)?; + let applied = apply_writes_tx(&tx, &space, writes, window)?; + tx.commit().map_err(sql)?; + Ok(applied) + }) + } + + async fn head(&self, space_uri: &str, did: &str) -> Result> { + self.with_store(did, |conn| { + let Some((rev, state)) = live_repo(conn, space_uri)? else { + return Ok(None); + }; + Ok(Some(RepoHead { + rev, + state: state_bytes(state)?, + })) + }) + } + + async fn get_record( + &self, + space_uri: &str, + did: &str, + collection: &str, + rkey: &str, + ) -> Result> { + self.with_store(did, |conn| { + conn.query_row( + "SELECT collection, rkey, cid, value FROM space_record \ + WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![space_uri, collection, rkey], + row_to_record, + ) + .optional() + .map_err(sql) + }) + } + + async fn list_records( + &self, + space_uri: &str, + did: &str, + collection: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result<(Vec, Option)> { + self.with_store(did, |conn| { + if live_repo(conn, space_uri)?.is_none() { + return Err(HostError::RepoNotFound); + } + let cursor = cursor.and_then(|c| c.split_once('/')); + let mut query = String::from( + "SELECT collection, rkey, cid, value FROM space_record WHERE space_uri = ?1", + ); + let mut args: Vec<&dyn rusqlite::ToSql> = vec![&space_uri]; + if let Some(ref collection) = collection { + query.push_str(&format!(" AND collection = ?{}", args.len() + 1)); + args.push(collection); + } + if let Some((ref c, ref r)) = cursor { + let base = args.len(); + query.push_str(&format!( + " AND (collection > ?{0} OR (collection = ?{0} AND rkey > ?{1}))", + base + 1, + base + 2 + )); + args.push(c); + args.push(r); + } + query.push_str(&format!(" ORDER BY collection, rkey LIMIT {limit}")); + let mut statement = conn.prepare(&query).map_err(sql)?; + let page = statement + .query_map(rusqlite::params_from_iter(args), row_to_record) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + let cursor = page_cursor(&page, limit, |r| r.path()); + Ok((page, cursor)) + }) + } + + async fn list_ops( + &self, + space_uri: &str, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result { + let after = parse_cursor(cursor)?; + self.with_store(did, |conn| { + let floor: Option = conn + .query_row( + "SELECT oplog_floor_rev FROM space_repo WHERE space_uri = ?1 AND deleted = 0", + [space_uri], + |row| row.get(0), + ) + .optional() + .map_err(sql)? + .ok_or(HostError::RepoNotFound)?; + if let Some(floor) = floor { + match since { + Some(since) if since >= floor.as_str() => {} + _ => return Err(HostError::HistoryUnavailable), + } + } + let mut statement = conn + .prepare( + "SELECT id, rev, collection, rkey, cid, prev FROM space_oplog \ + WHERE space_uri = ?1 AND (?2 IS NULL OR rev > ?2) \ + AND (?3 IS NULL OR id > ?3) \ + ORDER BY id LIMIT ?4", + ) + .map_err(sql)?; + let mut ops = statement + .query_map(params![space_uri, since, after, limit as i64 + 1], |row| { + Ok(StoredOp { + seq: row.get(0)?, + rev: row.get(1)?, + collection: row.get(2)?, + rkey: row.get(3)?, + cid: row.get(4)?, + prev: row.get(5)?, + }) + }) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + let complete = ops.len() <= limit as usize; + ops.truncate(limit as usize); + Ok(OpPage { + cursor: if complete { + None + } else { + page_cursor(&ops, limit, |o| o.seq.to_string()) + }, + complete, + ops, + }) + }) + } + + async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()> { + self.with_store(did, |conn| { + conn.execute( + "UPDATE space_repo SET deleted = 1 WHERE space_uri = ?1", + [space_uri], + ) + .map_err(sql)?; + Ok(()) + }) + } +} + +/// Apply one batch: evolve the LtHash state, upsert/delete record rows, append +/// oplog rows sharing one fresh revision, then compact the oplog. +fn apply_writes_tx( + tx: &Transaction, + space: &SpaceId, + writes: &[RepoWrite], + window: usize, +) -> Result { + let space_uri = space.uri(); + let existing: Option<(String, Vec, i64)> = tx + .query_row( + "SELECT rev, lthash_state, deleted FROM space_repo WHERE space_uri = ?1", + [&space_uri], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(sql)?; + let (prev_rev, mut lthash) = match existing { + Some((_, _, deleted)) if deleted != 0 => return Err(HostError::RepoNotFound), + Some((rev, state, _)) => (Some(rev), LtHash::from_state_bytes(&state_bytes(state)?)), + None => (None, LtHash::new()), + }; + let rev = TID::next_str(prev_rev.clone()).map_err(|error| store(error.to_string()))?; + let mut outcomes = Vec::with_capacity(writes.len()); + + for write in writes { + let (collection, rkey) = (write.collection(), write.rkey()); + let current: Option = tx + .query_row( + "SELECT cid FROM space_record WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![&space_uri, collection, rkey], + |row| row.get(0), + ) + .optional() + .map_err(sql)?; + let swap = match write { + RepoWrite::Create { .. } => None, + RepoWrite::Update { swap_record, .. } | RepoWrite::Delete { swap_record, .. } => { + Some(swap_record) + } + }; + if let Some(swap) = swap { + if swap.is_some() && swap.as_deref() != current.as_deref() { + return Err(HostError::InvalidSwap); + } + } + let path = record_path(collection, rkey); + match write { + RepoWrite::Create { .. } if current.is_some() => { + return Err(HostError::RecordExists(path)) + } + RepoWrite::Update { .. } | RepoWrite::Delete { .. } if current.is_none() => { + return Err(HostError::RecordNotFound(path)) + } + _ => {} + } + if let Some(ref old_cid) = current { + lthash.remove(&element(collection, rkey, old_cid)); + tx.execute( + "DELETE FROM space_blob_ref WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![&space_uri, collection, rkey], + ) + .map_err(sql)?; + } + let new = match write { + RepoWrite::Create { value, .. } | RepoWrite::Update { value, .. } => { + Some((dag_cbor_cid(value).to_string(), value)) + } + RepoWrite::Delete { .. } => None, + }; + match new { + Some((ref cid, value)) => { + lthash.add(&element(collection, rkey, cid)); + tx.execute( + "INSERT INTO space_record (space_uri, collection, rkey, cid, rev, value) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6) \ + ON CONFLICT (space_uri, collection, rkey) \ + DO UPDATE SET cid = excluded.cid, rev = excluded.rev, value = excluded.value", + params![&space_uri, collection, rkey, cid, rev, value], + ) + .map_err(sql)?; + } + None => { + tx.execute( + "DELETE FROM space_record WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![&space_uri, collection, rkey], + ) + .map_err(sql)?; + } + } + let cid = new.map(|(cid, _)| cid); + tx.execute( + "INSERT INTO space_oplog (space_uri, rev, collection, rkey, cid, prev) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![&space_uri, rev, collection, rkey, cid, current], + ) + .map_err(sql)?; + outcomes.push(match (cid, current) { + (Some(cid), None) => WriteOutcome::Created { cid }, + (Some(cid), Some(_)) => WriteOutcome::Updated { cid }, + (None, _) => WriteOutcome::Deleted, + }); + } + + let state = lthash.state_bytes().to_vec(); + if prev_rev.is_some() { + tx.execute( + "UPDATE space_repo SET rev = ?2, lthash_state = ?3 WHERE space_uri = ?1", + params![&space_uri, rev, state], + ) + .map_err(sql)?; + } else { + tx.execute( + "INSERT INTO space_repo \ + (space_uri, authority, space_type, skey, rev, lthash_state, oplog_floor_rev, deleted, created_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, 0, ?7)", + params![ + &space_uri, + space.authority, + space.space_type, + space.skey, + rev, + state, + rsky_common::now() + ], + ) + .map_err(sql)?; + } + compact_oplog(tx, &space_uri, window)?; + Ok(Applied { + rev, + hash: lthash.hash(), + outcomes, + }) +} + +/// Keep at most `window` oplog rows per repo, dropping whole revisions from the +/// oldest end and advancing `oplog_floor_rev` to the newest dropped revision. +fn compact_oplog(tx: &Transaction, space_uri: &str, window: usize) -> Result<()> { + let cutoff_rev: Option = tx + .query_row( + "SELECT rev FROM space_oplog WHERE space_uri = ?1 ORDER BY id DESC LIMIT 1 OFFSET ?2", + params![space_uri, (window - 1) as i64], + |row| row.get(0), + ) + .optional() + .map_err(sql)?; + let Some(cutoff_rev) = cutoff_rev else { + return Ok(()); + }; + let floor: Option = tx + .query_row( + "SELECT MAX(rev) FROM space_oplog WHERE space_uri = ?1 AND rev < ?2", + params![space_uri, cutoff_rev], + |row| row.get(0), + ) + .map_err(sql)?; + let Some(floor) = floor else { + return Ok(()); + }; + tx.execute( + "DELETE FROM space_oplog WHERE space_uri = ?1 AND rev < ?2", + params![space_uri, cutoff_rev], + ) + .map_err(sql)?; + tx.execute( + "UPDATE space_repo SET oplog_floor_rev = ?2 WHERE space_uri = ?1", + params![space_uri, floor], + ) + .map_err(sql)?; + Ok(()) +} + +fn live_repo(conn: &Connection, space_uri: &str) -> Result)>> { + conn.query_row( + "SELECT rev, lthash_state FROM space_repo WHERE space_uri = ?1 AND deleted = 0", + [space_uri], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(sql) +} + +fn row_to_record(row: &rusqlite::Row) -> rusqlite::Result { + Ok(StoredRecord { + collection: row.get(0)?, + rkey: row.get(1)?, + cid: row.get(2)?, + value: row.get(3)?, + }) +} + +fn state_bytes(state: Vec) -> Result<[u8; STATE_BYTES]> { + state + .try_into() + .map_err(|_| store("lthash state is not 2048 bytes".to_string())) +} + +fn store(message: String) -> HostError { + HostError::Store(message) +} + +fn sql(error: rusqlite::Error) -> HostError { + HostError::Store(error.to_string()) +} diff --git a/rsky-space-host/src/error.rs b/rsky-space-host/src/error.rs index d68350a3..19ff256c 100644 --- a/rsky-space-host/src/error.rs +++ b/rsky-space-host/src/error.rs @@ -30,6 +30,10 @@ pub enum HostError { AccountNotHosted(String), #[error("repo not found")] RepoNotFound, + #[error("record already exists: {0}")] + RecordExists(String), + #[error("record not found: {0}")] + RecordNotFound(String), #[error("swap cid did not match")] InvalidSwap, #[error("requested history is no longer available")] diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index a74324cf..d6299a2e 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -171,7 +171,9 @@ impl From for ApiError { "RepoNotFound", "repo not hosted here", ), - HostError::InvalidRequest(message) => Self::invalid_request(message.clone()), + HostError::InvalidRequest(message) + | HostError::RecordExists(message) + | HostError::RecordNotFound(message) => Self::invalid_request(message.clone()), HostError::InvalidSwap => Self::new( StatusCode::CONFLICT, "InvalidSwap", diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index 4d42afb4..081eab53 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -18,6 +18,7 @@ //! `com.atproto.space.*` DTOs from rsky-lexicon, backed by in-memory or SQLite //! [stores](store). +pub mod actor_repos; pub mod actor_schema; pub mod appaccess; pub mod attestation; diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index 59493242..b836526d 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -6,6 +6,7 @@ use rsky_identity::did::did_resolver::DidResolver; use rsky_identity::types::{DidResolverOpts, MemoryCache}; use rsky_oauth::dpop::{DpopManager, InMemoryReplayStore}; use rsky_space::space_id::SpaceId; +use rsky_space_host::actor_repos::ActorStoreRepos; use rsky_space_host::appaccess::AppAccess; use rsky_space_host::attestation::HttpMetadataFetcher; use rsky_space_host::authority::{ @@ -20,7 +21,6 @@ use rsky_space_host::notify::HttpNotifier; use rsky_space_host::pds_seam::PdsSeam; use rsky_space_host::policy::Policy; use rsky_space_host::registration::{HttpLifecycleAcker, LifecycleAcker}; -use rsky_space_host::repo::ActorStoreRepos; use rsky_space_host::signing::Signer; use rsky_space_host::store::{HostedSpaceStore, SqliteStore}; use std::sync::Arc; diff --git a/rsky-space-host/src/repo.rs b/rsky-space-host/src/repo.rs index e1152910..c194431f 100644 --- a/rsky-space-host/src/repo.rs +++ b/rsky-space-host/src/repo.rs @@ -22,7 +22,7 @@ use crate::error::{HostError, Result}; /// DAG-CBOR well-formedness are the only limits the host applies to a value. pub const MAX_RECORD_BYTES: usize = 64 * 1024; -const STATE_BYTES: usize = 2048; +pub(crate) const STATE_BYTES: usize = 2048; /// One mutation in an atomic batch. Values are already-encoded DAG-CBOR. #[derive(Debug, Clone, PartialEq, Eq)] @@ -180,67 +180,6 @@ pub trait RepoStore: Send + Sync { async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()>; } -pub struct ActorStoreRepos; - -impl ActorStoreRepos { - pub fn open(_directory: impl AsRef) -> Result { - Ok(Self) - } -} - -#[async_trait] -impl RepoStore for ActorStoreRepos { - async fn apply_writes( - &self, - _space_uri: &str, - _did: &str, - _rev: &str, - _writes: &[RepoWrite], - ) -> Result { - Err(HostError::Unimplemented) - } - - async fn head(&self, _space_uri: &str, _did: &str) -> Result> { - Err(HostError::Unimplemented) - } - - async fn get_record( - &self, - _space_uri: &str, - _did: &str, - _collection: &str, - _rkey: &str, - ) -> Result> { - Err(HostError::Unimplemented) - } - - async fn list_records( - &self, - _space_uri: &str, - _did: &str, - _collection: Option<&str>, - _cursor: Option<&str>, - _limit: u32, - ) -> Result<(Vec, Option)> { - Err(HostError::Unimplemented) - } - - async fn list_ops( - &self, - _space_uri: &str, - _did: &str, - _since: Option<&str>, - _cursor: Option<&str>, - _limit: u32, - ) -> Result { - Err(HostError::Unimplemented) - } - - async fn delete_repo(&self, _space_uri: &str, _did: &str) -> Result<()> { - Err(HostError::Unimplemented) - } -} - /// Fold one batch into an existing record set + digest. Shared by both /// backings so their semantics cannot drift. fn plan_batch( @@ -358,7 +297,7 @@ impl PlannedWrite { } } -fn page_cursor(page: &[T], limit: u32, key: impl Fn(&T) -> String) -> Option { +pub(crate) fn page_cursor(page: &[T], limit: u32, key: impl Fn(&T) -> String) -> Option { match page.last() { Some(last) if page.len() == limit as usize => Some(key(last)), _ => None, @@ -549,7 +488,7 @@ fn ensure_history(since: Option<&str>, earliest_retained: Option<&str>) -> Resul } } -fn parse_cursor(cursor: Option<&str>) -> Result> { +pub(crate) fn parse_cursor(cursor: Option<&str>) -> Result> { cursor .map(|c| { c.parse::() @@ -558,7 +497,7 @@ fn parse_cursor(cursor: Option<&str>) -> Result> { .transpose() } -fn finish_op_page(ops: Vec, limit: u32, last_seq: Option) -> OpPage { +pub(crate) fn finish_op_page(ops: Vec, limit: u32, last_seq: Option) -> OpPage { let reached_head = ops.last().map(|o| o.seq) == last_seq; let cursor = page_cursor(&ops, limit, |o| o.seq.to_string()); OpPage { diff --git a/rsky-spaces-parity/Cargo.toml b/rsky-spaces-parity/Cargo.toml index 964a7956..fc05722c 100644 --- a/rsky-spaces-parity/Cargo.toml +++ b/rsky-spaces-parity/Cargo.toml @@ -9,6 +9,7 @@ rsky-space-host = { path = "../rsky-space-host" } rsky-space = { path = "../rsky-space" } oracle-rsky-space = { package = "rsky-space", git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } rsky-pds = { git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } +anyhow = "1" serde_json = { workspace = true } tokio = { workspace = true } rusqlite = { workspace = true } diff --git a/rsky-spaces-parity/src/lib.rs b/rsky-spaces-parity/src/lib.rs index a65fc2d3..c7bd27a7 100644 --- a/rsky-spaces-parity/src/lib.rs +++ b/rsky-spaces-parity/src/lib.rs @@ -1,4 +1,5 @@ -use rsky_pds::actor_store::space::SpaceStore; +use rsky_pds::actor_store::space::{SpaceStore, SpaceStoreError}; +use rsky_space_host::error::HostError; use rsky_space_host::repo::RepoStore; #[derive(Debug, Clone, PartialEq, Eq)] @@ -8,21 +9,60 @@ pub struct RepoDump { pub ops: Vec<(String, String, Option, Option)>, } -pub async fn dump_shim(store: &dyn RepoStore, space_uri: &str, did: &str) -> RepoDump { +/// The classification a write batch or read is compared on: either it applied, +/// or both sides must refuse it for the same reason. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Outcome { + Applied, + RecordExists, + RecordNotFound, + InvalidSwap, + HistoryUnavailable, + RepoGone, + Other(String), +} + +pub fn shim_outcome(result: &Result) -> Outcome { + match result { + Ok(_) => Outcome::Applied, + Err(HostError::RecordExists(_)) => Outcome::RecordExists, + Err(HostError::RecordNotFound(_)) => Outcome::RecordNotFound, + Err(HostError::InvalidSwap) => Outcome::InvalidSwap, + Err(HostError::HistoryUnavailable) => Outcome::HistoryUnavailable, + Err(HostError::RepoNotFound) => Outcome::RepoGone, + Err(other) => Outcome::Other(other.to_string()), + } +} + +pub fn pds_outcome(result: &anyhow::Result) -> Outcome { + let Err(error) = result else { + return Outcome::Applied; + }; + match error.downcast_ref::() { + Some(SpaceStoreError::RecordExists(_)) => Outcome::RecordExists, + Some(SpaceStoreError::RecordNotFound(_)) => Outcome::RecordNotFound, + Some(SpaceStoreError::InvalidSwap(_)) => Outcome::InvalidSwap, + Some(SpaceStoreError::HistoryUnavailable) => Outcome::HistoryUnavailable, + Some(SpaceStoreError::SpaceNotFound(_)) | Some(SpaceStoreError::SpaceDeleted(_)) => { + Outcome::RepoGone + } + None => Outcome::Other(error.to_string()), + } +} + +/// `None` when the repo does not exist, so a scenario whose first batch is +/// rejected on both sides still compares. +pub async fn dump_shim(store: &dyn RepoStore, space_uri: &str, did: &str) -> Option { + let head = store.head(space_uri, did).await.expect("shim head")?; let (records, _) = store .list_records(space_uri, did, None, None, u32::MAX) .await .expect("shim records"); - let head = store - .head(space_uri, did) - .await - .expect("shim head") - .expect("shim repo"); let ops = store .list_ops(space_uri, did, None, None, u32::MAX) .await .expect("shim ops"); - RepoDump { + Some(RepoDump { records: records .into_iter() .map(|r| (r.collection, r.rkey, r.cid, r.value)) @@ -33,20 +73,20 @@ pub async fn dump_shim(store: &dyn RepoStore, space_uri: &str, did: &str) -> Rep .into_iter() .map(|o| (o.collection, o.rkey, o.cid, o.prev)) .collect(), - } + }) } -pub async fn dump_pds(store: &SpaceStore, space_uri: &str) -> RepoDump { +pub async fn dump_pds(store: &SpaceStore, space_uri: &str) -> Option { + let state = store.repo_state(space_uri).await.expect("pds repo state")?; + if state.deleted { + return None; + } let records = store.all_records(space_uri).await.expect("pds records"); - let state = store - .live_repo_state(space_uri) - .await - .expect("pds repo state"); let (ops, _) = store .list_repo_ops(space_uri, None, None, usize::MAX >> 1) .await .expect("pds ops"); - RepoDump { + Some(RepoDump { records: records .into_iter() .map(|r| (r.collection, r.rkey, r.cid, r.value)) @@ -56,10 +96,25 @@ pub async fn dump_pds(store: &SpaceStore, space_uri: &str) -> RepoDump { .into_iter() .map(|o| (o.collection, o.rkey, o.cid, o.prev)) .collect(), + }) +} + +pub fn assert_parity(name: &str, shim: &Option, pds: &Option) -> bool { + match (shim, pds) { + (None, None) => true, + (Some(shim), Some(pds)) => compare(name, shim, pds), + (shim, pds) => { + eprintln!( + "{name}: repo existence differs: shim={}, pds={}", + shim.is_some(), + pds.is_some() + ); + false + } } } -pub fn assert_parity(name: &str, shim: &RepoDump, pds: &RepoDump) -> bool { +fn compare(name: &str, shim: &RepoDump, pds: &RepoDump) -> bool { let mut equal = true; for (field, left, right) in [ ( diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs index 835c9a2f..6082e730 100644 --- a/rsky-spaces-parity/tests/parity.rs +++ b/rsky-spaces-parity/tests/parity.rs @@ -1,8 +1,9 @@ use oracle_rsky_space::space_id::SpaceId; use rsky_pds::actor_store::db::get_migrated_db; use rsky_pds::actor_store::space::{encode_record, oplog_window, SpaceStore, SpaceWrite}; -use rsky_space_host::repo::{ActorStoreRepos, RepoStore, RepoWrite}; -use rsky_spaces_parity::{assert_parity, dump_pds, dump_shim}; +use rsky_space_host::actor_repos::ActorStoreRepos; +use rsky_space_host::repo::{RepoStore, RepoWrite}; +use rsky_spaces_parity::{assert_parity, dump_pds, dump_shim, pds_outcome, shim_outcome}; use serde_json::{json, Value}; fn sqlite_master(path: &std::path::Path) -> Vec<(String, String, String)> { @@ -92,126 +93,140 @@ impl ScriptWrite { } } -async fn run(name: &str, script: Vec) -> bool { +fn create(rkey: &'static str, text: &str) -> ScriptWrite { + ScriptWrite::Create { + rkey, + value: json!({ "text": text }), + } +} + +fn cid_of(value: &Value) -> String { + encode_record(value).expect("record encoding").0 +} + +/// Drive both stores through the same batches, comparing the outcome of each +/// batch and then the two stores' contents. +async fn run(name: &str, batches: Vec>) -> bool { let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); let space_uri = space.uri(); let temp = tempfile::tempdir().expect("tempdir"); - let shim = ActorStoreRepos::open(temp.path()).expect("shim store"); + let shim = ActorStoreRepos::open(temp.path().join("shim")).expect("shim store"); let pds = SpaceStore::new( DID.into(), get_migrated_db(temp.path().join("store.sqlite")) .await .expect("pds db"), ); - let shim_result = shim - .apply_writes( - &space_uri, - DID, - "3shimrev", - &script.iter().map(ScriptWrite::shim).collect::>(), - ) - .await; - let pds_result = pds - .apply_writes( - &space, - script.iter().map(ScriptWrite::pds).collect(), - oplog_window(), - ) - .await; - let equal_outcome = match (shim_result, pds_result) { - (Ok(_), Ok(_)) => assert_parity( - name, - &dump_shim(&shim, &space_uri, DID).await, - &dump_pds(&pds, &space_uri).await, - ), - (Err(_), Err(_)) => false, - (left, right) => { + + for (index, batch) in batches.iter().enumerate() { + let shim_result = shim + .apply_writes( + &space_uri, + DID, + "3shimrev", + &batch.iter().map(ScriptWrite::shim).collect::>(), + ) + .await; + let pds_result = pds + .apply_writes( + &space, + batch.iter().map(ScriptWrite::pds).collect(), + oplog_window(), + ) + .await; + let (shim_outcome, pds_outcome) = (shim_outcome(&shim_result), pds_outcome(&pds_result)); + if shim_outcome != pds_outcome { eprintln!( - "{name}: outcomes differ: shim_ok={}, pds_ok={}", - left.is_ok(), - right.is_ok() + "{name}: batch {index} outcomes differ\n shim: {shim_outcome:?}\n pds: {pds_outcome:?}" ); - false + return false; } - }; - equal_outcome + } + + assert_parity( + name, + &dump_shim(&shim, &space_uri, DID).await, + &dump_pds(&pds, &space_uri).await, + ) } -#[tokio::test] -async fn scoreboard() { - let first = json!({"text": "first"}); - let first_cid = encode_record(&first).expect("record encoding").0; - let scenarios = [ +fn scenarios() -> Vec<(&'static str, Vec>)> { + let first = json!({ "text": "first" }); + vec![ ( - "S1 create", - vec![ScriptWrite::Create { - rkey: "one", - value: first.clone(), - }], + "S1 create single record", + vec![vec![create("one", "first")]], ), ( "S2 batch create", + vec![vec![ + create("one", "one"), + create("two", "two"), + create("three", "three"), + ]], + ), + ( + "S3 update with swap-cid success", vec![ - ScriptWrite::Create { + vec![create("one", "first")], + vec![ScriptWrite::Update { rkey: "one", - value: json!({"text":"one"}), - }, - ScriptWrite::Create { - rkey: "two", - value: json!({"text":"two"}), - }, - ScriptWrite::Create { - rkey: "three", - value: json!({"text":"three"}), - }, + value: json!({ "text": "second" }), + swap: Some(cid_of(&first)), + }], ], ), ( - "S3 update swap success", + "S4 swap-cid conflict", vec![ - ScriptWrite::Create { - rkey: "one", - value: first.clone(), - }, - ScriptWrite::Update { + vec![create("one", "first")], + vec![ScriptWrite::Update { rkey: "one", - value: json!({"text":"second"}), - swap: Some(first_cid.clone()), - }, + value: json!({ "text": "second" }), + swap: Some(cid_of(&json!({ "text": "stale" }))), + }], ], ), ( - "S4 swap conflict", + "S5 delete", vec![ - ScriptWrite::Create { + vec![create("one", "first"), create("two", "two")], + vec![ScriptWrite::Delete { rkey: "one", - value: first, - }, - ScriptWrite::Update { - rkey: "one", - value: json!({"text":"second"}), - swap: Some("bafyreinvalid".into()), - }, + swap: None, + }], ], ), ( - "S5 delete", + "S6 delete then recreate same rkey", vec![ - ScriptWrite::Create { + vec![create("one", "first")], + vec![ScriptWrite::Delete { rkey: "one", - value: json!({"text":"first"}), - }, - ScriptWrite::Delete { + swap: Some(cid_of(&first)), + }], + vec![create("one", "reborn")], + vec![ScriptWrite::Delete { rkey: "one", swap: None, - }, + }], + vec![ScriptWrite::Delete { + rkey: "one", + swap: None, + }], ], ), - ]; + ] +} + +#[tokio::test] +async fn scoreboard() { + let scenarios = scenarios(); + let total = scenarios.len(); let mut equal = 0; - for (name, script) in scenarios { - equal += usize::from(run(name, script).await); + for (name, batches) in scenarios { + equal += usize::from(run(name, batches).await); } - println!("parity: {equal}/5 scenarios byte-equal"); - assert_eq!(equal, 5, "parity harness must be red before convergence"); + println!("parity: {equal}/{total} scenarios byte-equal"); + assert_eq!(equal, total, "every scenario must be byte-equal"); } From 5e057a422e5bc44fe0f11a9668bbc6427dd82b20 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 14:00:38 -0400 Subject: [PATCH 34/56] test(spaces-parity): compare stored rows with normalized revisions Adds a second comparator over the space_* rows of both store files, with server-minted revisions, oplog ids and creation timestamps replaced by first-appearance placeholders, plus assertions that every revision is a valid TID and that oplog order agrees with revision order. Paged reads are now walked page by page on both sides. S7 (unicode and prefix-colliding keys), S8 (large record), S10 (two spaces, one author) and S11 (pagination) join the scoreboard: 10/10. --- rsky-spaces-parity/src/lib.rs | 237 ++++++++++++++++++ rsky-spaces-parity/tests/parity.rs | 374 +++++++++++++++++++++-------- 2 files changed, 514 insertions(+), 97 deletions(-) diff --git a/rsky-spaces-parity/src/lib.rs b/rsky-spaces-parity/src/lib.rs index c7bd27a7..bdfde644 100644 --- a/rsky-spaces-parity/src/lib.rs +++ b/rsky-spaces-parity/src/lib.rs @@ -1,6 +1,8 @@ use rsky_pds::actor_store::space::{SpaceStore, SpaceStoreError}; use rsky_space_host::error::HostError; use rsky_space_host::repo::RepoStore; +use std::collections::BTreeMap; +use std::path::Path; #[derive(Debug, Clone, PartialEq, Eq)] pub struct RepoDump { @@ -136,3 +138,238 @@ fn compare(name: &str, shim: &RepoDump, pds: &RepoDump) -> bool { } equal } + +// ------------------------------------------------------- stored-row comparison + +/// The `space_*` rows of one store file, rendered as strings with the values +/// that cannot match across two independent writers replaced by placeholders: +/// revisions (server-minted TIDs) by `R1, R2, …` in first-appearance order, +/// oplog row ids by `I1, I2, …`, and creation timestamps by `T`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableDump { + pub tables: Vec<(&'static str, Vec)>, + /// The distinct revisions in first-appearance order, unnormalized. + pub revs: Vec, + /// Revisions in oplog order, unnormalized. + pub oplog_revs: Vec, +} + +struct Normalizer { + revs: BTreeMap, + order: Vec, + ids: BTreeMap, +} + +impl Normalizer { + fn rev(&mut self, rev: &str) -> String { + if let Some(placeholder) = self.revs.get(rev) { + return placeholder.clone(); + } + let placeholder = format!("R{}", self.order.len() + 1); + self.revs.insert(rev.to_string(), placeholder.clone()); + self.order.push(rev.to_string()); + placeholder + } + + fn optional_rev(&mut self, rev: Option) -> String { + rev.map(|r| self.rev(&r)).unwrap_or_else(|| "-".to_string()) + } + + fn id(&mut self, id: i64) -> String { + let next = format!("I{}", self.ids.len() + 1); + self.ids.entry(id).or_insert(next).clone() + } +} + +pub fn dump_tables(path: &Path) -> TableDump { + let conn = rusqlite::Connection::open(path).expect("store connection"); + let mut norm = Normalizer { + revs: BTreeMap::new(), + order: Vec::new(), + ids: BTreeMap::new(), + }; + + let mut oplog_revs = Vec::new(); + let mut oplog = Vec::new(); + { + let mut statement = conn + .prepare( + "SELECT id, space_uri, rev, collection, rkey, cid, prev \ + FROM space_oplog ORDER BY id", + ) + .expect("oplog statement"); + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + )) + }) + .expect("oplog rows") + .collect::>>() + .expect("oplog values"); + for (id, space_uri, rev, collection, rkey, cid, prev) in rows { + oplog_revs.push(rev.clone()); + oplog.push(format!( + "{} {space_uri} {} {collection} {rkey} {} {}", + norm.id(id), + norm.rev(&rev), + cid.unwrap_or_else(|| "-".into()), + prev.unwrap_or_else(|| "-".into()), + )); + } + } + + let mut records = Vec::new(); + { + let mut statement = conn + .prepare( + "SELECT space_uri, collection, rkey, cid, rev, value \ + FROM space_record ORDER BY space_uri, collection, rkey", + ) + .expect("record statement"); + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Vec>(5)?, + )) + }) + .expect("record rows") + .collect::>>() + .expect("record values"); + for (space_uri, collection, rkey, cid, rev, value) in rows { + records.push(format!( + "{space_uri} {collection} {rkey} {cid} {} {}", + norm.rev(&rev), + render(&value) + )); + } + } + + let mut repos = Vec::new(); + { + let mut statement = conn + .prepare( + "SELECT space_uri, authority, space_type, skey, rev, lthash_state, \ + oplog_floor_rev, deleted FROM space_repo ORDER BY space_uri", + ) + .expect("repo statement"); + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Vec>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, i64>(7)?, + )) + }) + .expect("repo rows") + .collect::>>() + .expect("repo values"); + for (space_uri, authority, space_type, skey, rev, state, floor, deleted) in rows { + repos.push(format!( + "{space_uri} {authority} {space_type} {skey} {} {} {} {deleted}", + norm.rev(&rev), + render(&state), + norm.optional_rev(floor), + )); + } + } + + let blobs; + { + let mut statement = conn + .prepare( + "SELECT space_uri, blob_cid, collection, rkey FROM space_blob_ref \ + ORDER BY space_uri, blob_cid, collection, rkey", + ) + .expect("blob statement"); + blobs = statement + .query_map([], |row| { + Ok(format!( + "{} {} {} {}", + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)? + )) + }) + .expect("blob rows") + .collect::>>() + .expect("blob values"); + } + + TableDump { + tables: vec![ + ("space_repo", repos), + ("space_record", records), + ("space_oplog", oplog), + ("space_blob_ref", blobs), + ], + revs: norm.order, + oplog_revs, + } +} + +pub fn compare_tables(name: &str, shim: &TableDump, pds: &TableDump) -> bool { + let mut equal = true; + for ((table, left), (_, right)) in shim.tables.iter().zip(pds.tables.iter()) { + if left != right { + eprintln!("{name}: {table} rows differ\n shim: {left:?}\n pds: {right:?}"); + equal = false; + } + } + equal +} + +/// Every revision is a syntactically valid TID, and the oplog is ordered by +/// revision — the two properties the placeholders would otherwise hide. +pub fn revs_are_well_formed(name: &str, dump: &TableDump, side: &str) -> bool { + let mut sound = true; + for rev in &dump.revs { + if !is_tid(rev) { + eprintln!("{name}: {side} rev `{rev}` is not a valid TID"); + sound = false; + } + } + for pair in dump.oplog_revs.windows(2) { + if pair[0] > pair[1] { + eprintln!( + "{name}: {side} oplog order disagrees with rev order: `{}` then `{}`", + pair[0], pair[1] + ); + sound = false; + } + } + sound +} + +/// 13 characters of the sortable base32 alphabet, with the high bit of the +/// leading character clear. +pub fn is_tid(value: &str) -> bool { + const ALPHABET: &str = "234567abcdefghijklmnopqrstuvwxyz"; + value.len() == 13 + && value.chars().all(|c| ALPHABET.contains(c)) + && value + .chars() + .next() + .is_some_and(|c| "234567abcdefghij".contains(c)) +} + +fn render(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs index 6082e730..389de8c9 100644 --- a/rsky-spaces-parity/tests/parity.rs +++ b/rsky-spaces-parity/tests/parity.rs @@ -3,7 +3,10 @@ use rsky_pds::actor_store::db::get_migrated_db; use rsky_pds::actor_store::space::{encode_record, oplog_window, SpaceStore, SpaceWrite}; use rsky_space_host::actor_repos::ActorStoreRepos; use rsky_space_host::repo::{RepoStore, RepoWrite}; -use rsky_spaces_parity::{assert_parity, dump_pds, dump_shim, pds_outcome, shim_outcome}; +use rsky_spaces_parity::{ + assert_parity, compare_tables, dump_pds, dump_shim, dump_tables, pds_outcome, + revs_are_well_formed, shim_outcome, +}; use serde_json::{json, Value}; fn sqlite_master(path: &std::path::Path) -> Vec<(String, String, String)> { @@ -31,127 +34,274 @@ async fn actor_schema_matches_pinned_oracle() { const DID: &str = "did:plc:parityauthor"; const AUTHORITY: &str = "did:plc:parityauthority"; const COLLECTION: &str = "app.bsky.feed.post"; +const PAGE: u32 = 2; #[derive(Clone)] -enum ScriptWrite { - Create { - rkey: &'static str, - value: Value, - }, - Update { - rkey: &'static str, - value: Value, - swap: Option, - }, - Delete { - rkey: &'static str, - swap: Option, - }, +struct ScriptWrite { + space: usize, + collection: &'static str, + rkey: String, + action: Action, +} + +#[derive(Clone)] +enum Action { + Create(Value), + Update(Value, Option), + Delete(Option), } impl ScriptWrite { fn shim(&self) -> RepoWrite { - match self { - Self::Create { rkey, value } => RepoWrite::Create { - collection: COLLECTION.into(), - rkey: (*rkey).into(), - value: encode_record(value).expect("record encoding").1, + let (collection, rkey) = (self.collection.to_string(), self.rkey.clone()); + match &self.action { + Action::Create(value) => RepoWrite::Create { + collection, + rkey, + value: encoded(value), }, - Self::Update { rkey, value, swap } => RepoWrite::Update { - collection: COLLECTION.into(), - rkey: (*rkey).into(), - value: encode_record(value).expect("record encoding").1, + Action::Update(value, swap) => RepoWrite::Update { + collection, + rkey, + value: encoded(value), swap_record: swap.clone(), }, - Self::Delete { rkey, swap } => RepoWrite::Delete { - collection: COLLECTION.into(), - rkey: (*rkey).into(), + Action::Delete(swap) => RepoWrite::Delete { + collection, + rkey, swap_record: swap.clone(), }, } } fn pds(&self) -> SpaceWrite { - match self { - Self::Create { rkey, value } => SpaceWrite::Create { - collection: COLLECTION.into(), - rkey: (*rkey).into(), + let (collection, rkey) = (self.collection.to_string(), self.rkey.clone()); + match &self.action { + Action::Create(value) => SpaceWrite::Create { + collection, + rkey, value: value.clone(), }, - Self::Update { rkey, value, swap } => SpaceWrite::Update { - collection: COLLECTION.into(), - rkey: (*rkey).into(), + Action::Update(value, swap) => SpaceWrite::Update { + collection, + rkey, value: value.clone(), swap_cid: swap.clone(), }, - Self::Delete { rkey, swap } => SpaceWrite::Delete { - collection: COLLECTION.into(), - rkey: (*rkey).into(), + Action::Delete(swap) => SpaceWrite::Delete { + collection, + rkey, swap_cid: swap.clone(), }, } } } -fn create(rkey: &'static str, text: &str) -> ScriptWrite { - ScriptWrite::Create { - rkey, - value: json!({ "text": text }), - } +fn encoded(value: &Value) -> Vec { + encode_record(value).expect("record encoding").1 } fn cid_of(value: &Value) -> String { encode_record(value).expect("record encoding").0 } -/// Drive both stores through the same batches, comparing the outcome of each -/// batch and then the two stores' contents. +fn create(rkey: &str, text: &str) -> ScriptWrite { + ScriptWrite { + space: 0, + collection: COLLECTION, + rkey: rkey.to_string(), + action: Action::Create(json!({ "text": text })), + } +} + +fn update(rkey: &str, text: &str, swap: Option) -> ScriptWrite { + ScriptWrite { + space: 0, + collection: COLLECTION, + rkey: rkey.to_string(), + action: Action::Update(json!({ "text": text }), swap), + } +} + +fn delete(rkey: &str, swap: Option) -> ScriptWrite { + ScriptWrite { + space: 0, + collection: COLLECTION, + rkey: rkey.to_string(), + action: Action::Delete(swap), + } +} + +fn in_space(space: usize, write: ScriptWrite) -> ScriptWrite { + ScriptWrite { space, ..write } +} + +fn in_collection(collection: &'static str, write: ScriptWrite) -> ScriptWrite { + ScriptWrite { + collection, + ..write + } +} + +/// Drive both stores through the same batches, then compare: refusal reasons +/// per batch, the reads each side serves, the rows each side stored (with +/// server-minted revisions and oplog ids normalized), and paged reads. async fn run(name: &str, batches: Vec>) -> bool { - let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); - let space_uri = space.uri(); + let spaces = [ + SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"), + SpaceId::new(AUTHORITY, "community.blacksky.feed", "second"), + ]; let temp = tempfile::tempdir().expect("tempdir"); let shim = ActorStoreRepos::open(temp.path().join("shim")).expect("shim store"); + let pds_path = temp.path().join("store.sqlite"); let pds = SpaceStore::new( DID.into(), - get_migrated_db(temp.path().join("store.sqlite")) - .await - .expect("pds db"), + get_migrated_db(&pds_path).await.expect("pds db"), ); for (index, batch) in batches.iter().enumerate() { - let shim_result = shim - .apply_writes( - &space_uri, - DID, - "3shimrev", - &batch.iter().map(ScriptWrite::shim).collect::>(), - ) - .await; - let pds_result = pds - .apply_writes( - &space, - batch.iter().map(ScriptWrite::pds).collect(), - oplog_window(), + for (space_index, space) in spaces.iter().enumerate() { + let writes: Vec<&ScriptWrite> = + batch.iter().filter(|w| w.space == space_index).collect(); + if writes.is_empty() { + continue; + } + let shim_result = shim + .apply_writes( + &space.uri(), + DID, + "3shimrev", + &writes.iter().map(|w| w.shim()).collect::>(), + ) + .await; + let pds_result = pds + .apply_writes( + space, + writes.iter().map(|w| w.pds()).collect(), + oplog_window(), + ) + .await; + let (shim_kind, pds_kind) = (shim_outcome(&shim_result), pds_outcome(&pds_result)); + if shim_kind != pds_kind { + eprintln!( + "{name}: batch {index} space {space_index} outcomes differ\n \ + shim: {shim_kind:?}\n pds: {pds_kind:?}" + ); + return false; + } + } + } + + let shim_path = shim.store_path(DID).expect("shim store path"); + let (shim_tables, pds_tables) = (dump_tables(&shim_path), dump_tables(&pds_path)); + let mut equal = compare_tables(name, &shim_tables, &pds_tables) + && revs_are_well_formed(name, &shim_tables, "shim") + && revs_are_well_formed(name, &pds_tables, "pds"); + + for space in &spaces { + let uri = space.uri(); + equal = equal + && assert_parity( + name, + &dump_shim(&shim, &uri, DID).await, + &dump_pds(&pds, &uri).await, ) - .await; - let (shim_outcome, pds_outcome) = (shim_outcome(&shim_result), pds_outcome(&pds_result)); - if shim_outcome != pds_outcome { + && paged_reads_equal(name, &shim, &pds, &uri).await; + } + equal +} + +/// Walk both sides in `PAGE`-sized pages, comparing every page and the cursor +/// each side hands back. +async fn paged_reads_equal( + name: &str, + shim: &ActorStoreRepos, + pds: &SpaceStore, + space_uri: &str, +) -> bool { + if shim + .head(space_uri, DID) + .await + .expect("shim head") + .is_none() + { + return true; + } + + let mut cursor: Option = None; + loop { + let (page, next) = shim + .list_records(space_uri, DID, None, cursor.as_deref(), PAGE) + .await + .expect("shim record page"); + let mirror = pds + .list_records(space_uri, None, PAGE as usize, cursor.clone()) + .await + .expect("pds record page"); + let left: Vec<_> = page + .iter() + .map(|r| (&r.collection, &r.rkey, &r.cid, &r.value)) + .collect(); + let right: Vec<_> = mirror + .iter() + .map(|r| (&r.collection, &r.rkey, &r.cid, &r.value)) + .collect(); + if left != right { eprintln!( - "{name}: batch {index} outcomes differ\n shim: {shim_outcome:?}\n pds: {pds_outcome:?}" + "{name}: record page after {cursor:?} differs\n shim: {left:?}\n pds: {right:?}" ); return false; } + match next { + Some(next) if !page.is_empty() => cursor = Some(next), + _ => break, + } } - assert_parity( - name, - &dump_shim(&shim, &space_uri, DID).await, - &dump_pds(&pds, &space_uri).await, - ) + let mut cursor: Option = None; + loop { + let page = shim + .list_ops( + space_uri, + DID, + None, + cursor.map(|c| c.to_string()).as_deref(), + PAGE, + ) + .await + .expect("shim op page"); + let (mirror, more) = pds + .list_repo_ops(space_uri, None, cursor, PAGE as usize) + .await + .expect("pds op page"); + let left: Vec<_> = page + .ops + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + let right: Vec<_> = mirror + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + if left != right || page.complete == more { + eprintln!( + "{name}: op page after {cursor:?} differs\n \ + shim: {left:?} complete={}\n pds: {right:?} has_more={more}", + page.complete + ); + return false; + } + if !more { + break; + } + cursor = mirror.last().map(|o| o.id); + } + true } fn scenarios() -> Vec<(&'static str, Vec>)> { let first = json!({ "text": "first" }); + let long = "x".repeat(60 * 1024); vec![ ( "S1 create single record", @@ -169,53 +319,83 @@ fn scenarios() -> Vec<(&'static str, Vec>)> { "S3 update with swap-cid success", vec![ vec![create("one", "first")], - vec![ScriptWrite::Update { - rkey: "one", - value: json!({ "text": "second" }), - swap: Some(cid_of(&first)), - }], + vec![update("one", "second", Some(cid_of(&first)))], ], ), ( "S4 swap-cid conflict", vec![ vec![create("one", "first")], - vec![ScriptWrite::Update { - rkey: "one", - value: json!({ "text": "second" }), - swap: Some(cid_of(&json!({ "text": "stale" }))), - }], + vec![update( + "one", + "second", + Some(cid_of(&json!({ "text": "stale" }))), + )], ], ), ( "S5 delete", vec![ vec![create("one", "first"), create("two", "two")], - vec![ScriptWrite::Delete { - rkey: "one", - swap: None, - }], + vec![delete("one", None)], ], ), ( "S6 delete then recreate same rkey", vec![ vec![create("one", "first")], - vec![ScriptWrite::Delete { - rkey: "one", - swap: Some(cid_of(&first)), - }], + vec![delete("one", Some(cid_of(&first)))], vec![create("one", "reborn")], - vec![ScriptWrite::Delete { - rkey: "one", - swap: None, - }], - vec![ScriptWrite::Delete { - rkey: "one", - swap: None, - }], + vec![delete("one", None)], + vec![delete("one", None)], ], ), + ( + "S7 unicode and prefix-colliding keys", + vec![vec![ + create("é🌍", "unicode rkey"), + create("a-b", "hyphen sorts before slash"), + create("a", "bare"), + create("a.b", "dotted"), + in_collection("app.bsky.feed.pos", create("z", "shorter collection")), + in_collection("app.bsky.feed.post-x", create("a", "longer collection")), + in_collection("app.bsky.feed.post.deep", create("a", "deeper collection")), + ]], + ), + ( + "S8 large record", + vec![ + vec![create("big", &long)], + vec![update( + "big", + "small again", + Some(cid_of(&json!({"text": long}))), + )], + ], + ), + ( + "S10 two spaces one author", + vec![ + vec![ + create("one", "in first space"), + in_space(1, create("one", "in second space")), + ], + vec![ + in_space(1, create("two", "second space only")), + delete("one", None), + ], + ], + ), + ( + "S11 pagination across many revisions", + (0..7) + .map(|i| vec![create(&format!("r{i}"), &format!("body {i}"))]) + .chain(std::iter::once(vec![ + delete("r3", None), + update("r4", "edited", None), + ])) + .collect(), + ), ] } From e6bf488e78f8091e513fd360fdf8ed1ffc93373c Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 14:04:15 -0400 Subject: [PATCH 35/56] feat(space-host): compaction, blob divergence and cross-open parity S9 drives both stores past a three-row oplog window, so the shim now has to match the oracle's compaction, its oplog floor, and its refusal to serve history that fell out of the window. Every scenario additionally reopens the shim's own file with the oracle's SpaceStore and compares what it reads (S13), and probes history since every revision each side minted. S12 records the one accepted divergence: the oracle indexes blob references, the shim's storage keeps the bytes without indexing them and its write path refuses blob-bearing records outright. --- rsky-space-host/src/http.rs | 2 +- rsky-spaces-parity/src/lib.rs | 40 ++--- rsky-spaces-parity/tests/parity.rs | 238 +++++++++++++++++++++++++---- 3 files changed, 230 insertions(+), 50 deletions(-) diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index d6299a2e..262398b0 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -846,7 +846,7 @@ async fn create_record( )) } -fn contains_blob_ref(value: &Value) -> bool { +pub fn contains_blob_ref(value: &Value) -> bool { match value { Value::Array(values) => values.iter().any(contains_blob_ref), Value::Object(values) => { diff --git a/rsky-spaces-parity/src/lib.rs b/rsky-spaces-parity/src/lib.rs index bdfde644..2a5666fe 100644 --- a/rsky-spaces-parity/src/lib.rs +++ b/rsky-spaces-parity/src/lib.rs @@ -4,11 +4,15 @@ use rsky_space_host::repo::RepoStore; use std::collections::BTreeMap; use std::path::Path; +pub type OpTuple = (String, String, Option, Option); + #[derive(Debug, Clone, PartialEq, Eq)] pub struct RepoDump { pub records: Vec<(String, String, String, Vec)>, pub lthash_state: Vec, - pub ops: Vec<(String, String, Option, Option)>, + /// `Err` once compaction has dropped the start of history, which is itself + /// a value both sides must agree on. + pub ops: Result, Outcome>, } /// The classification a write batch or read is compared on: either it applied, @@ -60,21 +64,21 @@ pub async fn dump_shim(store: &dyn RepoStore, space_uri: &str, did: &str) -> Opt .list_records(space_uri, did, None, None, u32::MAX) .await .expect("shim records"); - let ops = store - .list_ops(space_uri, did, None, None, u32::MAX) - .await - .expect("shim ops"); + let ops = store.list_ops(space_uri, did, None, None, u32::MAX).await; Some(RepoDump { records: records .into_iter() .map(|r| (r.collection, r.rkey, r.cid, r.value)) .collect(), lthash_state: head.state.to_vec(), - ops: ops - .ops - .into_iter() - .map(|o| (o.collection, o.rkey, o.cid, o.prev)) - .collect(), + ops: match ops { + Ok(page) => Ok(page + .ops + .into_iter() + .map(|o| (o.collection, o.rkey, o.cid, o.prev)) + .collect()), + Err(error) => Err(shim_outcome::<()>(&Err(error))), + }, }) } @@ -84,20 +88,22 @@ pub async fn dump_pds(store: &SpaceStore, space_uri: &str) -> Option { return None; } let records = store.all_records(space_uri).await.expect("pds records"); - let (ops, _) = store + let ops = store .list_repo_ops(space_uri, None, None, usize::MAX >> 1) - .await - .expect("pds ops"); + .await; Some(RepoDump { records: records .into_iter() .map(|r| (r.collection, r.rkey, r.cid, r.value)) .collect(), lthash_state: state.lthash_state, - ops: ops - .into_iter() - .map(|o| (o.collection, o.rkey, o.cid, o.prev)) - .collect(), + ops: match ops { + Ok((page, _)) => Ok(page + .into_iter() + .map(|o| (o.collection, o.rkey, o.cid, o.prev)) + .collect()), + Err(error) => Err(pds_outcome::<()>(&Err(error))), + }, }) } diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs index 389de8c9..33098e75 100644 --- a/rsky-spaces-parity/tests/parity.rs +++ b/rsky-spaces-parity/tests/parity.rs @@ -1,6 +1,8 @@ use oracle_rsky_space::space_id::SpaceId; use rsky_pds::actor_store::db::get_migrated_db; -use rsky_pds::actor_store::space::{encode_record, oplog_window, SpaceStore, SpaceWrite}; +use rsky_pds::actor_store::space::{ + blob_refs_in_record, encode_record, SpaceStore, SpaceWrite, DEFAULT_OPLOG_WINDOW, +}; use rsky_space_host::actor_repos::ActorStoreRepos; use rsky_space_host::repo::{RepoStore, RepoWrite}; use rsky_spaces_parity::{ @@ -143,23 +145,41 @@ fn in_collection(collection: &'static str, write: ScriptWrite) -> ScriptWrite { } } +struct Scenario { + name: &'static str, + window: usize, + batches: Vec>, +} + +fn scenario(name: &'static str, batches: Vec>) -> Scenario { + Scenario { + name, + window: DEFAULT_OPLOG_WINDOW, + batches, + } +} + /// Drive both stores through the same batches, then compare: refusal reasons -/// per batch, the reads each side serves, the rows each side stored (with -/// server-minted revisions and oplog ids normalized), and paged reads. -async fn run(name: &str, batches: Vec>) -> bool { +/// per batch, the rows each side stored (with server-minted revisions and oplog +/// ids normalized), the reads each side serves whole and paged, the answers to +/// every `since` probe, and finally what the oracle reads back out of the +/// file the shim wrote. +async fn run(case: &Scenario) -> bool { + let name = case.name; let spaces = [ SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"), SpaceId::new(AUTHORITY, "community.blacksky.feed", "second"), ]; let temp = tempfile::tempdir().expect("tempdir"); - let shim = ActorStoreRepos::open(temp.path().join("shim")).expect("shim store"); + let shim = ActorStoreRepos::with_oplog_window(temp.path().join("shim"), case.window) + .expect("shim store"); let pds_path = temp.path().join("store.sqlite"); let pds = SpaceStore::new( DID.into(), get_migrated_db(&pds_path).await.expect("pds db"), ); - for (index, batch) in batches.iter().enumerate() { + for (index, batch) in case.batches.iter().enumerate() { for (space_index, space) in spaces.iter().enumerate() { let writes: Vec<&ScriptWrite> = batch.iter().filter(|w| w.space == space_index).collect(); @@ -175,11 +195,7 @@ async fn run(name: &str, batches: Vec>) -> bool { ) .await; let pds_result = pds - .apply_writes( - space, - writes.iter().map(|w| w.pds()).collect(), - oplog_window(), - ) + .apply_writes(space, writes.iter().map(|w| w.pds()).collect(), case.window) .await; let (shim_kind, pds_kind) = (shim_outcome(&shim_result), pds_outcome(&pds_result)); if shim_kind != pds_kind { @@ -198,6 +214,12 @@ async fn run(name: &str, batches: Vec>) -> bool { && revs_are_well_formed(name, &shim_tables, "shim") && revs_are_well_formed(name, &pds_tables, "pds"); + // The oracle, pointed at the shim's own file: the drop-in assertion. + let crossed = SpaceStore::new( + DID.into(), + get_migrated_db(&shim_path).await.expect("cross-open db"), + ); + for space in &spaces { let uri = space.uri(); equal = equal @@ -206,7 +228,14 @@ async fn run(name: &str, batches: Vec>) -> bool { &dump_shim(&shim, &uri, DID).await, &dump_pds(&pds, &uri).await, ) - && paged_reads_equal(name, &shim, &pds, &uri).await; + && assert_parity( + &format!("{name} cross-open"), + &dump_shim(&shim, &uri, DID).await, + &dump_pds(&crossed, &uri).await, + ) + && paged_reads_equal(name, &shim, &pds, &uri).await + && since_reads_equal(name, &shim, &pds, &uri, &shim_tables.revs, &pds_tables.revs) + .await; } equal } @@ -268,12 +297,21 @@ async fn paged_reads_equal( cursor.map(|c| c.to_string()).as_deref(), PAGE, ) - .await - .expect("shim op page"); - let (mirror, more) = pds + .await; + let mirror = pds .list_repo_ops(space_uri, None, cursor, PAGE as usize) - .await - .expect("pds op page"); + .await; + let (shim_kind, pds_kind) = (shim_outcome(&page), pds_outcome(&mirror)); + if shim_kind != pds_kind { + eprintln!( + "{name}: op page after {cursor:?} outcomes differ\n \ + shim: {shim_kind:?}\n pds: {pds_kind:?}" + ); + return false; + } + let (Ok(page), Ok((mirror, more))) = (page, mirror) else { + return true; + }; let left: Vec<_> = page .ops .iter() @@ -299,15 +337,64 @@ async fn paged_reads_equal( true } -fn scenarios() -> Vec<(&'static str, Vec>)> { +/// Ask both sides for history since each revision they minted. The revisions +/// differ between the two stores, so each side is probed with its own — the +/// nth revision on one side answers the nth on the other. +async fn since_reads_equal( + name: &str, + shim: &ActorStoreRepos, + pds: &SpaceStore, + space_uri: &str, + shim_revs: &[String], + pds_revs: &[String], +) -> bool { + for (index, (shim_rev, pds_rev)) in shim_revs.iter().zip(pds_revs).enumerate() { + let page = shim + .list_ops(space_uri, DID, Some(shim_rev), None, u32::MAX) + .await; + let mirror = pds + .list_repo_ops(space_uri, Some(pds_rev.clone()), None, usize::MAX >> 1) + .await; + let (shim_kind, pds_kind) = (shim_outcome(&page), pds_outcome(&mirror)); + if shim_kind != pds_kind { + eprintln!( + "{name}: history since revision {index} differs\n \ + shim: {shim_kind:?}\n pds: {pds_kind:?}" + ); + return false; + } + let (Ok(page), Ok((mirror, _))) = (page, mirror) else { + continue; + }; + let left: Vec<_> = page + .ops + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + let right: Vec<_> = mirror + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + if left != right { + eprintln!( + "{name}: history since revision {index} differs\n \ + shim: {left:?}\n pds: {right:?}" + ); + return false; + } + } + true +} + +fn scenarios() -> Vec { let first = json!({ "text": "first" }); let long = "x".repeat(60 * 1024); vec![ - ( + scenario( "S1 create single record", vec![vec![create("one", "first")]], ), - ( + scenario( "S2 batch create", vec![vec![ create("one", "one"), @@ -315,14 +402,14 @@ fn scenarios() -> Vec<(&'static str, Vec>)> { create("three", "three"), ]], ), - ( + scenario( "S3 update with swap-cid success", vec![ vec![create("one", "first")], vec![update("one", "second", Some(cid_of(&first)))], ], ), - ( + scenario( "S4 swap-cid conflict", vec![ vec![create("one", "first")], @@ -333,14 +420,14 @@ fn scenarios() -> Vec<(&'static str, Vec>)> { )], ], ), - ( + scenario( "S5 delete", vec![ vec![create("one", "first"), create("two", "two")], vec![delete("one", None)], ], ), - ( + scenario( "S6 delete then recreate same rkey", vec![ vec![create("one", "first")], @@ -350,7 +437,7 @@ fn scenarios() -> Vec<(&'static str, Vec>)> { vec![delete("one", None)], ], ), - ( + scenario( "S7 unicode and prefix-colliding keys", vec![vec![ create("é🌍", "unicode rkey"), @@ -362,18 +449,29 @@ fn scenarios() -> Vec<(&'static str, Vec>)> { in_collection("app.bsky.feed.post.deep", create("a", "deeper collection")), ]], ), - ( + scenario( "S8 large record", vec![ vec![create("big", &long)], vec![update( "big", "small again", - Some(cid_of(&json!({"text": long}))), + Some(cid_of(&json!({ "text": long }))), )], ], ), - ( + Scenario { + name: "S9 oplog compaction beyond the window", + window: 3, + batches: (0..6) + .map(|i| vec![create(&format!("r{i}"), &format!("body {i}"))]) + .chain(std::iter::once(vec![ + delete("r0", None), + delete("r1", None), + ])) + .collect(), + }, + scenario( "S10 two spaces one author", vec![ vec![ @@ -386,7 +484,7 @@ fn scenarios() -> Vec<(&'static str, Vec>)> { ], ], ), - ( + scenario( "S11 pagination across many revisions", (0..7) .map(|i| vec![create(&format!("r{i}"), &format!("body {i}"))]) @@ -396,17 +494,93 @@ fn scenarios() -> Vec<(&'static str, Vec>)> { ])) .collect(), ), + scenario( + "S13 cross-open after a mixed script", + vec![ + vec![create("keep", "kept"), create("gone", "removed")], + vec![ + update("keep", "edited", Some(cid_of(&json!({ "text": "kept" })))), + delete("gone", None), + in_collection("app.bsky.feed.like", create("l1", "liked")), + ], + vec![in_space(1, create("elsewhere", "other space"))], + ], + ), ] } +/// S12: blobs are the one documented divergence. The oracle accepts a +/// blob-bearing record and indexes the reference; the shim's storage keeps the +/// bytes but indexes nothing, and the host's write path refuses the record +/// outright, so a blob-carrying space cannot use the drop-in path. +#[tokio::test] +async fn s12_blob_bearing_record_is_a_documented_divergence() { + let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); + let temp = tempfile::tempdir().expect("tempdir"); + let shim = ActorStoreRepos::open(temp.path().join("shim")).expect("shim store"); + let pds = SpaceStore::new( + DID.into(), + get_migrated_db(temp.path().join("store.sqlite")) + .await + .expect("pds db"), + ); + let record = json!({ + "text": "with an image", + "image": { + "$type": "blob", + "ref": { "$link": "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egm4gv4a" }, + "mimeType": "image/png", + "size": 12345, + }, + }); + assert_eq!(blob_refs_in_record(&record).len(), 1); + assert!(rsky_space_host::http::contains_blob_ref(&record)); + + pds.apply_writes( + &space, + vec![SpaceWrite::Create { + collection: COLLECTION.into(), + rkey: "one".into(), + value: record.clone(), + }], + DEFAULT_OPLOG_WINDOW, + ) + .await + .expect("pds accepts blob-bearing records"); + shim.apply_writes( + &space.uri(), + DID, + "3shimrev", + &[RepoWrite::Create { + collection: COLLECTION.into(), + rkey: "one".into(), + value: encoded(&record), + }], + ) + .await + .expect("shim storage is shape-agnostic"); + + let shim_blobs = blob_refs(&shim.store_path(DID).expect("shim store path")); + let pds_blobs = blob_refs(&temp.path().join("store.sqlite")); + assert_eq!(pds_blobs, 1, "oracle indexes the blob reference"); + assert_eq!(shim_blobs, 0, "shim indexes no blob references"); +} + +fn blob_refs(path: &std::path::Path) -> i64 { + rusqlite::Connection::open(path) + .expect("store connection") + .query_row("SELECT COUNT(*) FROM space_blob_ref", [], |row| row.get(0)) + .expect("blob ref count") +} + #[tokio::test] async fn scoreboard() { let scenarios = scenarios(); let total = scenarios.len(); let mut equal = 0; - for (name, batches) in scenarios { - equal += usize::from(run(name, batches).await); + for case in &scenarios { + equal += usize::from(run(case).await); } - println!("parity: {equal}/{total} scenarios byte-equal"); + println!("parity: {equal}/{total} (+1 documented divergence) scenarios byte-equal"); assert_eq!(equal, total, "every scenario must be byte-equal"); } From 2f1111a6fd6bbdc39024176d739853548764712a Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 14:08:42 -0400 Subject: [PATCH 36/56] feat(space-host): legacy store converter with parity verification Converts a deployed multi-tenant store into per-account actor stores, carrying the LtHash state and every oplog row id across verbatim so readers see the same digest and syncers resume on the cursors they already hold. Record revisions, which the source schema does not carry, come from the newest operation that left the record at its current CID. S14 builds a fixture in the frozen source schema, converts it, and reads the result back with the oracle: heads, records, oplog ids and resume-from-cursor all verified. Scoreboard: 13/13 (+1 documented divergence). --- rsky-space-host/src/actor_repos.rs | 30 +-- rsky-space-host/src/bin/convert_store.rs | 23 ++ rsky-space-host/src/convert.rs | 190 +++++++++++++++++ rsky-space-host/src/lib.rs | 1 + rsky-spaces-parity/tests/parity.rs | 257 ++++++++++++++++++++++- 5 files changed, 485 insertions(+), 16 deletions(-) create mode 100644 rsky-space-host/src/bin/convert_store.rs create mode 100644 rsky-space-host/src/convert.rs diff --git a/rsky-space-host/src/actor_repos.rs b/rsky-space-host/src/actor_repos.rs index ef1322ba..e9bfb15b 100644 --- a/rsky-space-host/src/actor_repos.rs +++ b/rsky-space-host/src/actor_repos.rs @@ -25,6 +25,20 @@ use crate::repo::{ /// Oplog rows retained per repo before the oldest revisions are dropped. pub const DEFAULT_OPLOG_WINDOW: usize = 10_000; +/// `{root}/{sha256(did)[..2]}/{did}/store.sqlite`, the PDS actor layout. +pub fn store_path(directory: &std::path::Path, did: &str) -> Result { + if did.is_empty() + || !did.starts_with("did:") + || did.contains('/') + || did.contains('\\') + || did.contains("..") + { + return Err(HostError::InvalidRequest(format!("unusable did: {did}"))); + } + let digest = hex::encode(Sha256::digest(did.as_bytes())); + Ok(directory.join(&digest[..2]).join(did).join("store.sqlite")) +} + pub struct ActorStoreRepos { directory: PathBuf, oplog_window: usize, @@ -44,22 +58,8 @@ impl ActorStoreRepos { }) } - /// `{root}/{sha256(did)[..2]}/{did}/store.sqlite`, the PDS actor layout. pub fn store_path(&self, did: &str) -> Result { - if did.is_empty() - || !did.starts_with("did:") - || did.contains('/') - || did.contains('\\') - || did.contains("..") - { - return Err(HostError::InvalidRequest(format!("unusable did: {did}"))); - } - let digest = hex::encode(Sha256::digest(did.as_bytes())); - Ok(self - .directory - .join(&digest[..2]) - .join(did) - .join("store.sqlite")) + store_path(&self.directory, did) } fn with_store( diff --git a/rsky-space-host/src/bin/convert_store.rs b/rsky-space-host/src/bin/convert_store.rs new file mode 100644 index 00000000..f8d42fff --- /dev/null +++ b/rsky-space-host/src/bin/convert_store.rs @@ -0,0 +1,23 @@ +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(about = "Convert a multi-tenant space store into per-account actor stores")] +struct Args { + /// The deployed multi-tenant sqlite file, opened read-only. + #[arg(long)] + from: PathBuf, + /// Directory the per-account `store.sqlite` files are written under. + #[arg(long)] + into: PathBuf, +} + +fn main() -> Result<(), rsky_space_host::HostError> { + let args = Args::parse(); + let totals = rsky_space_host::convert::convert(&args.from, &args.into)?; + println!( + "converted {} accounts, {} repos, {} records, {} ops", + totals.accounts, totals.repos, totals.records, totals.ops + ); + Ok(()) +} diff --git a/rsky-space-host/src/convert.rs b/rsky-space-host/src/convert.rs new file mode 100644 index 00000000..0dc001de --- /dev/null +++ b/rsky-space-host/src/convert.rs @@ -0,0 +1,190 @@ +//! One-off conversion of a deployed multi-tenant store into per-account +//! actor stores. +//! +//! The source keeps every repo in one file with `did` as a column; the +//! destination is one `store.sqlite` per author DID. Two things are carried +//! across verbatim rather than recomputed: the LtHash state, because it is the +//! commit digest readers have already seen, and the operation-log row ids, +//! because they are the cursors syncers hold — a syncer must resume across the +//! swap without replaying or skipping. +//! +//! The source has no per-record revision. A record's revision is taken from +//! the newest operation that left it at its current CID, falling back to the +//! repo's revision when the log no longer reaches back that far. + +use rsky_space::space_id::SpaceId; +use rusqlite::{params, Connection, OpenFlags, OptionalExtension}; +use std::collections::HashMap; +use std::path::Path; + +use crate::error::{HostError, Result}; + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Converted { + pub accounts: usize, + pub repos: usize, + pub records: usize, + pub ops: usize, +} + +pub fn convert(legacy: &Path, directory: &Path) -> Result { + let source = + Connection::open_with_flags(legacy, OpenFlags::SQLITE_OPEN_READ_ONLY).map_err(sql)?; + let mut totals = Converted::default(); + + for did in accounts(&source)? { + let path = crate::actor_repos::store_path(directory, &did)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| HostError::Store(error.to_string()))?; + } + let mut target = crate::actor_schema::get_migrated_db(&path)?; + let tx = target.transaction().map_err(sql)?; + for (space_uri, rev, state) in repos(&source, &did)? { + let space = SpaceId::parse(&space_uri)?; + if state.len() != 2048 { + return Err(HostError::Store(format!( + "lthash state for {space_uri}/{did} is not 2048 bytes" + ))); + } + tx.execute( + "INSERT INTO space_repo \ + (space_uri, authority, space_type, skey, rev, lthash_state, oplog_floor_rev, deleted, created_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, 0, ?7)", + params![ + space_uri, + space.authority, + space.space_type, + space.skey, + rev, + state, + rsky_common::now() + ], + ) + .map_err(sql)?; + + let mut revisions = HashMap::new(); + for (seq, op_rev, collection, rkey, cid, prev) in ops(&source, &space_uri, &did)? { + if let Some(ref cid) = cid { + revisions.insert( + (collection.clone(), rkey.clone(), cid.clone()), + op_rev.clone(), + ); + } + tx.execute( + "INSERT INTO space_oplog (id, space_uri, rev, collection, rkey, cid, prev) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![seq, space_uri, op_rev, collection, rkey, cid, prev], + ) + .map_err(sql)?; + totals.ops += 1; + } + + for (collection, rkey, cid, value) in records(&source, &space_uri, &did)? { + let record_rev = revisions + .get(&(collection.clone(), rkey.clone(), cid.clone())) + .unwrap_or(&rev); + tx.execute( + "INSERT INTO space_record (space_uri, collection, rkey, cid, rev, value) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![space_uri, collection, rkey, cid, record_rev, value], + ) + .map_err(sql)?; + totals.records += 1; + } + totals.repos += 1; + } + tx.commit().map_err(sql)?; + totals.accounts += 1; + } + Ok(totals) +} + +/// The oplog id a syncer holding `cursor` before the conversion resumes from +/// after it. Ids are preserved, so the cursor is unchanged — the lookup exists +/// to prove that, and fails loudly if a converted store ever renumbers. +pub fn resumes_at(store: &Path, space_uri: &str, cursor: i64) -> Result> { + let conn = Connection::open_with_flags(store, OpenFlags::SQLITE_OPEN_READ_ONLY).map_err(sql)?; + conn.query_row( + "SELECT MIN(id) FROM space_oplog WHERE space_uri = ?1 AND id > ?2", + params![space_uri, cursor], + |row| row.get(0), + ) + .optional() + .map(Option::flatten) + .map_err(sql) +} + +fn accounts(source: &Connection) -> Result> { + let mut statement = source + .prepare("SELECT DISTINCT did FROM repo ORDER BY did") + .map_err(sql)?; + let rows = statement + .query_map([], |row| row.get(0)) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +type LegacyRepo = (String, String, Vec); + +fn repos(source: &Connection, did: &str) -> Result> { + let mut statement = source + .prepare("SELECT space_uri, rev, state FROM repo WHERE did = ?1 ORDER BY space_uri") + .map_err(sql)?; + let rows = statement + .query_map([did], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +type LegacyRecord = (String, String, String, Vec); + +fn records(source: &Connection, space_uri: &str, did: &str) -> Result> { + let mut statement = source + .prepare( + "SELECT collection, rkey, cid, value FROM record \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY collection, rkey", + ) + .map_err(sql)?; + let rows = statement + .query_map(params![space_uri, did], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) + }) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +type LegacyOp = (i64, String, String, String, Option, Option); + +fn ops(source: &Connection, space_uri: &str, did: &str) -> Result> { + let mut statement = source + .prepare( + "SELECT seq, rev, collection, rkey, cid, prev FROM repo_op \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY seq", + ) + .map_err(sql)?; + let rows = statement + .query_map(params![space_uri, did], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +fn sql(error: rusqlite::Error) -> HostError { + HostError::Store(error.to_string()) +} diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index 081eab53..dea5b5a1 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -25,6 +25,7 @@ pub mod attestation; pub mod authority; pub mod commits; pub mod config; +pub mod convert; pub mod error; pub mod http; pub mod keys; diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs index 33098e75..396ecdf7 100644 --- a/rsky-spaces-parity/tests/parity.rs +++ b/rsky-spaces-parity/tests/parity.rs @@ -576,11 +576,266 @@ fn blob_refs(path: &std::path::Path) -> i64 { #[tokio::test] async fn scoreboard() { let scenarios = scenarios(); - let total = scenarios.len(); + let total = scenarios.len() + 1; let mut equal = 0; for case in &scenarios { equal += usize::from(run(case).await); } + equal += usize::from(s14_legacy_converter().await); println!("parity: {equal}/{total} (+1 documented divergence) scenarios byte-equal"); assert_eq!(equal, total, "every scenario must be byte-equal"); } + +const LEGACY_SCHEMA: &str = "\ +CREATE TABLE repo (\ + space_uri TEXT NOT NULL, did TEXT NOT NULL, rev TEXT NOT NULL DEFAULT '', \ + state BLOB NOT NULL, PRIMARY KEY (space_uri, did));\ +CREATE TABLE record (\ + space_uri TEXT NOT NULL, did TEXT NOT NULL, path TEXT NOT NULL, \ + collection TEXT NOT NULL, rkey TEXT NOT NULL, cid TEXT NOT NULL, \ + value BLOB NOT NULL, PRIMARY KEY (space_uri, did, path));\ +CREATE TABLE repo_op (\ + seq INTEGER PRIMARY KEY AUTOINCREMENT, space_uri TEXT NOT NULL, \ + did TEXT NOT NULL, rev TEXT NOT NULL, collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, cid TEXT, prev TEXT);\ +CREATE INDEX repo_op_repo_seq ON repo_op (space_uri, did, seq);"; + +const SECOND_DID: &str = "did:plc:paritysecond"; + +struct LegacyOp { + space_uri: String, + did: &'static str, + rev: &'static str, + rkey: &'static str, + cid: Option, + prev: Option, +} + +/// S14: convert a store in the deployed multi-tenant schema into per-account +/// files, then read them back with the oracle. The LtHash state and every +/// oplog row id must survive the conversion, because they are respectively the +/// digest readers have seen and the cursors syncers hold. +async fn s14_legacy_converter() -> bool { + let name = "S14 legacy store converter"; + let temp = tempfile::tempdir().expect("tempdir"); + let legacy_path = temp.path().join("legacy.sqlite"); + let directory = temp.path().join("actors"); + let space_a = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); + let space_b = SpaceId::new(AUTHORITY, "community.blacksky.feed", "second"); + + let one_v1 = json!({ "text": "one" }); + let one_v2 = json!({ "text": "one edited" }); + let two = json!({ "text": "two" }); + let three = json!({ "text": "three" }); + let solo = json!({ "text": "solo" }); + let mine = json!({ "text": "mine" }); + + // Interleaved sequence numbers across accounts: the source numbers its + // oplog globally, the destination one file per account. + let ops = vec![ + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev1", + rkey: "one", + cid: Some(cid_of(&one_v1)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev1", + rkey: "two", + cid: Some(cid_of(&two)), + prev: None, + }, + LegacyOp { + space_uri: space_b.uri(), + did: DID, + rev: "3rev2", + rkey: "solo", + cid: Some(cid_of(&solo)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: SECOND_DID, + rev: "3rev3", + rkey: "mine", + cid: Some(cid_of(&mine)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev4", + rkey: "one", + cid: Some(cid_of(&one_v2)), + prev: Some(cid_of(&one_v1)), + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev5", + rkey: "three", + cid: Some(cid_of(&three)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev6", + rkey: "three", + cid: None, + prev: Some(cid_of(&three)), + }, + ]; + + let repos = [ + ( + space_a.uri(), + DID, + "3rev6", + vec![("one", &one_v2), ("two", &two)], + ), + (space_b.uri(), DID, "3rev2", vec![("solo", &solo)]), + (space_a.uri(), SECOND_DID, "3rev3", vec![("mine", &mine)]), + ]; + + { + let conn = rusqlite::Connection::open(&legacy_path).expect("legacy db"); + conn.execute_batch(LEGACY_SCHEMA).expect("legacy schema"); + for (space_uri, did, rev, records) in &repos { + let mut lthash = oracle_rsky_space::lthash::LtHash::new(); + for (rkey, value) in records { + let (cid, bytes) = encode_record(value).expect("record encoding"); + lthash.add(&oracle_rsky_space::lthash::element(COLLECTION, rkey, &cid)); + conn.execute( + "INSERT INTO record (space_uri, did, path, collection, rkey, cid, value) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + rusqlite::params![ + space_uri, + did, + format!("{COLLECTION}/{rkey}"), + COLLECTION, + rkey, + cid, + bytes + ], + ) + .expect("legacy record"); + } + conn.execute( + "INSERT INTO repo (space_uri, did, rev, state) VALUES (?1, ?2, ?3, ?4)", + rusqlite::params![space_uri, did, rev, lthash.state_bytes().to_vec()], + ) + .expect("legacy repo"); + } + for op in &ops { + conn.execute( + "INSERT INTO repo_op (space_uri, did, rev, collection, rkey, cid, prev) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + rusqlite::params![ + op.space_uri, + op.did, + op.rev, + COLLECTION, + op.rkey, + op.cid, + op.prev + ], + ) + .expect("legacy op"); + } + } + + let totals = rsky_space_host::convert::convert(&legacy_path, &directory).expect("conversion"); + if totals.accounts != 2 || totals.repos != 3 || totals.records != 4 || totals.ops != 7 { + eprintln!("{name}: unexpected conversion totals: {totals:?}"); + return false; + } + + for (space_uri, did, rev, records) in &repos { + let path = rsky_space_host::actor_repos::store_path(&directory, did).expect("store path"); + let store = SpaceStore::new( + (*did).into(), + get_migrated_db(&path).await.expect("converted db"), + ); + let state = store + .repo_state(space_uri) + .await + .expect("repo state") + .expect("repo row"); + let mut expected = oracle_rsky_space::lthash::LtHash::new(); + for (rkey, value) in records { + expected.add(&oracle_rsky_space::lthash::element( + COLLECTION, + rkey, + &cid_of(value), + )); + } + if state.rev != *rev || state.lthash_state != expected.state_bytes().to_vec() { + eprintln!("{name}: {space_uri}/{did} head did not survive conversion"); + return false; + } + let served = store.all_records(space_uri).await.expect("records"); + let expected_records: Vec<_> = records + .iter() + .map(|(rkey, value)| { + let (cid, bytes) = encode_record(value).expect("record encoding"); + (COLLECTION.to_string(), rkey.to_string(), cid, bytes) + }) + .collect(); + let served_records: Vec<_> = served + .iter() + .map(|r| { + ( + r.collection.clone(), + r.rkey.clone(), + r.cid.clone(), + r.value.clone(), + ) + }) + .collect(); + if served_records != expected_records { + eprintln!("{name}: {space_uri}/{did} records differ\n got: {served_records:?}\n want: {expected_records:?}"); + return false; + } + + // Oplog rows keep their source ids, so a held cursor still means the + // same position. + let expected_ids: Vec = ops + .iter() + .enumerate() + .filter(|(_, op)| op.space_uri == *space_uri && op.did == *did) + .map(|(index, _)| index as i64 + 1) + .collect(); + let (served_ops, _) = store + .list_repo_ops(space_uri, None, None, usize::MAX >> 1) + .await + .expect("ops"); + if served_ops.iter().map(|o| o.id).collect::>() != expected_ids { + eprintln!( + "{name}: {space_uri}/{did} oplog ids differ\n got: {:?}\n want: {expected_ids:?}", + served_ops.iter().map(|o| o.id).collect::>() + ); + return false; + } + + // A syncer holding the first id resumes at the next one, unchanged. + if let Some(&first) = expected_ids.first() { + let next = expected_ids.get(1).copied(); + let resumed = rsky_space_host::convert::resumes_at(&path, space_uri, first) + .expect("resume lookup"); + let (after, _) = store + .list_repo_ops(space_uri, None, Some(first), usize::MAX >> 1) + .await + .expect("ops after cursor"); + if resumed != next || after.first().map(|o| o.id) != next { + eprintln!("{name}: {space_uri}/{did} does not resume at {next:?}"); + return false; + } + } + } + true +} From 3324c3eac9e9386a77f21d10c1183b7b46378e15 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 14:12:01 -0400 Subject: [PATCH 37/56] refactor(space-host): align the in-memory backing with stored semantics The in-memory store no longer no-ops a delete of an absent record, and an update of an absent record is refused rather than upserted, matching what the actor-store backing and the pinned oracle do. WriteOutcome::Noop goes with it, so deleteRecord always records the write. --- rsky-space-host/src/http.rs | 4 +- rsky-space-host/src/repo.rs | 76 ++++++++++++++++++------------------- 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 262398b0..d2be49aa 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -883,9 +883,7 @@ async fn delete_record( }], ) .await?; - if !matches!(applied.outcomes[0], WriteOutcome::Noop) { - record_write(&state, &context, &space.uri(), &input.repo, &applied.rev).await?; - } + record_write(&state, &context, &space.uri(), &input.repo, &applied.rev).await?; Ok(Json( rsky_lexicon::com::atproto::space::DeleteRecordOutput { commit: Some(rsky_lexicon::com::atproto::space::CommitMeta { diff --git a/rsky-space-host/src/repo.rs b/rsky-space-host/src/repo.rs index c194431f..2d5fbc51 100644 --- a/rsky-space-host/src/repo.rs +++ b/rsky-space-host/src/repo.rs @@ -63,14 +63,13 @@ impl RepoWrite { } } -/// What a write did. A delete of an absent record is [`WriteOutcome::Noop`]: -/// it produces no oplog entry and leaves the digest untouched. +/// What a write did. Every applied write produces an oplog entry, so a batch +/// that touches an absent record is refused rather than silently doing nothing. #[derive(Debug, Clone, PartialEq, Eq)] pub enum WriteOutcome { Created { cid: String }, Updated { cid: String }, Deleted, - Noop, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -204,9 +203,7 @@ fn plan_batch( value, } => { if current.is_some() { - return Err(HostError::InvalidRequest(format!( - "record already exists: {path}" - ))); + return Err(HostError::RecordExists(path)); } let cid = dag_cbor_cid(value).to_string(); lt.add(&element(collection, rkey, &cid)); @@ -226,10 +223,11 @@ fn plan_batch( swap_record, } => { check_swap(swap_record.as_deref(), current.as_deref())?; + let Some(prev) = current.clone() else { + return Err(HostError::RecordNotFound(path)); + }; let cid = dag_cbor_cid(value).to_string(); - if let Some(prev) = ¤t { - lt.remove(&element(collection, rkey, prev)); - } + lt.remove(&element(collection, rkey, &prev)); lt.add(&element(collection, rkey, &cid)); PlannedWrite { collection: collection.clone(), @@ -247,15 +245,7 @@ fn plan_batch( } => { check_swap(swap_record.as_deref(), current.as_deref())?; let Some(prev) = current.clone() else { - planned.push(PlannedWrite { - collection: collection.clone(), - rkey: rkey.clone(), - cid: None, - prev: None, - value: None, - outcome: WriteOutcome::Noop, - }); - continue; + return Err(HostError::RecordNotFound(path)); }; lt.remove(&element(collection, rkey, &prev)); PlannedWrite { @@ -291,12 +281,6 @@ struct PlannedWrite { outcome: WriteOutcome, } -impl PlannedWrite { - fn is_noop(&self) -> bool { - self.outcome == WriteOutcome::Noop - } -} - pub(crate) fn page_cursor(page: &[T], limit: u32, key: impl Fn(&T) -> String) -> Option { match page.last() { Some(last) if page.len() == limit as usize => Some(key(last)), @@ -348,9 +332,6 @@ impl RepoStore for InMemoryRepos { let mut seq = self.next_seq.lock().unwrap(); for p in &planned { - if p.is_noop() { - continue; - } let path = record_path(&p.collection, &p.rkey); match &p.cid { Some(cid) => { @@ -380,9 +361,7 @@ impl RepoStore for InMemoryRepos { } repo.state = lt.state_bytes(); - if planned.iter().any(|p| !p.is_noop()) { - repo.rev = rev.to_string(); - } + repo.rev = rev.to_string(); Ok(Applied { rev: repo.rev.clone(), hash: lt.hash(), @@ -609,7 +588,7 @@ mod tests { store .apply_writes(SPACE, DID, "3rev2", &[create("a", "again")]) .await, - Err(HostError::InvalidRequest(_)) + Err(HostError::RecordExists(_)) )); assert!(matches!( store @@ -654,18 +633,35 @@ mod tests { assert_eq!(records.len(), 1); } - async fn exercise_noop_delete(store: &dyn RepoStore) { + async fn exercise_absent_record(store: &dyn RepoStore) { store .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) .await .unwrap(); - let applied = store - .apply_writes(SPACE, DID, "3rev2", &[delete("ghost")]) - .await - .unwrap(); - assert_eq!(applied.outcomes, vec![WriteOutcome::Noop]); - // A no-op neither advances the revision nor writes an oplog entry. - assert_eq!(applied.rev, "3rev1"); + assert!(matches!( + store + .apply_writes(SPACE, DID, "3rev2", &[delete("ghost")]) + .await, + Err(HostError::RecordNotFound(_)) + )); + assert!(matches!( + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Update { + collection: POST.to_string(), + rkey: "ghost".to_string(), + value: value("x"), + swap_record: None, + }] + ) + .await, + Err(HostError::RecordNotFound(_)) + )); + // A refused batch neither advances the revision nor logs an operation. + assert_eq!(store.head(SPACE, DID).await.unwrap().unwrap().rev, "3rev1"); let page = store.list_ops(SPACE, DID, None, None, 10).await.unwrap(); assert_eq!(page.ops.len(), 1); } @@ -857,7 +853,7 @@ mod tests { both_backings!(write_read_cycle, exercise_write_read_cycle); both_backings!(swap_and_conflict, exercise_swap_and_conflict); - both_backings!(noop_delete, exercise_noop_delete); + both_backings!(absent_record, exercise_absent_record); both_backings!(atomic_batch, exercise_atomic_batch); both_backings!(listing, exercise_listing); both_backings!(oplog_paging, exercise_oplog_paging); From 32e32122509f98bc1f58b0c018350a35c9abdf9f Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 14:12:15 -0400 Subject: [PATCH 38/56] chore(space-host): satisfy rustfmt --- rsky-space-host/src/authority.rs | 3 +-- rsky-space-host/src/config.rs | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rsky-space-host/src/authority.rs b/rsky-space-host/src/authority.rs index 21d9ce6a..c6c51776 100644 --- a/rsky-space-host/src/authority.rs +++ b/rsky-space-host/src/authority.rs @@ -36,8 +36,7 @@ impl AuthorityContext { /// Builds the [`AuthorityContext`] for an authority first seen at /// registration time; fails when the authority's signing key is unavailable. -pub type AuthorityFactory = - Arc Result> + Send + Sync>; +pub type AuthorityFactory = Arc Result> + Send + Sync>; /// The authorities this host answers for, keyed by authority DID. #[derive(Default)] diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index ac701762..aa7bc6f2 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -381,6 +381,9 @@ mod tests { let mut keyless = valid_unpinned(); keyless.actor_store_dir = String::new(); let message = keyless.validate().unwrap_err(); - assert!(message.contains("no space authority available"), "{message}"); + assert!( + message.contains("no space authority available"), + "{message}" + ); } } From d0742ff4694f7f97f528648e9de5d35c7d876b64 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 16:46:57 -0400 Subject: [PATCH 39/56] fix(space-host): match the oracle's xrpc error names and op hydration The two-process gate found three response-level divergences from rsky-pds that the in-process suite could not see: - RecordExists, RecordNotFound, InvalidSwap and HistoryUnavailable now answer with their own names and the oracle's status instead of being collapsed into InvalidRequest or answered with 409/410. - listRepoOps hydrates an op's value only when the op's cid still names the stored record, so a superseded op no longer carries the current record's value. - The local write path records the commit digest in the writer set, so listRepos returns the hash the oracle returns. --- Cargo.lock | 16 ++++++++++---- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/http.rs | 42 ++++++++++++++++++++++++++++++------- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6bca5cd..3991ebb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8354,7 +8354,7 @@ dependencies = [ "rsky-oauth 0.3.2", "rsky-repo 0.0.6", "rsky-space 0.4.2", - "rsky-space-host 0.7.1", + "rsky-space-host 0.7.2", "rsky-syntax 0.1.0", "rusqlite", "secp256k1", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.1" +version = "0.7.2" dependencies = [ "async-trait", "axum", @@ -8716,15 +8716,23 @@ dependencies = [ [[package]] name = "rsky-spaces-parity" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", + "base64 0.22.1", + "hmac", + "reqwest 0.12.23", + "rsky-crypto 0.2.0", + "rsky-oauth 0.3.2", "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", - "rsky-space-host 0.7.1", + "rsky-space-host 0.7.2", "rusqlite", + "secp256k1", + "serde", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", ] diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index ad24a37e..4cf9ee47 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.1" +version = "0.7.2" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index d2be49aa..6d64eccd 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -171,16 +171,22 @@ impl From for ApiError { "RepoNotFound", "repo not hosted here", ), - HostError::InvalidRequest(message) - | HostError::RecordExists(message) - | HostError::RecordNotFound(message) => Self::invalid_request(message.clone()), + HostError::InvalidRequest(message) => Self::invalid_request(message.clone()), + // These four names and statuses are what rsky-pds answers with, so + // a client cannot tell the two write paths apart. + HostError::RecordExists(message) => { + Self::new(StatusCode::BAD_REQUEST, "RecordExists", message.clone()) + } + HostError::RecordNotFound(message) => { + Self::new(StatusCode::BAD_REQUEST, "RecordNotFound", message.clone()) + } HostError::InvalidSwap => Self::new( - StatusCode::CONFLICT, + StatusCode::BAD_REQUEST, "InvalidSwap", "swap cid did not match", ), HostError::HistoryUnavailable => Self::new( - StatusCode::GONE, + StatusCode::BAD_REQUEST, "HistoryUnavailable", "requested history is no longer available", ), @@ -573,10 +579,13 @@ async fn list_repo_ops( let value = if params.exclude_values.unwrap_or(false) || op.cid.is_none() { None } else { + // Only the op that still holds the current record carries a value: + // a superseded op's cid no longer names what is stored. state .repos .get_record(&space.uri(), ¶ms.repo, &op.collection, &op.rkey) .await? + .filter(|record| Some(&record.cid) == op.cid.as_ref()) .map(|record| rsky_space::record::decode_record(&record.value)) .transpose() .map_err(HostError::from)? @@ -833,7 +842,15 @@ async fn create_record( WriteOutcome::Created { cid } => cid.clone(), _ => return Err(HostError::Store("create did not create".into()).into()), }; - record_write(&state, &context, &space.uri(), &input.repo, &applied.rev).await?; + record_write( + &state, + &context, + &space.uri(), + &input.repo, + &applied.rev, + Some(hex::encode(applied.hash)), + ) + .await?; Ok(Json( rsky_lexicon::com::atproto::space::CreateRecordOutput { uri: space.record_uri(&input.repo, &input.collection, &rkey), @@ -883,7 +900,15 @@ async fn delete_record( }], ) .await?; - record_write(&state, &context, &space.uri(), &input.repo, &applied.rev).await?; + record_write( + &state, + &context, + &space.uri(), + &input.repo, + &applied.rev, + Some(hex::encode(applied.hash)), + ) + .await?; Ok(Json( rsky_lexicon::com::atproto::space::DeleteRecordOutput { commit: Some(rsky_lexicon::com::atproto::space::CommitMeta { @@ -900,11 +925,12 @@ async fn record_write( space: &str, repo: &str, rev: &str, + hash: Option, ) -> Result<(), ApiError> { let now = (state.now)(); state .writers - .upsert_writer(space, repo, rev, None, now) + .upsert_writer(space, repo, rev, hash.as_deref(), now) .await?; let endpoints = state.registrations.endpoints(space, now).await?; fan_out_write( From 3ef2fdd578b48034edb4b3cc019c25c6777dbb17 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 16:47:07 -0400 Subject: [PATCH 40/56] test(spaces-parity): two-process layer 2 acceptance gate One command runs both real binaries and prints per-endpoint parity: run.sh builds rsky-pds from a detached build-only worktree at the pinned oracle revision, builds the space host from the working tree, then fires the same record script at each over XRPC and points the existing Layer 1 stored-row comparator at the two store files. The gate supplies the only support service itself: a stub DID directory on a loopback port, and the access tokens and space credentials each server demands. No database, container or network is involved. The four methods only rsky-pds routes are probed against it alone and reported as a surface difference rather than a parity failure; the probe is deliberately invalid so it cannot write to one side only. --- rsky-spaces-parity/Cargo.toml | 14 +- rsky-spaces-parity/layer2/README.md | 75 ++ rsky-spaces-parity/layer2/run.sh | 52 ++ rsky-spaces-parity/src/bin/layer2_gate.rs | 875 +++++++++++++++++++++ rsky-spaces-parity/src/layer2/car.rs | 87 ++ rsky-spaces-parity/src/layer2/directory.rs | 70 ++ rsky-spaces-parity/src/layer2/mod.rs | 126 +++ rsky-spaces-parity/src/layer2/normalize.rs | 80 ++ rsky-spaces-parity/src/layer2/process.rs | 154 ++++ rsky-spaces-parity/src/layer2/tokens.rs | 111 +++ rsky-spaces-parity/src/lib.rs | 2 + 11 files changed, 1645 insertions(+), 1 deletion(-) create mode 100644 rsky-spaces-parity/layer2/README.md create mode 100755 rsky-spaces-parity/layer2/run.sh create mode 100644 rsky-spaces-parity/src/bin/layer2_gate.rs create mode 100644 rsky-spaces-parity/src/layer2/car.rs create mode 100644 rsky-spaces-parity/src/layer2/directory.rs create mode 100644 rsky-spaces-parity/src/layer2/mod.rs create mode 100644 rsky-spaces-parity/src/layer2/normalize.rs create mode 100644 rsky-spaces-parity/src/layer2/process.rs create mode 100644 rsky-spaces-parity/src/layer2/tokens.rs diff --git a/rsky-spaces-parity/Cargo.toml b/rsky-spaces-parity/Cargo.toml index fc05722c..dd802888 100644 --- a/rsky-spaces-parity/Cargo.toml +++ b/rsky-spaces-parity/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-spaces-parity" -version = "0.1.0" +version = "0.2.0" edition = "2021" publish = false @@ -9,10 +9,22 @@ rsky-space-host = { path = "../rsky-space-host" } rsky-space = { path = "../rsky-space" } oracle-rsky-space = { package = "rsky-space", git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } rsky-pds = { git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } +rsky-oauth = { path = "../rsky-oauth" } +rsky-crypto = { workspace = true } anyhow = "1" serde_json = { workspace = true } tokio = { workspace = true } rusqlite = { workspace = true } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] } +base64 = "0.22" +hmac = "0.12" +sha2 = { workspace = true } +secp256k1 = { workspace = true } +serde = { workspace = true } [dev-dependencies] tempfile = "3" + +[[bin]] +name = "layer2-gate" +path = "src/bin/layer2_gate.rs" diff --git a/rsky-spaces-parity/layer2/README.md b/rsky-spaces-parity/layer2/README.md new file mode 100644 index 00000000..c9b19b70 --- /dev/null +++ b/rsky-spaces-parity/layer2/README.md @@ -0,0 +1,75 @@ +# Layer 2 — two-process acceptance gate + +Layer 1 (`cargo test -p rsky-spaces-parity`) drives both write paths in one +process. Layer 2 runs the two **real binaries** and compares what they actually +serve and store. + +```sh +./rsky-spaces-parity/layer2/run.sh +``` + +Exit code 0 means every scored check was equal. Nothing else is needed: no +Postgres, no Docker, no network. Both servers are SQLite-backed, and the gate +hosts the only support service (a stub DID directory on a loopback port). + +## What it does + +1. Creates a **detached, build-only** git worktree at the pinned oracle revision + `7ebd21ae788c550ee8510034d94eb19ede148738` and builds `rsky-pds` there. The + worktree is refused if it is dirty or at another revision, so the oracle can + never drift into the code under test. +2. Builds `rsky-space-host` from the working tree. +3. Runs `layer2-gate`, which: + - starts the stub DID directory, then the oracle PDS, then the space host; + - creates an account, activates it, opens a session, and creates the space + `at:///space/community.blacksky.feed/main` on the oracle; + - copies the oracle's actor-store directory so the space host has the account + signing keys, and replaces each `store.sqlite` with an empty one created by + the space host's own schema code; + - fires the same ten-step record script at each server over XRPC; + - compares five shared read endpoints; + - probes the four methods only the PDS routes; + - stops both servers and points the Layer 1 stored-row comparator + (`dump_tables` / `compare_tables` / `revs_are_well_formed`) at the two + `store.sqlite` files. + +Everything lands under `target/layer2/run`: `pds.log`, `shim.log`, `report.txt`, +and both store directories. The directory is wiped at the start of every run. + +## Credentials + +All local, all fixed, all created and destroyed inside the run directory; none +of it is a secret and none of it reaches a real service. + +- The oracle PDS takes its own session token on space writes. +- The space host verifies an access token it did not issue, so the gate acts as + the authorization server: it holds the same HS256 secret the host is + configured with (`SPACEHOST_OAUTH_HS256_SECRET`) and signs `at+jwt` tokens + with it, DPoP-bound to a fixed P-256 key. This is the same shape a PDS that is + its own authorization server issues. +- Read endpoints take a space credential on both sides: minted through + `getDelegationToken` → `getSpaceCredential` on the PDS, and through + `/admin/mintCredential` on the space host. + +## What is not compared, and why + +- `ikm`, `sig`, `mac` on a served commit: derived from fresh random key material + every serve. +- The paging `cursor`: an oplog row id, numbered per store. +- Revisions: server-minted TIDs. Substituted for `R1, R2, …` in first-appearance + order, exactly as the Layer 1 comparator does. Record keys are TIDs too and + compare literally. +- JSON object key order: the two decoders build maps in different orders from + identical DAG-CBOR bytes. Record identity is the CID, compared unnormalized in + the same response. +- `getSpace`: recorded as a documented divergence. The space definition is + configuration on the host and `space_def` rows on the PDS, which the storage + convergence leaves out of scope by design. + +## Running outside a sandbox + +On macOS both servers build a `reqwest` client whose proxy discovery calls +SystemConfiguration. A restricted sandbox denies that and the pinned +`hyper-util` panics on the null result. Run the gate with normal process +permissions. `sccache` is also unusable there, so `run.sh` clears +`RUSTC_WRAPPER`. diff --git a/rsky-spaces-parity/layer2/run.sh b/rsky-spaces-parity/layer2/run.sh new file mode 100755 index 00000000..703594b1 --- /dev/null +++ b/rsky-spaces-parity/layer2/run.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Layer 2 acceptance gate: build both real binaries, run the same record script +# at each over XRPC, then compare the two store files with the Layer 1 +# comparator. One command, no services beyond the two child processes. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +# The oracle is rsky-pds at this revision and nothing else. Its worktree is +# build-only: no edit ever lands there. +ORACLE_REV="${ORACLE_REV:-7ebd21ae788c550ee8510034d94eb19ede148738}" +ORACLE_TREE="${ORACLE_TREE:-/tmp/claude/rsky-layer2-oracle}" +RUN_DIR="${LAYER2_RUN_DIR:-$REPO_ROOT/target/layer2/run}" + +# sccache cannot open its cache in a sandboxed shell; the wrapper is only a +# build accelerator, so drop it rather than fail. +export RUSTC_WRAPPER="" +export CARGO_BUILD_RUSTC_WRAPPER="" + +say() { printf '\n== %s\n' "$1"; } + +say "oracle worktree at $ORACLE_REV" +if [ ! -d "$ORACLE_TREE/.git" ] && [ ! -f "$ORACLE_TREE/.git" ]; then + mkdir -p "$(dirname "$ORACLE_TREE")" + git worktree add --detach "$ORACLE_TREE" "$ORACLE_REV" +fi +HAVE="$(git -C "$ORACLE_TREE" rev-parse HEAD)" +if [ "$HAVE" != "$ORACLE_REV" ]; then + echo "oracle worktree is at $HAVE, expected $ORACLE_REV" >&2 + exit 1 +fi +if [ -n "$(git -C "$ORACLE_TREE" status --porcelain)" ]; then + echo "oracle worktree is dirty; it must stay build-only" >&2 + git -C "$ORACLE_TREE" status --short >&2 + exit 1 +fi + +say "building the pinned oracle pds" +( cd "$ORACLE_TREE" && cargo build -p rsky-pds --bin rsky-pds ) + +say "building the space host under test" +cargo build -p rsky-space-host --bin rsky-space-host + +say "building the gate" +cargo build -p rsky-spaces-parity --bin layer2-gate + +say "running the gate" +LAYER2_RUN_DIR="$RUN_DIR" \ +LAYER2_PDS_BIN="$ORACLE_TREE/target/debug/rsky-pds" \ +LAYER2_SHIM_BIN="$REPO_ROOT/target/debug/rsky-space-host" \ + ./target/debug/layer2-gate diff --git a/rsky-spaces-parity/src/bin/layer2_gate.rs b/rsky-spaces-parity/src/bin/layer2_gate.rs new file mode 100644 index 00000000..9b9d332c --- /dev/null +++ b/rsky-spaces-parity/src/bin/layer2_gate.rs @@ -0,0 +1,875 @@ +//! Layer 2 acceptance gate: the same record script fired at two real servers +//! over XRPC, then the Layer 1 comparator pointed at the two store files. +//! +//! Run it through `rsky-spaces-parity/layer2/run.sh`, which builds both binaries +//! and passes their paths in. + +use anyhow::{bail, Context, Result}; +use rsky_spaces_parity::layer2::normalize::{self, Revs}; +use rsky_spaces_parity::layer2::process::{copy_tree, free_port, reset_stores, Server}; +use rsky_spaces_parity::layer2::{car, directory::Directory, tokens, Scoreboard, Verdict}; +use rsky_spaces_parity::{compare_tables, dump_tables, revs_are_well_formed}; +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +const AUTHOR_DID: &str = "did:plc:layer2writeraaaaaaaaaaa"; +const HANDLE_DOMAIN: &str = ".layer2.test"; +const HANDLE: &str = "writer.layer2.test"; +const PASSWORD: &str = "layer2-local-password"; +const ADMIN_PASS: &str = "layer2-local-admin"; +const SPACE_TYPE: &str = "community.blacksky.feed"; +const SPACE_SKEY: &str = "main"; +const COLLECTION: &str = "com.example.post"; +const OTHER_COLLECTION: &str = "com.example.note"; +const HS256_SECRET: &str = "layer2-local-authorization-server-secret"; +const OAUTH_ISSUER: &str = "http://localhost:0/oauth"; +const PDS_SERVICE_DID: &str = "did:web:localho.st"; +const DAEMON_DID: &str = "did:plc:layer2daemonaaaaaaaaaaa"; + +/// Fixed local key material. None of it protects anything: the whole stack is +/// created and destroyed inside one run directory. +const AUTHORITY_SPACE_KEY: &str = + "1111111111111111111111111111111111111111111111111111111111111111"; +const DAEMON_KEY: &str = "2222222222222222222222222222222222222222222222222222222222222222"; +const PDS_JWT_KEY: &str = "9d5907143471e8f0e8df0f8b9512a8c5377878ee767f18fcf961055ecfc071cd"; +const PDS_ROTATION_KEY: &str = "fb478b39dd2ddf84bef135dd60f90381903eefadbb9df4b18a2b9b174ae72582"; +const PDS_SIGNING_KEY: &str = "71cfcf4882a6cff494c3d0affadd3858eb3a5838e7b5e15170e696a590a4fa01"; + +struct Gate { + client: reqwest::Client, + pds_url: String, + shim_url: String, + space: String, + session: String, + pds_credential: String, + shim_credential: String, +} + +/// One step of the write script, applied identically to both servers. +enum Step { + Create { + collection: &'static str, + rkey: &'static str, + record: Value, + }, + Delete { + collection: &'static str, + rkey: &'static str, + }, +} + +impl Step { + fn nsid(&self) -> &'static str { + match self { + Step::Create { .. } => "com.atproto.space.createRecord", + Step::Delete { .. } => "com.atproto.space.deleteRecord", + } + } + + fn label(&self) -> String { + match self { + Step::Create { + collection, rkey, .. + } => format!("createRecord {collection}/{rkey}"), + Step::Delete { collection, rkey } => format!("deleteRecord {collection}/{rkey}"), + } + } + + fn body(&self, space: &str, repo: &str) -> Value { + match self { + Step::Create { + collection, + rkey, + record, + } => json!({ + "space": space, + "repo": repo, + "collection": collection, + "rkey": rkey, + "record": record, + }), + Step::Delete { collection, rkey } => json!({ + "space": space, + "repo": repo, + "collection": collection, + "rkey": rkey, + }), + } + } +} + +fn script() -> Vec { + vec![ + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + record: json!({"text": "first", "n": 1}), + }, + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa2", + record: json!({"text": "second", "n": 2}), + }, + Step::Create { + collection: OTHER_COLLECTION, + rkey: "3kaaaaaaaaaa3", + record: json!({"text": "other collection", "nested": {"a": [1, 2, 3], "b": true}}), + }, + Step::Create { + collection: COLLECTION, + rkey: "unicode.rkey_1~", + record: json!({"text": "\u{e9}\u{4e16}\u{754c}\u{1f600}", "empty": ""}), + }, + // A duplicate rkey: both sides must refuse it the same way. + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + record: json!({"text": "duplicate"}), + }, + Step::Delete { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + }, + // Delete then recreate the same key. + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + record: json!({"text": "recreated"}), + }, + // A delete of something absent: both sides must refuse it the same way. + Step::Delete { + collection: COLLECTION, + rkey: "3kmissingaaaa", + }, + Step::Delete { + collection: OTHER_COLLECTION, + rkey: "3kaaaaaaaaaa3", + }, + Step::Create { + collection: OTHER_COLLECTION, + rkey: "3kaaaaaaaaaa4", + record: json!({"text": "after the delete"}), + }, + ] +} + +impl Gate { + async fn post( + &self, + base: &str, + nsid: &str, + headers: Vec<(&str, String)>, + body: &Value, + ) -> Result<(u16, Value)> { + let url = format!("{base}/xrpc/{nsid}"); + let mut request = self.client.post(&url).json(body); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request + .send() + .await + .with_context(|| format!("POST {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + Ok(( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + )) + } + + async fn get( + &self, + base: &str, + nsid: &str, + query: &str, + headers: Vec<(&str, String)>, + ) -> Result<(u16, Value)> { + let url = format!("{base}/xrpc/{nsid}?{query}"); + let mut request = self.client.get(&url); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request.send().await.with_context(|| format!("GET {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + Ok(( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + )) + } + + async fn get_bytes( + &self, + base: &str, + nsid: &str, + query: &str, + headers: Vec<(&str, String)>, + ) -> Result<(u16, Vec)> { + let url = format!("{base}/xrpc/{nsid}?{query}"); + let mut request = self.client.get(&url); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request.send().await.with_context(|| format!("GET {url}"))?; + let status = response.status().as_u16(); + Ok((status, response.bytes().await?.to_vec())) + } + + /// Write auth: the PDS takes its own session token, the space host takes a + /// DPoP-bound access token from the authorization server it trusts. + fn pds_write_headers(&self) -> Vec<(&'static str, String)> { + vec![("authorization", format!("Bearer {}", self.session))] + } + + fn shim_write_headers(&self, nsid: &str) -> Vec<(&'static str, String)> { + let token = tokens::access_token(HS256_SECRET, OAUTH_ISSUER, PDS_SERVICE_DID, AUTHOR_DID); + let proof = tokens::dpop_proof( + "POST", + &format!("{}/xrpc/{nsid}", self.shim_url), + Some(&token), + ); + vec![("authorization", format!("DPoP {token}")), ("dpop", proof)] + } + + /// Read auth: a space credential presented under DPoP on both sides. + fn pds_read_headers(&self, nsid: &str) -> Vec<(&'static str, String)> { + let proof = tokens::dpop_proof( + "GET", + &format!("{}/xrpc/{nsid}", self.pds_url), + Some(&self.pds_credential), + ); + vec![ + ("authorization", format!("DPoP {}", self.pds_credential)), + ("dpop", proof), + ] + } + + fn shim_read_headers(&self, nsid: &str) -> Vec<(&'static str, String)> { + let proof = tokens::dpop_proof( + "GET", + &format!("{}/xrpc/{nsid}", self.shim_url), + Some(&self.shim_credential), + ); + vec![ + ("authorization", format!("DPoP {}", self.shim_credential)), + ("dpop", proof), + ] + } +} + +fn env_path(key: &str, fallback: &str) -> PathBuf { + PathBuf::from(std::env::var(key).unwrap_or_else(|_| fallback.to_string())) +} + +fn multibase_of(hex_key: &str) -> Result { + let signer = rsky_space_host::signing::Signer::from_hex(hex_key) + .map_err(|error| anyhow::anyhow!("signer: {error}"))?; + Ok(signer + .did_key() + .strip_prefix("did:key:") + .unwrap_or(signer.did_key()) + .to_string()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let run_dir = env_path("LAYER2_RUN_DIR", "target/layer2/run"); + let pds_bin = env_path("LAYER2_PDS_BIN", ""); + let shim_bin = env_path("LAYER2_SHIM_BIN", "target/debug/rsky-space-host"); + if !pds_bin.is_file() { + bail!("LAYER2_PDS_BIN must point at the pinned oracle binary (got {pds_bin:?})"); + } + if !shim_bin.is_file() { + bail!("LAYER2_SHIM_BIN must point at the space-host binary (got {shim_bin:?})"); + } + + if run_dir.exists() { + std::fs::remove_dir_all(&run_dir).context("clear run directory")?; + } + std::fs::create_dir_all(&run_dir)?; + let run_dir = run_dir.canonicalize()?; + let pds_dir = run_dir.join("pds"); + let shim_dir = run_dir.join("shim"); + let pds_actors = pds_dir.join("actors"); + let shim_actors = shim_dir.join("actors"); + for dir in [&pds_dir, &shim_dir, &pds_dir.join("blobs")] { + std::fs::create_dir_all(dir)?; + } + + let mut keys = BTreeMap::new(); + keys.insert(DAEMON_DID.to_string(), multibase_of(DAEMON_KEY)?); + let directory = Directory::start(keys, HANDLE.to_string())?; + + let pds_port = free_port()?; + let shim_port = free_port()?; + let pds_url = format!("http://localhost:{pds_port}"); + let shim_url = format!("http://127.0.0.1:{shim_port}"); + + let pds_env: Vec<(String, String)> = vec![ + ("ROCKET_ADDRESS", "127.0.0.1".to_string()), + ("ROCKET_PORT", pds_port.to_string()), + ("PDS_PORT", pds_port.to_string()), + ("PDS_HOSTNAME", "localhost".to_string()), + ("PDS_SERVICE_DID", PDS_SERVICE_DID.to_string()), + ("PDS_SERVICE_HANDLE_DOMAINS", HANDLE_DOMAIN.to_string()), + ("PDS_ADMIN_PASS", ADMIN_PASS.to_string()), + ("PDS_INVITE_REQUIRED", "false".to_string()), + ("PDS_DID_PLC_URL", directory.url()), + ("PDS_JWT_KEY_K256_PRIVATE_KEY_HEX", PDS_JWT_KEY.to_string()), + ( + "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX", + PDS_ROTATION_KEY.to_string(), + ), + ( + "PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX", + PDS_SIGNING_KEY.to_string(), + ), + ( + "PDS_ACCOUNT_DB_LOCATION", + pds_dir.join("account.sqlite").display().to_string(), + ), + ( + "PDS_SEQUENCER_DB_LOCATION", + pds_dir.join("sequencer.sqlite").display().to_string(), + ), + ( + "PDS_DID_CACHE_DB_LOCATION", + pds_dir.join("did_cache.sqlite").display().to_string(), + ), + ( + "PDS_ACTOR_STORE_DIRECTORY", + pds_actors.display().to_string(), + ), + ( + "PDS_BLOBSTORE_DISK_LOCATION", + pds_dir.join("blobs").display().to_string(), + ), + ("RUST_LOG", "warn".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect(); + + let mut pds = Server::spawn( + "oracle pds", + &pds_bin, + &run_dir, + &pds_env, + &run_dir.join("pds.log"), + )?; + pds.wait_ready(&format!("{pds_url}/xrpc/_health"), Duration::from_secs(60)) + .await?; + println!("oracle pds ready on {pds_url}"); + + let client = rsky_spaces_parity::layer2::http_client()?; + let admin = format!("Basic {}", base64_standard(&format!("admin:{ADMIN_PASS}"))); + + // Account, activation, session. + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.server.createAccount"), + vec![("authorization", admin.clone())], + &json!({ + "did": AUTHOR_DID, + "email": "writer@layer2.test", + "handle": HANDLE, + "password": PASSWORD, + }), + ) + .await?; + if status != 200 { + bail!("createAccount failed ({status}): {body}\n{}", pds.tail()); + } + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.admin.updateSubjectStatus"), + vec![("authorization", admin.clone())], + &json!({ + "subject": {"$type": "com.atproto.admin.defs#repoRef", "did": AUTHOR_DID}, + "deactivated": {"applied": false}, + }), + ) + .await?; + if status != 200 { + bail!("activation failed ({status}): {body}\n{}", pds.tail()); + } + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.server.createSession"), + vec![], + &json!({"identifier": HANDLE, "password": PASSWORD}), + ) + .await?; + if status != 200 { + bail!("createSession failed ({status}): {body}\n{}", pds.tail()); + } + let session = body["accessJwt"] + .as_str() + .context("session has no accessJwt")? + .to_string(); + + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.simplespace.createSpace"), + vec![("authorization", format!("Bearer {session}"))], + &json!({"type": SPACE_TYPE, "skey": SPACE_SKEY}), + ) + .await?; + if status != 200 { + bail!("createSpace failed ({status}): {body}\n{}", pds.tail()); + } + let space = body["uri"] + .as_str() + .context("space has no uri")? + .to_string(); + let expected = format!("at://{AUTHOR_DID}/space/{SPACE_TYPE}/{SPACE_SKEY}"); + if space != expected { + bail!("space uri {space} is not the shared {expected}"); + } + println!("space created on the oracle: {space}"); + + // The space host reads account signing keys from a PDS-shaped actor store + // directory and writes its own stores beside them. + copy_tree(&pds_actors, &shim_actors)?; + let accounts = reset_stores(&shim_actors)?; + println!("space host store directory prepared for {accounts} account(s)"); + + let shim_env: Vec<(String, String)> = vec![ + ("SPACEHOST_BIND", format!("127.0.0.1:{shim_port}")), + ("SPACEHOST_PUBLIC_URL", shim_url.clone()), + ("SPACEHOST_AUTHORITY_DID", AUTHOR_DID.to_string()), + ("SPACEHOST_SIGNING_KEY_HEX", AUTHORITY_SPACE_KEY.to_string()), + ("SPACEHOST_POLICY", "public".to_string()), + ("SPACEHOST_PLC_URL", directory.url()), + ( + "SPACEHOST_DB_PATH", + shim_dir.join("space_host.db").display().to_string(), + ), + ( + "SPACEHOST_ACTOR_STORE_DIR", + shim_actors.display().to_string(), + ), + ("SPACEHOST_OAUTH_ISSUER", OAUTH_ISSUER.to_string()), + ("SPACEHOST_OAUTH_JWKS_URI", format!("{OAUTH_ISSUER}/jwks")), + ("SPACEHOST_OAUTH_AUDIENCE", PDS_SERVICE_DID.to_string()), + ("SPACEHOST_OAUTH_CLIENT_IDS", tokens::CLIENT_ID.to_string()), + ("SPACEHOST_OAUTH_HS256_SECRET", HS256_SECRET.to_string()), + ("SPACEHOST_MINT_TOKEN", "layer2-mint-token".to_string()), + ("SPACEHOST_DAEMON_SERVICE_DID", DAEMON_DID.to_string()), + ("SPACEHOST_APPVIEW_SERVICE_DID", DAEMON_DID.to_string()), + ("RUST_LOG", "warn".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect(); + + let mut shim = Server::spawn( + "space host", + &shim_bin, + &run_dir, + &shim_env, + &run_dir.join("shim.log"), + )?; + shim.wait_ready(&format!("{shim_url}/xrpc/_health"), Duration::from_secs(30)) + .await?; + println!("space host ready on {shim_url}"); + + let mut board = Scoreboard::default(); + + // Credentials for the read endpoints, minted by each side's own authority. + let pds_credential = mint_pds_credential(&client, &pds_url, &session, &space).await?; + let shim_credential = mint_shim_credential(&client, &shim_url, &space).await?; + println!("space credentials minted on both sides"); + + let gate = Gate { + client, + pds_url: pds_url.clone(), + shim_url: shim_url.clone(), + space: space.clone(), + session, + pds_credential, + shim_credential, + }; + + run_write_script(&gate, &mut board).await?; + compare_reads(&gate, &mut board).await?; + probe_pds_only_surface(&gate, &mut board).await?; + + // Both servers hold their stores open in WAL mode; stop them before the + // stored-row comparison so the files are quiescent. + shim.stop(); + pds.stop(); + + compare_stores(&pds_actors, &shim_actors, &mut board)?; + + let report = board.render(); + print!("{report}"); + std::fs::write(run_dir.join("report.txt"), &report)?; + println!("\nlogs: {}", run_dir.display()); + + if board.failures() > 0 { + bail!("layer 2 gate failed: {} check(s) differ", board.failures()); + } + Ok(()) +} + +fn base64_standard(text: &str) -> String { + use base64::engine::general_purpose::STANDARD; + use base64::Engine; + STANDARD.encode(text.as_bytes()) +} + +async fn post_raw( + client: &reqwest::Client, + url: &str, + headers: Vec<(&str, String)>, + body: &Value, +) -> Result<(u16, Value)> { + let mut request = client.post(url).json(body); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request + .send() + .await + .with_context(|| format!("POST {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + Ok(( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + )) +} + +/// getDelegationToken then getSpaceCredential, the flow a member's own PDS runs. +async fn mint_pds_credential( + client: &reqwest::Client, + pds_url: &str, + session: &str, + space: &str, +) -> Result { + let nsid = "com.atproto.space.getDelegationToken"; + let url = format!("{pds_url}/xrpc/{nsid}?space={space}"); + let response = client + .get(&url) + .header("authorization", format!("Bearer {session}")) + .send() + .await?; + let status = response.status().as_u16(); + let body: Value = serde_json::from_str(&response.text().await?).unwrap_or(Value::Null); + if status != 200 { + bail!("getDelegationToken failed ({status}): {body}"); + } + let delegation = body["token"] + .as_str() + .context("delegation response has no token")? + .to_string(); + + let nsid = "com.atproto.space.getSpaceCredential"; + let url = format!("{pds_url}/xrpc/{nsid}"); + let response = client + .post(&url) + .header("authorization", format!("Bearer {delegation}")) + .header("dpop", tokens::dpop_proof("POST", &url, None)) + .json(&json!({"space": space})) + .send() + .await?; + let status = response.status().as_u16(); + let body: Value = serde_json::from_str(&response.text().await?).unwrap_or(Value::Null); + if status != 200 { + bail!("getSpaceCredential failed ({status}): {body}"); + } + Ok(body["credential"] + .as_str() + .context("credential response has no credential")? + .to_string()) +} + +/// The space host's administrative mint, which stands in for a syncer asking +/// for a credential. +async fn mint_shim_credential( + client: &reqwest::Client, + shim_url: &str, + space: &str, +) -> Result { + let signer = rsky_space_host::signing::Signer::from_hex(DAEMON_KEY) + .map_err(|error| anyhow::anyhow!("daemon signer: {error}"))?; + let jwt = tokens::service_jwt( + &signer, + DAEMON_DID, + AUTHOR_DID, + "community.blacksky.space.mintCredential", + )?; + let url = format!("{shim_url}/admin/mintCredential"); + let response = client + .post(format!("{url}?space={space}")) + .header("authorization", format!("Bearer {jwt}")) + .header("x-spacehost-mint-token", "layer2-mint-token") + .header("dpop", tokens::dpop_proof("POST", &url, None)) + .send() + .await?; + let status = response.status().as_u16(); + let body: Value = serde_json::from_str(&response.text().await?).unwrap_or(Value::Null); + if status != 200 { + bail!("mintCredential failed ({status}): {body}"); + } + Ok(body["credential"] + .as_str() + .context("mint response has no credential")? + .to_string()) +} + +async fn run_write_script(gate: &Gate, board: &mut Scoreboard) -> Result<()> { + let mut shim_revs = Revs::default(); + let mut pds_revs = Revs::default(); + for (index, step) in script().into_iter().enumerate() { + let nsid = step.nsid(); + let body = step.body(&gate.space, AUTHOR_DID); + let (shim_status, shim_body) = gate + .post(&gate.shim_url, nsid, gate.shim_write_headers(nsid), &body) + .await?; + let (pds_status, pds_body) = gate + .post(&gate.pds_url, nsid, gate.pds_write_headers(), &body) + .await?; + + let name = format!("write {:02} {}", index + 1, step.label()); + let shim_view = write_view(shim_status, &shim_body, &mut shim_revs); + let pds_view = write_view(pds_status, &pds_body, &mut pds_revs); + board.equal_if( + name, + shim_view == pds_view, + if shim_view == pds_view { + shim_view.clone() + } else { + format!("shim: {shim_view}\npds: {pds_view}") + }, + ); + } + board.push( + "write revisions are TIDs", + if normalize::revs_are_tids(&shim_revs) && normalize::revs_are_tids(&pds_revs) { + Verdict::Equal + } else { + Verdict::Differs + }, + format!( + "shim minted {} revisions, pds minted {}", + shim_revs.count(), + pds_revs.count() + ), + ); + Ok(()) +} + +/// A write's comparable outcome: HTTP status, the XRPC error name when it +/// failed, and the record identity when it succeeded. +fn write_view(status: u16, body: &Value, revs: &mut Revs) -> String { + if status != 200 { + return format!( + "{status} {}", + body["error"].as_str().unwrap_or("") + ); + } + let normalized = normalize::normalize(body, revs); + let uri = normalized["uri"].as_str().unwrap_or("-").to_string(); + let cid = normalized["cid"].as_str().unwrap_or("-").to_string(); + let commit = &normalized["commit"]; + format!( + "200 uri={uri} cid={cid} rev={} hash={}", + commit["rev"].as_str().unwrap_or("-"), + commit["hash"].as_str().unwrap_or("-"), + ) +} + +async fn compare_reads(gate: &Gate, board: &mut Scoreboard) -> Result<()> { + let space = gate.space.clone(); + + for (nsid, query) in [ + ( + "com.atproto.space.listRepoOps", + format!("space={space}&repo={AUTHOR_DID}"), + ), + ( + "com.atproto.space.listRepoOps", + format!("space={space}&repo={AUTHOR_DID}&limit=2"), + ), + ( + "com.atproto.space.getLatestCommit", + format!("space={space}&repo={AUTHOR_DID}"), + ), + ("com.atproto.space.listRepos", format!("space={space}")), + ("com.atproto.space.getSpace", format!("space={space}")), + ] { + let (shim_status, shim_body) = gate + .get(&gate.shim_url, nsid, &query, gate.shim_read_headers(nsid)) + .await?; + let (pds_status, pds_body) = gate + .get(&gate.pds_url, nsid, &query, gate.pds_read_headers(nsid)) + .await?; + let mut shim_revs = Revs::default(); + let mut pds_revs = Revs::default(); + let shim_view = normalize::render(&normalize::normalize(&shim_body, &mut shim_revs)); + let pds_view = normalize::render(&normalize::normalize(&pds_body, &mut pds_revs)); + let label = match query.contains("limit=") { + true => format!("read {nsid} (paged)"), + false => format!("read {nsid}"), + }; + let equal = shim_status == pds_status && shim_view == pds_view; + // `getSpace` answers from the space definition, which the storage + // convergence deliberately leaves out of scope: the space host holds it + // in configuration, the PDS in its own `space_def` rows. + if nsid == "com.atproto.space.getSpace" && !equal { + board.push( + label, + Verdict::Documented, + format!( + "space definition is configured on the host and stored on the pds\n\ + shim: {shim_status} {shim_view}\npds: {pds_status} {pds_view}" + ), + ); + continue; + } + board.equal_if( + label, + equal, + if equal { + format!("{shim_status}") + } else { + format!("shim: {shim_status} {shim_view}\npds: {pds_status} {pds_view}") + }, + ); + } + + // getRepo returns a CAR whose commit block carries fresh random key + // material, so the record blocks are what can match. + let nsid = "com.atproto.space.getRepo"; + let query = format!("space={space}&repo={AUTHOR_DID}"); + let (shim_status, shim_car) = gate + .get_bytes(&gate.shim_url, nsid, &query, gate.shim_read_headers(nsid)) + .await?; + let (pds_status, pds_car) = gate + .get_bytes(&gate.pds_url, nsid, &query, gate.pds_read_headers(nsid)) + .await?; + if shim_status != 200 || pds_status != 200 { + board.equal_if( + format!("read {nsid}"), + false, + format!("shim: {shim_status}, pds: {pds_status}"), + ); + return Ok(()); + } + let diff = car::diff(&shim_car, &pds_car)?; + let commit_only = diff.only_shim.len() == 1 && diff.only_pds.len() == 1; + board.equal_if( + format!("read {nsid} record blocks"), + commit_only && diff.shared > 0, + format!( + "{} shared blocks; unique to shim: {:?}; unique to pds: {:?}", + diff.shared, diff.only_shim, diff.only_pds + ), + ); + Ok(()) +} + +/// The four methods the PDS serves and the space host does not. Recorded as a +/// surface difference: a client moving from the host to the PDS gains them. +async fn probe_pds_only_surface(gate: &Gate, board: &mut Scoreboard) -> Result<()> { + let space = gate.space.clone(); + let probes: Vec<(&str, &str, String, Option)> = vec![ + ( + "com.atproto.space.getRecord", + "GET", + format!("space={space}&repo={AUTHOR_DID}&collection={COLLECTION}&rkey=3kaaaaaaaaaa2"), + None, + ), + ( + "com.atproto.space.listRecords", + "GET", + format!("space={space}&repo={AUTHOR_DID}"), + None, + ), + ( + "com.atproto.space.getRepoState", + "GET", + format!("space={space}&repo={AUTHOR_DID}"), + None, + ), + ( + "com.atproto.space.applyWrites", + "POST", + String::new(), + // Deliberately invalid, so the probe cannot write to one side only: + // a routed method rejects it, an unrouted one is not found. + Some(json!({ + "space": space, + "repo": AUTHOR_DID, + "writes": [{ + "action": "not-an-action", + "collection": COLLECTION, + "rkey": "3kprobeaaaaaa", + }], + })), + ), + ]; + for (nsid, method, query, body) in probes { + let (pds_status, shim_status) = if method == "GET" { + let (pds_status, _) = gate + .get(&gate.pds_url, nsid, &query, gate.pds_read_headers(nsid)) + .await?; + let (shim_status, _) = gate + .get(&gate.shim_url, nsid, &query, gate.shim_read_headers(nsid)) + .await?; + (pds_status, shim_status) + } else { + let body = body.unwrap_or(Value::Null); + let (pds_status, _) = gate + .post(&gate.pds_url, nsid, gate.pds_write_headers(), &body) + .await?; + let (shim_status, _) = gate + .post(&gate.shim_url, nsid, gate.shim_write_headers(nsid), &body) + .await?; + (pds_status, shim_status) + }; + board.push( + format!("surface {nsid}"), + Verdict::Note, + format!("pds {pds_status}, space host {shim_status} (not routed)"), + ); + } + Ok(()) +} + +fn compare_stores(pds_actors: &Path, shim_actors: &Path, board: &mut Scoreboard) -> Result<()> { + let pds_store = rsky_space_host::actor_repos::store_path(pds_actors, AUTHOR_DID) + .map_err(|error| anyhow::anyhow!("pds store path: {error}"))?; + let shim_store = rsky_space_host::actor_repos::store_path(shim_actors, AUTHOR_DID) + .map_err(|error| anyhow::anyhow!("shim store path: {error}"))?; + for path in [&pds_store, &shim_store] { + if !path.exists() { + bail!("no store file at {}", path.display()); + } + } + let shim = dump_tables(&shim_store); + let pds = dump_tables(&pds_store); + let equal = compare_tables("store", &shim, &pds); + board.equal_if( + "stored space_* rows", + equal, + format!("{} / {}", shim_store.display(), pds_store.display()), + ); + let sound = + revs_are_well_formed("store", &shim, "shim") && revs_are_well_formed("store", &pds, "pds"); + board.equal_if( + "stored revisions well formed", + sound, + format!( + "shim {} distinct revisions, pds {}", + shim.revs.len(), + pds.revs.len() + ), + ); + Ok(()) +} diff --git a/rsky-spaces-parity/src/layer2/car.rs b/rsky-spaces-parity/src/layer2/car.rs new file mode 100644 index 00000000..438528a9 --- /dev/null +++ b/rsky-spaces-parity/src/layer2/car.rs @@ -0,0 +1,87 @@ +//! A CARv1 block reader, enough to compare what `getRepo` carries. +//! +//! The served commit embeds fresh random key material, so the two sides' commit +//! blocks never match by construction. The record blocks must, so the gate +//! compares the block sets and expects the difference to be exactly the one +//! commit block on each side. + +use anyhow::{bail, Result}; +use std::collections::BTreeSet; + +fn varint(bytes: &[u8], at: &mut usize) -> Result { + let mut value = 0u64; + let mut shift = 0u32; + loop { + let Some(byte) = bytes.get(*at) else { + bail!("truncated varint"); + }; + *at += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + shift += 7; + if shift > 63 { + bail!("varint too long"); + } + } +} + +/// Advance past one binary CIDv1 and return its bytes. +fn cid<'a>(bytes: &'a [u8], at: &mut usize) -> Result<&'a [u8]> { + let start = *at; + let version = varint(bytes, at)?; + if version != 1 { + bail!("expected CIDv1, got version {version}"); + } + let _codec = varint(bytes, at)?; + let _hash = varint(bytes, at)?; + let length = varint(bytes, at)? as usize; + *at += length; + if *at > bytes.len() { + bail!("truncated cid"); + } + Ok(&bytes[start..*at]) +} + +/// The `(cid, data)` pairs a CARv1 payload carries, roots excluded from the +/// return because the header is not what is being compared. +pub fn blocks(car: &[u8]) -> Result, Vec)>> { + let mut at = 0usize; + let header = varint(car, &mut at)? as usize; + at += header; + if at > car.len() { + bail!("truncated car header"); + } + let mut out = Vec::new(); + while at < car.len() { + let length = varint(car, &mut at)? as usize; + let end = at + length; + if end > car.len() { + bail!("truncated car block"); + } + let mut cursor = at; + let key = cid(car, &mut cursor)?.to_vec(); + out.push((key, car[cursor..end].to_vec())); + at = end; + } + Ok(out) +} + +pub struct CarDiff { + pub shared: usize, + pub only_shim: Vec, + pub only_pds: Vec, +} + +/// Compare two CAR payloads by block CID. +pub fn diff(shim: &[u8], pds: &[u8]) -> Result { + let left: BTreeSet> = blocks(shim)?.into_iter().map(|(cid, _)| cid).collect(); + let right: BTreeSet> = blocks(pds)?.into_iter().map(|(cid, _)| cid).collect(); + let hex = |bytes: &Vec| bytes.iter().map(|b| format!("{b:02x}")).collect::(); + Ok(CarDiff { + shared: left.intersection(&right).count(), + only_shim: left.difference(&right).map(hex).collect(), + only_pds: right.difference(&left).map(hex).collect(), + }) +} diff --git a/rsky-spaces-parity/src/layer2/directory.rs b/rsky-spaces-parity/src/layer2/directory.rs new file mode 100644 index 00000000..0f19de5b --- /dev/null +++ b/rsky-spaces-parity/src/layer2/directory.rs @@ -0,0 +1,70 @@ +//! A stand-in DID directory, so neither server reaches the network. +//! +//! It answers every path with a DID document for the requested DID: the +//! `#atproto` signing key the gate controls, and a PDS service endpoint +//! pointing back at itself, which absorbs best-effort write notifications. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::net::TcpListener; + +pub struct Directory { + pub port: u16, +} + +impl Directory { + /// Bind an ephemeral port and serve until the process exits. + pub fn start(keys: BTreeMap, handle: String) -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let mut buf = [0u8; 8192]; + let read = stream.read(&mut buf).unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + let path = request.split_whitespace().nth(1).unwrap_or("/").to_string(); + let did = path + .trim_start_matches('/') + .split('?') + .next() + .unwrap_or_default() + .replace("%3A", ":") + .replace("%3a", ":"); + let multibase = keys + .get(&did) + .or_else(|| keys.values().next()) + .cloned() + .unwrap_or_default(); + let body = serde_json::json!({ + "id": did, + "alsoKnownAs": [format!("at://{handle}")], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multibase, + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": format!("http://127.0.0.1:{port}"), + }], + }) + .to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + Ok(Self { port }) + } + + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } +} diff --git a/rsky-spaces-parity/src/layer2/mod.rs b/rsky-spaces-parity/src/layer2/mod.rs new file mode 100644 index 00000000..cb6fd532 --- /dev/null +++ b/rsky-spaces-parity/src/layer2/mod.rs @@ -0,0 +1,126 @@ +//! Support code for the two-process acceptance gate: token minting, a stub DID +//! directory, child-process supervision, and the response normalizers the gate +//! compares with. + +/// Every request in the gate is to a loopback port, so proxy discovery is both +/// useless and, on macOS, a hard failure inside a restricted sandbox. +pub fn http_client() -> anyhow::Result { + Ok(reqwest::Client::builder() + .no_proxy() + .timeout(std::time::Duration::from_secs(30)) + .build()?) +} + +pub mod car; +pub mod directory; +pub mod normalize; +pub mod process; +pub mod tokens; + +/// One line of the gate's scoreboard. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Check { + pub name: String, + pub verdict: Verdict, + pub detail: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// Both sides agreed. + Equal, + /// The sides disagreed in a way the gate treats as failure. + Differs, + /// The sides disagreed in a way this design accepts and records. + Documented, + /// Recorded for the report, not scored. + Note, +} + +impl Verdict { + pub fn tag(self) -> &'static str { + match self { + Verdict::Equal => "equal", + Verdict::Differs => "DIFFERS", + Verdict::Documented => "documented divergence", + Verdict::Note => "note", + } + } +} + +#[derive(Debug, Default)] +pub struct Scoreboard { + pub checks: Vec, +} + +impl Scoreboard { + pub fn push(&mut self, name: impl Into, verdict: Verdict, detail: impl Into) { + self.checks.push(Check { + name: name.into(), + verdict, + detail: detail.into(), + }); + } + + pub fn equal_if( + &mut self, + name: impl Into, + equal: bool, + detail: impl Into, + ) -> bool { + let verdict = if equal { + Verdict::Equal + } else { + Verdict::Differs + }; + self.push(name, verdict, detail); + equal + } + + pub fn scored(&self) -> usize { + self.checks + .iter() + .filter(|c| matches!(c.verdict, Verdict::Equal | Verdict::Differs)) + .count() + } + + pub fn passed(&self) -> usize { + self.checks + .iter() + .filter(|c| c.verdict == Verdict::Equal) + .count() + } + + pub fn failures(&self) -> usize { + self.checks + .iter() + .filter(|c| c.verdict == Verdict::Differs) + .count() + } + + pub fn documented(&self) -> usize { + self.checks + .iter() + .filter(|c| c.verdict == Verdict::Documented) + .count() + } + + pub fn render(&self) -> String { + let mut out = String::new(); + for check in &self.checks { + out.push_str(&format!(" [{}] {}\n", check.verdict.tag(), check.name)); + if !check.detail.is_empty() { + for line in check.detail.lines() { + out.push_str(&format!(" {line}\n")); + } + } + } + out.push_str(&format!( + "\nparity: {}/{} checks equal (+{} documented divergence)\n", + self.passed(), + self.scored(), + self.documented() + )); + out + } +} diff --git a/rsky-spaces-parity/src/layer2/normalize.rs b/rsky-spaces-parity/src/layer2/normalize.rs new file mode 100644 index 00000000..248f9ebf --- /dev/null +++ b/rsky-spaces-parity/src/layer2/normalize.rs @@ -0,0 +1,80 @@ +//! Response normalization, using the same first-appearance substitution the +//! stored-row comparator uses: each side's distinct revisions map in order to +//! `R1, R2, …`. Fields that cannot match between two independent writers are +//! replaced by a marker rather than dropped, so a field appearing on one side +//! only still shows up as a difference. + +use crate::is_tid; +use serde_json::{Map, Value}; +use std::collections::BTreeMap; + +/// Commit fields derived from fresh random key material on every serve, plus +/// the opaque paging cursor, whose numbering is per-store. +pub const NON_COMPARABLE: [&str; 4] = ["ikm", "sig", "mac", "cursor"]; + +#[derive(Default)] +pub struct Revs { + seen: BTreeMap, + order: Vec, +} + +impl Revs { + pub fn placeholder(&mut self, rev: &str) -> String { + if let Some(existing) = self.seen.get(rev) { + return existing.clone(); + } + let placeholder = format!("R{}", self.order.len() + 1); + self.seen.insert(rev.to_string(), placeholder.clone()); + self.order.push(rev.to_string()); + placeholder + } + + pub fn count(&self) -> usize { + self.order.len() + } + + /// The revisions in first-appearance order, unsubstituted. + pub fn raw(&self) -> &[String] { + &self.order + } +} + +/// Object keys are emitted in sorted order. The two decoders build their JSON +/// maps in different orders from the same DAG-CBOR bytes, and record identity is +/// the CID, which is compared unnormalized in the same response. +pub fn normalize(value: &Value, revs: &mut Revs) -> Value { + match value { + Value::Array(items) => Value::Array(items.iter().map(|v| normalize(v, revs)).collect()), + Value::Object(fields) => { + let mut sorted: Vec<(&String, &Value)> = fields.iter().collect(); + sorted.sort_by(|a, b| a.0.cmp(b.0)); + let mut out = Map::new(); + for (key, field) in sorted { + let replacement = if NON_COMPARABLE.contains(&key.as_str()) { + Value::String("".to_string()) + } else if key == "rev" { + // Only a revision is substituted. Record keys are TIDs too, + // and they must compare literally. + match field.as_str() { + Some(rev) => Value::String(revs.placeholder(rev)), + None => normalize(field, revs), + } + } else { + normalize(field, revs) + }; + out.insert(key.clone(), replacement); + } + Value::Object(out) + } + other => other.clone(), + } +} + +/// Every revision a response exposed is a syntactically valid TID. +pub fn revs_are_tids(revs: &Revs) -> bool { + revs.raw().iter().all(|rev| is_tid(rev)) +} + +pub fn render(value: &Value) -> String { + serde_json::to_string_pretty(value).unwrap_or_else(|_| value.to_string()) +} diff --git a/rsky-spaces-parity/src/layer2/process.rs b/rsky-spaces-parity/src/layer2/process.rs new file mode 100644 index 00000000..e88dc4c7 --- /dev/null +++ b/rsky-spaces-parity/src/layer2/process.rs @@ -0,0 +1,154 @@ +//! Child-process supervision: start a server, wait for it to answer, and make +//! sure it is gone when the gate finishes however the gate finishes. + +use anyhow::{bail, Context, Result}; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +pub struct Server { + pub name: &'static str, + child: Child, + pub log: std::path::PathBuf, +} + +impl Server { + pub fn spawn( + name: &'static str, + binary: &Path, + cwd: &Path, + env: &[(String, String)], + log: &Path, + ) -> Result { + let file = + std::fs::File::create(log).with_context(|| format!("create {}", log.display()))?; + let errors = file.try_clone()?; + let mut command = Command::new(binary); + command + .current_dir(cwd) + .env_clear() + .env("PATH", std::env::var("PATH").unwrap_or_default()) + .env("HOME", std::env::var("HOME").unwrap_or_default()) + .stdin(Stdio::null()) + .stdout(Stdio::from(file)) + .stderr(Stdio::from(errors)); + for (key, value) in env { + command.env(key, value); + } + let child = command + .spawn() + .with_context(|| format!("spawn {}", binary.display()))?; + Ok(Self { + name, + child, + log: log.to_path_buf(), + }) + } + + /// Poll `url` until it answers or the deadline passes; a child that has + /// already exited fails immediately with its log. + pub async fn wait_ready(&mut self, url: &str, timeout: Duration) -> Result<()> { + let client = crate::layer2::http_client()?; + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = self.child.try_wait()? { + bail!( + "{} exited early ({status}); log:\n{}", + self.name, + self.tail() + ); + } + if let Ok(response) = client.get(url).send().await { + if response.status().is_success() { + return Ok(()); + } + } + if Instant::now() >= deadline { + bail!( + "{} did not become ready at {url}; log:\n{}", + self.name, + self.tail() + ); + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + } + + pub fn tail(&self) -> String { + let text = std::fs::read_to_string(&self.log).unwrap_or_default(); + text.lines() + .rev() + .take(30) + .collect::>() + .into_iter() + .rev() + .collect::>() + .join("\n") + } + + pub fn stop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop(); + } +} + +pub fn free_port() -> Result { + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + Ok(listener.local_addr()?.port()) +} + +pub fn copy_tree(from: &Path, to: &Path) -> Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_tree(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), &target)?; + } + } + Ok(()) +} + +/// Replace every copied per-account store with an empty one created by the +/// space host's own schema code, keeping the signing keys beside it. The space +/// host requires the file to be present at startup, so it cannot simply be +/// deleted and left to appear on first write. +pub fn reset_stores(root: &Path) -> Result { + if !root.exists() { + return Ok(0); + } + let mut created = 0; + for entry in std::fs::read_dir(root)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + if entry.file_name().to_string_lossy().starts_with("did:") { + let account = entry.path(); + for stale in std::fs::read_dir(&account)? { + let stale = stale?; + if stale + .file_name() + .to_string_lossy() + .starts_with("store.sqlite") + { + std::fs::remove_file(stale.path())?; + } + } + rsky_space_host::actor_schema::get_migrated_db(account.join("store.sqlite")) + .map_err(|error| anyhow::anyhow!("migrate shim store: {error}"))?; + created += 1; + } else { + created += reset_stores(&entry.path())?; + } + } + Ok(created) +} diff --git a/rsky-spaces-parity/src/layer2/tokens.rs b/rsky-spaces-parity/src/layer2/tokens.rs new file mode 100644 index 00000000..ba9bd5cc --- /dev/null +++ b/rsky-spaces-parity/src/layer2/tokens.rs @@ -0,0 +1,111 @@ +//! The credentials the two servers demand, minted locally. +//! +//! The space host verifies an access token it did not issue, so the gate acts as +//! the authorization server: it holds the shared HS256 secret the host is +//! configured with and signs `at+jwt` tokens with it. DPoP proofs are ES256 over +//! a fixed P-256 key so a proof can be rebuilt for every request. + +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use hmac::{Hmac, Mac}; +use rsky_oauth::jwk::{EcCurve, Jwk}; +use rsky_oauth::jwt::{sign, JwtClaims, JwtHeader}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub const DPOP_KEY_BYTES: [u8; 32] = [0x42u8; 32]; +pub const CLIENT_ID: &str = "https://layer2.invalid/oauth-client-metadata.json"; + +static JTI: AtomicU64 = AtomicU64::new(0); + +fn next_jti(prefix: &str) -> String { + format!("{prefix}-{}", JTI.fetch_add(1, Ordering::SeqCst)) +} + +pub fn now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("clock before unix epoch") + .as_secs() +} + +pub fn dpop_key() -> Jwk { + Jwk::from_private_key_bytes(EcCurve::P256, &DPOP_KEY_BYTES).expect("dpop key") +} + +pub fn jwk_thumbprint_of_dpop_key() -> String { + let public = serde_json::to_value(dpop_key().to_public()).expect("jwk serializes"); + let canonical = format!( + r#"{{"crv":"{}","kty":"{}","x":"{}","y":"{}"}}"#, + public["crv"].as_str().unwrap_or_default(), + public["kty"].as_str().unwrap_or_default(), + public["x"].as_str().unwrap_or_default(), + public["y"].as_str().unwrap_or_default(), + ); + URL_SAFE_NO_PAD.encode(Sha256::digest(canonical.as_bytes())) +} + +/// A DPoP proof bound to `method` and `url`. `url` carries no query string: +/// both servers compare only scheme, host and path. +pub fn dpop_proof(method: &str, url: &str, access_token: Option<&str>) -> String { + let key = dpop_key(); + let mut header = JwtHeader::new("ES256"); + header.typ = Some("dpop+jwt".to_string()); + header.jwk = Some(key.to_public()); + let mut claims = JwtClaims { + iat: Some(now()), + jti: Some(next_jti("proof")), + ..Default::default() + }; + claims.extra.insert("htm".to_string(), method.into()); + claims.extra.insert("htu".to_string(), url.into()); + if let Some(token) = access_token { + claims.extra.insert( + "ath".to_string(), + URL_SAFE_NO_PAD + .encode(Sha256::digest(token.as_bytes())) + .into(), + ); + } + sign(&header, &claims, &key).expect("dpop proof signs") +} + +/// An HS256 `at+jwt` access token of the shape a PDS that is its own +/// authorization server issues: DID audience, no `scope`, DPoP-bound. +pub fn access_token(secret: &str, issuer: &str, audience: &str, subject: &str) -> String { + let header = json!({"typ": "at+jwt", "alg": "HS256"}); + let claims = json!({ + "iss": issuer, + "aud": audience, + "sub": subject, + "iat": now(), + "exp": now() + 3600, + "jti": next_jti("tok"), + "client_id": CLIENT_ID, + "cnf": { "jkt": jwk_thumbprint_of_dpop_key() }, + }); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).expect("header serializes")), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).expect("claims serialize")), + ); + let mut mac = as Mac>::new_from_slice(secret.as_bytes()).expect("hmac key"); + mac.update(input.as_bytes()); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) +} + +/// An inter-service auth JWT, minted with the space host's own issuer code so +/// the gate cannot drift from what the host verifies. +pub fn service_jwt( + signer: &rsky_space_host::signing::Signer, + iss: &str, + aud: &str, + lxm: &str, +) -> anyhow::Result { + rsky_space_host::service_jwt::mint(signer, iss, aud, lxm, now(), next_jti("svc")) + .map_err(|error| anyhow::anyhow!("service jwt: {error}")) +} diff --git a/rsky-spaces-parity/src/lib.rs b/rsky-spaces-parity/src/lib.rs index 2a5666fe..438d478b 100644 --- a/rsky-spaces-parity/src/lib.rs +++ b/rsky-spaces-parity/src/lib.rs @@ -1,3 +1,5 @@ +pub mod layer2; + use rsky_pds::actor_store::space::{SpaceStore, SpaceStoreError}; use rsky_space_host::error::HostError; use rsky_space_host::repo::RepoStore; From 445299f575b59a5a303e309fc9e556762583a041 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 16:47:14 -0400 Subject: [PATCH 41/56] docs(spaces-parity): layer 2 run report 18/18 checks equal (+1 documented divergence) with both real binaries running. Records the surface difference a client gains when it moves from the space host to rsky-pds, the three divergences the gate found, and the falsification evidence for each of its three comparison layers. --- .../layer2/report/2026-08-21.md | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 rsky-spaces-parity/layer2/report/2026-08-21.md diff --git a/rsky-spaces-parity/layer2/report/2026-08-21.md b/rsky-spaces-parity/layer2/report/2026-08-21.md new file mode 100644 index 00000000..b8b179fa --- /dev/null +++ b/rsky-spaces-parity/layer2/report/2026-08-21.md @@ -0,0 +1,165 @@ +# Layer 2 acceptance gate — run report + +**Date:** 2026-08-21 (America/New_York) +**Branch:** `feat/space-host-main-port` +**Oracle:** `rsky-pds` built from a detached, build-only worktree at +`7ebd21ae788c550ee8510034d94eb19ede148738` +**Under test:** `rsky-space-host` from the working tree +**Command:** `./rsky-spaces-parity/layer2/run.sh` (exit 0) + +## Result + +``` +parity: 18/18 checks equal (+1 documented divergence) +``` + +Both real binaries ran as separate processes. The same ten-step record script +was fired at each over XRPC, five shared read endpoints were compared, and the +Layer 1 stored-row comparator was pointed at the two on-disk `store.sqlite` +files. Every scored check was equal, including every record CID and every +LtHash digest at every step. + +## Surface difference: four methods the PDS routes and the space host does not + +This is the headline finding for anyone moving a client from the space host to +rsky-pds. The two write surfaces overlap; the read surfaces do not. + +| Method | rsky-pds | rsky-space-host | +|---|---|---| +| `com.atproto.space.createRecord` | routed | routed | +| `com.atproto.space.deleteRecord` | routed | routed | +| `com.atproto.space.getRepo` | routed | routed | +| `com.atproto.space.listRepoOps` | routed | routed | +| `com.atproto.space.getLatestCommit` | routed | routed | +| `com.atproto.space.listRepos` | routed | routed | +| `com.atproto.space.getSpace` | routed | routed | +| `com.atproto.space.applyWrites` | routed | **404** | +| `com.atproto.space.getRecord` | routed | **404** | +| `com.atproto.space.listRecords` | routed | **404** | +| `com.atproto.space.getRepoState` | routed | **404** | + +A client migrating from the space host to rsky-pds **gains** batch writes +(`applyWrites`) and direct record reads (`getRecord`, `listRecords`, +`getRepoState`); nothing is lost. Because the space host has no batch endpoint, +`applyWrites` cannot be part of a same-script comparison at all: a batch under +one revision on the PDS and N single writes on the host produce different +revision grouping and different oplog shapes by construction. It is therefore +probed PDS-only, with a deliberately invalid write so the probe cannot mutate +one side and not the other. + +## Divergences the gate found and this run fixed + +All three were invisible to Layer 1, which compares internal outcome enums and +stored rows rather than HTTP responses. All three were in `rsky-space-host` +(`rsky-pds` and `rsky-space` were not touched). + +1. **XRPC error names were collapsed.** The space host mapped + `RecordExists` and `RecordNotFound` onto `InvalidRequest`, and answered + `InvalidSwap` with 409 and `HistoryUnavailable` with 410 where the oracle + answers 400 with the specific name. A client could not distinguish "already + exists" from a malformed request. Now all four match the oracle's name and + status. +2. **`listRepoOps` attached the wrong value to a superseded operation.** The + host hydrated an op's `value` by `(collection, rkey)`, so an op whose record + had since been deleted and recreated carried the *current* record's value + beside its own historical CID. The oracle joins on the CID as well and omits + the value. Now the host does the same. +3. **The writer set omitted the repo digest.** `listRepos` returned `did` and + `rev` but no `hash`, because the local write path passed `None` where the + oracle passes the commit hash. Now it passes the digest; the notify-write + path still passes `None`, which is correct — a notification carries no hash. + +## Documented divergence + +`getSpace` answers from the space *definition*, which the storage convergence +leaves out of scope: the space host holds it in configuration, the PDS in its +own `space_def` rows (a table the stored-row comparator deliberately does not +read). In this run the host reported `policy: public` from its flags and the +PDS `policy: member-list` from its stored definition. + +## Falsification evidence + +The gate was observed failing before it was observed passing, on all three of +its comparison layers, which is what makes the green meaningful: + +- **Write responses** caught divergence 1 (`400 InvalidRequest` against + `400 RecordExists`). +- **Read responses** caught divergences 2 and 3. +- **Stored rows** caught an eleventh record written to the PDS only, when the + `applyWrites` probe still used a valid batch: `space_repo`, `space_record` + and `space_oplog` all reported differing rows and the run exited non-zero. + +## Scoreboard + +``` + [equal] write 01 createRecord com.example.post/3kaaaaaaaaaa1 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/3kaaaaaaaaaa1 cid=bafyreigiltmq54gboouigc54umca576hqldm2pn2qrh5goxafd6b3pa554 rev=R1 hash=54b495b7015c9acc2de6618cdc7451301d669189484db8a1add74cd8324d237b + [equal] write 02 createRecord com.example.post/3kaaaaaaaaaa2 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/3kaaaaaaaaaa2 cid=bafyreibrc4gkm3q4ovjdmcgi74sdptx34x2hrwwypvdkzk6jrb42ie3uc4 rev=R2 hash=18d0e789470666518aa2a70594180c2b8e28f47d241125a698940b734f1f973c + [equal] write 03 createRecord com.example.note/3kaaaaaaaaaa3 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.note/3kaaaaaaaaaa3 cid=bafyreih234nznbwegwspg7ezp4gqci246dg7kh6crwmxe7oogynryyf7n4 rev=R3 hash=d2a61fab058f3daed04ea9c36c70277d2401eee5680c9474a664eacc7050311b + [equal] write 04 createRecord com.example.post/unicode.rkey_1~ + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/unicode.rkey_1~ cid=bafyreigewh3bnk7sg2ufwm7ldx54ehr5fobiqw46q3hunv2mv4eeiue5vy rev=R4 hash=28bad2f99440f79e0afa6993f1dddcb3c35db0e4103a24b0b836d769f32f7c13 + [equal] write 05 createRecord com.example.post/3kaaaaaaaaaa1 + 400 RecordExists + [equal] write 06 deleteRecord com.example.post/3kaaaaaaaaaa1 + 200 uri=- cid=- rev=R5 hash=41facf48277e70f7044005777ce2db525e96b070804bb740e6e97f589ab5e3ad + [equal] write 07 createRecord com.example.post/3kaaaaaaaaaa1 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/3kaaaaaaaaaa1 cid=bafyreigjz7i56srnmyj7uyevkz26fqzbohsdvyu47l2oszxghyyxck5lf4 rev=R6 hash=e07880fdc27a57e83e361086e37a4d2cc69cc6e30202bda170b67894843661e3 + [equal] write 08 deleteRecord com.example.post/3kmissingaaaa + 400 RecordNotFound + [equal] write 09 deleteRecord com.example.note/3kaaaaaaaaaa3 + 200 uri=- cid=- rev=R7 hash=dfc8cb119ab18e2705cbb6f27fd0c49c0e2a9875a128670d56e41044d6d04ad6 + [equal] write 10 createRecord com.example.note/3kaaaaaaaaaa4 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.note/3kaaaaaaaaaa4 cid=bafyreiahahltekodrn4hj5vnsow6zb26kdjm5zrvsq57av45ipgqfyogx4 rev=R8 hash=bc2254cc03faccc88f91ff0b3d2b9d073d241d1db16e871cc61ebbdd45215e5b + [equal] write revisions are TIDs + shim minted 8 revisions, pds minted 8 + [equal] read com.atproto.space.listRepoOps + 200 + [equal] read com.atproto.space.listRepoOps (paged) + 200 + [equal] read com.atproto.space.getLatestCommit + 200 + [equal] read com.atproto.space.listRepos + 200 + [documented divergence] read com.atproto.space.getSpace + space definition is configured on the host and stored on the pds + shim: 200 { + "config": { + "$type": "com.atproto.simplespace.defs#config", + "appAccess": { + "$type": "com.atproto.simplespace.defs#appAccessOpen" + }, + "policy": "public" + }, + "space": "at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main" + } + pds: 200 { + "config": { + "$type": "com.atproto.simplespace.defs#config", + "appAccess": { + "$type": "com.atproto.simplespace.defs#appAccessOpen" + }, + "policy": "member-list" + }, + "space": "at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main" + } + [equal] read com.atproto.space.getRepo record blocks + 5 shared blocks; unique to shim: ["0171122058e7325f960239995ce4f6efeaeb507bdf201fbe377004bc64fbf9b90e01ec09"]; unique to pds: ["01711220f4ff946d631dfdc1f2ea0026620ddc853bc6ab254c157b9cc598bf5181f3ac16"] + [note] surface com.atproto.space.getRecord + pds 200, space host 404 (not routed) + [note] surface com.atproto.space.listRecords + pds 200, space host 404 (not routed) + [note] surface com.atproto.space.getRepoState + pds 200, space host 404 (not routed) + [note] surface com.atproto.space.applyWrites + pds 400, space host 404 (not routed) + [equal] stored space_* rows + /target/layer2/run/shim/actors/c6/did:plc:layer2writeraaaaaaaaaaa/store.sqlite / /target/layer2/run/pds/actors/c6/did:plc:layer2writeraaaaaaaaaaa/store.sqlite + [equal] stored revisions well formed + shim 8 distinct revisions, pds 8 + +parity: 18/18 checks equal (+1 documented divergence) + +logs: /target/layer2/run +``` From f2b6ed082727629731ccfa2a3eb9f7247bac8ce4 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Fri, 21 Aug 2026 17:56:57 -0400 Subject: [PATCH 42/56] test(spaces-parity): daemon resume-across-swap gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The converged space host deploys over data the legacy host wrote. This runs that upgrade with real processes — legacy host, real daemon, converter, converged host, same daemon — and asserts the daemon resumes on its existing cursors: exactly the operations after the cursor, once each, with no cursor refusal, divergence or full-state recovery. A cold daemon on the converted store is compared against it. --- Cargo.lock | 3 +- rsky-spaces-parity/Cargo.toml | 7 +- rsky-spaces-parity/resume-gate/README.md | 103 +++ .../resume-gate/report/2026-08-21.md | 106 +++ rsky-spaces-parity/resume-gate/run.sh | 58 ++ rsky-spaces-parity/src/bin/resume_gate.rs | 652 ++++++++++++++++++ rsky-spaces-parity/src/layer2/process.rs | 29 + rsky-spaces-parity/src/lib.rs | 1 + rsky-spaces-parity/src/resume/mod.rs | 199 ++++++ rsky-spaces-parity/src/resume/sink.rs | 151 ++++ 10 files changed, 1307 insertions(+), 2 deletions(-) create mode 100644 rsky-spaces-parity/resume-gate/README.md create mode 100644 rsky-spaces-parity/resume-gate/report/2026-08-21.md create mode 100755 rsky-spaces-parity/resume-gate/run.sh create mode 100644 rsky-spaces-parity/src/bin/resume_gate.rs create mode 100644 rsky-spaces-parity/src/resume/mod.rs create mode 100644 rsky-spaces-parity/src/resume/sink.rs diff --git a/Cargo.lock b/Cargo.lock index 3991ebb5..5fa82ad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8716,10 +8716,11 @@ dependencies = [ [[package]] name = "rsky-spaces-parity" -version = "0.2.0" +version = "0.3.0" dependencies = [ "anyhow", "base64 0.22.1", + "hex", "hmac", "reqwest 0.12.23", "rsky-crypto 0.2.0", diff --git a/rsky-spaces-parity/Cargo.toml b/rsky-spaces-parity/Cargo.toml index dd802888..c25c206b 100644 --- a/rsky-spaces-parity/Cargo.toml +++ b/rsky-spaces-parity/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-spaces-parity" -version = "0.2.0" +version = "0.3.0" edition = "2021" publish = false @@ -21,6 +21,7 @@ hmac = "0.12" sha2 = { workspace = true } secp256k1 = { workspace = true } serde = { workspace = true } +hex = "0.4" [dev-dependencies] tempfile = "3" @@ -28,3 +29,7 @@ tempfile = "3" [[bin]] name = "layer2-gate" path = "src/bin/layer2_gate.rs" + +[[bin]] +name = "resume-gate" +path = "src/bin/resume_gate.rs" diff --git a/rsky-spaces-parity/resume-gate/README.md b/rsky-spaces-parity/resume-gate/README.md new file mode 100644 index 00000000..eeab1b79 --- /dev/null +++ b/rsky-spaces-parity/resume-gate/README.md @@ -0,0 +1,103 @@ +# Resume-across-swap gate + +Layer 1 and Layer 2 compare the converged space host against the rsky-pds +oracle. This gate asks a different question: when the converged host deploys +over data the legacy host wrote, does the **existing** syncer keep going? + +```sh +./rsky-spaces-parity/resume-gate/run.sh +``` + +Exit code 0 means every scored check passed. No Postgres, no Docker, no +network: three child processes at a time plus two loopback stubs the gate hosts +itself. + +## What it does + +1. Creates a **detached, build-only** git worktree at `6da61ce`, the + pre-convergence tip of `feat/space-host-main-port`, and builds + `rsky-space-host` there. The worktree is refused if it is dirty or at + another revision. +2. Builds the converged `rsky-space-host`, `convert_store` and `rsky-daemon` + from the working tree. **The daemon is not modified for this gate** — that + is the point of it. +3. Runs `resume-gate`, which drives three eras against one space-host database, + one actor-store directory, one host port and one daemon index: + + **Phase A — legacy era.** The legacy host serves the multi-tenant + `space_host.db`. Two posts are created over XRPC. The real daemon runs + against it, projects both to a capturing sink, and persists its cursor. + The daemon is then killed and three more writes land — two creates and a + delete — none of which it ever acknowledged. + + **Phase B — the swap.** `convert_store` converts `space_host.db` into + per-account `store.sqlite` files. The account signing key is placed beside + the new store, as the deploy leaves it. + + **Phase C — converged era.** The converged host starts on the same port, + the same host database and the converted stores. The same daemon restarts + with the same index, the same DPoP key and identical arguments. One further + write lands at a server-minted revision. + + Finally a **cold daemon** — fresh index, fresh key, its own sink — syncs the + converted store from scratch. + +Everything lands under `target/resume/run`: a log per process +(`shim-legacy`, `shim-converged`, `daemon-legacy`, `daemon-converged`, +`daemon-cold`), the host and daemon databases, and `report.txt`. The directory +is wiped at the start of every run. + +## What it asserts + +- The legacy-era daemon projects what it saw, once each, and persists a cursor. +- Writes landing after its last acknowledgement leave that cursor behind. +- The conversion carries oplog ids and revisions across verbatim, and leaves + `oplog_floor_rev` open so no `since` can be refused. +- On resume the daemon projects **exactly** the operations after its cursor, + once each — including the three it never saw before the swap — and re-projects + nothing. +- A revision minted by the converged host sorts after the carried legacy one, + and its write projects once. +- No `HistoryUnavailable`, no divergence, no full-state recovery, no `prev` + mismatch in either daemon log. This is the sharpest check: a broken cursor is + survivable by falling back to `getRepo`, which would hide the fault behind a + correct-looking end state. +- A cold daemon reaches the same records, revisions and LtHash digest, and the + same projected end state. + +## Credentials + +All local, all fixed, all created and destroyed inside the run directory; none +of it is a secret and none of it reaches a real service. + +- The gate acts as the authorization server for writes, holding the same HS256 + secret the host is configured with and signing DPoP-bound `at+jwt` tokens — + the same shape as Layer 2. +- The daemon mints its own space credential through `/admin/mintCredential` + with its own service identity and DPoP key, so the credential path is the + real one and is exercised twice, once per era. +- The account's actor-store key is written by the gate; the stub DID directory + publishes the matching `#atproto` multikey so the daemon verifies real + commit signatures. + +## Falsification + +Both eras green on an unmutated tree proves nothing on its own, so the gate was +run against two deliberate converter faults: + +| Mutation | Result | +|---|---| +| `oplog_floor_rev` set to the head revision instead of left open | 10/14 — the daemon took `HistoryUnavailable` → full-state recovery; the log check, the store check and both cold-sync checks went red | +| oplog row ids renumbered (`seq + 1000`) | 13/14 — only the conversion check went red | + +The second is a finding, not a gap: the daemon's durable cursor is a +**revision**, not an oplog row id. Row ids are only within-request paging +cursors. Preserving them is still correct — any other consumer may hold one — +but revision continuity is what the resume depends on. + +## Running outside a sandbox + +Same as Layer 2: on macOS `reqwest` proxy discovery calls SystemConfiguration, +which a restricted sandbox denies and the pinned `hyper-util` panics on. Run +with ordinary process permissions. `sccache` is likewise unusable there, so +`run.sh` clears `RUSTC_WRAPPER`. diff --git a/rsky-spaces-parity/resume-gate/report/2026-08-21.md b/rsky-spaces-parity/resume-gate/report/2026-08-21.md new file mode 100644 index 00000000..73c23203 --- /dev/null +++ b/rsky-spaces-parity/resume-gate/report/2026-08-21.md @@ -0,0 +1,106 @@ +# Resume-across-swap gate — run report, 2026-08-21 + +**Verdict: pass.** `parity: 14/14 checks equal (+0 documented divergence)`, +exit 0. The rsky-daemon binary was not modified; the gate verifies it as-is. + +``` +./rsky-spaces-parity/resume-gate/run.sh +``` + +Legacy era built from a detached build-only worktree at `6da61ce`, the +pre-convergence tip of `feat/space-host-main-port`. Converged host, converter +and daemon built from the working tree. + +## Scoreboard + +``` + [equal] legacy era: the daemon projects the writes it saw, once each + [equal] legacy era: the cursor is durable + [equal] legacy era: writes after the last ack leave the cursor behind + [equal] conversion: oplog ids and revisions survive verbatim + [equal] conversion: no history is placed out of reach + [equal] resume: exactly the operations after the cursor, once each + [equal] resume: nothing already projected is projected again + [equal] converged era: a server-minted revision follows the carried one + [equal] converged era: the new write projects once + [equal] resume: no cursor refusal, divergence or full-state recovery + [equal] legacy-era daemon log was also clean + [equal] cold sync: the same records, revisions and digest as the resumed daemon + [equal] cold sync: the same projected end state as the resumed daemon + [equal] whole run: no operation is projected twice to either destination + [note] oplog window was never the constraint + +parity: 14/14 checks equal (+0 documented divergence) +``` + +## What actually happened + +| Era | Event | Revision | +|---|---|---| +| legacy | create `post1`, create `post2` | `3mtmphzq…`, head `3mtmphzqzcnd6` | +| legacy | daemon syncs, projects both, persists cursor `3mtmphzqzcnd6` | | +| legacy | daemon killed | | +| legacy | create `post3`, create `post4`, delete `post2` | head `3mtmpi44g7nd6` | +| swap | `convert_store`: 1 account, 1 repo, 3 records, 5 ops; ids `[1,2,3,4,5]`, floor `NULL` | | +| converged | same daemon restarts on the same index; projects `post2:delete`, `post3:create`, `post4:create` and nothing else | | +| converged | create `post5` | `3mtmpi6ssuanm` | +| converged | cold daemon syncs from scratch to the same head and digest | | + +The three risk edges named in the design, as the gate found them: + +- **Oplog cursor numbering.** Preserved verbatim (`[1,2,3,4,5]`), confirmed from + the consumer side. But see the finding below: the daemon does not use these. +- **Server-minted revisions replacing caller-supplied ones.** Both eras mint + TIDs (legacy from a `Ticker`, converged from `TID::next_str(prev)`), and the + converged host seeds its first revision from the carried legacy head, so + `3mtmpi6ssuanm > 3mtmpi44g7nd6`. Every consumer comparison that depends on + revision ordering — `listRepoOps`'s `rev > since`, the daemon's + `projector_cursor`'s `j.rev > c.rev` — therefore holds across the swap. +- **`list_ops` refusing history beyond the oplog window.** The converter leaves + `oplog_floor_rev` open, so no `since` can be refused on a freshly converted + store. Not exercised under a full window: `DEFAULT_OPLOG_WINDOW` is a + compiled-in 10 000 rows with no configuration hook, so a small-window run is + not reachable through the binary. The gate asserts the floor invariant + instead and records the gap as a note. + +## Falsification + +Green on an unmutated tree is not evidence, so the gate was run against two +deliberate converter faults and reverted after each. + +| Mutation | Result | Which checks caught it | +|---|---|---| +| `oplog_floor_rev` set to the head revision instead of left open | 10/14, exit 1 | the conversion floor check; `resume: no cursor refusal…` (both daemons logged `history unavailable` → full-state recovery); both cold-sync checks | +| oplog row ids renumbered (`seq + 1000`) | 13/14, exit 1 | the conversion check only | + +## Findings + +**1. The daemon's durable cursor is a revision, not an oplog row id.** The +design note (`Design/spaces-storage-parity`, Blast radius) frames the deploy +risk as "oplog cursor numbering … the ids syncers hold". It is not: the daemon +persists `sync_state.rev` and passes it as `since`; oplog row ids appear only as +within-request paging cursors, discarded when a sync finishes. Renumbering every +id changed nothing about the resume. Preserving ids is still right — another +consumer may hold one, and the converter's guarantee is cheap — but the +property the deploy depends on is **revision continuity and ordering**, and that +is what should be stated in the cutover plan. + +**2. Full-state recovery masks a broken cursor.** Under the floor mutation the +warm daemon took `HistoryUnavailable`, fell back to `getRepo`, and still +projected exactly the three expected operations with no duplicates — the +end-state checks alone would have passed it. What exposed it was the log check +and the cold daemon, which lost `post2`'s deletion entirely (recovery +reconstructs current state, so a delete that happened before a cold start is +never an event). Any future resume check must keep asserting on the absence of +recovery, not only on the projected result. + +**3. No daemon change was needed.** Nothing in the swap required touching +`rsky-daemon`; the stop condition was never reached. + +## Environment + +macOS, sandbox off (`reqwest` proxy discovery calls SystemConfiguration, which a +restricted sandbox denies and the pinned `hyper-util` panics on). +`RUSTC_WRAPPER` cleared by `run.sh` because `sccache` cannot open its cache +there. Zero diff in `rsky-pds/`, `rsky-space/` and `rsky-daemon/`. Local only — +nothing pushed, no PR. diff --git a/rsky-spaces-parity/resume-gate/run.sh b/rsky-spaces-parity/resume-gate/run.sh new file mode 100755 index 00000000..53cbed26 --- /dev/null +++ b/rsky-spaces-parity/resume-gate/run.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Resume-across-swap gate: build the pre-convergence space host from a detached +# build-only worktree, run it with the real daemon, convert the store, then +# restart both the converged host and the same daemon and check what it +# projected. One command, no services beyond the child processes. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +# The legacy era is the pre-convergence tip of this branch and nothing else. +# Its worktree is build-only: no edit ever lands there. +LEGACY_REV="${LEGACY_REV:-6da61ce}" +LEGACY_TREE="${LEGACY_TREE:-/tmp/claude/rsky-resume-legacy}" +RUN_DIR="${RESUME_RUN_DIR:-$REPO_ROOT/target/resume/run}" + +# sccache cannot open its cache in a sandboxed shell; the wrapper is only a +# build accelerator, so drop it rather than fail. +export RUSTC_WRAPPER="" +export CARGO_BUILD_RUSTC_WRAPPER="" + +say() { printf '\n== %s\n' "$1"; } + +LEGACY_SHA="$(git rev-parse "$LEGACY_REV")" + +say "legacy worktree at $LEGACY_SHA" +if [ ! -d "$LEGACY_TREE/.git" ] && [ ! -f "$LEGACY_TREE/.git" ]; then + mkdir -p "$(dirname "$LEGACY_TREE")" + git worktree add --detach "$LEGACY_TREE" "$LEGACY_SHA" +fi +HAVE="$(git -C "$LEGACY_TREE" rev-parse HEAD)" +if [ "$HAVE" != "$LEGACY_SHA" ]; then + echo "legacy worktree is at $HAVE, expected $LEGACY_SHA" >&2 + exit 1 +fi +if [ -n "$(git -C "$LEGACY_TREE" status --porcelain)" ]; then + echo "legacy worktree is dirty; it must stay build-only" >&2 + git -C "$LEGACY_TREE" status --short >&2 + exit 1 +fi + +say "building the legacy space host" +( cd "$LEGACY_TREE" && cargo build -p rsky-space-host --bin rsky-space-host ) + +say "building the converged space host, the converter and the daemon" +cargo build -p rsky-space-host --bin rsky-space-host --bin convert_store +cargo build -p rsky-daemon --bin rsky-daemon + +say "building the gate" +cargo build -p rsky-spaces-parity --bin resume-gate + +say "running the gate" +RESUME_RUN_DIR="$RUN_DIR" \ +RESUME_LEGACY_SHIM_BIN="$LEGACY_TREE/target/debug/rsky-space-host" \ +RESUME_SHIM_BIN="$REPO_ROOT/target/debug/rsky-space-host" \ +RESUME_CONVERT_BIN="$REPO_ROOT/target/debug/convert_store" \ +RESUME_DAEMON_BIN="$REPO_ROOT/target/debug/rsky-daemon" \ + ./target/debug/resume-gate diff --git a/rsky-spaces-parity/src/bin/resume_gate.rs b/rsky-spaces-parity/src/bin/resume_gate.rs new file mode 100644 index 00000000..434dcb30 --- /dev/null +++ b/rsky-spaces-parity/src/bin/resume_gate.rs @@ -0,0 +1,652 @@ +//! Resume-across-swap gate: the upgrade the storage convergence ships, +//! simulated end to end with real processes. +//! +//! Three eras run against one space-host database and one daemon index: +//! the legacy multi-tenant store, the one-off conversion, and the converged +//! per-account stores. The daemon keeps its durable cursors across all three, +//! and the gate asserts what it projected on the far side. +//! +//! Run it through `rsky-spaces-parity/resume-gate/run.sh`, which builds every +//! binary involved and passes their paths in. + +use anyhow::{bail, Context, Result}; +use rsky_spaces_parity::layer2::process::{free_port, Server}; +use rsky_spaces_parity::layer2::{directory::Directory, tokens, Scoreboard, Verdict}; +use rsky_spaces_parity::resume::sink::Sink; +use rsky_spaces_parity::resume::{ + converged_head, converged_oplog, duplicates, final_state, labels, legacy_oplog, read_index, + unclean_resume_lines, +}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +const AUTHOR_DID: &str = "did:plc:resumewriteraaaaaaaaaa"; +const DAEMON_DID: &str = "did:plc:resumedaemonaaaaaaaaaa"; +const HANDLE: &str = "writer.resume.test"; +const SPACE_TYPE: &str = "community.blacksky.feed"; +const SPACE_SKEY: &str = "main"; +const COLLECTION: &str = "app.bsky.feed.post"; +const HS256_SECRET: &str = "resume-local-authorization-server-secret"; +const OAUTH_ISSUER: &str = "http://localhost:0/oauth"; +const OAUTH_AUDIENCE: &str = "did:web:localho.st"; +const MINT_TOKEN: &str = "resume-mint-token"; + +/// Fixed local key material. None of it protects anything: the whole stack is +/// created and destroyed inside one run directory. +const AUTHOR_KEY: &str = "3333333333333333333333333333333333333333333333333333333333333333"; +const DAEMON_KEY: &str = "2222222222222222222222222222222222222222222222222222222222222222"; + +/// The record keys written, in the order the script writes them. +const POST_1: &str = "3kresumepost1"; +const POST_2: &str = "3kresumepost2"; +const POST_3: &str = "3kresumepost3"; +const POST_4: &str = "3kresumepost4"; +const POST_5: &str = "3kresumepost5"; + +const READY: Duration = Duration::from_secs(30); +const SETTLE: Duration = Duration::from_secs(45); + +struct Writer { + client: reqwest::Client, + shim_url: String, + space: String, +} + +impl Writer { + /// Create a record as the author, over XRPC, exactly as a client does. + async fn create(&self, rkey: &str, text: &str) -> Result { + let body = json!({ + "space": self.space, + "repo": AUTHOR_DID, + "collection": COLLECTION, + "rkey": rkey, + "record": { + "$type": COLLECTION, + "text": text, + "createdAt": "2026-08-21T00:00:00.000Z", + }, + }); + self.write("com.atproto.space.createRecord", &body).await + } + + async fn delete(&self, rkey: &str) -> Result { + let body = json!({ + "space": self.space, + "repo": AUTHOR_DID, + "collection": COLLECTION, + "rkey": rkey, + }); + self.write("com.atproto.space.deleteRecord", &body).await + } + + /// Returns the revision the host committed the write at. + async fn write(&self, nsid: &str, body: &Value) -> Result { + let url = format!("{}/xrpc/{nsid}", self.shim_url); + let token = tokens::access_token(HS256_SECRET, OAUTH_ISSUER, OAUTH_AUDIENCE, AUTHOR_DID); + let proof = tokens::dpop_proof("POST", &url, Some(&token)); + let response = self + .client + .post(&url) + .header("authorization", format!("DPoP {token}")) + .header("dpop", proof) + .json(body) + .send() + .await + .with_context(|| format!("POST {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + let parsed: Value = serde_json::from_str(&text).unwrap_or(Value::String(text.clone())); + if status != 200 { + bail!("{nsid} returned {status}: {parsed}"); + } + Ok(parsed["commit"]["rev"] + .as_str() + .with_context(|| format!("{nsid} response has no commit revision: {parsed}"))? + .to_string()) + } +} + +fn env_path(key: &str, fallback: &str) -> PathBuf { + PathBuf::from(std::env::var(key).unwrap_or_else(|_| fallback.to_string())) +} + +fn multibase_of(hex_key: &str) -> Result { + let signer = rsky_space_host::signing::Signer::from_hex(hex_key) + .map_err(|error| anyhow::anyhow!("signer: {error}"))?; + Ok(signer + .did_key() + .strip_prefix("did:key:") + .unwrap_or(signer.did_key()) + .to_string()) +} + +/// The PDS actor-store layout both eras of the space host read signing keys +/// from: `{root}/{sha256(did)[..2]}/{did}/{key,store.sqlite}`. +fn seed_actor_store(root: &Path, did: &str, key_hex: &str) -> Result { + let digest = hex::encode(Sha256::digest(did.as_bytes())); + let account = root.join(&digest[..2]).join(did); + std::fs::create_dir_all(&account)?; + let key = hex::decode(key_hex).context("actor key is not hex")?; + std::fs::write(account.join("key"), key)?; + let store = account.join("store.sqlite"); + rsky_space_host::actor_schema::get_migrated_db(&store) + .map_err(|error| anyhow::anyhow!("migrate empty store: {error}"))?; + Ok(store) +} + +fn shim_env( + port: u16, + public_url: &str, + db_path: &Path, + actors: &Path, + plc_url: &str, +) -> Vec<(String, String)> { + vec![ + ("SPACEHOST_BIND", format!("127.0.0.1:{port}")), + ("SPACEHOST_PUBLIC_URL", public_url.to_string()), + ("SPACEHOST_AUTHORITY_DID", AUTHOR_DID.to_string()), + ("SPACEHOST_SIGNING_KEY_HEX", AUTHOR_KEY.to_string()), + ("SPACEHOST_POLICY", "public".to_string()), + ("SPACEHOST_PLC_URL", plc_url.to_string()), + ("SPACEHOST_DB_PATH", db_path.display().to_string()), + ("SPACEHOST_ACTOR_STORE_DIR", actors.display().to_string()), + ("SPACEHOST_OAUTH_ISSUER", OAUTH_ISSUER.to_string()), + ("SPACEHOST_OAUTH_JWKS_URI", format!("{OAUTH_ISSUER}/jwks")), + ("SPACEHOST_OAUTH_AUDIENCE", OAUTH_AUDIENCE.to_string()), + ("SPACEHOST_OAUTH_CLIENT_IDS", tokens::CLIENT_ID.to_string()), + ("SPACEHOST_OAUTH_HS256_SECRET", HS256_SECRET.to_string()), + ("SPACEHOST_MINT_TOKEN", MINT_TOKEN.to_string()), + ("SPACEHOST_DAEMON_SERVICE_DID", DAEMON_DID.to_string()), + ("SPACEHOST_APPVIEW_SERVICE_DID", DAEMON_DID.to_string()), + ("RUST_LOG", "warn".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect() +} + +struct DaemonSetup { + index_db: PathBuf, + dpop_key: PathBuf, + notify_port: u16, + sink_url: String, +} + +fn daemon_env( + space: &str, + shim_url: &str, + setup: &DaemonSetup, + plc_url: &str, +) -> Vec<(String, String)> { + vec![ + ("DAEMON_SPACE_URI", space.to_string()), + ("DAEMON_SPACE_HOST_URL", shim_url.to_string()), + ("DAEMON_SERVICE_IDENTITY", DAEMON_DID.to_string()), + ("DAEMON_SERVICE_SIGNING_KEY_HEX", DAEMON_KEY.to_string()), + ("DAEMON_SPACE_HOST_MINT_TOKEN", MINT_TOKEN.to_string()), + ("DAEMON_DPOP_KEY_PATH", setup.dpop_key.display().to_string()), + ("DAEMON_INDEX_DB_PATH", setup.index_db.display().to_string()), + ( + "DAEMON_NOTIFY_BIND", + format!("127.0.0.1:{}", setup.notify_port), + ), + ("DAEMON_SWEEP_INTERVAL_SECS", "1".to_string()), + ("DAEMON_PLC_URL", plc_url.to_string()), + ("DAEMON_FEEDS_URL", setup.sink_url.clone()), + ("DAEMON_FEEDS_SERVICE_DID", DAEMON_DID.to_string()), + ("RUST_LOG", "info".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect() +} + +/// Poll `check` until it holds, or fail with what was last seen. +async fn wait_until(what: &str, timeout: Duration, mut check: F) -> Result<()> +where + F: FnMut() -> Result>, +{ + let deadline = Instant::now() + timeout; + loop { + let Some(state) = check()? else { + return Ok(()); + }; + if Instant::now() >= deadline { + bail!("timed out waiting for {what}; last seen: {state}"); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let run_dir = env_path("RESUME_RUN_DIR", "target/resume/run"); + let legacy_bin = env_path("RESUME_LEGACY_SHIM_BIN", ""); + let shim_bin = env_path("RESUME_SHIM_BIN", "target/debug/rsky-space-host"); + let daemon_bin = env_path("RESUME_DAEMON_BIN", "target/debug/rsky-daemon"); + let convert_bin = env_path("RESUME_CONVERT_BIN", "target/debug/convert_store"); + for (label, path) in [ + ("RESUME_LEGACY_SHIM_BIN", &legacy_bin), + ("RESUME_SHIM_BIN", &shim_bin), + ("RESUME_DAEMON_BIN", &daemon_bin), + ("RESUME_CONVERT_BIN", &convert_bin), + ] { + if !path.is_file() { + bail!("{label} must point at a built binary (got {path:?})"); + } + } + + if run_dir.exists() { + std::fs::remove_dir_all(&run_dir).context("clear run directory")?; + } + std::fs::create_dir_all(&run_dir)?; + let run_dir = run_dir.canonicalize()?; + let host_dir = run_dir.join("host"); + let legacy_actors = host_dir.join("actors-legacy"); + let converged_actors = host_dir.join("actors"); + let host_db = host_dir.join("space_host.db"); + std::fs::create_dir_all(&legacy_actors)?; + std::fs::create_dir_all(&converged_actors)?; + + // The legacy era keeps its repos in the multi-tenant `space_host.db`; the + // actor store beside it holds nothing but the account's signing key. + seed_actor_store(&legacy_actors, AUTHOR_DID, AUTHOR_KEY)?; + + let mut keys = BTreeMap::new(); + keys.insert(AUTHOR_DID.to_string(), multibase_of(AUTHOR_KEY)?); + keys.insert(DAEMON_DID.to_string(), multibase_of(DAEMON_KEY)?); + let directory = Directory::start(keys, HANDLE.to_string())?; + + let shim_port = free_port()?; + let shim_url = format!("http://127.0.0.1:{shim_port}"); + let space = format!("at://{AUTHOR_DID}/space/{SPACE_TYPE}/{SPACE_SKEY}"); + + let warm_sink = Sink::start()?; + let warm = DaemonSetup { + index_db: run_dir.join("daemon/index.sqlite"), + dpop_key: run_dir.join("daemon/dpop.json"), + notify_port: free_port()?, + sink_url: warm_sink.url(), + }; + std::fs::create_dir_all(run_dir.join("daemon"))?; + + let mut board = Scoreboard::default(); + let client = rsky_spaces_parity::layer2::http_client()?; + + // ---- Phase A: the legacy era ------------------------------------------ + let mut legacy = Server::spawn( + "legacy space host", + &legacy_bin, + &run_dir, + &shim_env( + shim_port, + &shim_url, + &host_db, + &legacy_actors, + &directory.url(), + ), + &run_dir.join("shim-legacy.log"), + )?; + legacy + .wait_ready(&format!("{shim_url}/xrpc/_health"), READY) + .await?; + println!("legacy space host ready on {shim_url}"); + + let writer = Writer { + client: client.clone(), + shim_url: shim_url.clone(), + space: space.clone(), + }; + let rev_1 = writer.create(POST_1, "legacy one").await?; + let rev_2 = writer.create(POST_2, "legacy two").await?; + println!("legacy writes committed at {rev_1}, {rev_2}"); + + let mut daemon = Server::spawn( + "daemon (legacy era)", + &daemon_bin, + &run_dir, + &daemon_env(&space, &shim_url, &warm, &directory.url()), + &run_dir.join("daemon-legacy.log"), + )?; + daemon.wait_log("daemon starting", READY).await?; + println!("daemon running against the legacy store"); + + // The daemon has caught up when its durable cursor is the head revision + // and the feeds projector has confirmed that revision. + let index_db = warm.index_db.clone(); + { + let (space, rev_2) = (space.clone(), rev_2.clone()); + let index_db = index_db.clone(); + wait_until( + "the daemon to persist its legacy-era cursor", + SETTLE, + move || { + let snapshot = read_index(&index_db, &space, AUTHOR_DID)?; + let acked = snapshot.head_rev.as_deref() == Some(rev_2.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_2.as_str()); + Ok((!acked).then(|| format!("{snapshot:?}"))) + }, + ) + .await?; + } + let legacy_projected = warm_sink.drain(); + let before_swap = read_index(&index_db, &space, AUTHOR_DID)?; + println!( + "daemon cursor persisted at {:?} after projecting {:?}", + before_swap.head_rev, + labels(&legacy_projected) + ); + + board.equal_if( + "legacy era: the daemon projects the writes it saw, once each", + labels(&legacy_projected) + == vec![post_label(POST_1, "create"), post_label(POST_2, "create")] + && duplicates(&legacy_projected).is_empty(), + format!("{:?}", labels(&legacy_projected)), + ); + board.equal_if( + "legacy era: the cursor is durable", + before_swap.head_rev.as_deref() == Some(rev_2.as_str()), + format!("cursor {:?}, head write {rev_2}", before_swap.head_rev), + ); + + // Stop the daemon mid-stream, then land writes it never acknowledged. + daemon.stop(); + let legacy_daemon_log = std::fs::read_to_string(&daemon.log).unwrap_or_default(); + println!("daemon stopped; landing writes it will never have seen"); + let rev_3 = writer.create(POST_3, "legacy three, unseen").await?; + let rev_4 = writer.create(POST_4, "legacy four, unseen").await?; + let rev_5 = writer.delete(POST_2).await?; + println!("unacknowledged legacy writes at {rev_3}, {rev_4}, {rev_5}"); + + let still = read_index(&index_db, &space, AUTHOR_DID)?; + board.equal_if( + "legacy era: writes after the last ack leave the cursor behind", + still.head_rev.as_deref() == Some(rev_2.as_str()) && rev_5 > rev_2, + format!("cursor {:?}, latest write {rev_5}", still.head_rev), + ); + + legacy.stop(); + println!("legacy space host stopped"); + + // ---- Phase B: the swap ------------------------------------------------ + let legacy_ops = legacy_oplog(&host_db, &space, AUTHOR_DID)?; + let convert = std::process::Command::new(&convert_bin) + .arg("--from") + .arg(&host_db) + .arg("--into") + .arg(&converged_actors) + .output() + .context("run the store converter")?; + if !convert.status.success() { + bail!( + "conversion failed: {}{}", + String::from_utf8_lossy(&convert.stdout), + String::from_utf8_lossy(&convert.stderr) + ); + } + let conversion = String::from_utf8_lossy(&convert.stdout).trim().to_string(); + println!("conversion: {conversion}"); + // The converted stores need the account key beside them, exactly as the + // deploy leaves it: the same actor-store directory, new store files. + let key_source = legacy_actors + .join(&hex::encode(Sha256::digest(AUTHOR_DID.as_bytes()))[..2]) + .join(AUTHOR_DID) + .join("key"); + let store = rsky_space_host::actor_repos::store_path(&converged_actors, AUTHOR_DID) + .map_err(|error| anyhow::anyhow!("store path: {error}"))?; + std::fs::copy( + &key_source, + store.parent().context("store has no parent")?.join("key"), + )?; + + let converged_ops = converged_oplog(&store, &space)?; + board.equal_if( + "conversion: oplog ids and revisions survive verbatim", + converged_ops == legacy_ops && !legacy_ops.is_empty(), + format!( + "{} legacy ops -> {} converged ops; ids {:?}", + legacy_ops.len(), + converged_ops.len(), + converged_ops.iter().map(|op| op.0).collect::>() + ), + ); + let (converged_rev, floor) = converged_head(&store, &space)?; + board.equal_if( + "conversion: no history is placed out of reach", + floor.is_none() && converged_rev == rev_5, + format!("head {converged_rev}, oplog floor {floor:?}"), + ); + + // ---- Phase C: the converged era --------------------------------------- + let mut converged = Server::spawn( + "converged space host", + &shim_bin, + &run_dir, + &shim_env( + shim_port, + &shim_url, + &host_db, + &converged_actors, + &directory.url(), + ), + &run_dir.join("shim-converged.log"), + )?; + converged + .wait_ready(&format!("{shim_url}/xrpc/_health"), READY) + .await?; + println!("converged space host ready on {shim_url}"); + + // The same daemon, the same index, the same DPoP key, the same arguments. + let mut daemon = Server::spawn( + "daemon (converged era)", + &daemon_bin, + &run_dir, + &daemon_env(&space, &shim_url, &warm, &directory.url()), + &run_dir.join("daemon-converged.log"), + )?; + daemon.wait_log("daemon starting", READY).await?; + println!("daemon restarted with its existing cursors"); + + { + let (space, rev_5) = (space.clone(), rev_5.clone()); + let index_db = index_db.clone(); + wait_until( + "the daemon to resume to the pre-swap head", + SETTLE, + move || { + let snapshot = read_index(&index_db, &space, AUTHOR_DID)?; + let caught_up = snapshot.head_rev.as_deref() == Some(rev_5.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_5.as_str()); + Ok((!caught_up).then(|| format!("{snapshot:?}"))) + }, + ) + .await?; + } + let resumed = warm_sink.drain(); + println!("resume projected {:?}", labels(&resumed)); + + let mut expected_resume = vec![ + post_label(POST_2, "delete"), + post_label(POST_3, "create"), + post_label(POST_4, "create"), + ]; + expected_resume.sort(); + let mut got_resume = labels(&resumed); + got_resume.sort(); + board.equal_if( + "resume: exactly the operations after the cursor, once each", + got_resume == expected_resume, + format!("expected {expected_resume:?}, got {got_resume:?}"), + ); + board.equal_if( + "resume: nothing already projected is projected again", + duplicates(&resumed).is_empty() + && !got_resume.contains(&post_label(POST_1, "create")) + && !got_resume.contains(&post_label(POST_2, "create")), + format!("duplicates {:?}", duplicates(&resumed)), + ); + + // A write in the converged era, at a server-minted revision. + let rev_6 = writer.create(POST_5, "converged five").await?; + board.equal_if( + "converged era: a server-minted revision follows the carried one", + rev_6 > rev_5, + format!("carried {rev_5}, minted {rev_6}"), + ); + { + let (space, rev_6) = (space.clone(), rev_6.clone()); + let index_db = index_db.clone(); + wait_until( + "the daemon to project the converged-era write", + SETTLE, + move || { + let snapshot = read_index(&index_db, &space, AUTHOR_DID)?; + let caught_up = snapshot.head_rev.as_deref() == Some(rev_6.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_6.as_str()); + Ok((!caught_up).then(|| format!("{snapshot:?}"))) + }, + ) + .await?; + } + let after_swap_write = warm_sink.drain(); + board.equal_if( + "converged era: the new write projects once", + labels(&after_swap_write) == vec![post_label(POST_5, "create")], + format!("{:?}", labels(&after_swap_write)), + ); + + daemon.stop(); + let converged_daemon_log = std::fs::read_to_string(&daemon.log).unwrap_or_default(); + let unclean = unclean_resume_lines(&converged_daemon_log); + board.equal_if( + "resume: no cursor refusal, divergence or full-state recovery", + unclean.is_empty(), + if unclean.is_empty() { + "the daemon advanced incrementally throughout".to_string() + } else { + unclean.join("\n") + }, + ); + board.push( + "legacy-era daemon log was also clean", + if unclean_resume_lines(&legacy_daemon_log).is_empty() { + Verdict::Equal + } else { + Verdict::Differs + }, + unclean_resume_lines(&legacy_daemon_log).join("\n"), + ); + + let warm_index = read_index(&index_db, &space, AUTHOR_DID)?; + + // ---- A cold daemon on the converted store ----------------------------- + let cold_sink = Sink::start()?; + let cold = DaemonSetup { + index_db: run_dir.join("cold/index.sqlite"), + dpop_key: run_dir.join("cold/dpop.json"), + notify_port: free_port()?, + sink_url: cold_sink.url(), + }; + std::fs::create_dir_all(run_dir.join("cold"))?; + let mut cold_daemon = Server::spawn( + "cold daemon", + &daemon_bin, + &run_dir, + &daemon_env(&space, &shim_url, &cold, &directory.url()), + &run_dir.join("daemon-cold.log"), + )?; + cold_daemon.wait_log("daemon starting", READY).await?; + { + let (space, rev_6) = (space.clone(), rev_6.clone()); + let cold_db = cold.index_db.clone(); + wait_until("the cold daemon to sync from scratch", SETTLE, move || { + let snapshot = read_index(&cold_db, &space, AUTHOR_DID)?; + let caught_up = snapshot.head_rev.as_deref() == Some(rev_6.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_6.as_str()); + Ok((!caught_up).then(|| format!("{snapshot:?}"))) + }) + .await?; + } + let cold_projected = cold_sink.seen(); + cold_daemon.stop(); + converged.stop(); + + let cold_index = read_index(&cold.index_db, &space, AUTHOR_DID)?; + board.equal_if( + "cold sync: the same records, revisions and digest as the resumed daemon", + cold_index.records == warm_index.records + && cold_index.head_rev == warm_index.head_rev + && cold_index.lthash_state == warm_index.lthash_state, + format!( + "warm {} records at {:?}, cold {} records at {:?}", + warm_index.records.len(), + warm_index.head_rev, + cold_index.records.len(), + cold_index.head_rev + ), + ); + + let warm_stream: Vec<_> = legacy_projected + .iter() + .chain(resumed.iter()) + .chain(after_swap_write.iter()) + .cloned() + .collect(); + board.equal_if( + "cold sync: the same projected end state as the resumed daemon", + final_state(&cold_projected) == final_state(&warm_stream), + format!( + "warm {:?}\ncold {:?}", + final_state(&warm_stream), + final_state(&cold_projected) + ), + ); + board.equal_if( + "whole run: no operation is projected twice to either destination", + duplicates(&warm_stream).is_empty() && duplicates(&cold_projected).is_empty(), + format!( + "warm {:?}, cold {:?}", + duplicates(&warm_stream), + duplicates(&cold_projected) + ), + ); + board.push( + "oplog window was never the constraint", + Verdict::Note, + format!( + "{} ops in a {} row window; the converted floor is open, so no `since` can be refused", + converged_ops.len(), + rsky_space_host::actor_repos::DEFAULT_OPLOG_WINDOW + ), + ); + + let report = format!( + "resume-across-swap gate\n\n\ + space: {space}\n\ + legacy head at hand-off: {rev_2}\n\ + unacknowledged legacy writes: {rev_3}, {rev_4}, {rev_5}\n\ + converged-era write: {rev_6}\n\ + {conversion}\n\n{}", + board.render() + ); + print!("{report}"); + std::fs::write(run_dir.join("report.txt"), &report)?; + println!("\nlogs: {}", run_dir.display()); + + if board.failures() > 0 { + bail!("resume gate failed: {} check(s) differ", board.failures()); + } + Ok(()) +} + +fn post_label(rkey: &str, operation: &str) -> String { + format!("{COLLECTION}/{rkey}:{operation}") +} diff --git a/rsky-spaces-parity/src/layer2/process.rs b/rsky-spaces-parity/src/layer2/process.rs index e88dc4c7..31efe2b6 100644 --- a/rsky-spaces-parity/src/layer2/process.rs +++ b/rsky-spaces-parity/src/layer2/process.rs @@ -74,6 +74,35 @@ impl Server { } } + /// Poll the log until `needle` appears, for a server with no health + /// endpoint to poll instead. + pub async fn wait_log(&mut self, needle: &str, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = self.child.try_wait()? { + bail!( + "{} exited early ({status}); log:\n{}", + self.name, + self.tail() + ); + } + if std::fs::read_to_string(&self.log) + .unwrap_or_default() + .contains(needle) + { + return Ok(()); + } + if Instant::now() >= deadline { + bail!( + "{} never logged {needle:?}; log:\n{}", + self.name, + self.tail() + ); + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + } + pub fn tail(&self) -> String { let text = std::fs::read_to_string(&self.log).unwrap_or_default(); text.lines() diff --git a/rsky-spaces-parity/src/lib.rs b/rsky-spaces-parity/src/lib.rs index 438d478b..2673a29d 100644 --- a/rsky-spaces-parity/src/lib.rs +++ b/rsky-spaces-parity/src/lib.rs @@ -1,4 +1,5 @@ pub mod layer2; +pub mod resume; use rsky_pds::actor_store::space::{SpaceStore, SpaceStoreError}; use rsky_space_host::error::HostError; diff --git a/rsky-spaces-parity/src/resume/mod.rs b/rsky-spaces-parity/src/resume/mod.rs new file mode 100644 index 00000000..c9a2b382 --- /dev/null +++ b/rsky-spaces-parity/src/resume/mod.rs @@ -0,0 +1,199 @@ +//! Support code for the resume-across-swap gate: the projection sink the +//! daemon is pointed at, and readers for the three durable artefacts the gate +//! reasons about — the daemon's index, the legacy multi-tenant store, and a +//! converted per-account store. + +pub mod sink; + +use anyhow::{Context, Result}; +use rusqlite::{params, Connection, OpenFlags}; +use std::collections::BTreeMap; +use std::path::Path; + +/// One oplog row, in the form both storage eras can be read into. +pub type OpRow = (i64, String, String, String, Option); + +/// What the daemon durably knows about one repo. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexSnapshot { + /// The syncer's cursor: `since` on the next `listRepoOps`. + pub head_rev: Option, + pub lthash_state: Option>, + /// `collection/rkey -> (cid, rev, value)`. + pub records: BTreeMap>)>, + /// Journalled batches, by revision, with the mutation count each carried. + pub journal: BTreeMap, + /// `projector -> rev`: how far each projection has been confirmed. + pub projector_cursors: BTreeMap, +} + +/// Read the daemon's index for one `(space, did)`. The daemon runs SQLite in +/// WAL mode, so this opens read-write to let the reader replay the log of a +/// process that was killed rather than shut down. +pub fn read_index(path: &Path, space: &str, did: &str) -> Result { + let conn = Connection::open(path).with_context(|| format!("open {}", path.display()))?; + let head: Option<(String, Vec)> = conn + .query_row( + "SELECT rev, lthash_state FROM sync_state WHERE space_uri = ?1 AND did = ?2", + params![space, did], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .ok(); + let mut records = BTreeMap::new(); + { + let mut statement = conn.prepare( + "SELECT collection, rkey, cid, rev, value FROM record \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY collection, rkey", + )?; + let mut rows = statement.query(params![space, did])?; + while let Some(row) = rows.next()? { + let collection: String = row.get(0)?; + let rkey: String = row.get(1)?; + records.insert( + format!("{collection}/{rkey}"), + (row.get(2)?, row.get(3)?, row.get(4)?), + ); + } + } + let mut journal = BTreeMap::new(); + { + let mut statement = conn.prepare( + "SELECT rev, mutations FROM projection_journal \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY rev", + )?; + let mut rows = statement.query(params![space, did])?; + while let Some(row) = rows.next()? { + let rev: String = row.get(0)?; + let mutations: Vec = row.get(1)?; + let count = serde_json::from_slice::(&mutations) + .ok() + .and_then(|value| value.as_array().map(Vec::len)) + .unwrap_or(0); + journal.insert(rev, count); + } + } + let mut projector_cursors = BTreeMap::new(); + { + let mut statement = conn.prepare( + "SELECT projector, rev FROM projector_cursor \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY projector", + )?; + let mut rows = statement.query(params![space, did])?; + while let Some(row) = rows.next()? { + projector_cursors.insert(row.get(0)?, row.get(1)?); + } + } + Ok(IndexSnapshot { + head_rev: head.as_ref().map(|(rev, _)| rev.clone()), + lthash_state: head.map(|(_, state)| state), + records, + journal, + projector_cursors, + }) +} + +/// The legacy multi-tenant oplog for one `(space, did)`, in row order. +pub fn legacy_oplog(db: &Path, space: &str, did: &str) -> Result> { + let conn = Connection::open_with_flags(db, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("open {}", db.display()))?; + let mut statement = conn.prepare( + "SELECT seq, rev, collection, rkey, cid FROM repo_op \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY seq", + )?; + let mut rows = statement.query(params![space, did])?; + let mut ops = Vec::new(); + while let Some(row) = rows.next()? { + ops.push(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )); + } + Ok(ops) +} + +/// The converged per-account oplog for one space, in row order. +pub fn converged_oplog(store: &Path, space: &str) -> Result> { + let conn = Connection::open_with_flags(store, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("open {}", store.display()))?; + let mut statement = conn.prepare( + "SELECT id, rev, collection, rkey, cid FROM space_oplog \ + WHERE space_uri = ?1 ORDER BY id", + )?; + let mut rows = statement.query(params![space])?; + let mut ops = Vec::new(); + while let Some(row) = rows.next()? { + ops.push(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )); + } + Ok(ops) +} + +/// `(rev, oplog_floor_rev)` for a converged repo. A floor of `None` is the +/// state in which no `since` can be refused as outside the oplog window. +pub fn converged_head(store: &Path, space: &str) -> Result<(String, Option)> { + let conn = Connection::open_with_flags(store, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("open {}", store.display()))?; + conn.query_row( + "SELECT rev, oplog_floor_rev FROM space_repo WHERE space_uri = ?1 AND deleted = 0", + params![space], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .with_context(|| format!("no live space_repo row for {space}")) +} + +/// Labels of a projected batch list, in arrival order. +pub fn labels(ops: &[sink::Projected]) -> Vec { + ops.iter().map(sink::Projected::label).collect() +} + +/// Labels that appear more than once across `ops` — a duplicated projection. +pub fn duplicates(ops: &[sink::Projected]) -> Vec { + let mut counts: BTreeMap = BTreeMap::new(); + for op in ops { + *counts.entry(op.label()).or_default() += 1; + } + counts + .into_iter() + .filter(|(_, count)| *count > 1) + .map(|(label, count)| format!("{label} x{count}")) + .collect() +} + +/// The final projected state per record path, which is what a destination +/// holds once a batch list has been applied in order. +pub fn final_state(ops: &[sink::Projected]) -> BTreeMap { + let mut state = BTreeMap::new(); + for op in ops { + state.insert(op.path(), op.operation.clone()); + } + state +} + +/// Lines of `log` that name a sync failure, cursor refusal, or full-state +/// recovery. Any of them means the resume was not clean. +pub fn unclean_resume_lines(log: &str) -> Vec { + const MARKERS: [&str; 5] = [ + "HistoryUnavailable", + "full-state recovery", + "diverged", + "sweep failed", + "prev does not match", + ]; + log.lines() + .filter(|line| { + let lowered = line.to_lowercase(); + MARKERS + .iter() + .any(|marker| lowered.contains(&marker.to_lowercase())) + }) + .map(str::to_string) + .collect() +} diff --git a/rsky-spaces-parity/src/resume/sink.rs b/rsky-spaces-parity/src/resume/sink.rs new file mode 100644 index 00000000..3e211ced --- /dev/null +++ b/rsky-spaces-parity/src/resume/sink.rs @@ -0,0 +1,151 @@ +//! A stand-in projection destination: it accepts the daemon's +//! `projectRecords` batches and keeps them, so the gate can say exactly which +//! operations were projected, in which order, and how many times. + +use serde_json::Value; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; + +/// One projected operation, reduced to what the gate compares. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Projected { + /// The batch this operation arrived in, counted from 1 per sink. + pub batch: usize, + pub uri: String, + pub operation: String, + pub revision: String, + pub cid: Option, +} + +impl Projected { + /// `collection/rkey`, the part of the URI that identifies the record. + pub fn path(&self) -> String { + let mut segments = self.uri.rsplitn(3, '/'); + let rkey = segments.next().unwrap_or_default(); + let collection = segments.next().unwrap_or_default(); + format!("{collection}/{rkey}") + } + + /// `collection/rkey:operation`, the label the gate asserts on. + pub fn label(&self) -> String { + format!("{}:{}", self.path(), self.operation) + } +} + +#[derive(Default)] +struct State { + ops: Vec, + batches: usize, + acknowledgements: usize, +} + +pub struct Sink { + port: u16, + state: Arc>, +} + +impl Sink { + /// Bind an ephemeral port and serve until the process exits. + pub fn start() -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let state = Arc::new(Mutex::new(State::default())); + let served = state.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let (path, body) = match read_request(&mut stream) { + Some(request) => request, + None => continue, + }; + record(&served, &path, &body); + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\ + Content-Length: 2\r\nConnection: close\r\n\r\n{}", + ); + } + }); + Ok(Self { port, state }) + } + + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Everything received so far, without removing it. + pub fn seen(&self) -> Vec { + self.state.lock().expect("sink state").ops.clone() + } + + /// Take everything received so far, leaving the sink empty, so the next + /// phase's projections are read in isolation. + pub fn drain(&self) -> Vec { + std::mem::take(&mut self.state.lock().expect("sink state").ops) + } + + pub fn batches(&self) -> usize { + self.state.lock().expect("sink state").batches + } + + pub fn acknowledgements(&self) -> usize { + self.state.lock().expect("sink state").acknowledgements + } +} + +fn record(state: &Arc>, path: &str, body: &str) { + let mut state = state.lock().expect("sink state"); + if path.contains("ackSyncersObserved") { + state.acknowledgements += 1; + return; + } + if !path.contains("projectRecords") { + return; + } + let Ok(parsed) = serde_json::from_str::(body) else { + return; + }; + state.batches += 1; + let batch = state.batches; + let ops = parsed["ops"].as_array().cloned().unwrap_or_default(); + for op in ops { + state.ops.push(Projected { + batch, + uri: op["uri"].as_str().unwrap_or_default().to_string(), + operation: op["operation"].as_str().unwrap_or_default().to_string(), + revision: op["revision"].as_str().unwrap_or_default().to_string(), + cid: op["cid"].as_str().map(str::to_string), + }); + } +} + +/// Read one request, returning its path and body. Only the request line and +/// `content-length` are interpreted; the daemon sends no chunked bodies. +fn read_request(stream: &mut std::net::TcpStream) -> Option<(String, String)> { + let mut reader = BufReader::new(stream.try_clone().ok()?); + let mut request_line = String::new(); + if reader.read_line(&mut request_line).ok()? == 0 { + return None; + } + let path = request_line.split_whitespace().nth(1)?.to_string(); + let mut length = 0usize; + loop { + let mut header = String::new(); + if reader.read_line(&mut header).ok()? == 0 { + break; + } + if header.trim().is_empty() { + break; + } + if let Some((name, value)) = header.split_once(':') { + if name.trim().eq_ignore_ascii_case("content-length") { + length = value.trim().parse().unwrap_or(0); + } + } + } + let mut body = vec![0u8; length]; + if length > 0 { + reader.read_exact(&mut body).ok()?; + } + Some((path, String::from_utf8_lossy(&body).to_string())) +} From 88057a9f6ead8151c5bdbe7c795cccf41a2d2a37 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Sun, 23 Aug 2026 23:15:21 -0400 Subject: [PATCH 43/56] fix(space-host): include parity crate manifest in image workspace resolution --- rsky-space-host/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsky-space-host/Dockerfile b/rsky-space-host/Dockerfile index acf71996..27843a78 100644 --- a/rsky-space-host/Dockerfile +++ b/rsky-space-host/Dockerfile @@ -23,6 +23,7 @@ COPY rsky-repo/Cargo.toml rsky-repo/Cargo.toml COPY rsky-satnav/Cargo.toml rsky-satnav/Cargo.toml COPY rsky-space/Cargo.toml rsky-space/Cargo.toml COPY rsky-space-host/Cargo.toml rsky-space-host/Cargo.toml +COPY rsky-spaces-parity/Cargo.toml rsky-spaces-parity/Cargo.toml COPY rsky-syntax/Cargo.toml rsky-syntax/Cargo.toml COPY rsky-video/Cargo.toml rsky-video/Cargo.toml COPY rsky-wintermute/Cargo.toml rsky-wintermute/Cargo.toml @@ -48,6 +49,7 @@ RUN mkdir -p \ echo 'fn main() {}' > $crate/src/main.rs; \ done && \ touch rsky-pds/src/lib.rs rsky-repo/src/lib.rs && \ + mkdir -p rsky-spaces-parity/src && touch rsky-spaces-parity/src/lib.rs && \ mkdir -p rsky-wintermute/src/bin && \ for bin in queue_backfill fix_blob_refs plc_import label_sync car_loader \ cleanup_stale_deactivated; do \ From 1d801458a7367a5aabd092fe8972d0af55250314 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Mon, 24 Aug 2026 14:27:42 -0400 Subject: [PATCH 44/56] fix(space-host): mint managing-app service auth from the actor-store key The managing app resolves an inbound service JWT against the issuer's #atproto verification method, so a call signed with the space key is rejected. Mint checkUserAccess and ackHostRegistered through PdsServiceJwtIssuer instead, and refuse at boot when a pinned authority under the managing-app policy has no actor-store key. --- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/config.rs | 85 +++++++++++++++++++++++++ rsky-space-host/src/main.rs | 23 ++++--- rsky-space-host/src/managing_app.rs | 98 ++++++++++++++++++++++++++--- rsky-space-host/src/pds_seam.rs | 21 ++++--- rsky-space-host/src/registration.rs | 83 ++++++++++++++++++++++++ 6 files changed, 289 insertions(+), 23 deletions(-) diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 4cf9ee47..3f0f5d8e 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.2" +version = "0.7.3" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index aa7bc6f2..15e11482 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -197,16 +197,51 @@ impl Config { } Ok(()) } + + /// The managing-app conversation is service auth, which is keyed on the + /// authority's `#atproto` key rather than the space key. A pinned authority + /// carries only the space key, so without an actor-store key for it the + /// policy can never mint a call the managing app will accept. Fail at boot + /// instead of at a member's first read. `has_service_key` answers whether + /// the actor store holds a signing key for that DID. + pub fn validate_pinned_service_key( + &self, + has_service_key: impl FnOnce(&str) -> bool, + ) -> Result<(), String> { + let Some((authority_did, _)) = self.bootstrap_pin() else { + return Ok(()); + }; + if self.policy != PolicyMode::ManagingApp { + return Ok(()); + } + if has_service_key(authority_did) { + return Ok(()); + } + Err(format!( + "managing-app policy needs an actor-store signing key for the pinned authority {authority_did}: SPACEHOST_ACTOR_STORE_DIR has none, so only the public and member-list policies are available to a pinned host" + )) + } } #[cfg(test)] mod tests { use super::*; + // `Config` falls back to env vars, and the env-var test below mutates + // process-global state, so every parse in this module is serialized. + static PARSE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn parse_lock() -> std::sync::MutexGuard<'static, ()> { + PARSE_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + // One sequential test: the env-var section mutates process-global state, // which would race sibling tests run in parallel. #[test] fn parses_args_env_and_requirements() { + let _guard = parse_lock(); let bare = Config::try_parse_from(["rsky-space-host"]).unwrap(); assert!(bare.bootstrap_pin().is_none()); assert!(bare.validate().is_err()); @@ -333,6 +368,7 @@ mod tests { } fn valid_unpinned() -> Config { + let _guard = parse_lock(); Config::try_parse_from([ "rsky-space-host", "--oauth-issuer", @@ -355,6 +391,55 @@ mod tests { .unwrap() } + #[test] + fn pinned_managing_app_needs_an_actor_store_service_key() { + fn pinned_managing_app(policy: PolicyMode) -> Config { + let mut cfg = valid_unpinned(); + cfg.authority_did = "did:plc:authority".to_string(); + cfg.signing_key_hex = "aa".repeat(32); + cfg.policy = policy; + cfg.managing_app = "did:web:feeds.example#bsky_fg".to_string(); + cfg.lifecycle_url = "https://feeds.example".to_string(); + cfg.lifecycle_service_did = "did:web:feeds.example".to_string(); + cfg + } + + let pinned = pinned_managing_app(PolicyMode::ManagingApp); + assert!(pinned.validate().is_ok()); + + // No actor-store key for the pinned authority: refuse at boot. + let message = pinned + .validate_pinned_service_key(|_| false) + .expect_err("must refuse"); + assert!(message.contains("did:plc:authority"), "{message}"); + assert!(message.contains("member-list"), "{message}"); + + // With a key, the same config is accepted, and the probe sees the + // pinned authority rather than some other DID. + let mut asked = String::new(); + pinned + .validate_pinned_service_key(|did| { + asked = did.to_string(); + true + }) + .unwrap(); + assert_eq!(asked, "did:plc:authority"); + + // The other policies are unaffected — that is the point of the fallback. + pinned_managing_app(PolicyMode::MemberList) + .validate_pinned_service_key(|_| false) + .unwrap(); + pinned_managing_app(PolicyMode::Public) + .validate_pinned_service_key(|_| false) + .unwrap(); + + // An unpinned host resolves its authorities from the actor store, so + // there is nothing to reject. + let mut unpinned = valid_unpinned(); + unpinned.policy = PolicyMode::ManagingApp; + unpinned.validate_pinned_service_key(|_| false).unwrap(); + } + #[test] fn bootstrap_pin_is_optional_but_all_or_nothing() { let cfg = valid_unpinned(); diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index b836526d..8311b16d 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -18,9 +18,10 @@ use rsky_space_host::keys::{DocKeyResolver, DocSource, ResolverDocSource}; use rsky_space_host::managing_app::HttpManagingApp; use rsky_space_host::membership::InMemoryMembership; use rsky_space_host::notify::HttpNotifier; -use rsky_space_host::pds_seam::PdsSeam; +use rsky_space_host::pds_seam::{PdsSeam, PdsServiceJwtIssuer}; use rsky_space_host::policy::Policy; use rsky_space_host::registration::{HttpLifecycleAcker, LifecycleAcker}; +use rsky_space_host::service_jwt::ServiceJwtIssuer; use rsky_space_host::signing::Signer; use rsky_space_host::store::{HostedSpaceStore, SqliteStore}; use std::sync::Arc; @@ -49,6 +50,7 @@ struct ContextBuilder { members: Vec, lifecycle_url: String, lifecycle_service_did: String, + seam: Arc, docs: Arc, now: Arc u64 + Send + Sync>, jti: Arc String + Send + Sync>, @@ -57,6 +59,13 @@ struct ContextBuilder { impl ContextBuilder { fn context(&self, space: SpaceId, signer: Signer) -> AuthorityContext { let authority_did = space.authority.clone(); + // Service auth to the managing app is keyed on the authority's + // `#atproto` key, not the space key, so it is minted from the actor + // store rather than from the configured space signer. + let service_issuer: Arc = Arc::new(PdsServiceJwtIssuer::new( + self.seam.clone(), + authority_did.clone(), + )); let policy = match self.policy { PolicyMode::Public => Policy::Public, PolicyMode::MemberList => { @@ -64,10 +73,9 @@ impl ContextBuilder { } PolicyMode::ManagingApp => Policy::ManagingApp { service_id: self.managing_app.clone(), - client: Arc::new(HttpManagingApp::new( + client: Arc::new(HttpManagingApp::with_issuer( self.managing_app.clone(), - authority_did.clone(), - signer.clone(), + service_issuer.clone(), self.docs.clone(), self.now.clone(), self.jti.clone(), @@ -84,11 +92,10 @@ impl ContextBuilder { self.jti.clone(), )), lifecycle_acker: (self.policy == PolicyMode::ManagingApp).then(|| { - Arc::new(HttpLifecycleAcker::new( + Arc::new(HttpLifecycleAcker::with_issuer( self.lifecycle_url.clone(), self.lifecycle_service_did.clone(), - authority_did, - signer, + service_issuer, self.now.clone(), self.jti.clone(), )) as Arc @@ -118,6 +125,7 @@ async fn main() -> Result<(), Box> { let store = Arc::new(SqliteStore::open(&cfg.db_path)?); let repos = Arc::new(ActorStoreRepos::open(&cfg.actor_store_dir)?); let seam = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); + cfg.validate_pinned_service_key(|did| seam.key_path(did).is_some_and(|path| path.exists()))?; let builder = Arc::new(ContextBuilder { policy: cfg.policy, @@ -125,6 +133,7 @@ async fn main() -> Result<(), Box> { members: cfg.member_dids(), lifecycle_url: cfg.lifecycle_url.clone(), lifecycle_service_did: cfg.lifecycle_service_did.clone(), + seam: seam.clone(), docs: docs.clone(), now: now.clone(), jti: jti.clone(), diff --git a/rsky-space-host/src/managing_app.rs b/rsky-space-host/src/managing_app.rs index eeba7234..8c8f4d7d 100644 --- a/rsky-space-host/src/managing_app.rs +++ b/rsky-space-host/src/managing_app.rs @@ -12,7 +12,7 @@ use std::time::Duration; use crate::error::{HostError, Result}; use crate::keys::{service_endpoint_from_doc, DocSource}; -use crate::service_jwt; +use crate::service_jwt::{ServiceJwtIssuer, SignerIssuer}; use crate::signing::Signer; pub const CHECK_USER_ACCESS_LXM: &str = "com.atproto.simplespace.checkUserAccess"; @@ -55,8 +55,7 @@ pub(crate) fn require_https(url: &str) -> Result<()> { /// its DID document and calls `checkUserAccess` with authority service auth. pub struct HttpManagingApp { service_id: String, - authority_did: String, - signer: Signer, + issuer: Arc, docs: Arc, http: reqwest::Client, now: Arc u64 + Send + Sync>, @@ -92,6 +91,34 @@ impl HttpManagingApp { now: Arc u64 + Send + Sync>, jti: Arc String + Send + Sync>, timeout: Duration, + ) -> Self { + Self::with_issuer_and_timeout( + service_id, + Arc::new(SignerIssuer::new(authority_did, signer)), + docs, + now, + jti, + timeout, + ) + } + + pub fn with_issuer( + service_id: String, + issuer: Arc, + docs: Arc, + now: Arc u64 + Send + Sync>, + jti: Arc String + Send + Sync>, + ) -> Self { + Self::with_issuer_and_timeout(service_id, issuer, docs, now, jti, DEFAULT_TIMEOUT) + } + + pub fn with_issuer_and_timeout( + service_id: String, + issuer: Arc, + docs: Arc, + now: Arc u64 + Send + Sync>, + jti: Arc String + Send + Sync>, + timeout: Duration, ) -> Self { let http = reqwest::Client::builder() .timeout(timeout) @@ -99,8 +126,7 @@ impl HttpManagingApp { .expect("reqwest client"); Self { service_id, - authority_did, - signer, + issuer, docs, http, now, @@ -126,9 +152,7 @@ impl ManagingAppClient for HttpManagingApp { client_id: Option<&str>, ) -> Result { let endpoint = self.endpoint().await?; - let token = service_jwt::mint( - &self.signer, - &self.authority_did, + let token = self.issuer.mint( &self.service_id, CHECK_USER_ACCESS_LXM, (self.now)(), @@ -267,6 +291,64 @@ mod tests { assert_eq!(payload["jti"], "jti-fixed"); } + #[tokio::test] + async fn check_user_access_is_signed_with_the_authority_account_key() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"allowed": true})), + ) + .mount(&server) + .await; + + const AUTHORITY: &str = "did:plc:auth"; + let secret = [9u8; 32]; + let directory = crate::pds_seam::test_actor_store(AUTHORITY, secret); + let seam = Arc::new(crate::pds_seam::PdsSeam::open(directory.path()).unwrap()); + let app = HttpManagingApp::with_issuer( + format!("{APP_DID}#managing_app"), + Arc::new(crate::pds_seam::PdsServiceJwtIssuer::new( + seam, + AUTHORITY.to_string(), + )), + Arc::new(AppDoc(server.uri())), + Arc::new(|| 1000), + Arc::new(|| "jti-fixed".to_string()), + ); + assert!(app + .check_user_access(SPACE, "did:plc:member", None) + .await + .unwrap()); + + let requests = server.received_requests().await.unwrap(); + let auth = requests[0].headers.get("authorization").unwrap(); + let jwt = auth.to_str().unwrap().strip_prefix("Bearer ").unwrap(); + + // The account key — what a managing app resolves as `#atproto`. + let account_key = + crate::signing::Signer::from_secret(secp256k1::SecretKey::from_slice(&secret).unwrap()); + let claims = crate::service_jwt::verify( + jwt, + &[&format!("{APP_DID}#managing_app")], + CHECK_USER_ACCESS_LXM, + account_key.did_key(), + 1000, + ) + .unwrap(); + assert_eq!(claims.iss, AUTHORITY); + + // The space key must NOT verify it: that mismatch is what a managing app + // rejects with a 401. + assert!(crate::service_jwt::verify( + jwt, + &[&format!("{APP_DID}#managing_app")], + CHECK_USER_ACCESS_LXM, + test_signer().did_key(), + 1000, + ) + .is_err()); + } + #[tokio::test] async fn deny_path_returns_false_without_client_id() { let server = MockServer::start().await; diff --git a/rsky-space-host/src/pds_seam.rs b/rsky-space-host/src/pds_seam.rs index 79e8a39a..8d848bf7 100644 --- a/rsky-space-host/src/pds_seam.rs +++ b/rsky-space-host/src/pds_seam.rs @@ -217,6 +217,19 @@ fn validate_actor_store_layout(root: &Path) -> Result<()> { Ok(()) } +/// A temporary actor store holding one account's signing key, laid out the way +/// the PDS lays out `{data}/actors`. +#[cfg(test)] +pub(crate) fn test_actor_store(did: &str, secret: [u8; SECRET_KEY_BYTES]) -> tempfile::TempDir { + let directory = tempfile::tempdir().unwrap(); + let digest = hex::encode(Sha256::digest(did.as_bytes())); + let actor = directory.path().join(&digest[..2]).join(did); + std::fs::create_dir_all(&actor).unwrap(); + std::fs::write(actor.join("key"), secret).unwrap(); + std::fs::write(actor.join("store.sqlite"), []).unwrap(); + directory +} + #[cfg(test)] mod tests { use super::*; @@ -227,13 +240,7 @@ mod tests { const SPACE: &str = "at://did:plc:authority/space/community.blacksky.feed/main"; fn actor_store(secret: [u8; 32]) -> tempfile::TempDir { - let directory = tempfile::tempdir().unwrap(); - let digest = hex::encode(Sha256::digest(DID.as_bytes())); - let actor = directory.path().join(&digest[..2]).join(DID); - std::fs::create_dir_all(&actor).unwrap(); - std::fs::write(actor.join("key"), secret).unwrap(); - std::fs::write(actor.join("store.sqlite"), []).unwrap(); - directory + test_actor_store(DID, secret) } #[test] diff --git a/rsky-space-host/src/registration.rs b/rsky-space-host/src/registration.rs index 1d88c0e5..83556689 100644 --- a/rsky-space-host/src/registration.rs +++ b/rsky-space-host/src/registration.rs @@ -90,3 +90,86 @@ impl LifecycleAcker for HttpLifecycleAcker { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::pds_seam::{test_actor_store, PdsSeam, PdsServiceJwtIssuer}; + use crate::service_jwt; + use crate::signing::test_signer; + use std::sync::Arc; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + const AUTHORITY: &str = "did:plc:auth"; + const FEEDS: &str = "did:web:feeds.test"; + const SPACE: &str = "at://did:plc:auth/space/community.blacksky.feed/main"; + + #[tokio::test] + async fn ack_is_signed_with_the_authority_account_key() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/xrpc/{ACK_HOST_REGISTERED_LXM}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({}))) + .mount(&server) + .await; + + let secret = [11u8; 32]; + let directory = test_actor_store(AUTHORITY, secret); + let seam = Arc::new(PdsSeam::open(directory.path()).unwrap()); + let acker = HttpLifecycleAcker::with_issuer( + server.uri(), + FEEDS, + Arc::new(PdsServiceJwtIssuer::new(seam, AUTHORITY.to_string())), + Arc::new(|| 1000), + Arc::new(|| "jti-fixed".to_string()), + ); + acker.ack_host_registered(SPACE, 1).await.unwrap(); + + let requests = server.received_requests().await.unwrap(); + let auth = requests[0].headers.get("authorization").unwrap(); + let jwt = auth.to_str().unwrap().strip_prefix("Bearer ").unwrap(); + + let account_key = + crate::signing::Signer::from_secret(secp256k1::SecretKey::from_slice(&secret).unwrap()); + let claims = service_jwt::verify( + jwt, + &[FEEDS], + ACK_HOST_REGISTERED_LXM, + account_key.did_key(), + 1000, + ) + .unwrap(); + assert_eq!(claims.iss, AUTHORITY); + + // The space key must not verify it. + assert!(service_jwt::verify( + jwt, + &[FEEDS], + ACK_HOST_REGISTERED_LXM, + test_signer().did_key(), + 1000, + ) + .is_err()); + } + + #[tokio::test] + async fn a_rejected_ack_is_an_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + let directory = test_actor_store(AUTHORITY, [11u8; 32]); + let seam = Arc::new(PdsSeam::open(directory.path()).unwrap()); + let acker = HttpLifecycleAcker::with_issuer( + server.uri(), + FEEDS, + Arc::new(PdsServiceJwtIssuer::new(seam, AUTHORITY.to_string())), + Arc::new(|| 1000), + Arc::new(|| "jti-fixed".to_string()), + ); + let error = acker.ack_host_registered(SPACE, 1).await.unwrap_err(); + assert!(matches!(error, HostError::ManagingApp(msg) if msg.contains("401"))); + } +} From a9fdb9f27d84bed6c1f323d56654206689a94662 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Mon, 24 Aug 2026 14:33:04 -0400 Subject: [PATCH 45/56] fix(space-host): resolve verification keys by purpose Service auth and delegation tokens are keyed on #atproto, but the shared resolver preferred #atproto_space and so selected the wrong key for any DID publishing both. Credential verification keeps the space-first order. --- Cargo.lock | 6 +-- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/authority.rs | 8 ++-- rsky-space-host/src/http.rs | 8 ++-- rsky-space-host/src/keys.rs | 64 +++++++++++++++++++++++--------- 5 files changed, 59 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fa82ad7..9a0e78df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8354,7 +8354,7 @@ dependencies = [ "rsky-oauth 0.3.2", "rsky-repo 0.0.6", "rsky-space 0.4.2", - "rsky-space-host 0.7.2", + "rsky-space-host 0.7.4", "rsky-syntax 0.1.0", "rusqlite", "secp256k1", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.2" +version = "0.7.4" dependencies = [ "async-trait", "axum", @@ -8728,7 +8728,7 @@ dependencies = [ "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", - "rsky-space-host 0.7.2", + "rsky-space-host 0.7.4", "rusqlite", "secp256k1", "serde", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 3f0f5d8e..b0dfe85c 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.3" +version = "0.7.4" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/authority.rs b/rsky-space-host/src/authority.rs index c6c51776..a9a80d07 100644 --- a/rsky-space-host/src/authority.rs +++ b/rsky-space-host/src/authority.rs @@ -97,7 +97,7 @@ impl AuthorityRegistry { /// to verify a delegation token minted by that user's PDS. #[async_trait] pub trait KeyResolver: Send + Sync { - async fn signing_key(&self, did: &str) -> Result; + async fn service_key(&self, did: &str) -> Result; } /// A space authority for one or more spaces of a single type. @@ -325,7 +325,7 @@ impl Authority { let decoded = credential::decode(delegation_jwt).map_err(|e| HostError::Delegation(e.to_string()))?; let user_did = decoded.claims.iss.clone(); - let user_key = keys.signing_key(&user_did).await?; + let user_key = keys.service_key(&user_did).await?; let verified_user = credential::verify_delegation_token( delegation_jwt, &space.uri(), @@ -501,7 +501,7 @@ mod tests { struct DenyAllKeys; #[async_trait] impl KeyResolver for DenyAllKeys { - async fn signing_key(&self, _did: &str) -> Result { + async fn service_key(&self, _did: &str) -> Result { Err(HostError::Membership("no key".into())) } } @@ -509,7 +509,7 @@ mod tests { struct FixedKey(String); #[async_trait] impl KeyResolver for FixedKey { - async fn signing_key(&self, _did: &str) -> Result { + async fn service_key(&self, _did: &str) -> Result { Ok(self.0.clone()) } } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 6d64eccd..147a6829 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -300,7 +300,7 @@ async fn mint_credential( )); } let (context, space) = require_this_space(&state, ¶ms.space)?; - let key = state.keys.signing_key(&claims.iss).await?; + let key = state.keys.service_key(&claims.iss).await?; service_jwt::verify( jwt, &[context.authority_did()], @@ -435,7 +435,7 @@ async fn require_service_auth( ) -> Result { let jwt = bearer(headers)?; let claims = service_jwt::claims(jwt)?; - let issuer_key = state.keys.signing_key(&claims.iss).await?; + let issuer_key = state.keys.service_key(&claims.iss).await?; let authority_did = context.authority_did(); let space_host_aud = format!("{authority_did}#atproto_space_host"); service_jwt::verify( @@ -962,7 +962,7 @@ async fn notify_write( "issuer does not match notified repo", )); } - let issuer_key = state.keys.signing_key(&claims.iss).await?; + let issuer_key = state.keys.service_key(&claims.iss).await?; let authority_did = context.authority_did(); let space_host_aud = format!("{authority_did}#atproto_space_host"); service_jwt::verify( @@ -1021,7 +1021,7 @@ mod tests { struct UserKeys; #[async_trait] impl KeyResolver for UserKeys { - async fn signing_key(&self, did: &str) -> HostResult { + async fn service_key(&self, did: &str) -> HostResult { if did == MEMBER { Ok(user_signer().did_key().to_string()) } else { diff --git a/rsky-space-host/src/keys.rs b/rsky-space-host/src/keys.rs index 5e7bd40c..39843086 100644 --- a/rsky-space-host/src/keys.rs +++ b/rsky-space-host/src/keys.rs @@ -46,14 +46,26 @@ fn fragment_of(id: &str) -> Option<&str> { id.rsplit_once('#').map(|(_, frag)| frag) } -/// The `did:key` for a doc's signing key: prefer `#atproto_space`, fall back to -/// `#atproto`. -pub fn signing_did_key_from_doc(doc: &DidDocument) -> Result { +/// The `did:key` that verifies a space authority's **credentials**: prefer +/// `#atproto_space`, fall back to `#atproto` (spec §Space authority). +pub fn credential_did_key_from_doc(doc: &DidDocument) -> Result { + did_key_by_fragment(doc, &["atproto_space", "atproto"]) +} + +/// The `did:key` that verifies **service auth and delegation tokens**, which are +/// keyed on the account's signing key alone. A delegation token's `kid` MUST be +/// `#atproto` (spec §Delegation token), and inter-service auth uses the same +/// key, so `#atproto_space` is not accepted here even when published. +pub fn service_did_key_from_doc(doc: &DidDocument) -> Result { + did_key_by_fragment(doc, &["atproto"]) +} + +fn did_key_by_fragment(doc: &DidDocument, fragments: &[&str]) -> Result { let methods = doc .verification_method .as_deref() .ok_or_else(|| HostError::Resolution(format!("{}: no verification methods", doc.id)))?; - let method = ["atproto_space", "atproto"] + let method = fragments .iter() .find_map(|frag| methods.iter().find(|m| fragment_of(&m.id) == Some(frag))) .ok_or_else(|| HostError::Resolution(format!("{}: no atproto signing key", doc.id)))?; @@ -83,7 +95,7 @@ pub fn service_endpoint_from_doc(doc: &DidDocument, fragment: &str) -> Result, } @@ -96,9 +108,9 @@ impl DocKeyResolver { #[async_trait] impl KeyResolver for DocKeyResolver { - async fn signing_key(&self, did: &str) -> Result { + async fn service_key(&self, did: &str) -> Result { let doc = self.docs.did_document(did).await?; - signing_did_key_from_doc(&doc) + service_did_key_from_doc(&doc) } } @@ -143,24 +155,42 @@ mod tests { } #[test] - fn prefers_atproto_space_over_atproto() { + fn credential_resolution_prefers_atproto_space_over_atproto() { let (space_m, space_key) = multikey_method("did:plc:subject#atproto_space", [0x41; 32]); let (atp_m, atp_key) = multikey_method("#atproto", [0x42; 32]); let d = doc(vec![atp_m.clone(), space_m], vec![]); - assert_eq!(signing_did_key_from_doc(&d).unwrap(), space_key); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), space_key); let d = doc(vec![atp_m], vec![]); - assert_eq!(signing_did_key_from_doc(&d).unwrap(), atp_key); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), atp_key); + } + + #[test] + fn service_resolution_ignores_atproto_space_when_both_are_published() { + let (space_m, space_key) = multikey_method("did:plc:subject#atproto_space", [0x41; 32]); + let (atp_m, atp_key) = multikey_method("#atproto", [0x42; 32]); + assert_ne!(space_key, atp_key); + + // An authority publishes both. Service auth and delegation tokens are + // keyed on #atproto, so the space key must not be selected. + let d = doc(vec![space_m.clone(), atp_m], vec![]); + assert_eq!(service_did_key_from_doc(&d).unwrap(), atp_key); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), space_key); + + // A doc with only the space key has no service key at all. + let d = doc(vec![space_m], vec![]); + assert!(service_did_key_from_doc(&d).is_err()); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), space_key); } #[test] fn missing_or_unusable_keys_are_errors() { assert!(matches!( - signing_did_key_from_doc(&doc(vec![], vec![])), + credential_did_key_from_doc(&doc(vec![], vec![])), Err(HostError::Resolution(_)) )); let (other, _) = multikey_method("#unrelated", [0x43; 32]); - assert!(signing_did_key_from_doc(&doc(vec![other], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![other], vec![])).is_err()); let no_multibase = VerificationMethod { id: "#atproto".to_string(), @@ -168,7 +198,7 @@ mod tests { controller: "did:plc:subject".to_string(), public_key_multibase: None, }; - assert!(signing_did_key_from_doc(&doc(vec![no_multibase], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![no_multibase], vec![])).is_err()); let bad_multibase = VerificationMethod { id: "#atproto".to_string(), @@ -176,7 +206,7 @@ mod tests { controller: "did:plc:subject".to_string(), public_key_multibase: Some("!!!".to_string()), }; - assert!(signing_did_key_from_doc(&doc(vec![bad_multibase], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![bad_multibase], vec![])).is_err()); let unknown_type = VerificationMethod { id: "#atproto".to_string(), @@ -184,7 +214,7 @@ mod tests { controller: "did:plc:subject".to_string(), public_key_multibase: Some("zunknown".to_string()), }; - assert!(signing_did_key_from_doc(&doc(vec![unknown_type], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![unknown_type], vec![])).is_err()); } #[test] @@ -215,7 +245,7 @@ mod tests { async fn doc_key_resolver_resolves_signing_key() { let (m, key) = multikey_method("#atproto", [0x44; 32]); let resolver = DocKeyResolver::new(Arc::new(FixedDoc(doc(vec![m], vec![])))); - assert_eq!(resolver.signing_key("did:plc:subject").await.unwrap(), key); + assert_eq!(resolver.service_key("did:plc:subject").await.unwrap(), key); } #[tokio::test] @@ -250,7 +280,7 @@ mod tests { did_cache: Arc::new(MemoryCache::new(None, None)), })); let got = source.did_document(&did).await.unwrap(); - assert_eq!(signing_did_key_from_doc(&got).unwrap(), key); + assert_eq!(credential_did_key_from_doc(&got).unwrap(), key); // Unresolvable DIDs surface as resolution errors. let missing = source.did_document("did:web:localhost%3A1").await; From 8db9aab3e0c43dda3d31c665b337fac429c5c3c5 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Mon, 24 Aug 2026 17:04:22 -0400 Subject: [PATCH 46/56] fix(space-host): write space stores to their own directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPACEHOST_ACTOR_STORE_DIR was both the signing-key source and the ActorStoreRepos write target, which made the shim write space tables into the PDS's own actor files — the rejected Option B, and impossible where that directory is mounted read-only. Writes now go to SPACEHOST_SPACE_STORE_DIR, the same path is refused at boot, and a store that already carries the base schema is adopted rather than re-migrated. Both gates only ever ran the two directories as one path and never opened a store the PDS created, so neither could see this: Layer 1 gains those two cases and Layer 2 now runs them separated and asserts the key directory is never written to. --- Cargo.lock | 6 +- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/actor_schema.rs | 31 +++++++- rsky-space-host/src/config.rs | 67 ++++++++++++++++++ rsky-space-host/src/main.rs | 2 +- rsky-spaces-parity/src/bin/layer2_gate.rs | 52 +++++++++++++- rsky-spaces-parity/tests/parity.rs | 86 +++++++++++++++++++++++ 7 files changed, 236 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a0e78df..f7799976 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8354,7 +8354,7 @@ dependencies = [ "rsky-oauth 0.3.2", "rsky-repo 0.0.6", "rsky-space 0.4.2", - "rsky-space-host 0.7.4", + "rsky-space-host 0.7.5", "rsky-syntax 0.1.0", "rusqlite", "secp256k1", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.4" +version = "0.7.5" dependencies = [ "async-trait", "axum", @@ -8728,7 +8728,7 @@ dependencies = [ "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", - "rsky-space-host 0.7.4", + "rsky-space-host 0.7.5", "rusqlite", "secp256k1", "serde", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index b0dfe85c..942de094 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.4" +version = "0.7.5" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/actor_schema.rs b/rsky-space-host/src/actor_schema.rs index 4d4af5d9..a7cc313f 100644 --- a/rsky-space-host/src/actor_schema.rs +++ b/rsky-space-host/src/actor_schema.rs @@ -1,4 +1,4 @@ -use rusqlite::Connection; +use rusqlite::{Connection, OptionalExtension}; use std::collections::HashSet; use std::path::Path; @@ -170,19 +170,46 @@ const MIGRATIONS: &[(&str, &str)] = &[ ), ]; +/// The first migration recreates the PDS's base schema. A store the PDS itself +/// created already has those tables under different bookkeeping, so applying it +/// there would both fail and be a write into a file this service does not own. +const BASELINE_MIGRATION: &str = "001"; + +/// Whether the base schema is already present, i.e. this file was created by +/// something other than these migrations. +fn baseline_present(tx: &rusqlite::Transaction<'_>) -> Result { + tx.query_row( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'repo_root'", + [], + |_| Ok(()), + ) + .optional() + .map(|found| found.is_some()) + .map_err(sql_err) +} + pub fn get_migrated_db(path: impl AsRef) -> Result { let mut connection = Connection::open(path).map_err(sql_err)?; let transaction = connection.transaction().map_err(sql_err)?; transaction .execute_batch("CREATE TABLE IF NOT EXISTS migrations (name TEXT PRIMARY KEY, \"appliedAt\" TEXT NOT NULL)") .map_err(sql_err)?; - let applied = transaction + let mut applied = transaction .prepare("SELECT name FROM migrations") .map_err(sql_err)? .query_map([], |row| row.get::<_, String>(0)) .map_err(sql_err)? .collect::>>() .map_err(sql_err)?; + if !applied.contains(BASELINE_MIGRATION) && baseline_present(&transaction)? { + transaction + .execute( + "INSERT INTO migrations (name, \"appliedAt\") VALUES (?1, datetime('now'))", + [BASELINE_MIGRATION], + ) + .map_err(sql_err)?; + applied.insert(BASELINE_MIGRATION.to_string()); + } for (name, sql) in MIGRATIONS { if !applied.contains(*name) { transaction.execute_batch(sql).map_err(sql_err)?; diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index 15e11482..2554c1dd 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -103,8 +103,15 @@ pub struct Config { hide_env_values = true )] pub oauth_hs256_secret: String, + /// Read-only source of per-account signing keys: the PDS's own `actors` + /// directory. Never written to. #[arg(long, env = "SPACEHOST_ACTOR_STORE_DIR", default_value = "")] pub actor_store_dir: String, + /// This service's own per-account store directory, holding the space + /// tables. Separate from the PDS's actors directory, which is mounted + /// read-only in production and belongs to another writer. + #[arg(long, env = "SPACEHOST_SPACE_STORE_DIR", default_value = "")] + pub space_store_dir: String, #[arg( long, env = "SPACEHOST_MINT_TOKEN", @@ -189,6 +196,15 @@ impl Config { if self.actor_store_dir.is_empty() { return Err("SPACEHOST_ACTOR_STORE_DIR is required".to_string()); } + if self.space_store_dir.is_empty() { + return Err("SPACEHOST_SPACE_STORE_DIR is required".to_string()); + } + if same_directory(&self.actor_store_dir, &self.space_store_dir) { + return Err( + "SPACEHOST_SPACE_STORE_DIR must not be SPACEHOST_ACTOR_STORE_DIR: space tables are written to this service's own per-account stores, never into the PDS's actor files" + .to_string(), + ); + } if self.mint_token.is_empty() || self.daemon_service_did.is_empty() || self.appview_service_did.is_empty() @@ -223,6 +239,18 @@ impl Config { } } +/// Whether two settings name the same directory. Resolved paths are compared +/// when both exist, so `/pds/actors` and `/pds/../pds/actors` are also caught. +fn same_directory(left: &str, right: &str) -> bool { + if left.trim_end_matches('/') == right.trim_end_matches('/') { + return true; + } + match (std::fs::canonicalize(left), std::fs::canonicalize(right)) { + (Ok(left), Ok(right)) => left == right, + _ => false, + } +} + #[cfg(test)] mod tests { use super::*; @@ -262,6 +290,8 @@ mod tests { "https://client.example", "--actor-store-dir", "/actors", + "--space-store-dir", + "/space-stores", "--mint-token", "token", "--daemon-service-did", @@ -320,6 +350,7 @@ mod tests { std::env::set_var("SPACEHOST_OAUTH_AUDIENCE", "did:web:pds.example"); std::env::set_var("SPACEHOST_OAUTH_CLIENT_IDS", "https://client.example"); std::env::set_var("SPACEHOST_ACTOR_STORE_DIR", "/actors"); + std::env::set_var("SPACEHOST_SPACE_STORE_DIR", "/space-stores"); std::env::set_var("SPACEHOST_MINT_TOKEN", "token"); std::env::set_var("SPACEHOST_DAEMON_SERVICE_DID", "did:plc:daemon"); std::env::set_var("SPACEHOST_APPVIEW_SERVICE_DID", "did:plc:appview"); @@ -341,6 +372,7 @@ mod tests { "SPACEHOST_OAUTH_AUDIENCE", "SPACEHOST_OAUTH_CLIENT_IDS", "SPACEHOST_ACTOR_STORE_DIR", + "SPACEHOST_SPACE_STORE_DIR", "SPACEHOST_MINT_TOKEN", "SPACEHOST_DAEMON_SERVICE_DID", "SPACEHOST_APPVIEW_SERVICE_DID", @@ -381,6 +413,8 @@ mod tests { "https://client.example", "--actor-store-dir", "/actors", + "--space-store-dir", + "/space-stores", "--mint-token", "token", "--daemon-service-did", @@ -391,6 +425,39 @@ mod tests { .unwrap() } + #[test] + fn the_space_store_must_not_be_the_pds_actor_store() { + let mut cfg = valid_unpinned(); + assert!(cfg.validate().is_ok()); + + // The PDS's actor directory is a key source, never a write target: it + // belongs to another writer and is mounted read-only in production. + cfg.space_store_dir = cfg.actor_store_dir.clone(); + let message = cfg.validate().expect_err("must refuse"); + assert!(message.contains("SPACEHOST_SPACE_STORE_DIR"), "{message}"); + + // Trailing-slash and traversal spellings of the same directory too. + cfg.space_store_dir = "/actors/".to_string(); + assert!(cfg.validate().is_err()); + + let existing = tempfile::tempdir().unwrap(); + let mut resolved = valid_unpinned(); + resolved.actor_store_dir = existing.path().display().to_string(); + resolved.space_store_dir = existing + .path() + .join("..") + .join(existing.path().file_name().unwrap()) + .display() + .to_string(); + assert!(resolved.validate().is_err()); + + // And it is required at all: silently defaulting it to the actor store + // is how the two jobs got conflated in the first place. + let mut missing = valid_unpinned(); + missing.space_store_dir = String::new(); + assert!(missing.validate().is_err()); + } + #[test] fn pinned_managing_app_needs_an_actor_store_service_key() { fn pinned_managing_app(policy: PolicyMode) -> Config { diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index 8311b16d..42da1b8f 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -123,7 +123,7 @@ async fn main() -> Result<(), Box> { did_cache: std::sync::Arc::new(MemoryCache::new(None, None)), }))); let store = Arc::new(SqliteStore::open(&cfg.db_path)?); - let repos = Arc::new(ActorStoreRepos::open(&cfg.actor_store_dir)?); + let repos = Arc::new(ActorStoreRepos::open(&cfg.space_store_dir)?); let seam = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); cfg.validate_pinned_service_key(|did| seam.key_path(did).is_some_and(|path| path.exists()))?; diff --git a/rsky-spaces-parity/src/bin/layer2_gate.rs b/rsky-spaces-parity/src/bin/layer2_gate.rs index 9b9d332c..5e8aff49 100644 --- a/rsky-spaces-parity/src/bin/layer2_gate.rs +++ b/rsky-spaces-parity/src/bin/layer2_gate.rs @@ -295,6 +295,7 @@ async fn main() -> Result<()> { let shim_dir = run_dir.join("shim"); let pds_actors = pds_dir.join("actors"); let shim_actors = shim_dir.join("actors"); + let shim_stores = shim_dir.join("space-stores"); for dir in [&pds_dir, &shim_dir, &pds_dir.join("blobs")] { std::fs::create_dir_all(dir)?; } @@ -432,9 +433,12 @@ async fn main() -> Result<()> { println!("space created on the oracle: {space}"); // The space host reads account signing keys from a PDS-shaped actor store - // directory and writes its own stores beside them. + // directory it never writes to, and writes its own per-account stores into + // a separate directory (Option A). Running both jobs off one path is the + // configuration that hid them being conflated, so the gate keeps them apart. copy_tree(&pds_actors, &shim_actors)?; - let accounts = reset_stores(&shim_actors)?; + let accounts = reset_stores(&shim_stores)?; + let keys_before = tree_digest(&shim_actors)?; println!("space host store directory prepared for {accounts} account(s)"); let shim_env: Vec<(String, String)> = vec![ @@ -452,6 +456,10 @@ async fn main() -> Result<()> { "SPACEHOST_ACTOR_STORE_DIR", shim_actors.display().to_string(), ), + ( + "SPACEHOST_SPACE_STORE_DIR", + shim_stores.display().to_string(), + ), ("SPACEHOST_OAUTH_ISSUER", OAUTH_ISSUER.to_string()), ("SPACEHOST_OAUTH_JWKS_URI", format!("{OAUTH_ISSUER}/jwks")), ("SPACEHOST_OAUTH_AUDIENCE", PDS_SERVICE_DID.to_string()), @@ -503,7 +511,12 @@ async fn main() -> Result<()> { shim.stop(); pds.stop(); - compare_stores(&pds_actors, &shim_actors, &mut board)?; + compare_stores(&pds_actors, &shim_stores, &mut board)?; + board.equal_if( + "key directory untouched", + tree_digest(&shim_actors)? == keys_before, + shim_actors.display().to_string(), + ); let report = board.render(); print!("{report}"); @@ -842,6 +855,39 @@ async fn probe_pds_only_surface(gate: &Gate, board: &mut Scoreboard) -> Result<( Ok(()) } +/// A content fingerprint of every file under `root`, so the gate can prove the +/// key directory is never written to. +fn tree_digest(root: &Path) -> Result> { + let mut out = BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + if !dir.exists() { + continue; + } + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_dir() { + stack.push(path); + continue; + } + let relative = path + .strip_prefix(root) + .unwrap_or(&path) + .display() + .to_string(); + let bytes = std::fs::read(&path)?; + let mut hash = 1469598103934665603u64; + for byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(1099511628211); + } + out.insert(relative, hash); + } + } + Ok(out) +} + fn compare_stores(pds_actors: &Path, shim_actors: &Path, board: &mut Scoreboard) -> Result<()> { let pds_store = rsky_space_host::actor_repos::store_path(pds_actors, AUTHOR_DID) .map_err(|error| anyhow::anyhow!("pds store path: {error}"))?; diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs index 396ecdf7..dd66502c 100644 --- a/rsky-spaces-parity/tests/parity.rs +++ b/rsky-spaces-parity/tests/parity.rs @@ -33,6 +33,92 @@ async fn actor_schema_matches_pinned_oracle() { assert_eq!(sqlite_master(&shim_path), sqlite_master(&pds_path)); } +/// S15: the shim's init is safe against a store the **PDS** created. +/// +/// Every other case here builds its own store tree, so none of them ever open +/// a file the oracle wrote first — which is how a build whose every live write +/// failed on `table repo_root already exists` passed this suite. Migration 001 +/// recreates the PDS base schema, so a PDS-created file must be adopted, not +/// re-migrated, and must come back byte-identical. +#[tokio::test] +async fn shim_init_adopts_a_pds_created_store_without_touching_it() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("pds-created.sqlite"); + get_migrated_db(&path).await.expect("pds migration"); + let before = sqlite_master(&path); + let before_bytes = std::fs::read(&path).expect("read"); + + // The shim opening it must succeed rather than re-running migration 001. + rsky_space_host::actor_schema::get_migrated_db(&path).expect("shim adopts the pds store"); + + let after = sqlite_master(&path); + for object in &before { + let (kind, name, _) = object; + assert!( + after.contains(object), + "shim init altered the pds object {kind} {name}" + ); + } + // Nothing pre-existing is rewritten, and the schema still carries the + // space tables the oracle put there. + assert_eq!(after, before, "shim init changed a pds-created schema"); + assert!( + after.iter().any(|(_, name, _)| name == "space_record"), + "the pds-created store is missing the space tables" + ); + // Opening it a second time is a no-op, so a restart is safe too. + let twice = std::fs::read(&path).expect("read"); + rsky_space_host::actor_schema::get_migrated_db(&path).expect("second open"); + assert_eq!(twice, std::fs::read(&path).expect("read")); + assert_ne!(before_bytes.len(), 0); +} + +/// S16: the shim writes to its own directory while reading keys from the PDS's. +/// +/// Option A (`Design/spaces-storage-parity.md`) keeps the two directories +/// separate; the suite only ever ran them as the same path, which is precisely +/// the configuration that hid one setting doing both jobs. +#[tokio::test] +async fn separated_key_and_store_directories_write_only_to_the_space_store() { + use rsky_space_host::pds_seam::PdsSeam; + + let temp = tempfile::tempdir().expect("tempdir"); + let actors = temp.path().join("pds-actors"); + let spaces = temp.path().join("space-stores"); + std::fs::create_dir_all(&actors).expect("actors dir"); + + // A PDS-shaped account: a key file and a store the PDS created. + let digest = ::digest(DID.as_bytes()); + let account = actors.join(&hex::encode(digest)[..2]).join(DID); + std::fs::create_dir_all(&account).expect("account dir"); + std::fs::write(account.join("key"), [7u8; 32]).expect("key"); + get_migrated_db(account.join("store.sqlite")) + .await + .expect("pds store"); + let pds_store_before = std::fs::read(account.join("store.sqlite")).expect("read"); + + let seam = PdsSeam::open(&actors).expect("seam opens the pds actors dir"); + seam.signer(DID).expect("signing key resolves"); + + let repos = ActorStoreRepos::open(&spaces).expect("space store opens"); + let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); + let write = create("s16", "written to the space store"); + repos + .apply_writes(&space.uri(), DID, "", &[write.shim()]) + .await + .expect("write lands in the space store"); + + // The write went to the shim's own file, and the PDS's is untouched. + let shim_path = repos.store_path(DID).expect("shim path"); + assert!(shim_path.starts_with(&spaces)); + assert!(shim_path.is_file()); + assert_eq!( + pds_store_before, + std::fs::read(account.join("store.sqlite")).expect("read"), + "the pds store must not be written to" + ); +} + const DID: &str = "did:plc:parityauthor"; const AUTHORITY: &str = "did:plc:parityauthority"; const COLLECTION: &str = "app.bsky.feed.post"; From 81a6a559821e28bd3bfe5172d689ff27316cb890 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Mon, 24 Aug 2026 18:47:39 -0400 Subject: [PATCH 47/56] fix(space-host): accept a high-S signature on client-presented JWSs WebCrypto emits a high-S ECDSA signature about half the time, so verifying a DPoP proof or access token under the repo-commit signature policy failed roughly every other request from a legitimate client. Verify those through shim-local helpers that pass allow_malleable_sig, and keep the strict verifiers for repo commits, where a malleable signature would be a second valid signature over the same content. --- Cargo.lock | 6 +- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/client_jws.rs | 130 ++++++++++++++++++++++++++++++ rsky-space-host/src/lib.rs | 1 + rsky-space-host/src/oauth.rs | 9 ++- 5 files changed, 140 insertions(+), 8 deletions(-) create mode 100644 rsky-space-host/src/client_jws.rs diff --git a/Cargo.lock b/Cargo.lock index f7799976..97def315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8354,7 +8354,7 @@ dependencies = [ "rsky-oauth 0.3.2", "rsky-repo 0.0.6", "rsky-space 0.4.2", - "rsky-space-host 0.7.5", + "rsky-space-host 0.7.6", "rsky-syntax 0.1.0", "rusqlite", "secp256k1", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.5" +version = "0.7.6" dependencies = [ "async-trait", "axum", @@ -8728,7 +8728,7 @@ dependencies = [ "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", - "rsky-space-host 0.7.5", + "rsky-space-host 0.7.6", "rusqlite", "secp256k1", "serde", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 942de094..0dc5bc61 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.5" +version = "0.7.6" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/client_jws.rs b/rsky-space-host/src/client_jws.rs new file mode 100644 index 00000000..3d9059b9 --- /dev/null +++ b/rsky-space-host/src/client_jws.rs @@ -0,0 +1,130 @@ +//! JWS verification for tokens a *client* presents: OAuth access tokens and +//! DPoP proofs. +//! +//! These are signed by browsers and SDKs, and WebCrypto emits a high-S ECDSA +//! signature about half the time. Such a signature is perfectly valid ECDSA, so +//! rejecting it fails roughly every other request from a legitimate client. +//! `rsky_space::jwk`'s verifiers are strict for a reason — they also serve repo +//! commits, where low-S is required because a malleable commit signature would +//! be a second valid signature over the same content — so these are separate +//! entry points rather than a relaxation of those. +//! +//! The asymmetry is the point: malleable here, strict for commits. + +use rsky_crypto::types::VerifyOptions; +use rsky_space::jwk::{EcJwk, CRV_P256, CRV_SECP256K1}; +use rsky_space::{Result, SpaceError}; +use sha2::{Digest, Sha256}; + +fn malleable() -> Option { + Some(VerifyOptions { + allow_malleable_sig: Some(true), + }) +} + +fn outcome(ok: bool) -> Result<()> { + if ok { + Ok(()) + } else { + Err(SpaceError::BadSignature) + } +} + +/// ES256 over a JWT signing input, accepting a high-S signature. +pub fn verify_client_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { + jwk.require_crv(CRV_P256)?; + let point = jwk.sec1_point()?; + let ok = rsky_crypto::p256::operations::verify_sig(&point, signing_input, sig, malleable()) + .map_err(|e| SpaceError::Crypto(e.to_string()))?; + outcome(ok) +} + +/// ES256K over a JWT signing input, accepting a high-S signature. +pub fn verify_client_es256k(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { + jwk.require_crv(CRV_SECP256K1)?; + let point = jwk.sec1_point()?; + let digest = Sha256::digest(signing_input); + let ok = rsky_crypto::secp256k1::operations::verify_sig(&point, &digest, sig, malleable()) + .map_err(|e| SpaceError::Crypto(e.to_string()))?; + outcome(ok) +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use base64::Engine; + use p256::ecdsa::signature::hazmat::PrehashSigner; + use p256::ecdsa::SigningKey; + + const SIGNING_INPUT: &[u8] = b"eyJhbGciOiJFUzI1NiJ9.eyJodG0iOiJQT1NUIn0"; + + /// A P-256 key plus the JWK a client would publish for it. + fn key_and_jwk(secret: [u8; 32]) -> (SigningKey, EcJwk) { + let key = SigningKey::from_slice(&secret).expect("key"); + let point = key.verifying_key().to_encoded_point(false); + let jwk = EcJwk { + kty: "EC".to_string(), + crv: CRV_P256.to_string(), + x: URL_SAFE_NO_PAD.encode(point.x().expect("x")), + y: URL_SAFE_NO_PAD.encode(point.y().expect("y")), + kid: None, + }; + (key, jwk) + } + + /// A deliberately high-S `r || s`, i.e. what WebCrypto may hand us. + fn high_s_signature(key: &SigningKey, input: &[u8]) -> Vec { + let digest = Sha256::digest(input); + let sig: p256::ecdsa::Signature = key.sign_prehash(&digest).expect("sign"); + // `normalize_s` yields Some only when it changed something, so this is + // the low-S form either way; its counterpart is then always high-S. + let low = sig.normalize_s().unwrap_or(sig); + malleable_counterpart(&low).to_bytes().to_vec() + } + + /// (r, n - s) — the other valid signature over the same message. + fn malleable_counterpart(sig: &p256::ecdsa::Signature) -> p256::ecdsa::Signature { + use p256::elliptic_curve::scalar::IsHigh; + let (r, s) = sig.split_scalars(); + let flipped = -*s; + assert!(bool::from(flipped.is_high()), "counterpart must be high-S"); + p256::ecdsa::Signature::from_scalars(*r, flipped).expect("signature") + } + + #[test] + fn high_s_es256_jws_signature_verifies() { + let (key, jwk) = key_and_jwk([0x31; 32]); + let sig = high_s_signature(&key, SIGNING_INPUT); + + // The whole point: a client's high-S proof is accepted here. + verify_client_es256(&jwk, SIGNING_INPUT, &sig).expect("high-S client jws must verify"); + + // And the strict path — the one repo commits go through — still refuses + // it. The asymmetry is deliberate, so assert both halves together. + assert!( + rsky_space::jwk::verify_es256(&jwk, SIGNING_INPUT, &sig).is_err(), + "the commit-path verifier must keep rejecting a high-S signature" + ); + } + + #[test] + fn a_low_s_signature_verifies_on_both_paths() { + let (key, jwk) = key_and_jwk([0x32; 32]); + let digest = Sha256::digest(SIGNING_INPUT); + let sig: p256::ecdsa::Signature = key.sign_prehash(&digest).expect("sign"); + let sig = sig.normalize_s().unwrap_or(sig); + let bytes = sig.to_bytes().to_vec(); + + verify_client_es256(&jwk, SIGNING_INPUT, &bytes).expect("client path"); + rsky_space::jwk::verify_es256(&jwk, SIGNING_INPUT, &bytes).expect("commit path"); + } + + #[test] + fn a_wrong_key_is_refused_on_the_client_path_too() { + let (key, _) = key_and_jwk([0x33; 32]); + let (_, other_jwk) = key_and_jwk([0x34; 32]); + let sig = high_s_signature(&key, SIGNING_INPUT); + assert!(verify_client_es256(&other_jwk, SIGNING_INPUT, &sig).is_err()); + } +} diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index dea5b5a1..1557b9f4 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -23,6 +23,7 @@ pub mod actor_schema; pub mod appaccess; pub mod attestation; pub mod authority; +pub mod client_jws; pub mod commits; pub mod config; pub mod convert; diff --git a/rsky-space-host/src/oauth.rs b/rsky-space-host/src/oauth.rs index ff141dd9..c7062107 100644 --- a/rsky-space-host/src/oauth.rs +++ b/rsky-space-host/src/oauth.rs @@ -15,9 +15,10 @@ //! `client_id` stands in for one. A token revoked before it expires still //! verifies until `exp`. +use crate::client_jws::{verify_client_es256, verify_client_es256k}; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; -use rsky_space::jwk::{verify_es256, verify_es256k, EcJwk, JwkSet}; +use rsky_space::jwk::{EcJwk, JwkSet}; use serde::Deserialize; use sha2::{Digest, Sha256}; @@ -325,7 +326,7 @@ async fn verify_as_signature( .first() .ok_or_else(|| auth_err("authorization server jwks is empty"))?, }; - verify_es256(jwk, &decoded.signing_input, &decoded.signature) + verify_client_es256(jwk, &decoded.signing_input, &decoded.signature) .map_err(|e| auth_err(format!("token signature: {e}"))) } @@ -355,8 +356,8 @@ async fn verify_dpop_proof( .as_ref() .ok_or_else(|| auth_err("proof carries no jwk"))?; let verified = match decoded.header.alg.as_str() { - ES256K => verify_es256k(jwk, &decoded.signing_input, &decoded.signature), - _ => verify_es256(jwk, &decoded.signing_input, &decoded.signature), + ES256K => verify_client_es256k(jwk, &decoded.signing_input, &decoded.signature), + _ => verify_client_es256(jwk, &decoded.signing_input, &decoded.signature), }; verified.map_err(|e| auth_err(format!("proof signature: {e}")))?; From fe76d115fbdfd90f9811c812f7bcedbaaf67b2d6 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Mon, 24 Aug 2026 19:04:10 -0400 Subject: [PATCH 48/56] fix(space-host): name a client JWS failure for what it is MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SpaceError::BadSignature reads "commit signature verification failed", which is accurate for a repo commit and misleading on a DPoP proof — where it kept surfacing and kept sending readers to look at commits. --- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/client_jws.rs | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 0dc5bc61..da4af76a 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.6" +version = "0.7.7" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/client_jws.rs b/rsky-space-host/src/client_jws.rs index 3d9059b9..7d5de890 100644 --- a/rsky-space-host/src/client_jws.rs +++ b/rsky-space-host/src/client_jws.rs @@ -13,8 +13,30 @@ use rsky_crypto::types::VerifyOptions; use rsky_space::jwk::{EcJwk, CRV_P256, CRV_SECP256K1}; -use rsky_space::{Result, SpaceError}; +use rsky_space::SpaceError; use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Distinct from `SpaceError::BadSignature`, whose message reads "commit +/// signature verification failed" — accurate for a repo commit, actively +/// misleading on a DPoP proof, which is where it kept surfacing. +#[derive(Debug, Error)] +pub enum ClientJwsError { + #[error("presented signature does not verify against the key in its header")] + BadSignature, + #[error("unusable key: {0}")] + Key(String), + #[error("verification failed: {0}")] + Crypto(String), +} + +type Result = std::result::Result; + +impl From for ClientJwsError { + fn from(error: SpaceError) -> Self { + ClientJwsError::Key(error.to_string()) + } +} fn malleable() -> Option { Some(VerifyOptions { @@ -26,7 +48,7 @@ fn outcome(ok: bool) -> Result<()> { if ok { Ok(()) } else { - Err(SpaceError::BadSignature) + Err(ClientJwsError::BadSignature) } } @@ -35,7 +57,7 @@ pub fn verify_client_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Res jwk.require_crv(CRV_P256)?; let point = jwk.sec1_point()?; let ok = rsky_crypto::p256::operations::verify_sig(&point, signing_input, sig, malleable()) - .map_err(|e| SpaceError::Crypto(e.to_string()))?; + .map_err(|e| ClientJwsError::Crypto(e.to_string()))?; outcome(ok) } @@ -45,7 +67,7 @@ pub fn verify_client_es256k(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Re let point = jwk.sec1_point()?; let digest = Sha256::digest(signing_input); let ok = rsky_crypto::secp256k1::operations::verify_sig(&point, &digest, sig, malleable()) - .map_err(|e| SpaceError::Crypto(e.to_string()))?; + .map_err(|e| ClientJwsError::Crypto(e.to_string()))?; outcome(ok) } From 9852e68cef8c3c5d5e760e5c57ea47e2ce0b07fb Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Mon, 24 Aug 2026 22:42:50 -0400 Subject: [PATCH 49/56] fix(space-host): normalise a client JWS signature on secp256k1 allow_malleable_sig only waives the encoding check on this curve, and libsecp256k1 refuses a high-S signature regardless, so roughly every other DPoP proof from a client whose key is secp256k1 was rejected. The DER path is left as it was. Tests now pin the asymmetry on both curves: a client's high-S proof verifies, and the commit path still refuses it. --- Cargo.lock | 6 +- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/client_jws.rs | 93 ++++++++++++++++++++++++++++++- 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 97def315..d0d32ca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8354,7 +8354,7 @@ dependencies = [ "rsky-oauth 0.3.2", "rsky-repo 0.0.6", "rsky-space 0.4.2", - "rsky-space-host 0.7.6", + "rsky-space-host 0.7.8", "rsky-syntax 0.1.0", "rusqlite", "secp256k1", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.6" +version = "0.7.8" dependencies = [ "async-trait", "axum", @@ -8728,7 +8728,7 @@ dependencies = [ "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", - "rsky-space-host 0.7.6", + "rsky-space-host 0.7.8", "rusqlite", "secp256k1", "serde", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index da4af76a..27231a0c 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.7" +version = "0.7.8" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/client_jws.rs b/rsky-space-host/src/client_jws.rs index 7d5de890..96cf3722 100644 --- a/rsky-space-host/src/client_jws.rs +++ b/rsky-space-host/src/client_jws.rs @@ -62,12 +62,28 @@ pub fn verify_client_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Res } /// ES256K over a JWT signing input, accepting a high-S signature. +/// +/// `allow_malleable_sig` is not enough on this curve: it only waives the +/// encoding check, and libsecp256k1 refuses a high-S signature regardless. So +/// normalise the scalar first. The P-256 verifier needs no equivalent because +/// RustCrypto's does not enforce low-S in the first place. pub fn verify_client_es256k(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { jwk.require_crv(CRV_SECP256K1)?; let point = jwk.sec1_point()?; let digest = Sha256::digest(signing_input); - let ok = rsky_crypto::secp256k1::operations::verify_sig(&point, &digest, sig, malleable()) - .map_err(|e| ClientJwsError::Crypto(e.to_string()))?; + // A JWS signature is the fixed-width `r || s`, but DER has been accepted + // here historically; leave that path exactly as it was. + let normalised = match secp256k1::ecdsa::Signature::from_compact(sig) { + Ok(mut parsed) => { + parsed.normalize_s(); + Some(parsed.serialize_compact()) + } + Err(_) => None, + }; + let presented = normalised.as_ref().map(|s| s.as_slice()).unwrap_or(sig); + let ok = + rsky_crypto::secp256k1::operations::verify_sig(&point, &digest, presented, malleable()) + .map_err(|e| ClientJwsError::Crypto(e.to_string()))?; outcome(ok) } @@ -142,6 +158,79 @@ mod tests { rsky_space::jwk::verify_es256(&jwk, SIGNING_INPUT, &bytes).expect("commit path"); } + /// A secp256k1 key plus the JWK a client publishes for it. + fn k256_key_and_jwk(secret: [u8; 32]) -> (secp256k1::SecretKey, EcJwk) { + let secret = secp256k1::SecretKey::from_slice(&secret).expect("key"); + let public = secp256k1::PublicKey::from_secret_key(&secp256k1::Secp256k1::new(), &secret); + let uncompressed = public.serialize_uncompressed(); + let jwk = EcJwk { + kty: "EC".to_string(), + crv: CRV_SECP256K1.to_string(), + x: URL_SAFE_NO_PAD.encode(&uncompressed[1..33]), + y: URL_SAFE_NO_PAD.encode(&uncompressed[33..65]), + kid: None, + }; + (secret, jwk) + } + + /// A deliberately high-S ES256K signature: what the OAuth client emits + /// about half the time. + fn high_s_k256_signature(secret: &secp256k1::SecretKey, input: &[u8]) -> Vec { + let digest = Sha256::digest(input); + let message = secp256k1::Message::from_digest_slice(&digest).expect("digest"); + let signature = secp256k1::Secp256k1::new().sign_ecdsa(&message, secret); + // rust-secp256k1 always hands back the normalised form, so flip it. + let mut bytes = signature.serialize_compact(); + let n = [ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, + 0xD0, 0x36, 0x41, 0x41, + ]; + let mut borrow = 0i16; + for i in (0..32).rev() { + let diff = n[i] as i16 - bytes[32 + i] as i16 - borrow; + if diff < 0 { + bytes[32 + i] = (diff + 256) as u8; + borrow = 1; + } else { + bytes[32 + i] = diff as u8; + borrow = 0; + } + } + bytes.to_vec() + } + + #[test] + fn high_s_es256k_jws_signature_verifies() { + let (secret, jwk) = k256_key_and_jwk([0x51; 32]); + let sig = high_s_k256_signature(&secret, SIGNING_INPUT); + + // The client's high-S proof is accepted here... + verify_client_es256k(&jwk, SIGNING_INPUT, &sig) + .expect("high-S client jws must verify on secp256k1"); + + // ...and the commit-path verifier still refuses it. Both curves are + // asserted because only the P-256 half was ever covered. + assert!( + rsky_space::jwk::verify_es256k(&jwk, SIGNING_INPUT, &sig).is_err(), + "the commit-path verifier must keep rejecting a high-S signature" + ); + } + + #[test] + fn a_low_s_es256k_signature_verifies_on_both_paths() { + let (secret, jwk) = k256_key_and_jwk([0x52; 32]); + let digest = Sha256::digest(SIGNING_INPUT); + let message = secp256k1::Message::from_digest_slice(&digest).expect("digest"); + let sig = secp256k1::Secp256k1::new() + .sign_ecdsa(&message, &secret) + .serialize_compact() + .to_vec(); + + verify_client_es256k(&jwk, SIGNING_INPUT, &sig).expect("client path"); + rsky_space::jwk::verify_es256k(&jwk, SIGNING_INPUT, &sig).expect("commit path"); + } + #[test] fn a_wrong_key_is_refused_on_the_client_path_too() { let (key, _) = key_and_jwk([0x33; 32]); From a8fd66b894284965e89a732ee661d008e4674eaf Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Tue, 25 Aug 2026 00:15:46 -0400 Subject: [PATCH 50/56] fix(daemon): retry an admission denial on its own slow budget A projectRecords 401 means the author is not admitted to the space, which is state, not a bad batch: it flips when a membership write propagates. Three fast attempts then dead-lettered the batch permanently, so a moment of FGA lag left a record unprojected for good. Denials now get their own error class, their own durable counter beside the poison one, and a slow lane: re-attempted once per sweep, parked after DAEMON_DENIAL_PARK_AFTER sweeps so a legitimate refusal still stops. A denied author's later batches wait behind the denied one, or the cursor would advance past it. Poison keeps its fast dead-letter unchanged. --- Cargo.lock | 4 +- rsky-daemon/Cargo.toml | 2 +- rsky-daemon/src/config.rs | 13 ++ rsky-daemon/src/error.rs | 10 ++ rsky-daemon/src/feeds.rs | 18 +++ rsky-daemon/src/index.rs | 92 ++++++++++--- rsky-daemon/src/journal.rs | 237 +++++++++++++++++++++++++++++++- rsky-daemon/src/lib.rs | 2 +- rsky-daemon/src/main.rs | 24 ++-- rsky-daemon/src/runner.rs | 4 +- rsky-daemon/src/sqlite_index.rs | 180 +++++++++++++++++++++++- 11 files changed, 547 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0d32ca5..f85031af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "rsky-daemon" -version = "0.6.0" +version = "0.6.1" dependencies = [ "async-trait", "axum", @@ -8716,7 +8716,7 @@ dependencies = [ [[package]] name = "rsky-spaces-parity" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index e9a6ba42..61c7639c 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-daemon" -version = "0.6.0" +version = "0.6.1" authors = ["Rudy Fraser "] description = "atproto permissioned-data syncer daemon: pulls, verifies, and indexes permissioned repos from members' PDSes" edition = "2021" diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index 95a5ee67..08bba700 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -121,6 +121,16 @@ pub struct Config { #[arg(long, env = "DAEMON_SWEEP_INTERVAL_SECS", default_value_t = 300)] pub sweep_interval_secs: u64, + /// Sweeps a batch refused with "author is not admitted" is re-attempted + /// for before it is parked. Admission can arrive late; it can also never + /// arrive, so the retry is slow but finite. + #[arg( + long, + env = "DAEMON_DENIAL_PARK_AFTER", + default_value_t = crate::journal::DENIAL_PARK_AFTER + )] + pub denial_park_after: u32, + /// PLC directory to resolve DIDs against. Empty uses the public one, /// which cannot know about a local or staging network. #[arg(long, env = "DAEMON_PLC_URL", default_value = "")] @@ -206,6 +216,7 @@ mod tests { let cfg = Config::try_parse_from(REQUIRED).unwrap(); assert_eq!(cfg.sweep_interval_secs, 300); + assert_eq!(cfg.denial_park_after, crate::journal::DENIAL_PARK_AFTER); assert_eq!(cfg.index_db_path, ""); assert_eq!(cfg.notify_bind, "127.0.0.1:8055"); assert_eq!(cfg.repo_host_url(), "https://host.example"); @@ -241,6 +252,7 @@ mod tests { ("DAEMON_NOTIFY_BIND", "0.0.0.0:9000"), ("DAEMON_INDEX_DB_PATH", "/data/space.sqlite"), ("DAEMON_SWEEP_INTERVAL_SECS", "60"), + ("DAEMON_DENIAL_PARK_AFTER", "7"), ("DAEMON_PLC_URL", "http://localhost:2582"), ("DAEMON_FEEDS_URL", "http://localhost:8080"), ("DAEMON_FEEDS_SERVICE_DID", "did:web:feeds.example"), @@ -264,6 +276,7 @@ mod tests { assert_eq!(cfg.notify_endpoint(), "http://0.0.0.0:9000"); assert_eq!(cfg.index_db_path, "/data/space.sqlite"); assert_eq!(cfg.sweep_interval_secs, 60); + assert_eq!(cfg.denial_park_after, 7); assert_eq!( cfg.feeds_projection(), Some(("http://localhost:8080", "did:web:feeds.example")) diff --git a/rsky-daemon/src/error.rs b/rsky-daemon/src/error.rs index b09f4f62..227af00f 100644 --- a/rsky-daemon/src/error.rs +++ b/rsky-daemon/src/error.rs @@ -21,6 +21,12 @@ pub enum DaemonError { /// budget: an outage is not a bad batch. #[error("projection destination unavailable: {0}")] RetryableProjection(String), + /// A projection destination refused the batch because its author is not + /// admitted to the space. Admission is state, not a property of the batch: + /// it flips when a membership write propagates, so this must not spend the + /// poison budget. It gets its own slow budget instead. + #[error("projection destination denied admission: {0}")] + AdmissionDenied(String), #[error(transparent)] Space(#[from] rsky_space::SpaceError), } @@ -29,6 +35,10 @@ impl DaemonError { pub fn is_retryable_projection(&self) -> bool { matches!(self, Self::RetryableProjection(_)) } + + pub fn is_admission_denied(&self) -> bool { + matches!(self, Self::AdmissionDenied(_)) + } } pub type Result = std::result::Result; diff --git a/rsky-daemon/src/feeds.rs b/rsky-daemon/src/feeds.rs index 1a2bb664..54484e2a 100644 --- a/rsky-daemon/src/feeds.rs +++ b/rsky-daemon/src/feeds.rs @@ -149,6 +149,9 @@ impl ProjectionIngress for HttpProjectionIngress { if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { return Err(DaemonError::RetryableProjection(message)); } + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(DaemonError::AdmissionDenied(message)); + } Err(DaemonError::Xrpc(message)) } } @@ -463,6 +466,21 @@ pub(crate) mod tests { .await .unwrap_err(); assert!(!error.is_retryable_projection()); + assert!(!error.is_admission_denied()); + + let denying = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(401).set_body_string( + r#"{"error":"NotAuthorized","message":"author is not admitted to this space"}"#, + )) + .mount(&denying) + .await; + let error = ingress(&denying) + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err(); + assert!(error.is_admission_denied()); + assert!(!error.is_retryable_projection()); let unreachable = HttpProjectionIngress::new( "feeds", diff --git a/rsky-daemon/src/index.rs b/rsky-daemon/src/index.rs index 2b1543bd..01b9911e 100644 --- a/rsky-daemon/src/index.rs +++ b/rsky-daemon/src/index.rs @@ -52,6 +52,10 @@ pub struct JournaledBatch { pub author: String, pub rev: String, pub mutations: Vec, + /// How many times this projector has been told the author is not admitted. + /// Non-zero puts the batch in the slow lane: retried once per sweep, never + /// on the fast drain. + pub denials: u32, } /// Per-author sync state + records the daemon holds for a space. @@ -112,9 +116,23 @@ pub trait SpaceIndex: Send + Sync { ) -> Result { Ok(0) } + /// Returns the durable denial count for this batch, parking it once the + /// count reaches `park_after`. Denials are counted separately from the + /// poison budget: a denial says the space's admission state is not (yet) + /// what the batch needs, not that the batch is bad. + async fn record_projection_denial( + &self, + _projector: &str, + _did: &str, + _rev: &str, + _error: &str, + _park_after: u32, + ) -> Result { + Ok(0) + } /// Drop journal rows every one of `projectors` has advanced past, along - /// with their retryable failure rows. Dead-lettered batches are retained - /// until explicitly cleared. + /// with their retryable failure rows. Dead-lettered and parked batches are + /// retained until explicitly cleared. async fn prune_journal(&self, _projectors: &[&str]) -> Result { Ok(0) } @@ -137,11 +155,25 @@ struct AuthorState { records: HashMap, } +#[derive(Default, Clone, Copy)] +struct FailureState { + attempts: u32, + dead_lettered: bool, + denials: u32, + parked: bool, +} + +impl FailureState { + fn retired(&self) -> bool { + self.dead_lettered || self.parked + } +} + #[derive(Default)] struct JournalState { batches: Vec, cursors: HashMap<(String, String), String>, - failures: HashMap<(String, String, String), (u32, bool)>, + failures: HashMap<(String, String, String), FailureState>, } /// In-memory [`SpaceIndex`] for tests and local runs. @@ -255,12 +287,20 @@ impl SpaceIndex for InMemoryIndex { author: did.to_string(), rev: rev.to_string(), mutations: mutations.to_vec(), + denials: 0, }); Ok(()) } async fn pending_batches(&self, projector: &str) -> Result> { let journal = self.journal.read().unwrap(); + let failure = |b: &JournaledBatch| { + journal + .failures + .get(&(projector.to_string(), b.author.clone(), b.rev.clone())) + .copied() + .unwrap_or_default() + }; let mut pending: Vec = journal .batches .iter() @@ -269,12 +309,12 @@ impl SpaceIndex for InMemoryIndex { .cursors .get(&(projector.to_string(), b.author.clone())) .is_none_or(|cursor| b.rev > *cursor) - && !journal - .failures - .get(&(projector.to_string(), b.author.clone(), b.rev.clone())) - .is_some_and(|(_, dead)| *dead) + && !failure(b).retired() + }) + .map(|b| JournaledBatch { + denials: failure(b).denials, + ..b.clone() }) - .cloned() .collect(); pending.sort_by(|a, b| (&a.author, &a.rev).cmp(&(&b.author, &b.rev))); Ok(pending) @@ -301,10 +341,28 @@ impl SpaceIndex for InMemoryIndex { let entry = journal .failures .entry((projector.to_string(), did.to_string(), rev.to_string())) - .or_insert((0, false)); - entry.0 += 1; - entry.1 = entry.0 >= dead_letter_after; - Ok(entry.0) + .or_default(); + entry.attempts += 1; + entry.dead_lettered = entry.attempts >= dead_letter_after; + Ok(entry.attempts) + } + + async fn record_projection_denial( + &self, + projector: &str, + did: &str, + rev: &str, + _error: &str, + park_after: u32, + ) -> Result { + let mut journal = self.journal.write().unwrap(); + let entry = journal + .failures + .entry((projector.to_string(), did.to_string(), rev.to_string())) + .or_default(); + entry.denials += 1; + entry.parked = entry.denials >= park_after; + Ok(entry.denials) } async fn prune_journal(&self, projectors: &[&str]) -> Result { @@ -324,16 +382,16 @@ impl SpaceIndex for InMemoryIndex { .get(&(projector.to_string(), b.author.clone())) .is_some_and(|cursor| *cursor >= b.rev) }); - let dead_lettered = projectors.iter().any(|projector| { + let retired = projectors.iter().any(|projector| { failures .get(&(projector.to_string(), b.author.clone(), b.rev.clone())) - .is_some_and(|(_, dead)| *dead) + .is_some_and(FailureState::retired) }); - !all_passed || dead_lettered + !all_passed || retired }); let pruned = before - batches.len(); - failures.retain(|(_, author, rev), (_, dead)| { - *dead || batches.iter().any(|b| b.author == *author && b.rev == *rev) + failures.retain(|(_, author, rev), state| { + state.retired() || batches.iter().any(|b| b.author == *author && b.rev == *rev) }); Ok(pruned) } diff --git a/rsky-daemon/src/journal.rs b/rsky-daemon/src/journal.rs index bacba71c..00d4f08a 100644 --- a/rsky-daemon/src/journal.rs +++ b/rsky-daemon/src/journal.rs @@ -1,5 +1,6 @@ //! Journal-driven projection delivery. +use std::collections::HashSet; use std::sync::Arc; use crate::error::Result; @@ -8,6 +9,15 @@ use crate::projection::Projector; use crate::router::Router; pub const DEAD_LETTER_AFTER: u32 = 3; +pub const DENIAL_PARK_AFTER: u32 = 20; + +/// Which batches a drain pass will attempt. Denied batches wait for a sweep: +/// admission changes at the pace of a membership write, not of a drain tick. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Lane { + Fast, + Sweep, +} /// Reads the batch journal for one projector and advances only after its /// destination has accepted the batch. Each projector gets an independent @@ -18,6 +28,7 @@ pub struct JournalConsumer { router: Router, projector: Box, dead_letter_after: u32, + denial_park_after: u32, } impl JournalConsumer { @@ -27,6 +38,7 @@ impl JournalConsumer { router, projector, dead_letter_after: DEAD_LETTER_AFTER, + denial_park_after: DENIAL_PARK_AFTER, } } @@ -35,14 +47,31 @@ impl JournalConsumer { self } + pub fn with_denial_park_after(mut self, sweeps: u32) -> Self { + self.denial_park_after = sweeps; + self + } + pub fn name(&self) -> &'static str { self.name } - async fn drain_with_status(&self, index: &dyn SpaceIndex) -> Result<(usize, bool)> { + async fn drain_with_status(&self, index: &dyn SpaceIndex, lane: Lane) -> Result<(usize, bool)> { let mut delivered = 0; let mut succeeded = true; + // An author whose batch was denied keeps its remaining batches in + // order: delivering a later one would advance the cursor past the + // denied batch and lose it for good once admission arrives. + let mut denied_authors: HashSet = HashSet::new(); for batch in index.pending_batches(self.name).await? { + if denied_authors.contains(&batch.author) { + continue; + } + if batch.denials > 0 && lane == Lane::Fast { + succeeded = false; + denied_authors.insert(batch.author.clone()); + continue; + } let events = self.router.route_batch(&batch.author, &batch.mutations); let result = if events.is_empty() { Ok(()) @@ -64,6 +93,24 @@ impl JournalConsumer { tracing::warn!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, error = %error, "projection destination unavailable; batch remains pending without consuming its failure budget"); continue; } + if error.is_admission_denied() { + let denials = index + .record_projection_denial( + self.name, + &batch.author, + &batch.rev, + &error.to_string(), + self.denial_park_after, + ) + .await?; + denied_authors.insert(batch.author.clone()); + if denials >= self.denial_park_after { + tracing::error!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, denials, error = %error, "projection batch parked: admission never granted"); + } else { + tracing::warn!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, denials, error = %error, "author not admitted; batch retries on a later sweep"); + } + continue; + } let attempts = index .record_projection_failure( self.name, @@ -85,13 +132,26 @@ impl JournalConsumer { } pub async fn drain(&self, index: &dyn SpaceIndex) -> Result { - self.drain_with_status(index) + self.drain_with_status(index, Lane::Fast) + .await + .map(|(delivered, _)| delivered) + } + + /// A drain that also re-attempts denied batches. Called once per sweep. + pub async fn drain_sweep(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index, Lane::Sweep) .await .map(|(delivered, _)| delivered) } pub async fn drain_succeeded(&self, index: &dyn SpaceIndex) -> Result { - self.drain_with_status(index) + self.drain_with_status(index, Lane::Fast) + .await + .map(|(_, succeeded)| succeeded) + } + + pub async fn drain_sweep_succeeded(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index, Lane::Sweep) .await .map(|(_, succeeded)| succeeded) } @@ -102,9 +162,26 @@ pub type SharedJournalConsumer = Arc; /// Drain every projector, then drop the journal rows all of them have passed. /// Returns whether every destination accepted everything pending for it. pub async fn drain_all(index: &dyn SpaceIndex, consumers: &[SharedJournalConsumer]) -> bool { + drain_all_in(index, consumers, Lane::Fast).await +} + +/// [`drain_all`] plus a re-attempt of every denied batch. +pub async fn drain_all_sweep(index: &dyn SpaceIndex, consumers: &[SharedJournalConsumer]) -> bool { + drain_all_in(index, consumers, Lane::Sweep).await +} + +async fn drain_all_in( + index: &dyn SpaceIndex, + consumers: &[SharedJournalConsumer], + lane: Lane, +) -> bool { let mut succeeded = true; for consumer in consumers { - match consumer.drain_succeeded(index).await { + let drained = match lane { + Lane::Fast => consumer.drain_succeeded(index).await, + Lane::Sweep => consumer.drain_sweep_succeeded(index).await, + }; + match drained { Ok(clean) => succeeded &= clean, Err(error) => { succeeded = false; @@ -137,6 +214,7 @@ mod tests { const AUTHORITY: &str = "did:plc:community"; const AUTHOR: &str = "did:plc:alice"; + const BOB: &str = "did:plc:bob"; fn router() -> Router { Router::new( @@ -166,6 +244,9 @@ mod tests { delivered: Mutex>, fail_next: AtomicUsize, retryable: bool, + /// Authors this destination refuses admission; cleared to admit them. + denied: Mutex>, + attempts: Mutex>, } #[async_trait] @@ -173,7 +254,16 @@ mod tests { fn name(&self) -> &'static str { "recorder" } - async fn project(&self, _did: &str, rev: &str, events: &[SyncEvent]) -> Result<()> { + async fn project(&self, did: &str, rev: &str, events: &[SyncEvent]) -> Result<()> { + self.attempts + .lock() + .unwrap() + .push((did.to_string(), rev.to_string())); + if self.denied.lock().unwrap().iter().any(|d| d == did) { + return Err(DaemonError::AdmissionDenied( + "appview projectRecords returned 401 Unauthorized".to_string(), + )); + } if self.fail_next.load(Ordering::SeqCst) > 0 { self.fail_next.fetch_sub(1, Ordering::SeqCst); return Err(if self.retryable { @@ -190,6 +280,31 @@ mod tests { } } + /// A handle onto a [`Recorder`] the test keeps, so admission can flip + /// between drains. + struct Shared(Arc); + + #[async_trait] + impl Projector for Shared { + fn name(&self) -> &'static str { + "recorder" + } + async fn project(&self, did: &str, rev: &str, events: &[SyncEvent]) -> Result<()> { + self.0.project(did, rev, events).await + } + } + + fn shared_consumer(recorder: &Arc) -> JournalConsumer { + JournalConsumer::new(router(), Box::new(Shared(recorder.clone()))) + } + + async fn journal(index: &InMemoryIndex, author: &str, rev: &str, rkey: &str) { + index + .journal_batch(author, rev, &[post_mutation(rkey)]) + .await + .unwrap(); + } + #[tokio::test] async fn a_delivered_batch_advances_its_cursor_once() { let index = InMemoryIndex::new(); @@ -289,4 +404,116 @@ mod tests { assert!(index.pending_batches("recorder").await.unwrap().is_empty()); assert_eq!(index.pending_batches("stalled").await.unwrap().len(), 1); } + + #[tokio::test] + async fn a_denied_batch_projects_once_when_admission_arrives() { + let index = InMemoryIndex::new(); + journal(&index, BOB, "3krev", "3ka").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder).with_denial_park_after(5); + + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + assert_eq!(index.pending_batches("recorder").await.unwrap().len(), 1); + assert_eq!(recorder.attempts.lock().unwrap().len(), 1); + + // The fast lane leaves a denied batch alone: admission moves at the + // pace of a membership write, not of a drain tick. + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + assert_eq!(recorder.attempts.lock().unwrap().len(), 1); + + recorder.denied.lock().unwrap().clear(); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 1); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + assert_eq!(recorder.delivered.lock().unwrap().len(), 1); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_denial_that_never_lifts_parks_after_its_budget() { + let index = InMemoryIndex::new(); + journal(&index, BOB, "3krev", "3ka").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder).with_denial_park_after(3); + + for _ in 0..3 { + assert!(!consumer.drain_sweep_succeeded(&index).await.unwrap()); + } + assert_eq!(recorder.attempts.lock().unwrap().len(), 3); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + + for _ in 0..2 { + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + } + assert_eq!( + recorder.attempts.lock().unwrap().len(), + 3, + "a parked batch is never attempted again" + ); + } + + #[tokio::test] + async fn a_denial_does_not_spend_the_poison_budget() { + let index = InMemoryIndex::new(); + journal(&index, BOB, "3krev", "3ka").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder) + .with_dead_letter_after(1) + .with_denial_park_after(10); + + for _ in 0..4 { + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + } + assert_eq!( + index.pending_batches("recorder").await.unwrap().len(), + 1, + "the one-attempt poison budget must not dead-letter a denial" + ); + + // The same budget still kills a genuinely poisoned batch on its first + // failure. + let index = InMemoryIndex::new(); + journal(&index, AUTHOR, "3krev", "3ka").await; + let poison = Recorder::default(); + poison.fail_next.store(5, Ordering::SeqCst); + let consumer = JournalConsumer::new(router(), Box::new(poison)).with_dead_letter_after(1); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_denial_holds_its_author_in_order_and_leaves_others_alone() { + let index = InMemoryIndex::new(); + journal(&index, AUTHOR, "3krev1", "3ka").await; + journal(&index, AUTHOR, "3krev2", "3kb").await; + journal(&index, BOB, "3krev3", "3kc").await; + journal(&index, BOB, "3krev4", "3kd").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder).with_denial_park_after(5); + + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 2); + assert_eq!( + *recorder.attempts.lock().unwrap(), + vec![ + (AUTHOR.to_string(), "3krev1".to_string()), + (AUTHOR.to_string(), "3krev2".to_string()), + (BOB.to_string(), "3krev3".to_string()), + ], + "bob's later batch must not overtake his denied one" + ); + + recorder.denied.lock().unwrap().clear(); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 2); + let delivered: Vec = recorder + .delivered + .lock() + .unwrap() + .iter() + .map(|(rev, _)| rev.clone()) + .collect(); + assert_eq!(delivered, vec!["3krev1", "3krev2", "3krev3", "3krev4"]); + } } diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index 6181924c..63460882 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -50,7 +50,7 @@ pub use feeds::{ ProjectionOperation, SpaceLifecycleAcker, }; pub use index::{InMemoryIndex, IndexMutation, JournaledBatch, SpaceIndex}; -pub use journal::{drain_all, JournalConsumer, SharedJournalConsumer}; +pub use journal::{drain_all, drain_all_sweep, JournalConsumer, SharedJournalConsumer}; pub use notify::{router as notify_router, NotifyState, WriteNotice}; pub use projection::Projector; pub use recovery::recover_repo; diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 262af6bc..3bc54684 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -70,6 +70,7 @@ impl CommitKeyResolver for DidKeyResolver { struct ProjectionConfig { service_identity: String, signing_key_hex: String, + denial_park_after: u32, feeds: Option<(String, String)>, appview: Option<(String, String)>, } @@ -96,10 +97,13 @@ impl ProjectionConfig { &self.signing_key_hex, )?); acker = Some(ingress.clone()); - consumers.push(Arc::new(JournalConsumer::new( - Router::new(space_id.clone(), space_id.authority.clone()), - Box::new(FeedsProjector::new(ingress, space)), - ))); + consumers.push(Arc::new( + JournalConsumer::new( + Router::new(space_id.clone(), space_id.authority.clone()), + Box::new(FeedsProjector::new(ingress, space)), + ) + .with_denial_park_after(self.denial_park_after), + )); } if let Some((url, audience)) = &self.appview { let ingress = HttpProjectionIngress::new( @@ -109,10 +113,13 @@ impl ProjectionConfig { audience, &self.signing_key_hex, )?; - consumers.push(Arc::new(JournalConsumer::new( - Router::new(space_id.clone(), space_id.authority.clone()), - Box::new(AppviewProjector::new(ingress, space)), - ))); + consumers.push(Arc::new( + JournalConsumer::new( + Router::new(space_id.clone(), space_id.authority.clone()), + Box::new(AppviewProjector::new(ingress, space)), + ) + .with_denial_park_after(self.denial_park_after), + )); } Ok((consumers, acker)) } @@ -218,6 +225,7 @@ async fn main() -> std::result::Result<(), Box> { let projection = ProjectionConfig { service_identity: cfg.service_identity.clone(), signing_key_hex: cfg.service_signing_key_hex.clone(), + denial_park_after: cfg.denial_park_after, feeds: cfg .feeds_projection() .map(|(url, aud)| (url.to_string(), aud.to_string())), diff --git a/rsky-daemon/src/runner.rs b/rsky-daemon/src/runner.rs index f1fd736b..7373edab 100644 --- a/rsky-daemon/src/runner.rs +++ b/rsky-daemon/src/runner.rs @@ -14,7 +14,7 @@ use crate::engine::{sync_repo, CommitKeyResolver, SyncOutcome}; use crate::error::{DaemonError, Result}; use crate::feeds::SpaceLifecycleAcker; use crate::index::SpaceIndex; -use crate::journal::{drain_all, SharedJournalConsumer}; +use crate::journal::{drain_all, drain_all_sweep, SharedJournalConsumer}; use crate::notify::WriteNotice; use crate::recovery::recover_repo; use crate::repohost::RepoHostClient; @@ -317,7 +317,7 @@ pub async fn run( keys.as_ref(), ) .await; - let projected = drain_all(index.as_ref(), &projectors).await; + let projected = drain_all_sweep(index.as_ref(), &projectors).await; if swept && projected && !acknowledged { if let Some(acker) = &lifecycle_acker { match acker.acknowledge_sync(&opts.space_uri, opts.generation).await { diff --git a/rsky-daemon/src/sqlite_index.rs b/rsky-daemon/src/sqlite_index.rs index 5aadc8b7..6426cedc 100644 --- a/rsky-daemon/src/sqlite_index.rs +++ b/rsky-daemon/src/sqlite_index.rs @@ -51,10 +51,16 @@ CREATE TABLE IF NOT EXISTS projection_failure ( attempts INTEGER NOT NULL, last_error TEXT NOT NULL, dead_lettered INTEGER NOT NULL DEFAULT 0, + denials INTEGER NOT NULL DEFAULT 0, + parked INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (projector, space_uri, did, rev) ); "; +/// Columns added to `projection_failure` after its first release; an index +/// created by an earlier build has the table but not these. +const FAILURE_COLUMNS: [&str; 2] = ["denials", "parked"]; + fn db_err(e: rusqlite::Error) -> DaemonError { DaemonError::Index(e.to_string()) } @@ -72,11 +78,40 @@ impl SqliteIndex { conn.pragma_update(None, "synchronous", "NORMAL") .map_err(db_err)?; conn.execute_batch(SCHEMA).map_err(db_err)?; + Self::add_missing_failure_columns(&conn)?; Ok(Self { conn: Mutex::new(conn), }) } + fn add_missing_failure_columns(conn: &Connection) -> Result<()> { + let mut present = Vec::new(); + { + let mut stmt = conn + .prepare("SELECT name FROM pragma_table_info('projection_failure')") + .map_err(db_err)?; + let rows = stmt + .query_map([], |row| row.get::<_, String>(0)) + .map_err(db_err)?; + for row in rows { + present.push(row.map_err(db_err)?); + } + } + for column in FAILURE_COLUMNS { + if !present.iter().any(|name| name == column) { + conn.execute( + &format!( + "ALTER TABLE projection_failure + ADD COLUMN {column} INTEGER NOT NULL DEFAULT 0" + ), + [], + ) + .map_err(db_err)?; + } + } + Ok(()) + } + /// A [`SpaceIndex`] handle scoped to one space. pub fn for_space(self: &Arc, space_uri: impl Into) -> SpaceScopedIndex { SpaceScopedIndex { @@ -199,7 +234,8 @@ impl SpaceIndex for SpaceScopedIndex { let conn = self.db.conn.lock().unwrap(); let mut stmt = conn .prepare( - "SELECT j.did, j.rev, j.mutations FROM projection_journal j + "SELECT j.did, j.rev, j.mutations, COALESCE(f.denials, 0) + FROM projection_journal j LEFT JOIN projector_cursor c ON c.projector = ?1 AND c.space_uri = j.space_uri AND c.did = j.did LEFT JOIN projection_failure f @@ -207,6 +243,7 @@ impl SpaceIndex for SpaceScopedIndex { WHERE j.space_uri = ?2 AND (c.rev IS NULL OR j.rev > c.rev) AND COALESCE(f.dead_lettered, 0) = 0 + AND COALESCE(f.parked, 0) = 0 ORDER BY j.did, j.rev", ) .map_err(db_err)?; @@ -219,6 +256,7 @@ impl SpaceIndex for SpaceScopedIndex { author: row.get(0)?, rev: row.get(1)?, mutations, + denials: row.get(3)?, }) }) .map_err(db_err)?; @@ -272,6 +310,36 @@ impl SpaceIndex for SpaceScopedIndex { .map_err(db_err) } + async fn record_projection_denial( + &self, + projector: &str, + did: &str, + rev: &str, + error: &str, + park_after: u32, + ) -> Result { + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projection_failure + (projector, space_uri, did, rev, attempts, last_error, dead_lettered, + denials, parked) + VALUES (?1, ?2, ?3, ?4, 0, ?5, 0, 1, CASE WHEN 1 >= ?6 THEN 1 ELSE 0 END) + ON CONFLICT (projector, space_uri, did, rev) DO UPDATE + SET denials = denials + 1, + last_error = ?5, + parked = CASE WHEN denials + 1 >= ?6 THEN 1 ELSE 0 END", + params![projector, self.space_uri, did, rev, error, park_after], + ) + .map_err(db_err)?; + conn.query_row( + "SELECT denials FROM projection_failure + WHERE projector = ?1 AND space_uri = ?2 AND did = ?3 AND rev = ?4", + params![projector, self.space_uri, did, rev], + |row| row.get(0), + ) + .map_err(db_err) + } + async fn prune_journal(&self, projectors: &[&str]) -> Result { if projectors.is_empty() { return Ok(0); @@ -293,7 +361,7 @@ impl SpaceIndex for SpaceScopedIndex { WHERE f.space_uri = projection_journal.space_uri AND f.did = projection_journal.did AND f.rev = projection_journal.rev - AND f.dead_lettered = 1)" + AND (f.dead_lettered = 1 OR f.parked = 1))" ); let count = projectors.len() as i64; let mut values: Vec<&dyn rusqlite::ToSql> = vec![&self.space_uri, &count]; @@ -302,7 +370,8 @@ impl SpaceIndex for SpaceScopedIndex { } let pruned = tx.execute(&sql, &values[..]).map_err(db_err)?; tx.execute( - "DELETE FROM projection_failure WHERE space_uri = ?1 AND dead_lettered = 0 + "DELETE FROM projection_failure + WHERE space_uri = ?1 AND dead_lettered = 0 AND parked = 0 AND NOT EXISTS (SELECT 1 FROM projection_journal j WHERE j.space_uri = projection_failure.space_uri AND j.did = projection_failure.did @@ -626,6 +695,111 @@ mod tests { assert!(index.pending_batches("appview").await.unwrap().is_empty()); } + #[tokio::test] + async fn denials_are_budgeted_apart_from_dead_letters_and_survive_a_prune() { + let dir = tempfile::tempdir().unwrap(); + let db = open_at(&dir); + let index = db.for_space(SPACE); + let mutation = IndexMutation::Delete { + collection: "app.bsky.feed.post".to_string(), + rkey: "3ka".to_string(), + }; + index + .journal_batch(AUTHOR, "3rev1", &[mutation]) + .await + .unwrap(); + + assert_eq!( + index + .record_projection_denial("appview", AUTHOR, "3rev1", "401", 2) + .await + .unwrap(), + 1 + ); + let pending = index.pending_batches("appview").await.unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].denials, 1); + + // A denial leaves the poison budget untouched. + let attempts: u32 = { + let conn = db.conn.lock().unwrap(); + conn.query_row( + "SELECT attempts FROM projection_failure + WHERE projector = 'appview' AND space_uri = ?1 AND did = ?2 AND rev = '3rev1'", + params![SPACE, AUTHOR], + |row| row.get(0), + ) + .unwrap() + }; + assert_eq!(attempts, 0); + + assert_eq!( + index + .record_projection_denial("appview", AUTHOR, "3rev1", "401", 2) + .await + .unwrap(), + 2 + ); + assert!(index.pending_batches("appview").await.unwrap().is_empty()); + + // A parked batch keeps its journal row, so it stays inspectable. + index + .advance_projector_cursor("appview", AUTHOR, "3rev1") + .await + .unwrap(); + assert_eq!(index.prune_journal(&["appview"]).await.unwrap(), 0); + } + + #[tokio::test] + async fn an_index_written_before_the_denial_columns_is_migrated_on_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("index.sqlite"); + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE projection_failure ( + projector TEXT NOT NULL, + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + attempts INTEGER NOT NULL, + last_error TEXT NOT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (projector, space_uri, did, rev) + );", + ) + .unwrap(); + conn.execute( + "INSERT INTO projection_failure + (projector, space_uri, did, rev, attempts, last_error, dead_lettered) + VALUES ('appview', ?1, ?2, '3rev1', 3, 'boom', 1)", + params![SPACE, AUTHOR], + ) + .unwrap(); + } + + let db = Arc::new(SqliteIndex::open(path.to_str().unwrap()).unwrap()); + let index = db.for_space(SPACE); + assert_eq!( + index + .record_projection_denial("appview", AUTHOR, "3rev2", "401", 5) + .await + .unwrap(), + 1 + ); + let (denials, parked, dead): (u32, u32, u32) = { + let conn = db.conn.lock().unwrap(); + conn.query_row( + "SELECT denials, parked, dead_lettered FROM projection_failure + WHERE rev = '3rev1'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap() + }; + assert_eq!((denials, parked, dead), (0, 0, 1)); + } + #[tokio::test] async fn corrupt_lthash_state_is_an_error() { let dir = tempfile::tempdir().unwrap(); From c9c5840dd4731b1dc430a795ee76af1439eed285 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Tue, 25 Aug 2026 00:15:46 -0400 Subject: [PATCH 51/56] test(spaces-parity): give the resume gate's converged host a store dir The converged space host has required SPACEHOST_SPACE_STORE_DIR since the two directories were separated; the gate still passed one path and so failed at boot. The legacy era predates the split and keeps one directory. --- rsky-spaces-parity/Cargo.toml | 2 +- rsky-spaces-parity/src/bin/resume_gate.rs | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/rsky-spaces-parity/Cargo.toml b/rsky-spaces-parity/Cargo.toml index c25c206b..01243db6 100644 --- a/rsky-spaces-parity/Cargo.toml +++ b/rsky-spaces-parity/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-spaces-parity" -version = "0.3.0" +version = "0.3.1" edition = "2021" publish = false diff --git a/rsky-spaces-parity/src/bin/resume_gate.rs b/rsky-spaces-parity/src/bin/resume_gate.rs index 434dcb30..bb9aaa98 100644 --- a/rsky-spaces-parity/src/bin/resume_gate.rs +++ b/rsky-spaces-parity/src/bin/resume_gate.rs @@ -137,14 +137,17 @@ fn seed_actor_store(root: &Path, did: &str, key_hex: &str) -> Result { Ok(store) } +/// `space_stores` is `None` for the legacy era, which predates the split of +/// the key source from the space-store write target. fn shim_env( port: u16, public_url: &str, db_path: &Path, actors: &Path, + space_stores: Option<&Path>, plc_url: &str, ) -> Vec<(String, String)> { - vec![ + let mut env = vec![ ("SPACEHOST_BIND", format!("127.0.0.1:{port}")), ("SPACEHOST_PUBLIC_URL", public_url.to_string()), ("SPACEHOST_AUTHORITY_DID", AUTHOR_DID.to_string()), @@ -164,8 +167,15 @@ fn shim_env( ("RUST_LOG", "warn".to_string()), ] .into_iter() - .map(|(key, value)| (key.to_string(), value)) - .collect() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect::>(); + if let Some(stores) = space_stores { + env.push(( + "SPACEHOST_SPACE_STORE_DIR".to_string(), + stores.display().to_string(), + )); + } + env } struct DaemonSetup { @@ -286,6 +296,7 @@ async fn main() -> Result<()> { &shim_url, &host_db, &legacy_actors, + None, &directory.url(), ), &run_dir.join("shim-legacy.log"), @@ -431,7 +442,8 @@ async fn main() -> Result<()> { shim_port, &shim_url, &host_db, - &converged_actors, + &legacy_actors, + Some(&converged_actors), &directory.url(), ), &run_dir.join("shim-converged.log"), From 7446b9267d85a784b3dba7e168c2be27e975e69d Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Tue, 25 Aug 2026 19:56:09 -0400 Subject: [PATCH 52/56] fix(space-host): tolerate partial actor-store account dirs at boot --- Cargo.lock | 6 ++-- rsky-space-host/Cargo.toml | 2 +- rsky-space-host/src/pds_seam.rs | 52 +++++++++++++++++++++++---------- 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f85031af..aa7d87c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8354,7 +8354,7 @@ dependencies = [ "rsky-oauth 0.3.2", "rsky-repo 0.0.6", "rsky-space 0.4.2", - "rsky-space-host 0.7.8", + "rsky-space-host 0.7.9", "rsky-syntax 0.1.0", "rusqlite", "secp256k1", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.8" +version = "0.7.9" dependencies = [ "async-trait", "axum", @@ -8728,7 +8728,7 @@ dependencies = [ "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", - "rsky-space-host 0.7.8", + "rsky-space-host 0.7.9", "rusqlite", "secp256k1", "serde", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 27231a0c..499c2cc2 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.8" +version = "0.7.9" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" diff --git a/rsky-space-host/src/pds_seam.rs b/rsky-space-host/src/pds_seam.rs index 8d848bf7..402bf437 100644 --- a/rsky-space-host/src/pds_seam.rs +++ b/rsky-space-host/src/pds_seam.rs @@ -175,6 +175,12 @@ impl CommitSigner for PdsSeam { } } +// A live PDS actor store accumulates partial account dirs (abandoned +// creations, deactivations, mid-migration state). Their presence is not this +// service's concern: key availability is enforced per authority when it +// registers or signs (`require_signer` → AccountNotHosted). Boot validation +// therefore only proves the mount itself is right; malformed entries are +// logged and skipped, never fatal. fn validate_actor_store_layout(root: &Path) -> Result<()> { if !root.is_dir() { return Err(HostError::Store(format!( @@ -189,15 +195,12 @@ fn validate_actor_store_layout(root: &Path) -> Result<()> { if name == "reserved_keys" { continue; } - if name.len() != 2 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { - return Err(HostError::Store(format!( - "unrecognized actor-store layout entry: {name}" - ))); - } - if !prefix.path().is_dir() { - return Err(HostError::Store(format!( - "actor-store shard is not a directory: {name}" - ))); + if name.len() != 2 + || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) + || !prefix.path().is_dir() + { + tracing::warn!(entry = %name, "skipping unrecognized actor-store layout entry"); + continue; } for actor in std::fs::read_dir(prefix.path()).map_err(|error| HostError::Store(error.to_string()))? @@ -208,9 +211,10 @@ fn validate_actor_store_layout(root: &Path) -> Result<()> { || !actor.path().join("store.sqlite").is_file() || !actor.path().join("key").is_file() { - return Err(HostError::Store(format!( - "unrecognized actor-store account layout: {actor_name}" - ))); + tracing::warn!( + actor = %actor_name, + "skipping malformed actor-store account dir" + ); } } } @@ -271,12 +275,28 @@ mod tests { } #[test] - fn layout_drift_refuses_startup() { - let directory = tempfile::tempdir().unwrap(); + fn layout_drift_is_skipped_not_fatal() { + // A live PDS actor store carries partial dirs (abandoned creations, + // deactivations). They must not brick the host: signing for such an + // account still fails per-actor, but boot proceeds. + let directory = actor_store([7u8; 32]); std::fs::create_dir(directory.path().join("unexpected-layout")).unwrap(); + let partial = directory.path().join("aa").join("did:plc:abandoned"); + std::fs::create_dir_all(&partial).unwrap(); + std::fs::write(partial.join("store.sqlite"), []).unwrap(); // no key file + let seam = PdsSeam::open(directory.path()).expect("boot survives layout drift"); + assert!(matches!( + seam.signer("did:plc:abandoned"), + Err(HostError::AccountNotHosted(_)) + )); + assert!(seam.signer(DID).is_ok()); + } + + #[test] + fn missing_actor_store_root_still_refuses_startup() { assert!(matches!( - PdsSeam::open(directory.path()), - Err(HostError::Store(message)) if message.contains("layout") + PdsSeam::open(std::path::Path::new("/nonexistent/actors")), + Err(HostError::Store(message)) if message.contains("not a directory") )); } From 46437edf413637a26010f6091c6e0002932c7018 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 26 Aug 2026 13:32:40 -0400 Subject: [PATCH 53/56] fix(rsky-daemon): publish service DID document --- Cargo.lock | 2 +- rsky-daemon/Cargo.toml | 2 +- rsky-daemon/src/main.rs | 1 + rsky-daemon/src/notify.rs | 107 +++++++++++++++++++++++++++++++++++++- 4 files changed, 109 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa7d87c4..d827ddf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "rsky-daemon" -version = "0.6.1" +version = "0.6.2" dependencies = [ "async-trait", "axum", diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index 61c7639c..99e81d86 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-daemon" -version = "0.6.1" +version = "0.6.2" authors = ["Rudy Fraser "] description = "atproto permissioned-data syncer daemon: pulls, verifies, and indexes permissioned repos from members' PDSes" edition = "2021" diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 3bc54684..27a45374 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -194,6 +194,7 @@ async fn main() -> std::result::Result<(), Box> { space_uri: cfg.space_uri.clone(), registry: registry.clone(), service_identity: cfg.service_identity.clone(), + service_signing_key_hex: cfg.service_signing_key_hex.clone(), resolver: keys.clone(), index: Arc::new(InMemoryIndex::new()), tx: notify_tx, diff --git a/rsky-daemon/src/notify.rs b/rsky-daemon/src/notify.rs index 3fc7968c..144778fc 100644 --- a/rsky-daemon/src/notify.rs +++ b/rsky-daemon/src/notify.rs @@ -5,7 +5,7 @@ use axum::extract::State; use axum::http::{header, HeaderMap, StatusCode}; -use axum::routing::post; +use axum::routing::{get, post}; use axum::{Json, Router}; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; @@ -92,6 +92,7 @@ pub struct NotifyState { pub registry: SpaceRegistry, /// This syncer's service identity: the required `aud` on inbound tokens. pub service_identity: String, + pub service_signing_key_hex: String, pub resolver: Arc, pub index: Arc, pub tx: mpsc::Sender, @@ -100,6 +101,7 @@ pub struct NotifyState { pub fn router(state: NotifyState) -> Router { Router::new() + .route("/.well-known/did.json", get(well_known)) .route("/xrpc/com.atproto.space.notifyWrite", post(notify_write)) .route( "/xrpc/com.atproto.space.notifySpaceDeleted", @@ -108,6 +110,48 @@ pub fn router(state: NotifyState) -> Router { .with_state(state) } +fn service_multikey(signing_key_hex: &str) -> Result { + let bytes = hex::decode(signing_key_hex.trim()) + .map_err(|error| DaemonError::Xrpc(error.to_string()))?; + let secret = secp256k1::SecretKey::from_slice(&bytes) + .map_err(|error| DaemonError::Xrpc(error.to_string()))?; + let public = secp256k1::PublicKey::from_secret_key(&secp256k1::Secp256k1::new(), &secret); + let did_key = rsky_crypto::utils::encode_did_key(&public); + Ok(did_key + .strip_prefix("did:key:") + .expect("rsky did:key encoder always includes its prefix") + .to_string()) +} + +async fn well_known(State(state): State) -> (StatusCode, Json) { + let multikey = match service_multikey(&state.service_signing_key_hex) { + Ok(multikey) => multikey, + Err(error) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + error_body("InternalServerError", error), + ) + } + }; + let did = state.service_identity; + ( + StatusCode::OK, + Json(json!({ + "@context": [ + "https://www.w3.org/ns/did/v1", + "https://w3id.org/security/multikey/v1" + ], + "id": did, + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }] + })), + ) +} + fn error_body(error: &str, message: impl std::fmt::Display) -> Json { Json(json!({ "error": error, "message": message.to_string() })) } @@ -201,6 +245,7 @@ mod tests { const SPACE: &str = "at://did:plc:authority/space/community.blacksky.feed/main"; const AUTHORITY: &str = "did:plc:authority"; const SYNCER: &str = "did:web:syncer.blacksky.community"; + const SYNCER_KEY: &str = "0707070707070707070707070707070707070707070707070707070707070707"; const NOW: u64 = 1_000_000; fn host_key() -> (SecretKey, String) { @@ -250,6 +295,7 @@ mod tests { registry }, service_identity: SYNCER.to_string(), + service_signing_key_hex: SYNCER_KEY.to_string(), resolver: Arc::new(FixedKey(did_key.to_string())), index, tx, @@ -257,6 +303,64 @@ mod tests { } } + #[tokio::test] + async fn well_known_publishes_the_service_atproto_multikey() { + let (_secret, did_key) = host_key(); + let (tx, _rx) = mpsc::channel(4); + let app = router(state(&did_key, Arc::new(InMemoryIndex::new()), tx)); + + let response = app + .oneshot( + Request::builder() + .uri("/.well-known/did.json") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let document: Value = serde_json::from_slice(&body).unwrap(); + let secret = SecretKey::from_slice(&[7u8; 32]).unwrap(); + let public = PublicKey::from_secret_key(&Secp256k1::new(), &secret); + let expected = rsky_crypto::utils::encode_did_key(&public) + .strip_prefix("did:key:") + .unwrap() + .to_string(); + + assert_eq!(document["id"], SYNCER); + assert_eq!( + document["verificationMethod"][0]["id"], + format!("{SYNCER}#atproto") + ); + assert_eq!(document["verificationMethod"][0]["type"], "Multikey"); + assert_eq!(document["verificationMethod"][0]["controller"], SYNCER); + assert_eq!( + document["verificationMethod"][0]["publicKeyMultibase"], + expected + ); + } + + #[tokio::test] + async fn well_known_rejects_an_invalid_signing_key() { + let (_secret, did_key) = host_key(); + let (tx, _rx) = mpsc::channel(4); + let mut notify_state = state(&did_key, Arc::new(InMemoryIndex::new()), tx); + notify_state.service_signing_key_hex = "not-hex".to_string(); + let response = router(notify_state) + .oneshot( + Request::builder() + .uri("/.well-known/did.json") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + } + fn request(path: &str, token: Option<&str>, body: Value) -> Request { let mut builder = Request::builder() .method("POST") @@ -522,6 +626,7 @@ mod tests { space_uri: SPACE.to_string(), registry, service_identity: SYNCER.to_string(), + service_signing_key_hex: SYNCER_KEY.to_string(), resolver: Arc::new(KeyMap(BTreeMap::from([ (AUTHORITY.to_string(), key_a), ("did:plc:authority2".to_string(), key_b), From 64b0d2259387280b10529a7280db8eadd67065d1 Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 26 Aug 2026 13:54:54 -0400 Subject: [PATCH 54/56] fix(rsky-daemon): include spaces-parity manifest in the image build --- rsky-daemon/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsky-daemon/Dockerfile b/rsky-daemon/Dockerfile index 2687940f..09d4a889 100644 --- a/rsky-daemon/Dockerfile +++ b/rsky-daemon/Dockerfile @@ -23,6 +23,7 @@ COPY rsky-repo/Cargo.toml rsky-repo/Cargo.toml COPY rsky-satnav/Cargo.toml rsky-satnav/Cargo.toml COPY rsky-space/Cargo.toml rsky-space/Cargo.toml COPY rsky-space-host/Cargo.toml rsky-space-host/Cargo.toml +COPY rsky-spaces-parity/Cargo.toml rsky-spaces-parity/Cargo.toml COPY rsky-syntax/Cargo.toml rsky-syntax/Cargo.toml COPY rsky-video/Cargo.toml rsky-video/Cargo.toml COPY rsky-wintermute/Cargo.toml rsky-wintermute/Cargo.toml @@ -48,6 +49,7 @@ RUN mkdir -p \ echo 'fn main() {}' > $crate/src/main.rs; \ done && \ touch rsky-pds/src/lib.rs rsky-repo/src/lib.rs && \ + mkdir -p rsky-spaces-parity/src && touch rsky-spaces-parity/src/lib.rs && \ mkdir -p rsky-wintermute/src/bin && \ for bin in queue_backfill fix_blob_refs plc_import label_sync car_loader \ cleanup_stale_deactivated; do \ From c89644bc003ec2190068c2978ccd8b065f31833a Mon Sep 17 00:00:00 2001 From: Rishi Balakrishnan Date: Wed, 26 Aug 2026 14:59:32 -0400 Subject: [PATCH 55/56] fix(rsky-daemon): publish syncer service endpoint --- Cargo.lock | 2 +- rsky-daemon/Cargo.toml | 2 +- rsky-daemon/src/main.rs | 4 +++- rsky-daemon/src/notify.rs | 19 ++++++++++++++++++- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d827ddf8..86bc3506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "rsky-daemon" -version = "0.6.2" +version = "0.6.3" dependencies = [ "async-trait", "axum", diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index 99e81d86..eea9cd4a 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-daemon" -version = "0.6.2" +version = "0.6.3" authors = ["Rudy Fraser "] description = "atproto permissioned-data syncer daemon: pulls, verifies, and indexes permissioned repos from members' PDSes" edition = "2021" diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 27a45374..5e51af25 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -190,11 +190,13 @@ async fn main() -> std::result::Result<(), Box> { let (notify_tx, notify_rx) = mpsc::channel(1024); let (shutdown_tx, shutdown_rx) = watch::channel(false); + let notify_endpoint = cfg.notify_endpoint(); let notify_state = NotifyState { space_uri: cfg.space_uri.clone(), registry: registry.clone(), service_identity: cfg.service_identity.clone(), service_signing_key_hex: cfg.service_signing_key_hex.clone(), + notify_endpoint: notify_endpoint.clone(), resolver: keys.clone(), index: Arc::new(InMemoryIndex::new()), tx: notify_tx, @@ -259,7 +261,7 @@ async fn main() -> std::result::Result<(), Box> { let opts = MultiRunnerOptions { refresh_interval_secs: cfg.sweep_interval_secs, sweep_interval_secs: cfg.sweep_interval_secs, - notify_endpoint: cfg.notify_endpoint(), + notify_endpoint, service_identity: cfg.service_identity.clone(), now_fn: rsky_daemon::unix_now, }; diff --git a/rsky-daemon/src/notify.rs b/rsky-daemon/src/notify.rs index 144778fc..3e43be0b 100644 --- a/rsky-daemon/src/notify.rs +++ b/rsky-daemon/src/notify.rs @@ -93,6 +93,7 @@ pub struct NotifyState { /// This syncer's service identity: the required `aud` on inbound tokens. pub service_identity: String, pub service_signing_key_hex: String, + pub notify_endpoint: String, pub resolver: Arc, pub index: Arc, pub tx: mpsc::Sender, @@ -134,6 +135,7 @@ async fn well_known(State(state): State) -> (StatusCode, Json) -> (StatusCode, Json Date: Wed, 26 Aug 2026 19:23:50 -0400 Subject: [PATCH 56/56] fix(space-host): allow browser space writes --- Cargo.lock | 7 ++-- rsky-space-host/Cargo.toml | 3 +- rsky-space-host/src/http.rs | 71 ++++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86bc3506..619bde7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8354,7 +8354,7 @@ dependencies = [ "rsky-oauth 0.3.2", "rsky-repo 0.0.6", "rsky-space 0.4.2", - "rsky-space-host 0.7.9", + "rsky-space-host 0.7.10", "rsky-syntax 0.1.0", "rusqlite", "secp256k1", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "rsky-space-host" -version = "0.7.9" +version = "0.7.10" dependencies = [ "async-trait", "axum", @@ -8708,6 +8708,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tower 0.4.13", + "tower-http 0.5.2", "tracing", "tracing-subscriber", "wiremock", @@ -8728,7 +8729,7 @@ dependencies = [ "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "rsky-space 0.4.1", "rsky-space 0.4.2", - "rsky-space-host 0.7.9", + "rsky-space-host 0.7.10", "rusqlite", "secp256k1", "serde", diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index 499c2cc2..324ba6ba 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.7.9" +version = "0.7.10" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" @@ -39,6 +39,7 @@ tokio = { workspace = true } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } clap = { version = "4", features = ["derive", "env"] } +tower-http = { version = "0.5", features = ["cors"] } [dev-dependencies] p256 = { version = "0.13.2", features = ["ecdsa"] } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 147a6829..c5aab632 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -4,7 +4,8 @@ //! rsky-lexicon; errors are XRPC-shaped `{error, message}` JSON. use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::http::header::{AUTHORIZATION, CONTENT_TYPE}; +use axum::http::{HeaderMap, HeaderName, Method, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; @@ -19,6 +20,8 @@ use rsky_space::credential; use serde_json::Value; use std::collections::BTreeMap; use std::sync::Arc; +use std::time::Duration; +use tower_http::cors::{Any, CorsLayer}; use crate::attestation::{JtiStore, MetadataFetcher}; use crate::authority::{AuthorityContext, AuthorityFactory, AuthorityRegistry, KeyResolver}; @@ -95,6 +98,19 @@ pub fn router(state: AppState) -> Router { .route("/xrpc/com.atproto.space.deleteRecord", post(delete_record)) .route("/admin/mintCredential", post(mint_credential)) .with_state(state) + .layer( + CorsLayer::new() + .allow_origin(Any) + .allow_methods([Method::GET, Method::POST]) + .allow_headers([ + HeaderName::from_static("atproto-accept-labelers"), + HeaderName::from_static("atproto-proxy"), + AUTHORIZATION, + CONTENT_TYPE, + HeaderName::from_static("dpop"), + ]) + .max_age(Duration::from_secs(24 * 60 * 60)), + ) } /// An XRPC-shaped error response: `{error, message}` with a matching status. @@ -1315,6 +1331,59 @@ mod tests { .unwrap() } + #[tokio::test] + async fn cors_preflight_allows_the_staging_space_write() { + let f = fixture(AppAccess::Open, &[]); + let response = router(f.state) + .oneshot( + Request::builder() + .method("OPTIONS") + .uri("/xrpc/com.atproto.space.createRecord") + .header("origin", "https://staging.blacksky.community") + .header("access-control-request-method", "POST") + .header( + "access-control-request-headers", + "atproto-accept-labelers, atproto-proxy, authorization, content-type, dpop", + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert!(response.status().is_success()); + assert_eq!( + response + .headers() + .get("access-control-allow-origin") + .unwrap(), + "*" + ); + let methods = response + .headers() + .get("access-control-allow-methods") + .unwrap() + .to_str() + .unwrap(); + assert!(methods.split(',').any(|method| method.trim() == "POST")); + let headers = response + .headers() + .get("access-control-allow-headers") + .unwrap() + .to_str() + .unwrap() + .to_ascii_lowercase(); + for header in [ + "atproto-accept-labelers", + "atproto-proxy", + "authorization", + "content-type", + "dpop", + ] { + assert!(headers.split(',').any(|value| value.trim() == header)); + } + } + #[tokio::test] async fn create_and_delete_records_verify_the_pds_session() { let f = fixture(AppAccess::Open, &[]);