diff --git a/Cargo.lock b/Cargo.lock index 9b6aade8..619bde7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7978,7 +7978,7 @@ dependencies = [ "rand 0.8.5", "rand_core 0.6.4", "regex", - "rsky-identity", + "rsky-identity 0.2.0", "secp256k1", "serde", "serde_ipld_dagcbor", @@ -7991,6 +7991,34 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "rsky-common" +version = "0.1.3" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "base64ct", + "chrono", + "cid", + "futures", + "indexmap 1.9.3", + "multihash", + "multihash-codetable", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "rsky-identity 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "secp256k1", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.16", + "tracing", + "url", + "urlencoding", +] + [[package]] name = "rsky-crypto" version = "0.2.0" @@ -8003,9 +8031,21 @@ dependencies = [ "unsigned-varint 0.8.0", ] +[[package]] +name = "rsky-crypto" +version = "0.2.0" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "multibase", + "p256 0.13.2", + "secp256k1", + "unsigned-varint 0.8.0", +] + [[package]] name = "rsky-daemon" -version = "0.4.0" +version = "0.6.3" dependencies = [ "async-trait", "axum", @@ -8016,13 +8056,13 @@ dependencies = [ "hex", "rand 0.8.5", "reqwest 0.12.23", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-oauth", - "rsky-space", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-oauth 0.3.2", + "rsky-space 0.4.2", + "rsky-syntax 0.1.0", "rusqlite", "secp256k1", "serde", @@ -8055,8 +8095,8 @@ dependencies = [ "reqwest 0.11.27", "rocket", "rocket_sync_db_pools", - "rsky-common", - "rsky-lexicon", + "rsky-common 0.1.3", + "rsky-lexicon 0.10.2", "serde", "serde_bytes", "serde_cbor", @@ -8083,7 +8123,7 @@ dependencies = [ "parking_lot", "reqwest 0.11.27", "retry", - "rsky-lexicon", + "rsky-lexicon 0.10.2", "serde", "serde_bytes", "serde_cbor", @@ -8106,7 +8146,7 @@ dependencies = [ "hickory-resolver 0.24.4", "multibase", "reqwest 0.12.23", - "rsky-crypto", + "rsky-crypto 0.2.0", "secp256k1", "serde", "serde_json", @@ -8116,6 +8156,23 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "rsky-identity" +version = "0.2.0" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "async-trait", + "hickory-resolver 0.24.4", + "reqwest 0.12.23", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "serde", + "serde_json", + "thiserror 1.0.69", + "url", + "urlencoding", +] + [[package]] name = "rsky-jetstream-subscriber" version = "0.1.0" @@ -8125,7 +8182,7 @@ dependencies = [ "dotenvy", "futures", "reqwest 0.11.27", - "rsky-lexicon", + "rsky-lexicon 0.10.2", "serde", "serde_derive", "serde_json", @@ -8155,8 +8212,8 @@ dependencies = [ "parking_lot", "reqwest 0.12.23", "retry", - "rsky-common", - "rsky-lexicon", + "rsky-common 0.1.3", + "rsky-lexicon 0.10.2", "serde", "serde_bytes", "serde_cbor", @@ -8190,16 +8247,37 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "rsky-lexicon" +version = "0.10.2" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "base64 0.22.1", + "chrono", + "cid", + "derive_builder 0.12.0", + "miette 5.10.0", + "parking_lot", + "secp256k1", + "serde", + "serde_bytes", + "serde_cbor", + "serde_derive", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "rsky-oauth" -version = "0.3.1" +version = "0.3.2" dependencies = [ "async-trait", "base64 0.22.1", "hex", "hmac", "p256 0.13.2", - "rsky-crypto", + "rsky-crypto 0.2.0", "secp256k1", "serde", "serde_json", @@ -8209,9 +8287,28 @@ dependencies = [ "url", ] +[[package]] +name = "rsky-oauth" +version = "0.3.2" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "async-trait", + "base64 0.22.1", + "hex", + "hmac", + "p256 0.13.2", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "secp256k1", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.16", + "url", +] + [[package]] name = "rsky-pds" -version = "0.13.16" +version = "0.13.17" dependencies = [ "anyhow", "argon2", @@ -8250,15 +8347,15 @@ dependencies = [ "reqwest 0.12.23", "rocket", "rocket_ws", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-oauth", - "rsky-repo", - "rsky-space", - "rsky-space-host", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-oauth 0.3.2", + "rsky-repo 0.0.6", + "rsky-space 0.4.2", + "rsky-space-host 0.7.10", + "rsky-syntax 0.1.0", "rusqlite", "secp256k1", "serde", @@ -8279,6 +8376,75 @@ dependencies = [ "url", ] +[[package]] +name = "rsky-pds" +version = "0.13.17" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "argon2", + "askama", + "async-event-emitter", + "async-trait", + "atrium-api 0.24.10", + "atrium-xrpc-client", + "aws-config", + "aws-sdk-s3", + "base64 0.22.1", + "base64-url", + "base64ct", + "chrono", + "cid", + "data-encoding", + "dotenvy", + "email_address", + "event-emitter-rs", + "futures", + "hex", + "hickory-resolver 0.24.4", + "image", + "indexmap 1.9.3", + "infer 0.15.0", + "ipld-core", + "jwt-simple", + "lazy_static", + "lru 0.14.0", + "mailchecker", + "mailgun-rs", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "reqwest 0.12.23", + "rocket", + "rocket_ws", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-identity 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-lexicon 0.10.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-oauth 0.3.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-repo 0.0.6 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-space 0.4.1", + "rsky-space-host 0.5.1", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rusqlite", + "secp256k1", + "serde", + "serde_bytes", + "serde_cbor", + "serde_derive", + "serde_ipld_dagcbor", + "serde_json", + "serde_repr", + "sha2 0.10.9", + "thiserror 1.0.69", + "time", + "tokio", + "toml 0.8.23", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "rsky-relay" version = "0.1.2" @@ -8310,8 +8476,8 @@ dependencies = [ "p256 0.13.2", "reqwest 0.12.23", "rs-car-sync", - "rsky-common", - "rsky-identity", + "rsky-common 0.1.3", + "rsky-identity 0.2.0", "rtrb", "rusqlite", "rustls 0.23.31", @@ -8356,10 +8522,44 @@ dependencies = [ "rand 0.8.5", "rand_core 0.6.4", "regex", - "rsky-common", - "rsky-crypto", - "rsky-lexicon", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-syntax 0.1.0", + "secp256k1", + "serde", + "serde_bytes", + "serde_cbor", + "serde_derive", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.10.9", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "rsky-repo" +version = "0.0.6" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "anyhow", + "async-recursion", + "async-stream", + "async-trait", + "cid", + "futures", + "integer-encoding", + "ipld-core", + "iroh-car", + "lazy_static", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-lexicon 0.10.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", "secp256k1", "serde", "serde_bytes", @@ -8396,6 +8596,31 @@ dependencies = [ [[package]] name = "rsky-space" version = "0.4.1" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" +dependencies = [ + "base64 0.22.1", + "blake3", + "cid", + "hex", + "hkdf", + "hmac", + "iroh-car", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.10.9", + "subtle", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "rsky-space" +version = "0.4.2" dependencies = [ "base64 0.22.1", "blake3", @@ -8403,11 +8628,12 @@ dependencies = [ "hex", "hkdf", "hmac", + "ipld-core", "iroh-car", "p256 0.13.2", - "rsky-common", - "rsky-crypto", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-syntax 0.1.0", "secp256k1", "serde", "serde_bytes", @@ -8422,6 +8648,7 @@ dependencies = [ [[package]] name = "rsky-space-host" version = "0.5.1" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" dependencies = [ "async-trait", "axum", @@ -8429,17 +8656,49 @@ dependencies = [ "chrono", "clap", "hex", + "rand 0.8.5", + "reqwest 0.12.23", + "rsky-common 0.1.3 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-crypto 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-identity 0.2.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-lexicon 0.10.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-oauth 0.3.2 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-space 0.4.1", + "rsky-syntax 0.1.0 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rusqlite", + "secp256k1", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "rsky-space-host" +version = "0.7.10" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "chrono", + "cid", + "clap", + "hex", + "hmac", "http-body-util", "p256 0.13.2", "rand 0.8.5", "reqwest 0.12.23", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-oauth", - "rsky-space", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-oauth 0.3.2", + "rsky-space 0.4.2", + "rsky-syntax 0.1.0", "rusqlite", "secp256k1", "serde", @@ -8449,14 +8708,55 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tower 0.4.13", + "tower-http 0.5.2", "tracing", "tracing-subscriber", "wiremock", + "zeroize", +] + +[[package]] +name = "rsky-spaces-parity" +version = "0.3.1" +dependencies = [ + "anyhow", + "base64 0.22.1", + "hex", + "hmac", + "reqwest 0.12.23", + "rsky-crypto 0.2.0", + "rsky-oauth 0.3.2", + "rsky-pds 0.13.17 (git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738)", + "rsky-space 0.4.1", + "rsky-space 0.4.2", + "rsky-space-host 0.7.10", + "rusqlite", + "secp256k1", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", +] + +[[package]] +name = "rsky-syntax" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "lazy_static", + "regex", + "serde", + "serde_derive", + "thiserror 1.0.69", + "url", ] [[package]] name = "rsky-syntax" version = "0.1.0" +source = "git+https://github.com/blacksky-algorithms/rsky.git?rev=7ebd21ae788c550ee8510034d94eb19ede148738#7ebd21ae788c550ee8510034d94eb19ede148738" dependencies = [ "anyhow", "chrono", @@ -8491,7 +8791,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "reqwest 0.12.23", - "rsky-syntax", + "rsky-syntax 0.1.0", "rustls 0.23.31", "sec1 0.7.3", "serde", @@ -8538,12 +8838,12 @@ dependencies = [ "prometheus", "rand 0.8.5", "reqwest 0.12.23", - "rsky-common", - "rsky-crypto", - "rsky-identity", - "rsky-lexicon", - "rsky-repo", - "rsky-syntax", + "rsky-common 0.1.3", + "rsky-crypto 0.2.0", + "rsky-identity 0.2.0", + "rsky-lexicon 0.10.2", + "rsky-repo 0.0.6", + "rsky-syntax 0.1.0", "rtrb", "rusqlite", "rustls 0.23.31", diff --git a/Cargo.toml b/Cargo.toml index b9fb3f6d..ebdb0533 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "rsky-satnav", "rsky-space", "rsky-space-host", + "rsky-spaces-parity", "rsky-syntax", "rsky-video", "rsky-wintermute", diff --git a/rsky-daemon/Cargo.toml b/rsky-daemon/Cargo.toml index 64428658..eea9cd4a 100644 --- a/rsky-daemon/Cargo.toml +++ b/rsky-daemon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-daemon" -version = "0.4.0" +version = "0.6.3" authors = ["Rudy Fraser "] description = "atproto permissioned-data syncer daemon: pulls, verifies, and indexes permissioned repos from members' PDSes" edition = "2021" @@ -36,6 +36,7 @@ chrono = { version = "0.4", features = ["serde"] } axum = "0.7" base64 = "0.22" sha2 = { workspace = true } +secp256k1 = { workspace = true } [dev-dependencies] secp256k1 = { workspace = true } diff --git a/rsky-daemon/Dockerfile b/rsky-daemon/Dockerfile new file mode 100644 index 00000000..09d4a889 --- /dev/null +++ b/rsky-daemon/Dockerfile @@ -0,0 +1,82 @@ +# Use the official Rust image. +# https://hub.docker.com/_/rust +FROM rust AS builder + +WORKDIR /usr/src/rsky + +# Copy workspace and all crate manifests for dependency resolution +COPY Cargo.toml Cargo.lock rust-toolchain ./ +COPY palomar-sync/Cargo.toml palomar-sync/Cargo.toml +COPY rsky-common/Cargo.toml rsky-common/Cargo.toml +COPY rsky-crypto/Cargo.toml rsky-crypto/Cargo.toml +COPY rsky-daemon/Cargo.toml rsky-daemon/Cargo.toml +COPY rsky-feedgen/Cargo.toml rsky-feedgen/Cargo.toml +COPY rsky-firehose/Cargo.toml rsky-firehose/Cargo.toml +COPY rsky-identity/Cargo.toml rsky-identity/Cargo.toml +COPY rsky-jetstream-subscriber/Cargo.toml rsky-jetstream-subscriber/Cargo.toml +COPY rsky-labeler/Cargo.toml rsky-labeler/Cargo.toml +COPY rsky-lexicon/Cargo.toml rsky-lexicon/Cargo.toml +COPY rsky-oauth/Cargo.toml rsky-oauth/Cargo.toml +COPY rsky-pds/Cargo.toml rsky-pds/Cargo.toml +COPY rsky-relay/Cargo.toml rsky-relay/Cargo.toml +COPY rsky-repo/Cargo.toml rsky-repo/Cargo.toml +COPY rsky-satnav/Cargo.toml rsky-satnav/Cargo.toml +COPY rsky-space/Cargo.toml rsky-space/Cargo.toml +COPY rsky-space-host/Cargo.toml rsky-space-host/Cargo.toml +COPY rsky-spaces-parity/Cargo.toml rsky-spaces-parity/Cargo.toml +COPY rsky-syntax/Cargo.toml rsky-syntax/Cargo.toml +COPY rsky-video/Cargo.toml rsky-video/Cargo.toml +COPY rsky-wintermute/Cargo.toml rsky-wintermute/Cargo.toml + +# Copy real source for library crates that rsky-daemon depends on +COPY rsky-common/src rsky-common/src +COPY rsky-crypto/src rsky-crypto/src +COPY rsky-identity/src rsky-identity/src +COPY rsky-lexicon/src rsky-lexicon/src +COPY rsky-oauth/src rsky-oauth/src +COPY rsky-space/src rsky-space/src +COPY rsky-syntax/src rsky-syntax/src + +# Stub out the remaining workspace members so cargo can resolve the workspace +RUN mkdir -p \ + palomar-sync/src rsky-feedgen/src rsky-firehose/src \ + rsky-jetstream-subscriber/src rsky-labeler/src rsky-pds/src \ + rsky-relay/src rsky-repo/src rsky-satnav/src rsky-space-host/src \ + rsky-video/src rsky-wintermute/src && \ + for crate in palomar-sync rsky-feedgen rsky-firehose \ + rsky-jetstream-subscriber rsky-labeler rsky-pds rsky-relay \ + rsky-satnav rsky-space-host rsky-video rsky-wintermute; do \ + echo 'fn main() {}' > $crate/src/main.rs; \ + done && \ + touch rsky-pds/src/lib.rs rsky-repo/src/lib.rs && \ + mkdir -p rsky-spaces-parity/src && touch rsky-spaces-parity/src/lib.rs && \ + mkdir -p rsky-wintermute/src/bin && \ + for bin in queue_backfill fix_blob_refs plc_import label_sync car_loader \ + cleanup_stale_deactivated; do \ + echo 'fn main() {}' > rsky-wintermute/src/bin/$bin.rs; \ + done + +# Create an empty src directory to trick Cargo into thinking it's a valid Rust project +RUN mkdir -p rsky-daemon/src && echo "fn main() {}" > rsky-daemon/src/main.rs + +# Install production dependencies and build a release artifact. +RUN cargo build --release --package rsky-daemon + +# Now copy the real source code and build the final binary +COPY rsky-daemon/src rsky-daemon/src + +RUN cargo build --release --package rsky-daemon + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /usr/src/rsky +COPY --from=builder /usr/src/rsky/target/release/rsky-daemon rsky-daemon +LABEL org.opencontainers.image.source=https://github.com/blacksky-algorithms/rsky +# State is sqlite + a persisted key: mount a volume covering DAEMON_INDEX_DB_PATH +# and DAEMON_DPOP_KEY_PATH. Required env: DAEMON_SPACE_HOST_URL, +# DAEMON_SERVICE_IDENTITY, DAEMON_SERVICE_SIGNING_KEY_HEX, +# DAEMON_SPACE_HOST_MINT_TOKEN, and DAEMON_SPACE_URI and/or DAEMON_SPACES_URL +# (+ DAEMON_SPACES_API_KEY). See rsky-daemon/src/config.rs for the full reference. +CMD ["sh", "-c", "DAEMON_NOTIFY_BIND=${DAEMON_NOTIFY_BIND:-0.0.0.0:8055} ./rsky-daemon"] diff --git a/rsky-daemon/src/appview.rs b/rsky-daemon/src/appview.rs new file mode 100644 index 00000000..70f079ae --- /dev/null +++ b/rsky-daemon/src/appview.rs @@ -0,0 +1,271 @@ +//! Projection to the appview's record ingress. + +use async_trait::async_trait; +use serde_json::Value; +use std::collections::BTreeMap; + +use crate::error::Result; +use crate::feeds::{ProjectRecord, ProjectRecordsRequest, ProjectionIngress, ProjectionOperation}; +use crate::projection::Projector; +use crate::router::{SyncEvent, LIKE_COLLECTION, MODERATION_ACTION_COLLECTION, POST_COLLECTION}; + +/// Projects a space's posts, likes and moderation state to the appview. +pub struct AppviewProjector { + ingress: I, + space: String, +} + +impl AppviewProjector { + pub fn new(ingress: I, space: impl Into) -> Self { + Self { + ingress, + space: space.into(), + } + } + + #[allow(clippy::too_many_arguments)] + fn op( + &self, + author: &str, + uri: &str, + cid: Option<&str>, + revision: &str, + operation: ProjectionOperation, + collection: &str, + record: Option, + ) -> ProjectRecord { + ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.to_string(), + cid: cid.map(str::to_string), + revision: revision.to_string(), + operation, + collection: collection.to_string(), + record, + action_uri: None, + neg: None, + } + } + + fn moderation_op( + &self, + author: &str, + revision: &str, + uri: &str, + record: &Value, + ) -> Option { + if record["val"].as_str() != Some("remove") { + return None; + } + // A lift is its own signed record carrying the action it reverses, + // so it names that action's URI rather than its own. + let neg = record["neg"].as_bool() == Some(true); + let (action_uri, cid) = if neg { + (record["action"]["uri"].as_str(), None) + } else { + (Some(uri), record["subject"]["cid"].as_str()) + }; + Some(ProjectRecord { + action_uri: Some(action_uri?.to_string()), + neg: Some(neg), + ..self.op( + author, + uri, + cid, + revision, + if neg { + ProjectionOperation::Unflag + } else { + ProjectionOperation::Flag + }, + MODERATION_ACTION_COLLECTION, + None, + ) + }) + } +} + +#[async_trait] +impl Projector for AppviewProjector { + fn name(&self) -> &'static str { + "appview" + } + + async fn project(&self, author: &str, revision: &str, events: &[SyncEvent]) -> Result<()> { + // One URI may change more than once in a batch; only its final state + // is worth sending. + let mut final_records: BTreeMap<&str, &SyncEvent> = BTreeMap::new(); + for event in events { + final_records.insert(event.uri(), event); + } + let ops: Vec = final_records + .into_values() + .filter_map(|event| match event { + SyncEvent::PostCreated { + uri, cid, record, .. + } => Some(self.op( + author, + uri, + Some(cid), + revision, + ProjectionOperation::Create, + POST_COLLECTION, + Some(record.clone()), + )), + SyncEvent::PostDeleted { uri, .. } => Some(self.op( + author, + uri, + None, + revision, + ProjectionOperation::Delete, + POST_COLLECTION, + None, + )), + SyncEvent::LikeCreated { + uri, cid, record, .. + } => Some(self.op( + author, + uri, + Some(cid), + revision, + ProjectionOperation::Create, + LIKE_COLLECTION, + Some(record.clone()), + )), + SyncEvent::LikeDeleted { uri, .. } => Some(self.op( + author, + uri, + None, + revision, + ProjectionOperation::Delete, + LIKE_COLLECTION, + None, + )), + SyncEvent::ModerationAction { + uri, + record: Some(record), + .. + } => self.moderation_op(author, revision, uri, record), + SyncEvent::ModerationAction { record: None, .. } => None, + }) + .collect(); + if ops.is_empty() { + return Ok(()); + } + self.ingress + .project_records(&ProjectRecordsRequest { ops }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::feeds::tests::{post_created, CapturingIngress, AUTHOR, SPACE}; + use serde_json::json; + + fn like_created(rkey: &str) -> SyncEvent { + SyncEvent::LikeCreated { + uri: format!("{SPACE}/{AUTHOR}/{LIKE_COLLECTION}/{rkey}"), + author: AUTHOR.to_string(), + cid: "bafylike".to_string(), + rev: "3krev".to_string(), + record: json!({ + "$type": LIKE_COLLECTION, + "subject": {"uri": format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), "cid": "bafypost"}, + "createdAt": "2026-08-19T00:00:00Z", + }), + } + } + + #[tokio::test] + async fn posts_likes_and_removals_all_project() { + let projector = AppviewProjector::new(CapturingIngress::default(), SPACE); + let action_uri = format!("{SPACE}/did:plc:community/{MODERATION_ACTION_COLLECTION}/3m"); + projector + .project( + AUTHOR, + "3krev", + &[ + post_created("3ka", "hello"), + like_created("3kl"), + SyncEvent::ModerationAction { + uri: action_uri.clone(), + rev: "3krev".to_string(), + record: Some(json!({ + "val": "remove", + "subject": {"uri": format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), "cid": "bafypost"}, + })), + }, + ], + ) + .await + .unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + assert_eq!(sent.len(), 1); + let ops = &sent[0].ops; + assert_eq!(ops.len(), 3); + let collections: Vec<&str> = ops.iter().map(|op| op.collection.as_str()).collect(); + assert!(collections.contains(&POST_COLLECTION)); + assert!(collections.contains(&LIKE_COLLECTION)); + assert!(collections.contains(&MODERATION_ACTION_COLLECTION)); + let flagged = ops + .iter() + .find(|op| op.collection == MODERATION_ACTION_COLLECTION) + .unwrap(); + assert_eq!(flagged.operation, ProjectionOperation::Flag); + assert_eq!(flagged.action_uri.as_deref(), Some(action_uri.as_str())); + let like = ops + .iter() + .find(|op| op.collection == LIKE_COLLECTION) + .unwrap(); + assert!(like.record.as_ref().unwrap()["subject"]["uri"] + .as_str() + .unwrap() + .contains("/space/")); + } + + #[tokio::test] + async fn only_the_final_state_of_a_uri_is_sent() { + let projector = AppviewProjector::new(CapturingIngress::default(), SPACE); + let uri = format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"); + projector + .project( + AUTHOR, + "3krev", + &[ + post_created("3ka", "first"), + SyncEvent::PostDeleted { + uri: uri.clone(), + author: AUTHOR.to_string(), + }, + ], + ) + .await + .unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + assert_eq!(sent[0].ops.len(), 1); + assert_eq!(sent[0].ops[0].operation, ProjectionOperation::Delete); + } + + #[tokio::test] + async fn a_batch_with_nothing_to_send_makes_no_request() { + let projector = AppviewProjector::new(CapturingIngress::default(), SPACE); + projector + .project( + AUTHOR, + "3krev", + &[SyncEvent::ModerationAction { + uri: format!("{SPACE}/did:plc:community/{MODERATION_ACTION_COLLECTION}/3m"), + rev: "3krev".to_string(), + record: None, + }], + ) + .await + .unwrap(); + assert!(projector.ingress.sent.lock().unwrap().is_empty()); + } +} diff --git a/rsky-daemon/src/config.rs b/rsky-daemon/src/config.rs index f30193b3..08bba700 100644 --- a/rsky-daemon/src/config.rs +++ b/rsky-daemon/src/config.rs @@ -9,9 +9,32 @@ use clap::Parser; )] pub struct Config { /// The space to sync: `at://{authority}/space/{type}/{skey}`. - #[arg(long, env = "DAEMON_SPACE_URI")] + #[arg(long, env = "DAEMON_SPACE_URI", default_value = "")] pub space_uri: String, + /// Managing-app base URL for dynamic space discovery. + #[arg(long, env = "DAEMON_SPACES_URL", default_value = "")] + pub spaces_url: String, + /// Managing-app API key for dynamic space discovery. + #[arg( + long, + env = "DAEMON_SPACES_API_KEY", + default_value = "", + hide_env_values = true + )] + pub spaces_api_key: String, + /// Optional authority filter for dynamic discovery; unset, the daemon + /// accepts spaces from every authority the managing app serves. + #[arg(long, env = "DAEMON_AUTHORITY_DID", default_value = "")] + pub authority_did: String, + /// Space type accepted from the managing app. + #[arg( + long, + env = "DAEMON_SPACE_TYPE", + default_value = "community.blacksky.feed" + )] + pub space_type: String, + /// The space host (authority) base URL, for listRepos + credential mint. #[arg(long, env = "DAEMON_SPACE_HOST_URL")] pub space_host_url: String, @@ -48,6 +71,39 @@ pub struct Config { )] pub static_credential: String, + #[arg( + long, + env = "DAEMON_SPACE_HOST_MINT_TOKEN", + default_value = "", + hide_env_values = true + )] + pub space_host_mint_token: String, + #[arg(long, env = "DAEMON_DPOP_KEY_PATH", default_value = "")] + pub dpop_key_path: String, + #[arg( + long, + env = "DAEMON_SERVICE_SIGNING_KEY_HEX", + default_value = "", + hide_env_values = true + )] + pub service_signing_key_hex: String, + + /// Feed service base URL for record projection; empty leaves the daemon + /// index-only. + #[arg(long, env = "DAEMON_FEEDS_URL", default_value = "")] + pub feeds_url: String, + /// The `aud` the feed service requires on projection calls. + #[arg(long, env = "DAEMON_FEEDS_SERVICE_DID", default_value = "")] + pub feeds_service_did: String, + + /// Appview base URL for record projection; empty leaves the daemon + /// index-only. + #[arg(long, env = "DAEMON_APPVIEW_URL", default_value = "")] + pub appview_url: String, + /// The `aud` the appview requires on projection calls. + #[arg(long, env = "DAEMON_APPVIEW_SERVICE_DID", default_value = "")] + pub appview_service_did: String, + /// Bind address for the notify listener. #[arg(long, env = "DAEMON_NOTIFY_BIND", default_value = "127.0.0.1:8055")] pub notify_bind: String, @@ -64,9 +120,61 @@ pub struct Config { /// Seconds between writer-set sweeps (self-healing when notifications drop). #[arg(long, env = "DAEMON_SWEEP_INTERVAL_SECS", default_value_t = 300)] pub sweep_interval_secs: u64, + + /// Sweeps a batch refused with "author is not admitted" is re-attempted + /// for before it is parked. Admission can arrive late; it can also never + /// arrive, so the retry is slow but finite. + #[arg( + long, + env = "DAEMON_DENIAL_PARK_AFTER", + default_value_t = crate::journal::DENIAL_PARK_AFTER + )] + pub denial_park_after: u32, + + /// PLC directory to resolve DIDs against. Empty uses the public one, + /// which cannot know about a local or staging network. + #[arg(long, env = "DAEMON_PLC_URL", default_value = "")] + pub plc_url: String, } impl Config { + pub fn validate(&self) -> std::result::Result<(), String> { + if self.space_uri.is_empty() && self.spaces_url.is_empty() { + return Err("DAEMON_SPACE_URI or DAEMON_SPACES_URL is required".into()); + } + if !self.spaces_url.is_empty() && self.spaces_api_key.is_empty() { + return Err("DAEMON_SPACES_API_KEY is required with DAEMON_SPACES_URL".into()); + } + if !self.feeds_url.is_empty() && self.feeds_service_did.is_empty() { + return Err("DAEMON_FEEDS_SERVICE_DID is required with DAEMON_FEEDS_URL".into()); + } + if !self.appview_url.is_empty() && self.appview_service_did.is_empty() { + return Err("DAEMON_APPVIEW_SERVICE_DID is required with DAEMON_APPVIEW_URL".into()); + } + Ok(()) + } + + /// `(base url, audience)` for the feed service, or `None` when the daemon + /// is configured index-only. + pub fn feeds_projection(&self) -> Option<(&str, &str)> { + (!self.feeds_url.is_empty()) + .then_some((self.feeds_url.as_str(), self.feeds_service_did.as_str())) + } + + /// `(base url, audience)` for the appview, or `None` when the daemon is + /// configured index-only. + pub fn appview_projection(&self) -> Option<(&str, &str)> { + (!self.appview_url.is_empty()) + .then_some((self.appview_url.as_str(), self.appview_service_did.as_str())) + } + + pub fn authority_filter(&self) -> Option { + (!self.authority_did.is_empty()).then(|| self.authority_did.clone()) + } + + pub fn plc_url(&self) -> Option { + (!self.plc_url.is_empty()).then(|| self.plc_url.clone()) + } pub fn repo_host_url(&self) -> &str { if self.repo_host_url.is_empty() { &self.space_host_url @@ -108,6 +216,7 @@ mod tests { let cfg = Config::try_parse_from(REQUIRED).unwrap(); assert_eq!(cfg.sweep_interval_secs, 300); + assert_eq!(cfg.denial_park_after, crate::journal::DENIAL_PARK_AFTER); assert_eq!(cfg.index_db_path, ""); assert_eq!(cfg.notify_bind, "127.0.0.1:8055"); assert_eq!(cfg.repo_host_url(), "https://host.example"); @@ -115,6 +224,9 @@ mod tests { assert_eq!(cfg.pds_url, ""); assert_eq!(cfg.pds_access_token, ""); assert_eq!(cfg.static_credential, ""); + assert_eq!(cfg.plc_url(), None); + assert_eq!(cfg.feeds_projection(), None); + assert_eq!(cfg.appview_projection(), None); let mut cfg = Config::try_parse_from(REQUIRED.into_iter().chain([ "--repo-host-url", @@ -140,6 +252,12 @@ mod tests { ("DAEMON_NOTIFY_BIND", "0.0.0.0:9000"), ("DAEMON_INDEX_DB_PATH", "/data/space.sqlite"), ("DAEMON_SWEEP_INTERVAL_SECS", "60"), + ("DAEMON_DENIAL_PARK_AFTER", "7"), + ("DAEMON_PLC_URL", "http://localhost:2582"), + ("DAEMON_FEEDS_URL", "http://localhost:8080"), + ("DAEMON_FEEDS_SERVICE_DID", "did:web:feeds.example"), + ("DAEMON_APPVIEW_URL", "http://localhost:2584"), + ("DAEMON_APPVIEW_SERVICE_DID", "did:web:appview.example"), ]; for (k, v) in env { std::env::set_var(k, v); @@ -154,8 +272,85 @@ mod tests { assert_eq!(cfg.pds_access_token, "access.jwt"); assert_eq!(cfg.static_credential, "sc.jwt"); assert_eq!(cfg.notify_bind, "0.0.0.0:9000"); + assert_eq!(cfg.plc_url(), Some("http://localhost:2582".to_string())); assert_eq!(cfg.notify_endpoint(), "http://0.0.0.0:9000"); assert_eq!(cfg.index_db_path, "/data/space.sqlite"); assert_eq!(cfg.sweep_interval_secs, 60); + assert_eq!(cfg.denial_park_after, 7); + assert_eq!( + cfg.feeds_projection(), + Some(("http://localhost:8080", "did:web:feeds.example")) + ); + assert_eq!( + cfg.appview_projection(), + Some(("http://localhost:2584", "did:web:appview.example")) + ); + assert!(cfg.validate().is_ok()); + + let mut audienceless = Config::try_parse_from( + REQUIRED + .into_iter() + .chain(["--feeds-url", "http://localhost:8080"]), + ) + .unwrap(); + assert!(audienceless.validate().is_err()); + audienceless + .try_update_from(["rsky-daemon", "--appview-url", "http://localhost:2584"]) + .unwrap(); + assert!(audienceless.validate().is_err()); + + let discovery_only = Config::try_parse_from([ + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + "--spaces-url", + "https://feeds.example", + "--spaces-api-key", + "key", + "--authority-did", + "did:plc:authority", + ]) + .unwrap(); + assert!(discovery_only.validate().is_ok()); + assert_eq!( + discovery_only.authority_filter().as_deref(), + Some("did:plc:authority") + ); + let all_authorities = Config::try_parse_from([ + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + "--spaces-url", + "https://feeds.example", + "--spaces-api-key", + "key", + ]) + .unwrap(); + assert!(all_authorities.validate().is_ok()); + assert!(all_authorities.authority_filter().is_none()); + let keyless_discovery = Config::try_parse_from([ + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + "--spaces-url", + "https://feeds.example", + ]) + .unwrap(); + assert!(keyless_discovery.validate().is_err()); + let neither = Config::try_parse_from([ + "rsky-daemon", + "--space-host-url", + "https://host.example", + "--service-identity", + "did:web:syncer.example", + ]) + .unwrap(); + assert!(neither.validate().is_err()); } } diff --git a/rsky-daemon/src/credentials.rs b/rsky-daemon/src/credentials.rs index 255ff33f..23ce9455 100644 --- a/rsky-daemon/src/credentials.rs +++ b/rsky-daemon/src/credentials.rs @@ -5,11 +5,13 @@ use async_trait::async_trait; use rsky_lexicon::com::atproto::space::GetDelegationTokenOutput; use rsky_space::credential::{decode, CREDENTIAL_TTL_SECS}; +use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Mutex; use crate::error::Result; use crate::xrpc::{check, http_client, net_err, SpaceHostClient}; +use crate::{service_jwt::ServiceJwtIssuer, HttpSpaceHost}; /// Seconds since the Unix epoch; the injectable-`now` boundary for tests. pub fn unix_now() -> u64 { @@ -77,6 +79,77 @@ impl CredentialSource for StaticCredential { } } +pub struct InternalCredentialProvider { + default_space: String, + mint_token: String, + issuer: ServiceJwtIssuer, + host: Arc, + cached: Mutex>, +} + +/// Adapts the shared internal provider to one worker's `CredentialSource`. +pub struct SpaceCredentialSource { + provider: Arc, + space: String, +} +impl SpaceCredentialSource { + pub fn new(provider: Arc, space: impl Into) -> Self { + Self { + provider, + space: space.into(), + } + } +} +#[async_trait] +impl CredentialSource for SpaceCredentialSource { + async fn credential(&self, now: u64) -> Result { + self.provider.credential_for(&self.space, now).await + } +} +impl InternalCredentialProvider { + pub fn new( + space: impl Into, + mint_token: impl Into, + issuer: ServiceJwtIssuer, + host: Arc, + ) -> Self { + Self { + default_space: space.into(), + mint_token: mint_token.into(), + issuer, + host, + cached: Mutex::new(HashMap::new()), + } + } + + /// Get a credential bound to this daemon's DPoP key for any discovered + /// space. The service identity is shared, but credentials never cross a + /// space boundary in the cache. + pub async fn credential_for(&self, space: &str, now: u64) -> Result { + let mut cached = self.cached.lock().await; + if let Some((jwt, exp)) = cached.get(space) { + if now < exp.saturating_sub(CREDENTIAL_TTL_SECS / 5) { + return Ok(jwt.clone()); + } + } + let authority = rsky_space::space_id::SpaceId::parse(space)?.authority; + let service_jwt = self.issuer.mint(&authority, now, &format!("mint-{now}"))?; + let jwt = self + .host + .mint_internal_credential(space, &service_jwt, &self.mint_token) + .await?; + let exp = decode(&jwt)?.claims.exp; + cached.insert(space.to_string(), (jwt.clone(), exp)); + Ok(jwt) + } +} +#[async_trait] +impl CredentialSource for InternalCredentialProvider { + async fn credential(&self, now: u64) -> Result { + self.credential_for(&self.default_space, now).await + } +} + /// Mints and caches a space credential, re-minting once 80% of the credential /// TTL has elapsed so the daemon never presents one near expiry. pub struct CredentialProvider { @@ -224,6 +297,33 @@ mod tests { assert_eq!(host.mints.load(Ordering::SeqCst), 2); } + #[tokio::test] + async fn internal_credentials_are_cached_per_space() { + let a = SPACE; + let b = "at://did:plc:authority/space/community.blacksky.feed/other"; + let provider = InternalCredentialProvider::new( + a, + "mint-token", + ServiceJwtIssuer::from_hex("did:plc:daemon", &"11".repeat(32)).unwrap(), + Arc::new(HttpSpaceHost::new( + "http://127.0.0.1:9", + Arc::new(crate::dpop::DpopSigner::generate().unwrap()), + )), + ); + provider.cached.lock().await.extend([ + (a.into(), (credential_jwt(1000), 8200)), + (b.into(), (credential_jwt(2000), 9200)), + ]); + assert_eq!( + provider.credential_for(a, 3000).await.unwrap(), + credential_jwt(1000) + ); + assert_eq!( + provider.credential_for(b, 3000).await.unwrap(), + credential_jwt(2000) + ); + } + struct GarbageHost; #[async_trait] impl SpaceHostClient for GarbageHost { diff --git a/rsky-daemon/src/dpop.rs b/rsky-daemon/src/dpop.rs index c649bff9..792d07aa 100644 --- a/rsky-daemon/src/dpop.rs +++ b/rsky-daemon/src/dpop.rs @@ -16,6 +16,7 @@ use base64::Engine as _; use rsky_oauth::jwk::{EcCurve, Jwk}; use rsky_oauth::jwt::{sign, JwtClaims, JwtHeader}; use sha2::{Digest, Sha256}; +use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -47,6 +48,30 @@ impl DpopSigner { )) } + /// Load the daemon's stable proof key, creating it once when absent. + pub fn load_or_generate(path: impl AsRef) -> Result { + let path = path.as_ref(); + if path.exists() { + let bytes = std::fs::read(path).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + return Jwk::from_private_key_bytes(EcCurve::P256, &bytes) + .map(|key| Self { + key, + counter: AtomicU64::new(0), + }) + .map_err(|e| DaemonError::Xrpc(e.to_string())); + } + let signer = Self::generate()?; + let bytes = signer + .key + .private_key_bytes() + .map_err(|e| DaemonError::Xrpc(e.to_string()))?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + } + std::fs::write(path, bytes).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + Ok(signer) + } + /// RFC 7638 thumbprint — what a credential minted for this signer carries /// in `cnf.jkt`. pub fn thumbprint(&self) -> String { diff --git a/rsky-daemon/src/engine.rs b/rsky-daemon/src/engine.rs index fce0e37e..3960aa8d 100644 --- a/rsky-daemon/src/engine.rs +++ b/rsky-daemon/src/engine.rs @@ -11,7 +11,7 @@ use rsky_space::commit::verify_commit; use rsky_space::lthash::element; use crate::error::{DaemonError, Result}; -use crate::index::SpaceIndex; +use crate::index::{IndexMutation, SpaceIndex}; use crate::repohost::RepoHostClient; /// Resolves an author's atproto signing `did:key` to verify their commit. @@ -49,6 +49,7 @@ pub async fn sync_repo( let mut prev_mismatches = 0usize; let mut cursor: Option = None; let mut last_rev: Option = None; + let mut mutations: Vec = Vec::new(); loop { let page = client @@ -70,20 +71,25 @@ pub async fn sync_repo( } match &op.cid { Some(cid) => { + let value = op.value.as_ref().map(|v| v.to_vec()); index - .upsert( - did, - &op.collection, - &op.rkey, - cid, - &op.rev, - op.value.as_ref().map(|v| v.to_vec()), - ) + .upsert(did, &op.collection, &op.rkey, cid, &op.rev, value.clone()) .await?; lth.add(&element(&op.collection, &op.rkey, cid)); + mutations.push(IndexMutation::Upsert { + collection: op.collection.clone(), + rkey: op.rkey.clone(), + cid: cid.clone(), + rev: op.rev.clone(), + value, + }); } None => { index.delete(did, &op.collection, &op.rkey).await?; + mutations.push(IndexMutation::Delete { + collection: op.collection.clone(), + rkey: op.rkey.clone(), + }); } } ops_applied += 1; @@ -109,6 +115,7 @@ pub async fn sync_repo( if lth.hash().as_slice() != commit.hash.as_slice() { return Err(DaemonError::Diverged(did.to_string())); } + index.journal_batch(did, &commit.rev, &mutations).await?; index.save_head(did, &commit.rev, <h).await?; return Ok(SyncOutcome { ops_applied, @@ -126,6 +133,10 @@ pub async fn sync_repo( // Exhausted the oplog without a terminal commit: advance to the last op's // rev without a hash check; a later sync carries the commit. if let Some(rev) = &last_rev { + // The head advances here without a commit to check it against, so a + // later sync will never replay these ops: journal them now or the + // projection loses them outright. + index.journal_batch(did, rev, &mutations).await?; index.save_head(did, rev, <h).await?; } Ok(SyncOutcome { diff --git a/rsky-daemon/src/error.rs b/rsky-daemon/src/error.rs index 608772c3..227af00f 100644 --- a/rsky-daemon/src/error.rs +++ b/rsky-daemon/src/error.rs @@ -16,8 +16,29 @@ pub enum DaemonError { /// full-state recovery (`getRepo`). #[error("history unavailable: {0}")] HistoryUnavailable(String), + /// A projection destination was unreachable or overloaded. Distinct from + /// [`DaemonError::Xrpc`] because it must not consume the batch's failure + /// budget: an outage is not a bad batch. + #[error("projection destination unavailable: {0}")] + RetryableProjection(String), + /// A projection destination refused the batch because its author is not + /// admitted to the space. Admission is state, not a property of the batch: + /// it flips when a membership write propagates, so this must not spend the + /// poison budget. It gets its own slow budget instead. + #[error("projection destination denied admission: {0}")] + AdmissionDenied(String), #[error(transparent)] Space(#[from] rsky_space::SpaceError), } +impl DaemonError { + pub fn is_retryable_projection(&self) -> bool { + matches!(self, Self::RetryableProjection(_)) + } + + pub fn is_admission_denied(&self) -> bool { + matches!(self, Self::AdmissionDenied(_)) + } +} + pub type Result = std::result::Result; diff --git a/rsky-daemon/src/feeds.rs b/rsky-daemon/src/feeds.rs new file mode 100644 index 00000000..54484e2a --- /dev/null +++ b/rsky-daemon/src/feeds.rs @@ -0,0 +1,499 @@ +//! Projection to the feed service's record ingress. + +use async_trait::async_trait; +use serde::Serialize; +use serde_json::Value; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use tracing::debug; + +use crate::error::{DaemonError, Result}; +use crate::projection::Projector; +use crate::router::{SyncEvent, POST_COLLECTION}; +use crate::service_jwt::ServiceJwtIssuer; +use crate::unix_now; + +pub const PROJECT_RECORDS_LXM: &str = "community.blacksky.space.projectRecords"; +pub const ACK_SYNCERS_OBSERVED_LXM: &str = "community.blacksky.space.ackSyncersObserved"; +/// The receiving side rejects a token whose lifetime exceeds five minutes; +/// the margin absorbs clock skew between the two services. +const TOKEN_TTL_SECS: u64 = 240; + +#[derive(Clone, Debug, Serialize, PartialEq)] +pub struct ProjectRecordsRequest { + pub ops: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ProjectionOperation { + Create, + Delete, + Flag, + Unflag, +} + +#[derive(Clone, Debug, Serialize, PartialEq)] +pub struct ProjectRecord { + pub space: String, + pub author: String, + pub uri: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub cid: Option, + pub revision: String, + pub operation: ProjectionOperation, + pub collection: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub record: Option, + #[serde(rename = "actionUri", skip_serializing_if = "Option::is_none")] + pub action_uri: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub neg: Option, +} + +#[async_trait] +pub trait ProjectionIngress: Send + Sync { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()>; +} + +#[async_trait] +impl ProjectionIngress for std::sync::Arc { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()> { + self.as_ref().project_records(request).await + } +} + +/// Tells the feed service this space now has a syncer keeping it current. +/// Until it hears that, it refuses the space's projections. +#[async_trait] +pub trait SpaceLifecycleAcker: Send + Sync { + async fn acknowledge_sync(&self, space: &str, generation: i64) -> Result<()>; +} + +/// Posts batches to a service that accepts `projectRecords`, authenticated +/// with this daemon's own service identity. +pub struct HttpProjectionIngress { + target: &'static str, + base_url: String, + audience: String, + issuer: ServiceJwtIssuer, + http: reqwest::Client, +} + +impl HttpProjectionIngress { + pub fn new( + target: &'static str, + base_url: impl Into, + service_identity: impl Into, + audience: impl Into, + signing_key_hex: &str, + ) -> Result { + Ok(Self { + target, + base_url: base_url.into().trim_end_matches('/').to_string(), + audience: audience.into(), + issuer: ServiceJwtIssuer::from_hex(service_identity, signing_key_hex)?, + http: reqwest::Client::new(), + }) + } + + fn service_jwt(&self, lxm: &str) -> Result { + static JTI: AtomicU64 = AtomicU64::new(1); + let now = unix_now(); + let jti = format!("{now}-{}", JTI.fetch_add(1, Ordering::Relaxed)); + self.issuer + .mint_for(&self.audience, lxm, now, TOKEN_TTL_SECS, &jti) + } +} + +#[async_trait] +impl SpaceLifecycleAcker for HttpProjectionIngress { + async fn acknowledge_sync(&self, space: &str, generation: i64) -> Result<()> { + let response = self + .http + .post(format!("{}/xrpc/{ACK_SYNCERS_OBSERVED_LXM}", self.base_url)) + .bearer_auth(self.service_jwt(ACK_SYNCERS_OBSERVED_LXM)?) + .json(&serde_json::json!({"space": space, "generation": generation})) + .send() + .await + .map_err(|error| DaemonError::Xrpc(error.to_string()))?; + if !response.status().is_success() { + return Err(DaemonError::Xrpc(format!( + "{} {ACK_SYNCERS_OBSERVED_LXM} returned {}", + self.target, + response.status() + ))); + } + Ok(()) + } +} + +#[async_trait] +impl ProjectionIngress for HttpProjectionIngress { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()> { + let response = self + .http + .post(format!("{}/xrpc/{PROJECT_RECORDS_LXM}", self.base_url)) + .bearer_auth(self.service_jwt(PROJECT_RECORDS_LXM)?) + .json(request) + .send() + .await + .map_err(|error| { + DaemonError::RetryableProjection(format!("{} unreachable: {error}", self.target)) + })?; + let status = response.status(); + if status.is_success() { + return Ok(()); + } + let message = format!("{} {PROJECT_RECORDS_LXM} returned {status}", self.target); + if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { + return Err(DaemonError::RetryableProjection(message)); + } + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(DaemonError::AdmissionDenied(message)); + } + Err(DaemonError::Xrpc(message)) + } +} + +/// Projects a space's posts and their moderation state to the feed service. +pub struct FeedsProjector { + ingress: I, + space: String, +} + +impl FeedsProjector { + pub fn new(ingress: I, space: impl Into) -> Self { + Self { + ingress, + space: space.into(), + } + } +} + +#[async_trait] +impl Projector for FeedsProjector { + fn name(&self) -> &'static str { + "feeds" + } + + async fn project(&self, author: &str, revision: &str, events: &[SyncEvent]) -> Result<()> { + // One URI may change more than once in a batch; only its final state + // is worth sending. + let mut final_posts: BTreeMap<&str, &SyncEvent> = BTreeMap::new(); + let mut ops = Vec::new(); + for event in events { + match event { + SyncEvent::PostCreated { .. } | SyncEvent::PostDeleted { .. } => { + final_posts.insert(event.uri(), event); + } + SyncEvent::ModerationAction { uri, record, .. } => { + if let Some(op) = self.moderation_op(author, revision, uri, record.as_ref()) { + ops.push(op); + } + } + other => debug!(uri = other.uri(), "feeds projection ignores this event"), + } + } + for event in final_posts.into_values() { + match event { + SyncEvent::PostCreated { + uri, cid, record, .. + } => ops.push(ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.clone(), + cid: Some(cid.clone()), + revision: revision.to_string(), + operation: ProjectionOperation::Create, + collection: POST_COLLECTION.to_string(), + record: Some(record.clone()), + action_uri: None, + neg: None, + }), + SyncEvent::PostDeleted { uri, .. } => ops.push(ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.clone(), + cid: None, + revision: revision.to_string(), + operation: ProjectionOperation::Delete, + collection: POST_COLLECTION.to_string(), + record: None, + action_uri: None, + neg: None, + }), + _ => unreachable!("only post events were retained"), + } + } + if ops.is_empty() { + return Ok(()); + } + self.ingress + .project_records(&ProjectRecordsRequest { ops }) + .await + } +} + +impl FeedsProjector { + fn moderation_op( + &self, + author: &str, + revision: &str, + uri: &str, + record: Option<&Value>, + ) -> Option { + let record = record?; + if record["val"].as_str() != Some("remove") { + return None; + } + // A lift is its own signed record carrying the action it reverses, + // so it names that action's URI rather than its own. + let neg = record["neg"].as_bool() == Some(true); + let (action_uri, cid) = if neg { + (record["action"]["uri"].as_str(), None) + } else { + (Some(uri), record["subject"]["cid"].as_str()) + }; + let Some(action_uri) = action_uri else { + debug!(%uri, "moderation action has no action URI"); + return None; + }; + if !neg && cid.is_none() { + debug!(%uri, "moderation action has no strong post reference"); + return None; + } + Some(ProjectRecord { + space: self.space.clone(), + author: author.to_string(), + uri: uri.to_string(), + cid: cid.map(str::to_string), + revision: revision.to_string(), + operation: if neg { + ProjectionOperation::Unflag + } else { + ProjectionOperation::Flag + }, + collection: POST_COLLECTION.to_string(), + record: None, + action_uri: Some(action_uri.to_string()), + neg: Some(neg), + }) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use serde_json::json; + use std::sync::Mutex; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + pub(crate) const SPACE: &str = "at://did:plc:community/space/community.blacksky.feed/private"; + pub(crate) const AUTHOR: &str = "did:plc:alice"; + + #[derive(Default)] + pub(crate) struct CapturingIngress { + pub sent: Mutex>, + } + + #[async_trait] + impl ProjectionIngress for CapturingIngress { + async fn project_records(&self, request: &ProjectRecordsRequest) -> Result<()> { + self.sent.lock().unwrap().push(request.clone()); + Ok(()) + } + } + + pub(crate) fn post_created(rkey: &str, text: &str) -> SyncEvent { + SyncEvent::PostCreated { + uri: format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/{rkey}"), + author: AUTHOR.to_string(), + cid: "bafypost".to_string(), + rev: "3krev".to_string(), + record: json!({"$type": POST_COLLECTION, "text": text, "createdAt": "2026-08-19T00:00:00Z"}), + } + } + + fn ingress(server: &MockServer) -> HttpProjectionIngress { + HttpProjectionIngress::new( + "feeds", + server.uri(), + "did:web:syncer.example", + "did:web:feeds.example", + &hex::encode([1_u8; 32]), + ) + .unwrap() + } + + #[tokio::test] + async fn a_batch_projects_final_post_state_once() { + let projector = FeedsProjector::new(CapturingIngress::default(), SPACE); + let deleted = SyncEvent::PostDeleted { + uri: format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), + author: AUTHOR.to_string(), + }; + projector + .project( + AUTHOR, + "3krev", + &[ + post_created("3ka", "first"), + deleted, + post_created("3kb", "second"), + ], + ) + .await + .unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + assert_eq!(sent.len(), 1); + let ops = &sent[0].ops; + assert_eq!(ops.len(), 2); + assert_eq!(ops[0].operation, ProjectionOperation::Delete); + assert_eq!( + ops[0].uri, + format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka") + ); + assert_eq!(ops[1].operation, ProjectionOperation::Create); + assert_eq!(ops[1].record.as_ref().unwrap()["text"], "second"); + assert_eq!(ops[1].space, SPACE); + assert_eq!(ops[1].revision, "3krev"); + } + + #[tokio::test] + async fn moderation_removals_and_lifts_become_flags() { + let projector = FeedsProjector::new(CapturingIngress::default(), SPACE); + let action_uri = + format!("{SPACE}/did:plc:community/community.blacksky.moderation.action/3m"); + let events = vec![ + SyncEvent::ModerationAction { + uri: action_uri.clone(), + rev: "3krev".to_string(), + record: Some(json!({ + "val": "remove", + "subject": {"uri": format!("{SPACE}/{AUTHOR}/{POST_COLLECTION}/3ka"), "cid": "bafypost"}, + })), + }, + SyncEvent::ModerationAction { + uri: format!("{SPACE}/did:plc:community/community.blacksky.moderation.action/3n"), + rev: "3krev".to_string(), + record: Some(json!({ + "val": "remove", + "neg": true, + "action": {"uri": action_uri}, + })), + }, + // Neither a removal nor decodable as one: dropped, not sent. + SyncEvent::ModerationAction { + uri: format!("{SPACE}/did:plc:community/community.blacksky.moderation.action/3o"), + rev: "3krev".to_string(), + record: Some(json!({"val": "spam"})), + }, + ]; + projector.project(AUTHOR, "3krev", &events).await.unwrap(); + + let sent = projector.ingress.sent.lock().unwrap(); + let ops = &sent[0].ops; + assert_eq!(ops.len(), 2); + assert_eq!(ops[0].operation, ProjectionOperation::Flag); + assert_eq!(ops[0].neg, Some(false)); + assert_eq!(ops[0].cid.as_deref(), Some("bafypost")); + assert_eq!(ops[1].operation, ProjectionOperation::Unflag); + assert_eq!(ops[1].neg, Some(true)); + assert_eq!(ops[1].action_uri.as_deref(), Some(action_uri.as_str())); + } + + #[tokio::test] + async fn a_batch_with_nothing_to_send_makes_no_request() { + let projector = FeedsProjector::new(CapturingIngress::default(), SPACE); + projector + .project( + AUTHOR, + "3krev", + &[SyncEvent::LikeDeleted { + uri: format!("{SPACE}/{AUTHOR}/app.bsky.feed.like/3ka"), + author: AUTHOR.to_string(), + }], + ) + .await + .unwrap(); + assert!(projector.ingress.sent.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_batch_reaches_the_ingress_with_a_method_bound_service_token() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/xrpc/{PROJECT_RECORDS_LXM}"))) + .and(wiremock::matchers::header_regex( + "authorization", + "^Bearer ", + )) + .and(wiremock::matchers::body_string_contains("\"create\"")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({}))) + .mount(&server) + .await; + + let projector = FeedsProjector::new(ingress(&server), SPACE); + projector + .project(AUTHOR, "3krev", &[post_created("3ka", "hello")]) + .await + .unwrap(); + } + + #[tokio::test] + async fn destination_outages_are_retryable_and_rejections_are_not() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let error = ingress(&server) + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err(); + assert!(error.is_retryable_projection()); + + let rejecting = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400)) + .mount(&rejecting) + .await; + let error = ingress(&rejecting) + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err(); + assert!(!error.is_retryable_projection()); + assert!(!error.is_admission_denied()); + + let denying = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(401).set_body_string( + r#"{"error":"NotAuthorized","message":"author is not admitted to this space"}"#, + )) + .mount(&denying) + .await; + let error = ingress(&denying) + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err(); + assert!(error.is_admission_denied()); + assert!(!error.is_retryable_projection()); + + let unreachable = HttpProjectionIngress::new( + "feeds", + "http://127.0.0.1:1", + "did:web:syncer.example", + "did:web:feeds.example", + &hex::encode([1_u8; 32]), + ) + .unwrap(); + assert!(unreachable + .project_records(&ProjectRecordsRequest { ops: vec![] }) + .await + .unwrap_err() + .is_retryable_projection()); + } +} diff --git a/rsky-daemon/src/index.rs b/rsky-daemon/src/index.rs index eac02a41..01b9911e 100644 --- a/rsky-daemon/src/index.rs +++ b/rsky-daemon/src/index.rs @@ -17,6 +17,47 @@ pub struct IndexedRecord { pub value: Option>, } +/// One record change from a synced batch, as journaled for projection. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum IndexMutation { + Upsert { + collection: String, + rkey: String, + cid: String, + rev: String, + value: Option>, + }, + Delete { + collection: String, + rkey: String, + }, +} + +impl IndexMutation { + pub fn collection(&self) -> &str { + match self { + Self::Upsert { collection, .. } | Self::Delete { collection, .. } => collection, + } + } + pub fn rkey(&self) -> &str { + match self { + Self::Upsert { rkey, .. } | Self::Delete { rkey, .. } => rkey, + } + } +} + +/// A journaled batch a projector has not yet delivered. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JournaledBatch { + pub author: String, + pub rev: String, + pub mutations: Vec, + /// How many times this projector has been told the author is not admitted. + /// Non-zero puts the batch in the slow lane: retried once per sweep, never + /// on the fast drain. + pub denials: u32, +} + /// Per-author sync state + records the daemon holds for a space. #[async_trait] pub trait SpaceIndex: Send + Sync { @@ -40,6 +81,61 @@ pub trait SpaceIndex: Send + Sync { async fn delete(&self, did: &str, collection: &str, rkey: &str) -> Result<()>; /// Persist the author's new head (rev + accumulator) after a synced batch. async fn save_head(&self, did: &str, rev: &str, lthash: &LtHash) -> Result<()>; + /// Record a synced batch for later projection, keyed by `(did, rev)`. + /// Journaling precedes the head write, so a crash between the two replays + /// the batch instead of losing it; the key makes that replay a no-op. + async fn journal_batch( + &self, + _did: &str, + _rev: &str, + _mutations: &[IndexMutation], + ) -> Result<()> { + Ok(()) + } + /// Batches past this projector's independent `(author, rev)` cursor. + async fn pending_batches(&self, _projector: &str) -> Result> { + Ok(Vec::new()) + } + async fn advance_projector_cursor( + &self, + _projector: &str, + _did: &str, + _rev: &str, + ) -> Result<()> { + Ok(()) + } + /// Returns the durable failure count for this batch, marking it + /// dead-lettered once the count reaches `dead_letter_after`. + async fn record_projection_failure( + &self, + _projector: &str, + _did: &str, + _rev: &str, + _error: &str, + _dead_letter_after: u32, + ) -> Result { + Ok(0) + } + /// Returns the durable denial count for this batch, parking it once the + /// count reaches `park_after`. Denials are counted separately from the + /// poison budget: a denial says the space's admission state is not (yet) + /// what the batch needs, not that the batch is bad. + async fn record_projection_denial( + &self, + _projector: &str, + _did: &str, + _rev: &str, + _error: &str, + _park_after: u32, + ) -> Result { + Ok(0) + } + /// Drop journal rows every one of `projectors` has advanced past, along + /// with their retryable failure rows. Dead-lettered and parked batches are + /// retained until explicitly cleared. + async fn prune_journal(&self, _projectors: &[&str]) -> Result { + Ok(0) + } /// Enumerate an author's indexed records as `(collection, rkey, cid)`, /// used to diff against a recovered full-state CAR. async fn list_paths(&self, did: &str) -> Result>; @@ -59,10 +155,32 @@ struct AuthorState { records: HashMap, } +#[derive(Default, Clone, Copy)] +struct FailureState { + attempts: u32, + dead_lettered: bool, + denials: u32, + parked: bool, +} + +impl FailureState { + fn retired(&self) -> bool { + self.dead_lettered || self.parked + } +} + +#[derive(Default)] +struct JournalState { + batches: Vec, + cursors: HashMap<(String, String), String>, + failures: HashMap<(String, String, String), FailureState>, +} + /// In-memory [`SpaceIndex`] for tests and local runs. #[derive(Default)] pub struct InMemoryIndex { authors: RwLock>, + journal: RwLock, } impl InMemoryIndex { @@ -156,6 +274,128 @@ impl SpaceIndex for InMemoryIndex { Ok(()) } + async fn journal_batch(&self, did: &str, rev: &str, mutations: &[IndexMutation]) -> Result<()> { + let mut journal = self.journal.write().unwrap(); + if journal + .batches + .iter() + .any(|b| b.author == did && b.rev == rev) + { + return Ok(()); + } + journal.batches.push(JournaledBatch { + author: did.to_string(), + rev: rev.to_string(), + mutations: mutations.to_vec(), + denials: 0, + }); + Ok(()) + } + + async fn pending_batches(&self, projector: &str) -> Result> { + let journal = self.journal.read().unwrap(); + let failure = |b: &JournaledBatch| { + journal + .failures + .get(&(projector.to_string(), b.author.clone(), b.rev.clone())) + .copied() + .unwrap_or_default() + }; + let mut pending: Vec = journal + .batches + .iter() + .filter(|b| { + journal + .cursors + .get(&(projector.to_string(), b.author.clone())) + .is_none_or(|cursor| b.rev > *cursor) + && !failure(b).retired() + }) + .map(|b| JournaledBatch { + denials: failure(b).denials, + ..b.clone() + }) + .collect(); + pending.sort_by(|a, b| (&a.author, &a.rev).cmp(&(&b.author, &b.rev))); + Ok(pending) + } + + async fn advance_projector_cursor(&self, projector: &str, did: &str, rev: &str) -> Result<()> { + self.journal + .write() + .unwrap() + .cursors + .insert((projector.to_string(), did.to_string()), rev.to_string()); + Ok(()) + } + + async fn record_projection_failure( + &self, + projector: &str, + did: &str, + rev: &str, + _error: &str, + dead_letter_after: u32, + ) -> Result { + let mut journal = self.journal.write().unwrap(); + let entry = journal + .failures + .entry((projector.to_string(), did.to_string(), rev.to_string())) + .or_default(); + entry.attempts += 1; + entry.dead_lettered = entry.attempts >= dead_letter_after; + Ok(entry.attempts) + } + + async fn record_projection_denial( + &self, + projector: &str, + did: &str, + rev: &str, + _error: &str, + park_after: u32, + ) -> Result { + let mut journal = self.journal.write().unwrap(); + let entry = journal + .failures + .entry((projector.to_string(), did.to_string(), rev.to_string())) + .or_default(); + entry.denials += 1; + entry.parked = entry.denials >= park_after; + Ok(entry.denials) + } + + async fn prune_journal(&self, projectors: &[&str]) -> Result { + if projectors.is_empty() { + return Ok(0); + } + let mut journal = self.journal.write().unwrap(); + let JournalState { + batches, + cursors, + failures, + } = &mut *journal; + let before = batches.len(); + batches.retain(|b| { + let all_passed = projectors.iter().all(|projector| { + cursors + .get(&(projector.to_string(), b.author.clone())) + .is_some_and(|cursor| *cursor >= b.rev) + }); + let retired = projectors.iter().any(|projector| { + failures + .get(&(projector.to_string(), b.author.clone(), b.rev.clone())) + .is_some_and(FailureState::retired) + }); + !all_passed || retired + }); + let pruned = before - batches.len(); + failures.retain(|(_, author, rev), state| { + state.retired() || batches.iter().any(|b| b.author == *author && b.rev == *rev) + }); + Ok(pruned) + } + async fn list_paths(&self, did: &str) -> Result> { Ok(self .authors @@ -176,6 +416,7 @@ impl SpaceIndex for InMemoryIndex { async fn purge_space(&self) -> Result<()> { self.authors.write().unwrap().clear(); + *self.journal.write().unwrap() = JournalState::default(); Ok(()) } } diff --git a/rsky-daemon/src/journal.rs b/rsky-daemon/src/journal.rs new file mode 100644 index 00000000..00d4f08a --- /dev/null +++ b/rsky-daemon/src/journal.rs @@ -0,0 +1,519 @@ +//! Journal-driven projection delivery. + +use std::collections::HashSet; +use std::sync::Arc; + +use crate::error::Result; +use crate::index::SpaceIndex; +use crate::projection::Projector; +use crate::router::Router; + +pub const DEAD_LETTER_AFTER: u32 = 3; +pub const DENIAL_PARK_AFTER: u32 = 20; + +/// Which batches a drain pass will attempt. Denied batches wait for a sweep: +/// admission changes at the pace of a membership write, not of a drain tick. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Lane { + Fast, + Sweep, +} + +/// Reads the batch journal for one projector and advances only after its +/// destination has accepted the batch. Each projector gets an independent +/// cursor, so a failed destination cannot hold the index or another +/// destination hostage. +pub struct JournalConsumer { + name: &'static str, + router: Router, + projector: Box, + dead_letter_after: u32, + denial_park_after: u32, +} + +impl JournalConsumer { + pub fn new(router: Router, projector: Box) -> Self { + Self { + name: projector.name(), + router, + projector, + dead_letter_after: DEAD_LETTER_AFTER, + denial_park_after: DENIAL_PARK_AFTER, + } + } + + pub fn with_dead_letter_after(mut self, attempts: u32) -> Self { + self.dead_letter_after = attempts; + self + } + + pub fn with_denial_park_after(mut self, sweeps: u32) -> Self { + self.denial_park_after = sweeps; + self + } + + pub fn name(&self) -> &'static str { + self.name + } + + async fn drain_with_status(&self, index: &dyn SpaceIndex, lane: Lane) -> Result<(usize, bool)> { + let mut delivered = 0; + let mut succeeded = true; + // An author whose batch was denied keeps its remaining batches in + // order: delivering a later one would advance the cursor past the + // denied batch and lose it for good once admission arrives. + let mut denied_authors: HashSet = HashSet::new(); + for batch in index.pending_batches(self.name).await? { + if denied_authors.contains(&batch.author) { + continue; + } + if batch.denials > 0 && lane == Lane::Fast { + succeeded = false; + denied_authors.insert(batch.author.clone()); + continue; + } + let events = self.router.route_batch(&batch.author, &batch.mutations); + let result = if events.is_empty() { + Ok(()) + } else { + self.projector + .project(&batch.author, &batch.rev, &events) + .await + }; + match result { + Ok(()) => { + index + .advance_projector_cursor(self.name, &batch.author, &batch.rev) + .await?; + delivered += 1; + } + Err(error) => { + succeeded = false; + if error.is_retryable_projection() { + tracing::warn!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, error = %error, "projection destination unavailable; batch remains pending without consuming its failure budget"); + continue; + } + if error.is_admission_denied() { + let denials = index + .record_projection_denial( + self.name, + &batch.author, + &batch.rev, + &error.to_string(), + self.denial_park_after, + ) + .await?; + denied_authors.insert(batch.author.clone()); + if denials >= self.denial_park_after { + tracing::error!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, denials, error = %error, "projection batch parked: admission never granted"); + } else { + tracing::warn!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, denials, error = %error, "author not admitted; batch retries on a later sweep"); + } + continue; + } + let attempts = index + .record_projection_failure( + self.name, + &batch.author, + &batch.rev, + &error.to_string(), + self.dead_letter_after, + ) + .await?; + if attempts >= self.dead_letter_after { + tracing::error!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, attempts, error = %error, "projection batch dead-lettered"); + } else { + tracing::warn!(projector = self.name, space = %self.router.space().uri(), author = %batch.author, rev = %batch.rev, attempts, error = %error, "projection batch will retry"); + } + } + } + } + Ok((delivered, succeeded)) + } + + pub async fn drain(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index, Lane::Fast) + .await + .map(|(delivered, _)| delivered) + } + + /// A drain that also re-attempts denied batches. Called once per sweep. + pub async fn drain_sweep(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index, Lane::Sweep) + .await + .map(|(delivered, _)| delivered) + } + + pub async fn drain_succeeded(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index, Lane::Fast) + .await + .map(|(_, succeeded)| succeeded) + } + + pub async fn drain_sweep_succeeded(&self, index: &dyn SpaceIndex) -> Result { + self.drain_with_status(index, Lane::Sweep) + .await + .map(|(_, succeeded)| succeeded) + } +} + +pub type SharedJournalConsumer = Arc; + +/// Drain every projector, then drop the journal rows all of them have passed. +/// Returns whether every destination accepted everything pending for it. +pub async fn drain_all(index: &dyn SpaceIndex, consumers: &[SharedJournalConsumer]) -> bool { + drain_all_in(index, consumers, Lane::Fast).await +} + +/// [`drain_all`] plus a re-attempt of every denied batch. +pub async fn drain_all_sweep(index: &dyn SpaceIndex, consumers: &[SharedJournalConsumer]) -> bool { + drain_all_in(index, consumers, Lane::Sweep).await +} + +async fn drain_all_in( + index: &dyn SpaceIndex, + consumers: &[SharedJournalConsumer], + lane: Lane, +) -> bool { + let mut succeeded = true; + for consumer in consumers { + let drained = match lane { + Lane::Fast => consumer.drain_succeeded(index).await, + Lane::Sweep => consumer.drain_sweep_succeeded(index).await, + }; + match drained { + Ok(clean) => succeeded &= clean, + Err(error) => { + succeeded = false; + tracing::warn!(projector = consumer.name(), error = %error, "projection drain failed"); + } + } + } + if consumers.is_empty() { + return succeeded; + } + let names: Vec<&str> = consumers.iter().map(|c| c.name()).collect(); + if let Err(error) = index.prune_journal(&names).await { + tracing::warn!(error = %error, "journal prune failed"); + } + succeeded +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::DaemonError; + use crate::index::{InMemoryIndex, IndexMutation}; + use crate::router::{SyncEvent, POST_COLLECTION}; + use async_trait::async_trait; + use rsky_space::record::encode_record; + use rsky_space::space_id::SpaceId; + use serde_json::json; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Mutex; + + const AUTHORITY: &str = "did:plc:community"; + const AUTHOR: &str = "did:plc:alice"; + const BOB: &str = "did:plc:bob"; + + fn router() -> Router { + Router::new( + SpaceId::new(AUTHORITY, "community.blacksky.feed", "private"), + AUTHORITY, + ) + } + + fn post_mutation(rkey: &str) -> IndexMutation { + IndexMutation::Upsert { + collection: POST_COLLECTION.to_string(), + rkey: rkey.to_string(), + cid: "bafypost".to_string(), + rev: "3krev".to_string(), + value: Some( + encode_record( + &json!({"$type": POST_COLLECTION, "text": "hi", "createdAt": "2026-08-19T00:00:00Z"}), + 64 * 1024, + ) + .unwrap(), + ), + } + } + + #[derive(Default)] + struct Recorder { + delivered: Mutex>, + fail_next: AtomicUsize, + retryable: bool, + /// Authors this destination refuses admission; cleared to admit them. + denied: Mutex>, + attempts: Mutex>, + } + + #[async_trait] + impl Projector for Recorder { + fn name(&self) -> &'static str { + "recorder" + } + async fn project(&self, did: &str, rev: &str, events: &[SyncEvent]) -> Result<()> { + self.attempts + .lock() + .unwrap() + .push((did.to_string(), rev.to_string())); + if self.denied.lock().unwrap().iter().any(|d| d == did) { + return Err(DaemonError::AdmissionDenied( + "appview projectRecords returned 401 Unauthorized".to_string(), + )); + } + if self.fail_next.load(Ordering::SeqCst) > 0 { + self.fail_next.fetch_sub(1, Ordering::SeqCst); + return Err(if self.retryable { + DaemonError::RetryableProjection("destination down".to_string()) + } else { + DaemonError::Xrpc("rejected".to_string()) + }); + } + self.delivered + .lock() + .unwrap() + .push((rev.to_string(), events.len())); + Ok(()) + } + } + + /// A handle onto a [`Recorder`] the test keeps, so admission can flip + /// between drains. + struct Shared(Arc); + + #[async_trait] + impl Projector for Shared { + fn name(&self) -> &'static str { + "recorder" + } + async fn project(&self, did: &str, rev: &str, events: &[SyncEvent]) -> Result<()> { + self.0.project(did, rev, events).await + } + } + + fn shared_consumer(recorder: &Arc) -> JournalConsumer { + JournalConsumer::new(router(), Box::new(Shared(recorder.clone()))) + } + + async fn journal(index: &InMemoryIndex, author: &str, rev: &str, rkey: &str) { + index + .journal_batch(author, rev, &[post_mutation(rkey)]) + .await + .unwrap(); + } + + #[tokio::test] + async fn a_delivered_batch_advances_its_cursor_once() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + let consumer = JournalConsumer::new(router(), Box::::default()); + + assert_eq!(consumer.drain(&index).await.unwrap(), 1); + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + } + + #[tokio::test] + async fn a_retryable_failure_keeps_the_batch_without_spending_its_budget() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + let projector = Recorder { + retryable: true, + ..Default::default() + }; + projector.fail_next.store(1, Ordering::SeqCst); + let consumer = + JournalConsumer::new(router(), Box::new(projector)).with_dead_letter_after(1); + + assert!(!consumer.drain_succeeded(&index).await.unwrap()); + assert_eq!(index.pending_batches("recorder").await.unwrap().len(), 1); + assert_eq!(consumer.drain(&index).await.unwrap(), 1); + } + + #[tokio::test] + async fn a_rejected_batch_dead_letters_after_its_budget() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + let projector = Recorder::default(); + projector.fail_next.store(5, Ordering::SeqCst); + let consumer = + JournalConsumer::new(router(), Box::new(projector)).with_dead_letter_after(2); + + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + assert_eq!(index.pending_batches("recorder").await.unwrap().len(), 1); + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_batch_with_nothing_to_project_still_advances() { + let index = InMemoryIndex::new(); + index + .journal_batch( + AUTHOR, + "3krev", + &[IndexMutation::Delete { + collection: "app.bsky.graph.follow".to_string(), + rkey: "3ka".to_string(), + }], + ) + .await + .unwrap(); + let consumer = JournalConsumer::new(router(), Box::::default()); + + assert_eq!(consumer.drain(&index).await.unwrap(), 1); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn one_stalled_projector_does_not_hold_back_another() { + let index = InMemoryIndex::new(); + index + .journal_batch(AUTHOR, "3krev", &[post_mutation("3ka")]) + .await + .unwrap(); + + struct Stalled; + #[async_trait] + impl Projector for Stalled { + fn name(&self) -> &'static str { + "stalled" + } + async fn project(&self, _did: &str, _rev: &str, _events: &[SyncEvent]) -> Result<()> { + Err(DaemonError::RetryableProjection("down".to_string())) + } + } + + let healthy: SharedJournalConsumer = + Arc::new(JournalConsumer::new(router(), Box::::default())); + let stalled: SharedJournalConsumer = + Arc::new(JournalConsumer::new(router(), Box::new(Stalled))); + drain_all(&index, &[healthy.clone(), stalled.clone()]).await; + + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + assert_eq!(index.pending_batches("stalled").await.unwrap().len(), 1); + } + + #[tokio::test] + async fn a_denied_batch_projects_once_when_admission_arrives() { + let index = InMemoryIndex::new(); + journal(&index, BOB, "3krev", "3ka").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder).with_denial_park_after(5); + + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + assert_eq!(index.pending_batches("recorder").await.unwrap().len(), 1); + assert_eq!(recorder.attempts.lock().unwrap().len(), 1); + + // The fast lane leaves a denied batch alone: admission moves at the + // pace of a membership write, not of a drain tick. + assert_eq!(consumer.drain(&index).await.unwrap(), 0); + assert_eq!(recorder.attempts.lock().unwrap().len(), 1); + + recorder.denied.lock().unwrap().clear(); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 1); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + assert_eq!(recorder.delivered.lock().unwrap().len(), 1); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_denial_that_never_lifts_parks_after_its_budget() { + let index = InMemoryIndex::new(); + journal(&index, BOB, "3krev", "3ka").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder).with_denial_park_after(3); + + for _ in 0..3 { + assert!(!consumer.drain_sweep_succeeded(&index).await.unwrap()); + } + assert_eq!(recorder.attempts.lock().unwrap().len(), 3); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + + for _ in 0..2 { + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + } + assert_eq!( + recorder.attempts.lock().unwrap().len(), + 3, + "a parked batch is never attempted again" + ); + } + + #[tokio::test] + async fn a_denial_does_not_spend_the_poison_budget() { + let index = InMemoryIndex::new(); + journal(&index, BOB, "3krev", "3ka").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder) + .with_dead_letter_after(1) + .with_denial_park_after(10); + + for _ in 0..4 { + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + } + assert_eq!( + index.pending_batches("recorder").await.unwrap().len(), + 1, + "the one-attempt poison budget must not dead-letter a denial" + ); + + // The same budget still kills a genuinely poisoned batch on its first + // failure. + let index = InMemoryIndex::new(); + journal(&index, AUTHOR, "3krev", "3ka").await; + let poison = Recorder::default(); + poison.fail_next.store(5, Ordering::SeqCst); + let consumer = JournalConsumer::new(router(), Box::new(poison)).with_dead_letter_after(1); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 0); + assert!(index.pending_batches("recorder").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_denial_holds_its_author_in_order_and_leaves_others_alone() { + let index = InMemoryIndex::new(); + journal(&index, AUTHOR, "3krev1", "3ka").await; + journal(&index, AUTHOR, "3krev2", "3kb").await; + journal(&index, BOB, "3krev3", "3kc").await; + journal(&index, BOB, "3krev4", "3kd").await; + let recorder = Arc::new(Recorder::default()); + recorder.denied.lock().unwrap().push(BOB.to_string()); + let consumer = shared_consumer(&recorder).with_denial_park_after(5); + + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 2); + assert_eq!( + *recorder.attempts.lock().unwrap(), + vec![ + (AUTHOR.to_string(), "3krev1".to_string()), + (AUTHOR.to_string(), "3krev2".to_string()), + (BOB.to_string(), "3krev3".to_string()), + ], + "bob's later batch must not overtake his denied one" + ); + + recorder.denied.lock().unwrap().clear(); + assert_eq!(consumer.drain_sweep(&index).await.unwrap(), 2); + let delivered: Vec = recorder + .delivered + .lock() + .unwrap() + .iter() + .map(|(rev, _)| rev.clone()) + .collect(); + assert_eq!(delivered, vec!["3krev1", "3krev2", "3krev3", "3krev4"]); + } +} diff --git a/rsky-daemon/src/lib.rs b/rsky-daemon/src/lib.rs index da741e1d..63460882 100644 --- a/rsky-daemon/src/lib.rs +++ b/rsky-daemon/src/lib.rs @@ -18,29 +18,50 @@ //! - [`runner`] — the loop composing all of the above. //! - [`index`] / [`sqlite_index`] — the synced record index. +pub mod appview; pub mod config; pub mod credentials; pub mod dpop; pub mod engine; pub mod error; +pub mod feeds; pub mod index; +pub mod journal; pub mod notify; +pub mod projection; pub mod recovery; pub mod repohost; +pub mod router; pub mod runner; +pub mod service_jwt; +pub mod spaces; pub mod sqlite_index; pub mod xrpc; +pub use appview::AppviewProjector; pub use credentials::{ - unix_now, CredentialProvider, CredentialSource, DelegationSource, PdsDelegationSource, - StaticCredential, + unix_now, CredentialProvider, CredentialSource, DelegationSource, InternalCredentialProvider, + PdsDelegationSource, SpaceCredentialSource, StaticCredential, }; pub use engine::{sync_repo, CommitKeyResolver, SyncOutcome}; pub use error::{DaemonError, Result}; -pub use index::{InMemoryIndex, SpaceIndex}; +pub use feeds::{ + FeedsProjector, HttpProjectionIngress, ProjectRecord, ProjectRecordsRequest, ProjectionIngress, + ProjectionOperation, SpaceLifecycleAcker, +}; +pub use index::{InMemoryIndex, IndexMutation, JournaledBatch, SpaceIndex}; +pub use journal::{drain_all, drain_all_sweep, JournalConsumer, SharedJournalConsumer}; pub use notify::{router as notify_router, NotifyState, WriteNotice}; +pub use projection::Projector; pub use recovery::recover_repo; pub use repohost::{HttpRepoHost, OplogPage, RepoHostClient}; -pub use runner::{run, sync_repo_healing, sync_space_once, RunnerOptions, SweepReport}; +pub use router::{Router, SyncEvent}; +pub use runner::{ + run, run_multi, sync_repo_healing, sync_space_once, MultiRunnerOptions, RunnerOptions, + SweepReport, +}; +pub use spaces::{ + CombinedSource, HttpSpaceSource, SpaceRegistry, SpaceSource, SpaceTarget, StaticSpaces, +}; pub use sqlite_index::{SpaceScopedIndex, SqliteIndex}; pub use xrpc::{HttpSpaceHost, SpaceHostClient}; diff --git a/rsky-daemon/src/main.rs b/rsky-daemon/src/main.rs index 70107da6..5e51af25 100644 --- a/rsky-daemon/src/main.rs +++ b/rsky-daemon/src/main.rs @@ -4,10 +4,13 @@ use clap::Parser; use rsky_daemon::config::Config; use rsky_daemon::engine::CommitKeyResolver; +use rsky_daemon::runner::SpaceWorkerParts; use rsky_daemon::{ - notify_router, run, CredentialProvider, CredentialSource, DaemonError, HttpRepoHost, - HttpSpaceHost, InMemoryIndex, NotifyState, PdsDelegationSource, Result, RunnerOptions, - SpaceIndex, SqliteIndex, StaticCredential, + notify_router, run_multi, AppviewProjector, CombinedSource, CredentialSource, DaemonError, + FeedsProjector, HttpProjectionIngress, HttpRepoHost, HttpSpaceHost, HttpSpaceSource, + InMemoryIndex, InternalCredentialProvider, JournalConsumer, MultiRunnerOptions, NotifyState, + Result, Router, SharedJournalConsumer, SpaceCredentialSource, SpaceIndex, SpaceLifecycleAcker, + SpaceRegistry, SqliteIndex, StaticCredential, StaticSpaces, }; use rsky_identity::did::atproto_data::{get_did_key_from_multibase, VerificationMaterial}; use rsky_identity::types::{IdentityResolverOpts, MemoryCache}; @@ -22,11 +25,11 @@ struct DidKeyResolver { } impl DidKeyResolver { - fn new() -> Self { + fn new(plc_url: Option) -> Self { Self { resolver: tokio::sync::Mutex::new(IdResolver::new(IdentityResolverOpts { timeout: None, - plc_url: None, + plc_url, did_cache: Some(std::sync::Arc::new(MemoryCache::new(None, None))), backup_nameservers: None, })), @@ -63,6 +66,65 @@ impl CommitKeyResolver for DidKeyResolver { } } +/// The projection destinations this process was configured with, if any. +struct ProjectionConfig { + service_identity: String, + signing_key_hex: String, + denial_park_after: u32, + feeds: Option<(String, String)>, + appview: Option<(String, String)>, +} + +type ProjectionParts = ( + Vec, + Option>, +); + +impl ProjectionConfig { + fn consumers(&self, space: &str) -> Result { + if self.feeds.is_none() && self.appview.is_none() { + return Ok((Vec::new(), None)); + } + let space_id = SpaceId::parse(space)?; + let mut consumers: Vec = Vec::new(); + let mut acker: Option> = None; + if let Some((url, audience)) = &self.feeds { + let ingress = Arc::new(HttpProjectionIngress::new( + "feeds", + url, + &self.service_identity, + audience, + &self.signing_key_hex, + )?); + acker = Some(ingress.clone()); + consumers.push(Arc::new( + JournalConsumer::new( + Router::new(space_id.clone(), space_id.authority.clone()), + Box::new(FeedsProjector::new(ingress, space)), + ) + .with_denial_park_after(self.denial_park_after), + )); + } + if let Some((url, audience)) = &self.appview { + let ingress = HttpProjectionIngress::new( + "appview", + url, + &self.service_identity, + audience, + &self.signing_key_hex, + )?; + consumers.push(Arc::new( + JournalConsumer::new( + Router::new(space_id.clone(), space_id.authority.clone()), + Box::new(AppviewProjector::new(ingress, space)), + ) + .with_denial_park_after(self.denial_park_after), + )); + } + Ok((consumers, acker)) + } +} + #[tokio::main] async fn main() -> std::result::Result<(), Box> { tracing_subscriber::fmt() @@ -72,48 +134,77 @@ async fn main() -> std::result::Result<(), Box> { .init(); let cfg = Config::parse(); - let space = SpaceId::parse(&cfg.space_uri)?; + cfg.validate()?; + let authority_filter = cfg.authority_filter(); // One proof-of-possession key for the process: the credential it mints is // bound to it, and every host it is presented to checks that binding. - let dpop = Arc::new(rsky_daemon::dpop::DpopSigner::generate()?); + if cfg.dpop_key_path.is_empty() { + return Err("DAEMON_DPOP_KEY_PATH is required".into()); + } + let dpop = Arc::new(rsky_daemon::dpop::DpopSigner::load_or_generate( + &cfg.dpop_key_path, + )?); let host = Arc::new(HttpSpaceHost::new(&cfg.space_host_url, dpop.clone())); - let keys: Arc = Arc::new(DidKeyResolver::new()); + let keys: Arc = Arc::new(DidKeyResolver::new(cfg.plc_url())); - let index: Arc = if cfg.index_db_path.is_empty() { - tracing::warn!("no DAEMON_INDEX_DB_PATH set; using a non-persistent in-memory index"); - Arc::new(InMemoryIndex::new()) + let db = if cfg.index_db_path.is_empty() { + None } else { - Arc::new(Arc::new(SqliteIndex::open(&cfg.index_db_path)?).for_space(&cfg.space_uri)) + Some(Arc::new(SqliteIndex::open(&cfg.index_db_path)?)) }; - let creds: Arc = if cfg.static_credential.is_empty() { - Arc::new(CredentialProvider::new( + let shared_creds = if cfg.static_credential.is_empty() { + if cfg.space_host_mint_token.is_empty() || cfg.service_signing_key_hex.is_empty() { + return Err( + "DAEMON_SPACE_HOST_MINT_TOKEN and DAEMON_SERVICE_SIGNING_KEY_HEX are required" + .into(), + ); + } + Some(Arc::new(InternalCredentialProvider::new( &cfg.space_uri, - Box::new(PdsDelegationSource::new( - &cfg.pds_url, - &cfg.pds_access_token, - )), + &cfg.space_host_mint_token, + rsky_daemon::service_jwt::ServiceJwtIssuer::from_hex( + &cfg.service_identity, + &cfg.service_signing_key_hex, + )?, host.clone(), - )) + ))) } else { - tracing::warn!("using a static space credential (dev mode)"); - Arc::new(StaticCredential(cfg.static_credential.clone())) + None }; + let mut sources: Vec> = Vec::new(); + if !cfg.space_uri.is_empty() { + sources.push(Box::new(StaticSpaces::new([cfg.space_uri.clone()]))); + } + if !cfg.spaces_url.is_empty() { + sources.push(Box::new(HttpSpaceSource::new( + &cfg.spaces_url, + &cfg.spaces_api_key, + authority_filter.clone(), + &cfg.space_type, + ))); + } + let source = Arc::new(CombinedSource(sources)); + let registry = SpaceRegistry::new(); + let (notify_tx, notify_rx) = mpsc::channel(1024); let (shutdown_tx, shutdown_rx) = watch::channel(false); + let notify_endpoint = cfg.notify_endpoint(); let notify_state = NotifyState { space_uri: cfg.space_uri.clone(), - authority_did: space.authority.clone(), + registry: registry.clone(), service_identity: cfg.service_identity.clone(), + service_signing_key_hex: cfg.service_signing_key_hex.clone(), + notify_endpoint: notify_endpoint.clone(), resolver: keys.clone(), - index: index.clone(), + index: Arc::new(InMemoryIndex::new()), tx: notify_tx, now_fn: rsky_daemon::unix_now, }; let listener = tokio::net::TcpListener::bind(&cfg.notify_bind).await?; tracing::info!( - space = %cfg.space_uri, + authority = %authority_filter.as_deref().unwrap_or("(any)"), host = %cfg.space_host_url, notify_bind = %cfg.notify_bind, sweep_secs = cfg.sweep_interval_secs, @@ -130,25 +221,56 @@ async fn main() -> std::result::Result<(), Box> { }); let repo_host_base = cfg.repo_host_url().to_string(); - let opts = RunnerOptions { - space_uri: cfg.space_uri.clone(), + let static_credential = cfg.static_credential.clone(); + let db_for_factory = db.clone(); + let dpop_for_factory = dpop.clone(); + let shared_for_factory = shared_creds.clone(); + let projection = ProjectionConfig { + service_identity: cfg.service_identity.clone(), + signing_key_hex: cfg.service_signing_key_hex.clone(), + denial_park_after: cfg.denial_park_after, + feeds: cfg + .feeds_projection() + .map(|(url, aud)| (url.to_string(), aud.to_string())), + appview: cfg + .appview_projection() + .map(|(url, aud)| (url.to_string(), aud.to_string())), + }; + let factory = Arc::new(move |space: &str| -> Result { + let creds: Arc = match &shared_for_factory { + Some(provider) => Arc::new(SpaceCredentialSource::new(provider.clone(), space)), + None => Arc::new(StaticCredential(static_credential.clone())), + }; + let index: Arc = match &db_for_factory { + Some(db) => Arc::new(db.for_space(space)), + None => Arc::new(InMemoryIndex::new()), + }; + let base = repo_host_base.clone(); + let proof = dpop_for_factory.clone(); + let (projectors, acker) = projection.consumers(space)?; + Ok(( + creds, + Box::new(move |credential| { + Arc::new(HttpRepoHost::new(base.clone(), credential, proof.clone())) + }), + index, + projectors, + acker, + )) + }); + let opts = MultiRunnerOptions { + refresh_interval_secs: cfg.sweep_interval_secs, sweep_interval_secs: cfg.sweep_interval_secs, - notify_endpoint: cfg.notify_endpoint(), + notify_endpoint, service_identity: cfg.service_identity.clone(), now_fn: rsky_daemon::unix_now, }; - let runner = tokio::spawn(run( + let runner = tokio::spawn(run_multi( opts, + source, + registry, + factory, host, - creds, - Box::new(move |credential| { - Arc::new(HttpRepoHost::new( - repo_host_base.clone(), - credential, - dpop.clone(), - )) - }), - index, keys, notify_rx, shutdown_rx, diff --git a/rsky-daemon/src/notify.rs b/rsky-daemon/src/notify.rs index 033090dc..3e43be0b 100644 --- a/rsky-daemon/src/notify.rs +++ b/rsky-daemon/src/notify.rs @@ -5,7 +5,7 @@ use axum::extract::State; use axum::http::{header, HeaderMap, StatusCode}; -use axum::routing::post; +use axum::routing::{get, post}; use axum::{Json, Router}; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; @@ -19,6 +19,7 @@ use tokio::sync::mpsc; use crate::engine::CommitKeyResolver; use crate::error::{DaemonError, Result}; use crate::index::SpaceIndex; +use crate::spaces::SpaceRegistry; /// A queued `(space, did)` write notice for the runner to pull. pub type WriteNotice = (String, String); @@ -88,10 +89,11 @@ pub fn decode_claims(jwt: &str) -> Result { #[derive(Clone)] pub struct NotifyState { pub space_uri: String, - /// The authority (space host) DID whose key signs inbound notifications. - pub authority_did: String, + pub registry: SpaceRegistry, /// This syncer's service identity: the required `aud` on inbound tokens. pub service_identity: String, + pub service_signing_key_hex: String, + pub notify_endpoint: String, pub resolver: Arc, pub index: Arc, pub tx: mpsc::Sender, @@ -100,6 +102,7 @@ pub struct NotifyState { pub fn router(state: NotifyState) -> Router { Router::new() + .route("/.well-known/did.json", get(well_known)) .route("/xrpc/com.atproto.space.notifyWrite", post(notify_write)) .route( "/xrpc/com.atproto.space.notifySpaceDeleted", @@ -108,11 +111,60 @@ pub fn router(state: NotifyState) -> Router { .with_state(state) } +fn service_multikey(signing_key_hex: &str) -> Result { + let bytes = hex::decode(signing_key_hex.trim()) + .map_err(|error| DaemonError::Xrpc(error.to_string()))?; + let secret = secp256k1::SecretKey::from_slice(&bytes) + .map_err(|error| DaemonError::Xrpc(error.to_string()))?; + let public = secp256k1::PublicKey::from_secret_key(&secp256k1::Secp256k1::new(), &secret); + let did_key = rsky_crypto::utils::encode_did_key(&public); + Ok(did_key + .strip_prefix("did:key:") + .expect("rsky did:key encoder always includes its prefix") + .to_string()) +} + +async fn well_known(State(state): State) -> (StatusCode, Json) { + let multikey = match service_multikey(&state.service_signing_key_hex) { + Ok(multikey) => multikey, + Err(error) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + error_body("InternalServerError", error), + ) + } + }; + let did = state.service_identity; + let notify_endpoint = state.notify_endpoint; + ( + StatusCode::OK, + Json(json!({ + "@context": [ + "https://www.w3.org/ns/did/v1", + "https://w3id.org/security/multikey/v1" + ], + "id": did, + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": format!("{did}#atproto_space_syncer"), + "type": "AtprotoSpaceService", + "serviceEndpoint": notify_endpoint + }] + })), + ) +} + fn error_body(error: &str, message: impl std::fmt::Display) -> Json { Json(json!({ "error": error, "message": message.to_string() })) } -async fn authenticate(headers: &HeaderMap, state: &NotifyState) -> Result<()> { +async fn authenticate(headers: &HeaderMap, state: &NotifyState, space_uri: &str) -> Result<()> { + let authority = rsky_space::space_id::SpaceId::parse(space_uri)?.authority; let jwt = headers .get(header::AUTHORIZATION) .and_then(|v| v.to_str().ok()) @@ -122,10 +174,10 @@ async fn authenticate(headers: &HeaderMap, state: &NotifyState) -> Result<()> { "missing bearer token".into(), )) })?; - let did_key = state.resolver.signing_key(&state.authority_did).await?; + let did_key = state.resolver.signing_key(&authority).await?; verify_service_auth( jwt, - &state.authority_did, + &authority, &state.service_identity, &did_key, (state.now_fn)(), @@ -137,16 +189,16 @@ async fn notify_write( headers: HeaderMap, Json(input): Json, ) -> (StatusCode, Json) { - if let Err(e) = authenticate(&headers, &state).await { + if !state.registry.contains(&input.space) { return ( - StatusCode::UNAUTHORIZED, - error_body("AuthenticationRequired", e), + StatusCode::BAD_REQUEST, + error_body("InvalidRequest", "space is not synced by this daemon"), ); } - if input.space != state.space_uri { + if let Err(e) = authenticate(&headers, &state, &input.space).await { return ( - StatusCode::BAD_REQUEST, - error_body("InvalidRequest", "space is not synced by this daemon"), + StatusCode::UNAUTHORIZED, + error_body("AuthenticationRequired", e), ); } tracing::debug!(space = %input.space, repo = %input.repo, rev = %input.rev, "write notice"); @@ -164,16 +216,16 @@ async fn notify_space_deleted( headers: HeaderMap, Json(input): Json, ) -> (StatusCode, Json) { - if let Err(e) = authenticate(&headers, &state).await { + if !state.registry.contains(&input.space) { return ( - StatusCode::UNAUTHORIZED, - error_body("AuthenticationRequired", e), + StatusCode::BAD_REQUEST, + error_body("InvalidRequest", "space is not synced by this daemon"), ); } - if input.space != state.space_uri { + if let Err(e) = authenticate(&headers, &state, &input.space).await { return ( - StatusCode::BAD_REQUEST, - error_body("InvalidRequest", "space is not synced by this daemon"), + StatusCode::UNAUTHORIZED, + error_body("AuthenticationRequired", e), ); } tracing::warn!(space = %input.space, "space deleted; purging all synced data"); @@ -200,6 +252,8 @@ mod tests { const SPACE: &str = "at://did:plc:authority/space/community.blacksky.feed/main"; const AUTHORITY: &str = "did:plc:authority"; const SYNCER: &str = "did:web:syncer.blacksky.community"; + const SYNCER_ENDPOINT: &str = "https://syncer.blacksky.community"; + const SYNCER_KEY: &str = "0707070707070707070707070707070707070707070707070707070707070707"; const NOW: u64 = 1_000_000; fn host_key() -> (SecretKey, String) { @@ -243,8 +297,14 @@ mod tests { ) -> NotifyState { NotifyState { space_uri: SPACE.to_string(), - authority_did: AUTHORITY.to_string(), + registry: { + let registry = SpaceRegistry::new(); + registry.insert(SPACE); + registry + }, service_identity: SYNCER.to_string(), + service_signing_key_hex: SYNCER_KEY.to_string(), + notify_endpoint: SYNCER_ENDPOINT.to_string(), resolver: Arc::new(FixedKey(did_key.to_string())), index, tx, @@ -252,6 +312,71 @@ mod tests { } } + #[tokio::test] + async fn well_known_publishes_the_service_identity() { + let (_secret, did_key) = host_key(); + let (tx, _rx) = mpsc::channel(4); + let app = router(state(&did_key, Arc::new(InMemoryIndex::new()), tx)); + + let response = app + .oneshot( + Request::builder() + .uri("/.well-known/did.json") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let document: Value = serde_json::from_slice(&body).unwrap(); + let secret = SecretKey::from_slice(&[7u8; 32]).unwrap(); + let public = PublicKey::from_secret_key(&Secp256k1::new(), &secret); + let expected = rsky_crypto::utils::encode_did_key(&public) + .strip_prefix("did:key:") + .unwrap() + .to_string(); + + assert_eq!(document["id"], SYNCER); + assert_eq!( + document["verificationMethod"][0]["id"], + format!("{SYNCER}#atproto") + ); + assert_eq!(document["verificationMethod"][0]["type"], "Multikey"); + assert_eq!(document["verificationMethod"][0]["controller"], SYNCER); + assert_eq!( + document["verificationMethod"][0]["publicKeyMultibase"], + expected + ); + assert_eq!(document["service"].as_array().unwrap().len(), 1); + assert_eq!( + document["service"][0]["id"], + format!("{SYNCER}#atproto_space_syncer") + ); + assert_eq!(document["service"][0]["type"], "AtprotoSpaceService"); + assert_eq!(document["service"][0]["serviceEndpoint"], SYNCER_ENDPOINT); + } + + #[tokio::test] + async fn well_known_rejects_an_invalid_signing_key() { + let (_secret, did_key) = host_key(); + let (tx, _rx) = mpsc::channel(4); + let mut notify_state = state(&did_key, Arc::new(InMemoryIndex::new()), tx); + notify_state.service_signing_key_hex = "not-hex".to_string(); + let response = router(notify_state) + .oneshot( + Request::builder() + .uri("/.well-known/did.json") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + } + fn request(path: &str, token: Option<&str>, body: Value) -> Request { let mut builder = Request::builder() .method("POST") @@ -488,6 +613,75 @@ mod tests { idx.list_paths("d").await.unwrap(); } + #[tokio::test] + async fn notifications_verify_against_each_space_authority() { + use std::collections::BTreeMap; + struct KeyMap(BTreeMap); + #[async_trait] + impl CommitKeyResolver for KeyMap { + async fn signing_key(&self, did: &str) -> Result { + self.0 + .get(did) + .cloned() + .ok_or_else(|| DaemonError::KeyResolution(format!("unknown did {did}"))) + } + } + + let other_space = "at://did:plc:authority2/space/community.blacksky.feed/main"; + let (secret_a, key_a) = host_key(); + let secret_b = SecretKey::from_slice(&[0x66u8; 32]).unwrap(); + let key_b = rsky_crypto::utils::encode_did_key(&PublicKey::from_secret_key( + &Secp256k1::new(), + &secret_b, + )); + let registry = SpaceRegistry::new(); + registry.insert(SPACE); + registry.insert(other_space); + let (tx, mut rx) = mpsc::channel(4); + let app = router(NotifyState { + space_uri: SPACE.to_string(), + registry, + service_identity: SYNCER.to_string(), + service_signing_key_hex: SYNCER_KEY.to_string(), + notify_endpoint: SYNCER_ENDPOINT.to_string(), + resolver: Arc::new(KeyMap(BTreeMap::from([ + (AUTHORITY.to_string(), key_a), + ("did:plc:authority2".to_string(), key_b), + ]))), + index: Arc::new(InMemoryIndex::new()), + tx, + now_fn: fixed_now, + }); + + // Each authority's key opens only its own space's notifications. + let jwt_b = service_jwt(&secret_b, "did:plc:authority2", SYNCER, NOW + 60); + let resp = app + .clone() + .oneshot(request( + "/xrpc/com.atproto.space.notifyWrite", + Some(&jwt_b), + json!({ "space": other_space, "repo": "did:plc:w2", "rev": "3krev" }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + rx.recv().await.unwrap(), + (other_space.to_string(), "did:plc:w2".to_string()) + ); + + let jwt_a_for_b = service_jwt(&secret_a, AUTHORITY, SYNCER, NOW + 60); + let resp = app + .oneshot(request( + "/xrpc/com.atproto.space.notifyWrite", + Some(&jwt_a_for_b), + json!({ "space": other_space, "repo": "did:plc:w2", "rev": "3krev" }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + #[test] fn decode_claims_rejects_malformed_tokens() { assert!(decode_claims("one.two").is_err()); diff --git a/rsky-daemon/src/projection.rs b/rsky-daemon/src/projection.rs new file mode 100644 index 00000000..5250c383 --- /dev/null +++ b/rsky-daemon/src/projection.rs @@ -0,0 +1,17 @@ +//! Projection adapter contract. Delivery is driven by the durable journal, +//! never by writing straight through from the sync path. + +use async_trait::async_trait; + +use crate::error::Result; +use crate::router::SyncEvent; + +/// Where a synced batch is sent. +#[async_trait] +pub trait Projector: Send + Sync { + fn name(&self) -> &'static str; + + /// Deliver the events of one batch. Must be idempotent: a retry after a + /// crash replays the same events. + async fn project(&self, did: &str, rev: &str, events: &[SyncEvent]) -> Result<()>; +} diff --git a/rsky-daemon/src/recovery.rs b/rsky-daemon/src/recovery.rs index abd316cf..16b8f8da 100644 --- a/rsky-daemon/src/recovery.rs +++ b/rsky-daemon/src/recovery.rs @@ -10,7 +10,7 @@ use rsky_space::lthash::{element, LtHash}; use crate::engine::{CommitKeyResolver, SyncOutcome}; use crate::error::{DaemonError, Result}; -use crate::index::SpaceIndex; +use crate::index::{IndexMutation, SpaceIndex}; use crate::repohost::RepoHostClient; /// Recover an author's repo from a full-state CAR: verify the commit with the @@ -43,6 +43,7 @@ pub async fn recover_repo( let mut lth = LtHash::new(); let mut keep: HashSet = HashSet::with_capacity(records.len()); let mut changed = 0usize; + let mut mutations: Vec = Vec::new(); for (path, cid, bytes) in &records { let (collection, rkey) = path .split_once('/') @@ -60,6 +61,13 @@ pub async fn recover_repo( ) .await?; changed += 1; + mutations.push(IndexMutation::Upsert { + collection: collection.to_string(), + rkey: rkey.to_string(), + cid: cid.clone(), + rev: commit.rev.clone(), + value: Some(bytes.clone()), + }); } lth.add(&element(collection, rkey, &cid)); keep.insert(path.clone()); @@ -68,8 +76,10 @@ pub async fn recover_repo( if !keep.contains(&format!("{collection}/{rkey}")) { index.delete(did, &collection, &rkey).await?; changed += 1; + mutations.push(IndexMutation::Delete { collection, rkey }); } } + index.journal_batch(did, &commit.rev, &mutations).await?; index.save_head(did, &commit.rev, <h).await?; tracing::info!(did, rev = %commit.rev, changed, "recovered repo from full-state CAR"); Ok(SyncOutcome { diff --git a/rsky-daemon/src/repohost.rs b/rsky-daemon/src/repohost.rs index 55180ca1..f7271dba 100644 --- a/rsky-daemon/src/repohost.rs +++ b/rsky-daemon/src/repohost.rs @@ -122,7 +122,7 @@ impl RepoHostClient for HttpRepoHost { since: Option<&str>, cursor: Option<&str>, ) -> Result { - let mut query = vec![("space", space), ("did", did)]; + let mut query = vec![("space", space), ("repo", did)]; if let Some(since) = since { query.push(("since", since)); } @@ -146,7 +146,7 @@ impl RepoHostClient for HttpRepoHost { let resp = self .get( "com.atproto.space.getRepo", - &[("space", space), ("did", did)], + &[("space", space), ("repo", did)], ) .await?; Ok(resp.bytes().await.map_err(net_err)?.to_vec()) @@ -156,7 +156,7 @@ impl RepoHostClient for HttpRepoHost { let out: wire::GetLatestCommitOutput = self .get( "com.atproto.space.getLatestCommit", - &[("space", space), ("did", did)], + &[("space", space), ("repo", did)], ) .await? .json() @@ -208,7 +208,7 @@ mod tests { Mock::given(method("GET")) .and(path("/xrpc/com.atproto.space.listRepoOps")) .and(query_param("space", SPACE)) - .and(query_param("did", AUTHOR)) + .and(query_param("repo", AUTHOR)) .and(query_param("since", "3ka")) .and(query_param("cursor", "c1")) .and(header("authorization", "DPoP sc.jwt")) @@ -311,7 +311,7 @@ mod tests { Mock::given(method("GET")) .and(path("/xrpc/com.atproto.space.getRepo")) .and(query_param("space", SPACE)) - .and(query_param("did", AUTHOR)) + .and(query_param("repo", AUTHOR)) .and(header("authorization", "DPoP sc.jwt")) .and(header_exists("dpop")) .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![0xCAu8, 0x11])) @@ -329,7 +329,7 @@ mod tests { Mock::given(method("GET")) .and(path("/xrpc/com.atproto.space.getLatestCommit")) .and(query_param("space", SPACE)) - .and(query_param("did", AUTHOR)) + .and(query_param("repo", AUTHOR)) .and(header("authorization", "DPoP sc.jwt")) .and(header_exists("dpop")) .respond_with( diff --git a/rsky-daemon/src/router.rs b/rsky-daemon/src/router.rs new file mode 100644 index 00000000..c27b3a86 --- /dev/null +++ b/rsky-daemon/src/router.rs @@ -0,0 +1,374 @@ +//! Typed routing over an authenticated batch. +//! +//! A projection cares about *what happened*, not about record paths, so the +//! router turns index mutations into events before anything downstream sees +//! them. Routing runs on verified batches only — an unroutable or unknown +//! record is dropped and logged rather than raised, since one record a build +//! has never heard of must not stall a whole repo. + +use rsky_space::record::decode_record; +use rsky_space::space_id::{RecordId, SpaceId}; +use serde_json::Value; +use std::sync::atomic::{AtomicU64, Ordering}; +use tracing::debug; + +use crate::index::IndexMutation; + +/// Records on collections this build projects that were lost anyway because +/// their bytes would not decode. Dropping unknown collections is policy; +/// losing a known collection is data loss and must alarm. +pub static KNOWN_COLLECTION_DECODE_FAILURES: AtomicU64 = AtomicU64::new(0); + +/// Stable event name for the loss alarm, for log-based alerting. +pub const LOSS_EVENT: &str = "space_known_collection_decode_failure"; + +pub const POST_COLLECTION: &str = "app.bsky.feed.post"; +pub const LIKE_COLLECTION: &str = "app.bsky.feed.like"; +pub const MODERATION_ACTION_COLLECTION: &str = "community.blacksky.moderation.action"; + +/// One routed change, in the terms a projection acts on. +#[derive(Debug, Clone, PartialEq)] +pub enum SyncEvent { + PostCreated { + uri: String, + author: String, + cid: String, + rev: String, + record: Value, + }, + PostDeleted { + uri: String, + author: String, + }, + LikeCreated { + uri: String, + author: String, + cid: String, + rev: String, + record: Value, + }, + LikeDeleted { + uri: String, + author: String, + }, + /// A moderation action from the space authority's own repo. The router + /// emits this only for the authority (D25), so a projection never has to + /// re-check who wrote it. A lift is its own signed record (`neg: true`), + /// not deletion of the action it reverses. + ModerationAction { + uri: String, + rev: String, + record: Option, + }, +} + +impl SyncEvent { + pub fn uri(&self) -> &str { + match self { + Self::PostCreated { uri, .. } + | Self::PostDeleted { uri, .. } + | Self::LikeCreated { uri, .. } + | Self::LikeDeleted { uri, .. } + | Self::ModerationAction { uri, .. } => uri, + } + } +} + +/// Turns the mutations of one author's verified batch into events. +pub struct Router { + space: SpaceId, + authority: String, +} + +impl Router { + pub fn new(space: SpaceId, authority: impl Into) -> Self { + Self { + space, + authority: authority.into(), + } + } + + pub fn space(&self) -> &SpaceId { + &self.space + } + + pub fn route_batch(&self, did: &str, mutations: &[IndexMutation]) -> Vec { + mutations + .iter() + .filter_map(|m| self.route(did, m)) + .collect() + } + + pub fn route(&self, did: &str, mutation: &IndexMutation) -> Option { + let uri = RecordId { + space: self.space.clone(), + author: did.to_string(), + collection: mutation.collection().to_string(), + rkey: mutation.rkey().to_string(), + } + .uri(); + + match (mutation.collection(), mutation) { + (POST_COLLECTION, IndexMutation::Delete { .. }) => Some(SyncEvent::PostDeleted { + uri, + author: did.to_string(), + }), + (LIKE_COLLECTION, IndexMutation::Delete { .. }) => Some(SyncEvent::LikeDeleted { + uri, + author: did.to_string(), + }), + ( + POST_COLLECTION | LIKE_COLLECTION, + IndexMutation::Upsert { + cid, rev, value, .. + }, + ) => { + let record = decode_value(&uri, value.as_deref())?; + let is_post = mutation.collection() == POST_COLLECTION; + Some(if is_post { + SyncEvent::PostCreated { + uri, + author: did.to_string(), + cid: cid.clone(), + rev: rev.clone(), + record, + } + } else { + SyncEvent::LikeCreated { + uri, + author: did.to_string(), + cid: cid.clone(), + rev: rev.clone(), + record, + } + }) + } + (MODERATION_ACTION_COLLECTION, _) if did != self.authority => { + debug!(%uri, %did, "dropping moderation action from a repo that is not the authority's"); + None + } + (MODERATION_ACTION_COLLECTION, IndexMutation::Delete { .. }) => { + Some(SyncEvent::ModerationAction { + uri, + rev: String::new(), + record: None, + }) + } + (MODERATION_ACTION_COLLECTION, IndexMutation::Upsert { rev, value, .. }) => { + let record = decode_value(&uri, value.as_deref())?; + Some(SyncEvent::ModerationAction { + uri, + rev: rev.clone(), + record: Some(record), + }) + } + (other, _) => { + debug!(%uri, collection = %other, "dropping a collection this build does not project"); + None + } + } + } +} + +/// The two sync paths inline a record's value in different encodings: the +/// oplog carries it as JSON, a full-state CAR as DAG-CBOR. Either may be what +/// the index holds for a given record, so both are accepted here. +fn decode_inlined(bytes: &[u8]) -> std::result::Result { + match decode_record(bytes) { + Ok(value) if value.is_object() => Ok(value), + cbor => serde_json::from_slice::(bytes) + .map_err(|json_err| match cbor { + Ok(_) => format!("dag-cbor value is not a record; json: {json_err}"), + Err(cbor_err) => format!("dag-cbor: {cbor_err}; json: {json_err}"), + }) + .and_then(|value| { + if value.is_object() { + Ok(value) + } else { + Err("value is not a record".to_string()) + } + }), + } +} + +fn decode_value(uri: &str, value: Option<&[u8]>) -> Option { + match value { + Some(bytes) => match decode_inlined(bytes) { + Ok(value) => Some(value), + Err(err) => { + let total = KNOWN_COLLECTION_DECODE_FAILURES.fetch_add(1, Ordering::Relaxed) + 1; + tracing::error!( + event = LOSS_EVENT, + %uri, + %err, + total, + "a record on a projected collection will not decode and is lost" + ); + None + } + }, + None => { + // The host inlines a record's value with its op; without one there + // is nothing to project, and the sweep will bring it back. + debug!(%uri, "dropping a write carrying no record value"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rsky_space::record::encode_record; + use serde_json::json; + + const AUTHORITY: &str = "did:plc:community"; + const MEMBER: &str = "did:plc:alice"; + + fn router() -> Router { + Router::new( + SpaceId::new(AUTHORITY, "community.blacksky.feed", "private"), + AUTHORITY, + ) + } + + fn upsert(collection: &str, value: Value) -> IndexMutation { + IndexMutation::Upsert { + collection: collection.to_string(), + rkey: "3krkey".to_string(), + cid: "bafyrecord".to_string(), + rev: "3krev".to_string(), + value: Some(encode_record(&value, 64 * 1024).unwrap()), + } + } + + fn delete(collection: &str) -> IndexMutation { + IndexMutation::Delete { + collection: collection.to_string(), + rkey: "3krkey".to_string(), + } + } + + fn post() -> Value { + json!({"$type": POST_COLLECTION, "text": "hello", "createdAt": "2026-08-09T00:00:00.000Z"}) + } + + #[test] + fn a_post_routes_to_its_space_uri() { + match router().route(MEMBER, &upsert(POST_COLLECTION, post())) { + Some(SyncEvent::PostCreated { uri, record, .. }) => { + assert_eq!( + uri, + format!("at://{AUTHORITY}/space/community.blacksky.feed/private/{MEMBER}/{POST_COLLECTION}/3krkey") + ); + assert_eq!(record["text"], "hello"); + } + other => panic!("expected a post create, got {other:?}"), + } + } + + #[test] + fn a_like_over_a_space_subject_survives_routing() { + // The record standard lexicon validation would reject: its subject is + // a space URI, which is not an at-uri (D29). + let like = json!({ + "$type": LIKE_COLLECTION, + "subject": {"uri": format!("at://{AUTHORITY}/space/x/y/{MEMBER}/{POST_COLLECTION}/3kz"), "cid": "bafy"}, + "createdAt": "2026-08-09T00:00:00.000Z", + }); + match router().route(MEMBER, &upsert(LIKE_COLLECTION, like)) { + Some(SyncEvent::LikeCreated { record, .. }) => { + assert!(record["subject"]["uri"] + .as_str() + .unwrap() + .contains("/space/")); + } + other => panic!("expected a like create, got {other:?}"), + } + } + + #[test] + fn deletes_route_by_kind() { + let r = router(); + assert!(matches!( + r.route(MEMBER, &delete(POST_COLLECTION)), + Some(SyncEvent::PostDeleted { .. }) + )); + assert!(matches!( + r.route(MEMBER, &delete(LIKE_COLLECTION)), + Some(SyncEvent::LikeDeleted { .. }) + )); + } + + #[test] + fn moderation_actions_come_only_from_the_authority() { + let action = json!({ + "$type": MODERATION_ACTION_COLLECTION, + "subject": {"uri": "at://a/space/t/s/did:plc:alice/app.bsky.feed.post/3k"}, + "event": {"$type": "community.blacksky.moderation.action#delete"}, + "createdAt": "2026-08-09T00:00:00.000Z", + }); + let r = router(); + assert!(matches!( + r.route( + AUTHORITY, + &upsert(MODERATION_ACTION_COLLECTION, action.clone()) + ), + Some(SyncEvent::ModerationAction { + record: Some(_), + .. + }) + )); + // A member's own copy of the same record is a forgery attempt. + assert!(r + .route(MEMBER, &upsert(MODERATION_ACTION_COLLECTION, action)) + .is_none()); + // Deletes do not lift an effect: only a signed negation may do that. + assert!(matches!( + r.route(AUTHORITY, &delete(MODERATION_ACTION_COLLECTION)), + Some(SyncEvent::ModerationAction { record: None, .. }) + )); + } + + #[test] + fn a_value_inlined_as_json_routes_like_one_inlined_as_dag_cbor() { + let mut json_valued = upsert(POST_COLLECTION, post()); + if let IndexMutation::Upsert { value, .. } = &mut json_valued { + *value = Some(serde_json::to_vec(&post()).unwrap()); + } + match router().route(MEMBER, &json_valued) { + Some(SyncEvent::PostCreated { record, .. }) => assert_eq!(record["text"], "hello"), + other => panic!("expected a post create, got {other:?}"), + } + } + + #[test] + fn only_unknown_collections_may_drop_silently() { + let r = router(); + + // Unknown collection: drop-and-log is policy, no alarm. + let before = KNOWN_COLLECTION_DECODE_FAILURES.load(Ordering::SeqCst); + assert!(r + .route( + MEMBER, + &upsert("app.bsky.graph.follow", json!({"$type": "x"})) + ) + .is_none()); + assert_eq!( + KNOWN_COLLECTION_DECODE_FAILURES.load(Ordering::SeqCst), + before + ); + + // A known collection failing to decode is data loss: it must raise + // the alarmed counter, never vanish silently. + let mut broken = upsert(POST_COLLECTION, post()); + if let IndexMutation::Upsert { value, .. } = &mut broken { + *value = Some(vec![0xff, 0xff]); + } + assert!(r.route(MEMBER, &broken).is_none()); + assert_eq!( + KNOWN_COLLECTION_DECODE_FAILURES.load(Ordering::SeqCst), + before + 1 + ); + } +} diff --git a/rsky-daemon/src/runner.rs b/rsky-daemon/src/runner.rs index 30410035..7373edab 100644 --- a/rsky-daemon/src/runner.rs +++ b/rsky-daemon/src/runner.rs @@ -3,6 +3,7 @@ //! to full-state recovery when incremental sync cannot proceed. use rsky_lexicon::com::atproto::space::RepoRef; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use tokio::sync::{mpsc, watch}; @@ -11,17 +12,88 @@ use tokio::time::Instant; use crate::credentials::CredentialSource; use crate::engine::{sync_repo, CommitKeyResolver, SyncOutcome}; use crate::error::{DaemonError, Result}; +use crate::feeds::SpaceLifecycleAcker; use crate::index::SpaceIndex; +use crate::journal::{drain_all, drain_all_sweep, SharedJournalConsumer}; use crate::notify::WriteNotice; use crate::recovery::recover_repo; use crate::repohost::RepoHostClient; +use crate::spaces::{SpaceRegistry, SpaceSource}; use crate::xrpc::SpaceHostClient; const REGISTER_RETRY_SECS: u64 = 30; const MIN_REREGISTER_SECS: u64 = 30; +/// How often pending batches are retried when no sync has happened, so a +/// destination that was down recovers without waiting for the next write. +const PROJECTION_DRAIN_SECS: u64 = 30; /// Builds a repo-host client bound to the current space credential. pub type RepoHostFactory = Box Arc + Send + Sync>; +pub type SpaceWorkerParts = ( + Arc, + RepoHostFactory, + Arc, + Vec, + Option>, +); +pub type MultiSpaceFactory = Arc Result + Send + Sync>; + +pub struct MultiRunnerOptions { + pub refresh_interval_secs: u64, + pub sweep_interval_secs: u64, + pub notify_endpoint: String, + pub service_identity: String, + pub now_fn: fn() -> u64, +} + +/// Supervises one existing `run` worker per discovered space. Source errors +/// retain the current worker set; a transient managing-app outage must not +/// silently stop private-feed projection. +#[allow(clippy::too_many_arguments)] +pub async fn run_multi( + opts: MultiRunnerOptions, + source: Arc, + registry: SpaceRegistry, + factory: MultiSpaceFactory, + host: Arc, + keys: Arc, + mut notices: mpsc::Receiver, + mut shutdown: watch::Receiver, +) { + struct Worker { + generation: i64, + stop: watch::Sender, + notices: mpsc::Sender, + handle: tokio::task::JoinHandle<()>, + } + let mut workers: HashMap = HashMap::new(); + let mut refresh = tokio::time::interval(Duration::from_secs(opts.refresh_interval_secs.max(1))); + loop { + tokio::select! { + _ = shutdown.changed() => break, + _ = refresh.tick() => { + let desired = match source.spaces().await { Ok(value) => value, Err(error) => { tracing::warn!(error = %error, "space source unavailable; keeping current workers"); continue; } }; + let stale: Vec<_> = workers.iter().filter(|(space, worker)| desired.get(*space).is_none_or(|target| target.generation != worker.generation)).map(|(space, _)| space.clone()).collect(); + for space in stale { if let Some(worker) = workers.remove(&space) { let _ = worker.stop.send(true); let _ = worker.handle.await; } } + for (space, target) in &desired { if workers.contains_key(space) { continue; } + let (creds, repo, index, projectors, acker) = match factory(space) { Ok(parts) => parts, Err(error) => { tracing::warn!(%space, error = %error, "cannot prepare space worker"); continue; } }; + let (tx, rx) = mpsc::channel(256); let (stop, stop_rx) = watch::channel(false); + let worker_opts = RunnerOptions { space_uri: space.clone(), sweep_interval_secs: opts.sweep_interval_secs, notify_endpoint: opts.notify_endpoint.clone(), service_identity: opts.service_identity.clone(), generation: target.generation, now_fn: opts.now_fn }; + let handle = tokio::spawn(run(worker_opts, host.clone(), creds, repo, index, keys.clone(), projectors, acker, rx, stop_rx)); + workers.insert(space.clone(), Worker { generation: target.generation, stop, notices: tx, handle }); + } + registry.replace(workers.keys().cloned().collect()); + } + Some(notice) = notices.recv() => { if let Some(worker) = workers.get(¬ice.0) { let _ = worker.notices.send(notice).await; } else { tracing::warn!(space = %notice.0, "notice for a space we do not sync"); } } + } + } + for (space, worker) in workers { + let _ = worker.stop.send(true); + let _ = worker.handle.await; + tracing::info!(%space, "stopped"); + } + registry.replace(Default::default()); +} #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct SweepReport { @@ -110,6 +182,9 @@ pub struct RunnerOptions { /// This syncer's own service identifier, so the host can address its /// deliveries to it. pub service_identity: String, + /// The space's generation as the managing app reported it; an + /// acknowledgement names the generation it observed. + pub generation: i64, pub now_fn: fn() -> u64, } @@ -151,7 +226,7 @@ async fn sweep( make_repo_host: &RepoHostFactory, index: &dyn SpaceIndex, keys: &dyn CommitKeyResolver, -) { +) -> bool { let attempt = async { let credential = creds.credential((opts.now_fn)()).await?; let client = make_repo_host(credential.clone()); @@ -166,8 +241,14 @@ async fn sweep( .await }; match attempt.await { - Ok(r) => tracing::info!(synced = %r.synced, recovered = %r.recovered, "sweep complete"), - Err(e) => tracing::warn!(error = %e, "sweep failed"), + Ok(r) => { + tracing::info!(synced = %r.synced, recovered = %r.recovered, "sweep complete"); + true + } + Err(e) => { + tracing::warn!(error = %e, "sweep failed"); + false + } } } @@ -206,12 +287,16 @@ pub async fn run( make_repo_host: RepoHostFactory, index: Arc, keys: Arc, + projectors: Vec, + lifecycle_acker: Option>, mut notify_rx: mpsc::Receiver, mut shutdown: watch::Receiver, ) { let mut sweep_timer = tokio::time::interval(Duration::from_secs(opts.sweep_interval_secs.max(1))); + let mut drain_timer = tokio::time::interval(Duration::from_secs(PROJECTION_DRAIN_SECS)); let mut register_at = Instant::now(); + let mut acknowledged = false; loop { tokio::select! { _ = shutdown.changed() => { @@ -223,7 +308,7 @@ pub async fn run( + register(&opts, host.as_ref(), creds.as_ref()).await; } _ = sweep_timer.tick() => { - sweep( + let swept = sweep( &opts, host.as_ref(), creds.as_ref(), @@ -232,6 +317,18 @@ pub async fn run( keys.as_ref(), ) .await; + let projected = drain_all_sweep(index.as_ref(), &projectors).await; + if swept && projected && !acknowledged { + if let Some(acker) = &lifecycle_acker { + match acker.acknowledge_sync(&opts.space_uri, opts.generation).await { + Ok(()) => acknowledged = true, + Err(error) => tracing::warn!(error = %error, "space lifecycle acknowledgement failed"), + } + } + } + } + _ = drain_timer.tick() => { + drain_all(index.as_ref(), &projectors).await; } Some(notice) = notify_rx.recv() => { handle_notice( @@ -243,6 +340,7 @@ pub async fn run( notice, ) .await; + drain_all(index.as_ref(), &projectors).await; } } } @@ -728,6 +826,7 @@ mod tests { sweep_interval_secs: sweep_secs, notify_endpoint: "https://syncer.example/notify".to_string(), service_identity: "did:web:syncer.example".to_string(), + generation: 1, now_fn: fixed_now, } } @@ -765,6 +864,8 @@ mod tests { make_repo_host, index.clone(), keys, + Vec::new(), + None, rx, shutdown_rx, )); @@ -794,6 +895,152 @@ mod tests { handle.await.unwrap(); } + #[tokio::test(start_paused = true)] + async fn a_worker_drains_each_projector_independently() { + use crate::feeds::tests::{post_created, CapturingIngress}; + use crate::feeds::FeedsProjector; + use crate::index::IndexMutation; + use crate::journal::JournalConsumer; + use crate::projection::Projector; + use crate::router::{Router, SyncEvent, POST_COLLECTION}; + use rsky_space::record::encode_record; + use rsky_space::space_id::SpaceId; + + struct Stalled; + #[async_trait] + impl Projector for Stalled { + fn name(&self) -> &'static str { + "appview" + } + async fn project(&self, _did: &str, _rev: &str, _events: &[SyncEvent]) -> Result<()> { + Err(DaemonError::RetryableProjection("down".to_string())) + } + } + + let space = SpaceId::parse(SPACE).unwrap(); + let router = || Router::new(space.clone(), space.authority.clone()); + let feeds = Arc::new(JournalConsumer::new( + router(), + Box::new(FeedsProjector::new(CapturingIngress::default(), SPACE)), + )); + let appview = Arc::new(JournalConsumer::new(router(), Box::new(Stalled))); + + let index = Arc::new(InMemoryIndex::new()); + index + .journal_batch( + AUTHOR, + "3krev", + &[IndexMutation::Upsert { + collection: POST_COLLECTION.to_string(), + rkey: "3ka".to_string(), + cid: "bafypost".to_string(), + rev: "3krev".to_string(), + value: Some( + encode_record( + &match post_created("3ka", "hello") { + SyncEvent::PostCreated { record, .. } => record, + _ => unreachable!("fixture is a post create"), + }, + 64 * 1024, + ) + .unwrap(), + ), + }], + ) + .await + .unwrap(); + + let a = author(); + let host = Arc::new(PagedSpaceHost::new(vec![ListReposOutput { + cursor: None, + repos: vec![], + }])); + let client: Arc = Arc::new(ScriptedRepoHost(HashMap::new())); + let make_repo_host: RepoHostFactory = Box::new(move |_| client.clone()); + let (_tx, rx) = mpsc::channel(8); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let handle = tokio::spawn(run( + options(3600), + host, + Arc::new(StaticCredential("sc.jwt".to_string())), + make_repo_host, + index.clone(), + Arc::new(FixedKey(a.did_key.clone())), + vec![feeds.clone(), appview.clone()], + None, + rx, + shutdown_rx, + )); + + for _ in 0..200 { + if index.pending_batches("feeds").await.unwrap().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(index.pending_batches("feeds").await.unwrap().is_empty()); + assert_eq!(index.pending_batches("appview").await.unwrap().len(), 1); + + shutdown_tx.send(true).unwrap(); + handle.await.unwrap(); + } + + #[tokio::test(start_paused = true)] + async fn a_clean_sweep_acknowledges_the_space_generation_once() { + #[derive(Default)] + struct RecordingAcker(std::sync::Mutex>); + #[async_trait] + impl SpaceLifecycleAcker for RecordingAcker { + async fn acknowledge_sync(&self, space: &str, generation: i64) -> Result<()> { + self.0.lock().unwrap().push((space.to_string(), generation)); + Ok(()) + } + } + + let a = author(); + let host = Arc::new(PagedSpaceHost::new(vec![ListReposOutput { + cursor: None, + repos: vec![], + }])); + let client: Arc = Arc::new(ScriptedRepoHost(HashMap::new())); + let make_repo_host: RepoHostFactory = Box::new(move |_| client.clone()); + let acker = Arc::new(RecordingAcker::default()); + let (_tx, rx) = mpsc::channel(8); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let handle = tokio::spawn(run( + RunnerOptions { + sweep_interval_secs: 1, + generation: 7, + ..options(1) + }, + host, + Arc::new(StaticCredential("sc.jwt".to_string())), + make_repo_host, + Arc::new(InMemoryIndex::new()), + Arc::new(FixedKey(a.did_key.clone())), + Vec::new(), + Some(acker.clone()), + rx, + shutdown_rx, + )); + + for _ in 0..200 { + if !acker.0.lock().unwrap().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + tokio::time::sleep(Duration::from_secs(5)).await; + shutdown_tx.send(true).unwrap(); + handle.await.unwrap(); + + assert_eq!( + *acker.0.lock().unwrap(), + vec![(SPACE.to_string(), 7_i64)], + "the acknowledgement is sent once, not on every sweep" + ); + } + struct FailingSpaceHost; #[async_trait] impl SpaceHostClient for FailingSpaceHost { @@ -848,6 +1095,8 @@ mod tests { make_repo_host, index, keys, + Vec::new(), + None, rx, shutdown_rx, )); @@ -895,6 +1144,8 @@ mod tests { make_repo_host, index, keys, + Vec::new(), + None, rx, shutdown_rx, )); diff --git a/rsky-daemon/src/service_jwt.rs b/rsky-daemon/src/service_jwt.rs new file mode 100644 index 00000000..0ee0ab2a --- /dev/null +++ b/rsky-daemon/src/service_jwt.rs @@ -0,0 +1,101 @@ +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine as _; +use secp256k1::{Message, Secp256k1, SecretKey}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::error::{DaemonError, Result}; + +pub const MINT_LXM: &str = "community.blacksky.space.mintCredential"; + +pub struct ServiceJwtIssuer { + did: String, + secret: SecretKey, +} + +impl ServiceJwtIssuer { + pub fn from_hex(did: impl Into, key: &str) -> Result { + let bytes = hex::decode(key.trim()).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + let secret = SecretKey::from_slice(&bytes).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + Ok(Self { + did: did.into(), + secret, + }) + } + + pub fn mint(&self, audience: &str, now: u64, jti: &str) -> Result { + self.mint_for(audience, MINT_LXM, now, 60, jti) + } + + pub fn mint_for( + &self, + audience: &str, + lxm: &str, + now: u64, + ttl_secs: u64, + jti: &str, + ) -> Result { + #[derive(Serialize)] + struct Header<'a> { + typ: &'a str, + alg: &'a str, + } + #[derive(Serialize)] + struct Claims<'a> { + iss: &'a str, + aud: &'a str, + exp: u64, + lxm: &'a str, + jti: &'a str, + iat: u64, + } + let h = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&Header { + typ: "JWT", + alg: "ES256K", + }) + .unwrap(), + ); + let c = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&Claims { + iss: &self.did, + aud: audience, + exp: now + ttl_secs, + lxm, + jti, + iat: now, + }) + .unwrap(), + ); + let input = format!("{h}.{c}"); + let digest = Sha256::digest(input.as_bytes()); + let message = + Message::from_digest_slice(&digest).map_err(|e| DaemonError::Xrpc(e.to_string()))?; + let mut sig = Secp256k1::new().sign_ecdsa(&message, &self.secret); + sig.normalize_s(); + Ok(format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(sig.serialize_compact()) + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn mints_a_short_lived_method_bound_token() { + let issuer = ServiceJwtIssuer::from_hex("did:plc:daemon", &hex::encode([7u8; 32])).unwrap(); + let jwt = issuer.mint("did:plc:authority", 1000, "jti").unwrap(); + let claims: serde_json::Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(jwt.split('.').nth(1).unwrap()) + .unwrap(), + ) + .unwrap(); + assert_eq!(claims["iss"], "did:plc:daemon"); + assert_eq!(claims["aud"], "did:plc:authority"); + assert_eq!(claims["lxm"], MINT_LXM); + assert_eq!(claims["exp"], 1060); + } +} diff --git a/rsky-daemon/src/spaces.rs b/rsky-daemon/src/spaces.rs new file mode 100644 index 00000000..3ec6c915 --- /dev/null +++ b/rsky-daemon/src/spaces.rs @@ -0,0 +1,279 @@ +//! Runtime discovery of the spaces a daemon should sync. + +use async_trait::async_trait; +use rsky_space::space_id::SpaceId; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, RwLock}; + +use crate::error::{DaemonError, Result}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SpaceTarget { + pub generation: i64, + pub state: String, +} + +#[async_trait] +pub trait SpaceSource: Send + Sync { + async fn spaces(&self) -> Result>; +} + +pub struct StaticSpaces(pub BTreeSet); + +impl StaticSpaces { + pub fn new, S: Into>(spaces: I) -> Self { + Self(spaces.into_iter().map(Into::into).collect()) + } +} + +#[async_trait] +impl SpaceSource for StaticSpaces { + async fn spaces(&self) -> Result> { + Ok(self + .0 + .iter() + .cloned() + .map(|space| { + ( + space, + SpaceTarget { + generation: 1, + state: "active".into(), + }, + ) + }) + .collect()) + } +} + +pub struct HttpSpaceSource { + url: String, + api_key: String, + authority_did: Option, + space_type: String, + http: reqwest::Client, +} + +impl HttpSpaceSource { + pub fn new( + url: impl Into, + api_key: impl Into, + authority_did: Option, + space_type: impl Into, + ) -> Self { + Self { + url: url.into().trim_end_matches('/').into(), + api_key: api_key.into(), + authority_did, + space_type: space_type.into(), + http: reqwest::Client::new(), + } + } +} + +#[derive(serde::Deserialize)] +struct SpacesResponse { + spaces: Vec, +} +#[derive(serde::Deserialize)] +struct SyncableSpace { + space: String, + generation: i64, + state: String, +} + +#[async_trait] +impl SpaceSource for HttpSpaceSource { + async fn spaces(&self) -> Result> { + let mut request = self + .http + .get(format!("{}/admin/sync-spaces", self.url)) + .header("X-RSKY-KEY", &self.api_key); + if let Some(authority) = &self.authority_did { + request = request.query(&[("authority", authority)]); + } + let response = request + .send() + .await + .map_err(|e| DaemonError::Xrpc(e.to_string()))?; + if !response.status().is_success() { + return Err(DaemonError::Xrpc(format!( + "space list returned {}", + response.status() + ))); + } + let body: SpacesResponse = response + .json() + .await + .map_err(|e| DaemonError::Xrpc(e.to_string()))?; + Ok(body + .spaces + .into_iter() + .filter_map(|entry| { + let space = SpaceId::parse(&entry.space).ok()?; + (entry.generation > 0 + && matches!( + entry.state.as_str(), + "host_registered" | "active" | "deleting" + ) + && self + .authority_did + .as_deref() + .is_none_or(|authority| space.authority == authority) + && space.space_type == self.space_type) + .then_some(( + space.uri(), + SpaceTarget { + generation: entry.generation, + state: entry.state, + }, + )) + }) + .collect()) + } +} + +pub struct CombinedSource(pub Vec>); + +#[async_trait] +impl SpaceSource for CombinedSource { + async fn spaces(&self) -> Result> { + let mut all = BTreeMap::new(); + let mut last_error = None; + for source in &self.0 { + match source.spaces().await { + Ok(spaces) => { + for (space, target) in spaces { + all.entry(space) + .and_modify(|current: &mut SpaceTarget| { + if target.generation >= current.generation { + *current = target.clone(); + } + }) + .or_insert(target); + } + } + Err(error) => { + tracing::warn!(error = %error, "a space source failed"); + last_error = Some(error); + } + } + } + match last_error { + Some(error) if all.is_empty() => Err(error), + _ => Ok(all), + } + } +} + +#[derive(Clone, Default)] +pub struct SpaceRegistry(Arc>>); +impl SpaceRegistry { + pub fn new() -> Self { + Self::default() + } + pub fn contains(&self, space: &str) -> bool { + self.0.read().expect("space registry").contains(space) + } + pub fn snapshot(&self) -> BTreeSet { + self.0.read().expect("space registry").clone() + } + pub fn replace(&self, spaces: BTreeSet) { + *self.0.write().expect("space registry") = spaces; + } + pub fn insert(&self, space: impl Into) { + self.0.write().expect("space registry").insert(space.into()); + } + pub fn remove(&self, space: &str) { + self.0.write().expect("space registry").remove(space); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use wiremock::matchers::{header, method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + const A: &str = "at://did:plc:c/space/community.blacksky.feed/a"; + const B: &str = "at://did:plc:c/space/community.blacksky.feed/b"; + fn source(url: String) -> HttpSpaceSource { + HttpSpaceSource::new( + url, + "key", + Some("did:plc:c".to_string()), + "community.blacksky.feed", + ) + } + #[tokio::test] + async fn filters_and_reads_managing_app_spaces() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/admin/sync-spaces")).and(query_param("authority", "did:plc:c")).and(header("X-RSKY-KEY", "key")).respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"spaces":[{"space":A,"generation":2,"state":"active"},{"space":B,"generation":3,"state":"deleting"},{"space":"at://did:plc:other/space/community.blacksky.feed/x","generation":1,"state":"active"}]}))).mount(&server).await; + assert_eq!(source(server.uri()).spaces().await.unwrap().len(), 2); + } + #[tokio::test] + async fn without_an_authority_filter_all_authorities_are_discovered() { + use wiremock::matchers::query_param_is_missing; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/admin/sync-spaces")) + .and(query_param_is_missing("authority")) + .and(header("X-RSKY-KEY", "key")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "spaces": [ + {"space": A, "generation": 2, "state": "active"}, + {"space": "at://did:plc:other/space/community.blacksky.feed/x", "generation": 1, "state": "active"}, + {"space": "at://did:plc:other/space/wrong.type/y", "generation": 1, "state": "active"}, + {"space": "at://did:plc:third/space/community.blacksky.feed/z", "generation": 1, "state": "retired"}, + ] + }))) + .mount(&server) + .await; + let spaces = HttpSpaceSource::new(server.uri(), "key", None, "community.blacksky.feed") + .spaces() + .await + .unwrap(); + assert_eq!( + spaces.keys().collect::>(), + vec![A, "at://did:plc:other/space/community.blacksky.feed/x"] + ); + } + + #[tokio::test] + async fn pin_survives_source_outage() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let spaces = CombinedSource(vec![ + Box::new(StaticSpaces::new([A])), + Box::new(source(server.uri())), + ]) + .spaces() + .await + .unwrap(); + assert!(spaces.contains_key(A)); + } + #[tokio::test] + async fn every_source_failure_errors() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + assert!(CombinedSource(vec![Box::new(source(server.uri()))]) + .spaces() + .await + .is_err()); + } + #[test] + fn registry_tracks_spaces() { + let registry = SpaceRegistry::new(); + registry.insert(A); + assert!(registry.contains(A)); + registry.replace(BTreeSet::from([B.into()])); + assert!(!registry.contains(A) && registry.contains(B)); + registry.remove(B); + assert!(registry.snapshot().is_empty()); + } +} diff --git a/rsky-daemon/src/sqlite_index.rs b/rsky-daemon/src/sqlite_index.rs index 1ea8afc5..6426cedc 100644 --- a/rsky-daemon/src/sqlite_index.rs +++ b/rsky-daemon/src/sqlite_index.rs @@ -9,7 +9,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use crate::error::{DaemonError, Result}; -use crate::index::SpaceIndex; +use crate::index::{IndexMutation, JournaledBatch, SpaceIndex}; const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS sync_state ( @@ -29,8 +29,38 @@ CREATE TABLE IF NOT EXISTS record ( value BLOB, PRIMARY KEY (space_uri, did, collection, rkey) ); +CREATE TABLE IF NOT EXISTS projection_journal ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + mutations BLOB NOT NULL, + PRIMARY KEY (space_uri, did, rev) +); +CREATE TABLE IF NOT EXISTS projector_cursor ( + projector TEXT NOT NULL, + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + PRIMARY KEY (projector, space_uri, did) +); +CREATE TABLE IF NOT EXISTS projection_failure ( + projector TEXT NOT NULL, + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + attempts INTEGER NOT NULL, + last_error TEXT NOT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0, + denials INTEGER NOT NULL DEFAULT 0, + parked INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (projector, space_uri, did, rev) +); "; +/// Columns added to `projection_failure` after its first release; an index +/// created by an earlier build has the table but not these. +const FAILURE_COLUMNS: [&str; 2] = ["denials", "parked"]; + fn db_err(e: rusqlite::Error) -> DaemonError { DaemonError::Index(e.to_string()) } @@ -48,11 +78,40 @@ impl SqliteIndex { conn.pragma_update(None, "synchronous", "NORMAL") .map_err(db_err)?; conn.execute_batch(SCHEMA).map_err(db_err)?; + Self::add_missing_failure_columns(&conn)?; Ok(Self { conn: Mutex::new(conn), }) } + fn add_missing_failure_columns(conn: &Connection) -> Result<()> { + let mut present = Vec::new(); + { + let mut stmt = conn + .prepare("SELECT name FROM pragma_table_info('projection_failure')") + .map_err(db_err)?; + let rows = stmt + .query_map([], |row| row.get::<_, String>(0)) + .map_err(db_err)?; + for row in rows { + present.push(row.map_err(db_err)?); + } + } + for column in FAILURE_COLUMNS { + if !present.iter().any(|name| name == column) { + conn.execute( + &format!( + "ALTER TABLE projection_failure + ADD COLUMN {column} INTEGER NOT NULL DEFAULT 0" + ), + [], + ) + .map_err(db_err)?; + } + } + Ok(()) + } + /// A [`SpaceIndex`] handle scoped to one space. pub fn for_space(self: &Arc, space_uri: impl Into) -> SpaceScopedIndex { SpaceScopedIndex { @@ -157,6 +216,173 @@ impl SpaceIndex for SpaceScopedIndex { Ok(()) } + async fn journal_batch(&self, did: &str, rev: &str, mutations: &[IndexMutation]) -> Result<()> { + let encoded = serde_json::to_vec(mutations) + .map_err(|error| DaemonError::Index(format!("journal encode: {error}")))?; + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projection_journal (space_uri, did, rev, mutations) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (space_uri, did, rev) DO NOTHING", + params![self.space_uri, did, rev, encoded], + ) + .map_err(db_err)?; + Ok(()) + } + + async fn pending_batches(&self, projector: &str) -> Result> { + let conn = self.db.conn.lock().unwrap(); + let mut stmt = conn + .prepare( + "SELECT j.did, j.rev, j.mutations, COALESCE(f.denials, 0) + FROM projection_journal j + LEFT JOIN projector_cursor c + ON c.projector = ?1 AND c.space_uri = j.space_uri AND c.did = j.did + LEFT JOIN projection_failure f + ON f.projector = ?1 AND f.space_uri = j.space_uri AND f.did = j.did AND f.rev = j.rev + WHERE j.space_uri = ?2 + AND (c.rev IS NULL OR j.rev > c.rev) + AND COALESCE(f.dead_lettered, 0) = 0 + AND COALESCE(f.parked, 0) = 0 + ORDER BY j.did, j.rev", + ) + .map_err(db_err)?; + let rows = stmt + .query_map(params![projector, self.space_uri], |row| { + let mutations: Vec = row.get(2)?; + let mutations = serde_json::from_slice(&mutations) + .map_err(|_| rusqlite::Error::InvalidQuery)?; + Ok(JournaledBatch { + author: row.get(0)?, + rev: row.get(1)?, + mutations, + denials: row.get(3)?, + }) + }) + .map_err(db_err)?; + rows.collect::>>().map_err(db_err) + } + + async fn advance_projector_cursor(&self, projector: &str, did: &str, rev: &str) -> Result<()> { + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projector_cursor (projector, space_uri, did, rev) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (projector, space_uri, did) DO UPDATE SET rev = ?4", + params![projector, self.space_uri, did, rev], + ) + .map_err(db_err)?; + Ok(()) + } + + async fn record_projection_failure( + &self, + projector: &str, + did: &str, + rev: &str, + error: &str, + dead_letter_after: u32, + ) -> Result { + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projection_failure + (projector, space_uri, did, rev, attempts, last_error, dead_lettered) + VALUES (?1, ?2, ?3, ?4, 1, ?5, CASE WHEN 1 >= ?6 THEN 1 ELSE 0 END) + ON CONFLICT (projector, space_uri, did, rev) DO UPDATE + SET attempts = attempts + 1, + last_error = ?5, + dead_lettered = CASE WHEN attempts + 1 >= ?6 THEN 1 ELSE 0 END", + params![ + projector, + self.space_uri, + did, + rev, + error, + dead_letter_after + ], + ) + .map_err(db_err)?; + conn.query_row( + "SELECT attempts FROM projection_failure + WHERE projector = ?1 AND space_uri = ?2 AND did = ?3 AND rev = ?4", + params![projector, self.space_uri, did, rev], + |row| row.get(0), + ) + .map_err(db_err) + } + + async fn record_projection_denial( + &self, + projector: &str, + did: &str, + rev: &str, + error: &str, + park_after: u32, + ) -> Result { + let conn = self.db.conn.lock().unwrap(); + conn.execute( + "INSERT INTO projection_failure + (projector, space_uri, did, rev, attempts, last_error, dead_lettered, + denials, parked) + VALUES (?1, ?2, ?3, ?4, 0, ?5, 0, 1, CASE WHEN 1 >= ?6 THEN 1 ELSE 0 END) + ON CONFLICT (projector, space_uri, did, rev) DO UPDATE + SET denials = denials + 1, + last_error = ?5, + parked = CASE WHEN denials + 1 >= ?6 THEN 1 ELSE 0 END", + params![projector, self.space_uri, did, rev, error, park_after], + ) + .map_err(db_err)?; + conn.query_row( + "SELECT denials FROM projection_failure + WHERE projector = ?1 AND space_uri = ?2 AND did = ?3 AND rev = ?4", + params![projector, self.space_uri, did, rev], + |row| row.get(0), + ) + .map_err(db_err) + } + + async fn prune_journal(&self, projectors: &[&str]) -> Result { + if projectors.is_empty() { + return Ok(0); + } + let mut conn = self.db.conn.lock().unwrap(); + let tx = conn.transaction().map_err(db_err)?; + let placeholders = (0..projectors.len()) + .map(|i| format!("?{}", i + 3)) + .collect::>() + .join(", "); + let sql = format!( + "DELETE FROM projection_journal WHERE space_uri = ?1 + AND (SELECT COUNT(*) FROM projector_cursor c + WHERE c.space_uri = projection_journal.space_uri + AND c.did = projection_journal.did + AND c.rev >= projection_journal.rev + AND c.projector IN ({placeholders})) = ?2 + AND NOT EXISTS (SELECT 1 FROM projection_failure f + WHERE f.space_uri = projection_journal.space_uri + AND f.did = projection_journal.did + AND f.rev = projection_journal.rev + AND (f.dead_lettered = 1 OR f.parked = 1))" + ); + let count = projectors.len() as i64; + let mut values: Vec<&dyn rusqlite::ToSql> = vec![&self.space_uri, &count]; + for projector in projectors { + values.push(projector); + } + let pruned = tx.execute(&sql, &values[..]).map_err(db_err)?; + tx.execute( + "DELETE FROM projection_failure + WHERE space_uri = ?1 AND dead_lettered = 0 AND parked = 0 + AND NOT EXISTS (SELECT 1 FROM projection_journal j + WHERE j.space_uri = projection_failure.space_uri + AND j.did = projection_failure.did + AND j.rev = projection_failure.rev)", + params![self.space_uri], + ) + .map_err(db_err)?; + tx.commit().map_err(db_err)?; + Ok(pruned) + } + async fn list_paths(&self, did: &str) -> Result> { let conn = self.db.conn.lock().unwrap(); let mut stmt = conn @@ -185,6 +411,17 @@ impl SpaceIndex for SpaceScopedIndex { params![self.space_uri], ) .map_err(db_err)?; + for table in [ + "projection_journal", + "projector_cursor", + "projection_failure", + ] { + conn.execute( + &format!("DELETE FROM {table} WHERE space_uri = ?1"), + params![self.space_uri], + ) + .map_err(db_err)?; + } Ok(()) } } @@ -381,6 +618,188 @@ mod tests { assert_eq!(other.list_paths(AUTHOR).await.unwrap().len(), 1); } + #[tokio::test] + async fn projector_cursors_are_independent_and_survive_dead_letters() { + let dir = tempfile::tempdir().unwrap(); + let db = open_at(&dir); + let index = db.for_space(SPACE); + let mutation = |rkey: &str| IndexMutation::Delete { + collection: "app.bsky.feed.post".to_string(), + rkey: rkey.to_string(), + }; + + index + .journal_batch(AUTHOR, "3rev1", &[mutation("3ka")]) + .await + .unwrap(); + index + .journal_batch(AUTHOR, "3rev2", &[mutation("3kb")]) + .await + .unwrap(); + // A replayed batch must not duplicate its journal row. + index + .journal_batch(AUTHOR, "3rev1", &[mutation("3ka")]) + .await + .unwrap(); + assert_eq!(index.pending_batches("feeds").await.unwrap().len(), 2); + + index + .advance_projector_cursor("feeds", AUTHOR, "3rev1") + .await + .unwrap(); + let pending = index.pending_batches("feeds").await.unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].rev, "3rev2"); + assert_eq!(pending[0].mutations, vec![mutation("3kb")]); + assert_eq!(index.pending_batches("appview").await.unwrap().len(), 2); + + assert_eq!( + index + .record_projection_failure("feeds", AUTHOR, "3rev2", "boom", 2) + .await + .unwrap(), + 1 + ); + assert_eq!(index.pending_batches("feeds").await.unwrap().len(), 1); + assert_eq!( + index + .record_projection_failure("feeds", AUTHOR, "3rev2", "boom", 2) + .await + .unwrap(), + 2 + ); + assert!(index.pending_batches("feeds").await.unwrap().is_empty()); + + // Nothing prunes while a dead-lettered batch is still on the journal. + index + .advance_projector_cursor("appview", AUTHOR, "3rev2") + .await + .unwrap(); + index + .advance_projector_cursor("feeds", AUTHOR, "3rev2") + .await + .unwrap(); + assert_eq!(index.prune_journal(&["feeds", "appview"]).await.unwrap(), 1); + let remaining: i64 = { + let conn = db.conn.lock().unwrap(); + conn.query_row( + "SELECT COUNT(*) FROM projection_journal WHERE space_uri = ?1", + params![SPACE], + |row| row.get(0), + ) + .unwrap() + }; + assert_eq!(remaining, 1); + + index.purge_space().await.unwrap(); + assert!(index.pending_batches("appview").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn denials_are_budgeted_apart_from_dead_letters_and_survive_a_prune() { + let dir = tempfile::tempdir().unwrap(); + let db = open_at(&dir); + let index = db.for_space(SPACE); + let mutation = IndexMutation::Delete { + collection: "app.bsky.feed.post".to_string(), + rkey: "3ka".to_string(), + }; + index + .journal_batch(AUTHOR, "3rev1", &[mutation]) + .await + .unwrap(); + + assert_eq!( + index + .record_projection_denial("appview", AUTHOR, "3rev1", "401", 2) + .await + .unwrap(), + 1 + ); + let pending = index.pending_batches("appview").await.unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].denials, 1); + + // A denial leaves the poison budget untouched. + let attempts: u32 = { + let conn = db.conn.lock().unwrap(); + conn.query_row( + "SELECT attempts FROM projection_failure + WHERE projector = 'appview' AND space_uri = ?1 AND did = ?2 AND rev = '3rev1'", + params![SPACE, AUTHOR], + |row| row.get(0), + ) + .unwrap() + }; + assert_eq!(attempts, 0); + + assert_eq!( + index + .record_projection_denial("appview", AUTHOR, "3rev1", "401", 2) + .await + .unwrap(), + 2 + ); + assert!(index.pending_batches("appview").await.unwrap().is_empty()); + + // A parked batch keeps its journal row, so it stays inspectable. + index + .advance_projector_cursor("appview", AUTHOR, "3rev1") + .await + .unwrap(); + assert_eq!(index.prune_journal(&["appview"]).await.unwrap(), 0); + } + + #[tokio::test] + async fn an_index_written_before_the_denial_columns_is_migrated_on_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("index.sqlite"); + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE projection_failure ( + projector TEXT NOT NULL, + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + rev TEXT NOT NULL, + attempts INTEGER NOT NULL, + last_error TEXT NOT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (projector, space_uri, did, rev) + );", + ) + .unwrap(); + conn.execute( + "INSERT INTO projection_failure + (projector, space_uri, did, rev, attempts, last_error, dead_lettered) + VALUES ('appview', ?1, ?2, '3rev1', 3, 'boom', 1)", + params![SPACE, AUTHOR], + ) + .unwrap(); + } + + let db = Arc::new(SqliteIndex::open(path.to_str().unwrap()).unwrap()); + let index = db.for_space(SPACE); + assert_eq!( + index + .record_projection_denial("appview", AUTHOR, "3rev2", "401", 5) + .await + .unwrap(), + 1 + ); + let (denials, parked, dead): (u32, u32, u32) = { + let conn = db.conn.lock().unwrap(); + conn.query_row( + "SELECT denials, parked, dead_lettered FROM projection_failure + WHERE rev = '3rev1'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap() + }; + assert_eq!((denials, parked, dead), (0, 0, 1)); + } + #[tokio::test] async fn corrupt_lthash_state_is_an_error() { let dir = tempfile::tempdir().unwrap(); diff --git a/rsky-daemon/src/xrpc.rs b/rsky-daemon/src/xrpc.rs index 77c954ab..b5f2af4d 100644 --- a/rsky-daemon/src/xrpc.rs +++ b/rsky-daemon/src/xrpc.rs @@ -100,6 +100,31 @@ impl HttpSpaceHost { fn url(&self, nsid: &str) -> String { format!("{}/xrpc/{nsid}", self.base_url) } + + pub async fn mint_internal_credential( + &self, + space: &str, + service_jwt: &str, + mint_token: &str, + ) -> Result { + let url = format!("{}/admin/mintCredential", self.base_url); + let out: GetSpaceCredentialOutput = check( + self.http + .post(&url) + .header("Authorization", format!("Bearer {service_jwt}")) + .header("X-Spacehost-Mint-Token", mint_token) + .header("DPoP", self.dpop.proof("POST", &url, None)?) + .query(&[("space", space)]) + .send() + .await + .map_err(net_err)?, + ) + .await? + .json() + .await + .map_err(net_err)?; + Ok(out.credential) + } } #[async_trait] diff --git a/rsky-pds/Cargo.toml b/rsky-pds/Cargo.toml index 866802b3..2ae66164 100644 --- a/rsky-pds/Cargo.toml +++ b/rsky-pds/Cargo.toml @@ -54,7 +54,7 @@ rsky-identity = { workspace = true } rsky-lexicon = { workspace = true } rsky-repo = { workspace = true } rsky-space = { path = "../rsky-space", version = "0.4.0" } -rsky-space-host = { path = "../rsky-space-host", version = "0.5.1" } +rsky-space-host = { path = "../rsky-space-host", version = "0.7.0" } rsky-syntax = { workspace = true } hickory-resolver = "0.24.1" secp256k1 = { workspace = true } @@ -80,4 +80,3 @@ ws = { package = "rocket_ws", version = "0.1.1" } tempfile = "3.10" http-auth-basic = { version = "0.3.5" } - diff --git a/rsky-space-host/Cargo.toml b/rsky-space-host/Cargo.toml index bdfdaede..324ba6ba 100644 --- a/rsky-space-host/Cargo.toml +++ b/rsky-space-host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space-host" -version = "0.5.1" +version = "0.7.10" authors = ["Rudy Fraser "] description = "atproto permissioned-data space authority/host: issues space credentials, manages a space, routes write notifications" edition = "2021" @@ -19,8 +19,10 @@ rsky-identity = { workspace = true } rsky-lexicon = { workspace = true } rsky-oauth = { path = "../rsky-oauth", version = "0.3.0" } rsky-syntax = { workspace = true } +lexicon_cid = { workspace = true } secp256k1 = { workspace = true } sha2 = { workspace = true } +hmac = "0.12" serde = { workspace = true } serde_json = { workspace = true } rusqlite = { workspace = true } @@ -29,6 +31,7 @@ async-trait = "0.1" hex = "0.4" base64 = "0.22" rand = { workspace = true } +zeroize = "1" chrono = { version = "0.4.24", features = ["serde"] } axum = "0.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] } @@ -36,6 +39,7 @@ tokio = { workspace = true } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } clap = { version = "4", features = ["derive", "env"] } +tower-http = { version = "0.5", features = ["cors"] } [dev-dependencies] p256 = { version = "0.13.2", features = ["ecdsa"] } diff --git a/rsky-space-host/Dockerfile b/rsky-space-host/Dockerfile new file mode 100644 index 00000000..27843a78 --- /dev/null +++ b/rsky-space-host/Dockerfile @@ -0,0 +1,83 @@ +# Use the official Rust image. +# https://hub.docker.com/_/rust +FROM rust AS builder + +WORKDIR /usr/src/rsky + +# Copy workspace and all crate manifests for dependency resolution +COPY Cargo.toml Cargo.lock rust-toolchain ./ +COPY palomar-sync/Cargo.toml palomar-sync/Cargo.toml +COPY rsky-common/Cargo.toml rsky-common/Cargo.toml +COPY rsky-crypto/Cargo.toml rsky-crypto/Cargo.toml +COPY rsky-daemon/Cargo.toml rsky-daemon/Cargo.toml +COPY rsky-feedgen/Cargo.toml rsky-feedgen/Cargo.toml +COPY rsky-firehose/Cargo.toml rsky-firehose/Cargo.toml +COPY rsky-identity/Cargo.toml rsky-identity/Cargo.toml +COPY rsky-jetstream-subscriber/Cargo.toml rsky-jetstream-subscriber/Cargo.toml +COPY rsky-labeler/Cargo.toml rsky-labeler/Cargo.toml +COPY rsky-lexicon/Cargo.toml rsky-lexicon/Cargo.toml +COPY rsky-oauth/Cargo.toml rsky-oauth/Cargo.toml +COPY rsky-pds/Cargo.toml rsky-pds/Cargo.toml +COPY rsky-relay/Cargo.toml rsky-relay/Cargo.toml +COPY rsky-repo/Cargo.toml rsky-repo/Cargo.toml +COPY rsky-satnav/Cargo.toml rsky-satnav/Cargo.toml +COPY rsky-space/Cargo.toml rsky-space/Cargo.toml +COPY rsky-space-host/Cargo.toml rsky-space-host/Cargo.toml +COPY rsky-spaces-parity/Cargo.toml rsky-spaces-parity/Cargo.toml +COPY rsky-syntax/Cargo.toml rsky-syntax/Cargo.toml +COPY rsky-video/Cargo.toml rsky-video/Cargo.toml +COPY rsky-wintermute/Cargo.toml rsky-wintermute/Cargo.toml + +# Copy real source for library crates that rsky-space-host depends on +COPY rsky-common/src rsky-common/src +COPY rsky-crypto/src rsky-crypto/src +COPY rsky-identity/src rsky-identity/src +COPY rsky-lexicon/src rsky-lexicon/src +COPY rsky-oauth/src rsky-oauth/src +COPY rsky-space/src rsky-space/src +COPY rsky-syntax/src rsky-syntax/src + +# Stub out the remaining workspace members so cargo can resolve the workspace +RUN mkdir -p \ + palomar-sync/src rsky-daemon/src rsky-feedgen/src rsky-firehose/src \ + rsky-jetstream-subscriber/src rsky-labeler/src rsky-pds/src \ + rsky-relay/src rsky-repo/src rsky-satnav/src rsky-video/src \ + rsky-wintermute/src && \ + for crate in palomar-sync rsky-daemon rsky-feedgen rsky-firehose \ + rsky-jetstream-subscriber rsky-labeler rsky-pds rsky-relay \ + rsky-satnav rsky-video rsky-wintermute; do \ + echo 'fn main() {}' > $crate/src/main.rs; \ + done && \ + touch rsky-pds/src/lib.rs rsky-repo/src/lib.rs && \ + mkdir -p rsky-spaces-parity/src && touch rsky-spaces-parity/src/lib.rs && \ + mkdir -p rsky-wintermute/src/bin && \ + for bin in queue_backfill fix_blob_refs plc_import label_sync car_loader \ + cleanup_stale_deactivated; do \ + echo 'fn main() {}' > rsky-wintermute/src/bin/$bin.rs; \ + done + +# Create an empty src directory to trick Cargo into thinking it's a valid Rust project +RUN mkdir -p rsky-space-host/src && echo "fn main() {}" > rsky-space-host/src/main.rs + +# Install production dependencies and build a release artifact. +RUN cargo build --release --package rsky-space-host + +# Now copy the real source code and build the final binary +COPY rsky-space-host/src rsky-space-host/src + +RUN cargo build --release --package rsky-space-host + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /usr/src/rsky +COPY --from=builder /usr/src/rsky/target/release/rsky-space-host rsky-space-host +LABEL org.opencontainers.image.source=https://github.com/blacksky-algorithms/rsky +# State is sqlite: mount a volume covering SPACEHOST_DB_PATH, and mount the PDS +# actor store read-only at SPACEHOST_ACTOR_STORE_DIR. Required env: +# SPACEHOST_PUBLIC_URL, SPACEHOST_MINT_TOKEN, and either the +# SPACEHOST_AUTHORITY_DID/SPACEHOST_SIGNING_KEY_HEX bootstrap pin or an actor +# store to acquire authorities from. See rsky-space-host/src/config.rs for the +# full reference. +CMD ["sh", "-c", "SPACEHOST_BIND=${SPACEHOST_BIND:-0.0.0.0:3600} ./rsky-space-host"] diff --git a/rsky-space-host/src/actor_repos.rs b/rsky-space-host/src/actor_repos.rs new file mode 100644 index 00000000..e9bfb15b --- /dev/null +++ b/rsky-space-host/src/actor_repos.rs @@ -0,0 +1,454 @@ +//! [`RepoStore`] over a directory of per-account actor stores. +//! +//! One `store.sqlite` per author DID, laid out as the PDS lays out its actor +//! files, carrying the `space_*` tables of [`crate::actor_schema`]. Revisions +//! are minted here rather than supplied by the caller, so a repo's `rev` is +//! always a TID monotonic in that repo's own history. + +use async_trait::async_trait; +use rsky_common::tid::TID; +use rsky_space::lthash::{element, LtHash}; +use rsky_space::record::dag_cbor_cid; +use rsky_space::space_id::SpaceId; +use rusqlite::{params, Connection, OptionalExtension, Transaction}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; + +use crate::error::{HostError, Result}; +use crate::repo::{ + page_cursor, parse_cursor, record_path, Applied, OpPage, RepoHead, RepoStore, RepoWrite, + StoredOp, StoredRecord, WriteOutcome, STATE_BYTES, +}; + +/// Oplog rows retained per repo before the oldest revisions are dropped. +pub const DEFAULT_OPLOG_WINDOW: usize = 10_000; + +/// `{root}/{sha256(did)[..2]}/{did}/store.sqlite`, the PDS actor layout. +pub fn store_path(directory: &std::path::Path, did: &str) -> Result { + if did.is_empty() + || !did.starts_with("did:") + || did.contains('/') + || did.contains('\\') + || did.contains("..") + { + return Err(HostError::InvalidRequest(format!("unusable did: {did}"))); + } + let digest = hex::encode(Sha256::digest(did.as_bytes())); + Ok(directory.join(&digest[..2]).join(did).join("store.sqlite")) +} + +pub struct ActorStoreRepos { + directory: PathBuf, + oplog_window: usize, + connections: Mutex>, +} + +impl ActorStoreRepos { + pub fn open(directory: impl Into) -> Result { + Self::with_oplog_window(directory, DEFAULT_OPLOG_WINDOW) + } + + pub fn with_oplog_window(directory: impl Into, oplog_window: usize) -> Result { + Ok(Self { + directory: directory.into(), + oplog_window: oplog_window.max(1), + connections: Mutex::new(HashMap::new()), + }) + } + + pub fn store_path(&self, did: &str) -> Result { + store_path(&self.directory, did) + } + + fn with_store( + &self, + did: &str, + act: impl FnOnce(&mut Connection) -> Result, + ) -> Result { + let mut connections = self.connections.lock().unwrap(); + if !connections.contains_key(did) { + let path = self.store_path(did)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| store(error.to_string()))?; + } + connections.insert(did.to_string(), crate::actor_schema::get_migrated_db(path)?); + } + act(connections.get_mut(did).expect("store just opened")) + } +} + +#[async_trait] +impl RepoStore for ActorStoreRepos { + /// `rev` is ignored: the store mints the revision every write in the batch + /// shares, and returns it on [`Applied`]. + async fn apply_writes( + &self, + space_uri: &str, + did: &str, + _rev: &str, + writes: &[RepoWrite], + ) -> Result { + let space = SpaceId::parse(space_uri)?; + let window = self.oplog_window; + self.with_store(did, move |conn| { + let tx = conn.transaction().map_err(sql)?; + let applied = apply_writes_tx(&tx, &space, writes, window)?; + tx.commit().map_err(sql)?; + Ok(applied) + }) + } + + async fn head(&self, space_uri: &str, did: &str) -> Result> { + self.with_store(did, |conn| { + let Some((rev, state)) = live_repo(conn, space_uri)? else { + return Ok(None); + }; + Ok(Some(RepoHead { + rev, + state: state_bytes(state)?, + })) + }) + } + + async fn get_record( + &self, + space_uri: &str, + did: &str, + collection: &str, + rkey: &str, + ) -> Result> { + self.with_store(did, |conn| { + conn.query_row( + "SELECT collection, rkey, cid, value FROM space_record \ + WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![space_uri, collection, rkey], + row_to_record, + ) + .optional() + .map_err(sql) + }) + } + + async fn list_records( + &self, + space_uri: &str, + did: &str, + collection: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result<(Vec, Option)> { + self.with_store(did, |conn| { + if live_repo(conn, space_uri)?.is_none() { + return Err(HostError::RepoNotFound); + } + let cursor = cursor.and_then(|c| c.split_once('/')); + let mut query = String::from( + "SELECT collection, rkey, cid, value FROM space_record WHERE space_uri = ?1", + ); + let mut args: Vec<&dyn rusqlite::ToSql> = vec![&space_uri]; + if let Some(ref collection) = collection { + query.push_str(&format!(" AND collection = ?{}", args.len() + 1)); + args.push(collection); + } + if let Some((ref c, ref r)) = cursor { + let base = args.len(); + query.push_str(&format!( + " AND (collection > ?{0} OR (collection = ?{0} AND rkey > ?{1}))", + base + 1, + base + 2 + )); + args.push(c); + args.push(r); + } + query.push_str(&format!(" ORDER BY collection, rkey LIMIT {limit}")); + let mut statement = conn.prepare(&query).map_err(sql)?; + let page = statement + .query_map(rusqlite::params_from_iter(args), row_to_record) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + let cursor = page_cursor(&page, limit, |r| r.path()); + Ok((page, cursor)) + }) + } + + async fn list_ops( + &self, + space_uri: &str, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result { + let after = parse_cursor(cursor)?; + self.with_store(did, |conn| { + let floor: Option = conn + .query_row( + "SELECT oplog_floor_rev FROM space_repo WHERE space_uri = ?1 AND deleted = 0", + [space_uri], + |row| row.get(0), + ) + .optional() + .map_err(sql)? + .ok_or(HostError::RepoNotFound)?; + if let Some(floor) = floor { + match since { + Some(since) if since >= floor.as_str() => {} + _ => return Err(HostError::HistoryUnavailable), + } + } + let mut statement = conn + .prepare( + "SELECT id, rev, collection, rkey, cid, prev FROM space_oplog \ + WHERE space_uri = ?1 AND (?2 IS NULL OR rev > ?2) \ + AND (?3 IS NULL OR id > ?3) \ + ORDER BY id LIMIT ?4", + ) + .map_err(sql)?; + let mut ops = statement + .query_map(params![space_uri, since, after, limit as i64 + 1], |row| { + Ok(StoredOp { + seq: row.get(0)?, + rev: row.get(1)?, + collection: row.get(2)?, + rkey: row.get(3)?, + cid: row.get(4)?, + prev: row.get(5)?, + }) + }) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + let complete = ops.len() <= limit as usize; + ops.truncate(limit as usize); + Ok(OpPage { + cursor: if complete { + None + } else { + page_cursor(&ops, limit, |o| o.seq.to_string()) + }, + complete, + ops, + }) + }) + } + + async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()> { + self.with_store(did, |conn| { + conn.execute( + "UPDATE space_repo SET deleted = 1 WHERE space_uri = ?1", + [space_uri], + ) + .map_err(sql)?; + Ok(()) + }) + } +} + +/// Apply one batch: evolve the LtHash state, upsert/delete record rows, append +/// oplog rows sharing one fresh revision, then compact the oplog. +fn apply_writes_tx( + tx: &Transaction, + space: &SpaceId, + writes: &[RepoWrite], + window: usize, +) -> Result { + let space_uri = space.uri(); + let existing: Option<(String, Vec, i64)> = tx + .query_row( + "SELECT rev, lthash_state, deleted FROM space_repo WHERE space_uri = ?1", + [&space_uri], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(sql)?; + let (prev_rev, mut lthash) = match existing { + Some((_, _, deleted)) if deleted != 0 => return Err(HostError::RepoNotFound), + Some((rev, state, _)) => (Some(rev), LtHash::from_state_bytes(&state_bytes(state)?)), + None => (None, LtHash::new()), + }; + let rev = TID::next_str(prev_rev.clone()).map_err(|error| store(error.to_string()))?; + let mut outcomes = Vec::with_capacity(writes.len()); + + for write in writes { + let (collection, rkey) = (write.collection(), write.rkey()); + let current: Option = tx + .query_row( + "SELECT cid FROM space_record WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![&space_uri, collection, rkey], + |row| row.get(0), + ) + .optional() + .map_err(sql)?; + let swap = match write { + RepoWrite::Create { .. } => None, + RepoWrite::Update { swap_record, .. } | RepoWrite::Delete { swap_record, .. } => { + Some(swap_record) + } + }; + if let Some(swap) = swap { + if swap.is_some() && swap.as_deref() != current.as_deref() { + return Err(HostError::InvalidSwap); + } + } + let path = record_path(collection, rkey); + match write { + RepoWrite::Create { .. } if current.is_some() => { + return Err(HostError::RecordExists(path)) + } + RepoWrite::Update { .. } | RepoWrite::Delete { .. } if current.is_none() => { + return Err(HostError::RecordNotFound(path)) + } + _ => {} + } + if let Some(ref old_cid) = current { + lthash.remove(&element(collection, rkey, old_cid)); + tx.execute( + "DELETE FROM space_blob_ref WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![&space_uri, collection, rkey], + ) + .map_err(sql)?; + } + let new = match write { + RepoWrite::Create { value, .. } | RepoWrite::Update { value, .. } => { + Some((dag_cbor_cid(value).to_string(), value)) + } + RepoWrite::Delete { .. } => None, + }; + match new { + Some((ref cid, value)) => { + lthash.add(&element(collection, rkey, cid)); + tx.execute( + "INSERT INTO space_record (space_uri, collection, rkey, cid, rev, value) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6) \ + ON CONFLICT (space_uri, collection, rkey) \ + DO UPDATE SET cid = excluded.cid, rev = excluded.rev, value = excluded.value", + params![&space_uri, collection, rkey, cid, rev, value], + ) + .map_err(sql)?; + } + None => { + tx.execute( + "DELETE FROM space_record WHERE space_uri = ?1 AND collection = ?2 AND rkey = ?3", + params![&space_uri, collection, rkey], + ) + .map_err(sql)?; + } + } + let cid = new.map(|(cid, _)| cid); + tx.execute( + "INSERT INTO space_oplog (space_uri, rev, collection, rkey, cid, prev) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![&space_uri, rev, collection, rkey, cid, current], + ) + .map_err(sql)?; + outcomes.push(match (cid, current) { + (Some(cid), None) => WriteOutcome::Created { cid }, + (Some(cid), Some(_)) => WriteOutcome::Updated { cid }, + (None, _) => WriteOutcome::Deleted, + }); + } + + let state = lthash.state_bytes().to_vec(); + if prev_rev.is_some() { + tx.execute( + "UPDATE space_repo SET rev = ?2, lthash_state = ?3 WHERE space_uri = ?1", + params![&space_uri, rev, state], + ) + .map_err(sql)?; + } else { + tx.execute( + "INSERT INTO space_repo \ + (space_uri, authority, space_type, skey, rev, lthash_state, oplog_floor_rev, deleted, created_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, 0, ?7)", + params![ + &space_uri, + space.authority, + space.space_type, + space.skey, + rev, + state, + rsky_common::now() + ], + ) + .map_err(sql)?; + } + compact_oplog(tx, &space_uri, window)?; + Ok(Applied { + rev, + hash: lthash.hash(), + outcomes, + }) +} + +/// Keep at most `window` oplog rows per repo, dropping whole revisions from the +/// oldest end and advancing `oplog_floor_rev` to the newest dropped revision. +fn compact_oplog(tx: &Transaction, space_uri: &str, window: usize) -> Result<()> { + let cutoff_rev: Option = tx + .query_row( + "SELECT rev FROM space_oplog WHERE space_uri = ?1 ORDER BY id DESC LIMIT 1 OFFSET ?2", + params![space_uri, (window - 1) as i64], + |row| row.get(0), + ) + .optional() + .map_err(sql)?; + let Some(cutoff_rev) = cutoff_rev else { + return Ok(()); + }; + let floor: Option = tx + .query_row( + "SELECT MAX(rev) FROM space_oplog WHERE space_uri = ?1 AND rev < ?2", + params![space_uri, cutoff_rev], + |row| row.get(0), + ) + .map_err(sql)?; + let Some(floor) = floor else { + return Ok(()); + }; + tx.execute( + "DELETE FROM space_oplog WHERE space_uri = ?1 AND rev < ?2", + params![space_uri, cutoff_rev], + ) + .map_err(sql)?; + tx.execute( + "UPDATE space_repo SET oplog_floor_rev = ?2 WHERE space_uri = ?1", + params![space_uri, floor], + ) + .map_err(sql)?; + Ok(()) +} + +fn live_repo(conn: &Connection, space_uri: &str) -> Result)>> { + conn.query_row( + "SELECT rev, lthash_state FROM space_repo WHERE space_uri = ?1 AND deleted = 0", + [space_uri], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(sql) +} + +fn row_to_record(row: &rusqlite::Row) -> rusqlite::Result { + Ok(StoredRecord { + collection: row.get(0)?, + rkey: row.get(1)?, + cid: row.get(2)?, + value: row.get(3)?, + }) +} + +fn state_bytes(state: Vec) -> Result<[u8; STATE_BYTES]> { + state + .try_into() + .map_err(|_| store("lthash state is not 2048 bytes".to_string())) +} + +fn store(message: String) -> HostError { + HostError::Store(message) +} + +fn sql(error: rusqlite::Error) -> HostError { + HostError::Store(error.to_string()) +} diff --git a/rsky-space-host/src/actor_schema.rs b/rsky-space-host/src/actor_schema.rs new file mode 100644 index 00000000..a7cc313f --- /dev/null +++ b/rsky-space-host/src/actor_schema.rs @@ -0,0 +1,230 @@ +use rusqlite::{Connection, OptionalExtension}; +use std::collections::HashSet; +use std::path::Path; + +use crate::error::{HostError, Result}; + +const MIGRATIONS: &[(&str, &str)] = &[ + ( + "001", + "\ + CREATE TABLE repo_root (\ + did TEXT PRIMARY KEY, \ + cid TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + \"indexedAt\" TEXT NOT NULL\ + );\ + CREATE TABLE repo_block (\ + cid TEXT PRIMARY KEY, \ + \"repoRev\" TEXT NOT NULL, \ + size INTEGER NOT NULL, \ + content BLOB NOT NULL\ + );\ + CREATE INDEX repo_block_repo_rev_idx ON repo_block (\"repoRev\", cid);\ + CREATE TABLE record (\ + uri TEXT PRIMARY KEY, \ + cid TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + \"repoRev\" TEXT NOT NULL, \ + \"indexedAt\" TEXT NOT NULL, \ + \"takedownRef\" TEXT\ + );\ + CREATE INDEX record_cid_idx ON record (cid);\ + CREATE INDEX record_collection_idx ON record (collection);\ + CREATE INDEX record_repo_rev_idx ON record (\"repoRev\");\ + CREATE TABLE blob (\ + cid TEXT PRIMARY KEY, \ + \"mimeType\" TEXT NOT NULL, \ + size INTEGER NOT NULL, \ + \"tempKey\" TEXT, \ + width INTEGER, \ + height INTEGER, \ + \"createdAt\" TEXT NOT NULL, \ + \"takedownRef\" TEXT\ + );\ + CREATE INDEX blob_tempkey_idx ON blob (\"tempKey\");\ + CREATE TABLE record_blob (\ + \"blobCid\" TEXT NOT NULL, \ + \"recordUri\" TEXT NOT NULL, \ + PRIMARY KEY (\"blobCid\", \"recordUri\")\ + );\ + CREATE TABLE backlink (\ + uri TEXT NOT NULL, \ + path TEXT NOT NULL, \ + \"linkTo\" TEXT NOT NULL, \ + PRIMARY KEY (uri, path)\ + );\ + CREATE INDEX backlink_link_to_idx ON backlink (path, \"linkTo\");\ + CREATE TABLE account_pref (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + name TEXT NOT NULL, \ + \"valueJson\" TEXT NOT NULL\ + );", + ), + ( + "002", + "\ + CREATE TABLE space_repo (\ + space_uri TEXT PRIMARY KEY, \ + authority TEXT NOT NULL, \ + space_type TEXT NOT NULL, \ + skey TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + lthash_state BLOB NOT NULL, \ + oplog_floor_rev TEXT, \ + deleted INTEGER NOT NULL DEFAULT 0, \ + created_at TEXT NOT NULL\ + );\ + CREATE TABLE space_record (\ + space_uri TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + cid TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + value BLOB NOT NULL, \ + PRIMARY KEY (space_uri, collection, rkey)\ + );\ + CREATE TABLE space_oplog (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + space_uri TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + cid TEXT, \ + prev TEXT\ + );\ + CREATE INDEX space_oplog_space_idx ON space_oplog (space_uri, id);\ + CREATE TABLE space_blob_ref (\ + space_uri TEXT NOT NULL, \ + blob_cid TEXT NOT NULL, \ + collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, \ + PRIMARY KEY (space_uri, blob_cid, collection, rkey)\ + );\ + CREATE TABLE space_repo_notify (\ + space_uri TEXT NOT NULL, \ + endpoint TEXT NOT NULL, \ + expires_at TEXT NOT NULL, \ + PRIMARY KEY (space_uri, endpoint)\ + );\ + CREATE TABLE space_def (\ + space_uri TEXT PRIMARY KEY, \ + space_type TEXT NOT NULL, \ + skey TEXT NOT NULL, \ + policy TEXT NOT NULL DEFAULT 'member-list', \ + app_access TEXT NOT NULL DEFAULT 'open', \ + allowed_clients TEXT, \ + managing_app TEXT, \ + deleted INTEGER NOT NULL DEFAULT 0, \ + created_at TEXT NOT NULL\ + );\ + CREATE TABLE space_member (\ + space_uri TEXT NOT NULL, \ + did TEXT NOT NULL, \ + PRIMARY KEY (space_uri, did)\ + );\ + CREATE TABLE space_writer (\ + space_uri TEXT NOT NULL, \ + did TEXT NOT NULL, \ + rev TEXT NOT NULL, \ + hash TEXT, \ + PRIMARY KEY (space_uri, did)\ + );\ + CREATE TABLE space_host_reg (\ + space_uri TEXT NOT NULL, \ + endpoint TEXT NOT NULL, \ + expires_at TEXT NOT NULL, \ + PRIMARY KEY (space_uri, endpoint)\ + );\ + CREATE TABLE space_used_jti (\ + jti TEXT PRIMARY KEY, \ + exp INTEGER NOT NULL\ + );", + ), + // A notification is delivered with service auth addressed to the + // subscriber, so a registration has to remember who the subscriber is. + // Rows written before this carry no service identifier and keep the + // pre-amendment behaviour (proposals#100). + ( + "003", + "\ + ALTER TABLE space_repo_notify ADD COLUMN service TEXT;\ + ALTER TABLE space_host_reg ADD COLUMN service TEXT;", + ), + // Member rows carry when and at what rev they were added, which clients + // surface; and each account keeps a local index of spaces it was enrolled + // in, because listSpaces is how a member discovers a shared space at all -- + // a repo row only exists after the member's first write. + ( + "004", + "\ + ALTER TABLE space_member ADD COLUMN member_rev TEXT NOT NULL DEFAULT '';\ + ALTER TABLE space_member ADD COLUMN added_at TEXT NOT NULL DEFAULT '';\ + CREATE TABLE space_joined (\ + space_uri TEXT PRIMARY KEY, \ + authority TEXT NOT NULL, \ + space_type TEXT NOT NULL, \ + created_at TEXT NOT NULL\ + );", + ), +]; + +/// The first migration recreates the PDS's base schema. A store the PDS itself +/// created already has those tables under different bookkeeping, so applying it +/// there would both fail and be a write into a file this service does not own. +const BASELINE_MIGRATION: &str = "001"; + +/// Whether the base schema is already present, i.e. this file was created by +/// something other than these migrations. +fn baseline_present(tx: &rusqlite::Transaction<'_>) -> Result { + tx.query_row( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'repo_root'", + [], + |_| Ok(()), + ) + .optional() + .map(|found| found.is_some()) + .map_err(sql_err) +} + +pub fn get_migrated_db(path: impl AsRef) -> Result { + let mut connection = Connection::open(path).map_err(sql_err)?; + let transaction = connection.transaction().map_err(sql_err)?; + transaction + .execute_batch("CREATE TABLE IF NOT EXISTS migrations (name TEXT PRIMARY KEY, \"appliedAt\" TEXT NOT NULL)") + .map_err(sql_err)?; + let mut applied = transaction + .prepare("SELECT name FROM migrations") + .map_err(sql_err)? + .query_map([], |row| row.get::<_, String>(0)) + .map_err(sql_err)? + .collect::>>() + .map_err(sql_err)?; + if !applied.contains(BASELINE_MIGRATION) && baseline_present(&transaction)? { + transaction + .execute( + "INSERT INTO migrations (name, \"appliedAt\") VALUES (?1, datetime('now'))", + [BASELINE_MIGRATION], + ) + .map_err(sql_err)?; + applied.insert(BASELINE_MIGRATION.to_string()); + } + for (name, sql) in MIGRATIONS { + if !applied.contains(*name) { + transaction.execute_batch(sql).map_err(sql_err)?; + transaction + .execute( + "INSERT INTO migrations (name, \"appliedAt\") VALUES (?1, datetime('now'))", + [name], + ) + .map_err(sql_err)?; + } + } + transaction.commit().map_err(sql_err)?; + Ok(connection) +} + +fn sql_err(error: rusqlite::Error) -> HostError { + HostError::Store(error.to_string()) +} diff --git a/rsky-space-host/src/authority.rs b/rsky-space-host/src/authority.rs index 58acc276..a9a80d07 100644 --- a/rsky-space-host/src/authority.rs +++ b/rsky-space-host/src/authority.rs @@ -8,31 +8,136 @@ use rsky_space::credential::{ self, Confirmation, JwtHeader, SpaceClaims, CREDENTIAL_TTL_SECS, CREDENTIAL_TYP, }; use rsky_space::space_id::SpaceId; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, RwLock}; use crate::appaccess::AppAccess; use crate::attestation::{verify_client_attestation, JtiStore, MetadataFetcher}; use crate::error::{HostError, Result}; +use crate::notify::Notifier; use crate::policy::Policy; +use crate::registration::LifecycleAcker; use crate::signing::Signer; +/// Everything derived from one space authority: the authority itself plus the +/// collaborators that mint, verify, or sign in its name. +pub struct AuthorityContext { + pub authority: Arc, + pub policy: Arc, + pub notifier: Arc, + pub lifecycle_acker: Option>, +} + +impl AuthorityContext { + pub fn authority_did(&self) -> &str { + self.authority.authority_did() + } +} + +/// Builds the [`AuthorityContext`] for an authority first seen at +/// registration time; fails when the authority's signing key is unavailable. +pub type AuthorityFactory = Arc Result> + Send + Sync>; + +/// The authorities this host answers for, keyed by authority DID. +#[derive(Default)] +pub struct AuthorityRegistry { + contexts: RwLock>>, +} + +impl AuthorityRegistry { + pub fn new() -> Self { + Self::default() + } + + pub fn insert(&self, context: Arc) { + self.contexts + .write() + .expect("authority registry") + .insert(context.authority_did().to_string(), context); + } + + /// Insert unless the authority is already present; returns the context + /// that ends up registered either way. + pub fn insert_if_absent(&self, context: Arc) -> Arc { + self.contexts + .write() + .expect("authority registry") + .entry(context.authority_did().to_string()) + .or_insert(context) + .clone() + } + + pub fn authority(&self, authority_did: &str) -> Result> { + self.contexts + .read() + .expect("authority registry") + .get(authority_did) + .cloned() + .ok_or_else(|| HostError::SpaceNotFound(authority_did.to_string())) + } + + pub fn for_space(&self, space_uri: &str) -> Result> { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + self.authority(&space.authority) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string())) + } + + pub fn contexts(&self) -> Vec> { + self.contexts + .read() + .expect("authority registry") + .values() + .cloned() + .collect() + } +} + /// Resolves an account's atproto signing `did:key` (from its DID document), used /// to verify a delegation token minted by that user's PDS. #[async_trait] pub trait KeyResolver: Send + Sync { - async fn signing_key(&self, did: &str) -> Result; + async fn service_key(&self, did: &str) -> Result; } -/// A space authority for a single space. +/// A space authority for one or more spaces of a single type. pub struct Authority { pub space: SpaceId, + hosted: RwLock>, + registered: RwLock>, pub signer: Signer, pub app_access: AppAccess, } impl Authority { pub fn new(space: SpaceId, signer: Signer, app_access: AppAccess) -> Self { + Self::new_many( + space.authority.clone(), + space.space_type.clone(), + [space.skey.clone()], + signer, + app_access, + ) + } + + pub fn new_many( + authority_did: impl Into, + space_type: impl Into, + skeys: impl IntoIterator, + signer: Signer, + app_access: AppAccess, + ) -> Self { + let authority_did = authority_did.into(); + let space_type = space_type.into(); + let hosted: BTreeSet = skeys.into_iter().collect(); + let first_skey = hosted + .first() + .expect("space authority needs at least one hosted space") + .clone(); Self { - space, + space: SpaceId::new(authority_did, space_type, first_skey), + registered: RwLock::new(hosted.clone()), + hosted: RwLock::new(hosted), signer, app_access, } @@ -46,6 +151,90 @@ impl Authority { self.space.uri() } + pub fn space(&self, skey: &str) -> SpaceId { + SpaceId::new(&self.space.authority, &self.space.space_type, skey) + } + + pub fn spaces(&self) -> Vec { + self.hosted + .read() + .expect("hosted spaces") + .iter() + .map(|skey| self.space(skey).uri()) + .collect() + } + + pub fn resolve(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority == self.space.authority + && space.space_type == self.space.space_type + && self + .hosted + .read() + .expect("hosted spaces") + .contains(&space.skey) + { + Ok(space) + } else { + Err(HostError::SpaceNotFound(space_uri.to_string())) + } + } + + pub fn register(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority != self.space.authority || space.space_type != self.space.space_type { + return Err(HostError::SpaceNotFound(space_uri.to_string())); + } + let newly_registered = self + .registered + .write() + .expect("registered spaces") + .insert(space.skey.clone()); + self.hosted + .write() + .expect("hosted spaces") + .insert(space.skey); + Ok(newly_registered) + } + + pub fn resolve_registered(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority == self.space.authority + && space.space_type == self.space.space_type + && self + .registered + .read() + .expect("registered spaces") + .contains(&space.skey) + { + Ok(space) + } else { + Err(HostError::SpaceNotFound(space_uri.to_string())) + } + } + + pub fn unregister(&self, space_uri: &str) -> Result { + let space = SpaceId::parse(space_uri) + .map_err(|_| HostError::SpaceNotFound(space_uri.to_string()))?; + if space.authority != self.space.authority || space.space_type != self.space.space_type { + return Err(HostError::SpaceNotFound(space_uri.to_string())); + } + let registered = self + .registered + .write() + .expect("registered spaces") + .remove(&space.skey); + let hosted = self + .hosted + .write() + .expect("hosted spaces") + .remove(&space.skey); + Ok(registered || hosted) + } + /// The simplespace config surfaced by `getSpace`. pub fn space_config(&self, policy: &Policy) -> SimplespaceConfig { let app_access = match &self.app_access { @@ -68,7 +257,13 @@ impl Authority { /// `dpop_jkt` must come from a DPoP proof this server verified, never from /// a request field: a field is an assertion anyone holding a delegation /// token can make about a key someone else controls. - pub fn mint_credential(&self, now: u64, jti: String, dpop_jkt: &str) -> Result { + pub fn mint_credential_for( + &self, + space: &SpaceId, + now: u64, + jti: String, + dpop_jkt: &str, + ) -> Result { let header = JwtHeader { typ: CREDENTIAL_TYP.to_string(), alg: rsky_crypto::constants::SECP256K1_JWT_ALG.to_string(), @@ -76,7 +271,7 @@ impl Authority { }; let claims = SpaceClaims { iss: self.authority_did().to_string(), - sub: self.space_uri(), + sub: space.uri(), aud: None, iat: now, exp: now + CREDENTIAL_TTL_SECS, @@ -89,12 +284,17 @@ impl Authority { Ok(jwt) } + pub fn mint_credential(&self, now: u64, jti: String, dpop_jkt: &str) -> Result { + self.mint_credential_for(&self.space, now, jti, dpop_jkt) + } + /// The full `getSpaceCredential` flow: verify the client attestation (when /// required or presented), apply appAccess, verify the delegation token, /// consult the policy, then mint. #[allow(clippy::too_many_arguments)] - pub async fn get_space_credential( + pub async fn get_space_credential_for( &self, + space: &SpaceId, delegation_jwt: &str, attestation_jwt: Option<&str>, policy: &Policy, @@ -125,10 +325,10 @@ impl Authority { let decoded = credential::decode(delegation_jwt).map_err(|e| HostError::Delegation(e.to_string()))?; let user_did = decoded.claims.iss.clone(); - let user_key = keys.signing_key(&user_did).await?; + let user_key = keys.service_key(&user_did).await?; let verified_user = credential::verify_delegation_token( delegation_jwt, - &self.space_uri(), + &space.uri(), self.authority_did(), &user_key, now, @@ -136,16 +336,40 @@ impl Authority { .map_err(|e| HostError::Delegation(e.to_string()))?; // User axis: the policy decision (member list, public, or managing app). if !policy - .authorizes( - &self.space_uri(), - &verified_user, - attested_client_id.as_deref(), - ) + .authorizes(&space.uri(), &verified_user, attested_client_id.as_deref()) .await? { return Err(HostError::NotAuthorized); } - self.mint_credential(now, jti, dpop_jkt) + self.mint_credential_for(space, now, jti, dpop_jkt) + } + + #[allow(clippy::too_many_arguments)] + pub async fn get_space_credential( + &self, + delegation_jwt: &str, + attestation_jwt: Option<&str>, + policy: &Policy, + keys: &dyn KeyResolver, + metadata: &dyn MetadataFetcher, + jti_store: &dyn JtiStore, + now: u64, + jti: String, + dpop_jkt: &str, + ) -> Result { + self.get_space_credential_for( + &self.space, + delegation_jwt, + attestation_jwt, + policy, + keys, + metadata, + jti_store, + now, + jti, + dpop_jkt, + ) + .await } } @@ -172,6 +396,16 @@ mod tests { Authority::new(space, test_signer(), AppAccess::Open) } + #[test] + fn registering_a_space_makes_it_immediately_serveable() { + let authority = authority(); + let space = "at://did:plc:communityauthority/space/community.blacksky.feed/new"; + + assert!(authority.register(space).unwrap()); + assert_eq!(authority.resolve(space).unwrap().skey, "new"); + assert!(authority.spaces().contains(&space.to_string())); + } + fn member_policy(dids: &[&str]) -> Policy { Policy::MemberList(Arc::new(InMemoryMembership::new( dids.iter().map(|d| d.to_string()), @@ -267,7 +501,7 @@ mod tests { struct DenyAllKeys; #[async_trait] impl KeyResolver for DenyAllKeys { - async fn signing_key(&self, _did: &str) -> Result { + async fn service_key(&self, _did: &str) -> Result { Err(HostError::Membership("no key".into())) } } @@ -275,7 +509,7 @@ mod tests { struct FixedKey(String); #[async_trait] impl KeyResolver for FixedKey { - async fn signing_key(&self, _did: &str) -> Result { + async fn service_key(&self, _did: &str) -> Result { Ok(self.0.clone()) } } diff --git a/rsky-space-host/src/bin/convert_store.rs b/rsky-space-host/src/bin/convert_store.rs new file mode 100644 index 00000000..f8d42fff --- /dev/null +++ b/rsky-space-host/src/bin/convert_store.rs @@ -0,0 +1,23 @@ +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(about = "Convert a multi-tenant space store into per-account actor stores")] +struct Args { + /// The deployed multi-tenant sqlite file, opened read-only. + #[arg(long)] + from: PathBuf, + /// Directory the per-account `store.sqlite` files are written under. + #[arg(long)] + into: PathBuf, +} + +fn main() -> Result<(), rsky_space_host::HostError> { + let args = Args::parse(); + let totals = rsky_space_host::convert::convert(&args.from, &args.into)?; + println!( + "converted {} accounts, {} repos, {} records, {} ops", + totals.accounts, totals.repos, totals.records, totals.ops + ); + Ok(()) +} diff --git a/rsky-space-host/src/client_jws.rs b/rsky-space-host/src/client_jws.rs new file mode 100644 index 00000000..96cf3722 --- /dev/null +++ b/rsky-space-host/src/client_jws.rs @@ -0,0 +1,241 @@ +//! JWS verification for tokens a *client* presents: OAuth access tokens and +//! DPoP proofs. +//! +//! These are signed by browsers and SDKs, and WebCrypto emits a high-S ECDSA +//! signature about half the time. Such a signature is perfectly valid ECDSA, so +//! rejecting it fails roughly every other request from a legitimate client. +//! `rsky_space::jwk`'s verifiers are strict for a reason — they also serve repo +//! commits, where low-S is required because a malleable commit signature would +//! be a second valid signature over the same content — so these are separate +//! entry points rather than a relaxation of those. +//! +//! The asymmetry is the point: malleable here, strict for commits. + +use rsky_crypto::types::VerifyOptions; +use rsky_space::jwk::{EcJwk, CRV_P256, CRV_SECP256K1}; +use rsky_space::SpaceError; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Distinct from `SpaceError::BadSignature`, whose message reads "commit +/// signature verification failed" — accurate for a repo commit, actively +/// misleading on a DPoP proof, which is where it kept surfacing. +#[derive(Debug, Error)] +pub enum ClientJwsError { + #[error("presented signature does not verify against the key in its header")] + BadSignature, + #[error("unusable key: {0}")] + Key(String), + #[error("verification failed: {0}")] + Crypto(String), +} + +type Result = std::result::Result; + +impl From for ClientJwsError { + fn from(error: SpaceError) -> Self { + ClientJwsError::Key(error.to_string()) + } +} + +fn malleable() -> Option { + Some(VerifyOptions { + allow_malleable_sig: Some(true), + }) +} + +fn outcome(ok: bool) -> Result<()> { + if ok { + Ok(()) + } else { + Err(ClientJwsError::BadSignature) + } +} + +/// ES256 over a JWT signing input, accepting a high-S signature. +pub fn verify_client_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { + jwk.require_crv(CRV_P256)?; + let point = jwk.sec1_point()?; + let ok = rsky_crypto::p256::operations::verify_sig(&point, signing_input, sig, malleable()) + .map_err(|e| ClientJwsError::Crypto(e.to_string()))?; + outcome(ok) +} + +/// ES256K over a JWT signing input, accepting a high-S signature. +/// +/// `allow_malleable_sig` is not enough on this curve: it only waives the +/// encoding check, and libsecp256k1 refuses a high-S signature regardless. So +/// normalise the scalar first. The P-256 verifier needs no equivalent because +/// RustCrypto's does not enforce low-S in the first place. +pub fn verify_client_es256k(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { + jwk.require_crv(CRV_SECP256K1)?; + let point = jwk.sec1_point()?; + let digest = Sha256::digest(signing_input); + // A JWS signature is the fixed-width `r || s`, but DER has been accepted + // here historically; leave that path exactly as it was. + let normalised = match secp256k1::ecdsa::Signature::from_compact(sig) { + Ok(mut parsed) => { + parsed.normalize_s(); + Some(parsed.serialize_compact()) + } + Err(_) => None, + }; + let presented = normalised.as_ref().map(|s| s.as_slice()).unwrap_or(sig); + let ok = + rsky_crypto::secp256k1::operations::verify_sig(&point, &digest, presented, malleable()) + .map_err(|e| ClientJwsError::Crypto(e.to_string()))?; + outcome(ok) +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use base64::Engine; + use p256::ecdsa::signature::hazmat::PrehashSigner; + use p256::ecdsa::SigningKey; + + const SIGNING_INPUT: &[u8] = b"eyJhbGciOiJFUzI1NiJ9.eyJodG0iOiJQT1NUIn0"; + + /// A P-256 key plus the JWK a client would publish for it. + fn key_and_jwk(secret: [u8; 32]) -> (SigningKey, EcJwk) { + let key = SigningKey::from_slice(&secret).expect("key"); + let point = key.verifying_key().to_encoded_point(false); + let jwk = EcJwk { + kty: "EC".to_string(), + crv: CRV_P256.to_string(), + x: URL_SAFE_NO_PAD.encode(point.x().expect("x")), + y: URL_SAFE_NO_PAD.encode(point.y().expect("y")), + kid: None, + }; + (key, jwk) + } + + /// A deliberately high-S `r || s`, i.e. what WebCrypto may hand us. + fn high_s_signature(key: &SigningKey, input: &[u8]) -> Vec { + let digest = Sha256::digest(input); + let sig: p256::ecdsa::Signature = key.sign_prehash(&digest).expect("sign"); + // `normalize_s` yields Some only when it changed something, so this is + // the low-S form either way; its counterpart is then always high-S. + let low = sig.normalize_s().unwrap_or(sig); + malleable_counterpart(&low).to_bytes().to_vec() + } + + /// (r, n - s) — the other valid signature over the same message. + fn malleable_counterpart(sig: &p256::ecdsa::Signature) -> p256::ecdsa::Signature { + use p256::elliptic_curve::scalar::IsHigh; + let (r, s) = sig.split_scalars(); + let flipped = -*s; + assert!(bool::from(flipped.is_high()), "counterpart must be high-S"); + p256::ecdsa::Signature::from_scalars(*r, flipped).expect("signature") + } + + #[test] + fn high_s_es256_jws_signature_verifies() { + let (key, jwk) = key_and_jwk([0x31; 32]); + let sig = high_s_signature(&key, SIGNING_INPUT); + + // The whole point: a client's high-S proof is accepted here. + verify_client_es256(&jwk, SIGNING_INPUT, &sig).expect("high-S client jws must verify"); + + // And the strict path — the one repo commits go through — still refuses + // it. The asymmetry is deliberate, so assert both halves together. + assert!( + rsky_space::jwk::verify_es256(&jwk, SIGNING_INPUT, &sig).is_err(), + "the commit-path verifier must keep rejecting a high-S signature" + ); + } + + #[test] + fn a_low_s_signature_verifies_on_both_paths() { + let (key, jwk) = key_and_jwk([0x32; 32]); + let digest = Sha256::digest(SIGNING_INPUT); + let sig: p256::ecdsa::Signature = key.sign_prehash(&digest).expect("sign"); + let sig = sig.normalize_s().unwrap_or(sig); + let bytes = sig.to_bytes().to_vec(); + + verify_client_es256(&jwk, SIGNING_INPUT, &bytes).expect("client path"); + rsky_space::jwk::verify_es256(&jwk, SIGNING_INPUT, &bytes).expect("commit path"); + } + + /// A secp256k1 key plus the JWK a client publishes for it. + fn k256_key_and_jwk(secret: [u8; 32]) -> (secp256k1::SecretKey, EcJwk) { + let secret = secp256k1::SecretKey::from_slice(&secret).expect("key"); + let public = secp256k1::PublicKey::from_secret_key(&secp256k1::Secp256k1::new(), &secret); + let uncompressed = public.serialize_uncompressed(); + let jwk = EcJwk { + kty: "EC".to_string(), + crv: CRV_SECP256K1.to_string(), + x: URL_SAFE_NO_PAD.encode(&uncompressed[1..33]), + y: URL_SAFE_NO_PAD.encode(&uncompressed[33..65]), + kid: None, + }; + (secret, jwk) + } + + /// A deliberately high-S ES256K signature: what the OAuth client emits + /// about half the time. + fn high_s_k256_signature(secret: &secp256k1::SecretKey, input: &[u8]) -> Vec { + let digest = Sha256::digest(input); + let message = secp256k1::Message::from_digest_slice(&digest).expect("digest"); + let signature = secp256k1::Secp256k1::new().sign_ecdsa(&message, secret); + // rust-secp256k1 always hands back the normalised form, so flip it. + let mut bytes = signature.serialize_compact(); + let n = [ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, + 0xD0, 0x36, 0x41, 0x41, + ]; + let mut borrow = 0i16; + for i in (0..32).rev() { + let diff = n[i] as i16 - bytes[32 + i] as i16 - borrow; + if diff < 0 { + bytes[32 + i] = (diff + 256) as u8; + borrow = 1; + } else { + bytes[32 + i] = diff as u8; + borrow = 0; + } + } + bytes.to_vec() + } + + #[test] + fn high_s_es256k_jws_signature_verifies() { + let (secret, jwk) = k256_key_and_jwk([0x51; 32]); + let sig = high_s_k256_signature(&secret, SIGNING_INPUT); + + // The client's high-S proof is accepted here... + verify_client_es256k(&jwk, SIGNING_INPUT, &sig) + .expect("high-S client jws must verify on secp256k1"); + + // ...and the commit-path verifier still refuses it. Both curves are + // asserted because only the P-256 half was ever covered. + assert!( + rsky_space::jwk::verify_es256k(&jwk, SIGNING_INPUT, &sig).is_err(), + "the commit-path verifier must keep rejecting a high-S signature" + ); + } + + #[test] + fn a_low_s_es256k_signature_verifies_on_both_paths() { + let (secret, jwk) = k256_key_and_jwk([0x52; 32]); + let digest = Sha256::digest(SIGNING_INPUT); + let message = secp256k1::Message::from_digest_slice(&digest).expect("digest"); + let sig = secp256k1::Secp256k1::new() + .sign_ecdsa(&message, &secret) + .serialize_compact() + .to_vec(); + + verify_client_es256k(&jwk, SIGNING_INPUT, &sig).expect("client path"); + rsky_space::jwk::verify_es256k(&jwk, SIGNING_INPUT, &sig).expect("commit path"); + } + + #[test] + fn a_wrong_key_is_refused_on_the_client_path_too() { + let (key, _) = key_and_jwk([0x33; 32]); + let (_, other_jwk) = key_and_jwk([0x34; 32]); + let sig = high_s_signature(&key, SIGNING_INPUT); + assert!(verify_client_es256(&other_jwk, SIGNING_INPUT, &sig).is_err()); + } +} diff --git a/rsky-space-host/src/commits.rs b/rsky-space-host/src/commits.rs new file mode 100644 index 00000000..2af8579f --- /dev/null +++ b/rsky-space-host/src/commits.rs @@ -0,0 +1,145 @@ +//! Minting signed commits for a hosted repo (spec §Commit signature). +//! +//! `ikm` is fresh per reader, so a commit is produced at serve time from the +//! repo's persisted `(rev, state)` rather than stored alongside them. +//! +//! The spec has the account's own signing key sign the commit context. A host +//! holding repos on behalf of accounts whose PDS does not implement +//! permissioned data has no access to those keys, so [`CommitSigner`] names the +//! signer explicitly instead of assuming it. + +use rsky_lexicon::com::atproto::space::SignedCommit; +use rsky_space::commit::{build_ctx, compute_mac}; + +use crate::error::{HostError, Result}; +use crate::signing::Signer; + +pub const COMMIT_VERSION: i64 = 1; +pub const IKM_BYTES: usize = 32; + +pub trait CommitSigner: Send + Sync { + /// The `did:key` a reader verifies commits against. + fn did_key(&self, author_did: &str) -> Result; + fn sign(&self, author_did: &str, space_uri: &str, rev: &str, message: &[u8]) + -> Result>; +} + +impl CommitSigner for Signer { + fn did_key(&self, _author_did: &str) -> Result { + Ok(Signer::did_key(self).to_string()) + } + + fn sign( + &self, + _author_did: &str, + _space_uri: &str, + _rev: &str, + message: &[u8], + ) -> Result> { + Signer::sign(self, message).map_err(HostError::Key) + } +} + +/// Build a signed commit over a repo's current digest. +pub fn mint_commit( + signer: &dyn CommitSigner, + space_uri: &str, + author_did: &str, + rev: &str, + hash: &[u8; 32], + ikm: [u8; IKM_BYTES], +) -> Result { + let ctx = build_ctx(space_uri, author_did, rev, &ikm); + let sig = signer.sign(author_did, space_uri, rev, &ctx)?; + let mac = compute_mac(&ikm, &ctx, hash)?; + Ok(SignedCommit { + ver: COMMIT_VERSION, + hash: hash.to_vec(), + ikm: ikm.to_vec(), + sig, + mac: mac.to_vec(), + rev: rev.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::signing::test_signer; + use rsky_space::commit::verify_commit; + + const SPACE: &str = "at://did:plc:auth/space/community.blacksky.feed/main"; + const AUTHOR: &str = "did:plc:member"; + + fn minted(hash: [u8; 32], ikm: [u8; 32]) -> (Signer, SignedCommit) { + let signer = test_signer(); + let commit = mint_commit(&signer, SPACE, AUTHOR, "3rev1", &hash, ikm).unwrap(); + (signer, commit) + } + + fn verify(signer: &Signer, commit: &SignedCommit, hash: &[u8]) -> rsky_space::Result<()> { + verify_commit( + &CommitSigner::did_key(signer, AUTHOR).unwrap(), + SPACE, + AUTHOR, + &commit.rev, + &commit.ikm, + &commit.sig, + &commit.mac, + hash, + ) + } + + #[test] + fn a_minted_commit_verifies() { + let hash = [7u8; 32]; + let (signer, commit) = minted(hash, [3u8; 32]); + assert_eq!(commit.ver, COMMIT_VERSION); + assert_eq!(commit.hash, hash.to_vec()); + verify(&signer, &commit, &hash).unwrap(); + } + + #[test] + fn a_tampered_hash_fails_the_mac() { + let (signer, commit) = minted([7u8; 32], [3u8; 32]); + assert!(verify(&signer, &commit, &[8u8; 32]).is_err()); + } + + #[test] + fn a_different_ikm_yields_a_different_signature() { + let hash = [7u8; 32]; + let (_, a) = minted(hash, [3u8; 32]); + let (signer, b) = minted(hash, [4u8; 32]); + assert_ne!(a.sig, b.sig); + assert_ne!(a.mac, b.mac); + verify(&signer, &b, &hash).unwrap(); + } + + #[test] + fn a_commit_does_not_verify_under_another_space_or_author() { + let hash = [7u8; 32]; + let (signer, commit) = minted(hash, [3u8; 32]); + assert!(verify_commit( + &CommitSigner::did_key(&signer, AUTHOR).unwrap(), + "at://did:plc:auth/space/community.blacksky.feed/other", + AUTHOR, + &commit.rev, + &commit.ikm, + &commit.sig, + &commit.mac, + &hash, + ) + .is_err()); + assert!(verify_commit( + &CommitSigner::did_key(&signer, AUTHOR).unwrap(), + SPACE, + "did:plc:someoneelse", + &commit.rev, + &commit.ikm, + &commit.sig, + &commit.mac, + &hash, + ) + .is_err()); + } +} diff --git a/rsky-space-host/src/config.rs b/rsky-space-host/src/config.rs index c903afc7..2554c1dd 100644 --- a/rsky-space-host/src/config.rs +++ b/rsky-space-host/src/config.rs @@ -1,5 +1,7 @@ //! Configuration for the space-host service (env prefix `SPACEHOST_`). +use crate::oauth::AuthConfig; +use crate::pds_seam::VerifyOnlyHs256Secret; use clap::Parser; /// The Blacksky community space (v1: a single typed space under the authority). @@ -19,12 +21,16 @@ pub enum PolicyMode { about = "atproto permissioned-data space authority/host" )] pub struct Config { - /// The space authority DID (dedicated community DID). - #[arg(long, env = "SPACEHOST_AUTHORITY_DID")] + /// Optional bootstrap authority pin: a space authority DID served from + /// startup with an explicit signing key. Set together with + /// `SPACEHOST_SIGNING_KEY_HEX`, or leave both unset and let authorities + /// arrive via registration. + #[arg(long, env = "SPACEHOST_AUTHORITY_DID", default_value = "")] pub authority_did: String, - /// Hex-encoded secp256k1 space signing key (`#atproto_space`). - #[arg(long, env = "SPACEHOST_SIGNING_KEY_HEX")] + /// Hex-encoded secp256k1 space signing key (`#atproto_space`) for the + /// pinned bootstrap authority. + #[arg(long, env = "SPACEHOST_SIGNING_KEY_HEX", default_value = "")] pub signing_key_hex: String, /// How the authority authorizes users at credential-mint time. @@ -49,6 +55,14 @@ pub struct Config { #[arg(long, env = "SPACEHOST_MEMBERSHIP_DB_URL", default_value = "")] pub membership_db_url: String, + /// Feeds base URL that receives host-registration acknowledgements. + #[arg(long, env = "SPACEHOST_LIFECYCLE_URL", default_value = "")] + pub lifecycle_url: String, + + /// Feeds service DID, used as the acknowledgement JWT audience. + #[arg(long, env = "SPACEHOST_LIFECYCLE_SERVICE_DID", default_value = "")] + pub lifecycle_service_did: String, + /// SQLite path for host state (writer set, registrations, used nonces). #[arg(long, env = "SPACEHOST_DB_PATH", default_value = "./space_host.db")] pub db_path: String, @@ -73,6 +87,42 @@ pub struct Config { default_value = "http://localhost:3600" )] pub public_url: String, + + #[arg(long, env = "SPACEHOST_OAUTH_ISSUER", default_value = "")] + pub oauth_issuer: String, + #[arg(long, env = "SPACEHOST_OAUTH_JWKS_URI", default_value = "")] + pub oauth_jwks_uri: String, + #[arg(long, env = "SPACEHOST_OAUTH_AUDIENCE", default_value = "")] + pub oauth_audience: String, + #[arg(long, env = "SPACEHOST_OAUTH_CLIENT_IDS", default_value = "")] + pub oauth_client_ids: String, + #[arg( + long, + env = "SPACEHOST_OAUTH_HS256_SECRET", + default_value = "", + hide_env_values = true + )] + pub oauth_hs256_secret: String, + /// Read-only source of per-account signing keys: the PDS's own `actors` + /// directory. Never written to. + #[arg(long, env = "SPACEHOST_ACTOR_STORE_DIR", default_value = "")] + pub actor_store_dir: String, + /// This service's own per-account store directory, holding the space + /// tables. Separate from the PDS's actors directory, which is mounted + /// read-only in production and belongs to another writer. + #[arg(long, env = "SPACEHOST_SPACE_STORE_DIR", default_value = "")] + pub space_store_dir: String, + #[arg( + long, + env = "SPACEHOST_MINT_TOKEN", + default_value = "", + hide_env_values = true + )] + pub mint_token: String, + #[arg(long, env = "SPACEHOST_DAEMON_SERVICE_DID", default_value = "")] + pub daemon_service_did: String, + #[arg(long, env = "SPACEHOST_APPVIEW_SERVICE_DID", default_value = "")] + pub appview_service_did: String, } impl Config { @@ -92,28 +142,137 @@ impl Config { .collect() } + pub fn auth_config(&self) -> AuthConfig { + AuthConfig { + issuer: self.oauth_issuer.clone(), + jwks_uri: self.oauth_jwks_uri.clone(), + audience: self.oauth_audience.clone(), + client_ids: self + .oauth_client_ids + .split(',') + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(str::to_string) + .collect(), + hs256_secret: VerifyOnlyHs256Secret::new(self.oauth_hs256_secret.as_bytes().to_vec()), + } + } + + pub fn bootstrap_pin(&self) -> Option<(&str, &str)> { + (!self.authority_did.is_empty() && !self.signing_key_hex.is_empty()) + .then_some((self.authority_did.as_str(), self.signing_key_hex.as_str())) + } + pub fn validate(&self) -> Result<(), String> { + if self.authority_did.is_empty() != self.signing_key_hex.is_empty() { + return Err( + "SPACEHOST_AUTHORITY_DID and SPACEHOST_SIGNING_KEY_HEX must be set together (bootstrap pin) or both left unset" + .to_string(), + ); + } + if self.bootstrap_pin().is_none() && self.actor_store_dir.is_empty() { + return Err( + "no space authority available: set SPACEHOST_ACTOR_STORE_DIR (authorities register with actor-store keys) or pin one with SPACEHOST_AUTHORITY_DID + SPACEHOST_SIGNING_KEY_HEX" + .to_string(), + ); + } if self.policy == PolicyMode::ManagingApp && !self.managing_app.contains('#') { return Err( "managing-app policy requires SPACEHOST_MANAGING_APP (did#fragment)".to_string(), ); } + if self.policy == PolicyMode::ManagingApp + && (self.lifecycle_url.is_empty() || self.lifecycle_service_did.is_empty()) + { + return Err( + "managing-app policy requires SPACEHOST_LIFECYCLE_URL and SPACEHOST_LIFECYCLE_SERVICE_DID" + .to_string(), + ); + } if self.public_url.trim_end_matches('/').is_empty() { return Err("SPACEHOST_PUBLIC_URL must be an absolute origin".to_string()); } + self.auth_config().validate()?; + if self.actor_store_dir.is_empty() { + return Err("SPACEHOST_ACTOR_STORE_DIR is required".to_string()); + } + if self.space_store_dir.is_empty() { + return Err("SPACEHOST_SPACE_STORE_DIR is required".to_string()); + } + if same_directory(&self.actor_store_dir, &self.space_store_dir) { + return Err( + "SPACEHOST_SPACE_STORE_DIR must not be SPACEHOST_ACTOR_STORE_DIR: space tables are written to this service's own per-account stores, never into the PDS's actor files" + .to_string(), + ); + } + if self.mint_token.is_empty() + || self.daemon_service_did.is_empty() + || self.appview_service_did.is_empty() + { + return Err("SPACEHOST_MINT_TOKEN, SPACEHOST_DAEMON_SERVICE_DID, and SPACEHOST_APPVIEW_SERVICE_DID are required".to_string()); + } Ok(()) } + + /// The managing-app conversation is service auth, which is keyed on the + /// authority's `#atproto` key rather than the space key. A pinned authority + /// carries only the space key, so without an actor-store key for it the + /// policy can never mint a call the managing app will accept. Fail at boot + /// instead of at a member's first read. `has_service_key` answers whether + /// the actor store holds a signing key for that DID. + pub fn validate_pinned_service_key( + &self, + has_service_key: impl FnOnce(&str) -> bool, + ) -> Result<(), String> { + let Some((authority_did, _)) = self.bootstrap_pin() else { + return Ok(()); + }; + if self.policy != PolicyMode::ManagingApp { + return Ok(()); + } + if has_service_key(authority_did) { + return Ok(()); + } + Err(format!( + "managing-app policy needs an actor-store signing key for the pinned authority {authority_did}: SPACEHOST_ACTOR_STORE_DIR has none, so only the public and member-list policies are available to a pinned host" + )) + } +} + +/// Whether two settings name the same directory. Resolved paths are compared +/// when both exist, so `/pds/actors` and `/pds/../pds/actors` are also caught. +fn same_directory(left: &str, right: &str) -> bool { + if left.trim_end_matches('/') == right.trim_end_matches('/') { + return true; + } + match (std::fs::canonicalize(left), std::fs::canonicalize(right)) { + (Ok(left), Ok(right)) => left == right, + _ => false, + } } #[cfg(test)] mod tests { use super::*; + // `Config` falls back to env vars, and the env-var test below mutates + // process-global state, so every parse in this module is serialized. + static PARSE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn parse_lock() -> std::sync::MutexGuard<'static, ()> { + PARSE_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + // One sequential test: the env-var section mutates process-global state, // which would race sibling tests run in parallel. #[test] fn parses_args_env_and_requirements() { - assert!(Config::try_parse_from(["rsky-space-host"]).is_err()); + let _guard = parse_lock(); + let bare = Config::try_parse_from(["rsky-space-host"]).unwrap(); + assert!(bare.bootstrap_pin().is_none()); + assert!(bare.validate().is_err()); let cfg = Config::try_parse_from([ "rsky-space-host", @@ -121,6 +280,24 @@ mod tests { "did:plc:authority", "--signing-key-hex", "aa".repeat(32).as_str(), + "--oauth-issuer", + "https://pds.example", + "--oauth-jwks-uri", + "https://pds.example/jwks", + "--oauth-audience", + "did:web:pds.example", + "--oauth-client-ids", + "https://client.example", + "--actor-store-dir", + "/actors", + "--space-store-dir", + "/space-stores", + "--mint-token", + "token", + "--daemon-service-did", + "did:plc:daemon", + "--appview-service-did", + "did:plc:appview", ]) .unwrap(); assert_eq!(cfg.authority_did, "did:plc:authority"); @@ -135,7 +312,13 @@ mod tests { assert!(format!("{cfg:?}").contains("did:plc:authority")); let mut cfg = cfg; - cfg.update_from(["rsky-space-host", "--bind", "127.0.0.1:9"]); + cfg.update_from([ + "rsky-space-host", + "--bind", + "127.0.0.1:9", + "--authority-did", + "did:plc:authority", + ]); assert_eq!(cfg.bind, "127.0.0.1:9"); assert_eq!(cfg.authority_did, "did:plc:authority"); @@ -157,9 +340,20 @@ mod tests { std::env::set_var("SPACEHOST_MANAGING_APP", "did:web:app#svc"); std::env::set_var("SPACEHOST_MEMBERS", "did:plc:aaa, did:plc:bbb,"); std::env::set_var("SPACEHOST_MEMBERSHIP_DB_URL", "postgres://env"); + std::env::set_var("SPACEHOST_LIFECYCLE_URL", "https://feeds.example"); + std::env::set_var("SPACEHOST_LIFECYCLE_SERVICE_DID", "did:web:feeds.example"); std::env::set_var("SPACEHOST_DB_PATH", "/tmp/space.db"); std::env::set_var("SPACEHOST_PLC_URL", "https://plc.example"); std::env::set_var("SPACEHOST_BIND", "127.0.0.1:1234"); + std::env::set_var("SPACEHOST_OAUTH_ISSUER", "https://pds.example"); + std::env::set_var("SPACEHOST_OAUTH_JWKS_URI", "https://pds.example/jwks"); + std::env::set_var("SPACEHOST_OAUTH_AUDIENCE", "did:web:pds.example"); + std::env::set_var("SPACEHOST_OAUTH_CLIENT_IDS", "https://client.example"); + std::env::set_var("SPACEHOST_ACTOR_STORE_DIR", "/actors"); + std::env::set_var("SPACEHOST_SPACE_STORE_DIR", "/space-stores"); + std::env::set_var("SPACEHOST_MINT_TOKEN", "token"); + std::env::set_var("SPACEHOST_DAEMON_SERVICE_DID", "did:plc:daemon"); + std::env::set_var("SPACEHOST_APPVIEW_SERVICE_DID", "did:plc:appview"); let cfg = Config::try_parse_from(["rsky-space-host"]).unwrap(); for k in [ "SPACEHOST_AUTHORITY_DID", @@ -168,9 +362,20 @@ mod tests { "SPACEHOST_MANAGING_APP", "SPACEHOST_MEMBERS", "SPACEHOST_MEMBERSHIP_DB_URL", + "SPACEHOST_LIFECYCLE_URL", + "SPACEHOST_LIFECYCLE_SERVICE_DID", "SPACEHOST_DB_PATH", "SPACEHOST_PLC_URL", "SPACEHOST_BIND", + "SPACEHOST_OAUTH_ISSUER", + "SPACEHOST_OAUTH_JWKS_URI", + "SPACEHOST_OAUTH_AUDIENCE", + "SPACEHOST_OAUTH_CLIENT_IDS", + "SPACEHOST_ACTOR_STORE_DIR", + "SPACEHOST_SPACE_STORE_DIR", + "SPACEHOST_MINT_TOKEN", + "SPACEHOST_DAEMON_SERVICE_DID", + "SPACEHOST_APPVIEW_SERVICE_DID", ] { std::env::remove_var(k); } @@ -182,6 +387,8 @@ mod tests { vec!["did:plc:aaa".to_string(), "did:plc:bbb".to_string()] ); assert_eq!(cfg.db_path, "/tmp/space.db"); + assert_eq!(cfg.lifecycle_url, "https://feeds.example"); + assert_eq!(cfg.lifecycle_service_did, "did:web:feeds.example"); assert_eq!(cfg.plc_url, "https://plc.example"); assert_eq!(cfg.bind, "127.0.0.1:1234"); assert!(cfg.validate().is_ok()); @@ -191,4 +398,144 @@ mod tests { invalid.managing_app = String::new(); assert!(invalid.validate().is_err()); } + + fn valid_unpinned() -> Config { + let _guard = parse_lock(); + Config::try_parse_from([ + "rsky-space-host", + "--oauth-issuer", + "https://pds.example", + "--oauth-jwks-uri", + "https://pds.example/jwks", + "--oauth-audience", + "did:web:pds.example", + "--oauth-client-ids", + "https://client.example", + "--actor-store-dir", + "/actors", + "--space-store-dir", + "/space-stores", + "--mint-token", + "token", + "--daemon-service-did", + "did:plc:daemon", + "--appview-service-did", + "did:plc:appview", + ]) + .unwrap() + } + + #[test] + fn the_space_store_must_not_be_the_pds_actor_store() { + let mut cfg = valid_unpinned(); + assert!(cfg.validate().is_ok()); + + // The PDS's actor directory is a key source, never a write target: it + // belongs to another writer and is mounted read-only in production. + cfg.space_store_dir = cfg.actor_store_dir.clone(); + let message = cfg.validate().expect_err("must refuse"); + assert!(message.contains("SPACEHOST_SPACE_STORE_DIR"), "{message}"); + + // Trailing-slash and traversal spellings of the same directory too. + cfg.space_store_dir = "/actors/".to_string(); + assert!(cfg.validate().is_err()); + + let existing = tempfile::tempdir().unwrap(); + let mut resolved = valid_unpinned(); + resolved.actor_store_dir = existing.path().display().to_string(); + resolved.space_store_dir = existing + .path() + .join("..") + .join(existing.path().file_name().unwrap()) + .display() + .to_string(); + assert!(resolved.validate().is_err()); + + // And it is required at all: silently defaulting it to the actor store + // is how the two jobs got conflated in the first place. + let mut missing = valid_unpinned(); + missing.space_store_dir = String::new(); + assert!(missing.validate().is_err()); + } + + #[test] + fn pinned_managing_app_needs_an_actor_store_service_key() { + fn pinned_managing_app(policy: PolicyMode) -> Config { + let mut cfg = valid_unpinned(); + cfg.authority_did = "did:plc:authority".to_string(); + cfg.signing_key_hex = "aa".repeat(32); + cfg.policy = policy; + cfg.managing_app = "did:web:feeds.example#bsky_fg".to_string(); + cfg.lifecycle_url = "https://feeds.example".to_string(); + cfg.lifecycle_service_did = "did:web:feeds.example".to_string(); + cfg + } + + let pinned = pinned_managing_app(PolicyMode::ManagingApp); + assert!(pinned.validate().is_ok()); + + // No actor-store key for the pinned authority: refuse at boot. + let message = pinned + .validate_pinned_service_key(|_| false) + .expect_err("must refuse"); + assert!(message.contains("did:plc:authority"), "{message}"); + assert!(message.contains("member-list"), "{message}"); + + // With a key, the same config is accepted, and the probe sees the + // pinned authority rather than some other DID. + let mut asked = String::new(); + pinned + .validate_pinned_service_key(|did| { + asked = did.to_string(); + true + }) + .unwrap(); + assert_eq!(asked, "did:plc:authority"); + + // The other policies are unaffected — that is the point of the fallback. + pinned_managing_app(PolicyMode::MemberList) + .validate_pinned_service_key(|_| false) + .unwrap(); + pinned_managing_app(PolicyMode::Public) + .validate_pinned_service_key(|_| false) + .unwrap(); + + // An unpinned host resolves its authorities from the actor store, so + // there is nothing to reject. + let mut unpinned = valid_unpinned(); + unpinned.policy = PolicyMode::ManagingApp; + unpinned.validate_pinned_service_key(|_| false).unwrap(); + } + + #[test] + fn bootstrap_pin_is_optional_but_all_or_nothing() { + let cfg = valid_unpinned(); + assert!(cfg.bootstrap_pin().is_none()); + assert!(cfg.validate().is_ok()); + + let mut half = valid_unpinned(); + half.authority_did = "did:plc:authority".to_string(); + assert!(half.validate().is_err()); + + let mut half = valid_unpinned(); + half.signing_key_hex = "aa".repeat(32); + assert!(half.validate().is_err()); + + let mut pinned = valid_unpinned(); + pinned.authority_did = "did:plc:authority".to_string(); + pinned.signing_key_hex = "aa".repeat(32); + assert_eq!( + pinned.bootstrap_pin(), + Some(("did:plc:authority", pinned.signing_key_hex.as_str())) + ); + assert!(pinned.validate().is_ok()); + + let mut keyless = valid_unpinned(); + keyless.actor_store_dir = String::new(); + let message = keyless.validate().unwrap_err(); + assert!( + message.contains("no space authority available"), + "{message}" + ); + } } diff --git a/rsky-space-host/src/convert.rs b/rsky-space-host/src/convert.rs new file mode 100644 index 00000000..0dc001de --- /dev/null +++ b/rsky-space-host/src/convert.rs @@ -0,0 +1,190 @@ +//! One-off conversion of a deployed multi-tenant store into per-account +//! actor stores. +//! +//! The source keeps every repo in one file with `did` as a column; the +//! destination is one `store.sqlite` per author DID. Two things are carried +//! across verbatim rather than recomputed: the LtHash state, because it is the +//! commit digest readers have already seen, and the operation-log row ids, +//! because they are the cursors syncers hold — a syncer must resume across the +//! swap without replaying or skipping. +//! +//! The source has no per-record revision. A record's revision is taken from +//! the newest operation that left it at its current CID, falling back to the +//! repo's revision when the log no longer reaches back that far. + +use rsky_space::space_id::SpaceId; +use rusqlite::{params, Connection, OpenFlags, OptionalExtension}; +use std::collections::HashMap; +use std::path::Path; + +use crate::error::{HostError, Result}; + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Converted { + pub accounts: usize, + pub repos: usize, + pub records: usize, + pub ops: usize, +} + +pub fn convert(legacy: &Path, directory: &Path) -> Result { + let source = + Connection::open_with_flags(legacy, OpenFlags::SQLITE_OPEN_READ_ONLY).map_err(sql)?; + let mut totals = Converted::default(); + + for did in accounts(&source)? { + let path = crate::actor_repos::store_path(directory, &did)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| HostError::Store(error.to_string()))?; + } + let mut target = crate::actor_schema::get_migrated_db(&path)?; + let tx = target.transaction().map_err(sql)?; + for (space_uri, rev, state) in repos(&source, &did)? { + let space = SpaceId::parse(&space_uri)?; + if state.len() != 2048 { + return Err(HostError::Store(format!( + "lthash state for {space_uri}/{did} is not 2048 bytes" + ))); + } + tx.execute( + "INSERT INTO space_repo \ + (space_uri, authority, space_type, skey, rev, lthash_state, oplog_floor_rev, deleted, created_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, 0, ?7)", + params![ + space_uri, + space.authority, + space.space_type, + space.skey, + rev, + state, + rsky_common::now() + ], + ) + .map_err(sql)?; + + let mut revisions = HashMap::new(); + for (seq, op_rev, collection, rkey, cid, prev) in ops(&source, &space_uri, &did)? { + if let Some(ref cid) = cid { + revisions.insert( + (collection.clone(), rkey.clone(), cid.clone()), + op_rev.clone(), + ); + } + tx.execute( + "INSERT INTO space_oplog (id, space_uri, rev, collection, rkey, cid, prev) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![seq, space_uri, op_rev, collection, rkey, cid, prev], + ) + .map_err(sql)?; + totals.ops += 1; + } + + for (collection, rkey, cid, value) in records(&source, &space_uri, &did)? { + let record_rev = revisions + .get(&(collection.clone(), rkey.clone(), cid.clone())) + .unwrap_or(&rev); + tx.execute( + "INSERT INTO space_record (space_uri, collection, rkey, cid, rev, value) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![space_uri, collection, rkey, cid, record_rev, value], + ) + .map_err(sql)?; + totals.records += 1; + } + totals.repos += 1; + } + tx.commit().map_err(sql)?; + totals.accounts += 1; + } + Ok(totals) +} + +/// The oplog id a syncer holding `cursor` before the conversion resumes from +/// after it. Ids are preserved, so the cursor is unchanged — the lookup exists +/// to prove that, and fails loudly if a converted store ever renumbers. +pub fn resumes_at(store: &Path, space_uri: &str, cursor: i64) -> Result> { + let conn = Connection::open_with_flags(store, OpenFlags::SQLITE_OPEN_READ_ONLY).map_err(sql)?; + conn.query_row( + "SELECT MIN(id) FROM space_oplog WHERE space_uri = ?1 AND id > ?2", + params![space_uri, cursor], + |row| row.get(0), + ) + .optional() + .map(Option::flatten) + .map_err(sql) +} + +fn accounts(source: &Connection) -> Result> { + let mut statement = source + .prepare("SELECT DISTINCT did FROM repo ORDER BY did") + .map_err(sql)?; + let rows = statement + .query_map([], |row| row.get(0)) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +type LegacyRepo = (String, String, Vec); + +fn repos(source: &Connection, did: &str) -> Result> { + let mut statement = source + .prepare("SELECT space_uri, rev, state FROM repo WHERE did = ?1 ORDER BY space_uri") + .map_err(sql)?; + let rows = statement + .query_map([did], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +type LegacyRecord = (String, String, String, Vec); + +fn records(source: &Connection, space_uri: &str, did: &str) -> Result> { + let mut statement = source + .prepare( + "SELECT collection, rkey, cid, value FROM record \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY collection, rkey", + ) + .map_err(sql)?; + let rows = statement + .query_map(params![space_uri, did], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) + }) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +type LegacyOp = (i64, String, String, String, Option, Option); + +fn ops(source: &Connection, space_uri: &str, did: &str) -> Result> { + let mut statement = source + .prepare( + "SELECT seq, rev, collection, rkey, cid, prev FROM repo_op \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY seq", + ) + .map_err(sql)?; + let rows = statement + .query_map(params![space_uri, did], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + Ok(rows) +} + +fn sql(error: rusqlite::Error) -> HostError { + HostError::Store(error.to_string()) +} diff --git a/rsky-space-host/src/error.rs b/rsky-space-host/src/error.rs index 3a6b0f60..19ff256c 100644 --- a/rsky-space-host/src/error.rs +++ b/rsky-space-host/src/error.rs @@ -22,6 +22,24 @@ pub enum HostError { Resolution(String), #[error("store error: {0}")] Store(String), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("space not hosted here: {0}")] + SpaceNotFound(String), + #[error("account not hosted here: {0}")] + AccountNotHosted(String), + #[error("repo not found")] + RepoNotFound, + #[error("record already exists: {0}")] + RecordExists(String), + #[error("record not found: {0}")] + RecordNotFound(String), + #[error("swap cid did not match")] + InvalidSwap, + #[error("requested history is no longer available")] + HistoryUnavailable, + #[error("not implemented")] + Unimplemented, #[error(transparent)] Space(#[from] rsky_space::SpaceError), } diff --git a/rsky-space-host/src/http.rs b/rsky-space-host/src/http.rs index 9c19a76a..c5aab632 100644 --- a/rsky-space-host/src/http.rs +++ b/rsky-space-host/src/http.rs @@ -4,28 +4,37 @@ //! rsky-lexicon; errors are XRPC-shaped `{error, message}` JSON. use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::http::header::{AUTHORIZATION, CONTENT_TYPE}; +use axum::http::{HeaderMap, HeaderName, Method, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use rsky_lexicon::com::atproto::space::{ - GetSpaceCredentialInput, GetSpaceCredentialOutput, GetSpaceOutput, GetSpaceParams, + GetLatestCommitOutput, GetLatestCommitParams, GetRepoParams, GetSpaceCredentialInput, + GetSpaceCredentialOutput, GetSpaceOutput, GetSpaceParams, ListRepoOpsOutput, ListRepoOpsParams, ListReposOutput, ListReposParams, NotifyWriteInput, RegisterNotifyInput, RegisterNotifyOutput, SpaceConfig, }; use rsky_oauth::dpop::{DpopManager, DpopProof, DpopRequest}; use rsky_space::credential; +use serde_json::Value; +use std::collections::BTreeMap; use std::sync::Arc; +use std::time::Duration; +use tower_http::cors::{Any, CorsLayer}; use crate::attestation::{JtiStore, MetadataFetcher}; -use crate::authority::{Authority, KeyResolver}; +use crate::authority::{AuthorityContext, AuthorityFactory, AuthorityRegistry, KeyResolver}; +use crate::commits::CommitSigner; use crate::error::HostError; use crate::keys::DocSource; use crate::managing_app::require_https; -use crate::notify::{fan_out_write, Notifier, NOTIFY_WRITE_LXM}; -use crate::policy::Policy; +use crate::notify::{fan_out_write, NOTIFY_WRITE_LXM}; +use crate::oauth::{verify_access, AuthConfig, RequestAuth}; +use crate::registration::REGISTER_SPACE_LXM; +use crate::repo::{RepoStore, RepoWrite, WriteOutcome, MAX_RECORD_BYTES}; use crate::service_jwt; -use crate::store::{RegistrationStore, Subscriber, WriterSetStore}; +use crate::store::{HostedSpaceStore, RegistrationStore, Subscriber, WriterSetStore}; pub const DEFAULT_REGISTRATION_TTL_SECS: u64 = 24 * 60 * 60; const DEFAULT_LIST_LIMIT: i64 = 100; @@ -33,8 +42,10 @@ const MAX_LIST_LIMIT: i64 = 1000; #[derive(Clone)] pub struct AppState { - pub authority: Arc, - pub policy: Arc, + pub registry: Arc, + /// Builds the context for an authority first seen at registration time. + pub authority_factory: AuthorityFactory, + pub hosted_spaces: Arc, pub keys: Arc, pub metadata: Arc, pub jti_store: Arc, @@ -42,7 +53,6 @@ pub struct AppState { pub registrations: Arc, /// Resolves a subscriber's service identifier to its delivery endpoint. pub docs: Arc, - pub notifier: Arc, /// Verifies DPoP proofs on the credential-issuance and credential-presenting /// paths. Space issuance does not challenge with nonces, so this manager /// carries none; the replay store is what makes a proof single-use. @@ -52,6 +62,12 @@ pub struct AppState { pub now: Arc u64 + Send + Sync>, pub jti: Arc String + Send + Sync>, pub registration_ttl_secs: u64, + pub repos: Arc, + pub commit_signer: Arc, + pub auth: AuthConfig, + pub rev: Arc String + Send + Sync>, + pub mint_token: String, + pub credential_mint_services: [String; 2], } pub fn router(state: AppState) -> Router { @@ -63,12 +79,38 @@ pub fn router(state: AppState) -> Router { post(get_space_credential), ) .route("/xrpc/com.atproto.space.listRepos", get(list_repos)) + .route("/xrpc/com.atproto.space.getRepo", get(get_repo)) + .route("/xrpc/com.atproto.space.listRepoOps", get(list_repo_ops)) + .route( + "/xrpc/com.atproto.space.getLatestCommit", + get(get_latest_commit), + ) .route( "/xrpc/com.atproto.space.registerNotify", post(register_notify), ) .route("/xrpc/com.atproto.space.notifyWrite", post(notify_write)) + .route( + "/xrpc/community.blacksky.space.register", + post(register_space), + ) + .route("/xrpc/com.atproto.space.createRecord", post(create_record)) + .route("/xrpc/com.atproto.space.deleteRecord", post(delete_record)) + .route("/admin/mintCredential", post(mint_credential)) .with_state(state) + .layer( + CorsLayer::new() + .allow_origin(Any) + .allow_methods([Method::GET, Method::POST]) + .allow_headers([ + HeaderName::from_static("atproto-accept-labelers"), + HeaderName::from_static("atproto-proxy"), + AUTHORIZATION, + CONTENT_TYPE, + HeaderName::from_static("dpop"), + ]) + .max_age(Duration::from_secs(24 * 60 * 60)), + ) } /// An XRPC-shaped error response: `{error, message}` with a matching status. @@ -95,6 +137,10 @@ impl ApiError { fn auth_required(message: impl Into) -> Self { Self::new(StatusCode::UNAUTHORIZED, "AuthenticationRequired", message) } + + fn forbidden(message: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, "Forbidden", message) + } } impl IntoResponse for ApiError { @@ -128,11 +174,44 @@ impl From for ApiError { "ClientNotAuthorized", "client not authorized for space", ), + HostError::SpaceNotFound(_) => Self::new( + StatusCode::NOT_FOUND, + "SpaceNotFound", + "space not hosted here", + ), + HostError::RepoNotFound => { + Self::new(StatusCode::NOT_FOUND, "RepoNotFound", "repo not found") + } + HostError::AccountNotHosted(_) => Self::new( + StatusCode::NOT_FOUND, + "RepoNotFound", + "repo not hosted here", + ), + HostError::InvalidRequest(message) => Self::invalid_request(message.clone()), + // These four names and statuses are what rsky-pds answers with, so + // a client cannot tell the two write paths apart. + HostError::RecordExists(message) => { + Self::new(StatusCode::BAD_REQUEST, "RecordExists", message.clone()) + } + HostError::RecordNotFound(message) => { + Self::new(StatusCode::BAD_REQUEST, "RecordNotFound", message.clone()) + } + HostError::InvalidSwap => Self::new( + StatusCode::BAD_REQUEST, + "InvalidSwap", + "swap cid did not match", + ), + HostError::HistoryUnavailable => Self::new( + StatusCode::BAD_REQUEST, + "HistoryUnavailable", + "requested history is no longer available", + ), HostError::Key(_) | HostError::Membership(_) | HostError::ManagingApp(_) | HostError::Resolution(_) | HostError::Store(_) + | HostError::Unimplemented | HostError::Space(_) => { tracing::error!(error = %e, "internal error"); Self::new( @@ -172,6 +251,16 @@ fn check_proof( access_token: Option<&str>, ) -> Result { let uri = format!("{}/xrpc/{nsid}", state.public_url.trim_end_matches('/')); + check_proof_uri(state, headers, method, &uri, access_token) +} + +fn check_proof_uri( + state: &AppState, + headers: &HeaderMap, + method: &str, + uri: &str, + access_token: Option<&str>, +) -> Result { let proofs: Vec<&str> = headers .get_all("dpop") .iter() @@ -198,6 +287,63 @@ fn check_proof( }) } +#[derive(serde::Deserialize)] +struct MintCredentialParams { + space: String, +} + +async fn mint_credential( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result, ApiError> { + let supplied = headers + .get("x-spacehost-mint-token") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + if !same_secret(supplied, &state.mint_token) { + return Err(ApiError::forbidden("invalid mint token")); + } + let jwt = bearer(&headers)?; + let claims = service_jwt::claims(jwt)?; + if !state + .credential_mint_services + .iter() + .any(|did| did == &claims.iss) + { + return Err(ApiError::forbidden( + "service is not allowed to mint credentials", + )); + } + let (context, space) = require_this_space(&state, ¶ms.space)?; + let key = state.keys.service_key(&claims.iss).await?; + service_jwt::verify( + jwt, + &[context.authority_did()], + "community.blacksky.space.mintCredential", + &key, + (state.now)(), + )?; + let uri = format!( + "{}/admin/mintCredential", + state.public_url.trim_end_matches('/') + ); + let proof = check_proof_uri(&state, &headers, "POST", &uri, None)?; + let credential = + context + .authority + .mint_credential_for(&space, (state.now)(), (state.jti)(), &proof.jkt)?; + Ok(Json(serde_json::json!({"credential": credential}))) +} + +fn same_secret(left: &str, right: &str) -> bool { + let mut diff = left.len() ^ right.len(); + for (a, b) in left.bytes().zip(right.bytes()) { + diff |= usize::from(a ^ b); + } + diff == 0 +} + /// Space-credential auth: verify the presented credential against this /// authority's own space key, then confirm the presenter holds the key it is /// bound to. @@ -211,13 +357,15 @@ fn require_space_credential( headers: &HeaderMap, method: &str, nsid: &str, -) -> Result<(), ApiError> { + space_uri: &str, +) -> Result<(Arc, rsky_space::space_id::SpaceId), ApiError> { + let (context, space) = require_this_space(state, space_uri)?; let jwt = dpop_credential(headers)?; let bound_jkt = credential::verify_space_credential( jwt, - &state.authority.space_uri(), - state.authority.authority_did(), - state.authority.signer.did_key(), + &space.uri(), + context.authority_did(), + context.authority.signer.did_key(), (state.now)(), ) .map_err(|e| ApiError::new(StatusCode::UNAUTHORIZED, "InvalidToken", e.to_string()))?; @@ -229,7 +377,7 @@ fn require_space_credential( "DPoP key thumbprint does not match the credential binding", )); } - Ok(()) + Ok((context, space)) } /// Resolve a `did:...#fragment` subscriber to its delivery endpoint. The @@ -258,13 +406,19 @@ async fn resolve_service_endpoint(docs: &dyn DocSource, service: &str) -> Result }) } -fn require_this_space(state: &AppState, space: &str) -> Result<(), ApiError> { - if space != state.authority.space_uri() { - return Err(ApiError::invalid_request(format!( - "space not hosted here: {space}" - ))); - } - Ok(()) +fn require_this_space( + state: &AppState, + space: &str, +) -> Result<(Arc, rsky_space::space_id::SpaceId), ApiError> { + let context = state + .registry + .for_space(space) + .map_err(|_| ApiError::invalid_request(format!("space not hosted here: {space}")))?; + let space = context + .authority + .resolve(space) + .map_err(|_| ApiError::invalid_request(format!("space not hosted here: {space}")))?; + Ok((context, space)) } async fn health() -> Json { @@ -276,20 +430,46 @@ async fn get_space( headers: HeaderMap, Query(params): Query, ) -> Result, ApiError> { - require_space_credential(&state, &headers, "GET", "com.atproto.space.getSpace")?; - require_this_space(&state, ¶ms.space)?; + let (context, space) = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.getSpace", + ¶ms.space, + )?; Ok(Json(GetSpaceOutput { - space: state.authority.space_uri(), - config: SpaceConfig::Simplespace(state.authority.space_config(&state.policy)), + space: space.uri(), + config: SpaceConfig::Simplespace(context.authority.space_config(&context.policy)), })) } +async fn require_service_auth( + state: &AppState, + context: &AuthorityContext, + headers: &HeaderMap, + expected_lxm: &str, +) -> Result { + let jwt = bearer(headers)?; + let claims = service_jwt::claims(jwt)?; + let issuer_key = state.keys.service_key(&claims.iss).await?; + let authority_did = context.authority_did(); + let space_host_aud = format!("{authority_did}#atproto_space_host"); + service_jwt::verify( + jwt, + &[authority_did, space_host_aud.as_str()], + expected_lxm, + &issuer_key, + (state.now)(), + ) + .map_err(|e| ApiError::new(StatusCode::UNAUTHORIZED, "InvalidToken", e.to_string())) +} + async fn get_space_credential( State(state): State, headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - require_this_space(&state, &input.space)?; + let (context, space) = require_this_space(&state, &input.space)?; let delegation_token = bearer(&headers)?; // Before the delegation token, so a caller with a bad proof does not burn // its single-use grant finding out. @@ -300,12 +480,13 @@ async fn get_space_credential( "com.atproto.space.getSpaceCredential", None, )?; - let credential = state + let credential = context .authority - .get_space_credential( + .get_space_credential_for( + &space, delegation_token, input.client_attestation.as_deref(), - &state.policy, + &context.policy, state.keys.as_ref(), state.metadata.as_ref(), state.jti_store.as_ref(), @@ -322,8 +503,13 @@ async fn list_repos( headers: HeaderMap, Query(params): Query, ) -> Result, ApiError> { - require_space_credential(&state, &headers, "GET", "com.atproto.space.listRepos")?; - require_this_space(&state, ¶ms.space)?; + require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.listRepos", + ¶ms.space, + )?; let limit = params .limit .unwrap_or(DEFAULT_LIST_LIMIT) @@ -335,13 +521,174 @@ async fn list_repos( Ok(Json(ListReposOutput { cursor, repos })) } +async fn signed_head( + state: &AppState, + space: &rsky_space::space_id::SpaceId, + repo: &str, +) -> Result { + let head = state + .repos + .head(&space.uri(), repo) + .await? + .ok_or(HostError::RepoNotFound)?; + crate::commits::mint_commit( + state.commit_signer.as_ref(), + &space.uri(), + repo, + &head.rev, + &head.hash(), + rand::random(), + ) + .map_err(Into::into) +} + +async fn get_latest_commit( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result, ApiError> { + let (_, space) = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.getLatestCommit", + ¶ms.space, + )?; + Ok(Json(GetLatestCommitOutput { + commit: signed_head(&state, &space, ¶ms.repo).await?, + })) +} + +async fn list_repo_ops( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result, ApiError> { + let (_, space) = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.listRepoOps", + ¶ms.space, + )?; + let limit = params + .limit + .unwrap_or(DEFAULT_LIST_LIMIT) + .clamp(1, MAX_LIST_LIMIT) as u32; + let page = state + .repos + .list_ops( + &space.uri(), + ¶ms.repo, + params.since.as_deref(), + params.cursor.as_deref(), + limit, + ) + .await?; + let commit = if page.complete { + Some(signed_head(&state, &space, ¶ms.repo).await?) + } else { + None + }; + let mut ops = Vec::with_capacity(page.ops.len()); + for op in page.ops { + let value = if params.exclude_values.unwrap_or(false) || op.cid.is_none() { + None + } else { + // Only the op that still holds the current record carries a value: + // a superseded op's cid no longer names what is stored. + state + .repos + .get_record(&space.uri(), ¶ms.repo, &op.collection, &op.rkey) + .await? + .filter(|record| Some(&record.cid) == op.cid.as_ref()) + .map(|record| rsky_space::record::decode_record(&record.value)) + .transpose() + .map_err(HostError::from)? + }; + ops.push(rsky_lexicon::com::atproto::space::RepoOp { + rev: op.rev, + collection: op.collection, + rkey: op.rkey, + cid: op.cid, + prev: op.prev, + value, + }); + } + Ok(Json(ListRepoOpsOutput { + cursor: page.cursor, + ops, + commit, + })) +} + +async fn get_repo( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result { + let (_, space) = require_space_credential( + &state, + &headers, + "GET", + "com.atproto.space.getRepo", + ¶ms.space, + )?; + let commit = signed_head(&state, &space, ¶ms.repo).await?; + let records = state + .repos + .list_records( + &space.uri(), + ¶ms.repo, + None, + None, + MAX_LIST_LIMIT as u32, + ) + .await? + .0; + let mut entries = BTreeMap::new(); + let mut blocks = BTreeMap::new(); + for record in records { + let cid: lexicon_cid::Cid = record.cid.parse().map_err(|_| { + ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "InternalError", + "stored record has invalid cid", + ) + })?; + entries.insert(record.path(), cid); + blocks.insert(cid, record.value); + } + let internal = rsky_space::types::SignedCommit { + ver: commit.ver as u8, + hash: commit.hash.into(), + ikm: commit.ikm.into(), + sig: commit.sig.into(), + mac: commit.mac.into(), + rev: commit.rev, + }; + let car = rsky_space::repo_car_bytes(&internal, &entries, |cid| blocks.get(cid).cloned()) + .await + .map_err(HostError::from)?; + Ok(( + [(axum::http::header::CONTENT_TYPE, "application/vnd.ipld.car")], + car, + ) + .into_response()) +} + async fn register_notify( State(state): State, headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - require_space_credential(&state, &headers, "POST", "com.atproto.space.registerNotify")?; - require_this_space(&state, &input.space)?; + require_space_credential( + &state, + &headers, + "POST", + "com.atproto.space.registerNotify", + &input.space, + )?; // `service` names the subscriber, which is both where to deliver and who // the delivery is addressed to (proposals#100); `endpoint` is the // pre-amendment shape and loses when both are sent. @@ -374,12 +721,252 @@ async fn register_notify( })) } +#[derive(serde::Deserialize)] +struct RegisterSpaceInput { + space: String, + generation: i64, +} + +async fn register_space( + State(state): State, + headers: HeaderMap, + Json(input): Json, +) -> Result, ApiError> { + if input.generation < 1 { + return Err(ApiError::invalid_request("generation must be positive")); + } + let (context, adopted) = match state.registry.for_space(&input.space) { + Ok(context) => (context, false), + Err(_) => { + let space = rsky_space::space_id::SpaceId::parse(&input.space).map_err(|_| { + ApiError::invalid_request(format!("invalid space uri: {}", input.space)) + })?; + let context = (state.authority_factory)(&space).map_err(|e| match e { + HostError::AccountNotHosted(did) => ApiError::invalid_request(format!( + "authority signing key does not resolve: {did}" + )), + HostError::SpaceNotFound(space) => { + ApiError::invalid_request(format!("space not hosted here: {space}")) + } + other => ApiError::from(other), + })?; + (context, true) + } + }; + let claims = require_service_auth(&state, &context, &headers, REGISTER_SPACE_LXM).await?; + let expected_issuer = context + .policy + .managing_app() + .and_then(|service| service.split_once('#').map(|(did, _)| did)) + .ok_or_else(|| ApiError::forbidden("space has no managing app"))?; + if claims.iss != expected_issuer { + return Err(ApiError::forbidden("issuer is not the managing app")); + } + let acker = context.lifecycle_acker.clone().ok_or_else(|| { + ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "LifecycleUnavailable", + "lifecycle acknowledgement is not configured", + ) + })?; + context.authority.register(&input.space)?; + let context = if adopted { + let winner = state.registry.insert_if_absent(context.clone()); + if !Arc::ptr_eq(&winner, &context) { + winner.authority.register(&input.space)?; + } + winner + } else { + context + }; + state + .hosted_spaces + .record_space(context.authority_did(), &input.space) + .await?; + acker + .ack_host_registered(&input.space, input.generation) + .await?; + Ok(Json(serde_json::json!({}))) +} + +async fn write_actor( + state: &AppState, + headers: &HeaderMap, + path: &str, + space: &str, + repo: &str, +) -> Result<(Arc, rsky_space::space_id::SpaceId), ApiError> { + let (context, space) = require_this_space(state, space)?; + let url = format!("{}{}", state.public_url.trim_end_matches('/'), path); + let access = verify_access( + &RequestAuth { + authorization: headers.get("authorization").and_then(|v| v.to_str().ok()), + dpop: headers.get("dpop").and_then(|v| v.to_str().ok()), + method: "POST", + url: &url, + }, + &state.auth, + state.metadata.as_ref(), + state.jti_store.as_ref(), + (state.now)(), + ) + .await?; + if access.did != repo { + return Err(ApiError::auth_required( + "session subject does not match repo", + )); + } + Ok((context, space)) +} + +async fn create_record( + State(state): State, + headers: HeaderMap, + Json(input): Json, +) -> Result, ApiError> { + let (context, space) = write_actor( + &state, + &headers, + "/xrpc/com.atproto.space.createRecord", + &input.space, + &input.repo, + ) + .await?; + if contains_blob_ref(&input.record) { + return Err(ApiError::invalid_request( + "blob references are not supported", + )); + } + let value = rsky_space::record::encode_record(&input.record, MAX_RECORD_BYTES) + .map_err(HostError::from)?; + let rev = (state.rev)(); + let rkey = input.rkey.unwrap_or_else(|| rev.clone()); + let applied = state + .repos + .apply_writes( + &space.uri(), + &input.repo, + &rev, + &[RepoWrite::Create { + collection: input.collection.clone(), + rkey: rkey.clone(), + value, + }], + ) + .await?; + let cid = match &applied.outcomes[0] { + WriteOutcome::Created { cid } => cid.clone(), + _ => return Err(HostError::Store("create did not create".into()).into()), + }; + record_write( + &state, + &context, + &space.uri(), + &input.repo, + &applied.rev, + Some(hex::encode(applied.hash)), + ) + .await?; + Ok(Json( + rsky_lexicon::com::atproto::space::CreateRecordOutput { + uri: space.record_uri(&input.repo, &input.collection, &rkey), + cid, + commit: Some(rsky_lexicon::com::atproto::space::CommitMeta { + rev: applied.rev, + hash: hex::encode(applied.hash), + }), + }, + )) +} + +pub fn contains_blob_ref(value: &Value) -> bool { + match value { + Value::Array(values) => values.iter().any(contains_blob_ref), + Value::Object(values) => { + values.get("$type").and_then(Value::as_str) == Some("blob") + || values.values().any(contains_blob_ref) + } + _ => false, + } +} + +async fn delete_record( + State(state): State, + headers: HeaderMap, + Json(input): Json, +) -> Result, ApiError> { + let (context, space) = write_actor( + &state, + &headers, + "/xrpc/com.atproto.space.deleteRecord", + &input.space, + &input.repo, + ) + .await?; + let applied = state + .repos + .apply_writes( + &space.uri(), + &input.repo, + &(state.rev)(), + &[RepoWrite::Delete { + collection: input.collection, + rkey: input.rkey, + swap_record: input.swap_record, + }], + ) + .await?; + record_write( + &state, + &context, + &space.uri(), + &input.repo, + &applied.rev, + Some(hex::encode(applied.hash)), + ) + .await?; + Ok(Json( + rsky_lexicon::com::atproto::space::DeleteRecordOutput { + commit: Some(rsky_lexicon::com::atproto::space::CommitMeta { + rev: applied.rev, + hash: hex::encode(applied.hash), + }), + }, + )) +} + +async fn record_write( + state: &AppState, + context: &AuthorityContext, + space: &str, + repo: &str, + rev: &str, + hash: Option, +) -> Result<(), ApiError> { + let now = (state.now)(); + state + .writers + .upsert_writer(space, repo, rev, hash.as_deref(), now) + .await?; + let endpoints = state.registrations.endpoints(space, now).await?; + fan_out_write( + context.notifier.clone(), + endpoints, + NotifyWriteInput { + space: space.to_string(), + repo: repo.to_string(), + rev: rev.to_string(), + }, + ); + Ok(()) +} + async fn notify_write( State(state): State, headers: HeaderMap, Json(input): Json, ) -> Result, ApiError> { - require_this_space(&state, &input.space)?; + let (context, _) = require_this_space(&state, &input.space)?; let jwt = bearer(&headers)?; let claims = service_jwt::claims(jwt)?; // The repo host signs with the member's own key, so a notification may only @@ -391,8 +978,8 @@ async fn notify_write( "issuer does not match notified repo", )); } - let issuer_key = state.keys.signing_key(&claims.iss).await?; - let authority_did = state.authority.authority_did(); + let issuer_key = state.keys.service_key(&claims.iss).await?; + let authority_did = context.authority_did(); let space_host_aud = format!("{authority_did}#atproto_space_host"); service_jwt::verify( jwt, @@ -408,7 +995,7 @@ async fn notify_write( .upsert_writer(&input.space, &input.repo, &input.rev, None, now) .await?; let endpoints = state.registrations.endpoints(&input.space, now).await?; - fan_out_write(state.notifier.clone(), endpoints, input); + fan_out_write(context.notifier.clone(), endpoints, input); Ok(Json(serde_json::json!({}))) } @@ -417,8 +1004,12 @@ mod tests { use super::*; use crate::appaccess::AppAccess; use crate::attestation::{ClientMetadata, InMemoryJtiStore}; + use crate::authority::Authority; use crate::error::Result as HostResult; use crate::membership::InMemoryMembership; + use crate::notify::Notifier; + use crate::policy::Policy; + use crate::registration::LifecycleAcker; use crate::signing::{test_signer, Signer}; use crate::store::{InMemoryRegistrations, InMemoryWriterSet}; use async_trait::async_trait; @@ -446,7 +1037,7 @@ mod tests { struct UserKeys; #[async_trait] impl KeyResolver for UserKeys { - async fn signing_key(&self, did: &str) -> HostResult { + async fn service_key(&self, did: &str) -> HostResult { if did == MEMBER { Ok(user_signer().did_key().to_string()) } else { @@ -527,6 +1118,30 @@ mod tests { writes: tokio::sync::mpsc::UnboundedReceiver<(String, NotifyWriteInput)>, } + #[derive(Default)] + struct RecordingAcker(std::sync::Mutex>); + + #[async_trait] + impl LifecycleAcker for RecordingAcker { + async fn ack_host_registered(&self, space: &str, generation: i64) -> HostResult<()> { + self.0.lock().unwrap().push((space.to_string(), generation)); + Ok(()) + } + } + + struct UnusedManagingApp; + + #[async_trait] + impl crate::managing_app::ManagingAppClient for UnusedManagingApp { + async fn check_user_access(&self, _: &str, _: &str, _: Option<&str>) -> HostResult { + Ok(false) + } + } + + fn ctx(state: &AppState) -> Arc { + state.registry.for_space(&space_uri()).unwrap() + } + fn fixture(app_access: AppAccess, members: &[&str]) -> Fixture { let space = SpaceId::new( "did:plc:communityauthority", @@ -535,18 +1150,27 @@ mod tests { ); let authority = Authority::new(space, test_signer(), app_access); let (tx, writes) = tokio::sync::mpsc::unbounded_channel(); - let state = AppState { + let registry = Arc::new(AuthorityRegistry::new()); + registry.insert(Arc::new(AuthorityContext { authority: Arc::new(authority), policy: Arc::new(Policy::MemberList(Arc::new(InMemoryMembership::new( members.iter().map(|m| m.to_string()), )))), + notifier: Arc::new(RecordingNotifier { tx }), + lifecycle_acker: None, + })); + let state = AppState { + registry, + authority_factory: Arc::new(|space: &SpaceId| { + Err(HostError::AccountNotHosted(space.authority.clone())) + }), + hosted_spaces: Arc::new(crate::store::InMemoryHostedSpaces::default()), keys: Arc::new(UserKeys), - metadata: Arc::new(NoFetch), + metadata: Arc::new(crate::oauth::tests::AsJwks), jti_store: Arc::new(InMemoryJtiStore::default()), writers: Arc::new(InMemoryWriterSet::default()), registrations: Arc::new(InMemoryRegistrations::default()), docs: Arc::new(NoDocs), - notifier: Arc::new(RecordingNotifier { tx }), dpop: Arc::new(rsky_oauth::dpop::DpopManager::new( None, Box::new(rsky_oauth::dpop::InMemoryReplayStore::default()), @@ -555,6 +1179,12 @@ mod tests { now: Arc::new(|| NOW), jti: Arc::new(|| "jti-fixed".to_string()), registration_ttl_secs: DEFAULT_REGISTRATION_TTL_SECS, + repos: Arc::new(crate::repo::InMemoryRepos::default()), + commit_signer: Arc::new(test_signer()), + auth: crate::oauth::tests::config(), + rev: Arc::new(|| "3jzfcijpj2z2c".to_string()), + mint_token: "test-mint-token".to_string(), + credential_mint_services: ["did:plc:daemon".to_string(), "did:plc:appview".to_string()], }; Fixture { state, writes } } @@ -598,7 +1228,7 @@ mod tests { } fn credential_for(state: &AppState) -> String { - state + ctx(state) .authority .mint_credential(NOW, "cred-jti".to_string(), &dpop_key().thumbprint()) .unwrap() @@ -610,13 +1240,11 @@ mod tests { alg: rsky_crypto::constants::SECP256K1_JWT_ALG.to_string(), kid: Some("#atproto".to_string()), }; + let context = ctx(state); let claims = SpaceClaims { iss: user.to_string(), - sub: state.authority.space_uri(), - aud: Some(format!( - "{}#atproto_space_host", - state.authority.authority_did() - )), + sub: context.authority.space_uri(), + aud: Some(format!("{}#atproto_space_host", context.authority_did())), iat: NOW, exp: NOW + 60, jti: "delegation-jti".to_string(), @@ -685,6 +1313,283 @@ mod tests { .unwrap() } + fn pds_write_req(path: &str, body: serde_json::Value) -> Request { + let token = crate::oauth::tests::token(); + let mut claims = crate::oauth::tests::proof_claims(&token); + claims["htu"] = serde_json::json!(format!("{PUBLIC_URL}{path}")); + claims["jti"] = serde_json::json!(format!( + "write-proof-{}", + PROOF_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + )); + Request::builder() + .method("POST") + .uri(path) + .header("content-type", "application/json") + .header("authorization", format!("DPoP {token}")) + .header("dpop", crate::oauth::tests::proof_with(claims)) + .body(Body::from(body.to_string())) + .unwrap() + } + + #[tokio::test] + async fn cors_preflight_allows_the_staging_space_write() { + let f = fixture(AppAccess::Open, &[]); + let response = router(f.state) + .oneshot( + Request::builder() + .method("OPTIONS") + .uri("/xrpc/com.atproto.space.createRecord") + .header("origin", "https://staging.blacksky.community") + .header("access-control-request-method", "POST") + .header( + "access-control-request-headers", + "atproto-accept-labelers, atproto-proxy, authorization, content-type, dpop", + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert!(response.status().is_success()); + assert_eq!( + response + .headers() + .get("access-control-allow-origin") + .unwrap(), + "*" + ); + let methods = response + .headers() + .get("access-control-allow-methods") + .unwrap() + .to_str() + .unwrap(); + assert!(methods.split(',').any(|method| method.trim() == "POST")); + let headers = response + .headers() + .get("access-control-allow-headers") + .unwrap() + .to_str() + .unwrap() + .to_ascii_lowercase(); + for header in [ + "atproto-accept-labelers", + "atproto-proxy", + "authorization", + "content-type", + "dpop", + ] { + assert!(headers.split(',').any(|value| value.trim() == header)); + } + } + + #[tokio::test] + async fn create_and_delete_records_verify_the_pds_session() { + let f = fixture(AppAccess::Open, &[]); + let mut state = f.state.clone(); + state.now = Arc::new(|| crate::oauth::tests::NOW); + let create_path = "/xrpc/com.atproto.space.createRecord"; + let create = serde_json::json!({ + "space": space_uri(), + "repo": MEMBER, + "collection": "app.bsky.feed.post", + "rkey": "3jzfcijpj2z2c", + "record": {"$type": "app.bsky.feed.post", "text": "private", "createdAt": "2026-08-19T00:00:00Z"}, + }); + let (status, out) = send(&state, pds_write_req(create_path, create)).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + out["uri"], + format!( + "{}/did:plc:member/app.bsky.feed.post/3jzfcijpj2z2c", + space_uri() + ) + ); + assert!(out["cid"].as_str().is_some()); + assert_eq!(out["commit"]["rev"], "3jzfcijpj2z2c"); + + let stored = f + .state + .repos + .get_record(&space_uri(), MEMBER, "app.bsky.feed.post", "3jzfcijpj2z2c") + .await + .unwrap(); + assert!(stored.is_some()); + + let delete = serde_json::json!({ + "space": space_uri(), + "repo": MEMBER, + "collection": "app.bsky.feed.post", + "rkey": "3jzfcijpj2z2c", + }); + let (status, out) = send( + &state, + pds_write_req("/xrpc/com.atproto.space.deleteRecord", delete), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(out["commit"]["rev"], "3jzfcijpj2z2c"); + assert!(f + .state + .repos + .get_record(&space_uri(), MEMBER, "app.bsky.feed.post", "3jzfcijpj2z2c") + .await + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn create_record_rejects_blobs_and_subject_mismatch() { + let f = fixture(AppAccess::Open, &[]); + let mut state = f.state.clone(); + state.now = Arc::new(|| crate::oauth::tests::NOW); + let blob = serde_json::json!({ + "space": space_uri(), "repo": MEMBER, "collection": "app.bsky.feed.post", "rkey": "x", + "record": {"$type": "app.bsky.feed.post", "embed": {"image": {"$type": "blob", "ref": {"$link": "bafk"}}}}, + }); + let (status, out) = send( + &state, + pds_write_req("/xrpc/com.atproto.space.createRecord", blob), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(out["error"], "InvalidRequest"); + + let mismatch = serde_json::json!({ + "space": space_uri(), "repo": "did:plc:other", "collection": "app.bsky.feed.post", "rkey": "x", "record": {"$type": "app.bsky.feed.post"}, + }); + let (status, out) = send( + &state, + pds_write_req("/xrpc/com.atproto.space.createRecord", mismatch), + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(out["error"], "AuthenticationRequired"); + } + + #[tokio::test] + async fn repo_reads_require_credentials_and_serve_sync_shapes() { + let f = fixture(AppAccess::Open, &[]); + let value = rsky_space::record::encode_record( + &serde_json::json!({"$type":"app.bsky.feed.post","text":"sync"}), + MAX_RECORD_BYTES, + ) + .unwrap(); + f.state + .repos + .apply_writes( + &space_uri(), + MEMBER, + "3jzfcijpj2z2c", + &[RepoWrite::Create { + collection: "app.bsky.feed.post".to_string(), + rkey: "3jzfcijpj2z2c".to_string(), + value, + }], + ) + .await + .unwrap(); + let credential = credential_for(&f.state); + let query = format!( + "space={}&repo={}", + urlencode(&space_uri()), + urlencode(MEMBER) + ); + + let (status, body) = send( + &f.state, + get_req( + &format!("/xrpc/com.atproto.space.getLatestCommit?{query}"), + None, + ), + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(body["error"], "AuthenticationRequired"); + + let (status, body) = send( + &f.state, + get_req( + &format!("/xrpc/com.atproto.space.listRepoOps?{query}"), + Some(&credential), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["ops"].as_array().unwrap().len(), 1); + assert!(body["commit"].is_object()); + + let (status, body) = send( + &f.state, + get_req( + &format!("/xrpc/com.atproto.space.getLatestCommit?{query}"), + Some(&credential), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["commit"]["rev"], "3jzfcijpj2z2c"); + + let response = router(f.state.clone()) + .oneshot(get_req( + &format!("/xrpc/com.atproto.space.getRepo?{query}"), + Some(&credential), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[axum::http::header::CONTENT_TYPE], + "application/vnd.ipld.car" + ); + assert!(!response + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .is_empty()); + } + + #[tokio::test] + async fn shared_read_surface_rejects_wrong_key_credentials() { + let f = fixture(AppAccess::Open, &[]); + let value = rsky_space::record::encode_record( + &serde_json::json!({"$type":"app.bsky.feed.post"}), + MAX_RECORD_BYTES, + ) + .unwrap(); + f.state + .repos + .apply_writes( + &space_uri(), + MEMBER, + "3jzfcijpj2z2c", + &[RepoWrite::Create { + collection: "app.bsky.feed.post".to_string(), + rkey: "3jzfcijpj2z2c".to_string(), + value, + }], + ) + .await + .unwrap(); + let path = format!( + "/xrpc/com.atproto.space.getLatestCommit?space={}&repo={}", + urlencode(&space_uri()), + urlencode(MEMBER) + ); + + let wrong_key = ctx(&f.state) + .authority + .mint_credential(NOW, "wrong-key".to_string(), "another-thumbprint") + .unwrap(); + let (status, body) = send(&f.state, get_req(&path, Some(&wrong_key))).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(body["error"], "InvalidDpopProof"); + assert!(body.get("message").and_then(Value::as_str).is_some()); + } + #[tokio::test] async fn health_reports_version() { let f = fixture(AppAccess::Open, &[]); @@ -729,8 +1634,7 @@ mod tests { assert_eq!(body["error"], "AuthenticationRequired"); // A credential bound to someone else's key is refused as well. - let other_binding = f - .state + let other_binding = ctx(&f.state) .authority .mint_credential(NOW, "other-jti".to_string(), "some-other-thumbprint") .unwrap(); @@ -768,8 +1672,8 @@ mod tests { credential::verify_space_credential( out["credential"].as_str().unwrap(), &space_uri(), - f.state.authority.authority_did(), - f.state.authority.signer.did_key(), + ctx(&f.state).authority_did(), + ctx(&f.state).authority.signer.did_key(), NOW, ) .unwrap(); @@ -957,7 +1861,7 @@ mod tests { ) .await .unwrap(); - let aud = format!("{}#atproto_space_host", f.state.authority.authority_did()); + let aud = format!("{}#atproto_space_host", ctx(&f.state).authority_did()); let token = member_service_jwt(&aud, NOTIFY_WRITE_LXM); let path = "/xrpc/com.atproto.space.notifyWrite"; let body = serde_json::json!({ @@ -988,7 +1892,7 @@ mod tests { async fn notify_write_auth_failures() { let f = fixture(AppAccess::Open, &[]); let path = "/xrpc/com.atproto.space.notifyWrite"; - let authority_did = f.state.authority.authority_did().to_string(); + let authority_did = ctx(&f.state).authority_did().to_string(); let body = serde_json::json!({ "space": space_uri(), "repo": MEMBER, @@ -1056,7 +1960,7 @@ mod tests { let mut broken = f.state.clone(); broken.writers = Arc::new(BrokenWriters); let token = member_service_jwt( - &format!("{}#atproto_space_host", broken.authority.authority_did()), + &format!("{}#atproto_space_host", ctx(&broken).authority_did()), NOTIFY_WRITE_LXM, ); let body = serde_json::json!({ @@ -1073,6 +1977,239 @@ mod tests { assert_eq!(out["error"], "InternalError"); } + #[tokio::test] + async fn registration_authenticates_the_managing_app_and_acks_before_activation() { + let f = fixture(AppAccess::Open, &[]); + let acker = Arc::new(RecordingAcker::default()); + let existing = ctx(&f.state); + f.state.registry.insert(Arc::new(AuthorityContext { + authority: existing.authority.clone(), + policy: Arc::new(Policy::ManagingApp { + service_id: format!("{MEMBER}#bsky_fg"), + client: Arc::new(UnusedManagingApp), + }), + notifier: existing.notifier.clone(), + lifecycle_acker: Some(acker.clone()), + })); + let space = "at://did:plc:communityauthority/space/community.blacksky.feed/new"; + let audience = format!("{}#atproto_space_host", ctx(&f.state).authority_did()); + let token = service_jwt::mint( + &user_signer(), + MEMBER, + &audience, + REGISTER_SPACE_LXM, + NOW, + "register-1".to_string(), + ) + .unwrap(); + let (status, body) = send( + &f.state, + post_req( + &format!("/xrpc/{REGISTER_SPACE_LXM}"), + Some(&token), + serde_json::json!({"space": space, "generation": 7}), + ), + ) + .await; + + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(ctx(&f.state).authority.resolve_registered(space).is_ok()); + assert_eq!( + acker.0.lock().unwrap().as_slice(), + &[(space.to_string(), 7)] + ); + } + + #[tokio::test] + async fn each_authority_mints_and_verifies_with_its_own_key() { + let f = fixture(AppAccess::Open, &[]); + let other_space_uri = "at://did:plc:otherauthority/space/community.blacksky.feed/main"; + let other_signer = + Signer::from_secret(secp256k1::SecretKey::from_slice(&[0x66u8; 32]).unwrap()); + assert_ne!(other_signer.did_key(), test_signer().did_key()); + let (tx, _writes) = tokio::sync::mpsc::unbounded_channel(); + f.state.registry.insert(Arc::new(AuthorityContext { + authority: Arc::new(Authority::new( + SpaceId::parse(other_space_uri).unwrap(), + other_signer, + AppAccess::Open, + )), + policy: Arc::new(Policy::Public), + notifier: Arc::new(RecordingNotifier { tx }), + lifecycle_acker: None, + })); + let other = f.state.registry.for_space(other_space_uri).unwrap(); + let path = |space: &str| { + format!( + "/xrpc/com.atproto.space.getSpace?space={}", + urlencode(space) + ) + }; + + // Each authority's credential serves its own space. + let own = ctx(&f.state) + .authority + .mint_credential(NOW, "cred-a".to_string(), &dpop_key().thumbprint()) + .unwrap(); + let (status, body) = send(&f.state, get_req(&path(&space_uri()), Some(&own))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["space"], space_uri()); + + let other_cred = other + .authority + .mint_credential(NOW, "cred-b".to_string(), &dpop_key().thumbprint()) + .unwrap(); + let (status, body) = + send(&f.state, get_req(&path(other_space_uri), Some(&other_cred))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["space"], other_space_uri); + assert_eq!(body["config"]["policy"], "public"); + + // A credential signed with authority A's key never opens authority B's + // space, even with matching claims. + let cross = ctx(&f.state) + .authority + .mint_credential_for( + &SpaceId::parse(other_space_uri).unwrap(), + NOW, + "cred-cross".to_string(), + &dpop_key().thumbprint(), + ) + .unwrap(); + let (status, body) = send(&f.state, get_req(&path(other_space_uri), Some(&cross))).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(body["error"], "InvalidToken"); + } + + const DYNAMIC_AUTHORITY: &str = "did:plc:dynauthority"; + const DYNAMIC_AUTHORITY_KEY: [u8; 32] = [0x33u8; 32]; + + fn dynamic_actor_store() -> tempfile::TempDir { + use sha2::Digest; + let directory = tempfile::tempdir().unwrap(); + let digest = hex::encode(sha2::Sha256::digest(DYNAMIC_AUTHORITY.as_bytes())); + let actor = directory.path().join(&digest[..2]).join(DYNAMIC_AUTHORITY); + std::fs::create_dir_all(&actor).unwrap(); + std::fs::write(actor.join("key"), DYNAMIC_AUTHORITY_KEY).unwrap(); + std::fs::write(actor.join("store.sqlite"), []).unwrap(); + directory + } + + fn seam_factory( + seam: Arc, + acker: Arc, + ) -> crate::authority::AuthorityFactory { + Arc::new(move |space: &SpaceId| { + let signer = seam.signer(&space.authority)?; + let (tx, _writes) = tokio::sync::mpsc::unbounded_channel(); + Ok(Arc::new(AuthorityContext { + authority: Arc::new(Authority::new(space.clone(), signer, AppAccess::Open)), + policy: Arc::new(Policy::ManagingApp { + service_id: format!("{MEMBER}#bsky_fg"), + client: Arc::new(UnusedManagingApp), + }), + notifier: Arc::new(RecordingNotifier { tx }), + lifecycle_acker: Some(acker.clone()), + })) + }) + } + + #[tokio::test] + async fn registration_creates_an_unknown_authority_from_the_actor_store() { + let directory = dynamic_actor_store(); + let seam = Arc::new(crate::pds_seam::PdsSeam::open(directory.path()).unwrap()); + let acker = Arc::new(RecordingAcker::default()); + let mut f = fixture(AppAccess::Open, &[]); + f.state.authority_factory = seam_factory(seam, acker.clone()); + let space = format!("at://{DYNAMIC_AUTHORITY}/space/community.blacksky.feed/main"); + let token = service_jwt::mint( + &user_signer(), + MEMBER, + &format!("{DYNAMIC_AUTHORITY}#atproto_space_host"), + REGISTER_SPACE_LXM, + NOW, + "register-dyn".to_string(), + ) + .unwrap(); + + let (status, body) = send( + &f.state, + post_req( + &format!("/xrpc/{REGISTER_SPACE_LXM}"), + Some(&token), + serde_json::json!({"space": space, "generation": 1}), + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let context = f.state.registry.authority(DYNAMIC_AUTHORITY).unwrap(); + assert!(context.authority.resolve_registered(&space).is_ok()); + // The new authority signs with the key resolved from the actor store. + let expected = + Signer::from_secret(secp256k1::SecretKey::from_slice(&DYNAMIC_AUTHORITY_KEY).unwrap()); + assert_eq!(context.authority.signer.did_key(), expected.did_key()); + let credential = context + .authority + .mint_credential(NOW, "dyn-jti".to_string(), "jkt") + .unwrap(); + credential::verify_space_credential( + &credential, + &space, + DYNAMIC_AUTHORITY, + expected.did_key(), + NOW, + ) + .unwrap(); + assert_eq!(acker.0.lock().unwrap().as_slice(), &[(space.clone(), 1)]); + assert_eq!( + f.state.hosted_spaces.hosted_spaces().await.unwrap(), + vec![(DYNAMIC_AUTHORITY.to_string(), space)] + ); + } + + #[tokio::test] + async fn registration_for_an_unresolvable_authority_fails_clean() { + let directory = dynamic_actor_store(); + let seam = Arc::new(crate::pds_seam::PdsSeam::open(directory.path()).unwrap()); + let mut f = fixture(AppAccess::Open, &[]); + f.state.authority_factory = seam_factory(seam, Arc::new(RecordingAcker::default())); + let space = "at://did:plc:keyless/space/community.blacksky.feed/main"; + let token = service_jwt::mint( + &user_signer(), + MEMBER, + "did:plc:keyless#atproto_space_host", + REGISTER_SPACE_LXM, + NOW, + "register-keyless".to_string(), + ) + .unwrap(); + + let (status, body) = send( + &f.state, + post_req( + &format!("/xrpc/{REGISTER_SPACE_LXM}"), + Some(&token), + serde_json::json!({"space": space, "generation": 1}), + ), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], "InvalidRequest"); + assert!(body["message"] + .as_str() + .unwrap() + .contains("did:plc:keyless")); + assert!(f.state.registry.authority("did:plc:keyless").is_err()); + assert!(f + .state + .hosted_spaces + .hosted_spaces() + .await + .unwrap() + .is_empty()); + } + #[tokio::test] async fn error_mapping_covers_every_lexicon_error_name() { for (err, status, name) in [ diff --git a/rsky-space-host/src/keys.rs b/rsky-space-host/src/keys.rs index 5e7bd40c..39843086 100644 --- a/rsky-space-host/src/keys.rs +++ b/rsky-space-host/src/keys.rs @@ -46,14 +46,26 @@ fn fragment_of(id: &str) -> Option<&str> { id.rsplit_once('#').map(|(_, frag)| frag) } -/// The `did:key` for a doc's signing key: prefer `#atproto_space`, fall back to -/// `#atproto`. -pub fn signing_did_key_from_doc(doc: &DidDocument) -> Result { +/// The `did:key` that verifies a space authority's **credentials**: prefer +/// `#atproto_space`, fall back to `#atproto` (spec §Space authority). +pub fn credential_did_key_from_doc(doc: &DidDocument) -> Result { + did_key_by_fragment(doc, &["atproto_space", "atproto"]) +} + +/// The `did:key` that verifies **service auth and delegation tokens**, which are +/// keyed on the account's signing key alone. A delegation token's `kid` MUST be +/// `#atproto` (spec §Delegation token), and inter-service auth uses the same +/// key, so `#atproto_space` is not accepted here even when published. +pub fn service_did_key_from_doc(doc: &DidDocument) -> Result { + did_key_by_fragment(doc, &["atproto"]) +} + +fn did_key_by_fragment(doc: &DidDocument, fragments: &[&str]) -> Result { let methods = doc .verification_method .as_deref() .ok_or_else(|| HostError::Resolution(format!("{}: no verification methods", doc.id)))?; - let method = ["atproto_space", "atproto"] + let method = fragments .iter() .find_map(|frag| methods.iter().find(|m| fragment_of(&m.id) == Some(frag))) .ok_or_else(|| HostError::Resolution(format!("{}: no atproto signing key", doc.id)))?; @@ -83,7 +95,7 @@ pub fn service_endpoint_from_doc(doc: &DidDocument, fragment: &str) -> Result, } @@ -96,9 +108,9 @@ impl DocKeyResolver { #[async_trait] impl KeyResolver for DocKeyResolver { - async fn signing_key(&self, did: &str) -> Result { + async fn service_key(&self, did: &str) -> Result { let doc = self.docs.did_document(did).await?; - signing_did_key_from_doc(&doc) + service_did_key_from_doc(&doc) } } @@ -143,24 +155,42 @@ mod tests { } #[test] - fn prefers_atproto_space_over_atproto() { + fn credential_resolution_prefers_atproto_space_over_atproto() { let (space_m, space_key) = multikey_method("did:plc:subject#atproto_space", [0x41; 32]); let (atp_m, atp_key) = multikey_method("#atproto", [0x42; 32]); let d = doc(vec![atp_m.clone(), space_m], vec![]); - assert_eq!(signing_did_key_from_doc(&d).unwrap(), space_key); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), space_key); let d = doc(vec![atp_m], vec![]); - assert_eq!(signing_did_key_from_doc(&d).unwrap(), atp_key); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), atp_key); + } + + #[test] + fn service_resolution_ignores_atproto_space_when_both_are_published() { + let (space_m, space_key) = multikey_method("did:plc:subject#atproto_space", [0x41; 32]); + let (atp_m, atp_key) = multikey_method("#atproto", [0x42; 32]); + assert_ne!(space_key, atp_key); + + // An authority publishes both. Service auth and delegation tokens are + // keyed on #atproto, so the space key must not be selected. + let d = doc(vec![space_m.clone(), atp_m], vec![]); + assert_eq!(service_did_key_from_doc(&d).unwrap(), atp_key); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), space_key); + + // A doc with only the space key has no service key at all. + let d = doc(vec![space_m], vec![]); + assert!(service_did_key_from_doc(&d).is_err()); + assert_eq!(credential_did_key_from_doc(&d).unwrap(), space_key); } #[test] fn missing_or_unusable_keys_are_errors() { assert!(matches!( - signing_did_key_from_doc(&doc(vec![], vec![])), + credential_did_key_from_doc(&doc(vec![], vec![])), Err(HostError::Resolution(_)) )); let (other, _) = multikey_method("#unrelated", [0x43; 32]); - assert!(signing_did_key_from_doc(&doc(vec![other], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![other], vec![])).is_err()); let no_multibase = VerificationMethod { id: "#atproto".to_string(), @@ -168,7 +198,7 @@ mod tests { controller: "did:plc:subject".to_string(), public_key_multibase: None, }; - assert!(signing_did_key_from_doc(&doc(vec![no_multibase], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![no_multibase], vec![])).is_err()); let bad_multibase = VerificationMethod { id: "#atproto".to_string(), @@ -176,7 +206,7 @@ mod tests { controller: "did:plc:subject".to_string(), public_key_multibase: Some("!!!".to_string()), }; - assert!(signing_did_key_from_doc(&doc(vec![bad_multibase], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![bad_multibase], vec![])).is_err()); let unknown_type = VerificationMethod { id: "#atproto".to_string(), @@ -184,7 +214,7 @@ mod tests { controller: "did:plc:subject".to_string(), public_key_multibase: Some("zunknown".to_string()), }; - assert!(signing_did_key_from_doc(&doc(vec![unknown_type], vec![])).is_err()); + assert!(credential_did_key_from_doc(&doc(vec![unknown_type], vec![])).is_err()); } #[test] @@ -215,7 +245,7 @@ mod tests { async fn doc_key_resolver_resolves_signing_key() { let (m, key) = multikey_method("#atproto", [0x44; 32]); let resolver = DocKeyResolver::new(Arc::new(FixedDoc(doc(vec![m], vec![])))); - assert_eq!(resolver.signing_key("did:plc:subject").await.unwrap(), key); + assert_eq!(resolver.service_key("did:plc:subject").await.unwrap(), key); } #[tokio::test] @@ -250,7 +280,7 @@ mod tests { did_cache: Arc::new(MemoryCache::new(None, None)), })); let got = source.did_document(&did).await.unwrap(); - assert_eq!(signing_did_key_from_doc(&got).unwrap(), key); + assert_eq!(credential_did_key_from_doc(&got).unwrap(), key); // Unresolvable DIDs surface as resolution errors. let missing = source.did_document("did:web:localhost%3A1").await; diff --git a/rsky-space-host/src/lib.rs b/rsky-space-host/src/lib.rs index 499d54f7..1557b9f4 100644 --- a/rsky-space-host/src/lib.rs +++ b/rsky-space-host/src/lib.rs @@ -18,21 +18,30 @@ //! `com.atproto.space.*` DTOs from rsky-lexicon, backed by in-memory or SQLite //! [stores](store). +pub mod actor_repos; +pub mod actor_schema; pub mod appaccess; pub mod attestation; pub mod authority; +pub mod client_jws; +pub mod commits; pub mod config; +pub mod convert; pub mod error; pub mod http; pub mod keys; pub mod managing_app; pub mod membership; pub mod notify; +pub mod oauth; +pub mod pds_seam; pub mod policy; +pub mod registration; +pub mod repo; pub mod service_jwt; pub mod signing; pub mod store; -pub use authority::{Authority, KeyResolver}; +pub use authority::{Authority, AuthorityContext, AuthorityRegistry, KeyResolver}; pub use error::{HostError, Result}; pub use policy::Policy; diff --git a/rsky-space-host/src/main.rs b/rsky-space-host/src/main.rs index 11192e65..42da1b8f 100644 --- a/rsky-space-host/src/main.rs +++ b/rsky-space-host/src/main.rs @@ -1,4 +1,4 @@ -//! Space-host service entrypoint: parse config, wire the authority, policy, +//! Space-host service entrypoint: parse config, wire the authority registry, //! stores, and HTTP surface, and serve until shutdown. use clap::Parser; @@ -6,18 +6,24 @@ use rsky_identity::did::did_resolver::DidResolver; use rsky_identity::types::{DidResolverOpts, MemoryCache}; use rsky_oauth::dpop::{DpopManager, InMemoryReplayStore}; use rsky_space::space_id::SpaceId; +use rsky_space_host::actor_repos::ActorStoreRepos; use rsky_space_host::appaccess::AppAccess; use rsky_space_host::attestation::HttpMetadataFetcher; -use rsky_space_host::authority::Authority; +use rsky_space_host::authority::{ + Authority, AuthorityContext, AuthorityFactory, AuthorityRegistry, +}; use rsky_space_host::config::{Config, PolicyMode}; use rsky_space_host::http::{router, AppState, DEFAULT_REGISTRATION_TTL_SECS}; -use rsky_space_host::keys::{DocKeyResolver, ResolverDocSource}; +use rsky_space_host::keys::{DocKeyResolver, DocSource, ResolverDocSource}; use rsky_space_host::managing_app::HttpManagingApp; use rsky_space_host::membership::InMemoryMembership; use rsky_space_host::notify::HttpNotifier; +use rsky_space_host::pds_seam::{PdsSeam, PdsServiceJwtIssuer}; use rsky_space_host::policy::Policy; +use rsky_space_host::registration::{HttpLifecycleAcker, LifecycleAcker}; +use rsky_space_host::service_jwt::ServiceJwtIssuer; use rsky_space_host::signing::Signer; -use rsky_space_host::store::SqliteStore; +use rsky_space_host::store::{HostedSpaceStore, SqliteStore}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; @@ -37,6 +43,67 @@ async fn shutdown_signal() { tracing::info!("shutdown signal received"); } +/// Builds one authority's context from the shared host configuration. +struct ContextBuilder { + policy: PolicyMode, + managing_app: String, + members: Vec, + lifecycle_url: String, + lifecycle_service_did: String, + seam: Arc, + docs: Arc, + now: Arc u64 + Send + Sync>, + jti: Arc String + Send + Sync>, +} + +impl ContextBuilder { + fn context(&self, space: SpaceId, signer: Signer) -> AuthorityContext { + let authority_did = space.authority.clone(); + // Service auth to the managing app is keyed on the authority's + // `#atproto` key, not the space key, so it is minted from the actor + // store rather than from the configured space signer. + let service_issuer: Arc = Arc::new(PdsServiceJwtIssuer::new( + self.seam.clone(), + authority_did.clone(), + )); + let policy = match self.policy { + PolicyMode::Public => Policy::Public, + PolicyMode::MemberList => { + Policy::MemberList(Arc::new(InMemoryMembership::new(self.members.clone()))) + } + PolicyMode::ManagingApp => Policy::ManagingApp { + service_id: self.managing_app.clone(), + client: Arc::new(HttpManagingApp::with_issuer( + self.managing_app.clone(), + service_issuer.clone(), + self.docs.clone(), + self.now.clone(), + self.jti.clone(), + )), + }, + }; + AuthorityContext { + authority: Arc::new(Authority::new(space, signer.clone(), AppAccess::Open)), + policy: Arc::new(policy), + notifier: Arc::new(HttpNotifier::new( + authority_did.clone(), + signer.clone(), + self.now.clone(), + self.jti.clone(), + )), + lifecycle_acker: (self.policy == PolicyMode::ManagingApp).then(|| { + Arc::new(HttpLifecycleAcker::with_issuer( + self.lifecycle_url.clone(), + self.lifecycle_service_did.clone(), + service_issuer, + self.now.clone(), + self.jti.clone(), + )) as Arc + }), + } + } +} + #[tokio::main] async fn main() -> Result<(), Box> { tracing_subscriber::fmt() @@ -47,13 +114,6 @@ async fn main() -> Result<(), Box> { let cfg = Config::parse(); cfg.validate()?; - let signer = Signer::from_hex(&cfg.signing_key_hex)?; - let space = SpaceId::new( - cfg.authority_did.clone(), - cfg.space_type().to_string(), - cfg.space_skey().to_string(), - ); - let authority = Authority::new(space, signer.clone(), AppAccess::Open); let now: Arc u64 + Send + Sync> = Arc::new(unix_now); let jti: Arc String + Send + Sync> = Arc::new(random_jti); @@ -62,39 +122,74 @@ async fn main() -> Result<(), Box> { plc_url: Some(cfg.plc_url.clone()), did_cache: std::sync::Arc::new(MemoryCache::new(None, None)), }))); - let policy = match cfg.policy { - PolicyMode::Public => Policy::Public, - PolicyMode::MemberList => { - Policy::MemberList(Arc::new(InMemoryMembership::new(cfg.member_dids()))) - } - PolicyMode::ManagingApp => Policy::ManagingApp { - service_id: cfg.managing_app.clone(), - client: Arc::new(HttpManagingApp::new( - cfg.managing_app.clone(), - cfg.authority_did.clone(), - signer.clone(), - docs.clone(), - now.clone(), - jti.clone(), - )), - }, - }; let store = Arc::new(SqliteStore::open(&cfg.db_path)?); + let repos = Arc::new(ActorStoreRepos::open(&cfg.space_store_dir)?); + let seam = Arc::new(PdsSeam::open(&cfg.actor_store_dir)?); + cfg.validate_pinned_service_key(|did| seam.key_path(did).is_some_and(|path| path.exists()))?; + + let builder = Arc::new(ContextBuilder { + policy: cfg.policy, + managing_app: cfg.managing_app.clone(), + members: cfg.member_dids(), + lifecycle_url: cfg.lifecycle_url.clone(), + lifecycle_service_did: cfg.lifecycle_service_did.clone(), + seam: seam.clone(), + docs: docs.clone(), + now: now.clone(), + jti: jti.clone(), + }); + let registry = Arc::new(AuthorityRegistry::new()); + if let Some((authority_did, signing_key_hex)) = cfg.bootstrap_pin() { + let signer = Signer::from_hex(signing_key_hex)?; + let space = SpaceId::new( + authority_did.to_string(), + cfg.space_type().to_string(), + cfg.space_skey().to_string(), + ); + registry.insert(Arc::new(builder.context(space, signer))); + } + let factory: AuthorityFactory = { + let builder = builder.clone(); + let seam = seam.clone(); + Arc::new(move |space: &SpaceId| { + let signer = seam.signer(&space.authority)?; + Ok(Arc::new(builder.context(space.clone(), signer))) + }) + }; + for (authority_did, space_uri) in store.hosted_spaces().await? { + let context = match registry.authority(&authority_did) { + Ok(context) => context, + Err(_) => { + let built = SpaceId::parse(&space_uri) + .map_err(|e| e.to_string()) + .and_then(|space| factory(&space).map_err(|e| e.to_string())); + match built { + Ok(context) => registry.insert_if_absent(context), + Err(error) => { + tracing::warn!(authority = %authority_did, space = %space_uri, error = %error, + "cannot re-serve persisted space"); + continue; + } + } + } + }; + if let Err(error) = context.authority.register(&space_uri) { + tracing::warn!(space = %space_uri, error = %error, + "cannot re-register persisted space"); + } + } + + let ticker = std::sync::Mutex::new(rsky_common::tid::Ticker::new()); let state = AppState { - authority: Arc::new(authority), - policy: Arc::new(policy), + registry: registry.clone(), + authority_factory: factory, + hosted_spaces: store.clone(), keys: Arc::new(DocKeyResolver::new(docs.clone())), docs, metadata: Arc::new(HttpMetadataFetcher::new()), jti_store: store.clone(), writers: store.clone(), registrations: store, - notifier: Arc::new(HttpNotifier::new( - cfg.authority_did.clone(), - signer, - now.clone(), - jti.clone(), - )), dpop: Arc::new(DpopManager::new( None, Box::new(InMemoryReplayStore::default()), @@ -103,12 +198,21 @@ async fn main() -> Result<(), Box> { now, jti, registration_ttl_secs: DEFAULT_REGISTRATION_TTL_SECS, + repos, + commit_signer: seam, + auth: cfg.auth_config(), + rev: Arc::new(move || ticker.lock().expect("ticker").next(None).to_string()), + mint_token: cfg.mint_token.clone(), + credential_mint_services: [ + cfg.daemon_service_did.clone(), + cfg.appview_service_did.clone(), + ], }; let listener = tokio::net::TcpListener::bind(&cfg.bind).await?; tracing::info!( - space = %state.authority.space_uri(), - authority_key = %state.authority.signer.did_key(), + authorities = registry.contexts().len(), + bootstrap = %cfg.authority_did, policy = ?cfg.policy, bind = %cfg.bind, db = %cfg.db_path, diff --git a/rsky-space-host/src/managing_app.rs b/rsky-space-host/src/managing_app.rs index eeba7234..8c8f4d7d 100644 --- a/rsky-space-host/src/managing_app.rs +++ b/rsky-space-host/src/managing_app.rs @@ -12,7 +12,7 @@ use std::time::Duration; use crate::error::{HostError, Result}; use crate::keys::{service_endpoint_from_doc, DocSource}; -use crate::service_jwt; +use crate::service_jwt::{ServiceJwtIssuer, SignerIssuer}; use crate::signing::Signer; pub const CHECK_USER_ACCESS_LXM: &str = "com.atproto.simplespace.checkUserAccess"; @@ -55,8 +55,7 @@ pub(crate) fn require_https(url: &str) -> Result<()> { /// its DID document and calls `checkUserAccess` with authority service auth. pub struct HttpManagingApp { service_id: String, - authority_did: String, - signer: Signer, + issuer: Arc, docs: Arc, http: reqwest::Client, now: Arc u64 + Send + Sync>, @@ -92,6 +91,34 @@ impl HttpManagingApp { now: Arc u64 + Send + Sync>, jti: Arc String + Send + Sync>, timeout: Duration, + ) -> Self { + Self::with_issuer_and_timeout( + service_id, + Arc::new(SignerIssuer::new(authority_did, signer)), + docs, + now, + jti, + timeout, + ) + } + + pub fn with_issuer( + service_id: String, + issuer: Arc, + docs: Arc, + now: Arc u64 + Send + Sync>, + jti: Arc String + Send + Sync>, + ) -> Self { + Self::with_issuer_and_timeout(service_id, issuer, docs, now, jti, DEFAULT_TIMEOUT) + } + + pub fn with_issuer_and_timeout( + service_id: String, + issuer: Arc, + docs: Arc, + now: Arc u64 + Send + Sync>, + jti: Arc String + Send + Sync>, + timeout: Duration, ) -> Self { let http = reqwest::Client::builder() .timeout(timeout) @@ -99,8 +126,7 @@ impl HttpManagingApp { .expect("reqwest client"); Self { service_id, - authority_did, - signer, + issuer, docs, http, now, @@ -126,9 +152,7 @@ impl ManagingAppClient for HttpManagingApp { client_id: Option<&str>, ) -> Result { let endpoint = self.endpoint().await?; - let token = service_jwt::mint( - &self.signer, - &self.authority_did, + let token = self.issuer.mint( &self.service_id, CHECK_USER_ACCESS_LXM, (self.now)(), @@ -267,6 +291,64 @@ mod tests { assert_eq!(payload["jti"], "jti-fixed"); } + #[tokio::test] + async fn check_user_access_is_signed_with_the_authority_account_key() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"allowed": true})), + ) + .mount(&server) + .await; + + const AUTHORITY: &str = "did:plc:auth"; + let secret = [9u8; 32]; + let directory = crate::pds_seam::test_actor_store(AUTHORITY, secret); + let seam = Arc::new(crate::pds_seam::PdsSeam::open(directory.path()).unwrap()); + let app = HttpManagingApp::with_issuer( + format!("{APP_DID}#managing_app"), + Arc::new(crate::pds_seam::PdsServiceJwtIssuer::new( + seam, + AUTHORITY.to_string(), + )), + Arc::new(AppDoc(server.uri())), + Arc::new(|| 1000), + Arc::new(|| "jti-fixed".to_string()), + ); + assert!(app + .check_user_access(SPACE, "did:plc:member", None) + .await + .unwrap()); + + let requests = server.received_requests().await.unwrap(); + let auth = requests[0].headers.get("authorization").unwrap(); + let jwt = auth.to_str().unwrap().strip_prefix("Bearer ").unwrap(); + + // The account key — what a managing app resolves as `#atproto`. + let account_key = + crate::signing::Signer::from_secret(secp256k1::SecretKey::from_slice(&secret).unwrap()); + let claims = crate::service_jwt::verify( + jwt, + &[&format!("{APP_DID}#managing_app")], + CHECK_USER_ACCESS_LXM, + account_key.did_key(), + 1000, + ) + .unwrap(); + assert_eq!(claims.iss, AUTHORITY); + + // The space key must NOT verify it: that mismatch is what a managing app + // rejects with a 401. + assert!(crate::service_jwt::verify( + jwt, + &[&format!("{APP_DID}#managing_app")], + CHECK_USER_ACCESS_LXM, + test_signer().did_key(), + 1000, + ) + .is_err()); + } + #[tokio::test] async fn deny_path_returns_false_without_client_id() { let server = MockServer::start().await; diff --git a/rsky-space-host/src/oauth.rs b/rsky-space-host/src/oauth.rs new file mode 100644 index 00000000..c7062107 --- /dev/null +++ b/rsky-space-host/src/oauth.rs @@ -0,0 +1,1119 @@ +//! Write-path authentication: DPoP-bound OAuth access tokens (D24). +//! +//! Space writes arrive with the same access token the account's PDS issued for +//! ordinary XRPC, so this verifies a token it did not mint: +//! +//! - the authorization server's signature over the token, against its JWKS; +//! - `iss` is the trusted authorization server; +//! - `aud` is the PDS service DID — a `did:` string, never an origin URL; +//! - the header `typ` is `at+jwt` and `exp` has not passed; +//! - `cnf.jkt` matches the thumbprint of the presented DPoP proof's key; +//! - `sub` is the account the caller claims to be writing as; +//! - `client_id` is on the first-party allowlist. +//! +//! Tokens carry no `scope` claim, so no scope check is possible and +//! `client_id` stands in for one. A token revoked before it expires still +//! verifies until `exp`. + +use crate::client_jws::{verify_client_es256, verify_client_es256k}; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use rsky_space::jwk::{EcJwk, JwkSet}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use crate::attestation::{JtiStore, MetadataFetcher, MAX_IAT_SKEW_SECS}; +use crate::error::{HostError, Result}; +use crate::pds_seam::VerifyOnlyHs256Secret; + +pub const ACCESS_TOKEN_TYP: &str = "at+jwt"; +pub const DPOP_TYP: &str = "dpop+jwt"; +pub const SUPPORTED_ALG: &str = "ES256"; +pub const HS256: &str = "HS256"; +/// What an access token may be signed with. `HS256` is not a weakening: it is +/// what a standalone PDS actually issues (see `verify_as_signature`). +pub const SUPPORTED_TOKEN_ALGS: [&str; 2] = ["ES256", "HS256"]; +pub const ES256K: &str = "ES256K"; +pub const SUPPORTED_PROOF_ALGS: [&str; 2] = ["ES256", ES256K]; + +/// How long a DPoP proof stays acceptable after its `iat`. +pub const MAX_DPOP_AGE_SECS: u64 = 300; + +/// The trust anchors the shim checks a token against. +#[derive(Debug, Clone)] +pub struct AuthConfig { + /// The authorization server's issuer identifier. + pub issuer: String, + /// The JWKS document the authorization server signs tokens with. + pub jwks_uri: String, + /// The PDS service DID tokens are audienced to (`did:web:…`). + pub audience: String, + /// First-party `client_id`s permitted to write into a space. + pub client_ids: Vec, + /// The authorization server's symmetric signing secret, present when the + /// PDS is its own authorization server (see `verify_as_signature`). + /// `None` selects the JWKS path. + pub hs256_secret: Option, +} + +impl AuthConfig { + pub fn validate(&self) -> std::result::Result<(), String> { + if !self.audience.starts_with("did:") { + return Err(format!( + "audience must be a service DID, got {}", + self.audience + )); + } + if self.client_ids.is_empty() { + return Err("client allowlist is empty".to_string()); + } + Ok(()) + } +} + +/// The parts of an inbound request the shim needs. +pub struct RequestAuth<'a> { + pub authorization: Option<&'a str>, + pub dpop: Option<&'a str>, + pub method: &'a str, + /// Absolute request URI as the client saw it, query and fragment stripped. + pub url: &'a str, +} + +/// A verified caller. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccessContext { + pub did: String, + pub client_id: String, + pub jkt: String, +} + +impl AccessContext { + /// A token authorizes writes only as its own subject, so a request naming + /// another author is rejected rather than silently rewritten. + pub fn require_author(&self, author_did: &str) -> Result<()> { + if self.did == author_did { + Ok(()) + } else { + Err(auth_err("token subject is not the named author")) + } + } +} + +#[derive(Debug, Deserialize)] +struct JwtHeader { + #[serde(default)] + typ: String, + #[serde(default)] + alg: String, + #[serde(default)] + kid: Option, + #[serde(default)] + jwk: Option, +} + +#[derive(Debug, Deserialize)] +struct AccessClaims { + iss: String, + aud: Audience, + sub: String, + exp: u64, + #[serde(default)] + client_id: Option, + #[serde(default)] + cnf: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum Audience { + One(String), + Many(Vec), +} + +impl Audience { + fn contains(&self, want: &str) -> bool { + match self { + Self::One(aud) => aud == want, + Self::Many(auds) => auds.iter().any(|a| a == want), + } + } +} + +#[derive(Debug, Deserialize)] +struct Confirmation { + #[serde(default)] + jkt: Option, +} + +#[derive(Debug, Deserialize)] +struct DpopClaims { + jti: String, + htm: String, + htu: String, + iat: u64, + #[serde(default)] + ath: Option, +} + +struct DecodedJwt { + header: JwtHeader, + claims: C, + signing_input: Vec, + signature: Vec, +} + +fn auth_err(msg: impl Into) -> HostError { + HostError::Delegation(msg.into()) +} + +fn decode_part Deserialize<'de>>(part: &str, what: &str) -> Result { + let bytes = URL_SAFE_NO_PAD + .decode(part) + .map_err(|e| auth_err(format!("{what} is not base64url: {e}")))?; + serde_json::from_slice(&bytes).map_err(|e| auth_err(format!("malformed {what}: {e}"))) +} + +fn decode_jwt Deserialize<'de>>(jwt: &str) -> Result> { + let parts: Vec<&str> = jwt.split('.').collect(); + if parts.len() != 3 { + return Err(auth_err("jwt must have three parts")); + } + Ok(DecodedJwt { + header: decode_part(parts[0], "jwt header")?, + claims: decode_part(parts[1], "jwt claims")?, + signing_input: format!("{}.{}", parts[0], parts[1]).into_bytes(), + signature: URL_SAFE_NO_PAD + .decode(parts[2]) + .map_err(|e| auth_err(format!("signature is not base64url: {e}")))?, + }) +} + +/// RFC 7638 JWK thumbprint: SHA-256 over the required members in +/// lexicographic order, base64url-encoded. +pub fn jwk_thumbprint(jwk: &EcJwk) -> String { + let canonical = format!( + r#"{{"crv":"{}","kty":"{}","x":"{}","y":"{}"}}"#, + jwk.crv, jwk.kty, jwk.x, jwk.y + ); + URL_SAFE_NO_PAD.encode(Sha256::digest(canonical.as_bytes())) +} + +fn access_token_hash(token: &str) -> String { + URL_SAFE_NO_PAD.encode(Sha256::digest(token.as_bytes())) +} + +fn split_scheme(header: &str) -> Result<(&str, &str)> { + header + .split_once(' ') + .map(|(scheme, value)| (scheme, value.trim())) + .ok_or_else(|| auth_err("malformed authorization header")) +} + +/// Strip query and fragment; `htu` compares only scheme, host and path. +fn htu_of(url: &str) -> &str { + let end = url.find(['?', '#']).unwrap_or(url.len()); + &url[..end] +} + +/// Verify a DPoP-bound access token on an inbound write and return the caller. +pub async fn verify_access( + request: &RequestAuth<'_>, + config: &AuthConfig, + fetcher: &dyn MetadataFetcher, + jti_store: &dyn JtiStore, + now: u64, +) -> Result { + let authorization = request + .authorization + .ok_or_else(|| auth_err("missing authorization header"))?; + let (scheme, token) = split_scheme(authorization)?; + if !scheme.eq_ignore_ascii_case("DPoP") { + return Err(auth_err(format!("unsupported auth scheme {scheme}"))); + } + let proof = request + .dpop + .ok_or_else(|| auth_err("missing DPoP proof header"))?; + + let jkt = verify_dpop_proof(proof, token, request, jti_store, now).await?; + let decoded: DecodedJwt = decode_jwt(token)?; + + if decoded.header.typ != ACCESS_TOKEN_TYP { + return Err(auth_err(format!( + "token typ {} != {ACCESS_TOKEN_TYP}", + decoded.header.typ + ))); + } + if !SUPPORTED_TOKEN_ALGS.contains(&decoded.header.alg.as_str()) { + return Err(auth_err(format!( + "unsupported token alg {}", + decoded.header.alg + ))); + } + let claims = &decoded.claims; + if claims.iss != config.issuer { + return Err(auth_err( + "token issuer is not the trusted authorization server", + )); + } + if !claims.aud.contains(&config.audience) { + return Err(auth_err("token audience is not this pds service did")); + } + if now >= claims.exp { + return Err(auth_err("token expired")); + } + let bound = claims + .cnf + .as_ref() + .and_then(|c| c.jkt.as_deref()) + .ok_or_else(|| auth_err("token is not dpop-bound"))?; + if bound != jkt { + return Err(auth_err("token is bound to a different key")); + } + let client_id = claims + .client_id + .clone() + .ok_or_else(|| auth_err("token carries no client_id"))?; + if !config.client_ids.iter().any(|c| *c == client_id) { + return Err(auth_err("client_id is not first-party")); + } + + verify_as_signature(&decoded, config, fetcher).await?; + + Ok(AccessContext { + did: claims.sub.clone(), + client_id, + jkt, + }) +} + +/// Verify the token's signature against the authorization server. +/// +/// **A PDS that is its own authorization server signs access tokens with +/// HS256**, using the same symmetric secret it uses for legacy session JWTs +/// (`pds/src/context.ts`, the `keyset` given to `OAuthProvider`). It therefore +/// publishes an *empty* JWKS: a symmetric key has no public half. The +/// asymmetric path applies only when an entryway issues the tokens instead. +/// +/// So a co-located verifier has two options, and only two: hold the same +/// secret, or ask the PDS. This holds the secret — the host already reads +/// every local account's signing key from the actor store, which is strictly +/// more sensitive, and asking would put a network call in every write. When +/// the secret is not configured the asymmetric path is used unchanged, so an +/// entryway deployment needs no different build. +async fn verify_as_signature( + decoded: &DecodedJwt, + config: &AuthConfig, + fetcher: &dyn MetadataFetcher, +) -> Result<()> { + if decoded.header.alg == HS256 { + let secret = config + .hs256_secret + .as_ref() + .ok_or_else(|| auth_err("token is HS256 but no shared secret is configured"))?; + return secret + .verify(&decoded.signing_input, &decoded.signature) + .then_some(()) + .ok_or_else(|| auth_err("token signature: hs256 verification failed")); + } + let jwks: JwkSet = fetcher.jwks(&config.jwks_uri).await?; + let jwk = match decoded.header.kid.as_deref() { + Some(kid) => jwks + .find(kid) + .ok_or_else(|| auth_err(format!("authorization server has no key {kid}")))?, + None => jwks + .keys + .first() + .ok_or_else(|| auth_err("authorization server jwks is empty"))?, + }; + verify_client_es256(jwk, &decoded.signing_input, &decoded.signature) + .map_err(|e| auth_err(format!("token signature: {e}"))) +} + +async fn verify_dpop_proof( + proof: &str, + token: &str, + request: &RequestAuth<'_>, + jti_store: &dyn JtiStore, + now: u64, +) -> Result { + let decoded: DecodedJwt = decode_jwt(proof)?; + if decoded.header.typ != DPOP_TYP { + return Err(auth_err(format!( + "proof typ {} != {DPOP_TYP}", + decoded.header.typ + ))); + } + if !SUPPORTED_PROOF_ALGS.contains(&decoded.header.alg.as_str()) { + return Err(auth_err(format!( + "unsupported proof alg {}", + decoded.header.alg + ))); + } + let jwk = decoded + .header + .jwk + .as_ref() + .ok_or_else(|| auth_err("proof carries no jwk"))?; + let verified = match decoded.header.alg.as_str() { + ES256K => verify_client_es256k(jwk, &decoded.signing_input, &decoded.signature), + _ => verify_client_es256(jwk, &decoded.signing_input, &decoded.signature), + }; + verified.map_err(|e| auth_err(format!("proof signature: {e}")))?; + + let claims = &decoded.claims; + if !claims.htm.eq_ignore_ascii_case(request.method) { + return Err(auth_err("proof htm does not match the request method")); + } + if htu_of(&claims.htu) != htu_of(request.url) { + return Err(auth_err("proof htu does not match the request url")); + } + if claims.ath.as_deref() != Some(access_token_hash(token).as_str()) { + return Err(auth_err("proof ath does not match the access token")); + } + if claims.iat > now + MAX_IAT_SKEW_SECS { + return Err(auth_err("proof iat is in the future")); + } + let expires = claims.iat.saturating_add(MAX_DPOP_AGE_SECS); + if now >= expires { + return Err(auth_err("proof is too old")); + } + if !jti_store.consume(&claims.jti, expires).await? { + return Err(auth_err("proof jti replayed")); + } + Ok(jwk_thumbprint(jwk)) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::attestation::{ClientMetadata, InMemoryJtiStore}; + use async_trait::async_trait; + use p256::ecdsa::signature::hazmat::PrehashSigner; + use p256::ecdsa::{Signature, SigningKey}; + use serde_json::json; + + pub(crate) const NOW: u64 = 1_700_000_000; + pub(crate) const ISSUER: &str = "https://pds.example.com"; + pub(crate) const PDS_DID: &str = "did:web:pds.example.com"; + pub(crate) const CLIENT: &str = "https://blacksky.community/oauth-client-metadata.json"; + pub(crate) const AUTHOR: &str = "did:plc:member"; + pub(crate) const URL: &str = "https://pds.example.com/xrpc/com.atproto.space.createRecord"; + + fn as_key() -> SigningKey { + SigningKey::from_slice(&[0x31u8; 32]).unwrap() + } + + pub(crate) fn client_key() -> SigningKey { + SigningKey::from_slice(&[0x32u8; 32]).unwrap() + } + + pub(crate) fn jwk_of(key: &SigningKey, kid: Option<&str>) -> EcJwk { + let point = key.verifying_key().to_encoded_point(false); + let bytes = point.as_bytes(); + EcJwk { + kty: "EC".to_string(), + crv: "P-256".to_string(), + x: URL_SAFE_NO_PAD.encode(&bytes[1..33]), + y: URL_SAFE_NO_PAD.encode(&bytes[33..65]), + kid: kid.map(str::to_string), + } + } + + pub(crate) fn sign_jwt( + key: &SigningKey, + header: serde_json::Value, + claims: serde_json::Value, + ) -> String { + let header = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).unwrap()); + let claims = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let input = format!("{header}.{claims}"); + let digest = Sha256::digest(input.as_bytes()); + let sig: Signature = key.sign_prehash(&digest).unwrap(); + let sig = sig.normalize_s().unwrap_or(sig); + format!("{input}.{}", URL_SAFE_NO_PAD.encode(sig.to_vec())) + } + + pub(crate) struct AsJwks; + #[async_trait] + impl MetadataFetcher for AsJwks { + async fn client_metadata(&self, _client_id: &str) -> Result { + Err(auth_err("not used")) + } + async fn jwks(&self, _url: &str) -> Result { + Ok(JwkSet { + keys: vec![jwk_of(&as_key(), Some("as-key-1"))], + }) + } + } + + pub(crate) fn config() -> AuthConfig { + AuthConfig { + issuer: ISSUER.to_string(), + jwks_uri: format!("{ISSUER}/oauth/jwks"), + audience: PDS_DID.to_string(), + client_ids: vec![CLIENT.to_string()], + hs256_secret: None, + } + } + + /// The claim set of a real stateful-mode token: no `scope`, DID `aud`. + pub(crate) fn token_claims() -> serde_json::Value { + json!({ + "iss": ISSUER, + "aud": PDS_DID, + "sub": AUTHOR, + "exp": NOW + 3600, + "iat": NOW, + "jti": "tok-1", + "client_id": CLIENT, + "cnf": {"jkt": jwk_thumbprint(&jwk_of(&client_key(), None))}, + }) + } + + pub(crate) fn token_with(claims: serde_json::Value) -> String { + sign_jwt( + &as_key(), + json!({"typ": ACCESS_TOKEN_TYP, "alg": "ES256", "kid": "as-key-1"}), + claims, + ) + } + + pub(crate) fn token() -> String { + token_with(token_claims()) + } + + pub(crate) fn proof_claims(token: &str) -> serde_json::Value { + json!({ + "jti": "proof-1", + "htm": "POST", + "htu": URL, + "iat": NOW, + "ath": access_token_hash(token), + }) + } + + pub(crate) fn proof_with(claims: serde_json::Value) -> String { + sign_jwt( + &client_key(), + json!({ + "typ": DPOP_TYP, + "alg": "ES256", + "jwk": jwk_of(&client_key(), None), + }), + claims, + ) + } + + async fn check_at( + token: &str, + proof: &str, + method: &str, + url: &str, + now: u64, + jti: &InMemoryJtiStore, + ) -> Result { + let authorization = format!("DPoP {token}"); + verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(proof), + method, + url, + }, + &config(), + &AsJwks, + jti, + now, + ) + .await + } + + async fn check(token: &str, proof: &str) -> Result { + check_at(token, proof, "POST", URL, NOW, &InMemoryJtiStore::default()).await + } + + pub(crate) fn k1_client_key() -> secp256k1::SecretKey { + secp256k1::SecretKey::from_slice(&[0x33u8; 32]).unwrap() + } + + pub(crate) fn k1_jwk_of(key: &secp256k1::SecretKey) -> EcJwk { + let point = key + .public_key(secp256k1::SECP256K1) + .serialize_uncompressed(); + EcJwk { + kty: "EC".to_string(), + crv: "secp256k1".to_string(), + x: URL_SAFE_NO_PAD.encode(&point[1..33]), + y: URL_SAFE_NO_PAD.encode(&point[33..65]), + kid: None, + } + } + + pub(crate) fn k1_sign_jwt( + key: &secp256k1::SecretKey, + header: serde_json::Value, + claims: serde_json::Value, + ) -> String { + let header = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).unwrap()); + let claims = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let input = format!("{header}.{claims}"); + let digest = Sha256::digest(input.as_bytes()); + let message = secp256k1::Message::from_digest_slice(&digest).unwrap(); + let sig = secp256k1::SECP256K1.sign_ecdsa(&message, key); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(sig.serialize_compact()) + ) + } + + /// A proof whose header names `jwk_key` but whose signature is made by + /// `signing_key`; equal keys give an ordinary well-formed proof. + fn k1_proof( + jwk_key: &secp256k1::SecretKey, + signing_key: &secp256k1::SecretKey, + claims: serde_json::Value, + ) -> String { + k1_sign_jwt( + signing_key, + json!({"typ": DPOP_TYP, "alg": ES256K, "jwk": k1_jwk_of(jwk_key)}), + claims, + ) + } + + fn token_bound_to(jkt: String) -> String { + let mut claims = token_claims(); + claims["cnf"] = json!({ "jkt": jkt }); + token_with(claims) + } + + #[tokio::test] + async fn accepts_an_es256k_dpop_proof() { + let key = k1_client_key(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_proof(&key, &key, proof_claims(&token)); + let context = check(&token, &proof).await.unwrap(); + assert_eq!(context.did, AUTHOR); + assert_eq!(context.jkt, jwk_thumbprint(&k1_jwk_of(&key))); + } + + #[tokio::test] + async fn rejects_an_es256k_proof_signed_by_another_key() { + let key = k1_client_key(); + let impostor = secp256k1::SecretKey::from_slice(&[0x34u8; 32]).unwrap(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_proof(&key, &impostor, proof_claims(&token)); + assert!(check(&token, &proof).await.is_err()); + } + + /// The bound key is the one the token names, not whichever key presents a + /// self-consistent proof. + #[tokio::test] + async fn rejects_an_es256k_proof_for_an_unbound_key() { + let bound = k1_client_key(); + let other = secp256k1::SecretKey::from_slice(&[0x35u8; 32]).unwrap(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&bound))); + let proof = k1_proof(&other, &other, proof_claims(&token)); + assert!(check(&token, &proof).await.is_err()); + } + + #[tokio::test] + async fn rejects_an_es256k_proof_carrying_a_p256_key() { + let token = token(); + let proof = sign_jwt( + &client_key(), + json!({"typ": DPOP_TYP, "alg": ES256K, "jwk": jwk_of(&client_key(), None)}), + proof_claims(&token), + ); + assert!(check(&token, &proof).await.is_err()); + } + + #[tokio::test] + async fn rejects_an_es256_proof_carrying_a_secp256k1_key() { + let key = k1_client_key(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_sign_jwt( + &key, + json!({"typ": DPOP_TYP, "alg": "ES256", "jwk": k1_jwk_of(&key)}), + proof_claims(&token), + ); + assert!(check(&token, &proof).await.is_err()); + } + + #[tokio::test] + async fn rejects_an_unsupported_proof_alg() { + let key = k1_client_key(); + let token = token_bound_to(jwk_thumbprint(&k1_jwk_of(&key))); + let proof = k1_sign_jwt( + &key, + json!({"typ": DPOP_TYP, "alg": "ES512", "jwk": k1_jwk_of(&key)}), + proof_claims(&token), + ); + assert!(check(&token, &proof).await.is_err()); + } + + const HS_SECRET: &[u8] = b"a-pds-jwt-secret"; + + fn hs256_token(claims: serde_json::Value) -> String { + use hmac::{Hmac, Mac}; + let header = json!({"typ": ACCESS_TOKEN_TYP, "alg": "HS256"}); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).unwrap()), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()) + ); + let mut mac = as Mac>::new_from_slice(HS_SECRET).unwrap(); + mac.update(input.as_bytes()); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) + } + + fn hs_config() -> AuthConfig { + AuthConfig { + hs256_secret: VerifyOnlyHs256Secret::new(HS_SECRET.to_vec()), + ..config() + } + } + + /// A PDS that is its own authorization server signs with HS256 and + /// publishes no public key, so this is the only path that can verify a + /// real token from one. + #[tokio::test] + async fn accepts_an_hs256_token_from_a_standalone_pds() { + let token = hs256_token(token_claims()); + let proof = proof_with(proof_claims(&token)); + let authorization = format!("DPoP {token}"); + let context = verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &hs_config(), + &AsJwks, + &InMemoryJtiStore::default(), + NOW, + ) + .await + .expect("hs256 token rejected"); + assert_eq!(context.did, AUTHOR); + } + + #[tokio::test] + async fn an_hs256_token_is_refused_without_the_secret_or_with_the_wrong_one() { + let token = hs256_token(token_claims()); + let proof = proof_with(proof_claims(&token)); + let authorization = format!("DPoP {token}"); + let attempt = |config: AuthConfig| { + let authorization = authorization.clone(); + let proof = proof.clone(); + async move { + verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config, + &AsJwks, + &InMemoryJtiStore::default(), + NOW, + ) + .await + } + }; + // Unconfigured: an HS256 token must not fall through to the JWKS path. + assert!(attempt(config()).await.is_err()); + assert!(attempt(AuthConfig { + hs256_secret: VerifyOnlyHs256Secret::new(b"not-the-secret".to_vec()), + ..config() + }) + .await + .is_err()); + } + + #[tokio::test] + async fn a_tampered_hs256_token_is_refused() { + let mut claims = token_claims(); + claims["sub"] = json!("did:plc:someone-else"); + let token = hs256_token(token_claims()); + let forged = format!( + "{}.{}.{}", + token.split('.').next().unwrap(), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()), + token.split('.').nth(2).unwrap() + ); + let proof = proof_with(proof_claims(&forged)); + let authorization = format!("DPoP {forged}"); + assert!(verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &hs_config(), + &AsJwks, + &InMemoryJtiStore::default(), + NOW, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn accepts_a_real_shaped_token() { + let token = token(); + let context = check(&token, &proof_with(proof_claims(&token))) + .await + .unwrap(); + assert_eq!(context.did, AUTHOR); + assert_eq!(context.client_id, CLIENT); + assert_eq!(context.jkt, jwk_thumbprint(&jwk_of(&client_key(), None))); + } + + #[tokio::test] + async fn accepts_a_token_with_no_scope_claim() { + // Stateful-mode tokens omit `scope` entirely; that must not be fatal. + let claims = token_claims(); + assert!(claims.get("scope").is_none()); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_ok()); + } + + #[tokio::test] + async fn rejects_an_origin_url_audience() { + // The regression this suite exists for: `aud` is the service DID, and a + // vector built from the origin would pass a lenient check but 401 in + // production. + let mut claims = token_claims(); + claims["aud"] = json!(ISSUER); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn accepts_a_did_audience_in_a_list() { + let mut claims = token_claims(); + claims["aud"] = json!(["did:web:other.example", PDS_DID]); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_ok()); + } + + #[tokio::test] + async fn rejects_bad_token_claims() { + for (name, mutate) in [ + ("wrong iss", json!("https://evil.example")), + ("wrong aud", json!("did:web:other.example")), + ] { + let mut claims = token_claims(); + let field = if name == "wrong iss" { "iss" } else { "aud" }; + claims[field] = mutate; + let token = token_with(claims); + assert!( + check(&token, &proof_with(proof_claims(&token))) + .await + .is_err(), + "{name} was accepted" + ); + } + + // Expired. + let mut claims = token_claims(); + claims["exp"] = json!(NOW - 1); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // Not dpop-bound. + let mut claims = token_claims(); + claims["cnf"] = json!({}); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // Bound to another key. + let mut claims = token_claims(); + claims["cnf"] = json!({"jkt": "someone-elses-thumbprint"}); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // No client_id. + let mut claims = token_claims(); + claims.as_object_mut().unwrap().remove("client_id"); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + + // Non-allowlisted client_id. + let mut claims = token_claims(); + claims["client_id"] = json!("https://third-party.example/client-metadata.json"); + let token = token_with(claims); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_a_wrong_token_typ() { + let token = sign_jwt( + &as_key(), + json!({"typ": "JWT", "alg": "ES256", "kid": "as-key-1"}), + token_claims(), + ); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_a_token_signed_by_another_key() { + let token = sign_jwt( + &client_key(), + json!({"typ": ACCESS_TOKEN_TYP, "alg": "ES256", "kid": "as-key-1"}), + token_claims(), + ); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_a_tampered_token_signature() { + let token = token(); + let mut tampered: Vec<&str> = token.split('.').collect(); + let flipped = format!("{}A", &tampered[2][..tampered[2].len() - 1]); + tampered[2] = &flipped; + let tampered = tampered.join("."); + assert!(check(&tampered, &proof_with(proof_claims(&tampered))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_an_unknown_signing_key_id() { + let token = sign_jwt( + &as_key(), + json!({"typ": ACCESS_TOKEN_TYP, "alg": "ES256", "kid": "rotated-away"}), + token_claims(), + ); + assert!(check(&token, &proof_with(proof_claims(&token))) + .await + .is_err()); + } + + #[tokio::test] + async fn rejects_bad_dpop_proofs() { + let token = token(); + + // Wrong htu. + let mut claims = proof_claims(&token); + claims["htu"] = json!("https://pds.example.com/xrpc/com.atproto.space.deleteRecord"); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Wrong htm. + let mut claims = proof_claims(&token); + claims["htm"] = json!("GET"); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // ath bound to a different token. + let mut claims = proof_claims(&token); + claims["ath"] = json!(access_token_hash("some.other.token")); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Stale proof. + let mut claims = proof_claims(&token); + claims["iat"] = json!(NOW - MAX_DPOP_AGE_SECS - 1); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Proof from the future. + let mut claims = proof_claims(&token); + claims["iat"] = json!(NOW + MAX_IAT_SKEW_SECS + 10); + assert!(check(&token, &proof_with(claims)).await.is_err()); + + // Proof signed by a key other than the one it embeds. + let forged = sign_jwt( + &as_key(), + json!({"typ": DPOP_TYP, "alg": "ES256", "jwk": jwk_of(&client_key(), None)}), + proof_claims(&token), + ); + assert!(check(&token, &forged).await.is_err()); + + // Proof with no embedded key. + let no_jwk = sign_jwt( + &client_key(), + json!({"typ": DPOP_TYP, "alg": "ES256"}), + proof_claims(&token), + ); + assert!(check(&token, &no_jwk).await.is_err()); + + // Wrong proof typ. + let wrong_typ = sign_jwt( + &client_key(), + json!({"typ": "JWT", "alg": "ES256", "jwk": jwk_of(&client_key(), None)}), + proof_claims(&token), + ); + assert!(check(&token, &wrong_typ).await.is_err()); + } + + #[tokio::test] + async fn rejects_a_replayed_proof() { + let jti = InMemoryJtiStore::default(); + let token = token(); + let proof = proof_with(proof_claims(&token)); + assert!(check_at(&token, &proof, "POST", URL, NOW, &jti) + .await + .is_ok()); + assert!(check_at(&token, &proof, "POST", URL, NOW, &jti) + .await + .is_err()); + } + + #[tokio::test] + async fn htu_ignores_query_and_fragment() { + let token = token(); + let mut claims = proof_claims(&token); + claims["htu"] = json!(format!("{URL}?x=1")); + let jti = InMemoryJtiStore::default(); + assert!(check_at( + &token, + &proof_with(claims), + "POST", + &format!("{URL}#frag"), + NOW, + &jti + ) + .await + .is_ok()); + } + + #[tokio::test] + async fn rejects_malformed_and_missing_headers() { + let token = token(); + let proof = proof_with(proof_claims(&token)); + let jti = InMemoryJtiStore::default(); + + // Bearer instead of DPoP. + let bearer = format!("Bearer {token}"); + assert!(verify_access( + &RequestAuth { + authorization: Some(&bearer), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // No authorization header. + assert!(verify_access( + &RequestAuth { + authorization: None, + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // No DPoP proof header. + let authorization = format!("DPoP {token}"); + assert!(verify_access( + &RequestAuth { + authorization: Some(&authorization), + dpop: None, + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // Header with no scheme separator. + assert!(verify_access( + &RequestAuth { + authorization: Some("DPoPnospace"), + dpop: Some(&proof), + method: "POST", + url: URL, + }, + &config(), + &AsJwks, + &jti, + NOW, + ) + .await + .is_err()); + + // Structurally broken JWTs. + assert!(check("not.a.jwt", &proof).await.is_err()); + assert!(check(&token, "only.two").await.is_err()); + } + + #[tokio::test] + async fn a_token_may_only_write_as_its_own_subject() { + let token = token(); + let context = check(&token, &proof_with(proof_claims(&token))) + .await + .unwrap(); + assert!(context.require_author(AUTHOR).is_ok()); + assert!(context.require_author("did:plc:someoneelse").is_err()); + } + + #[test] + fn config_requires_a_did_audience_and_a_client_allowlist() { + assert!(config().validate().is_ok()); + + let mut origin_aud = config(); + origin_aud.audience = ISSUER.to_string(); + assert!(origin_aud.validate().is_err()); + + let mut no_clients = config(); + no_clients.client_ids.clear(); + assert!(no_clients.validate().is_err()); + } + + #[test] + fn thumbprint_uses_the_canonical_member_order() { + let jwk = EcJwk { + kty: "EC".to_string(), + crv: "P-256".to_string(), + x: "eA".to_string(), + y: "eQ".to_string(), + kid: Some("ignored".to_string()), + }; + let expected = URL_SAFE_NO_PAD.encode(Sha256::digest( + r#"{"crv":"P-256","kty":"EC","x":"eA","y":"eQ"}"#, + )); + assert_eq!(jwk_thumbprint(&jwk), expected); + } +} diff --git a/rsky-space-host/src/pds_seam.rs b/rsky-space-host/src/pds_seam.rs new file mode 100644 index 00000000..402bf437 --- /dev/null +++ b/rsky-space-host/src/pds_seam.rs @@ -0,0 +1,338 @@ +use hmac::{Hmac, Mac}; +use secp256k1::SecretKey; +use sha2::{Digest, Sha256}; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use zeroize::{Zeroize, Zeroizing}; + +use crate::commits::CommitSigner; +use crate::error::{HostError, Result}; +use crate::service_jwt::{self, ServiceJwtIssuer}; +use crate::signing::Signer; + +const SECRET_KEY_BYTES: usize = 32; +pub const COMMIT_SIGN_AUDIT_EVENT: &str = "space_host_commit_signed"; + +#[derive(Clone)] +pub struct VerifyOnlyHs256Secret(Arc>>); + +impl VerifyOnlyHs256Secret { + pub fn new(secret: impl Into>) -> Option { + let secret = secret.into(); + (!secret.is_empty()).then(|| Self(Arc::new(Zeroizing::new(secret)))) + } + + pub(crate) fn verify(&self, signing_input: &[u8], signature: &[u8]) -> bool { + let Ok(mut mac) = as Mac>::new_from_slice(self.0.as_slice()) else { + return false; + }; + mac.update(signing_input); + mac.verify_slice(signature).is_ok() + } +} + +impl fmt::Debug for VerifyOnlyHs256Secret { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("VerifyOnlyHs256Secret([REDACTED])") + } +} + +pub trait SigningAudit: Send + Sync { + fn commit_signed(&self, did: &str, space: &str, rev: &str); +} + +struct TracingSigningAudit; + +impl SigningAudit for TracingSigningAudit { + fn commit_signed(&self, did: &str, space: &str, rev: &str) { + tracing::info!( + event = COMMIT_SIGN_AUDIT_EVENT, + did, + space, + rev, + "space repo commit signed" + ); + } +} + +pub struct PdsSeam { + directory: PathBuf, + audit: Arc, +} + +pub struct PdsServiceJwtIssuer { + seam: Arc, + issuer: String, +} + +impl PdsServiceJwtIssuer { + pub fn new(seam: Arc, issuer: impl Into) -> Self { + Self { + seam, + issuer: issuer.into(), + } + } +} + +impl ServiceJwtIssuer for PdsServiceJwtIssuer { + fn mint(&self, aud: &str, lxm: &str, now: u64, jti: String) -> Result { + let signer = self.seam.require_signer(&self.issuer)?; + service_jwt::mint(&signer, &self.issuer, aud, lxm, now, jti) + } +} + +impl PdsSeam { + pub fn open(directory: impl Into) -> Result { + let directory = directory.into(); + validate_actor_store_layout(&directory)?; + Ok(Self { + directory, + audit: Arc::new(TracingSigningAudit), + }) + } + + #[cfg(test)] + pub(crate) fn with_audit( + directory: impl Into, + audit: Arc, + ) -> Result { + let directory = directory.into(); + validate_actor_store_layout(&directory)?; + Ok(Self { directory, audit }) + } + + pub fn key_path(&self, author_did: &str) -> Option { + if author_did.is_empty() + || !author_did.starts_with("did:") + || author_did.contains('/') + || author_did.contains('\\') + || author_did.contains("..") + { + return None; + } + let digest = hex::encode(Sha256::digest(author_did.as_bytes())); + Some( + self.directory + .join(&digest[..2]) + .join(author_did) + .join("key"), + ) + } + + pub fn signer(&self, author_did: &str) -> Result { + self.require_signer(author_did) + } + + fn require_signer(&self, author_did: &str) -> Result { + let Some(path) = self.key_path(author_did) else { + return Err(HostError::InvalidRequest(format!( + "unusable did: {author_did}" + ))); + }; + let mut bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(HostError::AccountNotHosted(author_did.to_string())) + } + Err(error) => return Err(HostError::Key(error.to_string())), + }; + signer_from_bytes(&mut bytes) + } +} + +fn signer_from_bytes(bytes: &mut [u8]) -> Result { + let parsed = if bytes.len() == SECRET_KEY_BYTES { + SecretKey::from_slice(bytes).map_err(|error| HostError::Key(error.to_string())) + } else { + Err(HostError::Key(format!( + "actor key is {} bytes, expected {SECRET_KEY_BYTES}", + bytes.len() + ))) + }; + bytes.zeroize(); + parsed.map(Signer::from_secret) +} + +impl CommitSigner for PdsSeam { + fn did_key(&self, author_did: &str) -> Result { + Ok(self.require_signer(author_did)?.did_key().to_string()) + } + + fn sign( + &self, + author_did: &str, + space_uri: &str, + rev: &str, + message: &[u8], + ) -> Result> { + let signature = self + .require_signer(author_did)? + .sign(message) + .map_err(HostError::Key)?; + self.audit.commit_signed(author_did, space_uri, rev); + Ok(signature) + } +} + +// A live PDS actor store accumulates partial account dirs (abandoned +// creations, deactivations, mid-migration state). Their presence is not this +// service's concern: key availability is enforced per authority when it +// registers or signs (`require_signer` → AccountNotHosted). Boot validation +// therefore only proves the mount itself is right; malformed entries are +// logged and skipped, never fatal. +fn validate_actor_store_layout(root: &Path) -> Result<()> { + if !root.is_dir() { + return Err(HostError::Store(format!( + "actor store is not a directory: {}", + root.display() + ))); + } + for prefix in std::fs::read_dir(root).map_err(|error| HostError::Store(error.to_string()))? { + let prefix = prefix.map_err(|error| HostError::Store(error.to_string()))?; + let name = prefix.file_name(); + let name = name.to_string_lossy(); + if name == "reserved_keys" { + continue; + } + if name.len() != 2 + || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) + || !prefix.path().is_dir() + { + tracing::warn!(entry = %name, "skipping unrecognized actor-store layout entry"); + continue; + } + for actor in + std::fs::read_dir(prefix.path()).map_err(|error| HostError::Store(error.to_string()))? + { + let actor = actor.map_err(|error| HostError::Store(error.to_string()))?; + let actor_name = actor.file_name().to_string_lossy().to_string(); + if !actor_name.starts_with("did:") + || !actor.path().join("store.sqlite").is_file() + || !actor.path().join("key").is_file() + { + tracing::warn!( + actor = %actor_name, + "skipping malformed actor-store account dir" + ); + } + } + } + Ok(()) +} + +/// A temporary actor store holding one account's signing key, laid out the way +/// the PDS lays out `{data}/actors`. +#[cfg(test)] +pub(crate) fn test_actor_store(did: &str, secret: [u8; SECRET_KEY_BYTES]) -> tempfile::TempDir { + let directory = tempfile::tempdir().unwrap(); + let digest = hex::encode(Sha256::digest(did.as_bytes())); + let actor = directory.path().join(&digest[..2]).join(did); + std::fs::create_dir_all(&actor).unwrap(); + std::fs::write(actor.join("key"), secret).unwrap(); + std::fs::write(actor.join("store.sqlite"), []).unwrap(); + directory +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commits::mint_commit; + use std::sync::Mutex; + + const DID: &str = "did:plc:member"; + const SPACE: &str = "at://did:plc:authority/space/community.blacksky.feed/main"; + + fn actor_store(secret: [u8; 32]) -> tempfile::TempDir { + test_actor_store(DID, secret) + } + + #[test] + fn actor_key_buffer_is_zeroized_after_parsing() { + let mut bytes = [7u8; SECRET_KEY_BYTES]; + signer_from_bytes(&mut bytes).unwrap(); + assert_eq!(bytes, [0u8; SECRET_KEY_BYTES]); + } + + #[test] + fn service_auth_uses_the_authority_account_key() { + let directory = actor_store([7u8; SECRET_KEY_BYTES]); + let seam = Arc::new(PdsSeam::open(directory.path()).unwrap()); + let issuer = PdsServiceJwtIssuer::new(seam, DID); + let token = issuer + .mint("did:web:feeds.test", "test.method", 1_000, "jti-1".into()) + .unwrap(); + let account_key = Signer::from_secret(SecretKey::from_slice(&[7u8; 32]).unwrap()); + let claims = service_jwt::verify( + &token, + &["did:web:feeds.test"], + "test.method", + account_key.did_key(), + 1_001, + ) + .unwrap(); + assert_eq!(claims.iss, DID); + } + + #[test] + fn layout_drift_is_skipped_not_fatal() { + // A live PDS actor store carries partial dirs (abandoned creations, + // deactivations). They must not brick the host: signing for such an + // account still fails per-actor, but boot proceeds. + let directory = actor_store([7u8; 32]); + std::fs::create_dir(directory.path().join("unexpected-layout")).unwrap(); + let partial = directory.path().join("aa").join("did:plc:abandoned"); + std::fs::create_dir_all(&partial).unwrap(); + std::fs::write(partial.join("store.sqlite"), []).unwrap(); // no key file + let seam = PdsSeam::open(directory.path()).expect("boot survives layout drift"); + assert!(matches!( + seam.signer("did:plc:abandoned"), + Err(HostError::AccountNotHosted(_)) + )); + assert!(seam.signer(DID).is_ok()); + } + + #[test] + fn missing_actor_store_root_still_refuses_startup() { + assert!(matches!( + PdsSeam::open(std::path::Path::new("/nonexistent/actors")), + Err(HostError::Store(message)) if message.contains("not a directory") + )); + } + + #[derive(Default)] + struct RecordingAudit(Mutex>); + + impl SigningAudit for RecordingAudit { + fn commit_signed(&self, did: &str, space: &str, rev: &str) { + self.0 + .lock() + .unwrap() + .push((did.to_string(), space.to_string(), rev.to_string())); + } + } + + #[test] + fn signing_emits_the_stable_audit_event_payload() { + let directory = actor_store([9u8; 32]); + let audit = Arc::new(RecordingAudit::default()); + let seam = PdsSeam::with_audit(directory.path(), audit.clone()).unwrap(); + mint_commit(&seam, SPACE, DID, "3rev1", &[1u8; 32], [2u8; 32]).unwrap(); + assert_eq!( + *audit.0.lock().unwrap(), + vec![(DID.to_string(), SPACE.to_string(), "3rev1".to_string())] + ); + assert_eq!(COMMIT_SIGN_AUDIT_EVENT, "space_host_commit_signed"); + } + + #[test] + fn hs256_secret_exposes_verification_without_exposing_key_material() { + let secret = VerifyOnlyHs256Secret::new(b"verify-only".to_vec()).unwrap(); + let mut mac = as Mac>::new_from_slice(b"verify-only").unwrap(); + mac.update(b"header.payload"); + let signature = mac.finalize().into_bytes(); + assert!(secret.verify(b"header.payload", signature.as_slice())); + assert!(!secret.verify(b"other", signature.as_slice())); + assert_eq!(format!("{secret:?}"), "VerifyOnlyHs256Secret([REDACTED])"); + } +} diff --git a/rsky-space-host/src/registration.rs b/rsky-space-host/src/registration.rs new file mode 100644 index 00000000..83556689 --- /dev/null +++ b/rsky-space-host/src/registration.rs @@ -0,0 +1,175 @@ +use async_trait::async_trait; +use serde::Serialize; + +use crate::error::{HostError, Result}; +use crate::service_jwt::{ServiceJwtIssuer, SignerIssuer}; +use crate::signing::Signer; + +pub const REGISTER_SPACE_LXM: &str = "community.blacksky.space.register"; +pub const ACK_HOST_REGISTERED_LXM: &str = "community.blacksky.space.ackHostRegistered"; + +#[async_trait] +pub trait LifecycleAcker: Send + Sync { + async fn ack_host_registered(&self, space: &str, generation: i64) -> Result<()>; +} + +pub struct HttpLifecycleAcker { + base_url: String, + audience: String, + issuer: std::sync::Arc, + now: std::sync::Arc u64 + Send + Sync>, + jti: std::sync::Arc String + Send + Sync>, + http: reqwest::Client, +} + +impl HttpLifecycleAcker { + pub fn new( + base_url: impl Into, + audience: impl Into, + issuer: impl Into, + signer: Signer, + now: std::sync::Arc u64 + Send + Sync>, + jti: std::sync::Arc String + Send + Sync>, + ) -> Self { + Self::with_issuer( + base_url, + audience, + std::sync::Arc::new(SignerIssuer::new(issuer, signer)), + now, + jti, + ) + } + + pub fn with_issuer( + base_url: impl Into, + audience: impl Into, + issuer: std::sync::Arc, + now: std::sync::Arc u64 + Send + Sync>, + jti: std::sync::Arc String + Send + Sync>, + ) -> Self { + Self { + base_url: base_url.into().trim_end_matches('/').to_string(), + audience: audience.into(), + issuer, + now, + jti, + http: reqwest::Client::new(), + } + } +} + +#[derive(Serialize)] +struct Ack<'a> { + space: &'a str, + generation: i64, +} + +#[async_trait] +impl LifecycleAcker for HttpLifecycleAcker { + async fn ack_host_registered(&self, space: &str, generation: i64) -> Result<()> { + let token = self.issuer.mint( + &self.audience, + ACK_HOST_REGISTERED_LXM, + (self.now)(), + (self.jti)(), + )?; + let response = self + .http + .post(format!("{}/xrpc/{ACK_HOST_REGISTERED_LXM}", self.base_url)) + .bearer_auth(token) + .json(&Ack { space, generation }) + .send() + .await + .map_err(|e| HostError::ManagingApp(e.to_string()))?; + if !response.status().is_success() { + return Err(HostError::ManagingApp(format!( + "lifecycle acknowledgement returned {}", + response.status() + ))); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pds_seam::{test_actor_store, PdsSeam, PdsServiceJwtIssuer}; + use crate::service_jwt; + use crate::signing::test_signer; + use std::sync::Arc; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + const AUTHORITY: &str = "did:plc:auth"; + const FEEDS: &str = "did:web:feeds.test"; + const SPACE: &str = "at://did:plc:auth/space/community.blacksky.feed/main"; + + #[tokio::test] + async fn ack_is_signed_with_the_authority_account_key() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/xrpc/{ACK_HOST_REGISTERED_LXM}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({}))) + .mount(&server) + .await; + + let secret = [11u8; 32]; + let directory = test_actor_store(AUTHORITY, secret); + let seam = Arc::new(PdsSeam::open(directory.path()).unwrap()); + let acker = HttpLifecycleAcker::with_issuer( + server.uri(), + FEEDS, + Arc::new(PdsServiceJwtIssuer::new(seam, AUTHORITY.to_string())), + Arc::new(|| 1000), + Arc::new(|| "jti-fixed".to_string()), + ); + acker.ack_host_registered(SPACE, 1).await.unwrap(); + + let requests = server.received_requests().await.unwrap(); + let auth = requests[0].headers.get("authorization").unwrap(); + let jwt = auth.to_str().unwrap().strip_prefix("Bearer ").unwrap(); + + let account_key = + crate::signing::Signer::from_secret(secp256k1::SecretKey::from_slice(&secret).unwrap()); + let claims = service_jwt::verify( + jwt, + &[FEEDS], + ACK_HOST_REGISTERED_LXM, + account_key.did_key(), + 1000, + ) + .unwrap(); + assert_eq!(claims.iss, AUTHORITY); + + // The space key must not verify it. + assert!(service_jwt::verify( + jwt, + &[FEEDS], + ACK_HOST_REGISTERED_LXM, + test_signer().did_key(), + 1000, + ) + .is_err()); + } + + #[tokio::test] + async fn a_rejected_ack_is_an_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + let directory = test_actor_store(AUTHORITY, [11u8; 32]); + let seam = Arc::new(PdsSeam::open(directory.path()).unwrap()); + let acker = HttpLifecycleAcker::with_issuer( + server.uri(), + FEEDS, + Arc::new(PdsServiceJwtIssuer::new(seam, AUTHORITY.to_string())), + Arc::new(|| 1000), + Arc::new(|| "jti-fixed".to_string()), + ); + let error = acker.ack_host_registered(SPACE, 1).await.unwrap_err(); + assert!(matches!(error, HostError::ManagingApp(msg) if msg.contains("401"))); + } +} diff --git a/rsky-space-host/src/repo.rs b/rsky-space-host/src/repo.rs new file mode 100644 index 00000000..2d5fbc51 --- /dev/null +++ b/rsky-space-host/src/repo.rs @@ -0,0 +1,898 @@ +//! Permissioned repo storage (spec §Permissioned repos, §Incremental sync). +//! +//! One repo is one account's records within one space. A repo carries three +//! pieces of state, all advanced atomically by [`RepoStore::apply_writes`]: +//! +//! - the records themselves, keyed `(collection, rkey)`; +//! - the LtHash state, from which the commit `hash` is derived; +//! - an operation log, the transport optimization syncers page through. +//! +//! Commits are not stored. `ikm` is fresh per reader, so a commit is minted at +//! serve time from the persisted `(rev, state)` pair. + +use async_trait::async_trait; +use rsky_space::lthash::{element, LtHash}; +use rsky_space::record::dag_cbor_cid; +use std::collections::BTreeMap; +use std::sync::Mutex; + +use crate::error::{HostError, Result}; + +/// Size cap on a single stored record. Storage is shape-agnostic, so this and +/// DAG-CBOR well-formedness are the only limits the host applies to a value. +pub const MAX_RECORD_BYTES: usize = 64 * 1024; + +pub(crate) const STATE_BYTES: usize = 2048; + +/// One mutation in an atomic batch. Values are already-encoded DAG-CBOR. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RepoWrite { + Create { + collection: String, + rkey: String, + value: Vec, + }, + Update { + collection: String, + rkey: String, + value: Vec, + swap_record: Option, + }, + Delete { + collection: String, + rkey: String, + swap_record: Option, + }, +} + +impl RepoWrite { + pub fn collection(&self) -> &str { + match self { + Self::Create { collection, .. } + | Self::Update { collection, .. } + | Self::Delete { collection, .. } => collection, + } + } + + pub fn rkey(&self) -> &str { + match self { + Self::Create { rkey, .. } | Self::Update { rkey, .. } | Self::Delete { rkey, .. } => { + rkey + } + } + } +} + +/// What a write did. Every applied write produces an oplog entry, so a batch +/// that touches an absent record is refused rather than silently doing nothing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteOutcome { + Created { cid: String }, + Updated { cid: String }, + Deleted, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredRecord { + pub collection: String, + pub rkey: String, + pub cid: String, + pub value: Vec, +} + +impl StoredRecord { + /// The `{collection}/{rkey}` path used as the list cursor and CAR index key. + pub fn path(&self) -> String { + record_path(&self.collection, &self.rkey) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredOp { + pub seq: i64, + pub rev: String, + pub collection: String, + pub rkey: String, + pub cid: Option, + pub prev: Option, +} + +/// A repo's current position: the latest revision and the LtHash state behind +/// its commit digest. +#[derive(Clone)] +pub struct RepoHead { + pub rev: String, + pub state: [u8; STATE_BYTES], +} + +impl RepoHead { + pub fn hash(&self) -> [u8; 32] { + LtHash::from_state_bytes(&self.state).hash() + } +} + +/// A page of the operation log. `complete` is true when the page reaches the +/// repo's head, which is when a caller may attach the current commit. +pub struct OpPage { + pub ops: Vec, + pub cursor: Option, + pub complete: bool, +} + +/// The result of an atomic batch: the new head plus one outcome per write. +pub struct Applied { + pub rev: String, + pub hash: [u8; 32], + pub outcomes: Vec, +} + +pub fn record_path(collection: &str, rkey: &str) -> String { + format!("{collection}/{rkey}") +} + +#[async_trait] +pub trait RepoStore: Send + Sync { + /// Apply a batch atomically at revision `rev`. Every write in the batch + /// shares the revision, which is how syncers see them as one mutation. + async fn apply_writes( + &self, + space_uri: &str, + did: &str, + rev: &str, + writes: &[RepoWrite], + ) -> Result; + + async fn head(&self, space_uri: &str, did: &str) -> Result>; + + async fn get_record( + &self, + space_uri: &str, + did: &str, + collection: &str, + rkey: &str, + ) -> Result>; + + /// Records ordered by `{collection}/{rkey}`; `cursor` is the last path of + /// the previous page. + async fn list_records( + &self, + space_uri: &str, + did: &str, + collection: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result<(Vec, Option)>; + + /// Operations after `since` (a revision), ordered by insertion. Returns + /// [`HostError::HistoryUnavailable`] when `since` predates the retained + /// window, which is a syncer's signal to fall back to full-state recovery. + async fn list_ops( + &self, + space_uri: &str, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result; + + /// Drop a repo entirely (account deletion, space deletion). + async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()>; +} + +/// Fold one batch into an existing record set + digest. Shared by both +/// backings so their semantics cannot drift. +fn plan_batch( + existing: &BTreeMap, + lt: &mut LtHash, + writes: &[RepoWrite], +) -> Result> { + let mut planned = Vec::with_capacity(writes.len()); + let mut seen: BTreeMap> = BTreeMap::new(); + + for write in writes { + let path = record_path(write.collection(), write.rkey()); + let current = match seen.get(&path) { + Some(cid) => cid.clone(), + None => existing.get(&path).map(|r| r.cid.clone()), + }; + + let planned_write = match write { + RepoWrite::Create { + collection, + rkey, + value, + } => { + if current.is_some() { + return Err(HostError::RecordExists(path)); + } + let cid = dag_cbor_cid(value).to_string(); + lt.add(&element(collection, rkey, &cid)); + PlannedWrite { + collection: collection.clone(), + rkey: rkey.clone(), + cid: Some(cid.clone()), + prev: None, + value: Some(value.clone()), + outcome: WriteOutcome::Created { cid }, + } + } + RepoWrite::Update { + collection, + rkey, + value, + swap_record, + } => { + check_swap(swap_record.as_deref(), current.as_deref())?; + let Some(prev) = current.clone() else { + return Err(HostError::RecordNotFound(path)); + }; + let cid = dag_cbor_cid(value).to_string(); + lt.remove(&element(collection, rkey, &prev)); + lt.add(&element(collection, rkey, &cid)); + PlannedWrite { + collection: collection.clone(), + rkey: rkey.clone(), + cid: Some(cid.clone()), + prev: current.clone(), + value: Some(value.clone()), + outcome: WriteOutcome::Updated { cid }, + } + } + RepoWrite::Delete { + collection, + rkey, + swap_record, + } => { + check_swap(swap_record.as_deref(), current.as_deref())?; + let Some(prev) = current.clone() else { + return Err(HostError::RecordNotFound(path)); + }; + lt.remove(&element(collection, rkey, &prev)); + PlannedWrite { + collection: collection.clone(), + rkey: rkey.clone(), + cid: None, + prev: Some(prev), + value: None, + outcome: WriteOutcome::Deleted, + } + } + }; + + seen.insert(path, planned_write.cid.clone()); + planned.push(planned_write); + } + Ok(planned) +} + +fn check_swap(swap: Option<&str>, current: Option<&str>) -> Result<()> { + match swap { + Some(expected) if current != Some(expected) => Err(HostError::InvalidSwap), + _ => Ok(()), + } +} + +struct PlannedWrite { + collection: String, + rkey: String, + cid: Option, + prev: Option, + value: Option>, + outcome: WriteOutcome, +} + +pub(crate) fn page_cursor(page: &[T], limit: u32, key: impl Fn(&T) -> String) -> Option { + match page.last() { + Some(last) if page.len() == limit as usize => Some(key(last)), + _ => None, + } +} + +// ---------------------------------------------------------------- in memory + +struct MemRepo { + records: BTreeMap, + ops: Vec, + rev: String, + state: [u8; STATE_BYTES], +} + +impl Default for MemRepo { + fn default() -> Self { + Self { + records: BTreeMap::new(), + ops: Vec::new(), + rev: String::new(), + state: [0u8; STATE_BYTES], + } + } +} + +#[derive(Default)] +pub struct InMemoryRepos { + repos: Mutex>, + next_seq: Mutex, +} + +#[async_trait] +impl RepoStore for InMemoryRepos { + async fn apply_writes( + &self, + space_uri: &str, + did: &str, + rev: &str, + writes: &[RepoWrite], + ) -> Result { + let mut repos = self.repos.lock().unwrap(); + let repo = repos + .entry((space_uri.to_string(), did.to_string())) + .or_default(); + let mut lt = LtHash::from_state_bytes(&repo.state); + let planned = plan_batch(&repo.records, &mut lt, writes)?; + + let mut seq = self.next_seq.lock().unwrap(); + for p in &planned { + let path = record_path(&p.collection, &p.rkey); + match &p.cid { + Some(cid) => { + repo.records.insert( + path, + StoredRecord { + collection: p.collection.clone(), + rkey: p.rkey.clone(), + cid: cid.clone(), + value: p.value.clone().unwrap_or_default(), + }, + ); + } + None => { + repo.records.remove(&path); + } + } + *seq += 1; + repo.ops.push(StoredOp { + seq: *seq, + rev: rev.to_string(), + collection: p.collection.clone(), + rkey: p.rkey.clone(), + cid: p.cid.clone(), + prev: p.prev.clone(), + }); + } + + repo.state = lt.state_bytes(); + repo.rev = rev.to_string(); + Ok(Applied { + rev: repo.rev.clone(), + hash: lt.hash(), + outcomes: planned.into_iter().map(|p| p.outcome).collect(), + }) + } + + async fn head(&self, space_uri: &str, did: &str) -> Result> { + Ok(self + .repos + .lock() + .unwrap() + .get(&(space_uri.to_string(), did.to_string())) + .map(|r| RepoHead { + rev: r.rev.clone(), + state: r.state, + })) + } + + async fn get_record( + &self, + space_uri: &str, + did: &str, + collection: &str, + rkey: &str, + ) -> Result> { + Ok(self + .repos + .lock() + .unwrap() + .get(&(space_uri.to_string(), did.to_string())) + .and_then(|r| r.records.get(&record_path(collection, rkey)).cloned())) + } + + async fn list_records( + &self, + space_uri: &str, + did: &str, + collection: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result<(Vec, Option)> { + let repos = self.repos.lock().unwrap(); + let Some(repo) = repos.get(&(space_uri.to_string(), did.to_string())) else { + return Err(HostError::RepoNotFound); + }; + let page: Vec = repo + .records + .iter() + .filter(|(path, record)| { + collection.is_none_or(|c| record.collection == c) + && cursor.is_none_or(|c| path.as_str() > c) + }) + .take(limit as usize) + .map(|(_, record)| record.clone()) + .collect(); + let cursor = page_cursor(&page, limit, |r| r.path()); + Ok((page, cursor)) + } + + async fn list_ops( + &self, + space_uri: &str, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: u32, + ) -> Result { + let repos = self.repos.lock().unwrap(); + let Some(repo) = repos.get(&(space_uri.to_string(), did.to_string())) else { + return Err(HostError::RepoNotFound); + }; + ensure_history(since, repo.ops.first().map(|o| o.rev.as_str()))?; + let after = parse_cursor(cursor)?; + let page: Vec = repo + .ops + .iter() + .filter(|op| { + since.is_none_or(|s| op.rev.as_str() > s) && after.is_none_or(|c| op.seq > c) + }) + .take(limit as usize) + .cloned() + .collect(); + Ok(finish_op_page(page, limit, repo.ops.last().map(|o| o.seq))) + } + + async fn delete_repo(&self, space_uri: &str, did: &str) -> Result<()> { + self.repos + .lock() + .unwrap() + .remove(&(space_uri.to_string(), did.to_string())); + Ok(()) + } +} + +fn ensure_history(since: Option<&str>, earliest_retained: Option<&str>) -> Result<()> { + match (since, earliest_retained) { + // Every retained operation is already newer than `since`, so the + // caller's revision fell out of the window (or never existed here). + (Some(since), Some(earliest)) if earliest > since => Err(HostError::HistoryUnavailable), + (Some(_), None) => Err(HostError::HistoryUnavailable), + _ => Ok(()), + } +} + +pub(crate) fn parse_cursor(cursor: Option<&str>) -> Result> { + cursor + .map(|c| { + c.parse::() + .map_err(|_| HostError::InvalidRequest(format!("invalid cursor: {c}"))) + }) + .transpose() +} + +pub(crate) fn finish_op_page(ops: Vec, limit: u32, last_seq: Option) -> OpPage { + let reached_head = ops.last().map(|o| o.seq) == last_seq; + let cursor = page_cursor(&ops, limit, |o| o.seq.to_string()); + OpPage { + complete: reached_head, + cursor: if reached_head { None } else { cursor }, + ops, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const SPACE: &str = "at://did:plc:auth/space/community.blacksky.feed/main"; + const DID: &str = "did:plc:member"; + const POST: &str = "app.bsky.feed.post"; + const LIKE: &str = "app.bsky.feed.like"; + + fn value(text: &str) -> Vec { + rsky_space::record::encode_record(&json!({"text": text}), MAX_RECORD_BYTES).unwrap() + } + + fn create(rkey: &str, text: &str) -> RepoWrite { + RepoWrite::Create { + collection: POST.to_string(), + rkey: rkey.to_string(), + value: value(text), + } + } + + fn delete(rkey: &str) -> RepoWrite { + RepoWrite::Delete { + collection: POST.to_string(), + rkey: rkey.to_string(), + swap_record: None, + } + } + + async fn exercise_write_read_cycle(store: &dyn RepoStore) { + let applied = store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + assert_eq!(applied.rev, "3rev1"); + assert!(matches!(applied.outcomes[0], WriteOutcome::Created { .. })); + + let got = store + .get_record(SPACE, DID, POST, "a") + .await + .unwrap() + .unwrap(); + assert_eq!( + rsky_space::record::decode_record(&got.value).unwrap(), + json!({"text": "one"}) + ); + assert_eq!(got.cid, dag_cbor_cid(&value("one")).to_string()); + + // The digest matches an independent fold over the record set. + let head = store.head(SPACE, DID).await.unwrap().unwrap(); + let mut expected = LtHash::new(); + expected.add(&element(POST, "a", &got.cid)); + assert_eq!(head.hash(), expected.hash()); + + // Update swaps the element, delete removes it, digest returns to empty. + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Update { + collection: POST.to_string(), + rkey: "a".to_string(), + value: value("two"), + swap_record: Some(got.cid.clone()), + }], + ) + .await + .unwrap(); + let updated = store + .get_record(SPACE, DID, POST, "a") + .await + .unwrap() + .unwrap(); + assert_eq!(updated.cid, dag_cbor_cid(&value("two")).to_string()); + + store + .apply_writes(SPACE, DID, "3rev3", &[delete("a")]) + .await + .unwrap(); + assert!(store + .get_record(SPACE, DID, POST, "a") + .await + .unwrap() + .is_none()); + assert_eq!( + store.head(SPACE, DID).await.unwrap().unwrap().hash(), + LtHash::new().hash() + ); + } + + async fn exercise_swap_and_conflict(store: &dyn RepoStore) { + store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + + assert!(matches!( + store + .apply_writes(SPACE, DID, "3rev2", &[create("a", "again")]) + .await, + Err(HostError::RecordExists(_)) + )); + assert!(matches!( + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Delete { + collection: POST.to_string(), + rkey: "a".to_string(), + swap_record: Some("bafyreiwrong".to_string()), + }] + ) + .await, + Err(HostError::InvalidSwap) + )); + // A swap against an absent record is also a swap failure. + assert!(matches!( + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Update { + collection: POST.to_string(), + rkey: "missing".to_string(), + value: value("x"), + swap_record: Some("bafyreiwrong".to_string()), + }] + ) + .await, + Err(HostError::InvalidSwap) + )); + + // A rejected batch leaves nothing behind. + let head = store.head(SPACE, DID).await.unwrap().unwrap(); + assert_eq!(head.rev, "3rev1"); + let (records, _) = store + .list_records(SPACE, DID, None, None, 10) + .await + .unwrap(); + assert_eq!(records.len(), 1); + } + + async fn exercise_absent_record(store: &dyn RepoStore) { + store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + assert!(matches!( + store + .apply_writes(SPACE, DID, "3rev2", &[delete("ghost")]) + .await, + Err(HostError::RecordNotFound(_)) + )); + assert!(matches!( + store + .apply_writes( + SPACE, + DID, + "3rev2", + &[RepoWrite::Update { + collection: POST.to_string(), + rkey: "ghost".to_string(), + value: value("x"), + swap_record: None, + }] + ) + .await, + Err(HostError::RecordNotFound(_)) + )); + // A refused batch neither advances the revision nor logs an operation. + assert_eq!(store.head(SPACE, DID).await.unwrap().unwrap().rev, "3rev1"); + let page = store.list_ops(SPACE, DID, None, None, 10).await.unwrap(); + assert_eq!(page.ops.len(), 1); + } + + async fn exercise_atomic_batch(store: &dyn RepoStore) { + store + .apply_writes( + SPACE, + DID, + "3rev1", + &[ + create("a", "one"), + create("b", "two"), + RepoWrite::Create { + collection: LIKE.to_string(), + rkey: "l1".to_string(), + value: value("like"), + }, + ], + ) + .await + .unwrap(); + let page = store.list_ops(SPACE, DID, None, None, 10).await.unwrap(); + assert_eq!(page.ops.len(), 3); + // Operations mutated atomically share a revision. + assert!(page.ops.iter().all(|o| o.rev == "3rev1")); + assert!(page.complete); + + // A create-then-delete of the same path within one batch nets to nothing. + store + .apply_writes(SPACE, DID, "3rev2", &[create("c", "three"), delete("c")]) + .await + .unwrap(); + assert!(store + .get_record(SPACE, DID, POST, "c") + .await + .unwrap() + .is_none()); + } + + async fn exercise_listing(store: &dyn RepoStore) { + for (i, rkey) in ["a", "b", "c"].iter().enumerate() { + store + .apply_writes(SPACE, DID, &format!("3rev{i}"), &[create(rkey, rkey)]) + .await + .unwrap(); + } + store + .apply_writes( + SPACE, + DID, + "3rev9", + &[RepoWrite::Create { + collection: LIKE.to_string(), + rkey: "l1".to_string(), + value: value("like"), + }], + ) + .await + .unwrap(); + + let (page, cursor) = store.list_records(SPACE, DID, None, None, 2).await.unwrap(); + assert_eq!(page.len(), 2); + assert_eq!(page[0].collection, LIKE); + let (page, cursor2) = store + .list_records(SPACE, DID, None, cursor.as_deref(), 2) + .await + .unwrap(); + assert_eq!(page.len(), 2); + // A full final page still yields a cursor; the empty page ends paging. + let (page, cursor3) = store + .list_records(SPACE, DID, None, cursor2.as_deref(), 2) + .await + .unwrap(); + assert!(page.is_empty()); + assert!(cursor3.is_none()); + + let (posts, _) = store + .list_records(SPACE, DID, Some(POST), None, 10) + .await + .unwrap(); + assert_eq!(posts.len(), 3); + assert!(posts.iter().all(|r| r.collection == POST)); + + assert!(matches!( + store + .list_records(SPACE, "did:plc:nobody", None, None, 10) + .await, + Err(HostError::RepoNotFound) + )); + } + + async fn exercise_oplog_paging(store: &dyn RepoStore) { + for i in 0..5 { + store + .apply_writes( + SPACE, + DID, + &format!("3rev{i}"), + &[create(&format!("r{i}"), "x")], + ) + .await + .unwrap(); + } + + let page = store.list_ops(SPACE, DID, None, None, 2).await.unwrap(); + assert_eq!(page.ops.len(), 2); + assert!(!page.complete); + let page2 = store + .list_ops(SPACE, DID, None, page.cursor.as_deref(), 10) + .await + .unwrap(); + assert_eq!(page2.ops.len(), 3); + assert!(page2.complete); + assert!(page2.cursor.is_none()); + + // `since` yields strictly later revisions. + let page = store + .list_ops(SPACE, DID, Some("3rev2"), None, 10) + .await + .unwrap(); + assert_eq!( + page.ops.iter().map(|o| o.rev.as_str()).collect::>(), + vec!["3rev3", "3rev4"] + ); + + // A revision at or past the head yields an empty, complete page. + let page = store + .list_ops(SPACE, DID, Some("3rev4"), None, 10) + .await + .unwrap(); + assert!(page.ops.is_empty()); + + assert!(matches!( + store.list_ops(SPACE, DID, None, Some("abc"), 10).await, + Err(HostError::InvalidRequest(_)) + )); + assert!(matches!( + store + .list_ops(SPACE, "did:plc:nobody", None, None, 10) + .await, + Err(HostError::RepoNotFound) + )); + } + + async fn exercise_history_unavailable(store: &dyn RepoStore) { + store + .apply_writes(SPACE, DID, "3rev5", &[create("a", "one")]) + .await + .unwrap(); + // `since` predates every retained operation. + assert!(matches!( + store.list_ops(SPACE, DID, Some("3rev1"), None, 10).await, + Err(HostError::HistoryUnavailable) + )); + } + + async fn exercise_isolation_and_deletion(store: &dyn RepoStore) { + let other_space = "at://did:plc:auth/space/community.blacksky.feed/other"; + store + .apply_writes(SPACE, DID, "3rev1", &[create("a", "one")]) + .await + .unwrap(); + store + .apply_writes(other_space, DID, "3rev1", &[create("a", "other")]) + .await + .unwrap(); + store + .apply_writes(SPACE, "did:plc:other", "3rev1", &[create("a", "theirs")]) + .await + .unwrap(); + + store.delete_repo(SPACE, DID).await.unwrap(); + assert!(store.head(SPACE, DID).await.unwrap().is_none()); + assert!(store.head(other_space, DID).await.unwrap().is_some()); + assert!(store.head(SPACE, "did:plc:other").await.unwrap().is_some()); + // Deleting an absent repo is not an error. + store.delete_repo(SPACE, DID).await.unwrap(); + } + + macro_rules! both_backings { + ($name:ident, $exercise:ident) => { + #[tokio::test] + async fn $name() { + $exercise(&InMemoryRepos::default()).await; + } + }; + } + + both_backings!(write_read_cycle, exercise_write_read_cycle); + both_backings!(swap_and_conflict, exercise_swap_and_conflict); + both_backings!(absent_record, exercise_absent_record); + both_backings!(atomic_batch, exercise_atomic_batch); + both_backings!(listing, exercise_listing); + both_backings!(oplog_paging, exercise_oplog_paging); + both_backings!(history_unavailable, exercise_history_unavailable); + both_backings!(isolation_and_deletion, exercise_isolation_and_deletion); + + #[tokio::test] + async fn digest_is_order_independent_across_repos() { + let a = InMemoryRepos::default(); + a.apply_writes(SPACE, DID, "3rev1", &[create("x", "one")]) + .await + .unwrap(); + a.apply_writes(SPACE, DID, "3rev2", &[create("y", "two")]) + .await + .unwrap(); + + let b = InMemoryRepos::default(); + b.apply_writes(SPACE, DID, "3rev1", &[create("y", "two")]) + .await + .unwrap(); + b.apply_writes(SPACE, DID, "3rev2", &[create("x", "one")]) + .await + .unwrap(); + + assert_eq!( + a.head(SPACE, DID).await.unwrap().unwrap().hash(), + b.head(SPACE, DID).await.unwrap().unwrap().hash() + ); + } + + #[test] + fn history_window_and_cursor_edges() { + assert!(ensure_history(None, None).is_ok()); + assert!(ensure_history(Some("3rev1"), Some("3rev1")).is_ok()); + assert!(matches!( + ensure_history(Some("3rev1"), None), + Err(HostError::HistoryUnavailable) + )); + assert_eq!(parse_cursor(None).unwrap(), None); + assert_eq!(parse_cursor(Some("7")).unwrap(), Some(7)); + } +} diff --git a/rsky-space-host/src/service_jwt.rs b/rsky-space-host/src/service_jwt.rs index c5393ac3..172ac3ce 100644 --- a/rsky-space-host/src/service_jwt.rs +++ b/rsky-space-host/src/service_jwt.rs @@ -33,6 +33,30 @@ pub struct ServiceClaims { pub const SERVICE_JWT_TTL_SECS: u64 = 60; +pub trait ServiceJwtIssuer: Send + Sync { + fn mint(&self, aud: &str, lxm: &str, now: u64, jti: String) -> Result; +} + +pub struct SignerIssuer { + issuer: String, + signer: Signer, +} + +impl SignerIssuer { + pub fn new(issuer: impl Into, signer: Signer) -> Self { + Self { + issuer: issuer.into(), + signer, + } + } +} + +impl ServiceJwtIssuer for SignerIssuer { + fn mint(&self, aud: &str, lxm: &str, now: u64, jti: String) -> Result { + mint(&self.signer, &self.issuer, aud, lxm, now, jti) + } +} + /// Mint a service-auth JWT signed by `signer` (sha256 then ECDSA, compact r||s). pub fn mint( signer: &Signer, diff --git a/rsky-space-host/src/store.rs b/rsky-space-host/src/store.rs index e050b66a..5afdc802 100644 --- a/rsky-space-host/src/store.rs +++ b/rsky-space-host/src/store.rs @@ -68,6 +68,34 @@ pub trait RegistrationStore: Send + Sync { async fn endpoints(&self, space_uri: &str, now: u64) -> Result>; } +/// The spaces registered with this host (via `community.blacksky.space.register`), +/// grouped by authority, so a restart re-serves them. +#[async_trait] +pub trait HostedSpaceStore: Send + Sync { + async fn record_space(&self, authority_did: &str, space_uri: &str) -> Result<()>; + async fn hosted_spaces(&self) -> Result>; +} + +#[derive(Default)] +pub struct InMemoryHostedSpaces { + spaces: Mutex>, +} + +#[async_trait] +impl HostedSpaceStore for InMemoryHostedSpaces { + async fn record_space(&self, authority_did: &str, space_uri: &str) -> Result<()> { + self.spaces + .lock() + .unwrap() + .insert((authority_did.to_string(), space_uri.to_string()), ()); + Ok(()) + } + + async fn hosted_spaces(&self) -> Result> { + Ok(self.spaces.lock().unwrap().keys().cloned().collect()) + } +} + fn next_cursor(page: &[RepoRef], limit: u32) -> Option { if page.len() == limit as usize { page.last().map(|r| r.did.clone()) @@ -194,6 +222,11 @@ impl SqliteStore { CREATE TABLE IF NOT EXISTS used_jti ( jti TEXT PRIMARY KEY, exp INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS hosted_space ( + authority_did TEXT NOT NULL, + space_uri TEXT NOT NULL, + PRIMARY KEY (authority_did, space_uri) );", ) .map_err(sql_err)?; @@ -313,6 +346,36 @@ impl RegistrationStore for SqliteStore { } } +#[async_trait] +impl HostedSpaceStore for SqliteStore { + async fn record_space(&self, authority_did: &str, space_uri: &str) -> Result<()> { + self.conn + .lock() + .unwrap() + .execute( + "INSERT OR IGNORE INTO hosted_space (authority_did, space_uri) VALUES (?1, ?2)", + rusqlite::params![authority_did, space_uri], + ) + .map_err(sql_err)?; + Ok(()) + } + + async fn hosted_spaces(&self) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn + .prepare( + "SELECT authority_did, space_uri FROM hosted_space + ORDER BY authority_did ASC, space_uri ASC", + ) + .map_err(sql_err)?; + let rows = stmt + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .map_err(sql_err)?; + rows.collect::, _>>() + .map_err(sql_err) + } +} + #[async_trait] impl JtiStore for SqliteStore { async fn consume(&self, jti: &str, exp: u64) -> Result { @@ -451,6 +514,36 @@ mod tests { .unwrap()); } + async fn exercise_hosted_spaces(store: &dyn HostedSpaceStore) { + store.record_space("did:plc:auth", SPACE).await.unwrap(); + store.record_space("did:plc:auth", SPACE).await.unwrap(); + store + .record_space("did:plc:auth", OTHER_SPACE) + .await + .unwrap(); + store + .record_space("did:plc:other", "at://did:plc:other/space/t/main") + .await + .unwrap(); + assert_eq!( + store.hosted_spaces().await.unwrap(), + vec![ + ("did:plc:auth".to_string(), SPACE.to_string()), + ("did:plc:auth".to_string(), OTHER_SPACE.to_string()), + ( + "did:plc:other".to_string(), + "at://did:plc:other/space/t/main".to_string() + ), + ] + ); + } + + #[tokio::test] + async fn hosted_spaces_round_trip() { + exercise_hosted_spaces(&InMemoryHostedSpaces::default()).await; + exercise_hosted_spaces(&SqliteStore::open_in_memory().unwrap()).await; + } + #[tokio::test] async fn sqlite_persists_across_reopen() { let dir = tempfile::tempdir().unwrap(); diff --git a/rsky-space/Cargo.toml b/rsky-space/Cargo.toml index 88862d6e..921a8c5c 100644 --- a/rsky-space/Cargo.toml +++ b/rsky-space/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rsky-space" -version = "0.4.1" +version = "0.4.2" authors = ["Rudy Fraser "] description = "atproto permissioned-data (spaces) primitives: LtHash commits, credentials, space repos" edition = "2021" @@ -16,6 +16,7 @@ serde_json = { workspace = true } serde_bytes = { workspace = true } serde_ipld_dagcbor = { workspace = true } lexicon_cid = { workspace = true } +ipld-core = { workspace = true } iroh-car = "0.5.1" tokio = { workspace = true } sha2 = { workspace = true } diff --git a/rsky-space/src/error.rs b/rsky-space/src/error.rs index eb1ab649..ba2f07f6 100644 --- a/rsky-space/src/error.rs +++ b/rsky-space/src/error.rs @@ -35,6 +35,8 @@ pub enum SpaceError { Car(String), #[error("decode error: {0}")] Decode(String), + #[error("record is {size} bytes, over the {max}-byte limit")] + RecordTooLarge { size: usize, max: usize }, #[error("invalid jwk: {0}")] InvalidJwk(String), #[error("crypto error: {0}")] diff --git a/rsky-space/src/jwk.rs b/rsky-space/src/jwk.rs index 13858e2f..6df49335 100644 --- a/rsky-space/src/jwk.rs +++ b/rsky-space/src/jwk.rs @@ -1,20 +1,27 @@ -//! Minimal EC JWK (P-256) support for verifying ES256-signed JWTs +//! Minimal EC JWK support for verifying ES256- and ES256K-signed JWTs //! (proposal §Client attestation). //! //! A space authority verifies a client attestation by resolving the client's //! published JWKS and checking the JWT signature against the key named by the -//! attestation's `kid`. Only `kty: "EC"` / `crv: "P-256"` keys are supported, -//! matching the attestation's required `alg: "ES256"`. +//! attestation's `kid`. Only `kty: "EC"` keys on the `P-256` and `secp256k1` +//! curves are supported. Each verifier requires its own curve, so a key can +//! never be verified under an algorithm it was not issued for. use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use crate::error::{Result, SpaceError}; const COORD_LEN: usize = 32; -/// An EC public JWK restricted to the P-256 curve. +/// The JWK `crv` of an ES256 key. +pub const CRV_P256: &str = "P-256"; +/// The JWK `crv` of an ES256K key. +pub const CRV_SECP256K1: &str = "secp256k1"; + +/// An EC public JWK on a supported curve. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct EcJwk { pub kty: String, @@ -42,7 +49,7 @@ impl EcJwk { self.kty ))); } - if self.crv != "P-256" { + if self.crv != CRV_P256 && self.crv != CRV_SECP256K1 { return Err(SpaceError::InvalidJwk(format!( "unsupported crv: {}", self.crv @@ -51,6 +58,18 @@ impl EcJwk { Ok(()) } + /// Validate the key and require one specific curve. + pub fn require_crv(&self, crv: &str) -> Result<()> { + self.validate()?; + if self.crv != crv { + return Err(SpaceError::InvalidJwk(format!( + "expected crv {crv}, got {}", + self.crv + ))); + } + Ok(()) + } + /// The SEC1 uncompressed point: `0x04 || x || y`. pub fn sec1_point(&self) -> Result> { let x = decode_coord(&self.x, "x")?; @@ -82,7 +101,7 @@ fn decode_coord(b64: &str, name: &str) -> Result> { /// `sig` must be the compact 64-byte low-S `r || s` encoding; anything else is /// rejected as a bad signature. pub fn verify_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { - jwk.validate()?; + jwk.require_crv(CRV_P256)?; let point = jwk.sec1_point()?; let ok = rsky_crypto::p256::operations::verify_sig(&point, signing_input, sig, None) .map_err(|e| SpaceError::Crypto(e.to_string()))?; @@ -93,6 +112,24 @@ pub fn verify_es256(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> } } +/// Verify an ES256K signature over a JWT signing input +/// (`header_b64.payload_b64` bytes) against a secp256k1 JWK. +/// +/// `sig` must be the compact 64-byte low-S `r || s` encoding; anything else is +/// rejected as a bad signature. +pub fn verify_es256k(jwk: &EcJwk, signing_input: &[u8], sig: &[u8]) -> Result<()> { + jwk.require_crv(CRV_SECP256K1)?; + let point = jwk.sec1_point()?; + let digest = Sha256::digest(signing_input); + let ok = rsky_crypto::secp256k1::operations::verify_sig(&point, &digest, sig, None) + .map_err(|e| SpaceError::Crypto(e.to_string()))?; + if ok { + Ok(()) + } else { + Err(SpaceError::BadSignature) + } +} + /// A JWKS document (`jwks` / `jwks_uri` in client metadata). #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct JwkSet { @@ -255,6 +292,81 @@ mod tests { )); } + fn k1_key() -> secp256k1::SecretKey { + secp256k1::SecretKey::from_slice(&[0x63u8; 32]).unwrap() + } + + fn k1_jwk_for(key: &secp256k1::SecretKey) -> EcJwk { + let secp = secp256k1::Secp256k1::new(); + let point = key.public_key(&secp).serialize_uncompressed(); + EcJwk { + kty: "EC".to_string(), + crv: CRV_SECP256K1.to_string(), + x: URL_SAFE_NO_PAD.encode(&point[1..33]), + y: URL_SAFE_NO_PAD.encode(&point[33..65]), + kid: None, + } + } + + fn k1_sign(key: &secp256k1::SecretKey, input: &[u8]) -> Vec { + let secp = secp256k1::Secp256k1::new(); + let digest = Sha256::digest(input); + let message = secp256k1::Message::from_digest_slice(&digest).unwrap(); + secp.sign_ecdsa(&message, key).serialize_compact().to_vec() + } + + #[test] + fn es256k_verify_roundtrip() { + let key = k1_key(); + verify_es256k(&k1_jwk_for(&key), INPUT, &k1_sign(&key, INPUT)).unwrap(); + } + + #[test] + fn es256k_wrong_key_rejected() { + let sig = k1_sign(&k1_key(), INPUT); + let other = secp256k1::SecretKey::from_slice(&[0x64u8; 32]).unwrap(); + assert!(matches!( + verify_es256k(&k1_jwk_for(&other), INPUT, &sig), + Err(SpaceError::BadSignature) + )); + } + + #[test] + fn es256k_tampered_input_rejected() { + let key = k1_key(); + let sig = k1_sign(&key, INPUT); + let mut tampered = INPUT.to_vec(); + tampered[0] ^= 0xFF; + assert!(matches!( + verify_es256k(&k1_jwk_for(&key), &tampered, &sig), + Err(SpaceError::BadSignature) + )); + } + + #[test] + fn curve_and_algorithm_must_agree() { + let k1 = k1_key(); + let k1_sig = k1_sign(&k1, INPUT); + assert!(matches!( + verify_es256(&k1_jwk_for(&k1), INPUT, &k1_sig), + Err(SpaceError::InvalidJwk(msg)) if msg.contains("crv") + )); + + let p = signing_key(); + let p_sig = sign(&p, INPUT); + assert!(matches!( + verify_es256k(&jwk_for(&p, None), INPUT, &p_sig), + Err(SpaceError::InvalidJwk(msg)) if msg.contains("crv") + )); + } + + #[test] + fn es256k_jwk_survives_json_roundtrip() { + let jwk = k1_jwk_for(&k1_key()); + let json = serde_json::to_string(&jwk).unwrap(); + assert_eq!(EcJwk::from_json(&json).unwrap(), jwk); + } + #[test] fn jwk_set_find_by_kid() { let key = signing_key(); diff --git a/rsky-space/src/lib.rs b/rsky-space/src/lib.rs index 6e73f18c..7cb4d953 100644 --- a/rsky-space/src/lib.rs +++ b/rsky-space/src/lib.rs @@ -13,6 +13,7 @@ //! signature/MAC verification. //! - [`credential`] — delegation tokens, space credentials, and client //! attestations (the JWT envelope + verification). +//! - [`record`] — shape-agnostic permissioned-record encoding (DAG-CBOR). //! - [`jwk`] — minimal EC JWK (P-256) verification for ES256 client //! attestations. //! - [`space_id`] — space and permissioned-record `at://.../space/...` @@ -29,6 +30,7 @@ pub mod credential; pub mod error; pub mod jwk; pub mod lthash; +pub mod record; pub mod space_id; pub mod types; @@ -36,5 +38,6 @@ pub use car::{repo_car_bytes, write_repo_car, RepoCarValidator}; pub use error::{Result, SpaceError}; pub use jwk::{verify_es256, EcJwk, JwkSet}; pub use lthash::LtHash; +pub use record::{dag_cbor_cid, decode_record, encode_record}; pub use space_id::{is_space_uri, RecordId, SpaceId}; pub use types::{RepoOp, RepoRef, SignedCommit}; diff --git a/rsky-space/src/record.rs b/rsky-space/src/record.rs new file mode 100644 index 00000000..3a08fd58 --- /dev/null +++ b/rsky-space/src/record.rs @@ -0,0 +1,210 @@ +//! Permissioned-record bytes: the atproto data model over DAG-CBOR. +//! +//! A repo host stores permissioned records shape-agnostically — it checks that +//! the value is a well-formed atproto data-model object under a size cap and +//! nothing more. Lexicon validation of permissioned records happens at the +//! consuming boundary, not here, because records in a space legitimately carry +//! space URIs in fields a public lexicon declares as `at-uri`. +//! +//! JSON ↔ DAG-CBOR follows the atproto data model: `{"$link": "…"}` is a CID +//! link, `{"$bytes": "…"}` is base64 bytes, and floats are rejected. + +use base64::Engine; +use ipld_core::ipld::Ipld; +use lexicon_cid::multihash::Multihash; +use lexicon_cid::Cid; +use serde_json::{Map, Number, Value}; +use sha2::{Digest, Sha256}; + +use crate::error::{Result, SpaceError}; + +const SHA2_256: u64 = 0x12; +/// DAG-CBOR multicodec, the codec of every permissioned record block. +pub const DAG_CBOR: u64 = 0x71; + +/// The CID of an already-encoded DAG-CBOR block. +pub fn dag_cbor_cid(bytes: &[u8]) -> Cid { + let digest = Sha256::digest(bytes); + let multihash = Multihash::wrap(SHA2_256, &digest).expect("sha256 digest fits in multihash"); + Cid::new_v1(DAG_CBOR, multihash) +} + +fn b64() -> base64::engine::general_purpose::GeneralPurpose { + base64::engine::general_purpose::STANDARD +} + +fn decode_err(msg: impl Into) -> SpaceError { + SpaceError::Decode(msg.into()) +} + +fn json_to_ipld(value: &Value) -> Result { + Ok(match value { + Value::Null => Ipld::Null, + Value::Bool(b) => Ipld::Bool(*b), + Value::String(s) => Ipld::String(s.clone()), + Value::Number(n) => Ipld::Integer( + n.as_i64() + .map(i128::from) + .ok_or_else(|| decode_err("only integer numbers are representable"))?, + ), + Value::Array(items) => Ipld::List(items.iter().map(json_to_ipld).collect::>()?), + Value::Object(map) => match tagged(map) { + Some(("$link", Value::String(s))) => Ipld::Link( + s.parse::() + .map_err(|e| decode_err(format!("invalid $link: {e}")))?, + ), + Some(("$bytes", Value::String(s))) => Ipld::Bytes( + b64() + .decode(s) + .map_err(|e| decode_err(format!("invalid $bytes: {e}")))?, + ), + _ => Ipld::Map( + map.iter() + .map(|(k, v)| Ok((k.clone(), json_to_ipld(v)?))) + .collect::>()?, + ), + }, + }) +} + +/// A single-entry map carrying one of the data model's reserved `$` keys. +fn tagged(map: &Map) -> Option<(&str, &Value)> { + if map.len() != 1 { + return None; + } + let (key, value) = map.iter().next()?; + matches!(key.as_str(), "$link" | "$bytes").then_some((key.as_str(), value)) +} + +fn ipld_to_json(value: &Ipld) -> Result { + Ok(match value { + Ipld::Null => Value::Null, + Ipld::Bool(b) => Value::Bool(*b), + Ipld::String(s) => Value::String(s.clone()), + Ipld::Integer(i) => Value::Number(Number::from( + i64::try_from(*i).map_err(|_| decode_err("integer out of range"))?, + )), + Ipld::Float(_) => return Err(decode_err("floats are not part of the atproto data model")), + Ipld::Bytes(b) => serde_json::json!({ "$bytes": b64().encode(b) }), + Ipld::Link(cid) => serde_json::json!({ "$link": cid.to_string() }), + Ipld::List(items) => Value::Array(items.iter().map(ipld_to_json).collect::>()?), + Ipld::Map(map) => Value::Object( + map.iter() + .map(|(k, v)| Ok((k.clone(), ipld_to_json(v)?))) + .collect::>()?, + ), + }) +} + +/// Encode a record value to canonical DAG-CBOR. The value must be an object; +/// no other structural or lexicon constraint is applied. +pub fn encode_record(value: &Value, max_bytes: usize) -> Result> { + if !value.is_object() { + return Err(decode_err("record must be an object")); + } + let ipld = json_to_ipld(value)?; + let bytes = serde_ipld_dagcbor::to_vec(&ipld).map_err(|e| decode_err(e.to_string()))?; + if bytes.len() > max_bytes { + return Err(SpaceError::RecordTooLarge { + size: bytes.len(), + max: max_bytes, + }); + } + Ok(bytes) +} + +/// Decode stored DAG-CBOR record bytes back to their JSON representation. +pub fn decode_record(bytes: &[u8]) -> Result { + let ipld: Ipld = + serde_ipld_dagcbor::from_slice(bytes).map_err(|e| decode_err(e.to_string()))?; + ipld_to_json(&ipld) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const MAX: usize = 64 * 1024; + + #[test] + fn roundtrips_the_data_model() { + let value = json!({ + "$type": "app.bsky.feed.post", + "text": "hi", + "count": 3, + "flag": true, + "nothing": null, + "tags": ["a", "b"], + "embed": { + "image": {"$link": "bafkreiabc2hs4mmvxbtmvj47xrslw6ijbcbtoz2jyxbjyqmjkgu3f4z2wq"}, + "raw": {"$bytes": "AQID"} + } + }); + let bytes = encode_record(&value, MAX).unwrap(); + assert_eq!(decode_record(&bytes).unwrap(), value); + } + + #[test] + fn encoding_is_canonical_and_order_independent() { + let a = encode_record(&json!({"a": 1, "b": 2}), MAX).unwrap(); + let b = encode_record(&json!({"b": 2, "a": 1}), MAX).unwrap(); + assert_eq!(a, b); + assert_eq!(dag_cbor_cid(&a), dag_cbor_cid(&b)); + } + + #[test] + fn cid_is_a_dag_cbor_sha256_link() { + let cid = dag_cbor_cid(&encode_record(&json!({"a": 1}), MAX).unwrap()); + assert_eq!(cid.codec(), DAG_CBOR); + assert_eq!(cid.hash().code(), SHA2_256); + assert!(cid.to_string().starts_with("bafyrei")); + } + + #[test] + fn a_space_uri_is_stored_verbatim() { + let uri = + "at://did:plc:auth/space/community.blacksky.feed/main/did:plc:a/app.bsky.feed.post/3k"; + let value = json!({"subject": {"uri": uri, "cid": "bafyreia"}}); + let bytes = encode_record(&value, MAX).unwrap(); + assert_eq!(decode_record(&bytes).unwrap(), value); + } + + #[test] + fn rejects_non_objects_floats_and_oversize() { + assert!(matches!( + encode_record(&json!([1, 2]), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"n": 1.5}), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"$link": "not-a-cid"}), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"$bytes": "!!!"}), MAX), + Err(SpaceError::Decode(_)) + )); + assert!(matches!( + encode_record(&json!({"text": "x".repeat(100)}), 16), + Err(SpaceError::RecordTooLarge { .. }) + )); + } + + #[test] + fn multi_key_dollar_maps_stay_maps() { + let value = json!({"m": {"$link": "bafyreia", "extra": 1}}); + let bytes = encode_record(&value, MAX).unwrap(); + assert_eq!(decode_record(&bytes).unwrap(), value); + } + + #[test] + fn decode_rejects_floats_and_malformed_cbor() { + let float = serde_ipld_dagcbor::to_vec(&Ipld::Float(1.5)).unwrap(); + assert!(matches!(decode_record(&float), Err(SpaceError::Decode(_)))); + assert!(matches!(decode_record(&[0xff]), Err(SpaceError::Decode(_)))); + } +} diff --git a/rsky-spaces-parity/Cargo.toml b/rsky-spaces-parity/Cargo.toml new file mode 100644 index 00000000..01243db6 --- /dev/null +++ b/rsky-spaces-parity/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "rsky-spaces-parity" +version = "0.3.1" +edition = "2021" +publish = false + +[dependencies] +rsky-space-host = { path = "../rsky-space-host" } +rsky-space = { path = "../rsky-space" } +oracle-rsky-space = { package = "rsky-space", git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } +rsky-pds = { git = "https://github.com/blacksky-algorithms/rsky.git", rev = "7ebd21ae788c550ee8510034d94eb19ede148738" } +rsky-oauth = { path = "../rsky-oauth" } +rsky-crypto = { workspace = true } +anyhow = "1" +serde_json = { workspace = true } +tokio = { workspace = true } +rusqlite = { workspace = true } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] } +base64 = "0.22" +hmac = "0.12" +sha2 = { workspace = true } +secp256k1 = { workspace = true } +serde = { workspace = true } +hex = "0.4" + +[dev-dependencies] +tempfile = "3" + +[[bin]] +name = "layer2-gate" +path = "src/bin/layer2_gate.rs" + +[[bin]] +name = "resume-gate" +path = "src/bin/resume_gate.rs" diff --git a/rsky-spaces-parity/layer2/README.md b/rsky-spaces-parity/layer2/README.md new file mode 100644 index 00000000..c9b19b70 --- /dev/null +++ b/rsky-spaces-parity/layer2/README.md @@ -0,0 +1,75 @@ +# Layer 2 — two-process acceptance gate + +Layer 1 (`cargo test -p rsky-spaces-parity`) drives both write paths in one +process. Layer 2 runs the two **real binaries** and compares what they actually +serve and store. + +```sh +./rsky-spaces-parity/layer2/run.sh +``` + +Exit code 0 means every scored check was equal. Nothing else is needed: no +Postgres, no Docker, no network. Both servers are SQLite-backed, and the gate +hosts the only support service (a stub DID directory on a loopback port). + +## What it does + +1. Creates a **detached, build-only** git worktree at the pinned oracle revision + `7ebd21ae788c550ee8510034d94eb19ede148738` and builds `rsky-pds` there. The + worktree is refused if it is dirty or at another revision, so the oracle can + never drift into the code under test. +2. Builds `rsky-space-host` from the working tree. +3. Runs `layer2-gate`, which: + - starts the stub DID directory, then the oracle PDS, then the space host; + - creates an account, activates it, opens a session, and creates the space + `at:///space/community.blacksky.feed/main` on the oracle; + - copies the oracle's actor-store directory so the space host has the account + signing keys, and replaces each `store.sqlite` with an empty one created by + the space host's own schema code; + - fires the same ten-step record script at each server over XRPC; + - compares five shared read endpoints; + - probes the four methods only the PDS routes; + - stops both servers and points the Layer 1 stored-row comparator + (`dump_tables` / `compare_tables` / `revs_are_well_formed`) at the two + `store.sqlite` files. + +Everything lands under `target/layer2/run`: `pds.log`, `shim.log`, `report.txt`, +and both store directories. The directory is wiped at the start of every run. + +## Credentials + +All local, all fixed, all created and destroyed inside the run directory; none +of it is a secret and none of it reaches a real service. + +- The oracle PDS takes its own session token on space writes. +- The space host verifies an access token it did not issue, so the gate acts as + the authorization server: it holds the same HS256 secret the host is + configured with (`SPACEHOST_OAUTH_HS256_SECRET`) and signs `at+jwt` tokens + with it, DPoP-bound to a fixed P-256 key. This is the same shape a PDS that is + its own authorization server issues. +- Read endpoints take a space credential on both sides: minted through + `getDelegationToken` → `getSpaceCredential` on the PDS, and through + `/admin/mintCredential` on the space host. + +## What is not compared, and why + +- `ikm`, `sig`, `mac` on a served commit: derived from fresh random key material + every serve. +- The paging `cursor`: an oplog row id, numbered per store. +- Revisions: server-minted TIDs. Substituted for `R1, R2, …` in first-appearance + order, exactly as the Layer 1 comparator does. Record keys are TIDs too and + compare literally. +- JSON object key order: the two decoders build maps in different orders from + identical DAG-CBOR bytes. Record identity is the CID, compared unnormalized in + the same response. +- `getSpace`: recorded as a documented divergence. The space definition is + configuration on the host and `space_def` rows on the PDS, which the storage + convergence leaves out of scope by design. + +## Running outside a sandbox + +On macOS both servers build a `reqwest` client whose proxy discovery calls +SystemConfiguration. A restricted sandbox denies that and the pinned +`hyper-util` panics on the null result. Run the gate with normal process +permissions. `sccache` is also unusable there, so `run.sh` clears +`RUSTC_WRAPPER`. diff --git a/rsky-spaces-parity/layer2/report/2026-08-21.md b/rsky-spaces-parity/layer2/report/2026-08-21.md new file mode 100644 index 00000000..b8b179fa --- /dev/null +++ b/rsky-spaces-parity/layer2/report/2026-08-21.md @@ -0,0 +1,165 @@ +# Layer 2 acceptance gate — run report + +**Date:** 2026-08-21 (America/New_York) +**Branch:** `feat/space-host-main-port` +**Oracle:** `rsky-pds` built from a detached, build-only worktree at +`7ebd21ae788c550ee8510034d94eb19ede148738` +**Under test:** `rsky-space-host` from the working tree +**Command:** `./rsky-spaces-parity/layer2/run.sh` (exit 0) + +## Result + +``` +parity: 18/18 checks equal (+1 documented divergence) +``` + +Both real binaries ran as separate processes. The same ten-step record script +was fired at each over XRPC, five shared read endpoints were compared, and the +Layer 1 stored-row comparator was pointed at the two on-disk `store.sqlite` +files. Every scored check was equal, including every record CID and every +LtHash digest at every step. + +## Surface difference: four methods the PDS routes and the space host does not + +This is the headline finding for anyone moving a client from the space host to +rsky-pds. The two write surfaces overlap; the read surfaces do not. + +| Method | rsky-pds | rsky-space-host | +|---|---|---| +| `com.atproto.space.createRecord` | routed | routed | +| `com.atproto.space.deleteRecord` | routed | routed | +| `com.atproto.space.getRepo` | routed | routed | +| `com.atproto.space.listRepoOps` | routed | routed | +| `com.atproto.space.getLatestCommit` | routed | routed | +| `com.atproto.space.listRepos` | routed | routed | +| `com.atproto.space.getSpace` | routed | routed | +| `com.atproto.space.applyWrites` | routed | **404** | +| `com.atproto.space.getRecord` | routed | **404** | +| `com.atproto.space.listRecords` | routed | **404** | +| `com.atproto.space.getRepoState` | routed | **404** | + +A client migrating from the space host to rsky-pds **gains** batch writes +(`applyWrites`) and direct record reads (`getRecord`, `listRecords`, +`getRepoState`); nothing is lost. Because the space host has no batch endpoint, +`applyWrites` cannot be part of a same-script comparison at all: a batch under +one revision on the PDS and N single writes on the host produce different +revision grouping and different oplog shapes by construction. It is therefore +probed PDS-only, with a deliberately invalid write so the probe cannot mutate +one side and not the other. + +## Divergences the gate found and this run fixed + +All three were invisible to Layer 1, which compares internal outcome enums and +stored rows rather than HTTP responses. All three were in `rsky-space-host` +(`rsky-pds` and `rsky-space` were not touched). + +1. **XRPC error names were collapsed.** The space host mapped + `RecordExists` and `RecordNotFound` onto `InvalidRequest`, and answered + `InvalidSwap` with 409 and `HistoryUnavailable` with 410 where the oracle + answers 400 with the specific name. A client could not distinguish "already + exists" from a malformed request. Now all four match the oracle's name and + status. +2. **`listRepoOps` attached the wrong value to a superseded operation.** The + host hydrated an op's `value` by `(collection, rkey)`, so an op whose record + had since been deleted and recreated carried the *current* record's value + beside its own historical CID. The oracle joins on the CID as well and omits + the value. Now the host does the same. +3. **The writer set omitted the repo digest.** `listRepos` returned `did` and + `rev` but no `hash`, because the local write path passed `None` where the + oracle passes the commit hash. Now it passes the digest; the notify-write + path still passes `None`, which is correct — a notification carries no hash. + +## Documented divergence + +`getSpace` answers from the space *definition*, which the storage convergence +leaves out of scope: the space host holds it in configuration, the PDS in its +own `space_def` rows (a table the stored-row comparator deliberately does not +read). In this run the host reported `policy: public` from its flags and the +PDS `policy: member-list` from its stored definition. + +## Falsification evidence + +The gate was observed failing before it was observed passing, on all three of +its comparison layers, which is what makes the green meaningful: + +- **Write responses** caught divergence 1 (`400 InvalidRequest` against + `400 RecordExists`). +- **Read responses** caught divergences 2 and 3. +- **Stored rows** caught an eleventh record written to the PDS only, when the + `applyWrites` probe still used a valid batch: `space_repo`, `space_record` + and `space_oplog` all reported differing rows and the run exited non-zero. + +## Scoreboard + +``` + [equal] write 01 createRecord com.example.post/3kaaaaaaaaaa1 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/3kaaaaaaaaaa1 cid=bafyreigiltmq54gboouigc54umca576hqldm2pn2qrh5goxafd6b3pa554 rev=R1 hash=54b495b7015c9acc2de6618cdc7451301d669189484db8a1add74cd8324d237b + [equal] write 02 createRecord com.example.post/3kaaaaaaaaaa2 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/3kaaaaaaaaaa2 cid=bafyreibrc4gkm3q4ovjdmcgi74sdptx34x2hrwwypvdkzk6jrb42ie3uc4 rev=R2 hash=18d0e789470666518aa2a70594180c2b8e28f47d241125a698940b734f1f973c + [equal] write 03 createRecord com.example.note/3kaaaaaaaaaa3 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.note/3kaaaaaaaaaa3 cid=bafyreih234nznbwegwspg7ezp4gqci246dg7kh6crwmxe7oogynryyf7n4 rev=R3 hash=d2a61fab058f3daed04ea9c36c70277d2401eee5680c9474a664eacc7050311b + [equal] write 04 createRecord com.example.post/unicode.rkey_1~ + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/unicode.rkey_1~ cid=bafyreigewh3bnk7sg2ufwm7ldx54ehr5fobiqw46q3hunv2mv4eeiue5vy rev=R4 hash=28bad2f99440f79e0afa6993f1dddcb3c35db0e4103a24b0b836d769f32f7c13 + [equal] write 05 createRecord com.example.post/3kaaaaaaaaaa1 + 400 RecordExists + [equal] write 06 deleteRecord com.example.post/3kaaaaaaaaaa1 + 200 uri=- cid=- rev=R5 hash=41facf48277e70f7044005777ce2db525e96b070804bb740e6e97f589ab5e3ad + [equal] write 07 createRecord com.example.post/3kaaaaaaaaaa1 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.post/3kaaaaaaaaaa1 cid=bafyreigjz7i56srnmyj7uyevkz26fqzbohsdvyu47l2oszxghyyxck5lf4 rev=R6 hash=e07880fdc27a57e83e361086e37a4d2cc69cc6e30202bda170b67894843661e3 + [equal] write 08 deleteRecord com.example.post/3kmissingaaaa + 400 RecordNotFound + [equal] write 09 deleteRecord com.example.note/3kaaaaaaaaaa3 + 200 uri=- cid=- rev=R7 hash=dfc8cb119ab18e2705cbb6f27fd0c49c0e2a9875a128670d56e41044d6d04ad6 + [equal] write 10 createRecord com.example.note/3kaaaaaaaaaa4 + 200 uri=at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main/did:plc:layer2writeraaaaaaaaaaa/com.example.note/3kaaaaaaaaaa4 cid=bafyreiahahltekodrn4hj5vnsow6zb26kdjm5zrvsq57av45ipgqfyogx4 rev=R8 hash=bc2254cc03faccc88f91ff0b3d2b9d073d241d1db16e871cc61ebbdd45215e5b + [equal] write revisions are TIDs + shim minted 8 revisions, pds minted 8 + [equal] read com.atproto.space.listRepoOps + 200 + [equal] read com.atproto.space.listRepoOps (paged) + 200 + [equal] read com.atproto.space.getLatestCommit + 200 + [equal] read com.atproto.space.listRepos + 200 + [documented divergence] read com.atproto.space.getSpace + space definition is configured on the host and stored on the pds + shim: 200 { + "config": { + "$type": "com.atproto.simplespace.defs#config", + "appAccess": { + "$type": "com.atproto.simplespace.defs#appAccessOpen" + }, + "policy": "public" + }, + "space": "at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main" + } + pds: 200 { + "config": { + "$type": "com.atproto.simplespace.defs#config", + "appAccess": { + "$type": "com.atproto.simplespace.defs#appAccessOpen" + }, + "policy": "member-list" + }, + "space": "at://did:plc:layer2writeraaaaaaaaaaa/space/community.blacksky.feed/main" + } + [equal] read com.atproto.space.getRepo record blocks + 5 shared blocks; unique to shim: ["0171122058e7325f960239995ce4f6efeaeb507bdf201fbe377004bc64fbf9b90e01ec09"]; unique to pds: ["01711220f4ff946d631dfdc1f2ea0026620ddc853bc6ab254c157b9cc598bf5181f3ac16"] + [note] surface com.atproto.space.getRecord + pds 200, space host 404 (not routed) + [note] surface com.atproto.space.listRecords + pds 200, space host 404 (not routed) + [note] surface com.atproto.space.getRepoState + pds 200, space host 404 (not routed) + [note] surface com.atproto.space.applyWrites + pds 400, space host 404 (not routed) + [equal] stored space_* rows + /target/layer2/run/shim/actors/c6/did:plc:layer2writeraaaaaaaaaaa/store.sqlite / /target/layer2/run/pds/actors/c6/did:plc:layer2writeraaaaaaaaaaa/store.sqlite + [equal] stored revisions well formed + shim 8 distinct revisions, pds 8 + +parity: 18/18 checks equal (+1 documented divergence) + +logs: /target/layer2/run +``` diff --git a/rsky-spaces-parity/layer2/run.sh b/rsky-spaces-parity/layer2/run.sh new file mode 100755 index 00000000..703594b1 --- /dev/null +++ b/rsky-spaces-parity/layer2/run.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Layer 2 acceptance gate: build both real binaries, run the same record script +# at each over XRPC, then compare the two store files with the Layer 1 +# comparator. One command, no services beyond the two child processes. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +# The oracle is rsky-pds at this revision and nothing else. Its worktree is +# build-only: no edit ever lands there. +ORACLE_REV="${ORACLE_REV:-7ebd21ae788c550ee8510034d94eb19ede148738}" +ORACLE_TREE="${ORACLE_TREE:-/tmp/claude/rsky-layer2-oracle}" +RUN_DIR="${LAYER2_RUN_DIR:-$REPO_ROOT/target/layer2/run}" + +# sccache cannot open its cache in a sandboxed shell; the wrapper is only a +# build accelerator, so drop it rather than fail. +export RUSTC_WRAPPER="" +export CARGO_BUILD_RUSTC_WRAPPER="" + +say() { printf '\n== %s\n' "$1"; } + +say "oracle worktree at $ORACLE_REV" +if [ ! -d "$ORACLE_TREE/.git" ] && [ ! -f "$ORACLE_TREE/.git" ]; then + mkdir -p "$(dirname "$ORACLE_TREE")" + git worktree add --detach "$ORACLE_TREE" "$ORACLE_REV" +fi +HAVE="$(git -C "$ORACLE_TREE" rev-parse HEAD)" +if [ "$HAVE" != "$ORACLE_REV" ]; then + echo "oracle worktree is at $HAVE, expected $ORACLE_REV" >&2 + exit 1 +fi +if [ -n "$(git -C "$ORACLE_TREE" status --porcelain)" ]; then + echo "oracle worktree is dirty; it must stay build-only" >&2 + git -C "$ORACLE_TREE" status --short >&2 + exit 1 +fi + +say "building the pinned oracle pds" +( cd "$ORACLE_TREE" && cargo build -p rsky-pds --bin rsky-pds ) + +say "building the space host under test" +cargo build -p rsky-space-host --bin rsky-space-host + +say "building the gate" +cargo build -p rsky-spaces-parity --bin layer2-gate + +say "running the gate" +LAYER2_RUN_DIR="$RUN_DIR" \ +LAYER2_PDS_BIN="$ORACLE_TREE/target/debug/rsky-pds" \ +LAYER2_SHIM_BIN="$REPO_ROOT/target/debug/rsky-space-host" \ + ./target/debug/layer2-gate diff --git a/rsky-spaces-parity/resume-gate/README.md b/rsky-spaces-parity/resume-gate/README.md new file mode 100644 index 00000000..eeab1b79 --- /dev/null +++ b/rsky-spaces-parity/resume-gate/README.md @@ -0,0 +1,103 @@ +# Resume-across-swap gate + +Layer 1 and Layer 2 compare the converged space host against the rsky-pds +oracle. This gate asks a different question: when the converged host deploys +over data the legacy host wrote, does the **existing** syncer keep going? + +```sh +./rsky-spaces-parity/resume-gate/run.sh +``` + +Exit code 0 means every scored check passed. No Postgres, no Docker, no +network: three child processes at a time plus two loopback stubs the gate hosts +itself. + +## What it does + +1. Creates a **detached, build-only** git worktree at `6da61ce`, the + pre-convergence tip of `feat/space-host-main-port`, and builds + `rsky-space-host` there. The worktree is refused if it is dirty or at + another revision. +2. Builds the converged `rsky-space-host`, `convert_store` and `rsky-daemon` + from the working tree. **The daemon is not modified for this gate** — that + is the point of it. +3. Runs `resume-gate`, which drives three eras against one space-host database, + one actor-store directory, one host port and one daemon index: + + **Phase A — legacy era.** The legacy host serves the multi-tenant + `space_host.db`. Two posts are created over XRPC. The real daemon runs + against it, projects both to a capturing sink, and persists its cursor. + The daemon is then killed and three more writes land — two creates and a + delete — none of which it ever acknowledged. + + **Phase B — the swap.** `convert_store` converts `space_host.db` into + per-account `store.sqlite` files. The account signing key is placed beside + the new store, as the deploy leaves it. + + **Phase C — converged era.** The converged host starts on the same port, + the same host database and the converted stores. The same daemon restarts + with the same index, the same DPoP key and identical arguments. One further + write lands at a server-minted revision. + + Finally a **cold daemon** — fresh index, fresh key, its own sink — syncs the + converted store from scratch. + +Everything lands under `target/resume/run`: a log per process +(`shim-legacy`, `shim-converged`, `daemon-legacy`, `daemon-converged`, +`daemon-cold`), the host and daemon databases, and `report.txt`. The directory +is wiped at the start of every run. + +## What it asserts + +- The legacy-era daemon projects what it saw, once each, and persists a cursor. +- Writes landing after its last acknowledgement leave that cursor behind. +- The conversion carries oplog ids and revisions across verbatim, and leaves + `oplog_floor_rev` open so no `since` can be refused. +- On resume the daemon projects **exactly** the operations after its cursor, + once each — including the three it never saw before the swap — and re-projects + nothing. +- A revision minted by the converged host sorts after the carried legacy one, + and its write projects once. +- No `HistoryUnavailable`, no divergence, no full-state recovery, no `prev` + mismatch in either daemon log. This is the sharpest check: a broken cursor is + survivable by falling back to `getRepo`, which would hide the fault behind a + correct-looking end state. +- A cold daemon reaches the same records, revisions and LtHash digest, and the + same projected end state. + +## Credentials + +All local, all fixed, all created and destroyed inside the run directory; none +of it is a secret and none of it reaches a real service. + +- The gate acts as the authorization server for writes, holding the same HS256 + secret the host is configured with and signing DPoP-bound `at+jwt` tokens — + the same shape as Layer 2. +- The daemon mints its own space credential through `/admin/mintCredential` + with its own service identity and DPoP key, so the credential path is the + real one and is exercised twice, once per era. +- The account's actor-store key is written by the gate; the stub DID directory + publishes the matching `#atproto` multikey so the daemon verifies real + commit signatures. + +## Falsification + +Both eras green on an unmutated tree proves nothing on its own, so the gate was +run against two deliberate converter faults: + +| Mutation | Result | +|---|---| +| `oplog_floor_rev` set to the head revision instead of left open | 10/14 — the daemon took `HistoryUnavailable` → full-state recovery; the log check, the store check and both cold-sync checks went red | +| oplog row ids renumbered (`seq + 1000`) | 13/14 — only the conversion check went red | + +The second is a finding, not a gap: the daemon's durable cursor is a +**revision**, not an oplog row id. Row ids are only within-request paging +cursors. Preserving them is still correct — any other consumer may hold one — +but revision continuity is what the resume depends on. + +## Running outside a sandbox + +Same as Layer 2: on macOS `reqwest` proxy discovery calls SystemConfiguration, +which a restricted sandbox denies and the pinned `hyper-util` panics on. Run +with ordinary process permissions. `sccache` is likewise unusable there, so +`run.sh` clears `RUSTC_WRAPPER`. diff --git a/rsky-spaces-parity/resume-gate/report/2026-08-21.md b/rsky-spaces-parity/resume-gate/report/2026-08-21.md new file mode 100644 index 00000000..73c23203 --- /dev/null +++ b/rsky-spaces-parity/resume-gate/report/2026-08-21.md @@ -0,0 +1,106 @@ +# Resume-across-swap gate — run report, 2026-08-21 + +**Verdict: pass.** `parity: 14/14 checks equal (+0 documented divergence)`, +exit 0. The rsky-daemon binary was not modified; the gate verifies it as-is. + +``` +./rsky-spaces-parity/resume-gate/run.sh +``` + +Legacy era built from a detached build-only worktree at `6da61ce`, the +pre-convergence tip of `feat/space-host-main-port`. Converged host, converter +and daemon built from the working tree. + +## Scoreboard + +``` + [equal] legacy era: the daemon projects the writes it saw, once each + [equal] legacy era: the cursor is durable + [equal] legacy era: writes after the last ack leave the cursor behind + [equal] conversion: oplog ids and revisions survive verbatim + [equal] conversion: no history is placed out of reach + [equal] resume: exactly the operations after the cursor, once each + [equal] resume: nothing already projected is projected again + [equal] converged era: a server-minted revision follows the carried one + [equal] converged era: the new write projects once + [equal] resume: no cursor refusal, divergence or full-state recovery + [equal] legacy-era daemon log was also clean + [equal] cold sync: the same records, revisions and digest as the resumed daemon + [equal] cold sync: the same projected end state as the resumed daemon + [equal] whole run: no operation is projected twice to either destination + [note] oplog window was never the constraint + +parity: 14/14 checks equal (+0 documented divergence) +``` + +## What actually happened + +| Era | Event | Revision | +|---|---|---| +| legacy | create `post1`, create `post2` | `3mtmphzq…`, head `3mtmphzqzcnd6` | +| legacy | daemon syncs, projects both, persists cursor `3mtmphzqzcnd6` | | +| legacy | daemon killed | | +| legacy | create `post3`, create `post4`, delete `post2` | head `3mtmpi44g7nd6` | +| swap | `convert_store`: 1 account, 1 repo, 3 records, 5 ops; ids `[1,2,3,4,5]`, floor `NULL` | | +| converged | same daemon restarts on the same index; projects `post2:delete`, `post3:create`, `post4:create` and nothing else | | +| converged | create `post5` | `3mtmpi6ssuanm` | +| converged | cold daemon syncs from scratch to the same head and digest | | + +The three risk edges named in the design, as the gate found them: + +- **Oplog cursor numbering.** Preserved verbatim (`[1,2,3,4,5]`), confirmed from + the consumer side. But see the finding below: the daemon does not use these. +- **Server-minted revisions replacing caller-supplied ones.** Both eras mint + TIDs (legacy from a `Ticker`, converged from `TID::next_str(prev)`), and the + converged host seeds its first revision from the carried legacy head, so + `3mtmpi6ssuanm > 3mtmpi44g7nd6`. Every consumer comparison that depends on + revision ordering — `listRepoOps`'s `rev > since`, the daemon's + `projector_cursor`'s `j.rev > c.rev` — therefore holds across the swap. +- **`list_ops` refusing history beyond the oplog window.** The converter leaves + `oplog_floor_rev` open, so no `since` can be refused on a freshly converted + store. Not exercised under a full window: `DEFAULT_OPLOG_WINDOW` is a + compiled-in 10 000 rows with no configuration hook, so a small-window run is + not reachable through the binary. The gate asserts the floor invariant + instead and records the gap as a note. + +## Falsification + +Green on an unmutated tree is not evidence, so the gate was run against two +deliberate converter faults and reverted after each. + +| Mutation | Result | Which checks caught it | +|---|---|---| +| `oplog_floor_rev` set to the head revision instead of left open | 10/14, exit 1 | the conversion floor check; `resume: no cursor refusal…` (both daemons logged `history unavailable` → full-state recovery); both cold-sync checks | +| oplog row ids renumbered (`seq + 1000`) | 13/14, exit 1 | the conversion check only | + +## Findings + +**1. The daemon's durable cursor is a revision, not an oplog row id.** The +design note (`Design/spaces-storage-parity`, Blast radius) frames the deploy +risk as "oplog cursor numbering … the ids syncers hold". It is not: the daemon +persists `sync_state.rev` and passes it as `since`; oplog row ids appear only as +within-request paging cursors, discarded when a sync finishes. Renumbering every +id changed nothing about the resume. Preserving ids is still right — another +consumer may hold one, and the converter's guarantee is cheap — but the +property the deploy depends on is **revision continuity and ordering**, and that +is what should be stated in the cutover plan. + +**2. Full-state recovery masks a broken cursor.** Under the floor mutation the +warm daemon took `HistoryUnavailable`, fell back to `getRepo`, and still +projected exactly the three expected operations with no duplicates — the +end-state checks alone would have passed it. What exposed it was the log check +and the cold daemon, which lost `post2`'s deletion entirely (recovery +reconstructs current state, so a delete that happened before a cold start is +never an event). Any future resume check must keep asserting on the absence of +recovery, not only on the projected result. + +**3. No daemon change was needed.** Nothing in the swap required touching +`rsky-daemon`; the stop condition was never reached. + +## Environment + +macOS, sandbox off (`reqwest` proxy discovery calls SystemConfiguration, which a +restricted sandbox denies and the pinned `hyper-util` panics on). +`RUSTC_WRAPPER` cleared by `run.sh` because `sccache` cannot open its cache +there. Zero diff in `rsky-pds/`, `rsky-space/` and `rsky-daemon/`. Local only — +nothing pushed, no PR. diff --git a/rsky-spaces-parity/resume-gate/run.sh b/rsky-spaces-parity/resume-gate/run.sh new file mode 100755 index 00000000..53cbed26 --- /dev/null +++ b/rsky-spaces-parity/resume-gate/run.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Resume-across-swap gate: build the pre-convergence space host from a detached +# build-only worktree, run it with the real daemon, convert the store, then +# restart both the converged host and the same daemon and check what it +# projected. One command, no services beyond the child processes. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +# The legacy era is the pre-convergence tip of this branch and nothing else. +# Its worktree is build-only: no edit ever lands there. +LEGACY_REV="${LEGACY_REV:-6da61ce}" +LEGACY_TREE="${LEGACY_TREE:-/tmp/claude/rsky-resume-legacy}" +RUN_DIR="${RESUME_RUN_DIR:-$REPO_ROOT/target/resume/run}" + +# sccache cannot open its cache in a sandboxed shell; the wrapper is only a +# build accelerator, so drop it rather than fail. +export RUSTC_WRAPPER="" +export CARGO_BUILD_RUSTC_WRAPPER="" + +say() { printf '\n== %s\n' "$1"; } + +LEGACY_SHA="$(git rev-parse "$LEGACY_REV")" + +say "legacy worktree at $LEGACY_SHA" +if [ ! -d "$LEGACY_TREE/.git" ] && [ ! -f "$LEGACY_TREE/.git" ]; then + mkdir -p "$(dirname "$LEGACY_TREE")" + git worktree add --detach "$LEGACY_TREE" "$LEGACY_SHA" +fi +HAVE="$(git -C "$LEGACY_TREE" rev-parse HEAD)" +if [ "$HAVE" != "$LEGACY_SHA" ]; then + echo "legacy worktree is at $HAVE, expected $LEGACY_SHA" >&2 + exit 1 +fi +if [ -n "$(git -C "$LEGACY_TREE" status --porcelain)" ]; then + echo "legacy worktree is dirty; it must stay build-only" >&2 + git -C "$LEGACY_TREE" status --short >&2 + exit 1 +fi + +say "building the legacy space host" +( cd "$LEGACY_TREE" && cargo build -p rsky-space-host --bin rsky-space-host ) + +say "building the converged space host, the converter and the daemon" +cargo build -p rsky-space-host --bin rsky-space-host --bin convert_store +cargo build -p rsky-daemon --bin rsky-daemon + +say "building the gate" +cargo build -p rsky-spaces-parity --bin resume-gate + +say "running the gate" +RESUME_RUN_DIR="$RUN_DIR" \ +RESUME_LEGACY_SHIM_BIN="$LEGACY_TREE/target/debug/rsky-space-host" \ +RESUME_SHIM_BIN="$REPO_ROOT/target/debug/rsky-space-host" \ +RESUME_CONVERT_BIN="$REPO_ROOT/target/debug/convert_store" \ +RESUME_DAEMON_BIN="$REPO_ROOT/target/debug/rsky-daemon" \ + ./target/debug/resume-gate diff --git a/rsky-spaces-parity/src/bin/layer2_gate.rs b/rsky-spaces-parity/src/bin/layer2_gate.rs new file mode 100644 index 00000000..5e8aff49 --- /dev/null +++ b/rsky-spaces-parity/src/bin/layer2_gate.rs @@ -0,0 +1,921 @@ +//! Layer 2 acceptance gate: the same record script fired at two real servers +//! over XRPC, then the Layer 1 comparator pointed at the two store files. +//! +//! Run it through `rsky-spaces-parity/layer2/run.sh`, which builds both binaries +//! and passes their paths in. + +use anyhow::{bail, Context, Result}; +use rsky_spaces_parity::layer2::normalize::{self, Revs}; +use rsky_spaces_parity::layer2::process::{copy_tree, free_port, reset_stores, Server}; +use rsky_spaces_parity::layer2::{car, directory::Directory, tokens, Scoreboard, Verdict}; +use rsky_spaces_parity::{compare_tables, dump_tables, revs_are_well_formed}; +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +const AUTHOR_DID: &str = "did:plc:layer2writeraaaaaaaaaaa"; +const HANDLE_DOMAIN: &str = ".layer2.test"; +const HANDLE: &str = "writer.layer2.test"; +const PASSWORD: &str = "layer2-local-password"; +const ADMIN_PASS: &str = "layer2-local-admin"; +const SPACE_TYPE: &str = "community.blacksky.feed"; +const SPACE_SKEY: &str = "main"; +const COLLECTION: &str = "com.example.post"; +const OTHER_COLLECTION: &str = "com.example.note"; +const HS256_SECRET: &str = "layer2-local-authorization-server-secret"; +const OAUTH_ISSUER: &str = "http://localhost:0/oauth"; +const PDS_SERVICE_DID: &str = "did:web:localho.st"; +const DAEMON_DID: &str = "did:plc:layer2daemonaaaaaaaaaaa"; + +/// Fixed local key material. None of it protects anything: the whole stack is +/// created and destroyed inside one run directory. +const AUTHORITY_SPACE_KEY: &str = + "1111111111111111111111111111111111111111111111111111111111111111"; +const DAEMON_KEY: &str = "2222222222222222222222222222222222222222222222222222222222222222"; +const PDS_JWT_KEY: &str = "9d5907143471e8f0e8df0f8b9512a8c5377878ee767f18fcf961055ecfc071cd"; +const PDS_ROTATION_KEY: &str = "fb478b39dd2ddf84bef135dd60f90381903eefadbb9df4b18a2b9b174ae72582"; +const PDS_SIGNING_KEY: &str = "71cfcf4882a6cff494c3d0affadd3858eb3a5838e7b5e15170e696a590a4fa01"; + +struct Gate { + client: reqwest::Client, + pds_url: String, + shim_url: String, + space: String, + session: String, + pds_credential: String, + shim_credential: String, +} + +/// One step of the write script, applied identically to both servers. +enum Step { + Create { + collection: &'static str, + rkey: &'static str, + record: Value, + }, + Delete { + collection: &'static str, + rkey: &'static str, + }, +} + +impl Step { + fn nsid(&self) -> &'static str { + match self { + Step::Create { .. } => "com.atproto.space.createRecord", + Step::Delete { .. } => "com.atproto.space.deleteRecord", + } + } + + fn label(&self) -> String { + match self { + Step::Create { + collection, rkey, .. + } => format!("createRecord {collection}/{rkey}"), + Step::Delete { collection, rkey } => format!("deleteRecord {collection}/{rkey}"), + } + } + + fn body(&self, space: &str, repo: &str) -> Value { + match self { + Step::Create { + collection, + rkey, + record, + } => json!({ + "space": space, + "repo": repo, + "collection": collection, + "rkey": rkey, + "record": record, + }), + Step::Delete { collection, rkey } => json!({ + "space": space, + "repo": repo, + "collection": collection, + "rkey": rkey, + }), + } + } +} + +fn script() -> Vec { + vec![ + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + record: json!({"text": "first", "n": 1}), + }, + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa2", + record: json!({"text": "second", "n": 2}), + }, + Step::Create { + collection: OTHER_COLLECTION, + rkey: "3kaaaaaaaaaa3", + record: json!({"text": "other collection", "nested": {"a": [1, 2, 3], "b": true}}), + }, + Step::Create { + collection: COLLECTION, + rkey: "unicode.rkey_1~", + record: json!({"text": "\u{e9}\u{4e16}\u{754c}\u{1f600}", "empty": ""}), + }, + // A duplicate rkey: both sides must refuse it the same way. + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + record: json!({"text": "duplicate"}), + }, + Step::Delete { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + }, + // Delete then recreate the same key. + Step::Create { + collection: COLLECTION, + rkey: "3kaaaaaaaaaa1", + record: json!({"text": "recreated"}), + }, + // A delete of something absent: both sides must refuse it the same way. + Step::Delete { + collection: COLLECTION, + rkey: "3kmissingaaaa", + }, + Step::Delete { + collection: OTHER_COLLECTION, + rkey: "3kaaaaaaaaaa3", + }, + Step::Create { + collection: OTHER_COLLECTION, + rkey: "3kaaaaaaaaaa4", + record: json!({"text": "after the delete"}), + }, + ] +} + +impl Gate { + async fn post( + &self, + base: &str, + nsid: &str, + headers: Vec<(&str, String)>, + body: &Value, + ) -> Result<(u16, Value)> { + let url = format!("{base}/xrpc/{nsid}"); + let mut request = self.client.post(&url).json(body); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request + .send() + .await + .with_context(|| format!("POST {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + Ok(( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + )) + } + + async fn get( + &self, + base: &str, + nsid: &str, + query: &str, + headers: Vec<(&str, String)>, + ) -> Result<(u16, Value)> { + let url = format!("{base}/xrpc/{nsid}?{query}"); + let mut request = self.client.get(&url); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request.send().await.with_context(|| format!("GET {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + Ok(( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + )) + } + + async fn get_bytes( + &self, + base: &str, + nsid: &str, + query: &str, + headers: Vec<(&str, String)>, + ) -> Result<(u16, Vec)> { + let url = format!("{base}/xrpc/{nsid}?{query}"); + let mut request = self.client.get(&url); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request.send().await.with_context(|| format!("GET {url}"))?; + let status = response.status().as_u16(); + Ok((status, response.bytes().await?.to_vec())) + } + + /// Write auth: the PDS takes its own session token, the space host takes a + /// DPoP-bound access token from the authorization server it trusts. + fn pds_write_headers(&self) -> Vec<(&'static str, String)> { + vec![("authorization", format!("Bearer {}", self.session))] + } + + fn shim_write_headers(&self, nsid: &str) -> Vec<(&'static str, String)> { + let token = tokens::access_token(HS256_SECRET, OAUTH_ISSUER, PDS_SERVICE_DID, AUTHOR_DID); + let proof = tokens::dpop_proof( + "POST", + &format!("{}/xrpc/{nsid}", self.shim_url), + Some(&token), + ); + vec![("authorization", format!("DPoP {token}")), ("dpop", proof)] + } + + /// Read auth: a space credential presented under DPoP on both sides. + fn pds_read_headers(&self, nsid: &str) -> Vec<(&'static str, String)> { + let proof = tokens::dpop_proof( + "GET", + &format!("{}/xrpc/{nsid}", self.pds_url), + Some(&self.pds_credential), + ); + vec![ + ("authorization", format!("DPoP {}", self.pds_credential)), + ("dpop", proof), + ] + } + + fn shim_read_headers(&self, nsid: &str) -> Vec<(&'static str, String)> { + let proof = tokens::dpop_proof( + "GET", + &format!("{}/xrpc/{nsid}", self.shim_url), + Some(&self.shim_credential), + ); + vec![ + ("authorization", format!("DPoP {}", self.shim_credential)), + ("dpop", proof), + ] + } +} + +fn env_path(key: &str, fallback: &str) -> PathBuf { + PathBuf::from(std::env::var(key).unwrap_or_else(|_| fallback.to_string())) +} + +fn multibase_of(hex_key: &str) -> Result { + let signer = rsky_space_host::signing::Signer::from_hex(hex_key) + .map_err(|error| anyhow::anyhow!("signer: {error}"))?; + Ok(signer + .did_key() + .strip_prefix("did:key:") + .unwrap_or(signer.did_key()) + .to_string()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let run_dir = env_path("LAYER2_RUN_DIR", "target/layer2/run"); + let pds_bin = env_path("LAYER2_PDS_BIN", ""); + let shim_bin = env_path("LAYER2_SHIM_BIN", "target/debug/rsky-space-host"); + if !pds_bin.is_file() { + bail!("LAYER2_PDS_BIN must point at the pinned oracle binary (got {pds_bin:?})"); + } + if !shim_bin.is_file() { + bail!("LAYER2_SHIM_BIN must point at the space-host binary (got {shim_bin:?})"); + } + + if run_dir.exists() { + std::fs::remove_dir_all(&run_dir).context("clear run directory")?; + } + std::fs::create_dir_all(&run_dir)?; + let run_dir = run_dir.canonicalize()?; + let pds_dir = run_dir.join("pds"); + let shim_dir = run_dir.join("shim"); + let pds_actors = pds_dir.join("actors"); + let shim_actors = shim_dir.join("actors"); + let shim_stores = shim_dir.join("space-stores"); + for dir in [&pds_dir, &shim_dir, &pds_dir.join("blobs")] { + std::fs::create_dir_all(dir)?; + } + + let mut keys = BTreeMap::new(); + keys.insert(DAEMON_DID.to_string(), multibase_of(DAEMON_KEY)?); + let directory = Directory::start(keys, HANDLE.to_string())?; + + let pds_port = free_port()?; + let shim_port = free_port()?; + let pds_url = format!("http://localhost:{pds_port}"); + let shim_url = format!("http://127.0.0.1:{shim_port}"); + + let pds_env: Vec<(String, String)> = vec![ + ("ROCKET_ADDRESS", "127.0.0.1".to_string()), + ("ROCKET_PORT", pds_port.to_string()), + ("PDS_PORT", pds_port.to_string()), + ("PDS_HOSTNAME", "localhost".to_string()), + ("PDS_SERVICE_DID", PDS_SERVICE_DID.to_string()), + ("PDS_SERVICE_HANDLE_DOMAINS", HANDLE_DOMAIN.to_string()), + ("PDS_ADMIN_PASS", ADMIN_PASS.to_string()), + ("PDS_INVITE_REQUIRED", "false".to_string()), + ("PDS_DID_PLC_URL", directory.url()), + ("PDS_JWT_KEY_K256_PRIVATE_KEY_HEX", PDS_JWT_KEY.to_string()), + ( + "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX", + PDS_ROTATION_KEY.to_string(), + ), + ( + "PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX", + PDS_SIGNING_KEY.to_string(), + ), + ( + "PDS_ACCOUNT_DB_LOCATION", + pds_dir.join("account.sqlite").display().to_string(), + ), + ( + "PDS_SEQUENCER_DB_LOCATION", + pds_dir.join("sequencer.sqlite").display().to_string(), + ), + ( + "PDS_DID_CACHE_DB_LOCATION", + pds_dir.join("did_cache.sqlite").display().to_string(), + ), + ( + "PDS_ACTOR_STORE_DIRECTORY", + pds_actors.display().to_string(), + ), + ( + "PDS_BLOBSTORE_DISK_LOCATION", + pds_dir.join("blobs").display().to_string(), + ), + ("RUST_LOG", "warn".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect(); + + let mut pds = Server::spawn( + "oracle pds", + &pds_bin, + &run_dir, + &pds_env, + &run_dir.join("pds.log"), + )?; + pds.wait_ready(&format!("{pds_url}/xrpc/_health"), Duration::from_secs(60)) + .await?; + println!("oracle pds ready on {pds_url}"); + + let client = rsky_spaces_parity::layer2::http_client()?; + let admin = format!("Basic {}", base64_standard(&format!("admin:{ADMIN_PASS}"))); + + // Account, activation, session. + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.server.createAccount"), + vec![("authorization", admin.clone())], + &json!({ + "did": AUTHOR_DID, + "email": "writer@layer2.test", + "handle": HANDLE, + "password": PASSWORD, + }), + ) + .await?; + if status != 200 { + bail!("createAccount failed ({status}): {body}\n{}", pds.tail()); + } + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.admin.updateSubjectStatus"), + vec![("authorization", admin.clone())], + &json!({ + "subject": {"$type": "com.atproto.admin.defs#repoRef", "did": AUTHOR_DID}, + "deactivated": {"applied": false}, + }), + ) + .await?; + if status != 200 { + bail!("activation failed ({status}): {body}\n{}", pds.tail()); + } + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.server.createSession"), + vec![], + &json!({"identifier": HANDLE, "password": PASSWORD}), + ) + .await?; + if status != 200 { + bail!("createSession failed ({status}): {body}\n{}", pds.tail()); + } + let session = body["accessJwt"] + .as_str() + .context("session has no accessJwt")? + .to_string(); + + let (status, body) = post_raw( + &client, + &format!("{pds_url}/xrpc/com.atproto.simplespace.createSpace"), + vec![("authorization", format!("Bearer {session}"))], + &json!({"type": SPACE_TYPE, "skey": SPACE_SKEY}), + ) + .await?; + if status != 200 { + bail!("createSpace failed ({status}): {body}\n{}", pds.tail()); + } + let space = body["uri"] + .as_str() + .context("space has no uri")? + .to_string(); + let expected = format!("at://{AUTHOR_DID}/space/{SPACE_TYPE}/{SPACE_SKEY}"); + if space != expected { + bail!("space uri {space} is not the shared {expected}"); + } + println!("space created on the oracle: {space}"); + + // The space host reads account signing keys from a PDS-shaped actor store + // directory it never writes to, and writes its own per-account stores into + // a separate directory (Option A). Running both jobs off one path is the + // configuration that hid them being conflated, so the gate keeps them apart. + copy_tree(&pds_actors, &shim_actors)?; + let accounts = reset_stores(&shim_stores)?; + let keys_before = tree_digest(&shim_actors)?; + println!("space host store directory prepared for {accounts} account(s)"); + + let shim_env: Vec<(String, String)> = vec![ + ("SPACEHOST_BIND", format!("127.0.0.1:{shim_port}")), + ("SPACEHOST_PUBLIC_URL", shim_url.clone()), + ("SPACEHOST_AUTHORITY_DID", AUTHOR_DID.to_string()), + ("SPACEHOST_SIGNING_KEY_HEX", AUTHORITY_SPACE_KEY.to_string()), + ("SPACEHOST_POLICY", "public".to_string()), + ("SPACEHOST_PLC_URL", directory.url()), + ( + "SPACEHOST_DB_PATH", + shim_dir.join("space_host.db").display().to_string(), + ), + ( + "SPACEHOST_ACTOR_STORE_DIR", + shim_actors.display().to_string(), + ), + ( + "SPACEHOST_SPACE_STORE_DIR", + shim_stores.display().to_string(), + ), + ("SPACEHOST_OAUTH_ISSUER", OAUTH_ISSUER.to_string()), + ("SPACEHOST_OAUTH_JWKS_URI", format!("{OAUTH_ISSUER}/jwks")), + ("SPACEHOST_OAUTH_AUDIENCE", PDS_SERVICE_DID.to_string()), + ("SPACEHOST_OAUTH_CLIENT_IDS", tokens::CLIENT_ID.to_string()), + ("SPACEHOST_OAUTH_HS256_SECRET", HS256_SECRET.to_string()), + ("SPACEHOST_MINT_TOKEN", "layer2-mint-token".to_string()), + ("SPACEHOST_DAEMON_SERVICE_DID", DAEMON_DID.to_string()), + ("SPACEHOST_APPVIEW_SERVICE_DID", DAEMON_DID.to_string()), + ("RUST_LOG", "warn".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect(); + + let mut shim = Server::spawn( + "space host", + &shim_bin, + &run_dir, + &shim_env, + &run_dir.join("shim.log"), + )?; + shim.wait_ready(&format!("{shim_url}/xrpc/_health"), Duration::from_secs(30)) + .await?; + println!("space host ready on {shim_url}"); + + let mut board = Scoreboard::default(); + + // Credentials for the read endpoints, minted by each side's own authority. + let pds_credential = mint_pds_credential(&client, &pds_url, &session, &space).await?; + let shim_credential = mint_shim_credential(&client, &shim_url, &space).await?; + println!("space credentials minted on both sides"); + + let gate = Gate { + client, + pds_url: pds_url.clone(), + shim_url: shim_url.clone(), + space: space.clone(), + session, + pds_credential, + shim_credential, + }; + + run_write_script(&gate, &mut board).await?; + compare_reads(&gate, &mut board).await?; + probe_pds_only_surface(&gate, &mut board).await?; + + // Both servers hold their stores open in WAL mode; stop them before the + // stored-row comparison so the files are quiescent. + shim.stop(); + pds.stop(); + + compare_stores(&pds_actors, &shim_stores, &mut board)?; + board.equal_if( + "key directory untouched", + tree_digest(&shim_actors)? == keys_before, + shim_actors.display().to_string(), + ); + + let report = board.render(); + print!("{report}"); + std::fs::write(run_dir.join("report.txt"), &report)?; + println!("\nlogs: {}", run_dir.display()); + + if board.failures() > 0 { + bail!("layer 2 gate failed: {} check(s) differ", board.failures()); + } + Ok(()) +} + +fn base64_standard(text: &str) -> String { + use base64::engine::general_purpose::STANDARD; + use base64::Engine; + STANDARD.encode(text.as_bytes()) +} + +async fn post_raw( + client: &reqwest::Client, + url: &str, + headers: Vec<(&str, String)>, + body: &Value, +) -> Result<(u16, Value)> { + let mut request = client.post(url).json(body); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request + .send() + .await + .with_context(|| format!("POST {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + Ok(( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + )) +} + +/// getDelegationToken then getSpaceCredential, the flow a member's own PDS runs. +async fn mint_pds_credential( + client: &reqwest::Client, + pds_url: &str, + session: &str, + space: &str, +) -> Result { + let nsid = "com.atproto.space.getDelegationToken"; + let url = format!("{pds_url}/xrpc/{nsid}?space={space}"); + let response = client + .get(&url) + .header("authorization", format!("Bearer {session}")) + .send() + .await?; + let status = response.status().as_u16(); + let body: Value = serde_json::from_str(&response.text().await?).unwrap_or(Value::Null); + if status != 200 { + bail!("getDelegationToken failed ({status}): {body}"); + } + let delegation = body["token"] + .as_str() + .context("delegation response has no token")? + .to_string(); + + let nsid = "com.atproto.space.getSpaceCredential"; + let url = format!("{pds_url}/xrpc/{nsid}"); + let response = client + .post(&url) + .header("authorization", format!("Bearer {delegation}")) + .header("dpop", tokens::dpop_proof("POST", &url, None)) + .json(&json!({"space": space})) + .send() + .await?; + let status = response.status().as_u16(); + let body: Value = serde_json::from_str(&response.text().await?).unwrap_or(Value::Null); + if status != 200 { + bail!("getSpaceCredential failed ({status}): {body}"); + } + Ok(body["credential"] + .as_str() + .context("credential response has no credential")? + .to_string()) +} + +/// The space host's administrative mint, which stands in for a syncer asking +/// for a credential. +async fn mint_shim_credential( + client: &reqwest::Client, + shim_url: &str, + space: &str, +) -> Result { + let signer = rsky_space_host::signing::Signer::from_hex(DAEMON_KEY) + .map_err(|error| anyhow::anyhow!("daemon signer: {error}"))?; + let jwt = tokens::service_jwt( + &signer, + DAEMON_DID, + AUTHOR_DID, + "community.blacksky.space.mintCredential", + )?; + let url = format!("{shim_url}/admin/mintCredential"); + let response = client + .post(format!("{url}?space={space}")) + .header("authorization", format!("Bearer {jwt}")) + .header("x-spacehost-mint-token", "layer2-mint-token") + .header("dpop", tokens::dpop_proof("POST", &url, None)) + .send() + .await?; + let status = response.status().as_u16(); + let body: Value = serde_json::from_str(&response.text().await?).unwrap_or(Value::Null); + if status != 200 { + bail!("mintCredential failed ({status}): {body}"); + } + Ok(body["credential"] + .as_str() + .context("mint response has no credential")? + .to_string()) +} + +async fn run_write_script(gate: &Gate, board: &mut Scoreboard) -> Result<()> { + let mut shim_revs = Revs::default(); + let mut pds_revs = Revs::default(); + for (index, step) in script().into_iter().enumerate() { + let nsid = step.nsid(); + let body = step.body(&gate.space, AUTHOR_DID); + let (shim_status, shim_body) = gate + .post(&gate.shim_url, nsid, gate.shim_write_headers(nsid), &body) + .await?; + let (pds_status, pds_body) = gate + .post(&gate.pds_url, nsid, gate.pds_write_headers(), &body) + .await?; + + let name = format!("write {:02} {}", index + 1, step.label()); + let shim_view = write_view(shim_status, &shim_body, &mut shim_revs); + let pds_view = write_view(pds_status, &pds_body, &mut pds_revs); + board.equal_if( + name, + shim_view == pds_view, + if shim_view == pds_view { + shim_view.clone() + } else { + format!("shim: {shim_view}\npds: {pds_view}") + }, + ); + } + board.push( + "write revisions are TIDs", + if normalize::revs_are_tids(&shim_revs) && normalize::revs_are_tids(&pds_revs) { + Verdict::Equal + } else { + Verdict::Differs + }, + format!( + "shim minted {} revisions, pds minted {}", + shim_revs.count(), + pds_revs.count() + ), + ); + Ok(()) +} + +/// A write's comparable outcome: HTTP status, the XRPC error name when it +/// failed, and the record identity when it succeeded. +fn write_view(status: u16, body: &Value, revs: &mut Revs) -> String { + if status != 200 { + return format!( + "{status} {}", + body["error"].as_str().unwrap_or("") + ); + } + let normalized = normalize::normalize(body, revs); + let uri = normalized["uri"].as_str().unwrap_or("-").to_string(); + let cid = normalized["cid"].as_str().unwrap_or("-").to_string(); + let commit = &normalized["commit"]; + format!( + "200 uri={uri} cid={cid} rev={} hash={}", + commit["rev"].as_str().unwrap_or("-"), + commit["hash"].as_str().unwrap_or("-"), + ) +} + +async fn compare_reads(gate: &Gate, board: &mut Scoreboard) -> Result<()> { + let space = gate.space.clone(); + + for (nsid, query) in [ + ( + "com.atproto.space.listRepoOps", + format!("space={space}&repo={AUTHOR_DID}"), + ), + ( + "com.atproto.space.listRepoOps", + format!("space={space}&repo={AUTHOR_DID}&limit=2"), + ), + ( + "com.atproto.space.getLatestCommit", + format!("space={space}&repo={AUTHOR_DID}"), + ), + ("com.atproto.space.listRepos", format!("space={space}")), + ("com.atproto.space.getSpace", format!("space={space}")), + ] { + let (shim_status, shim_body) = gate + .get(&gate.shim_url, nsid, &query, gate.shim_read_headers(nsid)) + .await?; + let (pds_status, pds_body) = gate + .get(&gate.pds_url, nsid, &query, gate.pds_read_headers(nsid)) + .await?; + let mut shim_revs = Revs::default(); + let mut pds_revs = Revs::default(); + let shim_view = normalize::render(&normalize::normalize(&shim_body, &mut shim_revs)); + let pds_view = normalize::render(&normalize::normalize(&pds_body, &mut pds_revs)); + let label = match query.contains("limit=") { + true => format!("read {nsid} (paged)"), + false => format!("read {nsid}"), + }; + let equal = shim_status == pds_status && shim_view == pds_view; + // `getSpace` answers from the space definition, which the storage + // convergence deliberately leaves out of scope: the space host holds it + // in configuration, the PDS in its own `space_def` rows. + if nsid == "com.atproto.space.getSpace" && !equal { + board.push( + label, + Verdict::Documented, + format!( + "space definition is configured on the host and stored on the pds\n\ + shim: {shim_status} {shim_view}\npds: {pds_status} {pds_view}" + ), + ); + continue; + } + board.equal_if( + label, + equal, + if equal { + format!("{shim_status}") + } else { + format!("shim: {shim_status} {shim_view}\npds: {pds_status} {pds_view}") + }, + ); + } + + // getRepo returns a CAR whose commit block carries fresh random key + // material, so the record blocks are what can match. + let nsid = "com.atproto.space.getRepo"; + let query = format!("space={space}&repo={AUTHOR_DID}"); + let (shim_status, shim_car) = gate + .get_bytes(&gate.shim_url, nsid, &query, gate.shim_read_headers(nsid)) + .await?; + let (pds_status, pds_car) = gate + .get_bytes(&gate.pds_url, nsid, &query, gate.pds_read_headers(nsid)) + .await?; + if shim_status != 200 || pds_status != 200 { + board.equal_if( + format!("read {nsid}"), + false, + format!("shim: {shim_status}, pds: {pds_status}"), + ); + return Ok(()); + } + let diff = car::diff(&shim_car, &pds_car)?; + let commit_only = diff.only_shim.len() == 1 && diff.only_pds.len() == 1; + board.equal_if( + format!("read {nsid} record blocks"), + commit_only && diff.shared > 0, + format!( + "{} shared blocks; unique to shim: {:?}; unique to pds: {:?}", + diff.shared, diff.only_shim, diff.only_pds + ), + ); + Ok(()) +} + +/// The four methods the PDS serves and the space host does not. Recorded as a +/// surface difference: a client moving from the host to the PDS gains them. +async fn probe_pds_only_surface(gate: &Gate, board: &mut Scoreboard) -> Result<()> { + let space = gate.space.clone(); + let probes: Vec<(&str, &str, String, Option)> = vec![ + ( + "com.atproto.space.getRecord", + "GET", + format!("space={space}&repo={AUTHOR_DID}&collection={COLLECTION}&rkey=3kaaaaaaaaaa2"), + None, + ), + ( + "com.atproto.space.listRecords", + "GET", + format!("space={space}&repo={AUTHOR_DID}"), + None, + ), + ( + "com.atproto.space.getRepoState", + "GET", + format!("space={space}&repo={AUTHOR_DID}"), + None, + ), + ( + "com.atproto.space.applyWrites", + "POST", + String::new(), + // Deliberately invalid, so the probe cannot write to one side only: + // a routed method rejects it, an unrouted one is not found. + Some(json!({ + "space": space, + "repo": AUTHOR_DID, + "writes": [{ + "action": "not-an-action", + "collection": COLLECTION, + "rkey": "3kprobeaaaaaa", + }], + })), + ), + ]; + for (nsid, method, query, body) in probes { + let (pds_status, shim_status) = if method == "GET" { + let (pds_status, _) = gate + .get(&gate.pds_url, nsid, &query, gate.pds_read_headers(nsid)) + .await?; + let (shim_status, _) = gate + .get(&gate.shim_url, nsid, &query, gate.shim_read_headers(nsid)) + .await?; + (pds_status, shim_status) + } else { + let body = body.unwrap_or(Value::Null); + let (pds_status, _) = gate + .post(&gate.pds_url, nsid, gate.pds_write_headers(), &body) + .await?; + let (shim_status, _) = gate + .post(&gate.shim_url, nsid, gate.shim_write_headers(nsid), &body) + .await?; + (pds_status, shim_status) + }; + board.push( + format!("surface {nsid}"), + Verdict::Note, + format!("pds {pds_status}, space host {shim_status} (not routed)"), + ); + } + Ok(()) +} + +/// A content fingerprint of every file under `root`, so the gate can prove the +/// key directory is never written to. +fn tree_digest(root: &Path) -> Result> { + let mut out = BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + if !dir.exists() { + continue; + } + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_dir() { + stack.push(path); + continue; + } + let relative = path + .strip_prefix(root) + .unwrap_or(&path) + .display() + .to_string(); + let bytes = std::fs::read(&path)?; + let mut hash = 1469598103934665603u64; + for byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(1099511628211); + } + out.insert(relative, hash); + } + } + Ok(out) +} + +fn compare_stores(pds_actors: &Path, shim_actors: &Path, board: &mut Scoreboard) -> Result<()> { + let pds_store = rsky_space_host::actor_repos::store_path(pds_actors, AUTHOR_DID) + .map_err(|error| anyhow::anyhow!("pds store path: {error}"))?; + let shim_store = rsky_space_host::actor_repos::store_path(shim_actors, AUTHOR_DID) + .map_err(|error| anyhow::anyhow!("shim store path: {error}"))?; + for path in [&pds_store, &shim_store] { + if !path.exists() { + bail!("no store file at {}", path.display()); + } + } + let shim = dump_tables(&shim_store); + let pds = dump_tables(&pds_store); + let equal = compare_tables("store", &shim, &pds); + board.equal_if( + "stored space_* rows", + equal, + format!("{} / {}", shim_store.display(), pds_store.display()), + ); + let sound = + revs_are_well_formed("store", &shim, "shim") && revs_are_well_formed("store", &pds, "pds"); + board.equal_if( + "stored revisions well formed", + sound, + format!( + "shim {} distinct revisions, pds {}", + shim.revs.len(), + pds.revs.len() + ), + ); + Ok(()) +} diff --git a/rsky-spaces-parity/src/bin/resume_gate.rs b/rsky-spaces-parity/src/bin/resume_gate.rs new file mode 100644 index 00000000..bb9aaa98 --- /dev/null +++ b/rsky-spaces-parity/src/bin/resume_gate.rs @@ -0,0 +1,664 @@ +//! Resume-across-swap gate: the upgrade the storage convergence ships, +//! simulated end to end with real processes. +//! +//! Three eras run against one space-host database and one daemon index: +//! the legacy multi-tenant store, the one-off conversion, and the converged +//! per-account stores. The daemon keeps its durable cursors across all three, +//! and the gate asserts what it projected on the far side. +//! +//! Run it through `rsky-spaces-parity/resume-gate/run.sh`, which builds every +//! binary involved and passes their paths in. + +use anyhow::{bail, Context, Result}; +use rsky_spaces_parity::layer2::process::{free_port, Server}; +use rsky_spaces_parity::layer2::{directory::Directory, tokens, Scoreboard, Verdict}; +use rsky_spaces_parity::resume::sink::Sink; +use rsky_spaces_parity::resume::{ + converged_head, converged_oplog, duplicates, final_state, labels, legacy_oplog, read_index, + unclean_resume_lines, +}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +const AUTHOR_DID: &str = "did:plc:resumewriteraaaaaaaaaa"; +const DAEMON_DID: &str = "did:plc:resumedaemonaaaaaaaaaa"; +const HANDLE: &str = "writer.resume.test"; +const SPACE_TYPE: &str = "community.blacksky.feed"; +const SPACE_SKEY: &str = "main"; +const COLLECTION: &str = "app.bsky.feed.post"; +const HS256_SECRET: &str = "resume-local-authorization-server-secret"; +const OAUTH_ISSUER: &str = "http://localhost:0/oauth"; +const OAUTH_AUDIENCE: &str = "did:web:localho.st"; +const MINT_TOKEN: &str = "resume-mint-token"; + +/// Fixed local key material. None of it protects anything: the whole stack is +/// created and destroyed inside one run directory. +const AUTHOR_KEY: &str = "3333333333333333333333333333333333333333333333333333333333333333"; +const DAEMON_KEY: &str = "2222222222222222222222222222222222222222222222222222222222222222"; + +/// The record keys written, in the order the script writes them. +const POST_1: &str = "3kresumepost1"; +const POST_2: &str = "3kresumepost2"; +const POST_3: &str = "3kresumepost3"; +const POST_4: &str = "3kresumepost4"; +const POST_5: &str = "3kresumepost5"; + +const READY: Duration = Duration::from_secs(30); +const SETTLE: Duration = Duration::from_secs(45); + +struct Writer { + client: reqwest::Client, + shim_url: String, + space: String, +} + +impl Writer { + /// Create a record as the author, over XRPC, exactly as a client does. + async fn create(&self, rkey: &str, text: &str) -> Result { + let body = json!({ + "space": self.space, + "repo": AUTHOR_DID, + "collection": COLLECTION, + "rkey": rkey, + "record": { + "$type": COLLECTION, + "text": text, + "createdAt": "2026-08-21T00:00:00.000Z", + }, + }); + self.write("com.atproto.space.createRecord", &body).await + } + + async fn delete(&self, rkey: &str) -> Result { + let body = json!({ + "space": self.space, + "repo": AUTHOR_DID, + "collection": COLLECTION, + "rkey": rkey, + }); + self.write("com.atproto.space.deleteRecord", &body).await + } + + /// Returns the revision the host committed the write at. + async fn write(&self, nsid: &str, body: &Value) -> Result { + let url = format!("{}/xrpc/{nsid}", self.shim_url); + let token = tokens::access_token(HS256_SECRET, OAUTH_ISSUER, OAUTH_AUDIENCE, AUTHOR_DID); + let proof = tokens::dpop_proof("POST", &url, Some(&token)); + let response = self + .client + .post(&url) + .header("authorization", format!("DPoP {token}")) + .header("dpop", proof) + .json(body) + .send() + .await + .with_context(|| format!("POST {url}"))?; + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + let parsed: Value = serde_json::from_str(&text).unwrap_or(Value::String(text.clone())); + if status != 200 { + bail!("{nsid} returned {status}: {parsed}"); + } + Ok(parsed["commit"]["rev"] + .as_str() + .with_context(|| format!("{nsid} response has no commit revision: {parsed}"))? + .to_string()) + } +} + +fn env_path(key: &str, fallback: &str) -> PathBuf { + PathBuf::from(std::env::var(key).unwrap_or_else(|_| fallback.to_string())) +} + +fn multibase_of(hex_key: &str) -> Result { + let signer = rsky_space_host::signing::Signer::from_hex(hex_key) + .map_err(|error| anyhow::anyhow!("signer: {error}"))?; + Ok(signer + .did_key() + .strip_prefix("did:key:") + .unwrap_or(signer.did_key()) + .to_string()) +} + +/// The PDS actor-store layout both eras of the space host read signing keys +/// from: `{root}/{sha256(did)[..2]}/{did}/{key,store.sqlite}`. +fn seed_actor_store(root: &Path, did: &str, key_hex: &str) -> Result { + let digest = hex::encode(Sha256::digest(did.as_bytes())); + let account = root.join(&digest[..2]).join(did); + std::fs::create_dir_all(&account)?; + let key = hex::decode(key_hex).context("actor key is not hex")?; + std::fs::write(account.join("key"), key)?; + let store = account.join("store.sqlite"); + rsky_space_host::actor_schema::get_migrated_db(&store) + .map_err(|error| anyhow::anyhow!("migrate empty store: {error}"))?; + Ok(store) +} + +/// `space_stores` is `None` for the legacy era, which predates the split of +/// the key source from the space-store write target. +fn shim_env( + port: u16, + public_url: &str, + db_path: &Path, + actors: &Path, + space_stores: Option<&Path>, + plc_url: &str, +) -> Vec<(String, String)> { + let mut env = vec![ + ("SPACEHOST_BIND", format!("127.0.0.1:{port}")), + ("SPACEHOST_PUBLIC_URL", public_url.to_string()), + ("SPACEHOST_AUTHORITY_DID", AUTHOR_DID.to_string()), + ("SPACEHOST_SIGNING_KEY_HEX", AUTHOR_KEY.to_string()), + ("SPACEHOST_POLICY", "public".to_string()), + ("SPACEHOST_PLC_URL", plc_url.to_string()), + ("SPACEHOST_DB_PATH", db_path.display().to_string()), + ("SPACEHOST_ACTOR_STORE_DIR", actors.display().to_string()), + ("SPACEHOST_OAUTH_ISSUER", OAUTH_ISSUER.to_string()), + ("SPACEHOST_OAUTH_JWKS_URI", format!("{OAUTH_ISSUER}/jwks")), + ("SPACEHOST_OAUTH_AUDIENCE", OAUTH_AUDIENCE.to_string()), + ("SPACEHOST_OAUTH_CLIENT_IDS", tokens::CLIENT_ID.to_string()), + ("SPACEHOST_OAUTH_HS256_SECRET", HS256_SECRET.to_string()), + ("SPACEHOST_MINT_TOKEN", MINT_TOKEN.to_string()), + ("SPACEHOST_DAEMON_SERVICE_DID", DAEMON_DID.to_string()), + ("SPACEHOST_APPVIEW_SERVICE_DID", DAEMON_DID.to_string()), + ("RUST_LOG", "warn".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect::>(); + if let Some(stores) = space_stores { + env.push(( + "SPACEHOST_SPACE_STORE_DIR".to_string(), + stores.display().to_string(), + )); + } + env +} + +struct DaemonSetup { + index_db: PathBuf, + dpop_key: PathBuf, + notify_port: u16, + sink_url: String, +} + +fn daemon_env( + space: &str, + shim_url: &str, + setup: &DaemonSetup, + plc_url: &str, +) -> Vec<(String, String)> { + vec![ + ("DAEMON_SPACE_URI", space.to_string()), + ("DAEMON_SPACE_HOST_URL", shim_url.to_string()), + ("DAEMON_SERVICE_IDENTITY", DAEMON_DID.to_string()), + ("DAEMON_SERVICE_SIGNING_KEY_HEX", DAEMON_KEY.to_string()), + ("DAEMON_SPACE_HOST_MINT_TOKEN", MINT_TOKEN.to_string()), + ("DAEMON_DPOP_KEY_PATH", setup.dpop_key.display().to_string()), + ("DAEMON_INDEX_DB_PATH", setup.index_db.display().to_string()), + ( + "DAEMON_NOTIFY_BIND", + format!("127.0.0.1:{}", setup.notify_port), + ), + ("DAEMON_SWEEP_INTERVAL_SECS", "1".to_string()), + ("DAEMON_PLC_URL", plc_url.to_string()), + ("DAEMON_FEEDS_URL", setup.sink_url.clone()), + ("DAEMON_FEEDS_SERVICE_DID", DAEMON_DID.to_string()), + ("RUST_LOG", "info".to_string()), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect() +} + +/// Poll `check` until it holds, or fail with what was last seen. +async fn wait_until(what: &str, timeout: Duration, mut check: F) -> Result<()> +where + F: FnMut() -> Result>, +{ + let deadline = Instant::now() + timeout; + loop { + let Some(state) = check()? else { + return Ok(()); + }; + if Instant::now() >= deadline { + bail!("timed out waiting for {what}; last seen: {state}"); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let run_dir = env_path("RESUME_RUN_DIR", "target/resume/run"); + let legacy_bin = env_path("RESUME_LEGACY_SHIM_BIN", ""); + let shim_bin = env_path("RESUME_SHIM_BIN", "target/debug/rsky-space-host"); + let daemon_bin = env_path("RESUME_DAEMON_BIN", "target/debug/rsky-daemon"); + let convert_bin = env_path("RESUME_CONVERT_BIN", "target/debug/convert_store"); + for (label, path) in [ + ("RESUME_LEGACY_SHIM_BIN", &legacy_bin), + ("RESUME_SHIM_BIN", &shim_bin), + ("RESUME_DAEMON_BIN", &daemon_bin), + ("RESUME_CONVERT_BIN", &convert_bin), + ] { + if !path.is_file() { + bail!("{label} must point at a built binary (got {path:?})"); + } + } + + if run_dir.exists() { + std::fs::remove_dir_all(&run_dir).context("clear run directory")?; + } + std::fs::create_dir_all(&run_dir)?; + let run_dir = run_dir.canonicalize()?; + let host_dir = run_dir.join("host"); + let legacy_actors = host_dir.join("actors-legacy"); + let converged_actors = host_dir.join("actors"); + let host_db = host_dir.join("space_host.db"); + std::fs::create_dir_all(&legacy_actors)?; + std::fs::create_dir_all(&converged_actors)?; + + // The legacy era keeps its repos in the multi-tenant `space_host.db`; the + // actor store beside it holds nothing but the account's signing key. + seed_actor_store(&legacy_actors, AUTHOR_DID, AUTHOR_KEY)?; + + let mut keys = BTreeMap::new(); + keys.insert(AUTHOR_DID.to_string(), multibase_of(AUTHOR_KEY)?); + keys.insert(DAEMON_DID.to_string(), multibase_of(DAEMON_KEY)?); + let directory = Directory::start(keys, HANDLE.to_string())?; + + let shim_port = free_port()?; + let shim_url = format!("http://127.0.0.1:{shim_port}"); + let space = format!("at://{AUTHOR_DID}/space/{SPACE_TYPE}/{SPACE_SKEY}"); + + let warm_sink = Sink::start()?; + let warm = DaemonSetup { + index_db: run_dir.join("daemon/index.sqlite"), + dpop_key: run_dir.join("daemon/dpop.json"), + notify_port: free_port()?, + sink_url: warm_sink.url(), + }; + std::fs::create_dir_all(run_dir.join("daemon"))?; + + let mut board = Scoreboard::default(); + let client = rsky_spaces_parity::layer2::http_client()?; + + // ---- Phase A: the legacy era ------------------------------------------ + let mut legacy = Server::spawn( + "legacy space host", + &legacy_bin, + &run_dir, + &shim_env( + shim_port, + &shim_url, + &host_db, + &legacy_actors, + None, + &directory.url(), + ), + &run_dir.join("shim-legacy.log"), + )?; + legacy + .wait_ready(&format!("{shim_url}/xrpc/_health"), READY) + .await?; + println!("legacy space host ready on {shim_url}"); + + let writer = Writer { + client: client.clone(), + shim_url: shim_url.clone(), + space: space.clone(), + }; + let rev_1 = writer.create(POST_1, "legacy one").await?; + let rev_2 = writer.create(POST_2, "legacy two").await?; + println!("legacy writes committed at {rev_1}, {rev_2}"); + + let mut daemon = Server::spawn( + "daemon (legacy era)", + &daemon_bin, + &run_dir, + &daemon_env(&space, &shim_url, &warm, &directory.url()), + &run_dir.join("daemon-legacy.log"), + )?; + daemon.wait_log("daemon starting", READY).await?; + println!("daemon running against the legacy store"); + + // The daemon has caught up when its durable cursor is the head revision + // and the feeds projector has confirmed that revision. + let index_db = warm.index_db.clone(); + { + let (space, rev_2) = (space.clone(), rev_2.clone()); + let index_db = index_db.clone(); + wait_until( + "the daemon to persist its legacy-era cursor", + SETTLE, + move || { + let snapshot = read_index(&index_db, &space, AUTHOR_DID)?; + let acked = snapshot.head_rev.as_deref() == Some(rev_2.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_2.as_str()); + Ok((!acked).then(|| format!("{snapshot:?}"))) + }, + ) + .await?; + } + let legacy_projected = warm_sink.drain(); + let before_swap = read_index(&index_db, &space, AUTHOR_DID)?; + println!( + "daemon cursor persisted at {:?} after projecting {:?}", + before_swap.head_rev, + labels(&legacy_projected) + ); + + board.equal_if( + "legacy era: the daemon projects the writes it saw, once each", + labels(&legacy_projected) + == vec![post_label(POST_1, "create"), post_label(POST_2, "create")] + && duplicates(&legacy_projected).is_empty(), + format!("{:?}", labels(&legacy_projected)), + ); + board.equal_if( + "legacy era: the cursor is durable", + before_swap.head_rev.as_deref() == Some(rev_2.as_str()), + format!("cursor {:?}, head write {rev_2}", before_swap.head_rev), + ); + + // Stop the daemon mid-stream, then land writes it never acknowledged. + daemon.stop(); + let legacy_daemon_log = std::fs::read_to_string(&daemon.log).unwrap_or_default(); + println!("daemon stopped; landing writes it will never have seen"); + let rev_3 = writer.create(POST_3, "legacy three, unseen").await?; + let rev_4 = writer.create(POST_4, "legacy four, unseen").await?; + let rev_5 = writer.delete(POST_2).await?; + println!("unacknowledged legacy writes at {rev_3}, {rev_4}, {rev_5}"); + + let still = read_index(&index_db, &space, AUTHOR_DID)?; + board.equal_if( + "legacy era: writes after the last ack leave the cursor behind", + still.head_rev.as_deref() == Some(rev_2.as_str()) && rev_5 > rev_2, + format!("cursor {:?}, latest write {rev_5}", still.head_rev), + ); + + legacy.stop(); + println!("legacy space host stopped"); + + // ---- Phase B: the swap ------------------------------------------------ + let legacy_ops = legacy_oplog(&host_db, &space, AUTHOR_DID)?; + let convert = std::process::Command::new(&convert_bin) + .arg("--from") + .arg(&host_db) + .arg("--into") + .arg(&converged_actors) + .output() + .context("run the store converter")?; + if !convert.status.success() { + bail!( + "conversion failed: {}{}", + String::from_utf8_lossy(&convert.stdout), + String::from_utf8_lossy(&convert.stderr) + ); + } + let conversion = String::from_utf8_lossy(&convert.stdout).trim().to_string(); + println!("conversion: {conversion}"); + // The converted stores need the account key beside them, exactly as the + // deploy leaves it: the same actor-store directory, new store files. + let key_source = legacy_actors + .join(&hex::encode(Sha256::digest(AUTHOR_DID.as_bytes()))[..2]) + .join(AUTHOR_DID) + .join("key"); + let store = rsky_space_host::actor_repos::store_path(&converged_actors, AUTHOR_DID) + .map_err(|error| anyhow::anyhow!("store path: {error}"))?; + std::fs::copy( + &key_source, + store.parent().context("store has no parent")?.join("key"), + )?; + + let converged_ops = converged_oplog(&store, &space)?; + board.equal_if( + "conversion: oplog ids and revisions survive verbatim", + converged_ops == legacy_ops && !legacy_ops.is_empty(), + format!( + "{} legacy ops -> {} converged ops; ids {:?}", + legacy_ops.len(), + converged_ops.len(), + converged_ops.iter().map(|op| op.0).collect::>() + ), + ); + let (converged_rev, floor) = converged_head(&store, &space)?; + board.equal_if( + "conversion: no history is placed out of reach", + floor.is_none() && converged_rev == rev_5, + format!("head {converged_rev}, oplog floor {floor:?}"), + ); + + // ---- Phase C: the converged era --------------------------------------- + let mut converged = Server::spawn( + "converged space host", + &shim_bin, + &run_dir, + &shim_env( + shim_port, + &shim_url, + &host_db, + &legacy_actors, + Some(&converged_actors), + &directory.url(), + ), + &run_dir.join("shim-converged.log"), + )?; + converged + .wait_ready(&format!("{shim_url}/xrpc/_health"), READY) + .await?; + println!("converged space host ready on {shim_url}"); + + // The same daemon, the same index, the same DPoP key, the same arguments. + let mut daemon = Server::spawn( + "daemon (converged era)", + &daemon_bin, + &run_dir, + &daemon_env(&space, &shim_url, &warm, &directory.url()), + &run_dir.join("daemon-converged.log"), + )?; + daemon.wait_log("daemon starting", READY).await?; + println!("daemon restarted with its existing cursors"); + + { + let (space, rev_5) = (space.clone(), rev_5.clone()); + let index_db = index_db.clone(); + wait_until( + "the daemon to resume to the pre-swap head", + SETTLE, + move || { + let snapshot = read_index(&index_db, &space, AUTHOR_DID)?; + let caught_up = snapshot.head_rev.as_deref() == Some(rev_5.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_5.as_str()); + Ok((!caught_up).then(|| format!("{snapshot:?}"))) + }, + ) + .await?; + } + let resumed = warm_sink.drain(); + println!("resume projected {:?}", labels(&resumed)); + + let mut expected_resume = vec![ + post_label(POST_2, "delete"), + post_label(POST_3, "create"), + post_label(POST_4, "create"), + ]; + expected_resume.sort(); + let mut got_resume = labels(&resumed); + got_resume.sort(); + board.equal_if( + "resume: exactly the operations after the cursor, once each", + got_resume == expected_resume, + format!("expected {expected_resume:?}, got {got_resume:?}"), + ); + board.equal_if( + "resume: nothing already projected is projected again", + duplicates(&resumed).is_empty() + && !got_resume.contains(&post_label(POST_1, "create")) + && !got_resume.contains(&post_label(POST_2, "create")), + format!("duplicates {:?}", duplicates(&resumed)), + ); + + // A write in the converged era, at a server-minted revision. + let rev_6 = writer.create(POST_5, "converged five").await?; + board.equal_if( + "converged era: a server-minted revision follows the carried one", + rev_6 > rev_5, + format!("carried {rev_5}, minted {rev_6}"), + ); + { + let (space, rev_6) = (space.clone(), rev_6.clone()); + let index_db = index_db.clone(); + wait_until( + "the daemon to project the converged-era write", + SETTLE, + move || { + let snapshot = read_index(&index_db, &space, AUTHOR_DID)?; + let caught_up = snapshot.head_rev.as_deref() == Some(rev_6.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_6.as_str()); + Ok((!caught_up).then(|| format!("{snapshot:?}"))) + }, + ) + .await?; + } + let after_swap_write = warm_sink.drain(); + board.equal_if( + "converged era: the new write projects once", + labels(&after_swap_write) == vec![post_label(POST_5, "create")], + format!("{:?}", labels(&after_swap_write)), + ); + + daemon.stop(); + let converged_daemon_log = std::fs::read_to_string(&daemon.log).unwrap_or_default(); + let unclean = unclean_resume_lines(&converged_daemon_log); + board.equal_if( + "resume: no cursor refusal, divergence or full-state recovery", + unclean.is_empty(), + if unclean.is_empty() { + "the daemon advanced incrementally throughout".to_string() + } else { + unclean.join("\n") + }, + ); + board.push( + "legacy-era daemon log was also clean", + if unclean_resume_lines(&legacy_daemon_log).is_empty() { + Verdict::Equal + } else { + Verdict::Differs + }, + unclean_resume_lines(&legacy_daemon_log).join("\n"), + ); + + let warm_index = read_index(&index_db, &space, AUTHOR_DID)?; + + // ---- A cold daemon on the converted store ----------------------------- + let cold_sink = Sink::start()?; + let cold = DaemonSetup { + index_db: run_dir.join("cold/index.sqlite"), + dpop_key: run_dir.join("cold/dpop.json"), + notify_port: free_port()?, + sink_url: cold_sink.url(), + }; + std::fs::create_dir_all(run_dir.join("cold"))?; + let mut cold_daemon = Server::spawn( + "cold daemon", + &daemon_bin, + &run_dir, + &daemon_env(&space, &shim_url, &cold, &directory.url()), + &run_dir.join("daemon-cold.log"), + )?; + cold_daemon.wait_log("daemon starting", READY).await?; + { + let (space, rev_6) = (space.clone(), rev_6.clone()); + let cold_db = cold.index_db.clone(); + wait_until("the cold daemon to sync from scratch", SETTLE, move || { + let snapshot = read_index(&cold_db, &space, AUTHOR_DID)?; + let caught_up = snapshot.head_rev.as_deref() == Some(rev_6.as_str()) + && snapshot.projector_cursors.get("feeds").map(String::as_str) + == Some(rev_6.as_str()); + Ok((!caught_up).then(|| format!("{snapshot:?}"))) + }) + .await?; + } + let cold_projected = cold_sink.seen(); + cold_daemon.stop(); + converged.stop(); + + let cold_index = read_index(&cold.index_db, &space, AUTHOR_DID)?; + board.equal_if( + "cold sync: the same records, revisions and digest as the resumed daemon", + cold_index.records == warm_index.records + && cold_index.head_rev == warm_index.head_rev + && cold_index.lthash_state == warm_index.lthash_state, + format!( + "warm {} records at {:?}, cold {} records at {:?}", + warm_index.records.len(), + warm_index.head_rev, + cold_index.records.len(), + cold_index.head_rev + ), + ); + + let warm_stream: Vec<_> = legacy_projected + .iter() + .chain(resumed.iter()) + .chain(after_swap_write.iter()) + .cloned() + .collect(); + board.equal_if( + "cold sync: the same projected end state as the resumed daemon", + final_state(&cold_projected) == final_state(&warm_stream), + format!( + "warm {:?}\ncold {:?}", + final_state(&warm_stream), + final_state(&cold_projected) + ), + ); + board.equal_if( + "whole run: no operation is projected twice to either destination", + duplicates(&warm_stream).is_empty() && duplicates(&cold_projected).is_empty(), + format!( + "warm {:?}, cold {:?}", + duplicates(&warm_stream), + duplicates(&cold_projected) + ), + ); + board.push( + "oplog window was never the constraint", + Verdict::Note, + format!( + "{} ops in a {} row window; the converted floor is open, so no `since` can be refused", + converged_ops.len(), + rsky_space_host::actor_repos::DEFAULT_OPLOG_WINDOW + ), + ); + + let report = format!( + "resume-across-swap gate\n\n\ + space: {space}\n\ + legacy head at hand-off: {rev_2}\n\ + unacknowledged legacy writes: {rev_3}, {rev_4}, {rev_5}\n\ + converged-era write: {rev_6}\n\ + {conversion}\n\n{}", + board.render() + ); + print!("{report}"); + std::fs::write(run_dir.join("report.txt"), &report)?; + println!("\nlogs: {}", run_dir.display()); + + if board.failures() > 0 { + bail!("resume gate failed: {} check(s) differ", board.failures()); + } + Ok(()) +} + +fn post_label(rkey: &str, operation: &str) -> String { + format!("{COLLECTION}/{rkey}:{operation}") +} diff --git a/rsky-spaces-parity/src/layer2/car.rs b/rsky-spaces-parity/src/layer2/car.rs new file mode 100644 index 00000000..438528a9 --- /dev/null +++ b/rsky-spaces-parity/src/layer2/car.rs @@ -0,0 +1,87 @@ +//! A CARv1 block reader, enough to compare what `getRepo` carries. +//! +//! The served commit embeds fresh random key material, so the two sides' commit +//! blocks never match by construction. The record blocks must, so the gate +//! compares the block sets and expects the difference to be exactly the one +//! commit block on each side. + +use anyhow::{bail, Result}; +use std::collections::BTreeSet; + +fn varint(bytes: &[u8], at: &mut usize) -> Result { + let mut value = 0u64; + let mut shift = 0u32; + loop { + let Some(byte) = bytes.get(*at) else { + bail!("truncated varint"); + }; + *at += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + shift += 7; + if shift > 63 { + bail!("varint too long"); + } + } +} + +/// Advance past one binary CIDv1 and return its bytes. +fn cid<'a>(bytes: &'a [u8], at: &mut usize) -> Result<&'a [u8]> { + let start = *at; + let version = varint(bytes, at)?; + if version != 1 { + bail!("expected CIDv1, got version {version}"); + } + let _codec = varint(bytes, at)?; + let _hash = varint(bytes, at)?; + let length = varint(bytes, at)? as usize; + *at += length; + if *at > bytes.len() { + bail!("truncated cid"); + } + Ok(&bytes[start..*at]) +} + +/// The `(cid, data)` pairs a CARv1 payload carries, roots excluded from the +/// return because the header is not what is being compared. +pub fn blocks(car: &[u8]) -> Result, Vec)>> { + let mut at = 0usize; + let header = varint(car, &mut at)? as usize; + at += header; + if at > car.len() { + bail!("truncated car header"); + } + let mut out = Vec::new(); + while at < car.len() { + let length = varint(car, &mut at)? as usize; + let end = at + length; + if end > car.len() { + bail!("truncated car block"); + } + let mut cursor = at; + let key = cid(car, &mut cursor)?.to_vec(); + out.push((key, car[cursor..end].to_vec())); + at = end; + } + Ok(out) +} + +pub struct CarDiff { + pub shared: usize, + pub only_shim: Vec, + pub only_pds: Vec, +} + +/// Compare two CAR payloads by block CID. +pub fn diff(shim: &[u8], pds: &[u8]) -> Result { + let left: BTreeSet> = blocks(shim)?.into_iter().map(|(cid, _)| cid).collect(); + let right: BTreeSet> = blocks(pds)?.into_iter().map(|(cid, _)| cid).collect(); + let hex = |bytes: &Vec| bytes.iter().map(|b| format!("{b:02x}")).collect::(); + Ok(CarDiff { + shared: left.intersection(&right).count(), + only_shim: left.difference(&right).map(hex).collect(), + only_pds: right.difference(&left).map(hex).collect(), + }) +} diff --git a/rsky-spaces-parity/src/layer2/directory.rs b/rsky-spaces-parity/src/layer2/directory.rs new file mode 100644 index 00000000..0f19de5b --- /dev/null +++ b/rsky-spaces-parity/src/layer2/directory.rs @@ -0,0 +1,70 @@ +//! A stand-in DID directory, so neither server reaches the network. +//! +//! It answers every path with a DID document for the requested DID: the +//! `#atproto` signing key the gate controls, and a PDS service endpoint +//! pointing back at itself, which absorbs best-effort write notifications. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::net::TcpListener; + +pub struct Directory { + pub port: u16, +} + +impl Directory { + /// Bind an ephemeral port and serve until the process exits. + pub fn start(keys: BTreeMap, handle: String) -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let mut buf = [0u8; 8192]; + let read = stream.read(&mut buf).unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + let path = request.split_whitespace().nth(1).unwrap_or("/").to_string(); + let did = path + .trim_start_matches('/') + .split('?') + .next() + .unwrap_or_default() + .replace("%3A", ":") + .replace("%3a", ":"); + let multibase = keys + .get(&did) + .or_else(|| keys.values().next()) + .cloned() + .unwrap_or_default(); + let body = serde_json::json!({ + "id": did, + "alsoKnownAs": [format!("at://{handle}")], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multibase, + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": format!("http://127.0.0.1:{port}"), + }], + }) + .to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + Ok(Self { port }) + } + + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } +} diff --git a/rsky-spaces-parity/src/layer2/mod.rs b/rsky-spaces-parity/src/layer2/mod.rs new file mode 100644 index 00000000..cb6fd532 --- /dev/null +++ b/rsky-spaces-parity/src/layer2/mod.rs @@ -0,0 +1,126 @@ +//! Support code for the two-process acceptance gate: token minting, a stub DID +//! directory, child-process supervision, and the response normalizers the gate +//! compares with. + +/// Every request in the gate is to a loopback port, so proxy discovery is both +/// useless and, on macOS, a hard failure inside a restricted sandbox. +pub fn http_client() -> anyhow::Result { + Ok(reqwest::Client::builder() + .no_proxy() + .timeout(std::time::Duration::from_secs(30)) + .build()?) +} + +pub mod car; +pub mod directory; +pub mod normalize; +pub mod process; +pub mod tokens; + +/// One line of the gate's scoreboard. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Check { + pub name: String, + pub verdict: Verdict, + pub detail: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// Both sides agreed. + Equal, + /// The sides disagreed in a way the gate treats as failure. + Differs, + /// The sides disagreed in a way this design accepts and records. + Documented, + /// Recorded for the report, not scored. + Note, +} + +impl Verdict { + pub fn tag(self) -> &'static str { + match self { + Verdict::Equal => "equal", + Verdict::Differs => "DIFFERS", + Verdict::Documented => "documented divergence", + Verdict::Note => "note", + } + } +} + +#[derive(Debug, Default)] +pub struct Scoreboard { + pub checks: Vec, +} + +impl Scoreboard { + pub fn push(&mut self, name: impl Into, verdict: Verdict, detail: impl Into) { + self.checks.push(Check { + name: name.into(), + verdict, + detail: detail.into(), + }); + } + + pub fn equal_if( + &mut self, + name: impl Into, + equal: bool, + detail: impl Into, + ) -> bool { + let verdict = if equal { + Verdict::Equal + } else { + Verdict::Differs + }; + self.push(name, verdict, detail); + equal + } + + pub fn scored(&self) -> usize { + self.checks + .iter() + .filter(|c| matches!(c.verdict, Verdict::Equal | Verdict::Differs)) + .count() + } + + pub fn passed(&self) -> usize { + self.checks + .iter() + .filter(|c| c.verdict == Verdict::Equal) + .count() + } + + pub fn failures(&self) -> usize { + self.checks + .iter() + .filter(|c| c.verdict == Verdict::Differs) + .count() + } + + pub fn documented(&self) -> usize { + self.checks + .iter() + .filter(|c| c.verdict == Verdict::Documented) + .count() + } + + pub fn render(&self) -> String { + let mut out = String::new(); + for check in &self.checks { + out.push_str(&format!(" [{}] {}\n", check.verdict.tag(), check.name)); + if !check.detail.is_empty() { + for line in check.detail.lines() { + out.push_str(&format!(" {line}\n")); + } + } + } + out.push_str(&format!( + "\nparity: {}/{} checks equal (+{} documented divergence)\n", + self.passed(), + self.scored(), + self.documented() + )); + out + } +} diff --git a/rsky-spaces-parity/src/layer2/normalize.rs b/rsky-spaces-parity/src/layer2/normalize.rs new file mode 100644 index 00000000..248f9ebf --- /dev/null +++ b/rsky-spaces-parity/src/layer2/normalize.rs @@ -0,0 +1,80 @@ +//! Response normalization, using the same first-appearance substitution the +//! stored-row comparator uses: each side's distinct revisions map in order to +//! `R1, R2, …`. Fields that cannot match between two independent writers are +//! replaced by a marker rather than dropped, so a field appearing on one side +//! only still shows up as a difference. + +use crate::is_tid; +use serde_json::{Map, Value}; +use std::collections::BTreeMap; + +/// Commit fields derived from fresh random key material on every serve, plus +/// the opaque paging cursor, whose numbering is per-store. +pub const NON_COMPARABLE: [&str; 4] = ["ikm", "sig", "mac", "cursor"]; + +#[derive(Default)] +pub struct Revs { + seen: BTreeMap, + order: Vec, +} + +impl Revs { + pub fn placeholder(&mut self, rev: &str) -> String { + if let Some(existing) = self.seen.get(rev) { + return existing.clone(); + } + let placeholder = format!("R{}", self.order.len() + 1); + self.seen.insert(rev.to_string(), placeholder.clone()); + self.order.push(rev.to_string()); + placeholder + } + + pub fn count(&self) -> usize { + self.order.len() + } + + /// The revisions in first-appearance order, unsubstituted. + pub fn raw(&self) -> &[String] { + &self.order + } +} + +/// Object keys are emitted in sorted order. The two decoders build their JSON +/// maps in different orders from the same DAG-CBOR bytes, and record identity is +/// the CID, which is compared unnormalized in the same response. +pub fn normalize(value: &Value, revs: &mut Revs) -> Value { + match value { + Value::Array(items) => Value::Array(items.iter().map(|v| normalize(v, revs)).collect()), + Value::Object(fields) => { + let mut sorted: Vec<(&String, &Value)> = fields.iter().collect(); + sorted.sort_by(|a, b| a.0.cmp(b.0)); + let mut out = Map::new(); + for (key, field) in sorted { + let replacement = if NON_COMPARABLE.contains(&key.as_str()) { + Value::String("".to_string()) + } else if key == "rev" { + // Only a revision is substituted. Record keys are TIDs too, + // and they must compare literally. + match field.as_str() { + Some(rev) => Value::String(revs.placeholder(rev)), + None => normalize(field, revs), + } + } else { + normalize(field, revs) + }; + out.insert(key.clone(), replacement); + } + Value::Object(out) + } + other => other.clone(), + } +} + +/// Every revision a response exposed is a syntactically valid TID. +pub fn revs_are_tids(revs: &Revs) -> bool { + revs.raw().iter().all(|rev| is_tid(rev)) +} + +pub fn render(value: &Value) -> String { + serde_json::to_string_pretty(value).unwrap_or_else(|_| value.to_string()) +} diff --git a/rsky-spaces-parity/src/layer2/process.rs b/rsky-spaces-parity/src/layer2/process.rs new file mode 100644 index 00000000..31efe2b6 --- /dev/null +++ b/rsky-spaces-parity/src/layer2/process.rs @@ -0,0 +1,183 @@ +//! Child-process supervision: start a server, wait for it to answer, and make +//! sure it is gone when the gate finishes however the gate finishes. + +use anyhow::{bail, Context, Result}; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +pub struct Server { + pub name: &'static str, + child: Child, + pub log: std::path::PathBuf, +} + +impl Server { + pub fn spawn( + name: &'static str, + binary: &Path, + cwd: &Path, + env: &[(String, String)], + log: &Path, + ) -> Result { + let file = + std::fs::File::create(log).with_context(|| format!("create {}", log.display()))?; + let errors = file.try_clone()?; + let mut command = Command::new(binary); + command + .current_dir(cwd) + .env_clear() + .env("PATH", std::env::var("PATH").unwrap_or_default()) + .env("HOME", std::env::var("HOME").unwrap_or_default()) + .stdin(Stdio::null()) + .stdout(Stdio::from(file)) + .stderr(Stdio::from(errors)); + for (key, value) in env { + command.env(key, value); + } + let child = command + .spawn() + .with_context(|| format!("spawn {}", binary.display()))?; + Ok(Self { + name, + child, + log: log.to_path_buf(), + }) + } + + /// Poll `url` until it answers or the deadline passes; a child that has + /// already exited fails immediately with its log. + pub async fn wait_ready(&mut self, url: &str, timeout: Duration) -> Result<()> { + let client = crate::layer2::http_client()?; + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = self.child.try_wait()? { + bail!( + "{} exited early ({status}); log:\n{}", + self.name, + self.tail() + ); + } + if let Ok(response) = client.get(url).send().await { + if response.status().is_success() { + return Ok(()); + } + } + if Instant::now() >= deadline { + bail!( + "{} did not become ready at {url}; log:\n{}", + self.name, + self.tail() + ); + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + } + + /// Poll the log until `needle` appears, for a server with no health + /// endpoint to poll instead. + pub async fn wait_log(&mut self, needle: &str, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = self.child.try_wait()? { + bail!( + "{} exited early ({status}); log:\n{}", + self.name, + self.tail() + ); + } + if std::fs::read_to_string(&self.log) + .unwrap_or_default() + .contains(needle) + { + return Ok(()); + } + if Instant::now() >= deadline { + bail!( + "{} never logged {needle:?}; log:\n{}", + self.name, + self.tail() + ); + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + } + + pub fn tail(&self) -> String { + let text = std::fs::read_to_string(&self.log).unwrap_or_default(); + text.lines() + .rev() + .take(30) + .collect::>() + .into_iter() + .rev() + .collect::>() + .join("\n") + } + + pub fn stop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop(); + } +} + +pub fn free_port() -> Result { + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + Ok(listener.local_addr()?.port()) +} + +pub fn copy_tree(from: &Path, to: &Path) -> Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_tree(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), &target)?; + } + } + Ok(()) +} + +/// Replace every copied per-account store with an empty one created by the +/// space host's own schema code, keeping the signing keys beside it. The space +/// host requires the file to be present at startup, so it cannot simply be +/// deleted and left to appear on first write. +pub fn reset_stores(root: &Path) -> Result { + if !root.exists() { + return Ok(0); + } + let mut created = 0; + for entry in std::fs::read_dir(root)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + if entry.file_name().to_string_lossy().starts_with("did:") { + let account = entry.path(); + for stale in std::fs::read_dir(&account)? { + let stale = stale?; + if stale + .file_name() + .to_string_lossy() + .starts_with("store.sqlite") + { + std::fs::remove_file(stale.path())?; + } + } + rsky_space_host::actor_schema::get_migrated_db(account.join("store.sqlite")) + .map_err(|error| anyhow::anyhow!("migrate shim store: {error}"))?; + created += 1; + } else { + created += reset_stores(&entry.path())?; + } + } + Ok(created) +} diff --git a/rsky-spaces-parity/src/layer2/tokens.rs b/rsky-spaces-parity/src/layer2/tokens.rs new file mode 100644 index 00000000..ba9bd5cc --- /dev/null +++ b/rsky-spaces-parity/src/layer2/tokens.rs @@ -0,0 +1,111 @@ +//! The credentials the two servers demand, minted locally. +//! +//! The space host verifies an access token it did not issue, so the gate acts as +//! the authorization server: it holds the shared HS256 secret the host is +//! configured with and signs `at+jwt` tokens with it. DPoP proofs are ES256 over +//! a fixed P-256 key so a proof can be rebuilt for every request. + +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use hmac::{Hmac, Mac}; +use rsky_oauth::jwk::{EcCurve, Jwk}; +use rsky_oauth::jwt::{sign, JwtClaims, JwtHeader}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub const DPOP_KEY_BYTES: [u8; 32] = [0x42u8; 32]; +pub const CLIENT_ID: &str = "https://layer2.invalid/oauth-client-metadata.json"; + +static JTI: AtomicU64 = AtomicU64::new(0); + +fn next_jti(prefix: &str) -> String { + format!("{prefix}-{}", JTI.fetch_add(1, Ordering::SeqCst)) +} + +pub fn now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("clock before unix epoch") + .as_secs() +} + +pub fn dpop_key() -> Jwk { + Jwk::from_private_key_bytes(EcCurve::P256, &DPOP_KEY_BYTES).expect("dpop key") +} + +pub fn jwk_thumbprint_of_dpop_key() -> String { + let public = serde_json::to_value(dpop_key().to_public()).expect("jwk serializes"); + let canonical = format!( + r#"{{"crv":"{}","kty":"{}","x":"{}","y":"{}"}}"#, + public["crv"].as_str().unwrap_or_default(), + public["kty"].as_str().unwrap_or_default(), + public["x"].as_str().unwrap_or_default(), + public["y"].as_str().unwrap_or_default(), + ); + URL_SAFE_NO_PAD.encode(Sha256::digest(canonical.as_bytes())) +} + +/// A DPoP proof bound to `method` and `url`. `url` carries no query string: +/// both servers compare only scheme, host and path. +pub fn dpop_proof(method: &str, url: &str, access_token: Option<&str>) -> String { + let key = dpop_key(); + let mut header = JwtHeader::new("ES256"); + header.typ = Some("dpop+jwt".to_string()); + header.jwk = Some(key.to_public()); + let mut claims = JwtClaims { + iat: Some(now()), + jti: Some(next_jti("proof")), + ..Default::default() + }; + claims.extra.insert("htm".to_string(), method.into()); + claims.extra.insert("htu".to_string(), url.into()); + if let Some(token) = access_token { + claims.extra.insert( + "ath".to_string(), + URL_SAFE_NO_PAD + .encode(Sha256::digest(token.as_bytes())) + .into(), + ); + } + sign(&header, &claims, &key).expect("dpop proof signs") +} + +/// An HS256 `at+jwt` access token of the shape a PDS that is its own +/// authorization server issues: DID audience, no `scope`, DPoP-bound. +pub fn access_token(secret: &str, issuer: &str, audience: &str, subject: &str) -> String { + let header = json!({"typ": "at+jwt", "alg": "HS256"}); + let claims = json!({ + "iss": issuer, + "aud": audience, + "sub": subject, + "iat": now(), + "exp": now() + 3600, + "jti": next_jti("tok"), + "client_id": CLIENT_ID, + "cnf": { "jkt": jwk_thumbprint_of_dpop_key() }, + }); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).expect("header serializes")), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).expect("claims serialize")), + ); + let mut mac = as Mac>::new_from_slice(secret.as_bytes()).expect("hmac key"); + mac.update(input.as_bytes()); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) +} + +/// An inter-service auth JWT, minted with the space host's own issuer code so +/// the gate cannot drift from what the host verifies. +pub fn service_jwt( + signer: &rsky_space_host::signing::Signer, + iss: &str, + aud: &str, + lxm: &str, +) -> anyhow::Result { + rsky_space_host::service_jwt::mint(signer, iss, aud, lxm, now(), next_jti("svc")) + .map_err(|error| anyhow::anyhow!("service jwt: {error}")) +} diff --git a/rsky-spaces-parity/src/lib.rs b/rsky-spaces-parity/src/lib.rs new file mode 100644 index 00000000..2673a29d --- /dev/null +++ b/rsky-spaces-parity/src/lib.rs @@ -0,0 +1,384 @@ +pub mod layer2; +pub mod resume; + +use rsky_pds::actor_store::space::{SpaceStore, SpaceStoreError}; +use rsky_space_host::error::HostError; +use rsky_space_host::repo::RepoStore; +use std::collections::BTreeMap; +use std::path::Path; + +pub type OpTuple = (String, String, Option, Option); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoDump { + pub records: Vec<(String, String, String, Vec)>, + pub lthash_state: Vec, + /// `Err` once compaction has dropped the start of history, which is itself + /// a value both sides must agree on. + pub ops: Result, Outcome>, +} + +/// The classification a write batch or read is compared on: either it applied, +/// or both sides must refuse it for the same reason. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Outcome { + Applied, + RecordExists, + RecordNotFound, + InvalidSwap, + HistoryUnavailable, + RepoGone, + Other(String), +} + +pub fn shim_outcome(result: &Result) -> Outcome { + match result { + Ok(_) => Outcome::Applied, + Err(HostError::RecordExists(_)) => Outcome::RecordExists, + Err(HostError::RecordNotFound(_)) => Outcome::RecordNotFound, + Err(HostError::InvalidSwap) => Outcome::InvalidSwap, + Err(HostError::HistoryUnavailable) => Outcome::HistoryUnavailable, + Err(HostError::RepoNotFound) => Outcome::RepoGone, + Err(other) => Outcome::Other(other.to_string()), + } +} + +pub fn pds_outcome(result: &anyhow::Result) -> Outcome { + let Err(error) = result else { + return Outcome::Applied; + }; + match error.downcast_ref::() { + Some(SpaceStoreError::RecordExists(_)) => Outcome::RecordExists, + Some(SpaceStoreError::RecordNotFound(_)) => Outcome::RecordNotFound, + Some(SpaceStoreError::InvalidSwap(_)) => Outcome::InvalidSwap, + Some(SpaceStoreError::HistoryUnavailable) => Outcome::HistoryUnavailable, + Some(SpaceStoreError::SpaceNotFound(_)) | Some(SpaceStoreError::SpaceDeleted(_)) => { + Outcome::RepoGone + } + None => Outcome::Other(error.to_string()), + } +} + +/// `None` when the repo does not exist, so a scenario whose first batch is +/// rejected on both sides still compares. +pub async fn dump_shim(store: &dyn RepoStore, space_uri: &str, did: &str) -> Option { + let head = store.head(space_uri, did).await.expect("shim head")?; + let (records, _) = store + .list_records(space_uri, did, None, None, u32::MAX) + .await + .expect("shim records"); + let ops = store.list_ops(space_uri, did, None, None, u32::MAX).await; + Some(RepoDump { + records: records + .into_iter() + .map(|r| (r.collection, r.rkey, r.cid, r.value)) + .collect(), + lthash_state: head.state.to_vec(), + ops: match ops { + Ok(page) => Ok(page + .ops + .into_iter() + .map(|o| (o.collection, o.rkey, o.cid, o.prev)) + .collect()), + Err(error) => Err(shim_outcome::<()>(&Err(error))), + }, + }) +} + +pub async fn dump_pds(store: &SpaceStore, space_uri: &str) -> Option { + let state = store.repo_state(space_uri).await.expect("pds repo state")?; + if state.deleted { + return None; + } + let records = store.all_records(space_uri).await.expect("pds records"); + let ops = store + .list_repo_ops(space_uri, None, None, usize::MAX >> 1) + .await; + Some(RepoDump { + records: records + .into_iter() + .map(|r| (r.collection, r.rkey, r.cid, r.value)) + .collect(), + lthash_state: state.lthash_state, + ops: match ops { + Ok((page, _)) => Ok(page + .into_iter() + .map(|o| (o.collection, o.rkey, o.cid, o.prev)) + .collect()), + Err(error) => Err(pds_outcome::<()>(&Err(error))), + }, + }) +} + +pub fn assert_parity(name: &str, shim: &Option, pds: &Option) -> bool { + match (shim, pds) { + (None, None) => true, + (Some(shim), Some(pds)) => compare(name, shim, pds), + (shim, pds) => { + eprintln!( + "{name}: repo existence differs: shim={}, pds={}", + shim.is_some(), + pds.is_some() + ); + false + } + } +} + +fn compare(name: &str, shim: &RepoDump, pds: &RepoDump) -> bool { + let mut equal = true; + for (field, left, right) in [ + ( + "records", + format!("{:?}", shim.records), + format!("{:?}", pds.records), + ), + ( + "lthash_state", + format!("{:?}", shim.lthash_state), + format!("{:?}", pds.lthash_state), + ), + ("ops", format!("{:?}", shim.ops), format!("{:?}", pds.ops)), + ] { + if left != right { + eprintln!("{name}: {field} differs\n shim: {left}\n pds: {right}"); + equal = false; + } + } + equal +} + +// ------------------------------------------------------- stored-row comparison + +/// The `space_*` rows of one store file, rendered as strings with the values +/// that cannot match across two independent writers replaced by placeholders: +/// revisions (server-minted TIDs) by `R1, R2, …` in first-appearance order, +/// oplog row ids by `I1, I2, …`, and creation timestamps by `T`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableDump { + pub tables: Vec<(&'static str, Vec)>, + /// The distinct revisions in first-appearance order, unnormalized. + pub revs: Vec, + /// Revisions in oplog order, unnormalized. + pub oplog_revs: Vec, +} + +struct Normalizer { + revs: BTreeMap, + order: Vec, + ids: BTreeMap, +} + +impl Normalizer { + fn rev(&mut self, rev: &str) -> String { + if let Some(placeholder) = self.revs.get(rev) { + return placeholder.clone(); + } + let placeholder = format!("R{}", self.order.len() + 1); + self.revs.insert(rev.to_string(), placeholder.clone()); + self.order.push(rev.to_string()); + placeholder + } + + fn optional_rev(&mut self, rev: Option) -> String { + rev.map(|r| self.rev(&r)).unwrap_or_else(|| "-".to_string()) + } + + fn id(&mut self, id: i64) -> String { + let next = format!("I{}", self.ids.len() + 1); + self.ids.entry(id).or_insert(next).clone() + } +} + +pub fn dump_tables(path: &Path) -> TableDump { + let conn = rusqlite::Connection::open(path).expect("store connection"); + let mut norm = Normalizer { + revs: BTreeMap::new(), + order: Vec::new(), + ids: BTreeMap::new(), + }; + + let mut oplog_revs = Vec::new(); + let mut oplog = Vec::new(); + { + let mut statement = conn + .prepare( + "SELECT id, space_uri, rev, collection, rkey, cid, prev \ + FROM space_oplog ORDER BY id", + ) + .expect("oplog statement"); + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + )) + }) + .expect("oplog rows") + .collect::>>() + .expect("oplog values"); + for (id, space_uri, rev, collection, rkey, cid, prev) in rows { + oplog_revs.push(rev.clone()); + oplog.push(format!( + "{} {space_uri} {} {collection} {rkey} {} {}", + norm.id(id), + norm.rev(&rev), + cid.unwrap_or_else(|| "-".into()), + prev.unwrap_or_else(|| "-".into()), + )); + } + } + + let mut records = Vec::new(); + { + let mut statement = conn + .prepare( + "SELECT space_uri, collection, rkey, cid, rev, value \ + FROM space_record ORDER BY space_uri, collection, rkey", + ) + .expect("record statement"); + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Vec>(5)?, + )) + }) + .expect("record rows") + .collect::>>() + .expect("record values"); + for (space_uri, collection, rkey, cid, rev, value) in rows { + records.push(format!( + "{space_uri} {collection} {rkey} {cid} {} {}", + norm.rev(&rev), + render(&value) + )); + } + } + + let mut repos = Vec::new(); + { + let mut statement = conn + .prepare( + "SELECT space_uri, authority, space_type, skey, rev, lthash_state, \ + oplog_floor_rev, deleted FROM space_repo ORDER BY space_uri", + ) + .expect("repo statement"); + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Vec>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, i64>(7)?, + )) + }) + .expect("repo rows") + .collect::>>() + .expect("repo values"); + for (space_uri, authority, space_type, skey, rev, state, floor, deleted) in rows { + repos.push(format!( + "{space_uri} {authority} {space_type} {skey} {} {} {} {deleted}", + norm.rev(&rev), + render(&state), + norm.optional_rev(floor), + )); + } + } + + let blobs; + { + let mut statement = conn + .prepare( + "SELECT space_uri, blob_cid, collection, rkey FROM space_blob_ref \ + ORDER BY space_uri, blob_cid, collection, rkey", + ) + .expect("blob statement"); + blobs = statement + .query_map([], |row| { + Ok(format!( + "{} {} {} {}", + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)? + )) + }) + .expect("blob rows") + .collect::>>() + .expect("blob values"); + } + + TableDump { + tables: vec![ + ("space_repo", repos), + ("space_record", records), + ("space_oplog", oplog), + ("space_blob_ref", blobs), + ], + revs: norm.order, + oplog_revs, + } +} + +pub fn compare_tables(name: &str, shim: &TableDump, pds: &TableDump) -> bool { + let mut equal = true; + for ((table, left), (_, right)) in shim.tables.iter().zip(pds.tables.iter()) { + if left != right { + eprintln!("{name}: {table} rows differ\n shim: {left:?}\n pds: {right:?}"); + equal = false; + } + } + equal +} + +/// Every revision is a syntactically valid TID, and the oplog is ordered by +/// revision — the two properties the placeholders would otherwise hide. +pub fn revs_are_well_formed(name: &str, dump: &TableDump, side: &str) -> bool { + let mut sound = true; + for rev in &dump.revs { + if !is_tid(rev) { + eprintln!("{name}: {side} rev `{rev}` is not a valid TID"); + sound = false; + } + } + for pair in dump.oplog_revs.windows(2) { + if pair[0] > pair[1] { + eprintln!( + "{name}: {side} oplog order disagrees with rev order: `{}` then `{}`", + pair[0], pair[1] + ); + sound = false; + } + } + sound +} + +/// 13 characters of the sortable base32 alphabet, with the high bit of the +/// leading character clear. +pub fn is_tid(value: &str) -> bool { + const ALPHABET: &str = "234567abcdefghijklmnopqrstuvwxyz"; + value.len() == 13 + && value.chars().all(|c| ALPHABET.contains(c)) + && value + .chars() + .next() + .is_some_and(|c| "234567abcdefghij".contains(c)) +} + +fn render(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/rsky-spaces-parity/src/resume/mod.rs b/rsky-spaces-parity/src/resume/mod.rs new file mode 100644 index 00000000..c9a2b382 --- /dev/null +++ b/rsky-spaces-parity/src/resume/mod.rs @@ -0,0 +1,199 @@ +//! Support code for the resume-across-swap gate: the projection sink the +//! daemon is pointed at, and readers for the three durable artefacts the gate +//! reasons about — the daemon's index, the legacy multi-tenant store, and a +//! converted per-account store. + +pub mod sink; + +use anyhow::{Context, Result}; +use rusqlite::{params, Connection, OpenFlags}; +use std::collections::BTreeMap; +use std::path::Path; + +/// One oplog row, in the form both storage eras can be read into. +pub type OpRow = (i64, String, String, String, Option); + +/// What the daemon durably knows about one repo. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexSnapshot { + /// The syncer's cursor: `since` on the next `listRepoOps`. + pub head_rev: Option, + pub lthash_state: Option>, + /// `collection/rkey -> (cid, rev, value)`. + pub records: BTreeMap>)>, + /// Journalled batches, by revision, with the mutation count each carried. + pub journal: BTreeMap, + /// `projector -> rev`: how far each projection has been confirmed. + pub projector_cursors: BTreeMap, +} + +/// Read the daemon's index for one `(space, did)`. The daemon runs SQLite in +/// WAL mode, so this opens read-write to let the reader replay the log of a +/// process that was killed rather than shut down. +pub fn read_index(path: &Path, space: &str, did: &str) -> Result { + let conn = Connection::open(path).with_context(|| format!("open {}", path.display()))?; + let head: Option<(String, Vec)> = conn + .query_row( + "SELECT rev, lthash_state FROM sync_state WHERE space_uri = ?1 AND did = ?2", + params![space, did], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .ok(); + let mut records = BTreeMap::new(); + { + let mut statement = conn.prepare( + "SELECT collection, rkey, cid, rev, value FROM record \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY collection, rkey", + )?; + let mut rows = statement.query(params![space, did])?; + while let Some(row) = rows.next()? { + let collection: String = row.get(0)?; + let rkey: String = row.get(1)?; + records.insert( + format!("{collection}/{rkey}"), + (row.get(2)?, row.get(3)?, row.get(4)?), + ); + } + } + let mut journal = BTreeMap::new(); + { + let mut statement = conn.prepare( + "SELECT rev, mutations FROM projection_journal \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY rev", + )?; + let mut rows = statement.query(params![space, did])?; + while let Some(row) = rows.next()? { + let rev: String = row.get(0)?; + let mutations: Vec = row.get(1)?; + let count = serde_json::from_slice::(&mutations) + .ok() + .and_then(|value| value.as_array().map(Vec::len)) + .unwrap_or(0); + journal.insert(rev, count); + } + } + let mut projector_cursors = BTreeMap::new(); + { + let mut statement = conn.prepare( + "SELECT projector, rev FROM projector_cursor \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY projector", + )?; + let mut rows = statement.query(params![space, did])?; + while let Some(row) = rows.next()? { + projector_cursors.insert(row.get(0)?, row.get(1)?); + } + } + Ok(IndexSnapshot { + head_rev: head.as_ref().map(|(rev, _)| rev.clone()), + lthash_state: head.map(|(_, state)| state), + records, + journal, + projector_cursors, + }) +} + +/// The legacy multi-tenant oplog for one `(space, did)`, in row order. +pub fn legacy_oplog(db: &Path, space: &str, did: &str) -> Result> { + let conn = Connection::open_with_flags(db, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("open {}", db.display()))?; + let mut statement = conn.prepare( + "SELECT seq, rev, collection, rkey, cid FROM repo_op \ + WHERE space_uri = ?1 AND did = ?2 ORDER BY seq", + )?; + let mut rows = statement.query(params![space, did])?; + let mut ops = Vec::new(); + while let Some(row) = rows.next()? { + ops.push(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )); + } + Ok(ops) +} + +/// The converged per-account oplog for one space, in row order. +pub fn converged_oplog(store: &Path, space: &str) -> Result> { + let conn = Connection::open_with_flags(store, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("open {}", store.display()))?; + let mut statement = conn.prepare( + "SELECT id, rev, collection, rkey, cid FROM space_oplog \ + WHERE space_uri = ?1 ORDER BY id", + )?; + let mut rows = statement.query(params![space])?; + let mut ops = Vec::new(); + while let Some(row) = rows.next()? { + ops.push(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )); + } + Ok(ops) +} + +/// `(rev, oplog_floor_rev)` for a converged repo. A floor of `None` is the +/// state in which no `since` can be refused as outside the oplog window. +pub fn converged_head(store: &Path, space: &str) -> Result<(String, Option)> { + let conn = Connection::open_with_flags(store, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("open {}", store.display()))?; + conn.query_row( + "SELECT rev, oplog_floor_rev FROM space_repo WHERE space_uri = ?1 AND deleted = 0", + params![space], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .with_context(|| format!("no live space_repo row for {space}")) +} + +/// Labels of a projected batch list, in arrival order. +pub fn labels(ops: &[sink::Projected]) -> Vec { + ops.iter().map(sink::Projected::label).collect() +} + +/// Labels that appear more than once across `ops` — a duplicated projection. +pub fn duplicates(ops: &[sink::Projected]) -> Vec { + let mut counts: BTreeMap = BTreeMap::new(); + for op in ops { + *counts.entry(op.label()).or_default() += 1; + } + counts + .into_iter() + .filter(|(_, count)| *count > 1) + .map(|(label, count)| format!("{label} x{count}")) + .collect() +} + +/// The final projected state per record path, which is what a destination +/// holds once a batch list has been applied in order. +pub fn final_state(ops: &[sink::Projected]) -> BTreeMap { + let mut state = BTreeMap::new(); + for op in ops { + state.insert(op.path(), op.operation.clone()); + } + state +} + +/// Lines of `log` that name a sync failure, cursor refusal, or full-state +/// recovery. Any of them means the resume was not clean. +pub fn unclean_resume_lines(log: &str) -> Vec { + const MARKERS: [&str; 5] = [ + "HistoryUnavailable", + "full-state recovery", + "diverged", + "sweep failed", + "prev does not match", + ]; + log.lines() + .filter(|line| { + let lowered = line.to_lowercase(); + MARKERS + .iter() + .any(|marker| lowered.contains(&marker.to_lowercase())) + }) + .map(str::to_string) + .collect() +} diff --git a/rsky-spaces-parity/src/resume/sink.rs b/rsky-spaces-parity/src/resume/sink.rs new file mode 100644 index 00000000..3e211ced --- /dev/null +++ b/rsky-spaces-parity/src/resume/sink.rs @@ -0,0 +1,151 @@ +//! A stand-in projection destination: it accepts the daemon's +//! `projectRecords` batches and keeps them, so the gate can say exactly which +//! operations were projected, in which order, and how many times. + +use serde_json::Value; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; + +/// One projected operation, reduced to what the gate compares. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Projected { + /// The batch this operation arrived in, counted from 1 per sink. + pub batch: usize, + pub uri: String, + pub operation: String, + pub revision: String, + pub cid: Option, +} + +impl Projected { + /// `collection/rkey`, the part of the URI that identifies the record. + pub fn path(&self) -> String { + let mut segments = self.uri.rsplitn(3, '/'); + let rkey = segments.next().unwrap_or_default(); + let collection = segments.next().unwrap_or_default(); + format!("{collection}/{rkey}") + } + + /// `collection/rkey:operation`, the label the gate asserts on. + pub fn label(&self) -> String { + format!("{}:{}", self.path(), self.operation) + } +} + +#[derive(Default)] +struct State { + ops: Vec, + batches: usize, + acknowledgements: usize, +} + +pub struct Sink { + port: u16, + state: Arc>, +} + +impl Sink { + /// Bind an ephemeral port and serve until the process exits. + pub fn start() -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let state = Arc::new(Mutex::new(State::default())); + let served = state.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let (path, body) = match read_request(&mut stream) { + Some(request) => request, + None => continue, + }; + record(&served, &path, &body); + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\ + Content-Length: 2\r\nConnection: close\r\n\r\n{}", + ); + } + }); + Ok(Self { port, state }) + } + + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Everything received so far, without removing it. + pub fn seen(&self) -> Vec { + self.state.lock().expect("sink state").ops.clone() + } + + /// Take everything received so far, leaving the sink empty, so the next + /// phase's projections are read in isolation. + pub fn drain(&self) -> Vec { + std::mem::take(&mut self.state.lock().expect("sink state").ops) + } + + pub fn batches(&self) -> usize { + self.state.lock().expect("sink state").batches + } + + pub fn acknowledgements(&self) -> usize { + self.state.lock().expect("sink state").acknowledgements + } +} + +fn record(state: &Arc>, path: &str, body: &str) { + let mut state = state.lock().expect("sink state"); + if path.contains("ackSyncersObserved") { + state.acknowledgements += 1; + return; + } + if !path.contains("projectRecords") { + return; + } + let Ok(parsed) = serde_json::from_str::(body) else { + return; + }; + state.batches += 1; + let batch = state.batches; + let ops = parsed["ops"].as_array().cloned().unwrap_or_default(); + for op in ops { + state.ops.push(Projected { + batch, + uri: op["uri"].as_str().unwrap_or_default().to_string(), + operation: op["operation"].as_str().unwrap_or_default().to_string(), + revision: op["revision"].as_str().unwrap_or_default().to_string(), + cid: op["cid"].as_str().map(str::to_string), + }); + } +} + +/// Read one request, returning its path and body. Only the request line and +/// `content-length` are interpreted; the daemon sends no chunked bodies. +fn read_request(stream: &mut std::net::TcpStream) -> Option<(String, String)> { + let mut reader = BufReader::new(stream.try_clone().ok()?); + let mut request_line = String::new(); + if reader.read_line(&mut request_line).ok()? == 0 { + return None; + } + let path = request_line.split_whitespace().nth(1)?.to_string(); + let mut length = 0usize; + loop { + let mut header = String::new(); + if reader.read_line(&mut header).ok()? == 0 { + break; + } + if header.trim().is_empty() { + break; + } + if let Some((name, value)) = header.split_once(':') { + if name.trim().eq_ignore_ascii_case("content-length") { + length = value.trim().parse().unwrap_or(0); + } + } + } + let mut body = vec![0u8; length]; + if length > 0 { + reader.read_exact(&mut body).ok()?; + } + Some((path, String::from_utf8_lossy(&body).to_string())) +} diff --git a/rsky-spaces-parity/tests/parity.rs b/rsky-spaces-parity/tests/parity.rs new file mode 100644 index 00000000..dd66502c --- /dev/null +++ b/rsky-spaces-parity/tests/parity.rs @@ -0,0 +1,927 @@ +use oracle_rsky_space::space_id::SpaceId; +use rsky_pds::actor_store::db::get_migrated_db; +use rsky_pds::actor_store::space::{ + blob_refs_in_record, encode_record, SpaceStore, SpaceWrite, DEFAULT_OPLOG_WINDOW, +}; +use rsky_space_host::actor_repos::ActorStoreRepos; +use rsky_space_host::repo::{RepoStore, RepoWrite}; +use rsky_spaces_parity::{ + assert_parity, compare_tables, dump_pds, dump_shim, dump_tables, pds_outcome, + revs_are_well_formed, shim_outcome, +}; +use serde_json::{json, Value}; + +fn sqlite_master(path: &std::path::Path) -> Vec<(String, String, String)> { + let connection = rusqlite::Connection::open(path).expect("schema connection"); + let mut statement = connection + .prepare("SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name") + .expect("schema statement"); + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) + .expect("schema rows") + .collect::>>() + .expect("schema values") +} + +#[tokio::test] +async fn actor_schema_matches_pinned_oracle() { + let temp = tempfile::tempdir().expect("tempdir"); + let shim_path = temp.path().join("shim.sqlite"); + let pds_path = temp.path().join("pds.sqlite"); + rsky_space_host::actor_schema::get_migrated_db(&shim_path).expect("shim migration"); + get_migrated_db(&pds_path).await.expect("pds migration"); + assert_eq!(sqlite_master(&shim_path), sqlite_master(&pds_path)); +} + +/// S15: the shim's init is safe against a store the **PDS** created. +/// +/// Every other case here builds its own store tree, so none of them ever open +/// a file the oracle wrote first — which is how a build whose every live write +/// failed on `table repo_root already exists` passed this suite. Migration 001 +/// recreates the PDS base schema, so a PDS-created file must be adopted, not +/// re-migrated, and must come back byte-identical. +#[tokio::test] +async fn shim_init_adopts_a_pds_created_store_without_touching_it() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("pds-created.sqlite"); + get_migrated_db(&path).await.expect("pds migration"); + let before = sqlite_master(&path); + let before_bytes = std::fs::read(&path).expect("read"); + + // The shim opening it must succeed rather than re-running migration 001. + rsky_space_host::actor_schema::get_migrated_db(&path).expect("shim adopts the pds store"); + + let after = sqlite_master(&path); + for object in &before { + let (kind, name, _) = object; + assert!( + after.contains(object), + "shim init altered the pds object {kind} {name}" + ); + } + // Nothing pre-existing is rewritten, and the schema still carries the + // space tables the oracle put there. + assert_eq!(after, before, "shim init changed a pds-created schema"); + assert!( + after.iter().any(|(_, name, _)| name == "space_record"), + "the pds-created store is missing the space tables" + ); + // Opening it a second time is a no-op, so a restart is safe too. + let twice = std::fs::read(&path).expect("read"); + rsky_space_host::actor_schema::get_migrated_db(&path).expect("second open"); + assert_eq!(twice, std::fs::read(&path).expect("read")); + assert_ne!(before_bytes.len(), 0); +} + +/// S16: the shim writes to its own directory while reading keys from the PDS's. +/// +/// Option A (`Design/spaces-storage-parity.md`) keeps the two directories +/// separate; the suite only ever ran them as the same path, which is precisely +/// the configuration that hid one setting doing both jobs. +#[tokio::test] +async fn separated_key_and_store_directories_write_only_to_the_space_store() { + use rsky_space_host::pds_seam::PdsSeam; + + let temp = tempfile::tempdir().expect("tempdir"); + let actors = temp.path().join("pds-actors"); + let spaces = temp.path().join("space-stores"); + std::fs::create_dir_all(&actors).expect("actors dir"); + + // A PDS-shaped account: a key file and a store the PDS created. + let digest = ::digest(DID.as_bytes()); + let account = actors.join(&hex::encode(digest)[..2]).join(DID); + std::fs::create_dir_all(&account).expect("account dir"); + std::fs::write(account.join("key"), [7u8; 32]).expect("key"); + get_migrated_db(account.join("store.sqlite")) + .await + .expect("pds store"); + let pds_store_before = std::fs::read(account.join("store.sqlite")).expect("read"); + + let seam = PdsSeam::open(&actors).expect("seam opens the pds actors dir"); + seam.signer(DID).expect("signing key resolves"); + + let repos = ActorStoreRepos::open(&spaces).expect("space store opens"); + let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); + let write = create("s16", "written to the space store"); + repos + .apply_writes(&space.uri(), DID, "", &[write.shim()]) + .await + .expect("write lands in the space store"); + + // The write went to the shim's own file, and the PDS's is untouched. + let shim_path = repos.store_path(DID).expect("shim path"); + assert!(shim_path.starts_with(&spaces)); + assert!(shim_path.is_file()); + assert_eq!( + pds_store_before, + std::fs::read(account.join("store.sqlite")).expect("read"), + "the pds store must not be written to" + ); +} + +const DID: &str = "did:plc:parityauthor"; +const AUTHORITY: &str = "did:plc:parityauthority"; +const COLLECTION: &str = "app.bsky.feed.post"; +const PAGE: u32 = 2; + +#[derive(Clone)] +struct ScriptWrite { + space: usize, + collection: &'static str, + rkey: String, + action: Action, +} + +#[derive(Clone)] +enum Action { + Create(Value), + Update(Value, Option), + Delete(Option), +} + +impl ScriptWrite { + fn shim(&self) -> RepoWrite { + let (collection, rkey) = (self.collection.to_string(), self.rkey.clone()); + match &self.action { + Action::Create(value) => RepoWrite::Create { + collection, + rkey, + value: encoded(value), + }, + Action::Update(value, swap) => RepoWrite::Update { + collection, + rkey, + value: encoded(value), + swap_record: swap.clone(), + }, + Action::Delete(swap) => RepoWrite::Delete { + collection, + rkey, + swap_record: swap.clone(), + }, + } + } + + fn pds(&self) -> SpaceWrite { + let (collection, rkey) = (self.collection.to_string(), self.rkey.clone()); + match &self.action { + Action::Create(value) => SpaceWrite::Create { + collection, + rkey, + value: value.clone(), + }, + Action::Update(value, swap) => SpaceWrite::Update { + collection, + rkey, + value: value.clone(), + swap_cid: swap.clone(), + }, + Action::Delete(swap) => SpaceWrite::Delete { + collection, + rkey, + swap_cid: swap.clone(), + }, + } + } +} + +fn encoded(value: &Value) -> Vec { + encode_record(value).expect("record encoding").1 +} + +fn cid_of(value: &Value) -> String { + encode_record(value).expect("record encoding").0 +} + +fn create(rkey: &str, text: &str) -> ScriptWrite { + ScriptWrite { + space: 0, + collection: COLLECTION, + rkey: rkey.to_string(), + action: Action::Create(json!({ "text": text })), + } +} + +fn update(rkey: &str, text: &str, swap: Option) -> ScriptWrite { + ScriptWrite { + space: 0, + collection: COLLECTION, + rkey: rkey.to_string(), + action: Action::Update(json!({ "text": text }), swap), + } +} + +fn delete(rkey: &str, swap: Option) -> ScriptWrite { + ScriptWrite { + space: 0, + collection: COLLECTION, + rkey: rkey.to_string(), + action: Action::Delete(swap), + } +} + +fn in_space(space: usize, write: ScriptWrite) -> ScriptWrite { + ScriptWrite { space, ..write } +} + +fn in_collection(collection: &'static str, write: ScriptWrite) -> ScriptWrite { + ScriptWrite { + collection, + ..write + } +} + +struct Scenario { + name: &'static str, + window: usize, + batches: Vec>, +} + +fn scenario(name: &'static str, batches: Vec>) -> Scenario { + Scenario { + name, + window: DEFAULT_OPLOG_WINDOW, + batches, + } +} + +/// Drive both stores through the same batches, then compare: refusal reasons +/// per batch, the rows each side stored (with server-minted revisions and oplog +/// ids normalized), the reads each side serves whole and paged, the answers to +/// every `since` probe, and finally what the oracle reads back out of the +/// file the shim wrote. +async fn run(case: &Scenario) -> bool { + let name = case.name; + let spaces = [ + SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"), + SpaceId::new(AUTHORITY, "community.blacksky.feed", "second"), + ]; + let temp = tempfile::tempdir().expect("tempdir"); + let shim = ActorStoreRepos::with_oplog_window(temp.path().join("shim"), case.window) + .expect("shim store"); + let pds_path = temp.path().join("store.sqlite"); + let pds = SpaceStore::new( + DID.into(), + get_migrated_db(&pds_path).await.expect("pds db"), + ); + + for (index, batch) in case.batches.iter().enumerate() { + for (space_index, space) in spaces.iter().enumerate() { + let writes: Vec<&ScriptWrite> = + batch.iter().filter(|w| w.space == space_index).collect(); + if writes.is_empty() { + continue; + } + let shim_result = shim + .apply_writes( + &space.uri(), + DID, + "3shimrev", + &writes.iter().map(|w| w.shim()).collect::>(), + ) + .await; + let pds_result = pds + .apply_writes(space, writes.iter().map(|w| w.pds()).collect(), case.window) + .await; + let (shim_kind, pds_kind) = (shim_outcome(&shim_result), pds_outcome(&pds_result)); + if shim_kind != pds_kind { + eprintln!( + "{name}: batch {index} space {space_index} outcomes differ\n \ + shim: {shim_kind:?}\n pds: {pds_kind:?}" + ); + return false; + } + } + } + + let shim_path = shim.store_path(DID).expect("shim store path"); + let (shim_tables, pds_tables) = (dump_tables(&shim_path), dump_tables(&pds_path)); + let mut equal = compare_tables(name, &shim_tables, &pds_tables) + && revs_are_well_formed(name, &shim_tables, "shim") + && revs_are_well_formed(name, &pds_tables, "pds"); + + // The oracle, pointed at the shim's own file: the drop-in assertion. + let crossed = SpaceStore::new( + DID.into(), + get_migrated_db(&shim_path).await.expect("cross-open db"), + ); + + for space in &spaces { + let uri = space.uri(); + equal = equal + && assert_parity( + name, + &dump_shim(&shim, &uri, DID).await, + &dump_pds(&pds, &uri).await, + ) + && assert_parity( + &format!("{name} cross-open"), + &dump_shim(&shim, &uri, DID).await, + &dump_pds(&crossed, &uri).await, + ) + && paged_reads_equal(name, &shim, &pds, &uri).await + && since_reads_equal(name, &shim, &pds, &uri, &shim_tables.revs, &pds_tables.revs) + .await; + } + equal +} + +/// Walk both sides in `PAGE`-sized pages, comparing every page and the cursor +/// each side hands back. +async fn paged_reads_equal( + name: &str, + shim: &ActorStoreRepos, + pds: &SpaceStore, + space_uri: &str, +) -> bool { + if shim + .head(space_uri, DID) + .await + .expect("shim head") + .is_none() + { + return true; + } + + let mut cursor: Option = None; + loop { + let (page, next) = shim + .list_records(space_uri, DID, None, cursor.as_deref(), PAGE) + .await + .expect("shim record page"); + let mirror = pds + .list_records(space_uri, None, PAGE as usize, cursor.clone()) + .await + .expect("pds record page"); + let left: Vec<_> = page + .iter() + .map(|r| (&r.collection, &r.rkey, &r.cid, &r.value)) + .collect(); + let right: Vec<_> = mirror + .iter() + .map(|r| (&r.collection, &r.rkey, &r.cid, &r.value)) + .collect(); + if left != right { + eprintln!( + "{name}: record page after {cursor:?} differs\n shim: {left:?}\n pds: {right:?}" + ); + return false; + } + match next { + Some(next) if !page.is_empty() => cursor = Some(next), + _ => break, + } + } + + let mut cursor: Option = None; + loop { + let page = shim + .list_ops( + space_uri, + DID, + None, + cursor.map(|c| c.to_string()).as_deref(), + PAGE, + ) + .await; + let mirror = pds + .list_repo_ops(space_uri, None, cursor, PAGE as usize) + .await; + let (shim_kind, pds_kind) = (shim_outcome(&page), pds_outcome(&mirror)); + if shim_kind != pds_kind { + eprintln!( + "{name}: op page after {cursor:?} outcomes differ\n \ + shim: {shim_kind:?}\n pds: {pds_kind:?}" + ); + return false; + } + let (Ok(page), Ok((mirror, more))) = (page, mirror) else { + return true; + }; + let left: Vec<_> = page + .ops + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + let right: Vec<_> = mirror + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + if left != right || page.complete == more { + eprintln!( + "{name}: op page after {cursor:?} differs\n \ + shim: {left:?} complete={}\n pds: {right:?} has_more={more}", + page.complete + ); + return false; + } + if !more { + break; + } + cursor = mirror.last().map(|o| o.id); + } + true +} + +/// Ask both sides for history since each revision they minted. The revisions +/// differ between the two stores, so each side is probed with its own — the +/// nth revision on one side answers the nth on the other. +async fn since_reads_equal( + name: &str, + shim: &ActorStoreRepos, + pds: &SpaceStore, + space_uri: &str, + shim_revs: &[String], + pds_revs: &[String], +) -> bool { + for (index, (shim_rev, pds_rev)) in shim_revs.iter().zip(pds_revs).enumerate() { + let page = shim + .list_ops(space_uri, DID, Some(shim_rev), None, u32::MAX) + .await; + let mirror = pds + .list_repo_ops(space_uri, Some(pds_rev.clone()), None, usize::MAX >> 1) + .await; + let (shim_kind, pds_kind) = (shim_outcome(&page), pds_outcome(&mirror)); + if shim_kind != pds_kind { + eprintln!( + "{name}: history since revision {index} differs\n \ + shim: {shim_kind:?}\n pds: {pds_kind:?}" + ); + return false; + } + let (Ok(page), Ok((mirror, _))) = (page, mirror) else { + continue; + }; + let left: Vec<_> = page + .ops + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + let right: Vec<_> = mirror + .iter() + .map(|o| (&o.collection, &o.rkey, &o.cid, &o.prev)) + .collect(); + if left != right { + eprintln!( + "{name}: history since revision {index} differs\n \ + shim: {left:?}\n pds: {right:?}" + ); + return false; + } + } + true +} + +fn scenarios() -> Vec { + let first = json!({ "text": "first" }); + let long = "x".repeat(60 * 1024); + vec![ + scenario( + "S1 create single record", + vec![vec![create("one", "first")]], + ), + scenario( + "S2 batch create", + vec![vec![ + create("one", "one"), + create("two", "two"), + create("three", "three"), + ]], + ), + scenario( + "S3 update with swap-cid success", + vec![ + vec![create("one", "first")], + vec![update("one", "second", Some(cid_of(&first)))], + ], + ), + scenario( + "S4 swap-cid conflict", + vec![ + vec![create("one", "first")], + vec![update( + "one", + "second", + Some(cid_of(&json!({ "text": "stale" }))), + )], + ], + ), + scenario( + "S5 delete", + vec![ + vec![create("one", "first"), create("two", "two")], + vec![delete("one", None)], + ], + ), + scenario( + "S6 delete then recreate same rkey", + vec![ + vec![create("one", "first")], + vec![delete("one", Some(cid_of(&first)))], + vec![create("one", "reborn")], + vec![delete("one", None)], + vec![delete("one", None)], + ], + ), + scenario( + "S7 unicode and prefix-colliding keys", + vec![vec![ + create("é🌍", "unicode rkey"), + create("a-b", "hyphen sorts before slash"), + create("a", "bare"), + create("a.b", "dotted"), + in_collection("app.bsky.feed.pos", create("z", "shorter collection")), + in_collection("app.bsky.feed.post-x", create("a", "longer collection")), + in_collection("app.bsky.feed.post.deep", create("a", "deeper collection")), + ]], + ), + scenario( + "S8 large record", + vec![ + vec![create("big", &long)], + vec![update( + "big", + "small again", + Some(cid_of(&json!({ "text": long }))), + )], + ], + ), + Scenario { + name: "S9 oplog compaction beyond the window", + window: 3, + batches: (0..6) + .map(|i| vec![create(&format!("r{i}"), &format!("body {i}"))]) + .chain(std::iter::once(vec![ + delete("r0", None), + delete("r1", None), + ])) + .collect(), + }, + scenario( + "S10 two spaces one author", + vec![ + vec![ + create("one", "in first space"), + in_space(1, create("one", "in second space")), + ], + vec![ + in_space(1, create("two", "second space only")), + delete("one", None), + ], + ], + ), + scenario( + "S11 pagination across many revisions", + (0..7) + .map(|i| vec![create(&format!("r{i}"), &format!("body {i}"))]) + .chain(std::iter::once(vec![ + delete("r3", None), + update("r4", "edited", None), + ])) + .collect(), + ), + scenario( + "S13 cross-open after a mixed script", + vec![ + vec![create("keep", "kept"), create("gone", "removed")], + vec![ + update("keep", "edited", Some(cid_of(&json!({ "text": "kept" })))), + delete("gone", None), + in_collection("app.bsky.feed.like", create("l1", "liked")), + ], + vec![in_space(1, create("elsewhere", "other space"))], + ], + ), + ] +} + +/// S12: blobs are the one documented divergence. The oracle accepts a +/// blob-bearing record and indexes the reference; the shim's storage keeps the +/// bytes but indexes nothing, and the host's write path refuses the record +/// outright, so a blob-carrying space cannot use the drop-in path. +#[tokio::test] +async fn s12_blob_bearing_record_is_a_documented_divergence() { + let space = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); + let temp = tempfile::tempdir().expect("tempdir"); + let shim = ActorStoreRepos::open(temp.path().join("shim")).expect("shim store"); + let pds = SpaceStore::new( + DID.into(), + get_migrated_db(temp.path().join("store.sqlite")) + .await + .expect("pds db"), + ); + let record = json!({ + "text": "with an image", + "image": { + "$type": "blob", + "ref": { "$link": "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egm4gv4a" }, + "mimeType": "image/png", + "size": 12345, + }, + }); + assert_eq!(blob_refs_in_record(&record).len(), 1); + assert!(rsky_space_host::http::contains_blob_ref(&record)); + + pds.apply_writes( + &space, + vec![SpaceWrite::Create { + collection: COLLECTION.into(), + rkey: "one".into(), + value: record.clone(), + }], + DEFAULT_OPLOG_WINDOW, + ) + .await + .expect("pds accepts blob-bearing records"); + shim.apply_writes( + &space.uri(), + DID, + "3shimrev", + &[RepoWrite::Create { + collection: COLLECTION.into(), + rkey: "one".into(), + value: encoded(&record), + }], + ) + .await + .expect("shim storage is shape-agnostic"); + + let shim_blobs = blob_refs(&shim.store_path(DID).expect("shim store path")); + let pds_blobs = blob_refs(&temp.path().join("store.sqlite")); + assert_eq!(pds_blobs, 1, "oracle indexes the blob reference"); + assert_eq!(shim_blobs, 0, "shim indexes no blob references"); +} + +fn blob_refs(path: &std::path::Path) -> i64 { + rusqlite::Connection::open(path) + .expect("store connection") + .query_row("SELECT COUNT(*) FROM space_blob_ref", [], |row| row.get(0)) + .expect("blob ref count") +} + +#[tokio::test] +async fn scoreboard() { + let scenarios = scenarios(); + let total = scenarios.len() + 1; + let mut equal = 0; + for case in &scenarios { + equal += usize::from(run(case).await); + } + equal += usize::from(s14_legacy_converter().await); + println!("parity: {equal}/{total} (+1 documented divergence) scenarios byte-equal"); + assert_eq!(equal, total, "every scenario must be byte-equal"); +} + +const LEGACY_SCHEMA: &str = "\ +CREATE TABLE repo (\ + space_uri TEXT NOT NULL, did TEXT NOT NULL, rev TEXT NOT NULL DEFAULT '', \ + state BLOB NOT NULL, PRIMARY KEY (space_uri, did));\ +CREATE TABLE record (\ + space_uri TEXT NOT NULL, did TEXT NOT NULL, path TEXT NOT NULL, \ + collection TEXT NOT NULL, rkey TEXT NOT NULL, cid TEXT NOT NULL, \ + value BLOB NOT NULL, PRIMARY KEY (space_uri, did, path));\ +CREATE TABLE repo_op (\ + seq INTEGER PRIMARY KEY AUTOINCREMENT, space_uri TEXT NOT NULL, \ + did TEXT NOT NULL, rev TEXT NOT NULL, collection TEXT NOT NULL, \ + rkey TEXT NOT NULL, cid TEXT, prev TEXT);\ +CREATE INDEX repo_op_repo_seq ON repo_op (space_uri, did, seq);"; + +const SECOND_DID: &str = "did:plc:paritysecond"; + +struct LegacyOp { + space_uri: String, + did: &'static str, + rev: &'static str, + rkey: &'static str, + cid: Option, + prev: Option, +} + +/// S14: convert a store in the deployed multi-tenant schema into per-account +/// files, then read them back with the oracle. The LtHash state and every +/// oplog row id must survive the conversion, because they are respectively the +/// digest readers have seen and the cursors syncers hold. +async fn s14_legacy_converter() -> bool { + let name = "S14 legacy store converter"; + let temp = tempfile::tempdir().expect("tempdir"); + let legacy_path = temp.path().join("legacy.sqlite"); + let directory = temp.path().join("actors"); + let space_a = SpaceId::new(AUTHORITY, "community.blacksky.feed", "parity"); + let space_b = SpaceId::new(AUTHORITY, "community.blacksky.feed", "second"); + + let one_v1 = json!({ "text": "one" }); + let one_v2 = json!({ "text": "one edited" }); + let two = json!({ "text": "two" }); + let three = json!({ "text": "three" }); + let solo = json!({ "text": "solo" }); + let mine = json!({ "text": "mine" }); + + // Interleaved sequence numbers across accounts: the source numbers its + // oplog globally, the destination one file per account. + let ops = vec![ + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev1", + rkey: "one", + cid: Some(cid_of(&one_v1)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev1", + rkey: "two", + cid: Some(cid_of(&two)), + prev: None, + }, + LegacyOp { + space_uri: space_b.uri(), + did: DID, + rev: "3rev2", + rkey: "solo", + cid: Some(cid_of(&solo)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: SECOND_DID, + rev: "3rev3", + rkey: "mine", + cid: Some(cid_of(&mine)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev4", + rkey: "one", + cid: Some(cid_of(&one_v2)), + prev: Some(cid_of(&one_v1)), + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev5", + rkey: "three", + cid: Some(cid_of(&three)), + prev: None, + }, + LegacyOp { + space_uri: space_a.uri(), + did: DID, + rev: "3rev6", + rkey: "three", + cid: None, + prev: Some(cid_of(&three)), + }, + ]; + + let repos = [ + ( + space_a.uri(), + DID, + "3rev6", + vec![("one", &one_v2), ("two", &two)], + ), + (space_b.uri(), DID, "3rev2", vec![("solo", &solo)]), + (space_a.uri(), SECOND_DID, "3rev3", vec![("mine", &mine)]), + ]; + + { + let conn = rusqlite::Connection::open(&legacy_path).expect("legacy db"); + conn.execute_batch(LEGACY_SCHEMA).expect("legacy schema"); + for (space_uri, did, rev, records) in &repos { + let mut lthash = oracle_rsky_space::lthash::LtHash::new(); + for (rkey, value) in records { + let (cid, bytes) = encode_record(value).expect("record encoding"); + lthash.add(&oracle_rsky_space::lthash::element(COLLECTION, rkey, &cid)); + conn.execute( + "INSERT INTO record (space_uri, did, path, collection, rkey, cid, value) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + rusqlite::params![ + space_uri, + did, + format!("{COLLECTION}/{rkey}"), + COLLECTION, + rkey, + cid, + bytes + ], + ) + .expect("legacy record"); + } + conn.execute( + "INSERT INTO repo (space_uri, did, rev, state) VALUES (?1, ?2, ?3, ?4)", + rusqlite::params![space_uri, did, rev, lthash.state_bytes().to_vec()], + ) + .expect("legacy repo"); + } + for op in &ops { + conn.execute( + "INSERT INTO repo_op (space_uri, did, rev, collection, rkey, cid, prev) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + rusqlite::params![ + op.space_uri, + op.did, + op.rev, + COLLECTION, + op.rkey, + op.cid, + op.prev + ], + ) + .expect("legacy op"); + } + } + + let totals = rsky_space_host::convert::convert(&legacy_path, &directory).expect("conversion"); + if totals.accounts != 2 || totals.repos != 3 || totals.records != 4 || totals.ops != 7 { + eprintln!("{name}: unexpected conversion totals: {totals:?}"); + return false; + } + + for (space_uri, did, rev, records) in &repos { + let path = rsky_space_host::actor_repos::store_path(&directory, did).expect("store path"); + let store = SpaceStore::new( + (*did).into(), + get_migrated_db(&path).await.expect("converted db"), + ); + let state = store + .repo_state(space_uri) + .await + .expect("repo state") + .expect("repo row"); + let mut expected = oracle_rsky_space::lthash::LtHash::new(); + for (rkey, value) in records { + expected.add(&oracle_rsky_space::lthash::element( + COLLECTION, + rkey, + &cid_of(value), + )); + } + if state.rev != *rev || state.lthash_state != expected.state_bytes().to_vec() { + eprintln!("{name}: {space_uri}/{did} head did not survive conversion"); + return false; + } + let served = store.all_records(space_uri).await.expect("records"); + let expected_records: Vec<_> = records + .iter() + .map(|(rkey, value)| { + let (cid, bytes) = encode_record(value).expect("record encoding"); + (COLLECTION.to_string(), rkey.to_string(), cid, bytes) + }) + .collect(); + let served_records: Vec<_> = served + .iter() + .map(|r| { + ( + r.collection.clone(), + r.rkey.clone(), + r.cid.clone(), + r.value.clone(), + ) + }) + .collect(); + if served_records != expected_records { + eprintln!("{name}: {space_uri}/{did} records differ\n got: {served_records:?}\n want: {expected_records:?}"); + return false; + } + + // Oplog rows keep their source ids, so a held cursor still means the + // same position. + let expected_ids: Vec = ops + .iter() + .enumerate() + .filter(|(_, op)| op.space_uri == *space_uri && op.did == *did) + .map(|(index, _)| index as i64 + 1) + .collect(); + let (served_ops, _) = store + .list_repo_ops(space_uri, None, None, usize::MAX >> 1) + .await + .expect("ops"); + if served_ops.iter().map(|o| o.id).collect::>() != expected_ids { + eprintln!( + "{name}: {space_uri}/{did} oplog ids differ\n got: {:?}\n want: {expected_ids:?}", + served_ops.iter().map(|o| o.id).collect::>() + ); + return false; + } + + // A syncer holding the first id resumes at the next one, unchanged. + if let Some(&first) = expected_ids.first() { + let next = expected_ids.get(1).copied(); + let resumed = rsky_space_host::convert::resumes_at(&path, space_uri, first) + .expect("resume lookup"); + let (after, _) = store + .list_repo_ops(space_uri, None, Some(first), usize::MAX >> 1) + .await + .expect("ops after cursor"); + if resumed != next || after.first().map(|o| o.id) != next { + eprintln!("{name}: {space_uri}/{did} does not resume at {next:?}"); + return false; + } + } + } + true +}