mirror_worker: implement the add-entries submission API

Stream and verify entry packages against the target pending checkpoint,
persist them as tlog-tiles entry bundles and hash tiles (see commit.rs),
and advance the persisted-entry frontier. A complete upload cosigns the
mirror checkpoint and returns 200; a client-truncated upload persists the
verified prefix and returns 202 with the advanced next entry so the
client can resume (C2SP/C2SP#253).

Persist incrementally every `commit_packages` packages (default 32) so a
long or interrupted upload advances the frontier as it streams rather
than only at the end; each flush commits from the last persisted frontier
and is resumable.

Includes body-decoding (transparent gzip), a streaming buffer adapter,
per-origin prefixed R2 storage, and the ticket-sealer for recovering a
past pending checkpoint on resume.

Author: lukevalenta

Cargo.lock

@@ -121,6 +121,7 @@ der = { version = "0.8", features = ["oid"] }
 # (crypto-common 0.2 / signature 3.0), so it remains pinned to a pre-release.
 # The rest of the batch is now stable.
 ed25519-dalek = { version = "3.0.0", features = ["pem"] }
+flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
 futures-executor = "0.3.31"
 futures-util = "0.3.31"
 getrandom = { version = "0.4", features = ["wasm_js"] }
@@ -157,6 +158,7 @@ tlog_checkpoint = { path = "crates/tlog_checkpoint", version = "0.2.0" }
 tlog_core = { path = "crates/tlog_core", version = "0.2.0" }
 tlog_cosignature = { path = "crates/tlog_cosignature", version = "0.2.0" }
 tlog_entry = { path = "crates/tlog_entry", version = "0.2.0" }
+tlog_mirror = { path = "crates/tlog_mirror", version = "0.2.0" }
 tlog_tiles = { path = "crates/tlog_tiles", version = "0.2.0" }
 tlog_witness = { path = "crates/tlog_witness", version = "0.2.0" }
 tokio = { version = "1", features = ["sync"] }

Cargo.toml

@@ -1 +1,2 @@
 MIRROR_SIGNING_KEY="-----BEGIN PRIVATE KEY-----\nMDQCAQAwCwYJYIZIAWUDBAMRBCKAIEJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJC\nQkJCQkJC\n-----END PRIVATE KEY-----\n"
+MIRROR_TICKET_KEY="Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc="

crates/mirror_worker/.dev.vars

@@ -32,33 +32,39 @@ serde_json.workspace = true
 
 [dependencies]
 axum.workspace = true
+base64.workspace = true
 config = { path = "./config", package = "mirror_worker_config" }
 console_error_panic_hook.workspace = true
 console_log.workspace = true
+flate2.workspace = true
+futures-util.workspace = true
 generic_log_worker.workspace = true
 getrandom.workspace = true
 getrandom_03.workspace = true
 hex.workspace = true
+length_prefixed.workspace = true
 log.workspace = true
 ml-dsa.workspace = true
 pkcs8.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
+sha2.workspace = true
 signed_note.workspace = true
 tlog_checkpoint.workspace = true
 tlog_core.workspace = true
 tlog_cosignature.workspace = true
+tlog_mirror.workspace = true
+tlog_tiles.workspace = true
 tlog_witness.workspace = true
 tower-service.workspace = true
 worker = { workspace = true, features = ["http", "axum"] }
 
 [dev-dependencies]
-# base64 is used only by the dev-config pin tests; ed25519-dalek only to
-# check that a non-ML-DSA-44 MIRROR_SIGNING_KEY is rejected. The shipped
-# worker is ML-DSA-44 only.
-base64.workspace = true
+# Ed25519 is used only by unit tests, to check that a non-ML-DSA-44
+# MIRROR_SIGNING_KEY is rejected. The shipped worker is ML-DSA-44 only.
 ed25519-dalek.workspace = true
+tokio = { workspace = true, features = ["macros", "rt"] }
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = [

crates/mirror_worker/Cargo.toml

@@ -5,6 +5,7 @@
     "submission_prefix": "http://localhost:8787/",
     "monitoring_prefix": "http://localhost:8787/",
     "clean_interval_secs": 5,
+    "commit_packages": 2,
     "logs": {
         "oid/1.3.6.1.4.1.32473.2": {
             "description": "Dev-only MTC CA cosigner. Key name is the CA ID; the mirror serves log numbers 1-6 as origins oid/1.3.6.1.4.1.32473.2.0.<N>. log_public_keys holds a dev-only ML-DSA-44 SPKI.",

crates/mirror_worker/config.dev.json

@@ -30,6 +30,13 @@
             "default": 3600,
             "description": "How often (in seconds) the per-origin partial-tile cleaner wakes to clean orphaned partial tiles from object storage. Defaults to 3600 (one hour) when omitted."
         },
+        "commit_packages": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 1024,
+            "default": 32,
+            "description": "How many entry packages add-entries verifies before flushing them to storage and advancing the persisted-entry frontier. Bounds in-memory buffering and gives durable mid-request progress on large uploads. Defaults to 32 (the recommended per-request package budget) when omitted; capped at 1024 to bound worst-case buffering."
+        },
         "logs": {
             "type": "object",
             "description": "CAs this mirror mirrors, keyed by log_key_name: the CA cosigner's note-signature name (the CA ID) on the checkpoints it ingests. Used as a signed-note key name at runtime, so per c2sp.org/signed-note it MUST NOT contain '+', whitespace, or control characters.",

crates/mirror_worker/config.schema.json

@@ -62,6 +62,13 @@ pub struct AppConfig {
     /// back to a one-hour default (see [`Self::clean_interval_secs`]).
     /// Consumed by [`mirror_worker`](../mirror_worker/)'s `cleaner_do`.
     pub clean_interval_secs: Option<u64>,
+    /// How many entry packages the `add-entries` handler verifies before
+    /// flushing them to storage and advancing the persisted-entry
+    /// frontier. Bounds in-memory buffering and gives durable mid-request
+    /// progress on large uploads. `None` falls back to a default of 32
+    /// (see [`Self::commit_packages`]). Consumed by
+    /// [`mirror_worker`](../mirror_worker/)'s `add_entries`.
+    pub commit_packages: Option<u64>,
     /// CAs this mirror mirrors, keyed by `log_key_name`: the CA
     /// cosigner's note-signature name (the CA ID) carried by the
     /// checkpoints it ingests.
@@ -118,6 +125,17 @@ impl AppConfig {
         self.clean_interval_secs.unwrap_or(3600)
     }
 
+    /// How many entry packages `add-entries` commits per flush, falling
+    /// back to 32 when `commit_packages` is unset. 32 matches the
+    /// per-request package budget clients are recommended to stay within
+    /// (tlog-mirror "Implementation Considerations"), so a compliant
+    /// single-request upload still commits once, while larger uploads
+    /// flush every 32 packages instead of buffering the whole body.
+    #[must_use]
+    pub fn commit_packages(&self) -> u64 {
+        self.commit_packages.unwrap_or(32)
+    }
+
     /// Validate the configuration beyond what `serde` and the JSON schema
     /// can express.
     ///
@@ -141,6 +159,10 @@ impl AppConfig {
     ///    signed-note key name length cap, since each origin is itself
     ///    used as a checkpoint origin.
     ///
+    /// Simple single-field bounds (e.g. `commit_packages` and the
+    /// log-number ranges) are expressed in `config.schema.json` and
+    /// enforced by the build script, so they are not re-checked here.
+    ///
     /// `log_key_name` uniqueness across log entries is not checked here;
     /// it is enforced earlier, during deserialization (see
     /// [`deserialize_logs`]). A plain `serde_json` object silently keeps
@@ -336,6 +358,7 @@ mod tests {
             submission_prefix: "https://mirror.example/".to_owned(),
             monitoring_prefix: None,
             clean_interval_secs: None,
+            commit_packages: None,
             logs: HashMap::from([(
                 "example.com/log1".to_owned(),
                 LogParams {
@@ -460,6 +483,14 @@ mod tests {
             .expect("a valid log-number window is accepted");
     }
 
+    #[test]
+    fn commit_packages_defaults_to_32() {
+        let mut cfg = good_app_config();
+        assert_eq!(cfg.commit_packages(), 32);
+        cfg.commit_packages = Some(8);
+        assert_eq!(cfg.commit_packages(), 8);
+    }
+
     #[test]
     fn validate_rejects_inverted_window() {
         let cfg = with_log(|log| {
@@ -528,6 +559,7 @@ mod tests {
             submission_prefix: "https://mirror.example/".to_owned(),
             monitoring_prefix: None,
             clean_interval_secs: None,
+            commit_packages: None,
             logs: HashMap::from([(
                 "a".repeat(250),
                 LogParams {