mirror_worker: stream and incrementally commit add-entries uploads

crates/mirror_worker/config.dev.json

@@ -5,6 +5,7 @@
     "submission_prefix": "http://localhost:8787/",
     "monitoring_prefix": "http://localhost:8787/",
     "clean_interval_secs": 5,
+    "commit_packages": 2,
     "logs": {
         "oid/1.3.6.1.4.1.32473.2": {
             "description": "Dev-only MTC CA cosigner. Key name is the CA ID; the mirror serves log numbers 1-6 as origins oid/1.3.6.1.4.1.32473.2.0.<N>. log_public_keys holds a dev-only ML-DSA-44 SPKI.",

crates/mirror_worker/config.schema.json

@@ -30,6 +30,19 @@
             "default": 3600,
             "description": "How often (in seconds) the per-origin partial-tile cleaner wakes to clean orphaned partial tiles from object storage. Defaults to 3600 (one hour) when omitted."
         },
+        "commit_packages": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 1024,
+            "default": 32,
+            "description": "How many entry packages add-entries verifies before flushing them to storage and advancing the persisted-entry frontier. Bounds in-memory buffering and gives durable mid-request progress on large uploads. Defaults to 32 (the recommended per-request package budget) when omitted; capped at 1024 to bound worst-case buffering."
+        },
+        "max_chunk_bytes": {
+            "type": "integer",
+            "minimum": 1,
+            "default": 16777216,
+            "description": "Byte ceiling on the entries buffered between flushes. add-entries flushes early once buffered entry bytes reach this many, even if fewer than commit_packages packages have accumulated. Bounds peak memory independent of package sizes, since a single package can hold up to ~16 MiB. Defaults to 16777216 (16 MiB) when omitted."
+        },
         "logs": {
             "type": "object",
             "description": "CAs this mirror mirrors, keyed by log_key_name: the CA cosigner's note-signature name (the CA ID) on the checkpoints it ingests. Used as a signed-note key name at runtime, so per c2sp.org/signed-note it MUST NOT contain '+', whitespace, or control characters.",

crates/mirror_worker/config/src/lib.rs

@@ -62,6 +62,23 @@ pub struct AppConfig {
     /// back to a one-hour default (see [`Self::clean_interval_secs`]).
     /// Consumed by [`mirror_worker`](../mirror_worker/)'s `cleaner_do`.
     pub clean_interval_secs: Option<u64>,
+    /// How many entry packages the `add-entries` handler verifies before
+    /// flushing them to storage and advancing the persisted-entry
+    /// frontier. Bounds in-memory buffering and gives durable mid-request
+    /// progress on large uploads. `None` falls back to a default of 32
+    /// (see [`Self::commit_packages`]). Consumed by
+    /// [`mirror_worker`](../mirror_worker/)'s `add_entries`.
+    pub commit_packages: Option<u64>,
+    /// Byte ceiling on the entries buffered between flushes. `add-entries`
+    /// flushes early once the buffered entry bytes reach this many, even if
+    /// fewer than `commit_packages` packages have accumulated. This bounds
+    /// peak memory independent of package sizes: `commit_packages` alone
+    /// caps only the package *count*, and a single package can hold up to
+    /// 256 entries of 65535 bytes (~16 MiB), so a count-only bound can
+    /// exceed the isolate's memory ceiling. `None` falls back to a default
+    /// (see [`Self::max_chunk_bytes`]). Consumed by
+    /// [`mirror_worker`](../mirror_worker/)'s `add_entries`.
+    pub max_chunk_bytes: Option<u64>,
     /// CAs this mirror mirrors, keyed by `log_key_name`: the CA
     /// cosigner's note-signature name (the CA ID) carried by the
     /// checkpoints it ingests.
@@ -118,6 +135,29 @@ impl AppConfig {
         self.clean_interval_secs.unwrap_or(3600)
     }
 
+    /// How many entry packages `add-entries` commits per flush, falling
+    /// back to 32 when `commit_packages` is unset. 32 matches the
+    /// per-request package budget clients are recommended to stay within
+    /// (tlog-mirror "Implementation Considerations"), so a compliant
+    /// single-request upload still commits once, while larger uploads
+    /// flush every 32 packages instead of buffering the whole body.
+    #[must_use]
+    pub fn commit_packages(&self) -> u64 {
+        self.commit_packages.unwrap_or(32)
+    }
+
+    /// Byte ceiling on buffered entries before `add-entries` flushes early,
+    /// falling back to 16 MiB when `max_chunk_bytes` is unset. This bounds
+    /// peak in-memory buffering regardless of how large individual packages
+    /// are, complementing the `commit_packages` count cap. 16 MiB matches
+    /// the worst-case size of a single spec-maximal package (256 entries *
+    /// 65535 bytes), so the default never flushes mid-package for a
+    /// compliant upload yet still caps a pathological one.
+    #[must_use]
+    pub fn max_chunk_bytes(&self) -> u64 {
+        self.max_chunk_bytes.unwrap_or(16 * 1024 * 1024)
+    }
+
     /// Validate the configuration beyond what `serde` and the JSON schema
     /// can express.
     ///
@@ -141,9 +181,9 @@ impl AppConfig {
     ///    signed-note key name length cap, since each origin is itself
     ///    used as a checkpoint origin.
     ///
-    /// Simple single-field bounds (e.g. the log-number ranges) are
-    /// expressed in `config.schema.json` and enforced by the build
-    /// script, so they are not re-checked here.
+    /// Simple single-field bounds (e.g. `commit_packages` and the
+    /// log-number ranges) are expressed in `config.schema.json` and
+    /// enforced by the build script, so they are not re-checked here.
     ///
     /// `log_key_name` uniqueness across log entries is not checked here;
     /// it is enforced earlier, during deserialization (see
@@ -340,6 +380,8 @@ mod tests {
             submission_prefix: "https://mirror.example/".to_owned(),
             monitoring_prefix: None,
             clean_interval_secs: None,
+            commit_packages: None,
+            max_chunk_bytes: None,
             logs: HashMap::from([(
                 "example.com/log1".to_owned(),
                 LogParams {
@@ -464,6 +506,22 @@ mod tests {
             .expect("a valid log-number window is accepted");
     }
 
+    #[test]
+    fn commit_packages_defaults_to_32() {
+        let mut cfg = good_app_config();
+        assert_eq!(cfg.commit_packages(), 32);
+        cfg.commit_packages = Some(8);
+        assert_eq!(cfg.commit_packages(), 8);
+    }
+
+    #[test]
+    fn max_chunk_bytes_defaults_to_16_mib() {
+        let mut cfg = good_app_config();
+        assert_eq!(cfg.max_chunk_bytes(), 16 * 1024 * 1024);
+        cfg.max_chunk_bytes = Some(1024);
+        assert_eq!(cfg.max_chunk_bytes(), 1024);
+    }
+
     #[test]
     fn validate_rejects_inverted_window() {
         let cfg = with_log(|log| {
@@ -532,6 +590,8 @@ mod tests {
             submission_prefix: "https://mirror.example/".to_owned(),
             monitoring_prefix: None,
             clean_interval_secs: None,
+            commit_packages: None,
+            max_chunk_bytes: None,
             logs: HashMap::from([(
                 "a".repeat(250),
                 LogParams {