Refactor pool logic

crates/ct_worker/config.schema.json

@@ -64,17 +64,16 @@
                             "type": "string",
                             "description": "Provide a hint to place the log in a specific geographic location. See https://developers.cloudflare.com/durable-objects/reference/data-location/ for supported locations. If unspecified, the Durable Object will be created in proximity to the first request."
                         },
-                        "pool_size": {
+                        "sequence_interval_seconds": {
                             "type": "integer",
                             "minimum": 1,
-                            "default": 4000,
-                            "description": "The maximum number of entries to sequence at a time. See lib.rs for more information on the default."
+                            "default": 1,
+                            "description": "The duration in between sequencing operations, in seconds."
                         },
-                        "sequence_interval": {
+                        "max_pending_entry_holds": {
                             "type": "integer",
-                            "minimum": 1,
                             "default": 1,
-                            "description": "The duration in between sequencing operations, in seconds."
+                            "description": "The maximum number of times a pending entry can be held back from sequencing to avoid creating partial tiles. If non-zero, pending entries may be delayed by a multiple of sequence interval."
                         }
                     },
                     "required": [

crates/ct_worker/config/src/lib.rs

@@ -27,18 +27,16 @@ pub struct LogParams {
     pub submission_url: String,
     pub temporal_interval: TemporalInterval,
     pub location_hint: Option<String>,
-    #[serde(default = "default_pool_size_seconds")]
-    pub pool_size: usize,
     #[serde(default = "default_sequence_interval_seconds")]
-    pub sequence_interval: u64,
+    pub sequence_interval_seconds: u64,
+    #[serde(default = "default_max_pending_entry_holds")]
+    pub max_pending_entry_holds: usize,
 }
 
-// Limit on the number of entries per batch. Tune this parameter to avoid running into various size limitations.
-// For instance, unexpectedly large leaves (e.g., with PQ signatures) could cause us to exceed the 128MB Workers memory limit. Storing 4000 10KB certificates is 40MB.
-fn default_pool_size_seconds() -> usize {
-    4000
+fn default_sequence_interval_seconds() -> u64 {
+    1
 }
 
-fn default_sequence_interval_seconds() -> u64 {
+fn default_max_pending_entry_holds() -> usize {
     1
 }

crates/ct_worker/src/batcher_do.rs

@@ -9,7 +9,7 @@
 use crate::{get_stub, load_cache_kv, LookupKey, QueryParams, SequenceMetadata};
 use base64::prelude::*;
 use futures_util::future::{join_all, select, Either};
-use static_ct_api::LogEntry;
+use static_ct_api::PendingLogEntry;
 use std::{
     collections::{HashMap, HashSet},
     time::Duration,
@@ -37,7 +37,7 @@ struct Batcher {
 
 // A batch of entries to be submitted to the Sequencer together.
 struct Batch {
-    pending_leaves: Vec<LogEntry>,
+    pending_leaves: Vec<PendingLogEntry>,
     by_hash: HashSet<LookupKey>,
     done: Sender<HashMap<LookupKey, SequenceMetadata>>,
 }
@@ -68,7 +68,7 @@ impl DurableObject for Batcher {
         match req.path().as_str() {
             "/add_leaf" => {
                 let name = &req.query::<QueryParams>()?.name;
-                let entry: LogEntry = req.json().await?;
+                let entry: PendingLogEntry = req.json().await?;
                 let key = entry.lookup_key();
 
                 if self.in_flight >= MAX_IN_FLIGHT {