Add option to disable deduplication checks

Add an option to the configuration to disable the deduplication cache,
which is useful for running tests and benchmarks against a log.

Author: lukevalenta

crates/ct_worker/config.schema.json

@@ -13,6 +13,11 @@
             ],
             "description": "Log verbosity."
         },
+        "disable_dedup": {
+            "type": "boolean",
+            "default": false,
+            "description": "Disable checking the deduplication cache for add-(pre-)chain requests (e.g., for tests or benchmarks)."
+        },
         "logs": {
             "type": "object",
             "description": "Dictionary CT log shard names to configurations.",

crates/ct_worker/config/src/lib.rs

@@ -15,6 +15,8 @@ pub struct TemporalInterval {
 #[derive(Deserialize, Debug)]
 pub struct AppConfig {
     pub logging_level: Option<String>,
+    #[serde(default)]
+    pub disable_dedup: bool,
     pub logs: HashMap<String, LogParams>,
 }
 

crates/ct_worker/src/ctlog.rs

@@ -21,7 +21,7 @@
 use crate::{
     metrics::{millis_diff_as_secs, AsF64, Metrics},
     util::now_millis,
-    CacheRead, CacheWrite, LockBackend, LookupKey, ObjectBackend, SequenceMetadata,
+    CacheRead, CacheWrite, LockBackend, LookupKey, ObjectBackend, SequenceMetadata, CONFIG,
 };
 use anyhow::{anyhow, bail};
 use ed25519_dalek::SigningKey as Ed25519SigningKey;
@@ -444,7 +444,10 @@ pub(crate) fn add_leaf_to_pool(
 ) -> AddLeafResult {
     let hash = entry.lookup_key();
 
-    if let Some(result) = state.check(&hash) {
+    if CONFIG.disable_dedup {
+        // Bypass deduplication and rate limit checks.
+        state.add(hash, entry)
+    } else if let Some(result) = state.check(&hash) {
         // Entry is already pending or being sequenced.
         result
     } else if let Some(v) = cache.get_entry(&hash) {

crates/ct_worker/src/frontend_worker.rs

@@ -202,21 +202,23 @@ async fn add_chain_or_pre_chain(
 
     // Check if entry is cached and return right away if so.
     let kv = load_cache_kv(env, name)?;
-    if let Some((leaf_index, timestamp)) = kv
-        .get(&BASE64_STANDARD.encode(lookup_key))
-        .bytes_with_metadata::<SequenceMetadata>()
-        .await?
-        .1
-    {
-        debug!("{name}: Entry is cached");
-        let entry = LogEntry {
-            inner: pending_entry,
-            leaf_index,
-            timestamp,
-        };
-        let sct = static_ct_api::signed_certificate_timestamp(signing_key, &entry)
-            .map_err(|e| e.to_string())?;
-        return Response::from_json(&sct);
+    if !CONFIG.disable_dedup {
+        if let Some((leaf_index, timestamp)) = kv
+            .get(&BASE64_STANDARD.encode(lookup_key))
+            .bytes_with_metadata::<SequenceMetadata>()
+            .await?
+            .1
+        {
+            debug!("{name}: Entry is cached");
+            let entry = LogEntry {
+                inner: pending_entry,
+                leaf_index,
+                timestamp,
+            };
+            let sct = static_ct_api::signed_certificate_timestamp(signing_key, &entry)
+                .map_err(|e| e.to_string())?;
+            return Response::from_json(&sct);
+        }
     }
 
     // Entry is not cached, so we need to sequence it.