Add ability to automatically fetch roots from CCADB, fixes #65

- Add a cron job that once a day fetches roots from CCADB that are
  included in root stores and saves them into per-log root files (only
  appending certs for each log shard, but never removing them). This is
  similar to the sunlight implementation.
- Add config flag (enabled by default) to specify if a log will attempt
  to load its roots from the CCADB roots file in Workers KV.
- Remove roots.default.pem and require per-environment roots files to be
  non-empty if they exist. These can be used to add additional accepted
  roots on top of the CCADB list, for instance, to allow test/staging
  CAs to submit log entries.
- Add more functionality to x509util::CertPool to support adding and
  checking inclusion of certs in an existing CertPool.

Author: lukevalenta

Cargo.lock

@@ -32,12 +32,14 @@ debug = 1
 [workspace.dependencies]
 anyhow = "1.0"
 base64 = "0.22"
+base64ct = "1.8.0"
 bitcode = { version = "0.6.6", features = ["serde"] }
 byteorder = "1.5"
 chrono = { version = "0.4", features = ["serde"] }
 console_error_panic_hook = "0.1.1"
 console_log = { version = "1.0" }
 criterion = { version = "0.5", features = ["html_reports"] }
+csv = "1.3.1"
 generic_log_worker = { path = "crates/generic_log_worker", version = "0.2.0" }
 der = "0.7.10"
 ed25519-dalek = { version = "2.1.1", features = ["pem"] }

Cargo.toml

@@ -58,6 +58,9 @@ worker.workspace = true
 x509-cert.workspace = true
 x509_util.workspace = true
 prometheus.workspace = true
+chrono.workspace = true
+base64ct.workspace = true
+csv.workspace = true
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = [

crates/ct_worker/Cargo.toml

@@ -150,7 +150,7 @@ Follow these instructions to deploy to a custom domain, suitable for running a p
 
 1.  Create a file `config.${ENV}.json` with the configuration for the log shards.
 
-1.  (Optional) Create a file `roots.${ENV}.pem` with any custom accepted roots for the log shards. By default, `roots.default.pem` will be used. All logs shards deployed within the same Worker script use the same set of roots. Roots can be updated later.
+1.  (Optional) Create a file `roots.${ENV}.pem` with any custom accepted roots for the log shards, in PEM format. If `enable_ccadb_roots` is set to true, these roots are used in addition to roots auto-pulled from the CCADB list. If `enable_ccadb_roots` is set to false for any logs in the deployment, `roots.${ENV}.pem` is required to exist and contain at least one certificate. All logs shards deployed within the same Worker script use the same set of additional roots. Roots can be updated later, but roots should generally not be removed once added.
 
 1.  First set environment variables to specify the log shard name and deployment environment as below and then follow the [instructions above](#deployment-to-a-workersdev-subdomain) to create resources for each log shard.
 

crates/ct_worker/README.md

@@ -32,7 +32,7 @@ fn main() {
     let conf = serde_json::from_str::<AppConfig>(config_contents).unwrap_or_else(|e| {
         panic!("failed to deserialize JSON config '{config_file}': {e}");
     });
-    for (name, params) in conf.logs {
+    for (name, params) in &conf.logs {
         // Chrome's CT policy (https://googlechrome.github.io/CertificateTransparency/log_policy.html) states:
         // "The certificate expiry ranges for CT Logs must be no longer than one calendar year and should be no shorter than six months."
         assert!(
@@ -58,20 +58,35 @@ fn main() {
         }
     }
 
-    // Get and validate roots. Use 'roots.default.pem' if no environment-specific roots file is found.
-    let mut roots_file: &str = &format!("roots.{env}.pem");
-    if !fs::exists(roots_file).expect("failed to check if file exists") {
-        roots_file = "roots.default.pem";
+    // Get and validate roots from an embedded roots file, which must exist but
+    // can be empty unless 'enable_ccadb_roots' is set to false. If
+    // 'enable_ccadb_roots' is set to true, the log shard will combine trusted
+    // roots from the embedded roots file and from a roots file loaded from
+    // Workers KV.
+    let roots_file: &str = &format!("roots.{env}.pem");
+    let roots_file_exists = fs::exists(roots_file).expect("failed to check if file exists");
+    // If 'enable_ccadb_roots' is
+    if roots_file_exists {
+        let roots =
+            Certificate::load_pem_chain(&fs::read(roots_file).expect("failed to read roots file"))
+                .expect("unable to decode certificates");
+        assert!(
+            !roots.is_empty(),
+            "roots file does not contain any certificates"
+        );
+    } else if conf.logs.values().any(|params| !params.enable_ccadb_roots) {
+        // If any log shards have 'enable_ccadb_roots' set to false, require a roots file.
+        panic!("{roots_file} must exist and contain at least one certificate if any logs have 'enable_ccadb_roots' set to false");
     }
-    let roots =
-        Certificate::load_pem_chain(&fs::read(roots_file).expect("failed to read roots file"))
-            .expect("unable to decode certificates");
-    assert!(roots.len() > 50, "Roots file has too few entries");
 
     // Copy to OUT_DIR.
     let out_dir = env::var("OUT_DIR").unwrap();
     fs::copy(config_file, format!("{out_dir}/config.json")).expect("failed to copy config file");
-    fs::copy(roots_file, format!("{out_dir}/roots.pem")).expect("failed to copy roots file");
+    if roots_file_exists {
+        fs::copy(roots_file, format!("{out_dir}/roots.pem")).expect("failed to copy roots file");
+    } else {
+        fs::write(format!("{out_dir}/roots.pem"), "").expect("failed to write empty roots file");
+    }
 
     println!("cargo::rerun-if-env-changed=DEPLOY_ENV");
     println!("cargo::rerun-if-changed=config.schema.json");

crates/ct_worker/build.rs

@@ -108,6 +108,11 @@
                             "type": "boolean",
                             "default": true,
                             "description": "Enables checking the deduplication cache for add-(pre-)chain requests. Can be disabled for tests and benchmarks. If disabled, `kv_namespaces` can be omitted from `wrangler.jsonc`."
+                        },
+                        "enable_ccadb_roots": {
+                            "type": "boolean",
+                            "default": true,
+                            "description": "Enables loading root store trusted roots from the CCADB list, in addition to any roots configured in `roots.<env>.pem`. If enabled, requires a KV namespace with the binding `ccadb_roots` to be configured in `wrangler.jsonc`, as well as a cron trigger so that the CCADB list auto-updates."
                         }
                     },
                     "required": [

crates/ct_worker/config.schema.json

@@ -40,6 +40,8 @@ pub struct LogParams {
     pub max_batch_entries: usize,
     #[serde(default = "default_bool::<true>")]
     pub enable_dedup: bool,
+    #[serde(default = "default_bool::<true>")]
+    pub enable_ccadb_roots: bool,
 }
 
 fn default_bool<const V: bool>() -> bool {