diff --git a/Cargo.lock b/Cargo.lock index 1c9f648e..bcd7ed78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1454,7 +1454,6 @@ dependencies = [ name = "mtc_api" version = "0.2.0" dependencies = [ - "anyhow", "byteorder", "der", "ed25519-dalek", diff --git a/crates/ct_worker/src/sequencer_do.rs b/crates/ct_worker/src/sequencer_do.rs index f455faff..24013301 100644 --- a/crates/ct_worker/src/sequencer_do.rs +++ b/crates/ct_worker/src/sequencer_do.rs @@ -7,7 +7,8 @@ use std::time::Duration; use crate::{load_checkpoint_signers, load_origin, CONFIG}; use generic_log_worker::{ - get_durable_object_name, GenericSequencer, SequencerConfig, SEQUENCER_BINDING, + empty_checkpoint_callback, get_durable_object_name, GenericSequencer, SequencerConfig, + SEQUENCER_BINDING, }; use static_ct_api::StaticCTLogEntry; #[allow(clippy::wildcard_imports)] @@ -36,6 +37,7 @@ impl DurableObject for Sequencer { enable_dedup: params.enable_dedup, sequence_skip_threshold_millis: params.sequence_skip_threshold_millis, location_hint: params.location_hint.clone(), + checkpoint_callback: empty_checkpoint_callback(), }; Sequencer(GenericSequencer::new(state, env, config)) diff --git a/crates/generic_log_worker/src/log_ops.rs b/crates/generic_log_worker/src/log_ops.rs index 23bf38e9..a7c9f941 100644 --- a/crates/generic_log_worker/src/log_ops.rs +++ b/crates/generic_log_worker/src/log_ops.rs @@ -39,12 +39,11 @@ use std::{ }; use thiserror::Error; use tlog_tiles::{ - ConsistencyProof, Hash, HashReader, InclusionProof, LogEntry, PendingLogEntry, - PreloadedTlogTileReader, TileHashReader, TileIterator, TlogError, TlogTile, TlogTileRecorder, - TreeWithTimestamp, UnixTimestamp, HASH_SIZE, + Hash, HashReader, LogEntry, PendingLogEntry, PreloadedTlogTileReader, Proof, Subtree, + TileHashReader, TileIterator, TlogError, TlogTile, TlogTileRecorder, TreeWithTimestamp, + UnixTimestamp, HASH_SIZE, }; use tokio::sync::watch::{channel, Receiver, Sender}; -use worker::Error as WorkerError; /// The maximum tile level is 63 (), so safe to use [`u8::MAX`] as /// the special level for data tiles. The Go implementation uses -1. @@ -475,7 +474,7 @@ impl SequenceState { /// Proves inclusion of the last leaf in the current tree. #[cfg(test)] - pub(crate) fn prove_inclusion_of_last_elem(&self) -> InclusionProof { + pub(crate) fn prove_inclusion_of_last_elem(&self) -> Proof { let tree_size = self.tree.size(); let reader = HashReaderWithOverlay { edge_tiles: &self.edge_tiles, @@ -483,7 +482,7 @@ impl SequenceState { }; // We can unwrap because edge_tiles is guaranteed to contain the tiles // necessary to prove this. - tlog_tiles::prove_inclusion(tree_size, tree_size - 1, &reader).unwrap() + tlog_tiles::inclusion_proof(tree_size, tree_size - 1, &reader).unwrap() } /// Proves that this tree of size n is compatible with the subtree of size @@ -493,75 +492,124 @@ impl SequenceState { /// Errors when the last tree was size 0. We cannot prove consistency with /// respect to an empty tree #[cfg(test)] - pub(crate) fn prove_consistency_of_single_append(&self) -> Result { + pub(crate) fn prove_consistency_of_single_append(&self) -> Result { let tree_size = self.tree.size(); let reader = HashReaderWithOverlay { edge_tiles: &self.edge_tiles, overlay: &HashMap::default(), }; - tlog_tiles::prove_consistency(tree_size, tree_size - 1, &reader) + tlog_tiles::consistency_proof(tree_size, tree_size - 1, &reader) } } -/// Returns an inclusion proof for the given leaf index, tree size, and root hash. +#[derive(Error, Debug)] +pub enum ProofError { + #[error(transparent)] + Tlog(#[from] tlog_tiles::TlogError), + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +/// Returns an inclusion proof that the leaf at index `leaf_index` is included +/// in the current tree of size `cur_tree_size` with hash `cur_tree_hash`. /// /// # Errors -/// Errors when the leaf index equals or exceeds the number of leaves, or -/// the desired tiles do not exist as bucket objects. +/// +/// Errors when the leaf index is not within the tree, or the desired tiles do +/// not exist as bucket objects. pub async fn prove_inclusion( - tree_size: u64, - tree_hash: Hash, + cur_tree_size: u64, + cur_tree_hash: Hash, leaf_index: u64, object: &impl ObjectBackend, -) -> Result { - if leaf_index >= tree_size { - return Err(WorkerError::RustError( - "leaf index exceeds number of leaves in the tree".to_string(), - )); - } +) -> Result { + prove_subtree_inclusion( + cur_tree_size, + cur_tree_hash, + 0, + cur_tree_size, + leaf_index, + object, + ) + .await +} +/// Returns an inclusion proof that the leaf at index `leaf_index` is included +/// in the subtree `[start, end)`. `cur_tree_size` and `cur_tree_hash` allow us +/// to select the correct partial tiles. +/// +/// # Errors +/// +/// Errors when the leaf index is not within the subtree, or the desired tiles +/// do not exist as bucket objects. +pub async fn prove_subtree_inclusion( + cur_tree_size: u64, + cur_tree_hash: Hash, + start: u64, + end: u64, + leaf_index: u64, + object: &impl ObjectBackend, +) -> Result { // Fetch the tiles needed for the proof. - let indexes = tlog_tiles::inclusion_proof_indexes(0, tree_size, leaf_index, Vec::new()); - let tile_reader = tile_reader_for_indexes(tree_size, &indexes, object) - .await - .map_err(|e| e.to_string())?; + let n = &Subtree::new(start, end)?; + let indexes = tlog_tiles::subtree_inclusion_proof_indexes(n, leaf_index)?; + let tile_reader = tile_reader_for_indexes(cur_tree_size, &indexes, object).await?; + let hash_reader = TileHashReader::new(cur_tree_size, cur_tree_hash, &tile_reader); - // Verify the proof. - let hash_reader = TileHashReader::new(tree_size, tree_hash, &tile_reader); - tlog_tiles::prove_inclusion(tree_size, leaf_index, &hash_reader) - .map_err(|e| WorkerError::RustError(e.to_string())) + // Construct the proof. + Ok(tlog_tiles::subtree_inclusion_proof( + n, + leaf_index, + &hash_reader, + )?) } -/// Returns a consistency proof, proving the tree with `cur_tree_size` and hash +/// Returns a consistency proof that the tree with size `cur_tree_size` and hash /// `cur_tree_hash` is an extension of the tree with `prev_tree_size`. /// /// # Errors -/// Errors when the desired tiles do not exist as bucket objects, or it's not -/// the case that `1 <= prev_tree_size <= cur_tree_size`. +/// +/// Errors when the desired tiles do not exist as bucket objects, or if the +/// proof fails. pub async fn prove_consistency( cur_tree_hash: Hash, cur_tree_size: u64, prev_tree_size: u64, object: &impl ObjectBackend, -) -> Result { - if !(1..=cur_tree_size).contains(&prev_tree_size) { - return Err("condition not met: 1 <= prev_tree_size <= cur_tree_size".into()); - } - // Fetch the tiles needed for the proof. - let indexes = - tlog_tiles::consistency_proof_indexes(0, cur_tree_size, prev_tree_size, Vec::new()); - let tile_reader = tile_reader_for_indexes(cur_tree_size, &indexes, object) - .await - .map_err(|e| e.to_string())?; +) -> Result { + prove_subtree_consistency(cur_tree_hash, cur_tree_size, 0, prev_tree_size, object).await +} - // Verify the proof. +/// Returns a consistency proof that the tree with size `cur_tree_size` and hash +/// `cur_tree_hash` is consistent with the subtree `[start, end)`. +/// +/// # Errors +/// +/// Errors when the desired tiles do not exist as bucket objects, or if the +/// proof fails. +pub async fn prove_subtree_consistency( + cur_tree_hash: Hash, + cur_tree_size: u64, + start: u64, + end: u64, + object: &impl ObjectBackend, +) -> Result { + let m = &Subtree::new(start, end)?; + // Fetch the tiles needed for the proof. + let indexes = tlog_tiles::subtree_consistency_proof_indexes(cur_tree_size, m)?; + let tile_reader = tile_reader_for_indexes(cur_tree_size, &indexes, object).await?; let hash_reader = TileHashReader::new(cur_tree_size, cur_tree_hash, &tile_reader); - tlog_tiles::prove_consistency(cur_tree_size, prev_tree_size, &hash_reader) - .map_err(|e| WorkerError::RustError(e.to_string())) + + // Construct the proof. + Ok(tlog_tiles::subtree_consistency_proof( + cur_tree_size, + m, + &hash_reader, + )?) } -/// Fetch the tree tiles containing the nodes the requested indexes, as well as -/// all tiles needed to verify those nodes. +/// Fetch the tree tiles containing the nodes at the requested indexes, as well +/// as all tiles needed to verify those nodes. async fn tile_reader_for_indexes( tree_size: u64, indexes: &[u64], @@ -1004,6 +1052,12 @@ async fn sequence_entries( ); } + // Call the checkpoint callback. This is a no-op for CT, but is used to + // update landmark checkpoints for MTC. + if let Err(e) = (config.checkpoint_callback)(n, old_time, timestamp).await { + warn!("{name}: Checkpoint callback failed: {e}"); + } + for tile in new.edge_tiles { trace!("{name}: Edge tile: {tile:?}"); } @@ -1121,6 +1175,68 @@ async fn read_edge_tiles( Ok(edge_tiles) } +/// Read and verify a single log entry at `leaf_index`. +/// +/// # Errors +/// +/// Returns an error if the leaf is not successfully read or verified. +pub async fn read_leaf( + object: &impl ObjectBackend, + leaf_index: u64, + tree_size: u64, + tree_hash: &Hash, +) -> Result { + let leaf_stored_hash_index = tlog_tiles::stored_hash_index(0, leaf_index); + let tile_reader = tile_reader_for_indexes(tree_size, &[leaf_stored_hash_index], object).await?; + + // Verify the leaf tile against the tree hash. + let hash_reader = TileHashReader::new(tree_size, *tree_hash, &tile_reader); + let hashes = hash_reader + .read_hashes(&[leaf_stored_hash_index]) + .map_err(|e| anyhow!(e))?; + let leaf_hash = hashes.first().ok_or(anyhow!("too many hashes read"))?; + + // Get the level-0 tile. There will be two level-0 tiles in the reader, so get the one matching the requested hash. + let Some((level0_tile, level0_tile_bytes)) = tile_reader.0.into_iter().find(|(tile, b)| { + tile.level() == 0 + && tile + .hash_at_index(b, leaf_stored_hash_index) + .is_ok_and(|h| h == *leaf_hash) + }) else { + bail!("failed to get level-0 tile"); + }; + + // Get the data tile. + let data_tile = level0_tile.with_data_path(L::Pending::DATA_TILE_PATH); + let data_tile_bytes = object + .fetch(&data_tile.path()) + .await? + .ok_or(anyhow!("no data tile in object storage"))?; + + // Verify the data tile against the level 0 tile. + let start = u64::from(TlogTile::FULL_WIDTH) * data_tile.level_index(); + for (i, entry_res) in + TileIterator::::new(&data_tile_bytes, data_tile.width() as usize).enumerate() + { + let entry = entry_res?; + let got = entry.merkle_tree_leaf(); + let exp = level0_tile.hash_at_index( + &level0_tile_bytes, + tlog_tiles::stored_hash_index(0, start + i as u64), + )?; + if got != exp { + bail!( + "tile leaf entry {} hashes to {got}, level 0 hash is {exp}", + start + i as u64, + ); + } + if leaf_index == start + i as u64 { + return Ok(entry); + } + } + bail!("did not find leaf") +} + /// Returns hashes from `edge_tiles` or from the overlay cache. #[derive(Debug)] struct HashReaderWithOverlay<'a> { @@ -1244,7 +1360,7 @@ pub async fn upload_issuers( #[cfg(test)] mod tests { use super::*; - use crate::util; + use crate::{empty_checkpoint_callback, util}; use anyhow::ensure; use ed25519_dalek::SigningKey as Ed25519SigningKey; @@ -1301,7 +1417,7 @@ mod tests { let leaf_edge = &sequence_state.edge_tiles.get(&0u8).unwrap().b; Hash(leaf_edge[leaf_edge.len() - HASH_SIZE..].try_into().unwrap()) }; - tlog_tiles::check_inclusion( + tlog_tiles::verify_inclusion_proof( &inc_proof, tree_size, new_tree_hash, @@ -1315,11 +1431,10 @@ mod tests { // Make a consistency proof. Just can't do it with a size-0 subtree if i > 0 { - let consistency_proof = - sequence_state.prove_consistency_of_single_append().unwrap(); + let proof = sequence_state.prove_consistency_of_single_append().unwrap(); // Verify the proof - tlog_tiles::check_consistency( - &consistency_proof, + tlog_tiles::verify_consistency_proof( + &proof, tree_size, new_tree_hash, tree_size - 1, @@ -1327,14 +1442,14 @@ mod tests { ) .unwrap(); // Check that the other way of constructing the consistency proof is the same - let consistency_proof2 = block_on(prove_consistency( + let proof2 = block_on(prove_consistency( new_tree_hash, tree_size, tree_size - 1, &log.object, )) .unwrap(); - assert_eq!(consistency_proof, consistency_proof2); + assert_eq!(proof, proof2); } } // Check that the static CT log is valid @@ -1368,7 +1483,7 @@ mod tests { ) }; // Verify the inclusion proof - tlog_tiles::check_inclusion(&proof, n, tree_hash, i, leaf_hash).unwrap(); + tlog_tiles::verify_inclusion_proof(&proof, n, tree_hash, i, leaf_hash).unwrap(); } // Check that we can make a consistency proof for random spans in the tree @@ -1383,7 +1498,7 @@ mod tests { &log.object, )) .unwrap(); - tlog_tiles::check_consistency( + tlog_tiles::verify_consistency_proof( &consistency_proof, new_tree_size, tree_hashes[usize::try_from(new_tree_size).unwrap()], @@ -2312,6 +2427,7 @@ mod tests { enable_dedup: true, sequence_skip_threshold_millis: None, location_hint: None, + checkpoint_callback: empty_checkpoint_callback(), }; let pool_state = RefCell::new(PoolState::default()); block_on(create_log(&config, &object, &lock)).unwrap(); diff --git a/crates/generic_log_worker/src/sequencer_do.rs b/crates/generic_log_worker/src/sequencer_do.rs index eb1507c1..bb479263 100644 --- a/crates/generic_log_worker/src/sequencer_do.rs +++ b/crates/generic_log_worker/src/sequencer_do.rs @@ -3,7 +3,7 @@ //! Sequencer is the 'brain' of the CT log, responsible for sequencing entries and maintaining log state. -use std::{cell::RefCell, time::Duration}; +use std::{cell::RefCell, future::Future, pin::Pin, time::Duration}; use crate::{ deserialize, get_durable_object_stub, load_public_bucket, @@ -54,6 +54,7 @@ pub struct SequencerConfig { pub sequence_skip_threshold_millis: Option, pub enable_dedup: bool, pub location_hint: Option, + pub checkpoint_callback: CheckpointCallbacker, } impl GenericSequencer { @@ -287,3 +288,36 @@ impl GenericSequencer { Response::ok(buffer) } } + +/// A callback function that gets called each time the sequencer updates the +/// checkpoint. Currently, this is used only to update the landmark checkpoint +/// sequence for MTC, but could be extended in the future to collect +/// cosignatures or perform other application-specific actions. +/// +/// This is a `Fn` closure that returns nothing, which might be surprising. The +/// callback is meant to capture values with interior mutability (e.g., Buckets) +/// so that it can have side-effects. +/// +/// The parameters are as follows: +/// - `tree_size: u64`: The tree size of the latest checkpoint. +/// - `old_time: UnixTimestamp`: The timestamp of the previous checkpoint. +/// - `new_time: UnixTimestamp`: The timestamp of the latest checkpoint. +pub type CheckpointCallbacker = Box< + dyn Fn( + u64, + UnixTimestamp, + UnixTimestamp, + ) -> Pin> + 'static>> + + 'static, +>; + +/// A no-op checkpoint callback that can be used in applications like CT that +/// don't need to perform any action after the checkpoint is updated. +pub fn empty_checkpoint_callback() -> CheckpointCallbacker { + Box::new( + move |_tree_size: u64, _old_time: UnixTimestamp, _new_time: UnixTimestamp| { + Box::pin(async move { Ok(()) }) + as Pin>>> + }, + ) +} diff --git a/crates/mtc_api/Cargo.toml b/crates/mtc_api/Cargo.toml index 4d17c245..cc1c1753 100644 --- a/crates/mtc_api/Cargo.toml +++ b/crates/mtc_api/Cargo.toml @@ -10,7 +10,6 @@ repository.workspace = true description.workspace = true [dependencies] -anyhow.workspace = true byteorder.workspace = true der.workspace = true ed25519-dalek.workspace = true diff --git a/crates/mtc_api/src/landmark.rs b/crates/mtc_api/src/landmark.rs new file mode 100644 index 00000000..eb87bf7e --- /dev/null +++ b/crates/mtc_api/src/landmark.rs @@ -0,0 +1,236 @@ +use crate::MtcError; +use std::{collections::VecDeque, fmt::Write}; +use tlog_tiles::Subtree; + +#[derive(Debug, PartialEq, Clone)] +pub struct LandmarkSequence { + max_landmarks: usize, + last_landmark: usize, + landmarks: VecDeque, +} + +/// The location in object storage for the landmark bundle. +pub static LANDMARK_KEY: &str = "landmark"; + +impl LandmarkSequence { + /// Create a new landmark sequence with the given `max_landmarks` and an + /// initial landmark with id 0 and tree size 0. + pub fn create(max_landmarks: usize) -> Self { + Self { + max_landmarks, + last_landmark: 0, + landmarks: VecDeque::from(vec![0]), + } + } + /// Get the first index that is covered by the landmark sequence. + /// + /// # Panics + /// + /// Panics if the landmark sequence is empty, which should never happen. + pub fn first_index(&self) -> u64 { + *self.landmarks.front().expect("landmark sequence is empty") + } + /// Add a new landmark with the given tree size, removing a landmark if the + /// maximum size would be exceeded. Returns true if the new landmark is + /// added, or false otherwise. + /// + /// # Errors + /// + /// Will return an error if the tree size is smaller than the last landmark + /// tree size. + pub fn add(&mut self, tree_size: u64) -> Result { + if let Some(last) = self.landmarks.back() { + if tree_size == *last { + // The last landmark is unchanged. + return Ok(false); + } + if tree_size < *last { + return Err(MtcError::Dynamic( + "landmark sequence must be strictly increasing".into(), + )); + } + } + // Keep `max_landmarks + 1` tree sizes, since we want `max_landmarks` + // landmark intervals. + if self.landmarks.len() == self.max_landmarks + 1 { + self.landmarks.pop_front(); + } + self.landmarks.push_back(tree_size); + self.last_landmark += 1; + Ok(true) + } + + /// Return the landmark ID and subtree covering `leaf_index`, or `None` if + /// the `leaf_index` is not covered by a landmark range. + /// + /// # Panics + /// + /// Will panic if landmarks are not sorted or are not unique. + pub fn subtree_for_index(&self, leaf_index: u64) -> Option<(usize, Subtree)> { + // Find the index of the first landmark greater than the leaf index. + let hi_index = self + .landmarks + .partition_point(|&landmark| landmark <= leaf_index); + + // Get the lower index, if it exists. + let lo_index = hi_index.checked_sub(1)?; + + // Return the ID of the higher landmark. + let landmark_id = hi_index + (self.last_landmark + 1 - self.landmarks.len()); + + // Get lo and hi landmarks, if they exist. + let &lo = self.landmarks.get(lo_index)?; + let &hi = self.landmarks.get(hi_index)?; + + // Find which landmark subtree within `[lo, hi)` contains the leaf. + let (left, right) = Subtree::split_interval(lo, hi).unwrap(); + if left.contains(leaf_index) { + Some((landmark_id, left)) + } else { + right.map(|tree| (landmark_id, tree)) + } + } + + /// Serialize according to + /// . + /// + /// # Errors + /// + /// Will return an error if writing to the buffer fails. + pub fn to_bytes(&self) -> Result, MtcError> { + let mut buffer = format!("{} {}\n", self.last_landmark, self.landmarks.len() - 1); + for landmark in self.landmarks.iter().rev() { + writeln!(buffer, "{landmark}")?; + } + Ok(buffer.into_bytes()) + } + + /// Deserialize according to + /// . + /// + /// # Errors + /// + /// Will return an error if the landmark sequence is invalid or if + /// `data.len() > 10_000`. + pub fn from_bytes(data: &[u8], max_landmarks: usize) -> Result { + // Note: `lines()` will return the same thing whether or not there's a + // newline after the last line, and whether or not there are carriage + // returns preceding each newline. + + // Set some upper limit on what we're willing to process. + if data.len() > 10_000 { + return Err(MtcError::Dynamic("too much data".into())); + } + let mut iter = std::str::from_utf8(data)?.lines(); + let first = iter + .next() + .ok_or(MtcError::Dynamic("missing first line".into()))? + .split_once(' ') + .ok_or(MtcError::Dynamic("malformed first line".into()))?; + let last_landmark = first.0.parse::()?; + let num_active_landmarks = first.1.parse::()?; + + if num_active_landmarks > max_landmarks { + return Err(MtcError::Dynamic( + "num_active_landmarks must not be greater than max_landmarks".into(), + )); + } + if num_active_landmarks > last_landmark { + return Err(MtcError::Dynamic( + "num_active_landmarks must not be greater than last_landmark".into(), + )); + } + + let mut landmarks = VecDeque::with_capacity(num_active_landmarks + 1); + for i in 0..=num_active_landmarks { + let landmark = iter + .next() + .ok_or(MtcError::Dynamic("malformed landmark line".into()))? + .parse::()?; + if i > 0 && landmark >= landmarks[0] { + return Err(MtcError::Dynamic( + "landmarks must be in decreasing order".into(), + )); + } + landmarks.push_front(landmark); + } + if iter.next().is_some() { + return Err(MtcError::Dynamic( + "trailing data in landmark sequence".into(), + )); + } + Ok(Self { + max_landmarks, + last_landmark, + landmarks, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_subtree_for_index() { + let mut seq = LandmarkSequence::create(10); + assert_eq!(seq.first_index(), 0); + // Only have a single landmark so no subtrees yet. + assert!(seq.subtree_for_index(0).is_none()); + // Check landmark sequence at partial capacity. + for i in 1..=5 { + seq.add(i * 10).unwrap(); + } + assert_eq!(seq.first_index(), 0); + // At first landmark. + assert_eq!( + seq.subtree_for_index(0), + Some((1, Subtree::new(0, 8).unwrap())) + ); + // Past last landmark. + assert!(seq.subtree_for_index(50).is_none()); + // Valid landmark, left subtree aligned with lower landmark tree size. + assert_eq!( + seq.subtree_for_index(31), + Some((4, Subtree::new(30, 32).unwrap())) + ); + // Valid landmark, left subtree extending beyond lower landmark tree + // size. + assert_eq!( + seq.subtree_for_index(12), + Some((2, Subtree::new(8, 16).unwrap())) + ); + // Valid landmark, right subtree. + assert_eq!( + seq.subtree_for_index(33), + Some((4, Subtree::new(32, 40).unwrap())) + ); + + // New tree size matching the last landmark tree size is ignored. + let old_seq = seq.clone(); + seq.add(50).unwrap(); + assert_eq!(seq, old_seq); + // Error if we try to add a smaller tree size. + assert!(seq.add(49).is_err()); + + // Put landmark sequence at full capacity. + for i in 6..=20 { + seq.add(i * 10).unwrap(); + } + assert_eq!(seq.first_index(), 100); + // Before first landmark. + assert!(seq.subtree_for_index(99).is_none()); + // Just within first landmark. + assert_eq!( + seq.subtree_for_index(100), + Some((11, Subtree::new(100, 104).unwrap())) + ); + // At last landmark. + assert_eq!( + seq.subtree_for_index(199), + Some((20, Subtree::new(192, 200).unwrap())) + ); + // Past last landmark. + assert!(seq.subtree_for_index(200).is_none()); + } +} diff --git a/crates/mtc_api/src/lib.rs b/crates/mtc_api/src/lib.rs index 9cad1b1a..db91787d 100644 --- a/crates/mtc_api/src/lib.rs +++ b/crates/mtc_api/src/lib.rs @@ -1,17 +1,18 @@ // Copyright (c) 2025 Cloudflare, Inc. // Licensed under the BSD-3-Clause license found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause +mod landmark; mod relative_oid; mod subtree_cosignature; +pub use landmark::*; pub use relative_oid::*; pub use subtree_cosignature::*; -use anyhow::{anyhow, bail, ensure}; -use byteorder::{BigEndian, ReadBytesExt}; +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use der::{ asn1::{BitString, OctetString}, oid::{db::rfc5280, ObjectIdentifier}, - Decode, Encode, Sequence, ValueOrd, + Any, Decode, Encode, Sequence, ValueOrd, }; use length_prefixed::WriteLengthPrefixedBytesExt; use serde::{Deserialize, Serialize}; @@ -24,8 +25,8 @@ use std::{ }; use thiserror::Error; use tlog_tiles::{ - Hash, LeafIndex, LogEntry, PathElem, PendingLogEntry, SequenceMetadata, TlogError, - TlogTilesLogEntry, TlogTilesPendingLogEntry, UnixTimestamp, + Hash, LeafIndex, LogEntry, PathElem, PendingLogEntry, Proof, SequenceMetadata, Subtree, + TlogError, TlogTilesLogEntry, TlogTilesPendingLogEntry, UnixTimestamp, }; use x509_cert::{ certificate::Version, @@ -34,6 +35,8 @@ use x509_cert::{ Extension, }, name::RdnSequence, + serial_number::SerialNumber, + spki::{AlgorithmIdentifier, SubjectPublicKeyInfo}, time::Validity, Certificate, TbsCertificate, }; @@ -44,6 +47,65 @@ use x509_util::CertPool; pub const ID_RDNA_TRUSTANCHOR_ID: ObjectIdentifier = ObjectIdentifier::new_unwrap("1.3.6.1.4.1.44363.47.1"); +// The OID to use for experimentaion. Eventually, we'll switch to "1.3.6.1.5.5.7.6.TBD" +// as described in . +pub const ID_ALG_MTCPROOF: ObjectIdentifier = + ObjectIdentifier::new_unwrap("1.3.6.1.4.1.44363.47.0"); + +// MTCSignature from . +struct MtcSignature { + cosigner_id: TrustAnchorID, + signature: Vec, +} + +impl MtcSignature { + fn to_bytes(&self) -> Vec { + let mut buffer = Vec::new(); + buffer + .write_length_prefixed(&self.cosigner_id.0, 1) + .unwrap(); + buffer.write_length_prefixed(&self.signature, 2).unwrap(); + buffer + } +} + +// MTCProof from . +struct MtcProof { + start: u64, + end: u64, + inclusion_proof: Proof, + signatures: Vec, +} + +impl MtcProof { + fn to_bytes(&self) -> Vec { + let mut buffer = Vec::new(); + buffer.write_u64::(self.start).unwrap(); + buffer.write_u64::(self.end).unwrap(); + buffer + .write_length_prefixed( + &self + .inclusion_proof + .iter() + .flat_map(|h| h.0.to_vec()) + .collect::>(), + 2, + ) + .unwrap(); + buffer + .write_length_prefixed( + &self + .signatures + .iter() + .flat_map(MtcSignature::to_bytes) + .collect::>(), + 2, + ) + .unwrap(); + buffer + } +} + /// Add-entry request. Chain is a certificate from which to bootstrap the /// request in the same format as RFC6962 add-chain requests. #[serde_as] @@ -123,10 +185,65 @@ impl LogEntry for BootstrapMtcLogEntry { } } +/// Return the serialized DER-encoded bytes of a signatureless certificate. +/// +/// # Errors +/// +/// Will return an error if the hash of `spki` does not match that in the log +/// entry, or if there are any serialization errors. +pub fn serialize_signatureless_cert( + log_entry: &BootstrapMtcLogEntry, + leaf_index: LeafIndex, + spki_der: &[u8], + subtree: &Subtree, + inclusion_proof: Proof, +) -> Result, MtcError> { + let entry = match MerkleTreeCertEntry::decode(&log_entry.0.inner.data)? { + MerkleTreeCertEntry::TbsCertEntry(entry) => entry, + MerkleTreeCertEntry::NullEntry => { + return Err(MtcError::Dynamic("no tbs cert entry for null entry".into())) + } + }; + let spki: SubjectPublicKeyInfo = SubjectPublicKeyInfo::from_der(spki_der)?; + let spki_hash = OctetString::new(Sha256::digest(spki_der).as_slice())?; + if spki_hash != entry.subject_public_key_info_hash { + return Err(MtcError::Dynamic("spki hash mismatch".to_string())); + } + let signature_algorithm = AlgorithmIdentifier { + oid: ID_ALG_MTCPROOF, + parameters: None, + }; + + let tbs_certificate = TbsCertificate { + version: entry.version, + serial_number: SerialNumber::new(&leaf_index.to_be_bytes())?, + signature: signature_algorithm.clone(), + issuer: entry.issuer, + validity: entry.validity, + subject: entry.subject, + subject_public_key_info: spki, + issuer_unique_id: entry.issuer_unique_id, + subject_unique_id: entry.subject_unique_id, + extensions: entry.extensions, + }; + let certificate = Certificate { + tbs_certificate, + signature_algorithm, + signature: BitString::from_bytes( + &MtcProof { + start: subtree.lo(), + end: subtree.hi(), + inclusion_proof, + signatures: Vec::new(), + } + .to_bytes(), + )?, + }; + Ok(certificate.to_der()?) +} + #[derive(Debug, Error)] pub enum MtcError { - #[error(transparent)] - Anyhow(#[from] anyhow::Error), #[error(transparent)] Tlog(#[from] TlogError), #[error(transparent)] @@ -134,13 +251,13 @@ pub enum MtcError { #[error(transparent)] IO(#[from] std::io::Error), #[error(transparent)] + Fmt(#[from] std::fmt::Error), + #[error(transparent)] + Utf8(#[from] std::str::Utf8Error), + #[error(transparent)] ParseInt(#[from] ParseIntError), - #[error("empty chain")] - EmptyChain, - #[error("invalid relative OID")] - InvalidRelativeOID, - #[error("unknown entry type")] - UnknownEntryType, + #[error("mtc: {0}")] + Dynamic(String), } #[repr(u16)] @@ -156,7 +273,7 @@ impl TryFrom for MerkleTreeCertEntryType { match value { 0x0000 => Ok(MerkleTreeCertEntryType::NullEntry), 0x0001 => Ok(MerkleTreeCertEntryType::TbsCertEntry), - _ => Err(MtcError::UnknownEntryType), + _ => Err(MtcError::Dynamic("unknown entry type".into())), } } } @@ -179,7 +296,7 @@ impl MerkleTreeCertEntry { /// # Errors /// /// Will return an error if there are issues encoding the entry. - pub fn encode(&self) -> Result, anyhow::Error> { + pub fn encode(&self) -> Result, MtcError> { match &self { Self::NullEntry => Ok((MerkleTreeCertEntryType::NullEntry as u16) .to_be_bytes() @@ -199,13 +316,15 @@ impl MerkleTreeCertEntry { /// # Errors /// /// Returns an error if the entry cannot be decoded. - pub fn decode(mut data: &[u8]) -> Result { + pub fn decode(mut data: &[u8]) -> Result { match MerkleTreeCertEntryType::try_from(data.read_u16::()?)? { MerkleTreeCertEntryType::NullEntry => { if data.is_empty() { Ok(Self::NullEntry) } else { - Err(anyhow!("data for null entry must be empty")) + Err(MtcError::Dynamic( + "data for null entry must be empty".into(), + )) } } MerkleTreeCertEntryType::TbsCertEntry => { @@ -229,7 +348,7 @@ pub struct TbsCertificateLogEntry { } // Validate and filter extended key usage extension. -fn filter_ext_key_usage(extension: &mut Extension) -> Result<(), anyhow::Error> { +fn filter_ext_key_usage(extension: &mut Extension) -> Result<(), MtcError> { let mut eku = ExtendedKeyUsage::from_der(extension.extn_value.as_bytes())?; // Require ip-kp-serverAuth, filter id-kp-clientAuth, and disallow everything else. // @@ -242,21 +361,28 @@ fn filter_ext_key_usage(extension: &mut Extension) -> Result<(), anyhow::Error> false } }); - ensure!(!is_err, "unexpected key usage"); - ensure!(!eku.0.is_empty(), "key usage missing id-kp-serverAuth"); + if is_err { + return Err(MtcError::Dynamic("unexpected key usage".into())); + } + if eku.0.is_empty() { + return Err(MtcError::Dynamic( + "key usage missing id-kp-serverAuth".into(), + )); + } extension.extn_value = OctetString::new(eku.to_der()?)?; Ok(()) } // Validate and filter key usage extension. -fn filter_key_usage(extension: &mut Extension) -> Result<(), anyhow::Error> { +fn filter_key_usage(extension: &mut Extension) -> Result<(), MtcError> { let mut ku = KeyUsage::from_der(extension.extn_value.as_bytes())?; // Require digital_signature, allow key_encipherment, and filter everything else. ku.0 &= KeyUsages::DigitalSignature | KeyUsages::KeyEncipherment; - ensure!( - ku.0.contains(KeyUsages::DigitalSignature), - "key usage missing DigitalSignature" - ); + if !ku.0.contains(KeyUsages::DigitalSignature) { + return Err(MtcError::Dynamic( + "key usage missing DigitalSignature".into(), + )); + } extension.extn_value = OctetString::new(ku.to_der()?)?; Ok(()) } @@ -267,12 +393,12 @@ fn filter_key_usage(extension: &mut Extension) -> Result<(), anyhow::Error> { // // Will return an error if there are any duplicate extensions, or if there are // any critical extensions that cannot be filtered out. -fn filter_extensions(extensions: &mut Vec) -> Result<(), anyhow::Error> { +fn filter_extensions(extensions: &mut Vec) -> Result<(), MtcError> { let mut result = Ok(()); let mut oids = BTreeSet::new(); extensions.retain_mut(|extension| { if oids.contains(&extension.extn_id) { - result = Err(anyhow!("duplicate extension")); + result = Err(MtcError::Dynamic("duplicate extension".into())); return false; } oids.insert(extension.extn_id); @@ -298,7 +424,9 @@ fn filter_extensions(extensions: &mut Vec) -> Result<(), anyhow::Erro | rfc5280::ID_CE_BASIC_CONSTRAINTS => false, id => { if extension.critical { - result = Err(anyhow!("unsupported critical extension {id}")); + result = Err(MtcError::Dynamic(format!( + "unsupported critical extension {id}" + ))); } false } @@ -318,16 +446,28 @@ pub fn tbs_cert_to_log_entry( bootstrap: TbsCertificate, issuer: RdnSequence, validity: Validity, -) -> Result { - ensure!(bootstrap.version == Version::V3); - ensure!(validity +) -> Result { + if bootstrap.version != Version::V3 { + return Err(MtcError::Dynamic("bootstrap version must be v3".into())); + } + if validity .not_before .to_unix_duration() - .ge(&bootstrap.validity.not_before.to_unix_duration())); - ensure!(validity + .lt(&bootstrap.validity.not_before.to_unix_duration()) + { + return Err(MtcError::Dynamic( + "entry not_before must not be less than bootstrap not_before".into(), + )); + } + if validity .not_after .to_unix_duration() - .le(&bootstrap.validity.not_after.to_unix_duration())); + .gt(&bootstrap.validity.not_after.to_unix_duration()) + { + return Err(MtcError::Dynamic( + "entry not_after must not be greater than bootstrap not_after".into(), + )); + }; let extensions = if let Some(mut bootstrap_extensions) = bootstrap.extensions { filter_extensions(&mut bootstrap_extensions)?; @@ -359,35 +499,66 @@ pub fn tbs_cert_to_log_entry( pub fn validate_correspondence( log_entry: &TbsCertificateLogEntry, chain: &[Certificate], -) -> Result<(), anyhow::Error> { +) -> Result<(), MtcError> { // TODO validate bootstrap chain - ensure!(!chain.is_empty()); + if chain.is_empty() { + return Err(MtcError::Dynamic( + "bootstrap chain must not be empty".into(), + )); + } let bootstrap = chain[0].tbs_certificate.clone(); - ensure!(log_entry.version == bootstrap.version && log_entry.version == Version::V3); + if !(log_entry.version == bootstrap.version && log_entry.version == Version::V3) { + return Err(MtcError::Dynamic( + "entry and bootstrap versions must be v3".into(), + )); + } // Make sure the validity is contained within the validity of every cert in // the chain. for cert in chain { - ensure!(log_entry.validity.not_after.to_unix_duration().le(&cert + if log_entry.validity.not_before.to_unix_duration().lt(&cert .tbs_certificate .validity - .not_after - .to_unix_duration())); - ensure!(log_entry.validity.not_before.to_unix_duration().ge(&cert + .not_before + .to_unix_duration()) + { + return Err(MtcError::Dynamic( + "entry not_before must not be less than bootstrap chain cert not_before".into(), + )); + } + if log_entry.validity.not_after.to_unix_duration().gt(&cert .tbs_certificate .validity - .not_before - .to_unix_duration())); + .not_after + .to_unix_duration()) + { + return Err(MtcError::Dynamic( + "entry not_after must not be greater than bootstrap chain cert not_after".into(), + )); + } + } + if log_entry.subject != bootstrap.subject { + return Err(MtcError::Dynamic( + "entry subject must match bootstrap subject".into(), + )); + } + if log_entry.subject_public_key_info_hash + != OctetString::new(Sha256::digest(bootstrap.subject_public_key_info.to_der()?).as_slice())? + { + return Err(MtcError::Dynamic( + "entry spki hash must match hash of bootstrap spki".into(), + )); + } + if log_entry.issuer_unique_id != bootstrap.issuer_unique_id { + return Err(MtcError::Dynamic( + "entry issuer unique ID must match bootstrap issuer unique ID".into(), + )); + } + if log_entry.subject_unique_id != bootstrap.subject_unique_id { + return Err(MtcError::Dynamic( + "entry subject unique ID must match bootstrap subject unique ID".into(), + )); } - ensure!(log_entry.subject == bootstrap.subject); - ensure!( - log_entry.subject_public_key_info_hash - == OctetString::new( - Sha256::digest(bootstrap.subject_public_key_info.to_der()?,).as_slice(), - )? - ); - ensure!(log_entry.issuer_unique_id == bootstrap.issuer_unique_id); - ensure!(log_entry.subject_unique_id == bootstrap.subject_unique_id); match (log_entry.extensions.as_ref(), bootstrap.extensions) { (None, None) => {} @@ -396,7 +567,12 @@ pub fn validate_correspondence( filter_extensions(&mut bootstrap_extensions)?; // Make sure the filtered bootstrap extensions cover those of the log entry. - ensure!(log_entry_extensions.len() == bootstrap_extensions.len()); + if log_entry_extensions.len() != bootstrap_extensions.len() { + return Err(MtcError::Dynamic( + "bootstrap extension lengths differ".into(), + )); + }; + let bootstrap_extensions_map = bootstrap_extensions .into_iter() .map(|extn| (extn.extn_id, extn)) @@ -412,16 +588,26 @@ pub fn validate_correspondence( // bootstrap cert with the key usage // DigitalSignature+KeyEncipherment to cover a log // entry with only the DigitalSignature key usage. - ensure!(extension == bootstrap_extension); + if extension != bootstrap_extension { + return Err(MtcError::Dynamic(format!( + "boostrap extension mismatch {id}" + ))); + } } else { - bail!("bootstrap missing extension {id}"); + return Err(MtcError::Dynamic(format!( + "bootstrap missing extension {id}" + ))); } } - id => bail!("log entry has unsupported extension {id}"), + id => { + return Err(MtcError::Dynamic(format!( + "log entry has unsupported extension {id}" + ))) + } } } } - _ => bail!("mismatched extensions"), + _ => return Err(MtcError::Dynamic("mismatched extensions".into())), } Ok(()) @@ -441,7 +627,7 @@ pub fn validate_chain( let mut iter = raw_chain.iter(); let leaf: Certificate = match iter.next() { Some(v) => Certificate::from_der(v)?, - None => return Err(MtcError::EmptyChain), + None => return Err(MtcError::Dynamic("empty bootstrap chain".into())), }; // TODO actually validate chain @@ -449,7 +635,17 @@ pub fn validate_chain( .map(|x| Certificate::from_der(x)) .collect::, der::Error>>()?; + // Adjust the validity bound to the overlapping part of validity periods of + // all certificates in the chain. for cert in std::iter::once(&leaf).chain(&chain) { + if validity.not_before.to_unix_duration().lt(&cert + .tbs_certificate + .validity + .not_before + .to_unix_duration()) + { + validity.not_before = cert.tbs_certificate.validity.not_before; + } if validity.not_after.to_unix_duration().gt(&cert .tbs_certificate .validity @@ -458,6 +654,17 @@ pub fn validate_chain( { validity.not_after = cert.tbs_certificate.validity.not_after; } + // Check that we still have a non-empty validity period. + if validity + .not_after + .to_unix_duration() + .le(&validity.not_before.to_unix_duration()) + { + // There is no remaining validity period. + return Err(MtcError::Dynamic( + "overlap in validity with bootstrap chain must not be empty".into(), + )); + } } let mut bootstrap = Vec::new(); diff --git a/crates/mtc_api/src/relative_oid.rs b/crates/mtc_api/src/relative_oid.rs index a57b54d7..ad03f760 100644 --- a/crates/mtc_api/src/relative_oid.rs +++ b/crates/mtc_api/src/relative_oid.rs @@ -27,7 +27,7 @@ impl RelativeOid { } } if ber.len() > 255 { - return Err(MtcError::InvalidRelativeOID); + return Err(MtcError::Dynamic("invalid relative OID".into())); } Ok(Self { ber }) } diff --git a/crates/mtc_worker/config.dev.json b/crates/mtc_worker/config.dev.json index 9db0abb1..59f48aec 100644 --- a/crates/mtc_worker/config.dev.json +++ b/crates/mtc_worker/config.dev.json @@ -13,7 +13,9 @@ "log_id": "13335.2", "submission_url": "http://localhost:8787/logs/dev2/", "location_hint": "enam", - "enable_dedup": false + "enable_dedup": false, + "max_certificate_lifetime_secs": 100, + "landmark_interval_secs": 10 } } } \ No newline at end of file diff --git a/crates/mtc_worker/config.schema.json b/crates/mtc_worker/config.schema.json index 2c4be3cc..2d0d716e 100644 --- a/crates/mtc_worker/config.schema.json +++ b/crates/mtc_worker/config.schema.json @@ -30,10 +30,16 @@ "type": "string", "description": "The log name (a trust anchor ID) in dotted decimal notation (e.g., 32473.1)." }, - "validity_interval_seconds": { + "max_certificate_lifetime_secs": { "type": "integer", "default": 604800, - "description": "The maximum validity interval for issued certificates. The actual validity window could be less, for example, to fit within the bootstrap certificate's validity." + "description": "The maximum lifetime for issued certificates. The actual lifetime could be less, for example, to fit within the bootstrap certificate's validity window." + }, + "landmark_interval_secs": { + "type": "integer", + "minimum": 1, + "default": 3600, + "description": "The time between publishing landmarks. This is used to calculate `max_landmarks` as `ceil(max_certificate_timetime_secs / landmark_interval_secs) + 1`." }, "submission_url": { "type": "string", diff --git a/crates/mtc_worker/config/src/lib.rs b/crates/mtc_worker/config/src/lib.rs index c6d2c116..73285142 100644 --- a/crates/mtc_worker/config/src/lib.rs +++ b/crates/mtc_worker/config/src/lib.rs @@ -15,8 +15,10 @@ pub struct AppConfig { pub struct LogParams { pub description: Option, pub log_id: String, - #[serde(default = "default_u64::<604_800>")] - pub validity_interval_seconds: u64, + #[serde(default = "default_usize::<604_800>")] + pub max_certificate_lifetime_secs: usize, + #[serde(default = "default_usize::<3600>")] + pub landmark_interval_secs: usize, #[serde(default)] pub monitoring_url: String, pub submission_url: String, diff --git a/crates/mtc_worker/src/frontend_worker.rs b/crates/mtc_worker/src/frontend_worker.rs index 2170a2fe..7669dd45 100644 --- a/crates/mtc_worker/src/frontend_worker.rs +++ b/crates/mtc_worker/src/frontend_worker.rs @@ -14,12 +14,15 @@ use der::{ use generic_log_worker::{ batcher_id_from_lookup_key, deserialize, get_cached_metadata, get_durable_object_stub, init_logging, load_cache_kv, load_public_bucket, - log_ops::{prove_inclusion, CHECKPOINT_KEY}, + log_ops::{prove_subtree_inclusion, read_leaf, ProofError, CHECKPOINT_KEY}, put_cache_entry_metadata, serialize, util::now_millis, ObjectBackend, ObjectBucket, ENTRY_ENDPOINT, METRICS_ENDPOINT, }; -use mtc_api::{AddEntryRequest, AddEntryResponse, RelativeOid, ID_RDNA_TRUSTANCHOR_ID}; +use mtc_api::{ + serialize_signatureless_cert, AddEntryRequest, AddEntryResponse, BootstrapMtcLogEntry, + LandmarkSequence, RelativeOid, ID_RDNA_TRUSTANCHOR_ID, LANDMARK_KEY, +}; use p256::pkcs8::EncodePublicKey; use serde::{Deserialize, Serialize}; use serde_with::{base64::Base64, serde_as}; @@ -49,18 +52,23 @@ struct MetadataResponse<'a> { monitoring_url: &'a str, } -/// GET query structure for the `/prove_inclusion` endpoint +// POST body structure for the `/get-certificate` endpoint +#[serde_as] #[derive(Serialize, Deserialize)] -pub struct ProveInclusionQuery { +pub struct GetCertificateRequest { pub leaf_index: LeafIndex, + + #[serde_as(as = "Base64")] + pub spki_der: Vec, } -/// GET response structure for the `/prove_inclusion` endpoint +/// GET response structure for the `/get-certificate` endpoint #[serde_as] #[derive(Serialize, Deserialize)] -pub struct ProveInclusionResponse { - #[serde_as(as = "Vec")] - pub proof: Vec>, +pub struct GetCertificateResponse { + #[serde_as(as = "Base64")] + pub data: Vec, + pub landmark_id: usize, } /// Start is the first code run when the Wasm module is loaded. @@ -99,11 +107,16 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { .post_async("/logs/:log/add-entry", |req, ctx| async move { add_entry(req, &ctx.env, ctx.data).await }) - .get_async("/logs/:log/prove-inclusion", |req, ctx| async move { + .post_async("/logs/:log/get-certificate", |mut req, ctx| async move { let name = ctx.data; - let bucket = load_public_bucket(&ctx.env, name)?; - let ProveInclusionQuery { leaf_index } = req.query()?; - let object_backend = ObjectBucket::new(bucket); + let params = &CONFIG.logs[name]; + let GetCertificateRequest { + leaf_index, + spki_der, + } = req.json().await?; + let object_backend = ObjectBucket::new(load_public_bucket(&ctx.env, name)?); + // Fetch the current checkpoint to know which tiles to fetch + // (full or partials). let checkpoint_bytes = object_backend .fetch(CHECKPOINT_KEY) .await? @@ -124,23 +137,80 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { .map_err(|e| e.to_string())? .0; if leaf_index >= checkpoint.size() { - return Response::error( - "Leaf index is greater than current log size.", - 422, - ); + return Response::error("Leaf index is not in log", 422); } - let proof = prove_inclusion( + + let seq = if let Some(bytes) = object_backend.fetch(LANDMARK_KEY).await? { + let max_landmarks = params + .max_certificate_lifetime_secs + .div_ceil(params.landmark_interval_secs) + + 1; + LandmarkSequence::from_bytes(&bytes, max_landmarks) + .map_err(|e| e.to_string())? + } else { + return Err("failed to get landmark sequence".into()); + }; + if leaf_index < seq.first_index() { + return Response::error("Leaf index is before first active landmark", 422); + } + let Some((landmark_id, landmark_subtree)) = seq.subtree_for_index(leaf_index) + else { + // The leaf index might be between the latest landmark + // and the current tree size. Set Retry-After to the + // expected time for the next landmark so the client can + // try again later. + let headers = Headers::new(); + let i = params.landmark_interval_secs as u64; + headers + .set("Retry-After", &format!("{}", i - (now_millis() / 1000) % i))?; + return Response::error("Leaf index will be covered by next landmark", 503) + .map(|r| r.with_headers(headers)); + }; + + // Fetch the log entry for the leaf index. + let log_entry = read_leaf::( + &object_backend, + leaf_index, + checkpoint.size(), + checkpoint.hash(), + ) + .await + .map_err(|e| e.to_string())?; + + // Get the inclusion proof. + let proof = match prove_subtree_inclusion( checkpoint.size(), *checkpoint.hash(), + landmark_subtree.lo(), + landmark_subtree.hi(), leaf_index, &object_backend, ) - .await?; - let proof_bytestrings = - proof.into_iter().map(|h| h.0.to_vec()).collect::>(); - Response::from_json(&ProveInclusionResponse { - proof: proof_bytestrings, - }) + .await + { + Ok(p) => p, + Err(ProofError::Tlog(s)) => return Response::error(s.to_string(), 422), + Err(ProofError::Other(e)) => return Err(e.to_string().into()), + }; + + // Construct the signatureless certificate. + let data = match serialize_signatureless_cert( + &log_entry, + leaf_index, + &spki_der, + &landmark_subtree, + proof, + ) { + Ok(data) => data, + Err(e) => { + return Response::error( + format!("Failed to serialize signatureless cert: {e}"), + 422, + ) + } + }; + + Response::from_json(&GetCertificateResponse { data, landmark_id }) }) .get("/logs/:log/metadata", |_req, ctx| { let name = ctx.data; @@ -245,7 +315,7 @@ async fn add_entry(mut req: Request, env: &Env, name: &str) -> Result not_before: Time::UtcTime(UtcTime::from_unix_duration(now).map_err(|e| e.to_string())?), not_after: Time::UtcTime( UtcTime::from_unix_duration( - now + Duration::from_secs(params.validity_interval_seconds), + now + Duration::from_secs(params.max_certificate_lifetime_secs as u64), ) .map_err(|e| e.to_string())?, ), diff --git a/crates/mtc_worker/src/sequencer_do.rs b/crates/mtc_worker/src/sequencer_do.rs index 1e707c09..b291230a 100644 --- a/crates/mtc_worker/src/sequencer_do.rs +++ b/crates/mtc_worker/src/sequencer_do.rs @@ -3,13 +3,15 @@ //! Sequencer is the 'brain' of the CT log, responsible for sequencing entries and maintaining log state. -use std::time::Duration; +use std::{future::Future, pin::Pin, time::Duration}; use crate::{load_checkpoint_signers, load_origin, CONFIG}; use generic_log_worker::{ - get_durable_object_name, GenericSequencer, SequencerConfig, SEQUENCER_BINDING, + get_durable_object_name, load_public_bucket, CheckpointCallbacker, GenericSequencer, + SequencerConfig, SEQUENCER_BINDING, }; -use mtc_api::BootstrapMtcLogEntry; +use mtc_api::{BootstrapMtcLogEntry, LandmarkSequence, LANDMARK_KEY}; +use tlog_tiles::UnixTimestamp; #[allow(clippy::wildcard_imports)] use worker::*; @@ -36,6 +38,7 @@ impl DurableObject for Sequencer { enable_dedup: params.enable_dedup, sequence_skip_threshold_millis: params.sequence_skip_threshold_millis, location_hint: params.location_hint.clone(), + checkpoint_callback: checkpoint_callback(&env, name), }; Sequencer(GenericSequencer::new(state, env, config)) @@ -49,3 +52,60 @@ impl DurableObject for Sequencer { self.0.alarm().await } } + +/// Return a callback function that gets passed into the generic sequencer and +/// called each time a new checkpoint is created. For MTC, this is used to +/// periodically update the landmark checkpoint sequence. +fn checkpoint_callback(env: &Env, name: &str) -> CheckpointCallbacker { + let params = &CONFIG.logs[name]; + let bucket = load_public_bucket(env, name).unwrap(); + Box::new( + move |tree_size: u64, old_time: UnixTimestamp, new_time: UnixTimestamp| { + Box::pin({ + // We have to clone each time since the bucket gets moved into + // the async function. + let bucket_clone = bucket.clone(); + async move { + if old_time > new_time { + return Err("condition not met: `old_time <= new_time`".into()); + } + // Check if we crossed a landmark epoch between the old and + // new checkpoints. (Ideally `old_time` would be the time + // that the last landmark was added, but we don't have that + // handy so can use the previous checkpoint time instead.) + if new_time / (1000 * params.landmark_interval_secs as u64) + == old_time / (1000 * params.landmark_interval_secs as u64) + { + // Not yet time to add a new landmark. + return Ok(()); + } + + // Time to add a new landmark. + let max_landmarks = params + .max_certificate_lifetime_secs + .div_ceil(params.landmark_interval_secs) + + 1; + + // Load current landmark sequence. + let mut seq = + if let Some(obj) = bucket_clone.get(LANDMARK_KEY).execute().await? { + let bytes = obj.body().ok_or("missing object body")?.bytes().await?; + LandmarkSequence::from_bytes(&bytes, max_landmarks) + .map_err(|e| e.to_string())? + } else { + LandmarkSequence::create(max_landmarks) + }; + // Add the new landmark. + if seq.add(tree_size).map_err(|e| e.to_string())? { + // The landmark sequence was updated. Publish the result. + bucket_clone + .put(LANDMARK_KEY, seq.to_bytes().map_err(|e| e.to_string())?) + .execute() + .await?; + } + Ok(()) + } + }) as Pin>>> + }, + ) +} diff --git a/crates/tlog_tiles/src/checkpoint.rs b/crates/tlog_tiles/src/checkpoint.rs index 92d84575..58409589 100644 --- a/crates/tlog_tiles/src/checkpoint.rs +++ b/crates/tlog_tiles/src/checkpoint.rs @@ -365,7 +365,7 @@ impl CheckpointSigner for Ed25519CheckpointSigner { } /// Open and verify a serialized checkpoint encoded as a [note](c2sp.org/signed-note), returning a -/// [CheckpointText] and the latest timestamp of any of its cosignatures (if defined). +/// [`CheckpointText`] and the latest timestamp of any of its cosignatures (if defined). /// /// # Errors /// diff --git a/crates/tlog_tiles/src/lib.rs b/crates/tlog_tiles/src/lib.rs index 519b546f..b7588937 100644 --- a/crates/tlog_tiles/src/lib.rs +++ b/crates/tlog_tiles/src/lib.rs @@ -103,7 +103,7 @@ mod tests { "http://ct.googleapis.com/logs/argon2020/ct/v1/get-entries?start=10000&end=10000", ); - let hash = tlog::record_hash(&leaf.entries[0].data); + let hash = record_hash(&leaf.entries[0].data); let url = format!( "http://ct.googleapis.com/logs/argon2020/ct/v1/get-proof-by-hash?tree_size={}&hash={}", @@ -112,7 +112,7 @@ mod tests { ); let rp: CtInclusionProof = http_get(&url); - tlog::check_inclusion(&rp.proof, root.size, root.hash, 10000, hash)?; + verify_inclusion_proof(&rp.proof, root.size, root.hash, 10000, hash)?; let url = format!( "http://ct.googleapis.com/logs/argon2020/ct/v1/get-sth-consistency?first=3654490&second={}", @@ -120,7 +120,7 @@ mod tests { let tp: CtConsistencyProof = http_get(&url); let oh = Hash::parse_hash("AuIZ5V6sDUj1vn3Y1K85oOaQ7y+FJJKtyRTl1edIKBQ=")?; - tlog::check_consistency(&tp.consistency, root.size, root.hash, 3_654_490, oh)?; + verify_consistency_proof(&tp.consistency, root.size, root.hash, 3_654_490, oh)?; Ok(()) } diff --git a/crates/tlog_tiles/src/tile.rs b/crates/tlog_tiles/src/tile.rs index 8f887bcb..922379d5 100644 --- a/crates/tlog_tiles/src/tile.rs +++ b/crates/tlog_tiles/src/tile.rs @@ -16,7 +16,7 @@ //! - [tile_test.go](https://cs.opensource.google/go/x/mod/+/refs/tags/v0.21.0:sumdb/tlog/tile_test.go) use crate::tlog::{ - node_hash, split_stored_hash_index, stored_hash_index, subtree_indexes, Hash, HashReader, + node_hash, split_stored_hash_index, stored_hash_index, tree_hash_indexes, Hash, HashReader, TlogError, HASH_SIZE, }; use std::cell::RefCell; @@ -570,7 +570,7 @@ impl HashReader for TileHashReader<'_> { // Plan to fetch tiles necessary to recompute tree hash. If it matches, // those tiles are authenticated. - let stx = subtree_indexes(0, self.tree_size, vec![]); + let stx = tree_hash_indexes(self.tree_size); let mut stx_tile_order = vec![0; stx.len()]; for (i, &x) in stx.iter().enumerate() { @@ -708,7 +708,7 @@ impl TileReader for TlogTileRecorder { if t.height() == TlogTile::HEIGHT { Ok(TlogTile::new(t.level(), t.level_index(), t.width(), None)) } else { - Err(TlogError::InvalidInput( + Err(TlogError::ConditionNotMet( "TlogTileRecorder cannot read tiles of height not equal to 8".to_string(), )) } @@ -745,13 +745,13 @@ impl TileReader for PreloadedTlogTileReader { for tile in tiles { // Convert the tile to a tlog-tile, ie one where height=8 and data=false if tile.height() != TlogTile::HEIGHT { - return Err(TlogError::InvalidInput( + return Err(TlogError::ConditionNotMet( "PreloadedTlogTileReader cannot read tiles of height not equal to 8" .to_string(), )); } if tile.is_data() { - return Err(TlogError::InvalidInput( + return Err(TlogError::ConditionNotMet( "PreloadedTlogTileReader cannot read data tiles".to_string(), )); } @@ -759,7 +759,7 @@ impl TileReader for PreloadedTlogTileReader { // Record the tile's contents let Some(contents) = self.0.get(&tlog_tile) else { - return Err(TlogError::InvalidInput(format!( + return Err(TlogError::ConditionNotMet(format!( "PreloadedTlogTileReader cannot find {}", tlog_tile.path() ))); diff --git a/crates/tlog_tiles/src/tlog.rs b/crates/tlog_tiles/src/tlog.rs index dfb50561..4cac8d5f 100644 --- a/crates/tlog_tiles/src/tlog.rs +++ b/crates/tlog_tiles/src/tlog.rs @@ -24,6 +24,42 @@ use sha2::{Digest, Sha256}; use std::fmt; use thiserror::Error; +#[derive(Error, Debug)] +pub enum TlogError { + #[error("invalid transparency proof")] + InvalidProof, + #[error("malformed hash")] + MalformedHash, + #[error("invalid tile")] + InvalidTile, + #[error("bad math")] + BadMath, + #[error("recorded but did not read tiles")] + RecordedTilesOnly, + #[error("downloaded inconsistent tile")] + InconsistentTile, + #[error("indexes not in tree")] + IndexesNotInTree, + #[error("indexes out of order")] + IndexesOutOfOrder, + #[error("unmet input condition: {0}")] + ConditionNotMet(String), + #[error("missing verifier signature")] + MissingVerifierSignature, + #[error("timestamp is after current time")] + InvalidTimestamp, + #[error("checkpoint origin does not match")] + OriginMismatch, + #[error(transparent)] + Note(#[from] signed_note::NoteError), + #[error(transparent)] + MalformedCheckpoint(#[from] crate::MalformedCheckpointTextError), + #[error(transparent)] + InvalidBase64(#[from] base64::DecodeError), + #[error(transparent)] + IO(#[from] std::io::Error), +} + /// `HashSize` is the size of a Hash in bytes. pub const HASH_SIZE: usize = 32; @@ -31,6 +67,10 @@ pub const HASH_SIZE: usize = 32; #[derive(Copy, Clone, Default, PartialEq)] pub struct Hash(pub [u8; HASH_SIZE]); +/// A `Proof` is a verifiable Merkle Tree (subtree) inclusion or consistency +/// proof. +pub type Proof = Vec; + impl fmt::Display for Hash { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", BASE64_STANDARD.encode(self.0))?; @@ -128,12 +168,14 @@ pub fn node_hash(left: Hash, right: Hash) -> Hash { Hash(result.into()) } -/// Maps the tree coordinates `(level, n)` to a dense linear ordering that can be used for hash -/// storage. Hash storage implementations that store hashes in sequential storage can use this -/// function to compute where to read or write a given hash. +/// Maps the tree coordinates `(level, n)` to a dense linear ordering that can +/// be used for hash storage. Hash storage implementations that store hashes in +/// sequential storage can use this function to compute where to read or write a +/// given hash. /// -/// For information about the stored hash index ordering, see section 3.3 of Crosby and Wallach's -/// paper ["Efficient Data Structures for Tamper-Evident +/// The stored hash index ordering is given by post-order (leaf, right, root) +/// traversal of the nodes in the tree. For information, see section 3.3 of +/// Crosby and Wallach's paper ["Efficient Data Structures for Tamper-Evident /// Logging"](https://www.usenix.org/legacy/event/sec09/tech/full_papers/crosby.pdf). pub fn stored_hash_index(level: u8, n: u64) -> u64 { // Level L's n'th hash is written right after level L+1's 2n+1'th hash. @@ -166,7 +208,7 @@ pub fn split_stored_hash_index(index: u64) -> (u8, u64) { // so the n we want is in [index/2, index/2+log₂(index)]. let mut n = index / 2; let mut index_n = stored_hash_index(0, n); - assert!(index_n <= index, "bad math"); + debug_assert!(index_n <= index, "bad math"); loop { // Each new record n adds 1 + trailingZeros(n) hashes. let x = index_n + 1 + u64::from((n + 1).trailing_zeros()); @@ -242,7 +284,7 @@ pub fn stored_hashes_for_record_hash( // Fetch hashes. let old = r.read_hashes(&indexes)?; - assert_eq!(old.len(), indexes.len(), "bad read_hashes implementation"); + debug_assert_eq!(old.len(), indexes.len(), "bad read_hashes implementation"); // Build new hashes. let mut h = h; @@ -291,71 +333,53 @@ pub fn tree_hash(n: u64, r: &R) -> Result { if n == 0 { return Ok(EMPTY_HASH); } - let indexes = subtree_indexes(0, n, vec![]); - let hashes = r.read_hashes(&indexes)?; - assert_eq!( - hashes.len(), - indexes.len(), - "bad read_hashes implementation" - ); - let (hash, remaining_hashes) = subtree_hash(0, n, &hashes); - assert!(remaining_hashes.is_empty(), "bad math in tree_hash"); - Ok(hash) + subtree_hash(&Subtree::new(0, n)?, r) } -/// Returns the storage indexes needed to compute the hash for the subtree containing records [lo, -/// hi), appending them to need and returning the result. See -/// . -/// -/// # Panics -/// -/// Panics if there are internal math errors. -pub fn subtree_indexes(lo: u64, hi: u64, mut need: Vec) -> Vec { - // See subtree_hash below for commentary. - let mut lo = lo; - while lo < hi { - let (k, level) = maxpow2(hi - lo + 1); - assert!(lo & (k - 1) == 0, "bad math in subtree_indexes"); - need.push(stored_hash_index(level, lo >> level)); - lo += k; +/// Computes the indexes needed to compute the hash of the tree with `n` records. +pub fn tree_hash_indexes(n: u64) -> Vec { + if n == 0 { + return vec![]; } - need + Subtree { lo: 0, hi: n }.hash_indexes() } -/// Computes the hash for the subtree containing records [lo, hi), assuming that -/// hashes are the hashes corresponding to the indexes returned by -/// `subtree_indexes(lo, hi)`. It returns any leftover hashes. -/// -/// May panic if there are internal math errors. -fn subtree_hash(lo: u64, hi: u64, hashes: &[Hash]) -> (Hash, Vec) { - // Repeatedly partition the tree into a left side with 2^level nodes, - // for as large a level as possible, and a right side with the fringe. - // The left hash is stored directly and can be read from storage. - // The right side needs further computation. - let mut num_tree = 0; - let mut lo = lo; - while lo < hi { - let (k, _) = maxpow2(hi - lo + 1); - assert!(lo & (k - 1) == 0 && lo < hi, "bad math in subtree_hash"); - num_tree += 1; - lo += k; - } - - assert!(hashes.len() >= num_tree, "bad index math in subtree_hash"); - - // Reconstruct hash. - let mut h = hashes[num_tree - 1]; - for i in (0..num_tree - 1).rev() { - h = node_hash(hashes[i], h); - } - (h, hashes[num_tree..].to_vec()) +/// Returns the storage indexes needed to compute the hash for the subtree. +/// See . +pub fn subtree_hash_indexes(n: &Subtree) -> Vec { + n.hash_indexes() } -/// A `InclusionProof` is a verifiable proof that a particular log root contains a particular record. -/// RFC 6962 calls this a “Merkle audit path.” -pub type InclusionProof = Vec; +/// Computes the hash for the root of the subtree `[lo, hi)`, using the +/// [`HashReader`] to obtain previously stored hashes (those returned by +/// [`stored_hashes`] during the writes of those `hi-lo` records). `tree_hash` +/// makes a single call to [`HashReader::read_hashes`] requesting at most `1 + +/// log₂ (hi-lo)` hashes. +/// +/// # Errors +/// +/// Returns an error if `read_hashes` fails to read hashes or if the subtree is +/// invalid. +/// +/// # Panics +/// +/// Panics if `read_hashes` returns a slice of hashes that is not the same +/// length as the requested indexes, or if there are internal math errors. +pub fn subtree_hash(n: &Subtree, r: &R) -> Result { + let indexes = n.hash_indexes(); + let mut hashes = r.read_hashes(&indexes)?; + debug_assert_eq!( + hashes.len(), + indexes.len(), + "bad read_hashes implementation" + ); + let hash = n.hash(&mut hashes); + debug_assert!(hashes.is_empty(), "bad math in subtree_hash"); + Ok(hash) +} -/// Returns the proof that the tree of size `t` contains the record with index `n`. +/// Returns the proof that the tree of size `n` contains the record with +/// index `leaf_index`. /// /// # Errors /// @@ -365,12 +389,12 @@ pub type InclusionProof = Vec; /// /// Panics if `read_hashes` returns a slice of hashes that is not the same /// length as the requested indexes, or if there are internal math errors. -pub fn prove_inclusion(t: u64, n: u64, r: &R) -> Result { - prove_subtree_inclusion(0, t, n, r) +pub fn inclusion_proof(n: u64, leaf_index: u64, r: &R) -> Result { + subtree_inclusion_proof(&Subtree::new(0, n)?, leaf_index, r) } -/// Returns the proof that the subtree with leaves [lo, hi) contains the record -/// with index `n`. +/// Returns the proof that the subtree `n` contains the record with index +/// `leaf_index`. /// /// # Errors /// @@ -380,387 +404,645 @@ pub fn prove_inclusion(t: u64, n: u64, r: &R) -> Result( - lo: u64, - hi: u64, - n: u64, +pub fn subtree_inclusion_proof( + n: &Subtree, + leaf_index: u64, r: &R, -) -> Result { - if n >= hi { - return Err(TlogError::InvalidInput("n >= t".into())); - } - let indexes = inclusion_proof_indexes(lo, hi, n, vec![]); +) -> Result { + let m = &Subtree::new(leaf_index, leaf_index + 1)?; + + // SUBTREE_PROOF(start, start + 1, D_n) = PATH(start, D_n) + let indexes = n.subproof_indexes(m, true)?; + if indexes.is_empty() { return Ok(vec![]); } - let hashes = r.read_hashes(&indexes)?; - assert_eq!( + let mut hashes = r.read_hashes(&indexes)?; + debug_assert_eq!( hashes.len(), indexes.len(), "bad read_hashes implementation" ); - let (proof, remaining_hashes) = inclusion_proof(lo, hi, n, hashes); - assert!( - remaining_hashes.is_empty(), + // SUBTREE_PROOF(start, start + 1, D_n) = PATH(start, D_n) + let proof = n.subproof(m, &mut hashes, true)?; + debug_assert!( + hashes.is_empty(), "bad index math in prove_subtree_inclusion" ); Ok(proof) } -/// Builds the list of indexes needed to construct the proof -/// that leaf n is contained in the subtree with leaves [lo, hi). -/// It appends those indexes to need and returns the result. -/// See . +/// Returns the indexes required for the proof that the tree of size `n` +/// contains the record with index `leaf_index`. /// -/// # Panics -/// May panic if there are internal math errors. -pub fn inclusion_proof_indexes(lo: u64, hi: u64, n: u64, mut need: Vec) -> Vec { - // See inclusion_proof below for commentary. - assert!(lo <= n && n < hi, "bad math in inclusion_proof_indexes"); - if lo + 1 == hi { - return need; - } - let (k, _) = maxpow2(hi - lo); - if n < lo + k { - need = inclusion_proof_indexes(lo, lo + k, n, need); - need = subtree_indexes(lo + k, hi, need); - } else { - need = subtree_indexes(lo, lo + k, need); - need = inclusion_proof_indexes(lo + k, hi, n, need); - } - need -} - -/// Constructs the proof that leaf n is contained in the subtree with leaves [lo, hi). -/// It returns any leftover hashes as well. -/// See . +/// # Errors /// -/// May panic if there are internal math errors. -fn inclusion_proof(lo: u64, hi: u64, n: u64, mut hashes: Vec) -> (InclusionProof, Vec) { - // We must have lo <= n < hi or else the code here has a bug. - assert!(lo <= n && n < hi, "bad math in inclusion_proof"); - - if lo + 1 == hi { - // n == lo - // Reached the leaf node. - // The verifier knows what the leaf hash is, so we don't need to send it. - return (vec![], hashes); - } - - // Walk down the tree toward n. - // Record the hash of the path not taken (needed for verifying the proof). - let mut proof: InclusionProof; - let th: Hash; - let (k, _) = maxpow2(hi - lo); - if n < lo + k { - // n is on left side - (proof, hashes) = inclusion_proof(lo, lo + k, n, hashes); - (th, hashes) = subtree_hash(lo + k, hi, &hashes); - } else { - // n is on right side - (th, hashes) = subtree_hash(lo, lo + k, &hashes); - (proof, hashes) = inclusion_proof(lo + k, hi, n, hashes); - } - - proof.push(th); - (proof, hashes) +/// Returns an error if the `[lo, hi)` is not a valid subtree, or if +/// `leaf_index` is not in that subtree. +pub fn inclusion_proof_indexes(n: u64, leaf_index: u64) -> Result, TlogError> { + subtree_inclusion_proof_indexes(&Subtree::new(0, n)?, leaf_index) } -#[derive(Error, Debug)] -pub enum TlogError { - #[error("invalid transparency proof")] - InvalidProof, - #[error("malformed hash")] - MalformedHash, - #[error("invalid tile")] - InvalidTile, - #[error("bad math")] - BadMath, - #[error("recorded but did not read tiles")] - RecordedTilesOnly, - #[error("downloaded inconsistent tile")] - InconsistentTile, - #[error("indexes not in tree")] - IndexesNotInTree, - #[error("indexes out of order")] - IndexesOutOfOrder, - #[error("unmet input condition: {0}")] - InvalidInput(String), - #[error("missing verifier signature")] - MissingVerifierSignature, - #[error("timestamp is after current time")] - InvalidTimestamp, - #[error("checkpoint origin does not match")] - OriginMismatch, - #[error(transparent)] - Note(#[from] signed_note::NoteError), - #[error(transparent)] - MalformedCheckpoint(#[from] crate::MalformedCheckpointTextError), - #[error(transparent)] - InvalidBase64(#[from] base64::DecodeError), - #[error(transparent)] - IO(#[from] std::io::Error), +/// Returns the indexes required for the proof that the subtree `[lo, hi)` +/// contains the record with index `leaf_index`. +/// +/// # Errors +/// +/// Returns an error if the `[lo, hi)` is not a valid subtree, or if +/// `leaf_index` is not in that subtree. +pub fn subtree_inclusion_proof_indexes( + n: &Subtree, + leaf_index: u64, +) -> Result, TlogError> { + // SUBTREE_PROOF(start, start + 1, D_n) = PATH(start, D_n) + n.subproof_indexes(&Subtree::new(leaf_index, leaf_index + 1)?, true) } -/// Verifies that `p` is a valid proof that the tree of size `t` with hash `th` has an `n`'th -/// record with hash `h`. +/// Verify an inclusion proof that the tree of size `tree_size` with root hash +/// `root_hash` contains a leaf at index `leaf_index` with hash `hash`. This +/// follows . /// /// # Errors /// -/// Returns an error if the inputs or proof are invalid. +/// Will return an error if proof verification fails. /// /// # Panics /// -/// Panics if there are internal math errors. -pub fn check_inclusion( - p: &InclusionProof, - t: u64, - th: Hash, - n: u64, - h: Hash, +/// Will panic if there are internal math errors. +pub fn verify_inclusion_proof( + proof: &Proof, + tree_size: u64, + root_hash: Hash, + leaf_index: u64, + leaf_hash: Hash, ) -> Result<(), TlogError> { - if n >= t { - return Err(TlogError::InvalidInput("n >= t".into())); + // 1. Compare leaf_index from the inclusion_proof_v2 structure against tree_size. If leaf_index is greater than or equal to tree_size, then fail the proof verification. + if leaf_index >= tree_size { + return Err(TlogError::InvalidProof); } - let th2 = run_inclusion_proof(p, 0, t, n, h)?; - if th2 == th { + // 2. Set fn to leaf_index and sn to tree_size - 1. + let mut f_n = leaf_index; + let mut s_n = tree_size - 1; + // 3. Set r to hash. + let mut r = leaf_hash; + // 4. For each value p in the inclusion_path array: + for p in proof { + // a. If sn is 0, then stop the iteration and fail the proof verification. + if s_n == 0 { + return Err(TlogError::InvalidProof); + } + // b. If LSB(fn) is set, or if fn is equal to sn, then: + if lsb_set(f_n) || f_n == s_n { + // i. Set r to HASH(0x01 || p || r). + r = node_hash(*p, r); + // ii. If LSB(fn) is not set, then right-shift both fn and sn equally until either LSB(fn) is set or fn is 0. + // + // NOTE: It must be the case that fn is non-zero, so we can simplify. + while !lsb_set(f_n) { + f_n >>= 1; + s_n >>= 1; + } + } else { + // i. Set r to HASH(0x01 || r || p). + r = node_hash(r, *p); + } + // c. Finally, right-shift both fn and sn one time. + f_n >>= 1; + s_n >>= 1; + } + // 5. Compare sn to 0. Compare r against the root_hash. If sn is equal to 0 and r and the root_hash are equal, then the log has proven the inclusion of hash. Otherwise, fail the proof verification. + if s_n == 0 && r == root_hash { Ok(()) } else { Err(TlogError::InvalidProof) } } -/// Runs the proof p that leaf n is contained in the subtree with leaves [lo, hi). -/// Running the proof means constructing and returning the implied hash of that -/// subtree. +/// Verify the proof that a leaf at index `leaf_index` and hash `leaf_hash` is +/// included in the subtree `[n_lo, n_hi)` with hash `n_hash`, following +/// . /// -/// # Panics +/// # Errors /// -/// Panics if there are internal math errors. -fn run_inclusion_proof( - p: &InclusionProof, - lo: u64, - hi: u64, - n: u64, +/// Will return an error if proof verification fails. +pub fn verify_subtree_inclusion_proof( + proof: &Proof, + n: &Subtree, + n_hash: Hash, + leaf_index: u64, leaf_hash: Hash, -) -> Result { - // We must have lo <= n < hi or else the code here has a bug. - assert!((lo..hi).contains(&n), "bad math in run_inclusion_proof"); - - if lo + 1 == hi { - // m == lo - // Reached the leaf node. - // The proof must not have any unnecessary hashes. - if !p.is_empty() { - return Err(TlogError::InvalidProof); - } - return Ok(leaf_hash); - } - - if p.is_empty() { - return Err(TlogError::InvalidProof); - } - - let (k, _) = maxpow2(hi - lo); - if n < lo + k { - let th = run_inclusion_proof(&p[..p.len() - 1].to_vec(), lo, lo + k, n, leaf_hash)?; - Ok(node_hash(th, p[p.len() - 1])) - } else { - let th = run_inclusion_proof(&p[..p.len() - 1].to_vec(), lo + k, hi, n, leaf_hash)?; - Ok(node_hash(p[p.len() - 1], th)) - } +) -> Result<(), TlogError> { + verify_inclusion_proof(proof, n.hi - n.lo, n_hash, leaf_index - n.lo, leaf_hash) } -/// A `ConsistencyProof` is a verifiable proof that a particular log tree contains -/// as a prefix all records present in an earlier tree. -/// RFC 6962 calls this a “Merkle consistency proof.” -pub type ConsistencyProof = Vec; +/// Returns the proof that the tree of size `n` contains as a prefix all the +/// records from the tree of smaller size `m`. +/// +/// # Errors +/// +/// Returns an error if the inputs or proof are invalid or if `read_hashes` +/// fails to read hashes. +/// +/// # Panics +/// +/// Panics if `read_hashes` returns a slice of hashes that is not the same +/// length as the requested indexes, or if there are internal math errors. +pub fn consistency_proof(n: u64, m: u64, r: &R) -> Result { + // SUBTREE_PROOF(0, end, D_n) = PROOF(end, D_n) + subtree_consistency_proof(n, &Subtree::new(0, m)?, r) +} -/// Returns the proof that the tree of size `t` contains -/// as a prefix all the records from the tree of smaller size `n`. +/// Returns the proof that the tree of size `tree_size` is consistent with the +/// subtree `m` following +/// . /// /// # Errors /// -/// Returns an error if the inputs or proof are invalid or if `read_hashes` fails to read hashes. +/// Returns an error if the inputs or proof are invalid or if `read_hashes` +/// fails to read hashes. /// /// # Panics /// /// Panics if `read_hashes` returns a slice of hashes that is not the same /// length as the requested indexes, or if there are internal math errors. -pub fn prove_consistency( - t: u64, - n: u64, - h: &R, -) -> Result { - if !(1..=t).contains(&n) { - return Err(TlogError::InvalidInput("1 <= n <= t".into())); - } - let indexes = consistency_proof_indexes(0, t, n, vec![]); +pub fn subtree_consistency_proof( + tree_size: u64, + m: &Subtree, + r: &R, +) -> Result { + let n = Subtree::new(0, tree_size)?; + let indexes = n.subproof_indexes(m, true)?; if indexes.is_empty() { return Ok(vec![]); } - let hashes = h.read_hashes(&indexes)?; - assert_eq!( + let mut hashes = r.read_hashes(&indexes)?; + debug_assert_eq!( hashes.len(), indexes.len(), "bad read_hashes implementation" ); - let (p, remaining_hashes) = consistency_proof(0, t, n, hashes); - assert!( - remaining_hashes.is_empty(), - "bad index math in prove_consistency" + let proof = n.subproof(m, &mut hashes, true)?; + debug_assert!( + hashes.is_empty(), + "bad index math in subtree_consistency_proof" ); - Ok(p) + Ok(proof) } -/// Builds the list of indexes needed to construct -/// the sub-proof related to the subtree containing records [lo, hi). -/// See . +/// Builds the list of indexes needed to construct the proof that +/// the tree of size `n` contains as a prefix all the records from the tree of +/// smaller size `m`. /// -/// # Panics +/// # Errors /// -/// Panics if there are internal math errors. -pub fn consistency_proof_indexes(lo: u64, hi: u64, n: u64, mut need: Vec) -> Vec { - // See treeProof below for commentary. - assert!( - (lo + 1..=hi).contains(&n), - "bad math in consistency_proof_indexes" - ); - - if n == hi { - if lo == 0 { - return need; - } - return subtree_indexes(lo, hi, need); - } +/// Will return an error if the parameters are invalid. +pub fn consistency_proof_indexes(n: u64, m: u64) -> Result, TlogError> { + subtree_consistency_proof_indexes(n, &Subtree::new(0, m)?) +} - let (k, _) = maxpow2(hi - lo); - if n <= lo + k { - need = consistency_proof_indexes(lo, lo + k, n, need); - need = subtree_indexes(lo + k, hi, need); - } else { - need = subtree_indexes(lo, lo + k, need); - need = consistency_proof_indexes(lo + k, hi, n, need); - } - need +/// Builds the list of indexes needed to construct the proof that the tree of +/// size `tree_size` is consistent with the subtree `m`. +/// +/// # Errors +/// +/// Will return an error if the parameters are invalid. +pub fn subtree_consistency_proof_indexes( + tree_size: u64, + m: &Subtree, +) -> Result, TlogError> { + Subtree::new(0, tree_size)?.subproof_indexes(m, true) } -/// Constructs the sub-proof related to the subtree containing records [lo, hi). -/// It returns any leftover hashes as well. -/// See . +/// Verify a consistency proof that the tree of size `n` with hash `root_hash` +/// contains the tree of size `m` with hash `m_hash` as a prefix. This follows +/// . /// -/// May panic if there are internal math errors. -fn consistency_proof( - lo: u64, - hi: u64, +/// # Errors +/// +/// Will return an error if proof verification fails. +pub fn verify_consistency_proof( + proof: &Proof, n: u64, - mut hashes: Vec, -) -> (ConsistencyProof, Vec) { - assert!((lo + 1..=hi).contains(&n), "bad math in consistency_proof"); - - // Reached common ground. - if n == hi { - if lo == 0 { - // This subtree corresponds exactly to the old tree. - // The verifier knows that hash, so we don't need to send it. - return (vec![], hashes); - } - let (th, hashes) = subtree_hash(lo, hi, &hashes); - return (vec![th], hashes); - } - - // Interior node for the proof. - // Decide whether to walk down the left or right side. - let mut p: ConsistencyProof; - let th: Hash; - let (k, _) = maxpow2(hi - lo); - if n <= lo + k { - // m is on left side - (p, hashes) = consistency_proof(lo, lo + k, n, hashes); - (th, hashes) = subtree_hash(lo + k, hi, &hashes); - } else { - // m is on right side - (th, hashes) = subtree_hash(lo, lo + k, &hashes); - (p, hashes) = consistency_proof(lo + k, hi, n, hashes); + root_hash: Hash, + m: u64, + m_hash: Hash, +) -> Result<(), TlogError> { + // Special case for proving consistency with an empty tree. + if m == 0 { + return if proof.is_empty() && m_hash == EMPTY_HASH && (n != 0 || root_hash == EMPTY_HASH) { + Ok(()) + } else { + Err(TlogError::InvalidProof) + }; } - p.push(th); - (p, hashes) + verify_subtree_consistency_proof(proof, n, root_hash, &Subtree::new(0, m)?, m_hash) } -/// Verifies that `p` is a valid proof that the tree of size `t` with hash `th` -/// contains as a prefix the tree of size `n` with hash `h`. +/// Verify a subtree consistency proof that the tree of size `n` with hash +/// `root_hash` is consistent with the subtree `m` with hash +/// `subtree_hash`. This follows +/// . /// /// # Errors /// -/// Returns an error if the consistency proof is invalid. +/// Will return an error if proof verification fails. /// -///# Panics +/// # Panics /// -/// Panics if there are internal math errors. -pub fn check_consistency( - p: &ConsistencyProof, - t: u64, - th: Hash, +/// Will panic if there are internal math errors. +pub fn verify_subtree_consistency_proof( + proof: &Proof, n: u64, - h: Hash, + root_hash: Hash, + m: &Subtree, + subtree_hash: Hash, ) -> Result<(), TlogError> { - if !(1..=t).contains(&n) { - return Err(TlogError::InvalidInput("1 <= n <= t".into())); + let Subtree { lo: start, hi: end } = *m; + // 1. If end is n, run the following: + if end == n { + // 1. Set fn to start and sn to end - 1. + let mut f_n = start; + let mut s_n = end - 1; + // 2. Set r to node_hash. + let mut r = subtree_hash; + // 3. Until LSB(fn) is set or sn is 0, right-shift fn and sn equally. + while !lsb_set(f_n) && s_n != 0 { + f_n >>= 1; + s_n >>= 1; + } + // 4. For each value p in the proof array: + for p in proof { + // 1. If sn is 0, then stop iteration and fail the proof verification. + if s_n == 0 { + return Err(TlogError::InvalidProof); + } + // 2. Set r to HASH(0x01, || p || r). + r = node_hash(*p, r); + // 3. Until LSB(sn) is set, right-shift sn. + while !lsb_set(s_n) { + s_n >>= 1; + } + // 4. Right-shift sn once more. + s_n >>= 1; + } + // 5. Compare sn to 0 and r to root_hash. If either is not equal, fail the proof verification. If all are equal, accept the proof. + if s_n == 0 && r == root_hash { + Ok(()) + } else { + Err(TlogError::InvalidProof) + } } - let (h2, th2) = run_consistency_proof(p, 0, t, n, h)?; - if th2 == th && h2 == h { - Ok(()) - } else { - Err(TlogError::InvalidProof) + // 2. Otherwise, run the following: + else { + // 1. If proof is an empty array, stop and fail verification. + if proof.is_empty() { + return Err(TlogError::InvalidProof); + } + // 2. If end - start is an exact power of 2, prepend node_hash to the proof array. + let mut proof = proof.clone(); + if (end - start).is_power_of_two() { + proof.insert(0, subtree_hash); + } + // 3. Set fn to start, sn to end - 1, and tn to n - 1. + let mut f_n = start; + let mut s_n = end - 1; + let mut t_n = n - 1; + // 4. Until LSB(sn) is not set or fn is equal to sn, right-shift fn, sn, and tn equally. + while lsb_set(s_n) && f_n != s_n { + f_n >>= 1; + s_n >>= 1; + t_n >>= 1; + } + // 5. Set both fr and sr to the first value in the proof array. + let mut f_r = proof[0]; + let mut s_r = proof[0]; + // 6. For each subsequent value c in the proof array: + for c in proof.into_iter().skip(1) { + // 1. If tn is 0, then stop the iteration and fail the proof verification. + if t_n == 0 { + return Err(TlogError::InvalidProof); + } + // 2. If LSB(sn) is set, or if sn is equal to tn, then: + if lsb_set(s_n) || s_n == t_n { + // 1. If fn < sn, set fr to HASH(0x01 || c || fr). + if f_n < s_n { + f_r = node_hash(c, f_r); + } + // 2. Set sr to HASH(0x01 || c || sr). + s_r = node_hash(c, s_r); + // 3. Until LSB(sn) is set, right-shift fn, sn, and tn equally. + while !lsb_set(s_n) { + f_n >>= 1; + s_n >>= 1; + t_n >>= 1; + } + } + // 3. Otherwise: + else { + // 1. Set sr to HASH(0x01 || sr || c). + s_r = node_hash(s_r, c); + } + // 4. Right-shift fn, sn, and tn once more. + f_n >>= 1; + s_n >>= 1; + t_n >>= 1; + } + // 7. Compare tn to 0, fr to node_hash, and sr to root_hash. If any are not equal, fail the proof verification. If all are equal, accept the proof. + if t_n == 0 && f_r == subtree_hash && s_r == root_hash { + Ok(()) + } else { + Err(TlogError::InvalidProof) + } } } -/// Runs the sub-proof p related to the subtree containing records [lo, hi), -/// where old is the hash of the old tree with n records. -/// Running the proof means constructing and returning the implied hashes of that -/// subtree in both the old and new tree. -/// -/// # Panics +// Return whether LSB(i) is set. +fn lsb_set(i: u64) -> bool { + (i & 1) == 1 +} + +/// A subtree of a Merkle Tree of size `n` is defined by two integers `lo` and +/// `hi` such that: +/// - 0 ≤ lo < hi ≤ n +/// - if `s` is the smallest power of two `≥ hi - lo`, `lo` is a multple of `s` /// -/// Panics if there are internal math errors. -fn run_consistency_proof( - p: &ConsistencyProof, +/// +#[derive(Debug, PartialEq, Eq)] +pub struct Subtree { lo: u64, hi: u64, - n: u64, - old: Hash, -) -> Result<(Hash, Hash), TlogError> { - assert!( - (lo + 1..=hi).contains(&n), - "bad math in run_consistency_proof" - ); +} - // Reached common ground. - if n == hi { - if lo == 0 { - if !p.is_empty() { - return Err(TlogError::InvalidProof); - } - return Ok((old, old)); +impl fmt::Display for Subtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[{}, {})", self.lo, self.hi) + } +} + +impl Subtree { + /// Returns a subtree for the given range. + /// + /// # Errors + /// + /// Will return an error if `[lo, hi)` is not a valid subtree. + pub fn new(lo: u64, hi: u64) -> Result { + if lo >= hi { + return Err(TlogError::ConditionNotMet("`lo < hi`".into())); } - if p.len() != 1 { - return Err(TlogError::InvalidProof); + // `s` is the next power of 2 greater than or equal to `hi - lo`. + let s = (hi - lo).next_power_of_two(); + if lo & (s - 1) != 0 { + return Err(TlogError::ConditionNotMet( + "`lo` must be a multiple of the next power of two ≥ `hi - lo`".into(), + )); + } + Ok(Self { lo, hi }) + } + /// Return the lower (inclusive) bound on indices in the subtree. + pub fn lo(&self) -> u64 { + self.lo + } + /// Return the upper (exclusive) bound on indices in the subtree. + pub fn hi(&self) -> u64 { + self.hi + } + /// Return whether or not the subtree contains the given leaf index. + pub fn contains(&self, leaf_index: u64) -> bool { + (self.lo..self.hi).contains(&leaf_index) + } + /// Return whether or not the subtree contains the given subtree. + fn contains_subtree(&self, other: &Subtree) -> bool { + (self.lo..self.hi).contains(&other.lo) && (self.lo + 1..=self.hi).contains(&other.hi) + } + /// Return left and right children. + fn children(&self) -> (Self, Self) { + let (k, _) = maxpow2(self.hi - self.lo); + ( + Self { + lo: self.lo, + hi: self.lo + k, + }, + Self { + lo: self.lo + k, + hi: self.hi, + }, + ) + } + /// Returns a list of one or two subtrees that efficiently cover `[lo, hi)`. + /// + /// # Errors + /// + /// Will return an error if `lo ≤ hi`. + pub fn split_interval(lo: u64, hi: u64) -> Result<(Self, Option), TlogError> { + if lo >= hi { + return Err(TlogError::ConditionNotMet("`lo < hi`".into())); + } + if hi - lo == 1 { + return Ok((Self { lo, hi }, None)); } - return Ok((p[0], p[0])); + let last = hi - 1; + // Find where `lo` and `last`'s tree paths diverge. The two subtrees + // will be on either side of the split. + // SAFETY: `lo ^ last` is guaranteed to be non-zero, so `ilog2` won't panic. + let split = (lo ^ last).ilog2(); + let mask = (1 << split) - 1; + let mid = last & !mask; + // Maximize the left endpoint. This is just before `lo`'s path leaves + // the right edge of its new subtree. + let left_split = (!lo & mask).ilog2() + 1; + let left = lo & !((1 << left_split) - 1); + + Ok((Self { lo: left, hi: mid }, Some(Self { lo: mid, hi }))) } +} - if p.is_empty() { - return Err(TlogError::InvalidProof); +// Strategy used for combining items in `walk_subproof`. +#[derive(Clone, Copy)] +enum CombinationStrategy { + /// For subproofs of indexes. Order is [sibling, recursive] on + /// right-recursion in order to preserve index ordering. + Index, + /// For subproofs of hashes. Order is always [recursive, sibling]. + Hash, +} + +impl Subtree { + /// Helper function to compute the `MTH` traversal logic from + /// . + /// + /// Repeatedly partition the tree into a left side with 2^level nodes, for + /// as large a level as possible, and a right side with the fringe and call + /// `f` to update some state for each level. + /// + /// This function is generic over: + /// - `F`: The type of closure that performs the action. + fn walk_hash(&self, f: &mut F) + where + F: FnMut(u8, u64), + { + let mut lo = self.lo; + while lo < self.hi { + let (k, level) = maxpow2(self.hi - lo + 1); + debug_assert!(lo & (k - 1) == 0 && lo < self.hi, "bad math in walk_hash"); + f(level, lo); + lo += k; + } } - // Interior node for the proof. - let (k, _) = maxpow2(hi - lo); - if n <= lo + k { - let (oh, th) = run_consistency_proof(&p[..p.len() - 1].to_vec(), lo, lo + k, n, old)?; - Ok((oh, node_hash(th, p[p.len() - 1]))) - } else { - let (oh, th) = run_consistency_proof(&p[..p.len() - 1].to_vec(), lo + k, hi, n, old)?; - Ok((node_hash(p[p.len() - 1], oh), node_hash(p[p.len() - 1], th))) + /// Returns the storage indexes needed to compute the subtree's root hash. + /// See . + /// + /// # Panics + /// + /// Panics if there are internal math errors. + fn hash_indexes(&self) -> Vec { + let mut need = Vec::new(); + let mut get_indexes = |level: u8, lo: u64| { + need.push(stored_hash_index(level, lo >> level)); + }; + self.walk_hash(&mut get_indexes); + + need + } + + /// Computes the subtree's root hash, assuming that `hashes` are the hashes + /// corresponding to the indexes returned by `hash_indexes`. It consumes + /// the requisite hashes from `hashes`. + /// + /// # Panics + /// + /// Panics if there are internal math errors. + fn hash(&self, hashes: &mut Vec) -> Hash { + let mut num_hashes = 0; + let mut get_hash = |_: u8, _: u64| { + num_hashes += 1; + }; + self.walk_hash(&mut get_hash); + + debug_assert!( + hashes.len() >= num_hashes, + "not enough hashes for reconstruction" + ); + + // The indexes are sorted in increasing order. In order to compute the + // root hash, start from the rightmost index and hash up. + let root_hash = hashes + .drain(0..num_hashes) + .rev() + .reduce(|fringe, sibling| node_hash(sibling, fringe)) + // This expect is safe because the loop to calculate num_tree ensures + // it's > 0 if the subtree has a non-zero range. + .expect("num_tree must be positive for a valid subtree range"); + + root_hash + } + + /// Helper function to implement the `SUBTREE_SUBPROOF` traversal logic from + /// . + /// + /// This can used to construct (sub)tree inclusion proofs and (sub)tree + /// consistency proofs. This implementation of the algorithm uses absolute + /// indexes as opposed to the relative indexes described in the spec. + /// + /// In order to calculate either the storage hash indexes needed for the + /// subproof or the subproof hashes, this function is generic over: + /// - `T`: The type of item to be collected (`u64` or `Hash`). + /// - `F`: The type of the closure that performs the action. + /// + /// When computing indexes, `strategy` indicates how to combine items. + /// Indexes are kept in increasing order as that may make retrieval from + /// backend storage more efficient, but for hashes the sibling node's hash + /// is always appended to the recursive proof according to the spec. + fn walk_subproof( + &self, + m: &Subtree, + known: bool, + f: &mut F, + strategy: CombinationStrategy, + ) -> Result, TlogError> + where + F: FnMut(&Subtree) -> Vec, + { + if !self.contains_subtree(m) { + return Err(TlogError::ConditionNotMet(format!( + "{self} does not contain {m}" + ))); + } + // Base case: the subtrees are equal. + if m == self { + // If the subtree was one of the inputs it is `known` and there is + // no need to return it. + return if known { Ok(vec![]) } else { Ok(f(self)) }; + } + + // Recursive step: traverse the children. + let (left, right) = self.children(); + if left.contains_subtree(m) { + // `m` is fully included in the left child. + // + // Recurse on the left, fully include the right. + let (mut recursive_proof, mut sibling) = + (left.walk_subproof(m, known, f, strategy)?, f(&right)); + recursive_proof.append(&mut sibling); + Ok(recursive_proof) + } else { + let mut sibling = f(&left); + let mut recursive_proof = if right.contains_subtree(m) { + // `m` is fully included in the right child. + // + // Fully include the left, recurse on the right. + right.walk_subproof(m, known, f, strategy)? + } else { + // `m` is fully included in `self`, but not fully included in + // either the left or right child. This implies `m` has the left + // child as a prefix and spills over into the right child + // (otherwise, `m` would not be a valid subtree). + // + // Fully include the left, recurse on right child of `m` with + // `known` set to false as the right child of `m` was not one of + // the inputs to the algorithm. + let (m_left, m_right) = m.children(); + debug_assert!(m_left == left, "expected left children to match"); + right.walk_subproof(&m_right, false, f, strategy)? + }; + + match strategy { + CombinationStrategy::Hash => { + // Always append the sibling to the end of the recursive proof. + recursive_proof.append(&mut sibling); + Ok(recursive_proof) + } + CombinationStrategy::Index => { + // Prepend the indexes needed to compute the sibling in + // order to keep the indexes in increasing order. + sibling.append(&mut recursive_proof); + Ok(sibling) + } + } + } + } + + /// Returns the storage hash indexes needed for the subproof. + fn subproof_indexes(&self, m: &Subtree, known: bool) -> Result, TlogError> { + // Get all hash indexes for a given subtree. + let mut get_indexes = |t: &Subtree| -> Vec { t.hash_indexes() }; + + self.walk_subproof(m, known, &mut get_indexes, CombinationStrategy::Index) + } + + /// Returns the hashes for the subproof. + fn subproof( + &self, + m: &Subtree, + hashes: &mut Vec, + known: bool, + ) -> Result { + // Reconstruct a single hash for a subtree and wrap it in a Vec. + // The closure captures the mutable `hashes` vector to pass it to `reconstruct_hash`. + let mut get_hash = |t: &Subtree| -> Vec { vec![t.hash(hashes)] }; + + // The closure's error type is Infallible, so we can safely unwrap. + self.walk_subproof(m, known, &mut get_hash, CombinationStrategy::Hash) } } @@ -827,13 +1109,14 @@ mod tests { } } + #[allow(clippy::too_many_lines)] #[test] fn test_tree() { const TEST_H: u8 = 2; let mut trees = Vec::new(); let mut leafhashes = Vec::new(); - let mut storage = Vec::new(); + let mut storage = TestHashStorage::new(); let mut tiles = HashMap::>::new(); for i in 0..100 { @@ -892,15 +1175,22 @@ mod tests { // Check that inclusion proofs work, for all trees and leaves so far. for j in 0..=i { - let mut p = prove_inclusion(i + 1, j, &storage).unwrap(); - check_inclusion(&p, i + 1, th, j, leafhashes[usize::try_from(j).unwrap()]).unwrap(); + let mut p = inclusion_proof(i + 1, j, &storage).unwrap(); + verify_inclusion_proof(&p, i + 1, th, j, leafhashes[usize::try_from(j).unwrap()]) + .unwrap(); for k in 0..p.len() { p[k].0[0] ^= 1; assert!( - check_inclusion(&p, i + 1, th, j, leafhashes[usize::try_from(j).unwrap()]) - .is_err(), - "check_record({}, {j}) succeeded with corrupt proof hash #{k}!", + verify_inclusion_proof( + &p, + i + 1, + th, + j, + leafhashes[usize::try_from(j).unwrap()] + ) + .is_err(), + "verify_inclusion_proof({}, {j}) succeeded with corrupt proof hash #{k}!", i + 1 ); p[k].0[0] ^= 1; @@ -919,8 +1209,9 @@ mod tests { assert_eq!(h[0], leafhashes[usize::try_from(j).unwrap()], "wrong hash"); // Even though reading the hash suffices, check we can generate the proof too. - let p = prove_inclusion(i + 1, j, &thr).unwrap(); - check_inclusion(&p, i + 1, th, j, leafhashes[usize::try_from(j).unwrap()]).unwrap(); + let p = inclusion_proof(i + 1, j, &thr).unwrap(); + verify_inclusion_proof(&p, i + 1, th, j, leafhashes[usize::try_from(j).unwrap()]) + .unwrap(); } assert_eq!(tile_storage.unsaved.get(), 0, "did not save tiles"); @@ -938,21 +1229,63 @@ mod tests { assert_eq!(h, trees[usize::try_from(j).unwrap()]); // Even though computing the subtree hash suffices, check that we can generate the proof too. - let mut p = prove_consistency(i + 1, j + 1, &thr).unwrap(); - check_consistency(&p, i + 1, th, j + 1, trees[usize::try_from(j).unwrap()]) + let mut p = consistency_proof(i + 1, j + 1, &thr).unwrap(); + verify_consistency_proof(&p, i + 1, th, j + 1, trees[usize::try_from(j).unwrap()]) .unwrap(); for k in 0..p.len() { p[k].0[0] ^= 1; assert!( - check_inclusion(&p, i + 1, th, j + 1, trees[usize::try_from(j).unwrap()]) - .is_err(), - "check_record({}, {j}) succeeded with corrupt proof hash #{k}!", + verify_consistency_proof( + &p, + i + 1, + th, + j + 1, + trees[usize::try_from(j).unwrap()] + ) + .is_err(), + "verify_consistency_proof({}, {j}) succeeded with corrupt proof hash #{k}!", i + 1 ); p[k].0[0] ^= 1; } } assert_eq!(tile_storage.unsaved.get(), 0, "did not save tiles"); + + // Check that subtree consistency proofs work, for all valid subtrees up to the current tree size. + for lo in 0..i { + let max_hi = if lo == 0 { + u64::MAX + } else { + // If `lo` is non-zero, find the maximum power of 2 that divides `lo`. + // This is a bitwise trick that isolates the lowest set bit. + let max_size = lo & lo.wrapping_neg(); + lo + max_size + }; + for hi in lo + 1..=i.min(max_hi) { + let m = Subtree::new(lo, hi).unwrap(); + let m_hash = subtree_hash(&m, &storage).unwrap(); + // Prove that the subtree is consistent with the tree of size `n`. + verify_subtree_consistency_proof( + &subtree_consistency_proof(i + 1, &m, &storage).unwrap(), + i + 1, + th, + &m, + m_hash, + ) + .unwrap(); + for leaf_index in lo..hi { + // Prove that each leaf in the subtree is included in the subtree. + verify_subtree_inclusion_proof( + &subtree_inclusion_proof(&m, leaf_index, &storage).unwrap(), + &m, + m_hash, + leaf_index, + leafhashes[usize::try_from(leaf_index).unwrap()], + ) + .unwrap(); + } + } + } } } @@ -968,9 +1301,45 @@ mod tests { } } + #[test] + fn test_new_subtree() { + // Valid subtrees. + assert!(Subtree::new(0, 1).is_ok()); + assert!(Subtree::new(36, 39).is_ok()); + + // Invalid subtrees. + assert!(Subtree::new(39, 36).is_err()); + assert!(Subtree::new(123, 456).is_err()); + assert!(Subtree::new(0, 0).is_err()); + } + #[test] fn test_empty_tree() { - let h = tree_hash(0, &TestHashStorage::new()).unwrap(); - assert_eq!(h, EMPTY_HASH); + assert_eq!(tree_hash(0, &TestHashStorage::new()).unwrap(), EMPTY_HASH); + + // Empty tree. + verify_consistency_proof(&vec![], 0, EMPTY_HASH, 0, EMPTY_HASH).unwrap(); + verify_consistency_proof(&vec![], 0, EMPTY_HASH, 1, EMPTY_HASH).unwrap_err(); + verify_consistency_proof(&vec![], 0, Hash::default(), 0, EMPTY_HASH).unwrap_err(); + + // Tree with single leaf. + verify_inclusion_proof(&vec![], 1, Hash::default(), 0, Hash::default()).unwrap(); + verify_consistency_proof(&vec![], 1, Hash::default(), 1, Hash::default()).unwrap(); + } + + #[test] + fn test_subtrees_split_interval() { + assert_eq!( + Subtree::split_interval(123, 124).unwrap(), + (Subtree::new(123, 124).unwrap(), None) + ); + + assert_eq!( + Subtree::split_interval(1200, 1300).unwrap(), + ( + Subtree::new(1152, 1280).unwrap(), + Some(Subtree::new(1280, 1300).unwrap()) + ) + ); } }