implement the specified sparse merkletree (#82)

* wip

* prototype custom predicates 1b

* feat: implement custom pred recursion

* files reorg, add github CI for rustfmt checks

* start sparsemerkletree. impl add_leaf method, initial Leaf & Intermediate types with methods

* mt: add hash computation of all the nodes in the tree, add method to print the tree to visualize it as a graphviz

* mt: add  (till the leaf) method which is used by get,contains,prove methods

* mt: add verify (of inclusion) method

* mt: update 'down' method to reuse siblings, update get,contains,prove methods (the three use 'down' under the hood)

* Add nonexistence proofs and iterator

* Add iterator test

* migrate usage of old merkletree to the new merkletree impl in POD2 code

---------

Co-authored-by: Eduard S. <eduardsanou@posteo.net>
Co-authored-by: Ahmad <root@ahmadafuni.com>

Author: 3 people

src/backends/mock_main/mod.rs

@@ -485,7 +485,7 @@ pub mod tests {
             pk: "ZooDeel".into(),
         };
         let pay_stub_pod = pay_stub_builder.sign(&mut signer)?;
-        let kyc_builder = zu_kyc_pod_builder(&params, &gov_id_pod, &pay_stub_pod);
+        let kyc_builder = zu_kyc_pod_builder(&params, &gov_id_pod, &pay_stub_pod)?;
 
         let mut prover = MockProver {};
         let kyc_pod = kyc_builder.prove(&mut prover)?;
@@ -501,7 +501,7 @@ pub mod tests {
 
     #[test]
     fn test_mock_main_great_boy() -> Result<()> {
-        let great_boy_builder = great_boy_pod_full_flow();
+        let great_boy_builder = great_boy_pod_full_flow()?;
 
         let mut prover = MockProver {};
         let great_boy_pod = great_boy_builder.prove(&mut prover)?;
@@ -520,7 +520,7 @@ pub mod tests {
 
     #[test]
     fn test_mock_main_tickets() -> Result<()> {
-        let tickets_builder = tickets_pod_full_flow();
+        let tickets_builder = tickets_pod_full_flow()?;
         let mut prover = MockProver {};
         let proof_pod = tickets_builder.prove(&mut prover)?;
         let pod = proof_pod.pod.into_any().downcast::<MockMainPod>().unwrap();

src/backends/mock_signed.rs

@@ -2,6 +2,7 @@ use anyhow::Result;
 use std::any::Any;
 use std::collections::HashMap;
 
+use crate::constants::MAX_DEPTH;
 use crate::middleware::{
     containers::Dictionary, hash_str, AnchoredKey, Hash, Params, Pod, PodId, PodSigner, PodType,
     Statement, Value, KEY_SIGNER, KEY_TYPE,
@@ -19,7 +20,7 @@ impl PodSigner for MockSigner {
         kvs.insert(hash_str(&KEY_SIGNER), Value(pk_hash.0));
         kvs.insert(hash_str(&KEY_TYPE), Value::from(PodType::MockSigned));
 
-        let dict = Dictionary::new(&kvs);
+        let dict = Dictionary::new(&kvs)?;
         let id = PodId(dict.commitment());
         let signature = format!("{}_signed_by_{}", id, pk_hash);
         Ok(Box::new(MockSignedPod {
@@ -49,13 +50,17 @@ impl Pod for MockSignedPod {
         }
 
         // Verify id
-        let mt = MerkleTree::new(
+        let mt = match MerkleTree::new(
+            MAX_DEPTH,
             &self
                 .dict
                 .iter()
                 .map(|(&k, &v)| (k, v))
                 .collect::<HashMap<Value, Value>>(),
-        );
+        ) {
+            Ok(mt) => mt,
+            Err(_) => return false,
+        };
         let id = PodId(mt.root());
         if id != self.id {
             return false;
@@ -93,14 +98,16 @@ impl Pod for MockSignedPod {
 
 #[cfg(test)]
 pub mod tests {
+    use plonky2::field::types::Field;
+    use std::iter;
+
     use super::*;
+    use crate::constants::MAX_DEPTH;
     use crate::frontend;
     use crate::middleware::{self, F, NULL};
-    use plonky2::field::types::Field;
-    use std::iter;
 
     #[test]
-    fn test_mock_signed_0() {
+    fn test_mock_signed_0() -> Result<()> {
         let params = middleware::Params::default();
         let mut pod = frontend::SignedPodBuilder::new(&params);
         pod.insert("idNumber", "4242424242");
@@ -131,7 +138,7 @@ pub mod tests {
             .map(|(AnchoredKey(_, k), v)| (Value(k.0), v))
             .chain(iter::once(bad_kv))
             .collect::<HashMap<Value, Value>>();
-        let bad_mt = MerkleTree::new(&bad_kvs_mt);
+        let bad_mt = MerkleTree::new(MAX_DEPTH, &bad_kvs_mt)?;
         bad_pod.dict.mt = bad_mt;
         assert_eq!(bad_pod.verify(), false);
 
@@ -143,8 +150,10 @@ pub mod tests {
             .map(|(AnchoredKey(_, k), v)| (Value(k.0), v))
             .chain(iter::once(bad_kv))
             .collect::<HashMap<Value, Value>>();
-        let bad_mt = MerkleTree::new(&bad_kvs_mt);
+        let bad_mt = MerkleTree::new(MAX_DEPTH, &bad_kvs_mt)?;
         bad_pod.dict.mt = bad_mt;
         assert_eq!(bad_pod.verify(), false);
+
+        Ok(())
     }
 }

src/constants.rs

@@ -0,0 +1 @@
+pub const MAX_DEPTH: usize = 32;

src/examples.rs

@@ -1,3 +1,4 @@
+use anyhow::Result;
 use std::collections::HashMap;
 
 use crate::backends::mock_signed::MockSigner;
@@ -24,8 +25,8 @@ pub fn zu_kyc_pod_builder(
     params: &Params,
     gov_id: &SignedPod,
     pay_stub: &SignedPod,
-) -> MainPodBuilder {
-    let sanction_list = Value::Dictionary(Dictionary::new(&HashMap::new())); // empty dictionary
+) -> Result<MainPodBuilder> {
+    let sanction_list = Value::Dictionary(Dictionary::new(&HashMap::new())?); // empty dictionary
     let now_minus_18y: i64 = 1169909388;
     let now_minus_1y: i64 = 1706367566;
 
@@ -41,7 +42,7 @@ pub fn zu_kyc_pod_builder(
     ));
     kyc.pub_op(op!(eq, (pay_stub, "startDate"), now_minus_1y));
 
-    kyc
+    Ok(kyc)
 }
 
 // GreatBoy
@@ -130,7 +131,7 @@ pub fn great_boy_pod_builder(
     great_boy
 }
 
-pub fn great_boy_pod_full_flow() -> MainPodBuilder {
+pub fn great_boy_pod_full_flow() -> Result<MainPodBuilder> {
     let params = Params {
         max_input_signed_pods: 6,
         max_statements: 100,
@@ -179,8 +180,8 @@ pub fn great_boy_pod_full_flow() -> MainPodBuilder {
     alice_friend_pods.push(friend.sign(&mut bob_signer).unwrap());
     alice_friend_pods.push(friend.sign(&mut charlie_signer).unwrap());
 
-    let good_boy_issuers_dict = Value::Dictionary(Dictionary::new(&HashMap::new())); // empty
-    great_boy_pod_builder(
+    let good_boy_issuers_dict = Value::Dictionary(Dictionary::new(&HashMap::new())?); // empty
+    Ok(great_boy_pod_builder(
         &params,
         [
             &bob_good_boys[0],
@@ -191,7 +192,7 @@ pub fn great_boy_pod_full_flow() -> MainPodBuilder {
         [&alice_friend_pods[0], &alice_friend_pods[1]],
         &good_boy_issuers_dict,
         alice,
-    )
+    ))
 }
 
 // Tickets
@@ -229,15 +230,15 @@ pub fn tickets_pod_builder(
     builder
 }
 
-pub fn tickets_pod_full_flow() -> MainPodBuilder {
+pub fn tickets_pod_full_flow() -> Result<MainPodBuilder> {
     let params = Params::default();
     let builder = tickets_sign_pod_builder(&params);
     let signed_pod = builder.sign(&mut MockSigner { pk: "test".into() }).unwrap();
-    tickets_pod_builder(
+    Ok(tickets_pod_builder(
         &params,
         &signed_pod,
         123,
         true,
-        &Value::Dictionary(Dictionary::new(&HashMap::new())),
-    )
+        &Value::Dictionary(Dictionary::new(&HashMap::new())?),
+    ))
 }

src/frontend/mod.rs

@@ -510,7 +510,7 @@ pub mod tests {
         let pay_stub = pay_stub.sign(&mut signer).unwrap();
         println!("{}", pay_stub);
 
-        let kyc = zu_kyc_pod_builder(&params, &gov_id, &pay_stub);
+        let kyc = zu_kyc_pod_builder(&params, &gov_id, &pay_stub)?;
         println!("{}", kyc);
 
         // TODO: prove kyc with MockProver and print it
@@ -520,7 +520,7 @@ pub mod tests {
 
     #[test]
     fn test_front_great_boy() -> Result<()> {
-        let great_boy = great_boy_pod_full_flow();
+        let great_boy = great_boy_pod_full_flow()?;
         println!("{}", great_boy);
 
         // TODO: prove kyc with MockProver and print it
@@ -530,7 +530,7 @@ pub mod tests {
 
     #[test]
     fn test_front_tickets() -> Result<()> {
-        let builder = tickets_pod_full_flow();
+        let builder = tickets_pod_full_flow()?;
         println!("{}", builder);
 
         Ok(())

src/lib.rs

@@ -1,4 +1,5 @@
 pub mod backends;
+pub mod constants;
 pub mod frontend;
 pub mod middleware;
 pub mod primitives;

src/middleware/containers.rs

@@ -6,6 +6,7 @@ use plonky2::plonk::config::Hasher;
 use std::collections::HashMap;
 
 use super::{Hash, Value, EMPTY};
+use crate::constants::MAX_DEPTH;
 use crate::primitives::merkletree::{MerkleProof, MerkleTree};
 
 /// Dictionary: the user original keys and values are hashed to be used in the leaf.
@@ -18,37 +19,37 @@ pub struct Dictionary {
 }
 
 impl Dictionary {
-    pub fn new(kvs: &HashMap<Hash, Value>) -> Self {
+    pub fn new(kvs: &HashMap<Hash, Value>) -> Result<Self> {
         let kvs: HashMap<Value, Value> = kvs.into_iter().map(|(&k, &v)| (Value(k.0), v)).collect();
-        Self {
-            mt: MerkleTree::new(&kvs),
-        }
+        Ok(Self {
+            mt: MerkleTree::new(MAX_DEPTH, &kvs)?,
+        })
     }
     pub fn commitment(&self) -> Hash {
         self.mt.root()
     }
     pub fn get(&self, key: &Value) -> Result<Value> {
         self.mt.get(key)
     }
-    pub fn prove(&self, key: &Value) -> Result<MerkleProof> {
+    pub fn prove(&self, key: &Value) -> Result<(Value, MerkleProof)> {
         self.mt.prove(key)
     }
     pub fn prove_nonexistence(&self, key: &Value) -> Result<MerkleProof> {
         self.mt.prove_nonexistence(key)
     }
     pub fn verify(root: Hash, proof: &MerkleProof, key: &Value, value: &Value) -> Result<()> {
-        MerkleTree::verify(root, proof, key, value)
+        MerkleTree::verify(MAX_DEPTH, root, proof, key, value)
     }
     pub fn verify_nonexistence(root: Hash, proof: &MerkleProof, key: &Value) -> Result<()> {
-        MerkleTree::verify_nonexistence(root, proof, key)
+        MerkleTree::verify_nonexistence(MAX_DEPTH, root, proof, key)
     }
-    pub fn iter(&self) -> std::collections::hash_map::Iter<Value, Value> {
+    pub fn iter(&self) -> crate::primitives::merkletree::Iter {
         self.mt.iter()
     }
 }
 impl<'a> IntoIterator for &'a Dictionary {
     type Item = (&'a Value, &'a Value);
-    type IntoIter = std::collections::hash_map::Iter<'a, Value, Value>;
+    type IntoIter = crate::primitives::merkletree::Iter<'a>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.mt.iter()
@@ -71,37 +72,38 @@ pub struct Set {
 }
 
 impl Set {
-    pub fn new(set: &Vec<Value>) -> Self {
+    pub fn new(set: &Vec<Value>) -> Result<Self> {
         let kvs: HashMap<Value, Value> = set
             .into_iter()
             .map(|e| {
                 let h = PoseidonHash::hash_no_pad(&e.0).elements;
                 (Value(h), EMPTY)
             })
             .collect();
-        Self {
-            mt: MerkleTree::new(&kvs),
-        }
+        Ok(Self {
+            mt: MerkleTree::new(MAX_DEPTH, &kvs)?,
+        })
     }
     pub fn commitment(&self) -> Hash {
         self.mt.root()
     }
-    pub fn contains(&self, value: &Value) -> bool {
+    pub fn contains(&self, value: &Value) -> Result<bool> {
         self.mt.contains(value)
     }
     pub fn prove(&self, value: &Value) -> Result<MerkleProof> {
-        self.mt.prove(value)
+        let (_, proof) = self.mt.prove(value)?;
+        Ok(proof)
     }
     pub fn prove_nonexistence(&self, value: &Value) -> Result<MerkleProof> {
         self.mt.prove_nonexistence(value)
     }
     pub fn verify(root: Hash, proof: &MerkleProof, value: &Value) -> Result<()> {
-        MerkleTree::verify(root, proof, value, &EMPTY)
+        MerkleTree::verify(MAX_DEPTH, root, proof, value, &EMPTY)
     }
     pub fn verify_nonexistence(root: Hash, proof: &MerkleProof, value: &Value) -> Result<()> {
-        MerkleTree::verify_nonexistence(root, proof, value)
+        MerkleTree::verify_nonexistence(MAX_DEPTH, root, proof, value)
     }
-    pub fn iter(&self) -> std::collections::hash_map::Iter<Value, Value> {
+    pub fn iter(&self) -> crate::primitives::merkletree::Iter {
         self.mt.iter()
     }
 }
@@ -123,30 +125,30 @@ pub struct Array {
 }
 
 impl Array {
-    pub fn new(array: &Vec<Value>) -> Self {
+    pub fn new(array: &Vec<Value>) -> Result<Self> {
         let kvs: HashMap<Value, Value> = array
             .into_iter()
             .enumerate()
             .map(|(i, &e)| (Value::from(i as i64), e))
             .collect();
 
-        Self {
-            mt: MerkleTree::new(&kvs),
-        }
+        Ok(Self {
+            mt: MerkleTree::new(MAX_DEPTH, &kvs)?,
+        })
     }
     pub fn commitment(&self) -> Hash {
         self.mt.root()
     }
     pub fn get(&self, i: usize) -> Result<Value> {
         self.mt.get(&Value::from(i as i64))
     }
-    pub fn prove(&self, i: usize) -> Result<MerkleProof> {
+    pub fn prove(&self, i: usize) -> Result<(Value, MerkleProof)> {
         self.mt.prove(&Value::from(i as i64))
     }
     pub fn verify(root: Hash, proof: &MerkleProof, i: usize, value: &Value) -> Result<()> {
-        MerkleTree::verify(root, proof, &Value::from(i as i64), value)
+        MerkleTree::verify(MAX_DEPTH, root, proof, &Value::from(i as i64), value)
     }
-    pub fn iter(&self) -> std::collections::hash_map::Iter<Value, Value> {
+    pub fn iter(&self) -> crate::primitives::merkletree::Iter {
         self.mt.iter()
     }
 }

src/middleware/mod.rs

@@ -48,6 +48,15 @@ pub type Entry = (String, Value);
 #[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq)]
 pub struct Value(pub [F; 4]);
 
+impl Value {
+    pub fn to_bytes(self) -> Vec<u8> {
+        self.0
+            .iter()
+            .flat_map(|e| e.to_canonical_u64().to_le_bytes())
+            .collect()
+    }
+}
+
 impl Ord for Value {
     fn cmp(&self, other: &Self) -> Ordering {
         for (lhs, rhs) in self.0.iter().zip(other.0.iter()).rev() {