Fe contains (#145)

* Contains should take three arguments (root, key, value)

* Add a test for frontend Dictionaries

* Separate frontend and middleware operations

* Make tests pass: add arg to contains

* Cargo fmt

* Merkleproof verify circuit (#143)

* merkletree: add keypath circuit

* merkletree-circuit: implement proof of existence verification in-circuit

* parametrize max_depth at the tree circuit

* Constrain selectors in-circuit

* implement merketree nonexistence proof circuit, and add edgecase tests

* add non-existence proofs documentation in the mdbook, mv EMPTY->EMPTY_VALUE & NULL->EMPTY_HASH, dependency clean and public exposure methods

* review comments, some extra polishing and add a test that expects wrong proofs to fail

* Add circuit to check only merkleproofs-of-existence

With this, the merkletree_circuit module offers two different circuits:
- `MerkleProofCircuit`: allows to verify both proofs of existence and proofs
non-existence with the same circuit.
- `MerkleProofExistenceCircuit`: allows to verify proofs of existence only.

In this way, if only proofs of existence are needed,
`MerkleProofExistenceCircuit` should be used, which requires less amount
of constraints than `MerkleProofCircuit`.

* Code review

---------

Co-authored-by: Ahmad <root@ahmadafuni.com>

* Towards Contains/NotContains in middleware and backend

* Fix build

* Adding error handling to deal with op compile introduce extra ops

* Incorporate Merkle proofs into MockMainPod

* Merkleproof verify circuit (#143)

* merkletree: add keypath circuit

* merkletree-circuit: implement proof of existence verification in-circuit

* parametrize max_depth at the tree circuit

* Constrain selectors in-circuit

* implement merketree nonexistence proof circuit, and add edgecase tests

* add non-existence proofs documentation in the mdbook, mv EMPTY->EMPTY_VALUE & NULL->EMPTY_HASH, dependency clean and public exposure methods

* review comments, some extra polishing and add a test that expects wrong proofs to fail

* Add circuit to check only merkleproofs-of-existence

With this, the merkletree_circuit module offers two different circuits:
- `MerkleProofCircuit`: allows to verify both proofs of existence and proofs
non-existence with the same circuit.
- `MerkleProofExistenceCircuit`: allows to verify proofs of existence only.

In this way, if only proofs of existence are needed,
`MerkleProofExistenceCircuit` should be used, which requires less amount
of constraints than `MerkleProofCircuit`.

* Code review

---------

Co-authored-by: Ahmad <root@ahmadafuni.com>

* Towards Contains/NotContains in middleware and backend

* Frontend compound types -- allow one frontend operation to produce multiple middleware statements (in progress)

* Incorporate Merkle proofs into MockMainPod

* Incorporate Merkle proof op arg into frontend

* Compile one statement to many, in progress

* Fix remaining tests

* Minor clean-up

* Oops I did a bunch of work in the middle of a rebase, committing

* Incorporate Merkle proof op arg into frontend

* still working on frontend compound types, refactor compile() to output multiple statements

* Contains statements for frontend types: code compiles

* Tests pass

* Examples use front-end compound types

* Remove old Contains and NotContains from frontend

* Add nin to typos

* Code review

---------

Co-authored-by: arnaucube <git@arnaucube.com>
Co-authored-by: Ahmad <root@ahmadafuni.com>

Author: 3 people

.github/workflows/typos.toml

@@ -4,3 +4,4 @@ BA = "BA"
 Ded = "Ded" # "ANDed", it thought "Ded" should be "Dead"
 OT = "OT"
 aks = "aks" # anchored keys
+nin = "nin" # not in

src/backends/plonky2/circuits/mainpod.rs

@@ -491,7 +491,10 @@ impl MainPodVerifyCircuit {
 mod tests {
     use super::*;
     use crate::backends::plonky2::mock::mainpod;
-    use crate::backends::plonky2::{basetypes::C, mock::mainpod::OperationArg};
+    use crate::backends::plonky2::{
+        basetypes::C,
+        mock::mainpod::{OperationArg, OperationAux},
+    };
     use crate::middleware::{OperationType, PodId};
     use plonky2::plonk::{circuit_builder::CircuitBuilder, circuit_data::CircuitConfig};
 
@@ -571,20 +574,29 @@ mod tests {
     fn test_operation_verify() -> Result<()> {
         // None
         let st: mainpod::Statement = Statement::None.into();
-        let op = mainpod::Operation(OperationType::Native(NativeOperation::None), vec![]);
+        let op = mainpod::Operation(
+            OperationType::Native(NativeOperation::None),
+            vec![],
+            OperationAux::None,
+        );
         let prev_statements = vec![Statement::None.into()];
         operation_verify(st.clone(), op, prev_statements.clone())?;
 
         // NewEntry
         let st1: mainpod::Statement =
             Statement::ValueOf(AnchoredKey(SELF, "hello".into()), 55.into()).into();
-        let op = mainpod::Operation(OperationType::Native(NativeOperation::NewEntry), vec![]);
+        let op = mainpod::Operation(
+            OperationType::Native(NativeOperation::NewEntry),
+            vec![],
+            OperationAux::None,
+        );
         operation_verify(st1.clone(), op, vec![])?;
 
         // Copy
         let op = mainpod::Operation(
             OperationType::Native(NativeOperation::CopyStatement),
             vec![OperationArg::Index(0)],
+            OperationAux::None,
         );
         operation_verify(st, op, prev_statements)?;
 
@@ -602,6 +614,7 @@ mod tests {
         let op = mainpod::Operation(
             OperationType::Native(NativeOperation::EqualFromEntries),
             vec![OperationArg::Index(0), OperationArg::Index(1)],
+            OperationAux::None,
         );
         let prev_statements = vec![st1.clone(), st2];
         operation_verify(st, op, prev_statements)?;
@@ -620,6 +633,7 @@ mod tests {
         let op = mainpod::Operation(
             OperationType::Native(NativeOperation::LtFromEntries),
             vec![OperationArg::Index(0), OperationArg::Index(1)],
+            OperationAux::None,
         );
         let prev_statements = vec![st1.clone(), st2];
         operation_verify(st, op, prev_statements)?;

src/backends/plonky2/mock/mainpod/mod.rs

@@ -8,10 +8,13 @@ use serde::{Deserialize, Serialize};
 use std::any::Any;
 use std::fmt;
 
-use crate::middleware::{
-    self, hash_str, AnchoredKey, Hash, MainPodInputs, NativeOperation, NativePredicate, NonePod,
-    OperationType, Params, Pod, PodId, PodProver, Predicate, StatementArg, ToFields, KEY_TYPE,
-    SELF,
+use crate::{
+    backends::plonky2::primitives::merkletree::MerkleProof,
+    middleware::{
+        self, hash_str, AnchoredKey, Hash, MainPodInputs, NativeOperation, NativePredicate,
+        NonePod, OperationType, Params, Pod, PodId, PodProver, Predicate, StatementArg, ToFields,
+        KEY_TYPE, SELF,
+    },
 };
 
 mod operation;
@@ -41,6 +44,9 @@ pub struct MockMainPod {
     operations: Vec<Operation>,
     // All statements (inherited + new)
     statements: Vec<Statement>,
+    // All Merkle proofs
+    // TODO: Use a backend-specific representation
+    merkle_proofs: Vec<MerkleProof>,
 }
 
 impl fmt::Display for MockMainPod {
@@ -243,9 +249,28 @@ impl MockMainPod {
         }
     }
 
+    fn find_op_aux(
+        merkle_proofs: &[MerkleProof],
+        op_aux: &middleware::OperationAux,
+    ) -> Result<OperationAux> {
+        match op_aux {
+            middleware::OperationAux::None => Ok(OperationAux::None),
+            middleware::OperationAux::MerkleProof(pf_arg) => merkle_proofs
+                .iter()
+                .enumerate()
+                .find_map(|(i, pf)| (pf == pf_arg).then_some(i))
+                .map(OperationAux::MerkleProofIndex)
+                .ok_or(anyhow!(
+                    "Merkle proof corresponding to op arg {} not found",
+                    op_aux
+                )),
+        }
+    }
+
     fn process_private_statements_operations(
         params: &Params,
         statements: &[Statement],
+        merkle_proofs: &[MerkleProof],
         input_operations: &[middleware::Operation],
     ) -> Result<Vec<Operation>> {
         let mut operations = Vec::new();
@@ -259,8 +284,12 @@ impl MockMainPod {
                 .iter()
                 .map(|mid_arg| Self::find_op_arg(statements, mid_arg))
                 .collect::<Result<Vec<_>>>()?;
+
+            let mid_aux = op.aux();
+            let aux = Self::find_op_aux(merkle_proofs, &mid_aux)?;
+
             Self::pad_operation_args(params, &mut args);
-            operations.push(Operation(op.op_type(), args));
+            operations.push(Operation(op.op_type(), args, aux));
         }
         Ok(operations)
     }
@@ -278,17 +307,22 @@ impl MockMainPod {
         operations.push(Operation(
             OperationType::Native(NativeOperation::NewEntry),
             vec![],
+            OperationAux::None,
         ));
         for i in 0..(params.max_public_statements - 1) {
             let st = &statements[offset_public_statements + i + 1];
             let mut op = if st.is_none() {
-                Operation(OperationType::Native(NativeOperation::None), vec![])
+                Operation(
+                    OperationType::Native(NativeOperation::None),
+                    vec![],
+                    OperationAux::None,
+                )
             } else {
                 let mid_arg = st.clone();
                 Operation(
                     OperationType::Native(NativeOperation::CopyStatement),
-                    // TODO
-                    vec![Self::find_op_arg(statements, &mid_arg.try_into().unwrap())?],
+                    vec![Self::find_op_arg(statements, &mid_arg.try_into()?)?],
+                    OperationAux::None,
                 )
             };
             fill_pad(&mut op.1, OperationArg::None, params.max_operation_args);
@@ -304,8 +338,21 @@ impl MockMainPod {
         // TODO: Insert a new public statement of ValueOf with `key=KEY_TYPE,
         // value=PodType::MockMainPod`
         let statements = Self::layout_statements(params, &inputs);
-        let operations =
-            Self::process_private_statements_operations(params, &statements, inputs.operations)?;
+        let merkle_proofs = inputs
+            .operations
+            .iter()
+            .flat_map(|op| match op {
+                middleware::Operation::ContainsFromEntries(_, _, _, pf) => Some(pf.clone()),
+                middleware::Operation::NotContainsFromEntries(_, _, pf) => Some(pf.clone()),
+                _ => None,
+            })
+            .collect::<Vec<_>>();
+        let operations = Self::process_private_statements_operations(
+            params,
+            &statements,
+            &merkle_proofs,
+            inputs.operations,
+        )?;
         let operations =
             Self::process_public_statements_operations(params, &statements, operations)?;
 
@@ -340,6 +387,7 @@ impl MockMainPod {
             public_statements,
             statements,
             operations,
+            merkle_proofs,
         })
     }
 
@@ -350,7 +398,11 @@ impl MockMainPod {
     }
 
     fn operation_none(params: &Params) -> Operation {
-        let mut op = Operation(OperationType::Native(NativeOperation::None), vec![]);
+        let mut op = Operation(
+            OperationType::Native(NativeOperation::None),
+            vec![],
+            OperationAux::None,
+        );
         fill_pad(&mut op.1, OperationArg::None, params.max_operation_args);
         op
     }
@@ -444,7 +496,10 @@ impl Pod for MockMainPod {
             .enumerate()
             .map(|(i, s)| {
                 self.operations[i]
-                    .deref(&self.statements[..input_statement_offset + i])
+                    .deref(
+                        &self.statements[..input_statement_offset + i],
+                        &self.merkle_proofs,
+                    )
                     .unwrap()
                     .check_and_log(&self.params, &s.clone().try_into().unwrap())
             })
@@ -513,13 +568,18 @@ pub mod tests {
         zu_kyc_sign_pod_builders,
     };
     use crate::middleware;
+    use crate::middleware::containers::Set;
 
     #[test]
     fn test_mock_main_zu_kyc() -> Result<()> {
         let params = middleware::Params::default();
+        let sanctions_values = ["A343434340"].map(|s| crate::frontend::Value::from(s));
+        let sanction_set = crate::frontend::Value::Set(crate::frontend::containers::Set::new(
+            sanctions_values.to_vec(),
+        )?);
 
         let (gov_id_builder, pay_stub_builder, sanction_list_builder) =
-            zu_kyc_sign_pod_builders(&params);
+            zu_kyc_sign_pod_builders(&params, &sanction_set);
         let mut signer = MockSigner {
             pk: "ZooGov".into(),
         };

src/backends/plonky2/mock/mainpod/operation.rs

@@ -1,6 +1,9 @@
 use super::Statement;
-use crate::middleware::{self, OperationType, Params, ToFields, F};
-use anyhow::Result;
+use crate::{
+    backends::plonky2::primitives::merkletree::MerkleProof,
+    middleware::{self, OperationType, Params, ToFields, F},
+};
+use anyhow::{anyhow, Result};
 use plonky2::field::types::{Field, PrimeField64};
 use serde::{Deserialize, Serialize};
 use std::fmt;
@@ -28,7 +31,13 @@ impl OperationArg {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Operation(pub OperationType, pub Vec<OperationArg>);
+pub enum OperationAux {
+    None,
+    MerkleProofIndex(usize),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Operation(pub OperationType, pub Vec<OperationArg>, pub OperationAux);
 
 impl Operation {
     pub fn op_type(&self) -> OperationType {
@@ -37,16 +46,28 @@ impl Operation {
     pub fn args(&self) -> &[OperationArg] {
         &self.1
     }
-    pub fn deref(&self, statements: &[Statement]) -> Result<crate::middleware::Operation> {
+    pub fn deref(
+        &self,
+        statements: &[Statement],
+        merkle_proofs: &[MerkleProof],
+    ) -> Result<crate::middleware::Operation> {
         let deref_args = self
             .1
             .iter()
             .flat_map(|arg| match arg {
                 OperationArg::None => None,
                 OperationArg::Index(i) => Some(statements[*i].clone().try_into()),
             })
-            .collect::<Result<Vec<crate::middleware::Statement>>>()?;
-        middleware::Operation::op(self.0.clone(), &deref_args)
+            .collect::<Result<Vec<_>>>()?;
+        let deref_aux = match self.2 {
+            OperationAux::None => Ok(crate::middleware::OperationAux::None),
+            OperationAux::MerkleProofIndex(i) => merkle_proofs
+                .get(i)
+                .cloned()
+                .ok_or(anyhow!("Missing Merkle proof index {}", i))
+                .map(crate::middleware::OperationAux::MerkleProof),
+        }?;
+        middleware::Operation::op(self.0.clone(), &deref_args, &deref_aux)
     }
 }
 
@@ -64,6 +85,10 @@ impl fmt::Display for Operation {
                 }
             }
         }
+        match self.2 {
+            OperationAux::None => (),
+            OperationAux::MerkleProofIndex(i) => write!(f, "merkle_proof_{:02}", i)?,
+        }
         Ok(())
     }
 }

src/backends/plonky2/mock/mainpod/statement.rs

@@ -60,8 +60,8 @@ impl TryFrom<Statement> for middleware::Statement {
                 }
                 (NP::Gt, (Some(SA::Key(ak1)), Some(SA::Key(ak2)), None), 2) => S::Gt(ak1, ak2),
                 (NP::Lt, (Some(SA::Key(ak1)), Some(SA::Key(ak2)), None), 2) => S::Lt(ak1, ak2),
-                (NP::Contains, (Some(SA::Key(ak1)), Some(SA::Key(ak2)), None), 2) => {
-                    S::Contains(ak1, ak2)
+                (NP::Contains, (Some(SA::Key(ak1)), Some(SA::Key(ak2)), Some(SA::Key(ak3))), 3) => {
+                    S::Contains(ak1, ak2, ak3)
                 }
                 (NP::NotContains, (Some(SA::Key(ak1)), Some(SA::Key(ak2)), None), 2) => {
                     S::NotContains(ak1, ak2)

src/backends/plonky2/primitives/merkletree.rs

@@ -2,6 +2,7 @@
 //! https://0xparc.github.io/pod2/merkletree.html .
 use anyhow::{anyhow, Result};
 use plonky2::field::types::Field;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt;
 use std::iter::IntoIterator;
@@ -207,7 +208,7 @@ impl fmt::Display for MerkleTree {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct MerkleProof {
     // note: currently we don't use the `_existence` field, we would use if we merge the methods
     // `verify` and `verify_nonexistence` into a single one