Add verifier-datas tree (set) & in-circuit verification (#274)

* containers: add method to create new {Dict,Set,Array} with custom max_depth

* add vds_tree computation, update tree circuit interface

* add VDTree struct, add DEFAULT_VD_TREE, integrate it with MainPod,EmptyPod,frontend,etc.

* adapt frontend/serialization tests to new containers field (max_depth)

* adapt interfaces to allow using custom vd_tree in frontend & backend constructors

* rename VDTree to VDSet (and derivate namings too)

* containers 'new' always with param 'max_depth', use params.max_depth_mt_containers instead of the global constant MAX_DEPTH

* adapt after rebasing the branch to main latest changes

* apply review suggestions from @ed255

* use emptypod vd_mt_proofs (using vd_set as circuit input), merge the two existing set_targets methods of MainPodVerifyTarget

* document VDSet & vds_root

Author: arnaucube

src/backends/plonky2/basetypes.rs

@@ -36,3 +36,91 @@ pub type VerifierCircuitData = circuit_data::VerifierCircuitData<F, C, D>;
 pub type CircuitBuilder = circuit_builder::CircuitBuilder<F, D>;
 pub type Proof = proof::Proof<F, C, D>;
 pub type ProofWithPublicInputs = proof::ProofWithPublicInputs<F, C, D>;
+
+use std::{collections::HashMap, sync::LazyLock};
+
+use itertools::Itertools;
+use plonky2::hash::hash_types::HashOut;
+
+use crate::{
+    backends::plonky2::{
+        emptypod::STANDARD_EMPTY_POD_DATA, primitives::merkletree::MerkleClaimAndProof,
+        DEFAULT_PARAMS, STANDARD_REC_MAIN_POD_CIRCUIT_DATA,
+    },
+    middleware::{containers::Array, Hash, RawValue, Result, Value},
+};
+
+pub static DEFAULT_VD_SET: LazyLock<VDSet> = LazyLock::new(|| {
+    let params = &*DEFAULT_PARAMS;
+
+    let vds = vec![
+        STANDARD_REC_MAIN_POD_CIRCUIT_DATA.verifier_only.clone(),
+        STANDARD_EMPTY_POD_DATA.1.verifier_only.clone(),
+    ];
+    VDSet::new(params.max_depth_mt_vds, &vds).unwrap()
+});
+
+/// VDSet is the set of the allowed verifier_data hashes. When proving a
+/// MainPod, the circuit will enforce that all the used verifier_datas for
+/// verifying the recursive proofs of previous PODs appears in the VDSet.
+/// The VDSet struct that allows to get the specific merkle proofs for the given
+/// verifier_data.
+#[derive(Clone, Debug)]
+pub struct VDSet {
+    root: Hash,
+    // (verifier_data, merkleproof)
+    proofs_map: HashMap<HashOut<F>, MerkleClaimAndProof>,
+}
+impl VDSet {
+    /// builds the verifier_datas tree, and returns the root and the proofs
+    pub fn new(tree_depth: usize, vds: &[VerifierOnlyCircuitData]) -> Result<Self> {
+        // first of all, sort the vds, so that each set of verifier_datas gets
+        // the same root
+        let vds: Vec<&VerifierOnlyCircuitData> = vds
+            .iter()
+            .sorted_by_key(|vd| RawValue(vd.circuit_digest.elements))
+            .collect::<Vec<_>>();
+
+        let array = Array::new(
+            tree_depth,
+            vds.iter()
+                .map(|vd| Value::from(RawValue(vd.circuit_digest.elements)))
+                .collect(),
+        )?;
+
+        let root = array.commitment();
+        let mut proofs_map = HashMap::<HashOut<F>, MerkleClaimAndProof>::new();
+
+        for (i, vd) in vds.iter().enumerate() {
+            let (value, proof) = array.prove(i)?;
+            let p = MerkleClaimAndProof {
+                root,
+                key: RawValue::from(i as i64),
+                value: value.raw(),
+                proof,
+            };
+            proofs_map.insert(vd.circuit_digest, p);
+        }
+        Ok(Self { root, proofs_map })
+    }
+    pub fn root(&self) -> Hash {
+        self.root
+    }
+    /// returns the vector of merkle proofs corresponding to the given verifier_datas
+    pub fn get_vds_proofs(
+        &self,
+        vds: &[VerifierOnlyCircuitData],
+    ) -> Result<Vec<MerkleClaimAndProof>> {
+        let mut proofs: Vec<MerkleClaimAndProof> = vec![];
+        for vd in vds {
+            let p =
+                self.proofs_map
+                    .get(&vd.circuit_digest)
+                    .ok_or(crate::middleware::Error::custom(
+                        "verifier_data not found in VDSet".to_string(),
+                    ))?;
+            proofs.push(p.clone());
+        }
+        Ok(proofs)
+    }
+}

src/backends/plonky2/circuits/common.rs

@@ -40,7 +40,7 @@ use crate::{
 pub const CODE_SIZE: usize = HASH_SIZE + 2;
 const NUM_BITS: usize = 32;
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub struct ValueTarget {
     pub elements: [Target; VALUE_SIZE],
 }

src/backends/plonky2/circuits/mainpod.rs

@@ -17,7 +17,7 @@ use plonky2::{
 
 use crate::{
     backends::plonky2::{
-        basetypes::CircuitBuilder,
+        basetypes::{CircuitBuilder, VDSet},
         circuits::{
             common::{
                 CircuitBuilderPod, CustomPredicateBatchTarget, CustomPredicateEntryTarget,
@@ -28,7 +28,7 @@ use crate::{
             },
             signedpod::{SignedPodVerifyGadget, SignedPodVerifyTarget},
         },
-        emptypod::EmptyPod,
+        emptypod::{EmptyPod, STANDARD_EMPTY_POD_DATA},
         error::Result,
         mainpod::{self, pad_statement},
         primitives::merkletree::{
@@ -39,10 +39,9 @@ use crate::{
     },
     measure_gates_begin, measure_gates_end,
     middleware::{
-        AnchoredKey, CustomPredicate, CustomPredicateBatch, CustomPredicateRef, Hash,
-        NativeOperation, NativePredicate, Params, PodType, PredicatePrefix, Statement,
-        StatementArg, ToFields, Value, WildcardValue, EMPTY_VALUE, F, HASH_SIZE, KEY_TYPE, SELF,
-        VALUE_SIZE,
+        AnchoredKey, CustomPredicate, CustomPredicateBatch, CustomPredicateRef, NativeOperation,
+        NativePredicate, Params, PodType, PredicatePrefix, Statement, StatementArg, ToFields,
+        Value, WildcardValue, EMPTY_VALUE, F, HASH_SIZE, KEY_TYPE, SELF, VALUE_SIZE,
     },
 };
 
@@ -1220,9 +1219,31 @@ impl MainPodVerifyGadget {
         }
 
         let vds_root = builder.add_virtual_hash();
-        // TODO: verify that all input pod proofs use verifier data from the public input VD array
-        // This requires merkle proofs
-        // https://github.com/0xPARC/pod2/issues/250
+
+        // verify that all input pod proofs use verifier data from the public input VD array This
+        // requires merkle proofs
+        let mut vd_mt_proofs: Vec<MerkleClaimAndProofTarget> = vec![];
+        for verified_proof in verified_proofs {
+            // add target for the vd_mt_proof
+            let vd_mt_proof = MerkleProofGadget {
+                max_depth: params.max_depth_mt_vds,
+            }
+            .eval(builder);
+
+            // ensure that mt_proof is enabled
+            let true_targ = builder._true();
+            builder.connect(vd_mt_proof.enabled.target, true_targ.target);
+            // connect the vd_mt_proof's root to the actual vds_root, to ensure that the mt proof
+            // verifies against the vds_root
+            builder.connect_hashes(vds_root, vd_mt_proof.root);
+            // connect vd_mt_proof's value with the verified_proof.verifier_data_hash
+            builder.connect_hashes(
+                verified_proof.verifier_data_hash,
+                HashOutTarget::from_vec(vd_mt_proof.value.elements.to_vec()),
+            );
+
+            vd_mt_proofs.push(vd_mt_proof);
+        }
 
         // Verify that VD array that input pod uses is the same we use now.
         for verified_proof in verified_proofs {
@@ -1247,11 +1268,11 @@ impl MainPodVerifyGadget {
 
         // Add Merkle claim/proof targets
         let mp_gadget = MerkleProofGadget {
-            max_depth: params.max_depth_mt_gadget,
+            max_depth: params.max_depth_mt_containers,
         };
-        let merkle_proofs: Vec<_> = (0..params.max_merkle_proofs)
+        let merkle_proofs: Vec<_> = (0..params.max_merkle_proofs_containers)
             .map(|_| mp_gadget.eval(builder))
-            .collect::<Result<_>>()?;
+            .collect();
         let merkle_claims: Vec<_> = merkle_proofs
             .clone()
             .into_iter()
@@ -1310,6 +1331,7 @@ impl MainPodVerifyGadget {
         Ok(MainPodVerifyTarget {
             params: params.clone(),
             vds_root,
+            vd_mt_proofs,
             id,
             signed_pods,
             input_pods_self_statements,
@@ -1325,6 +1347,7 @@ impl MainPodVerifyGadget {
 pub struct MainPodVerifyTarget {
     params: Params,
     vds_root: HashOutTarget,
+    vd_mt_proofs: Vec<MerkleClaimAndProofTarget>,
     id: HashOutTarget,
     signed_pods: Vec<SignedPodVerifyTarget>,
     input_pods_self_statements: Vec<Vec<StatementTarget>>,
@@ -1344,7 +1367,11 @@ pub struct CustomPredicateVerification {
 }
 
 pub struct MainPodVerifyInput {
-    pub vds_root: Hash,
+    pub vds_set: VDSet,
+    // field containing the `vd_mt_proofs` aside from the `vds_set`, because
+    // inide the MainPodVerifyTarget circuit, since it is the InnerCircuit for
+    // the RecursiveCircuit, we don't have access to the used verifier_datas.
+    pub vd_mt_proofs: Vec<MerkleClaimAndProof>,
     pub signed_pods: Vec<SignedPod>,
     pub recursive_pods_pub_self_statements: Vec<Vec<Statement>>,
     pub statements: Vec<mainpod::Statement>,
@@ -1378,13 +1405,58 @@ fn set_targets_input_pods_self_statements(
     Ok(())
 }
 
-impl MainPodVerifyTarget {
-    pub fn set_targets(
+pub struct MainPodVerifyCircuit {
+    pub params: Params,
+}
+
+// TODO: Remove this type and implement it's logic directly in `impl InnerCircuit for MainPodVerifyTarget`
+impl MainPodVerifyCircuit {
+    pub fn eval(
         &self,
-        pw: &mut PartialWitness<F>,
-        input: &MainPodVerifyInput,
-    ) -> Result<()> {
-        pw.set_target_arr(&self.vds_root.elements, &input.vds_root.0)?;
+        builder: &mut CircuitBuilder,
+        verified_proofs: &[VerifiedProofTarget],
+    ) -> Result<MainPodVerifyTarget> {
+        let main_pod = MainPodVerifyGadget {
+            params: self.params.clone(),
+        }
+        .eval(builder, verified_proofs)?;
+        builder.register_public_inputs(&main_pod.id.elements);
+        builder.register_public_inputs(&main_pod.vds_root.elements);
+        Ok(main_pod)
+    }
+}
+
+impl InnerCircuit for MainPodVerifyTarget {
+    type Input = MainPodVerifyInput;
+    type Params = Params;
+
+    fn build(
+        builder: &mut CircuitBuilder,
+        params: &Self::Params,
+        verified_proofs: &[VerifiedProofTarget],
+    ) -> Result<Self> {
+        MainPodVerifyCircuit {
+            params: params.clone(),
+        }
+        .eval(builder, verified_proofs)
+    }
+
+    /// assigns the values to the targets
+    fn set_targets(&self, pw: &mut PartialWitness<F>, input: &Self::Input) -> Result<()> {
+        let vds_root = input.vds_set.root();
+        pw.set_target_arr(&self.vds_root.elements, &vds_root.0)?;
+
+        for (i, vd_mt_proof) in input.vd_mt_proofs.iter().enumerate() {
+            self.vd_mt_proofs[i].set_targets(pw, true, vd_mt_proof)?;
+        }
+        // the rest of vd_mt_proofs set them to the empty_pod vd_mt_proof
+        let vd_emptypod_mt_proof = input
+            .vds_set
+            .get_vds_proofs(&[STANDARD_EMPTY_POD_DATA.1.verifier_only.clone()])?;
+        let vd_emptypod_mt_proof = vd_emptypod_mt_proof[0].clone();
+        for i in input.vd_mt_proofs.len()..self.vd_mt_proofs.len() {
+            self.vd_mt_proofs[i].set_targets(pw, true, &vd_emptypod_mt_proof)?;
+        }
 
         assert!(input.signed_pods.len() <= self.params.max_input_signed_pods);
         for (i, signed_pod) in input.signed_pods.iter().enumerate() {
@@ -1414,7 +1486,7 @@ impl MainPodVerifyTarget {
         }
         // Padding
         if input.recursive_pods_pub_self_statements.len() != self.params.max_input_recursive_pods {
-            let empty_pod = EmptyPod::new_boxed(&self.params, input.vds_root);
+            let empty_pod = EmptyPod::new_boxed(&self.params, input.vds_set.root());
             let empty_pod_statements = empty_pod.pub_statements();
             for i in
                 input.recursive_pods_pub_self_statements.len()..self.params.max_input_recursive_pods
@@ -1434,13 +1506,13 @@ impl MainPodVerifyTarget {
             self.operations[i].set_targets(pw, &self.params, op)?;
         }
 
-        assert!(input.merkle_proofs.len() <= self.params.max_merkle_proofs);
+        assert!(input.merkle_proofs.len() <= self.params.max_merkle_proofs_containers);
         for (i, mp) in input.merkle_proofs.iter().enumerate() {
             self.merkle_proofs[i].set_targets(pw, true, mp)?;
         }
         // Padding
         let pad_mp = MerkleClaimAndProof::empty();
-        for i in input.merkle_proofs.len()..self.params.max_merkle_proofs {
+        for i in input.merkle_proofs.len()..self.params.max_merkle_proofs_containers {
             self.merkle_proofs[i].set_targets(pw, false, &pad_mp)?;
         }
 
@@ -1487,48 +1559,6 @@ impl MainPodVerifyTarget {
     }
 }
 
-pub struct MainPodVerifyCircuit {
-    pub params: Params,
-}
-
-// TODO: Remove this type and implement it's logic directly in `impl InnerCircuit for MainPodVerifyTarget`
-impl MainPodVerifyCircuit {
-    pub fn eval(
-        &self,
-        builder: &mut CircuitBuilder,
-        verified_proofs: &[VerifiedProofTarget],
-    ) -> Result<MainPodVerifyTarget> {
-        let main_pod = MainPodVerifyGadget {
-            params: self.params.clone(),
-        }
-        .eval(builder, verified_proofs)?;
-        builder.register_public_inputs(&main_pod.id.elements);
-        builder.register_public_inputs(&main_pod.vds_root.elements);
-        Ok(main_pod)
-    }
-}
-
-impl InnerCircuit for MainPodVerifyTarget {
-    type Input = MainPodVerifyInput;
-    type Params = Params;
-
-    fn build(
-        builder: &mut CircuitBuilder,
-        params: &Self::Params,
-        verified_proofs: &[VerifiedProofTarget],
-    ) -> Result<Self> {
-        MainPodVerifyCircuit {
-            params: params.clone(),
-        }
-        .eval(builder, verified_proofs)
-    }
-
-    /// assigns the values to the targets
-    fn set_targets(&self, pw: &mut PartialWitness<F>, input: &Self::Input) -> Result<()> {
-        self.set_targets(pw, input)
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::{iter, ops::Not};
@@ -1567,7 +1597,7 @@ mod tests {
             ..Default::default()
         };
         let mp_gadget = MerkleProofGadget {
-            max_depth: params.max_depth_mt_gadget,
+            max_depth: params.max_depth_mt_containers,
         };
 
         let config = CircuitConfig::standard_recursion_config();
@@ -1581,7 +1611,7 @@ mod tests {
         let merkle_proofs_target: Vec<_> = merkle_proofs
             .iter()
             .map(|_| mp_gadget.eval(&mut builder))
-            .collect::<Result<_>>()?;
+            .collect();
         let merkle_claims_target: Vec<_> = merkle_proofs_target
             .clone()
             .into_iter()
@@ -2360,7 +2390,7 @@ mod tests {
         ]
         .into_iter()
         .collect();
-        let mt = MerkleTree::new(params.max_depth_mt_gadget, &kvs)?;
+        let mt = MerkleTree::new(params.max_depth_mt_containers, &kvs)?;
 
         let root = Value::from(mt.root());
         let root_ak = AnchoredKey::from((PodId(RawValue::from(88).into()), "merkle root"));
@@ -2400,7 +2430,7 @@ mod tests {
         ]
         .into_iter()
         .collect();
-        let mt = MerkleTree::new(params.max_depth_mt_gadget, &kvs)?;
+        let mt = MerkleTree::new(params.max_depth_mt_containers, &kvs)?;
 
         let root = Value::from(mt.root());
         let root_ak = AnchoredKey::from((PodId(RawValue::from(88).into()), "merkle root"));

src/backends/plonky2/circuits/signedpod.rs

@@ -41,7 +41,7 @@ impl SignedPodVerifyGadget {
         let mut mt_proofs = Vec::new();
         for _ in 0..self.params.max_signed_pod_values {
             let mt_proof = MerkleProofExistenceGadget {
-                max_depth: self.params.max_depth_mt_gadget,
+                max_depth: self.params.max_depth_mt_containers,
             }
             .eval(builder)?;
             builder.connect_hashes(id, mt_proof.root);