Re-implement serialization (#201)

* Serialization tests now pass again

* Tidy up and test more edge-cases

* Use attributes rather than custom serializer for arrays

* Add JSON Schema support

* Tests for JSON Schema generation and validation

* Add comments

* Support custom predicates

* Clippy fixes

* Make deserialization/constructor functions pub(crate)

Author: robknight

Cargo.toml

@@ -22,12 +22,17 @@ plonky2 = { git = "https://github.com/0xPolygonZero/plonky2", optional = true }
 serde = "1.0.219"
 serde_json = "1.0.140"
 base64 = "0.22.1"
-schemars = "1.0.0-alpha.17"
+schemars = "0.8.22"
 
 # Uncomment for debugging with https://github.com/ed255/plonky2/ at branch `feat/debug`.  The repo directory needs to be checked out next to the pod2 repo directory.
 # [patch."https://github.com/0xPolygonZero/plonky2"]
 # plonky2 = { path = "../plonky2/plonky2" }
 
+[dev-dependencies]
+pretty_assertions = "1.4.1"
+# Used only for testing JSON Schema generation and validation.
+jsonschema = "0.30.0"
+
 [features]
 default = ["backend_plonky2"]
 backend_plonky2 = ["plonky2"]

src/backends/plonky2/mainpod/operation.rs

@@ -2,14 +2,14 @@ use std::fmt;
 
 use anyhow::{anyhow, Result};
 use plonky2::field::types::Field;
+use serde::{Deserialize, Serialize};
 
-// use serde::{Deserialize, Serialize};
 use crate::{
     backends::plonky2::{mainpod::Statement, primitives::merkletree::MerkleClaimAndProof},
     middleware::{self, OperationType, Params, ToFields, F},
 };
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum OperationArg {
     None,
     Index(usize),
@@ -31,7 +31,7 @@ impl OperationArg {
     }
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum OperationAux {
     None,
     MerkleProofIndex(usize),
@@ -47,7 +47,7 @@ impl ToFields for OperationAux {
     }
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct Operation(pub OperationType, pub Vec<OperationArg>, pub OperationAux);
 
 impl Operation {

src/backends/plonky2/mainpod/statement.rs

@@ -1,13 +1,13 @@
 use std::fmt;
 
 use anyhow::{anyhow, Result};
+use serde::{Deserialize, Serialize};
 
-// use serde::{Deserialize, Serialize};
 use crate::middleware::{
     self, NativePredicate, Params, Predicate, StatementArg, ToFields, WildcardValue,
 };
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct Statement(pub Predicate, pub Vec<StatementArg>);
 
 impl Statement {

src/backends/plonky2/mock/mainpod.rs

@@ -5,15 +5,15 @@
 use std::{any::Any, fmt};
 
 use anyhow::{anyhow, Result};
+use base64::{prelude::BASE64_STANDARD, Engine};
+use serde::{Deserialize, Serialize};
 
-// use base64::prelude::*;
-// use serde::{Deserialize, Serialize};
-use crate::backends::plonky2::mainpod::process_private_statements_operations;
 use crate::{
     backends::plonky2::{
         mainpod::{
             extract_merkle_proofs, hash_statements, layout_statements, normalize_statement,
-            process_public_statements_operations, Operation, Statement,
+            process_private_statements_operations, process_public_statements_operations, Operation,
+            Statement,
         },
         primitives::merkletree::MerkleClaimAndProof,
     },
@@ -31,7 +31,7 @@ impl PodProver for MockProver {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct MockMainPod {
     params: Params,
     id: PodId,
@@ -166,14 +166,17 @@ impl MockMainPod {
         })
     }
 
-    // pub fn deserialize(serialized: String) -> Result<Self> {
-    //     let proof = String::from_utf8(BASE64_STANDARD.decode(&serialized)?)
-    //         .map_err(|e| anyhow::anyhow!("Invalid base64 encoding: {}", e))?;
-    //     let pod: MockMainPod = serde_json::from_str(&proof)
-    //         .map_err(|e| anyhow::anyhow!("Failed to parse proof: {}", e))?;
+    // MockMainPods include some internal private state which is necessary
+    // for verification. In non-mock Pods, this state will not be necessary,
+    // as the public statements can be verified using a ZK proof.
+    pub(crate) fn deserialize(serialized: String) -> Result<Self> {
+        let proof = String::from_utf8(BASE64_STANDARD.decode(&serialized)?)
+            .map_err(|e| anyhow::anyhow!("Invalid base64 encoding: {}", e))?;
+        let pod: MockMainPod = serde_json::from_str(&proof)
+            .map_err(|e| anyhow::anyhow!("Failed to parse proof: {}", e))?;
 
-    //     Ok(pod)
-    // }
+        Ok(pod)
+    }
 }
 
 impl Pod for MockMainPod {
@@ -282,8 +285,7 @@ impl Pod for MockMainPod {
     }
 
     fn serialized_proof(&self) -> String {
-        todo!()
-        // BASE64_STANDARD.encode(serde_json::to_string(self).unwrap())
+        BASE64_STANDARD.encode(serde_json::to_string(self).unwrap())
     }
 }
 

src/backends/plonky2/mock/signedpod.rs

@@ -43,15 +43,11 @@ pub struct MockSignedPod {
     kvs: HashMap<Key, Value>,
 }
 
-// impl MockSignedPod {
-//     pub fn deserialize(id: PodId, signature: String, dict: Dictionary) -> Self {
-//         Self {
-//             id,
-//             signature,
-//             dict,
-//         }
-//     }
-// }
+impl MockSignedPod {
+    pub(crate) fn new(id: PodId, signature: String, kvs: HashMap<Key, Value>) -> Self {
+        Self { id, signature, kvs }
+    }
+}
 
 impl Pod for MockSignedPod {
     fn verify(&self) -> Result<()> {

src/backends/plonky2/primitives/merkletree.rs

@@ -4,8 +4,8 @@ use std::{collections::HashMap, fmt, iter::IntoIterator};
 
 use anyhow::{anyhow, Result};
 use plonky2::field::types::Field;
+use serde::{Deserialize, Serialize};
 
-// use serde::{Deserialize, Serialize};
 pub use super::merkletree_circuit::*;
 use crate::middleware::{hash_fields, Hash, RawValue, EMPTY_HASH, EMPTY_VALUE, F};
 
@@ -208,7 +208,7 @@ impl fmt::Display for MerkleTree {
     }
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct MerkleProof {
     // note: currently we don't use the `_existence` field, we would use if we merge the methods
     // `verify` and `verify_nonexistence` into a single one
@@ -260,7 +260,7 @@ impl MerkleProof {
     }
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct MerkleClaimAndProof {
     pub root: Hash,
     pub key: RawValue,

src/frontend/custom.rs

@@ -4,7 +4,6 @@ use std::{collections::HashMap, fmt, hash as h, iter, iter::zip, sync::Arc};
 use anyhow::{anyhow, Result};
 use schemars::JsonSchema;
 
-// use serde::{Deserialize, Serialize};
 use crate::{
     frontend::{AnchoredKey, Statement, StatementArg},
     middleware::{

src/frontend/mod.rs

@@ -5,10 +5,8 @@ use std::{collections::HashMap, convert::From, fmt};
 
 use anyhow::{anyhow, Result};
 use itertools::Itertools;
+use serde::{Deserialize, Serialize};
 
-// use schemars::JsonSchema;
-
-// use serde::{Deserialize, Serialize};
 use crate::middleware::{
     self, check_st_tmpl, hash_str, AnchoredKey, Key, MainPodInputs, NativeOperation,
     NativePredicate, OperationAux, OperationType, Params, PodId, PodProver, PodSigner, Predicate,
@@ -17,8 +15,10 @@ use crate::middleware::{
 
 mod custom;
 mod operation;
+mod serialization;
 pub use custom::*;
 pub use operation::*;
+use serialization::*;
 
 /// This type is just for presentation purposes.
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
@@ -66,8 +66,8 @@ impl SignedPodBuilder {
 
 /// SignedPod is a wrapper on top of backend::SignedPod, which additionally stores the
 /// string<-->hash relation of the keys.
-#[derive(Debug, Clone)]
-// #[serde(try_from = "SignedPodHelper", into = "SignedPodHelper")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(try_from = "SignedPodHelper", into = "SignedPodHelper")]
 pub struct SignedPod {
     pub pod: Box<dyn middleware::Pod>,
     // We store a copy of the key values for quick access
@@ -591,8 +591,8 @@ impl MainPodBuilder {
     }
 }
 
-#[derive(Debug, Clone)]
-// #[serde(try_from = "MainPodHelper", into = "MainPodHelper")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(try_from = "MainPodHelper", into = "MainPodHelper")]
 pub struct MainPod {
     pub pod: Box<dyn middleware::Pod>,
     pub public_statements: Vec<Statement>,

src/frontend/operation.rs

@@ -1,6 +1,5 @@
 use std::fmt;
 
-// use serde::{Deserialize, Serialize};
 use crate::{
     frontend::SignedPod,
     middleware::{AnchoredKey, OperationAux, OperationType, Statement, Value},