feat: Streaming serialize / deserialize (WIP) (#20072)

Public dispatch performance was incredibly bad due to
serialize/deserialize inefficiencies. This PR switches to a streaming
(reader/writer) pattern to improve performance.
Thanks @Thunkar as macro wizard for help navigating comptime problems

Author: AztecBot

noir-projects/aztec-nr/aztec/src/macros/dispatch.nr

@@ -52,7 +52,7 @@ pub comptime fn generate_public_dispatch(m: Module) -> Quoted {
             let param_name = f"arg{parameter_index_value}".quoted_contents();
             let param_type = param.1;
             let read = quote {
-                let $param_name: $param_type = reader.read_struct(aztec::protocol::traits::Deserialize::deserialize);
+                let $param_name: $param_type = aztec::protocol::traits::Deserialize::stream_deserialize(&mut reader);
             };
             *parameter_index += 1;
             quote { $read }

noir-projects/aztec-nr/aztec/src/test/mocks/mock_struct.nr

@@ -2,6 +2,7 @@
 
 use crate::protocol::traits::{Deserialize, Empty, Packable, Serialize};
 
+#[derive(Deserialize, Serialize)]
 pub struct MockStruct {
     pub a: Field,
     pub b: Field,
@@ -25,22 +26,6 @@ impl Empty for MockStruct {
     }
 }
 
-impl Serialize for MockStruct {
-    let N: u32 = 2;
-
-    fn serialize(self) -> [Field; Self::N] {
-        [self.a, self.b]
-    }
-}
-
-impl Deserialize for MockStruct {
-    let N: u32 = 2;
-
-    fn deserialize(fields: [Field; Self::N]) -> Self {
-        Self { a: fields[0], b: fields[1] }
-    }
-}
-
 impl Packable for MockStruct {
     let N: u32 = 2;
 

noir-projects/noir-protocol-circuits/Nargo.template.toml

@@ -46,4 +46,5 @@ members = [
     "crates/rollup-checkpoint-padding",
     "crates/rollup-checkpoint-merge",
     "crates/rollup-root",
+    "crates/serde"
 ]

noir-projects/noir-protocol-circuits/crates/blob/src/abis/blob_accumulator.nr

@@ -10,7 +10,7 @@ use types::{
     constants::BLOB_ACCUMULATOR_LENGTH,
     hash::{poseidon2_hash, sha256_to_field},
     traits::{Deserialize, Empty, Serialize},
-    utils::reader::Reader,
+    utils::{reader::Reader, writer::Writer},
 };
 
 /**
@@ -180,17 +180,22 @@ impl Eq for BlobAccumulator {
 impl Serialize for BlobAccumulator {
     let N: u32 = BLOB_ACCUMULATOR_LENGTH;
 
-    fn serialize(self) -> [Field; BLOB_ACCUMULATOR_LENGTH] {
-        let mut fields: BoundedVec<Field, BLOB_ACCUMULATOR_LENGTH> = BoundedVec::new();
-        fields.push(self.blob_commitments_hash_acc);
-        fields.push(self.z_acc);
-        fields.extend_from_array(self.y_acc.get_limbs().map(|l| l as Field));
-        fields.extend_from_array(self.c_acc.x.get_limbs().map(|l| l as Field));
-        fields.extend_from_array(self.c_acc.y.get_limbs().map(|l| l as Field));
-        fields.push(self.c_acc.is_infinity as Field);
-        fields.push(self.gamma_acc);
-        fields.extend_from_array(self.gamma_pow_acc.get_limbs().map(|l| l as Field));
-        fields.storage()
+    fn serialize(self) -> [Field; Self::N] {
+        let mut writer: Writer<Self::N> = Writer::new();
+        self.stream_serialize(&mut writer);
+        writer.finish()
+    }
+
+    #[inline_always]
+    fn stream_serialize<let K: u32>(self, writer: &mut Writer<K>) {
+        writer.write(self.blob_commitments_hash_acc);
+        writer.write(self.z_acc);
+        writer.write_array(self.y_acc.get_limbs().map(|l| l as Field));
+        writer.write_array(self.c_acc.x.get_limbs().map(|l| l as Field));
+        writer.write_array(self.c_acc.y.get_limbs().map(|l| l as Field));
+        writer.write(self.c_acc.is_infinity as Field);
+        writer.write(self.gamma_acc);
+        writer.write_array(self.gamma_pow_acc.get_limbs().map(|l| l as Field));
     }
 }
 
@@ -201,7 +206,14 @@ impl Deserialize for BlobAccumulator {
 
     fn deserialize(fields: [Field; Self::N]) -> Self {
         let mut reader = Reader::new(fields);
-        let mut item = Self {
+        let result = Self::stream_deserialize(&mut reader);
+        reader.finish();
+        result
+    }
+
+    #[inline_always]
+    fn stream_deserialize<let K: u32>(reader: &mut Reader<K>) -> Self {
+        Self {
             blob_commitments_hash_acc: reader.read(),
             z_acc: reader.read(),
             y_acc: BLS12_381_Fr::from_limbs(reader.read_array().map(|e| e as u128)),
@@ -212,7 +224,26 @@ impl Deserialize for BlobAccumulator {
             },
             gamma_acc: reader.read(),
             gamma_pow_acc: BLS12_381_Fr::from_limbs(reader.read_array().map(|e| e as u128)),
-        };
-        item
+        }
     }
 }
+
+#[test]
+fn serde_test() {
+    let blob_accumulator = BlobAccumulator {
+        blob_commitments_hash_acc: 1,
+        z_acc: 2,
+        y_acc: BLS12_381_Fr::from_limbs([3, 4, 5]),
+        c_acc: BLSPoint {
+            x: BLS12_381_Fq::from_limbs([6, 7, 8, 0]),
+            y: BLS12_381_Fq::from_limbs([10, 11, 12, 13]),
+            is_infinity: false,
+        },
+        gamma_acc: 42,
+        gamma_pow_acc: BLS12_381_Fr::from_limbs([14, 15, 16]),
+    };
+
+    let serialized = blob_accumulator.serialize();
+    let deserialized = BlobAccumulator::deserialize(serialized);
+    assert_eq(blob_accumulator, deserialized);
+}

noir-projects/noir-protocol-circuits/crates/blob/src/abis/final_blob_batching_challenges.nr

@@ -2,7 +2,7 @@ use bignum::{BigNum, BLS12_381_Fr};
 use types::{
     constants::BLS12_FR_LIMBS,
     traits::{Deserialize, Empty, Serialize},
-    utils::reader::Reader,
+    utils::{reader::Reader, writer::Writer},
 };
 
 /**
@@ -41,10 +41,15 @@ impl Serialize for FinalBlobBatchingChallenges {
     let N: u32 = BLS12_FR_LIMBS + 1;
 
     fn serialize(self) -> [Field; Self::N] {
-        let mut fields: BoundedVec<Field, BLS12_FR_LIMBS + 1> = BoundedVec::new();
-        fields.push(self.z);
-        fields.extend_from_array(self.gamma.get_limbs().map(|l| l as Field));
-        fields.storage()
+        let mut writer: Writer<Self::N> = Writer::new();
+        self.stream_serialize(&mut writer);
+        writer.finish()
+    }
+
+    #[inline_always]
+    fn stream_serialize<let K: u32>(self, writer: &mut Writer<K>) {
+        writer.write(self.z);
+        writer.write_array(self.gamma.get_limbs().map(|l| l as Field));
     }
 }
 
@@ -53,10 +58,25 @@ impl Deserialize for FinalBlobBatchingChallenges {
 
     fn deserialize(fields: [Field; Self::N]) -> Self {
         let mut reader = Reader::new(fields);
-        let mut item = Self {
+        let result = Self::stream_deserialize(&mut reader);
+        reader.finish();
+        result
+    }
+
+    #[inline_always]
+    fn stream_deserialize<let K: u32>(reader: &mut Reader<K>) -> Self {
+        Self {
             z: reader.read(),
             gamma: BLS12_381_Fr::from_limbs(reader.read_array().map(|e| e as u128)),
-        };
-        item
+        }
     }
 }
+
+#[test]
+fn serde_test() {
+    let final_blob_batching_challenges =
+        FinalBlobBatchingChallenges { z: 1, gamma: BLS12_381_Fr::from_limbs([2, 3, 4]) };
+    let serialized = final_blob_batching_challenges.serialize();
+    let deserialized = FinalBlobBatchingChallenges::deserialize(serialized);
+    assert_eq(final_blob_batching_challenges, deserialized);
+}

noir-projects/noir-protocol-circuits/crates/serde/Nargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "serde"
+type = "lib"
+authors = [""]
+compiler_version = ">=0.18.0"
+
+[dependencies]

noir-projects/noir-protocol-circuits/crates/serde/src/lib.nr

@@ -0,0 +1,5 @@
+pub mod reader;
+pub mod writer;
+pub mod serialization;
+pub mod type_impls;
+mod tests;

…ircuits/crates/types/src/utils/reader.nr …ocol-circuits/crates/serde/src/reader.nrnoir-projects/noir-protocol-circuits/crates/types/src/utils/reader.nr renamed to noir-projects/noir-protocol-circuits/crates/serde/src/reader.nr noir-projects/noir-protocol-circuits/crates/types/src/utils/reader.nr renamed to noir-projects/noir-protocol-circuits/crates/serde/src/reader.nr

@@ -51,6 +51,14 @@ impl<let N: u32> Reader<N> {
         result
     }
 
+    pub fn peek_offset(&mut self, offset: u32) -> Field {
+        self.data[self.offset + offset]
+    }
+
+    pub fn advance_offset(&mut self, offset: u32) {
+        self.offset += offset;
+    }
+
     pub fn finish(self) {
         assert_eq(self.offset, self.data.len(), "Reader did not read all data");
     }