[consensus/coding] Prevent Proposer Withholding (#3338)

Author: clabby

coding/Cargo.toml

@@ -60,3 +60,13 @@ path = "src/benches/bench.rs"
 name = "coding_scheme_sizes"
 harness = false
 path = "src/benches/bench_size.rs"
+
+[[bench]]
+name = "phased_coding_scheme_times"
+harness = false
+path = "src/benches/bench_phased.rs"
+
+[[bench]]
+name = "phased_coding_scheme_sizes"
+harness = false
+path = "src/benches/bench_phased_size.rs"

coding/conformance.toml

@@ -1,11 +1,3 @@
-["commonware_coding::no_coding::conformance::CodecConformance<StrongShard>"]
-n_cases = 65536
-hash = "edc22446bb2952609d0c8daccf2d22f8ad2b71eedfcf45296c3f4db49d78404a"
-
-["commonware_coding::no_coding::conformance::CodecConformance<WeakShard>"]
-n_cases = 65536
-hash = "07854d2fef297a06ba81685e660c332de36d5d18d546927d30daad6d7fda1541"
-
 ["commonware_coding::reed_solomon::tests::conformance::CodecConformance<Chunk<Sha256Digest>>"]
 n_cases = 65536
 hash = "45545e4d4aeb18b8bdb019e630fb9a1fa6dda9ed32b2d529a9213ec07ccab07c"

coding/fuzz/Cargo.toml

@@ -29,13 +29,6 @@ test = false
 doc = false
 bench = false
 
-[[bin]]
-name = "no_coding"
-path = "fuzz_targets/no_coding.rs"
-test = false
-doc = false
-bench = false
-
 [[bin]]
 name = "zoda"
 path = "fuzz_targets/zoda.rs"

coding/fuzz/fuzz_targets/no_coding.rs

@@ -1,10 +1,10 @@
 #![no_main]
 
 use commonware_coding::Zoda;
-use commonware_coding_fuzz::{fuzz, FuzzInput};
+use commonware_coding_fuzz::{fuzz_phased, FuzzInput};
 use commonware_cryptography::Sha256;
 use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|input: FuzzInput| {
-    fuzz::<Zoda<Sha256>>(input);
+    fuzz_phased::<Zoda<Sha256>>(input);
 });

coding/fuzz/fuzz_targets/zoda.rs

@@ -1,5 +1,5 @@
 use arbitrary::{Arbitrary, Unstructured};
-use commonware_coding::{Config, Scheme};
+use commonware_coding::{Config, PhasedScheme, Scheme};
 use commonware_parallel::Sequential;
 use commonware_utils::NZU16;
 use std::iter;
@@ -74,16 +74,45 @@ pub fn fuzz<S: Scheme>(input: FuzzInput) {
     };
     let (commitment, shards) = S::encode(&config, data.as_slice(), &STRATEGY).unwrap();
     assert_eq!(shards.len(), (recovery + min) as usize);
-    // We don't use enumerate to get u16s.
     let mut shards = (0u16..).zip(shards).collect::<Vec<_>>();
     shuffle.shuffle(&mut shards);
+
+    // Check and collect `to_use` shards.
+    let checked_shards = shards
+        .into_iter()
+        .take(to_use as usize)
+        .map(|(i, shard)| S::check(&config, &commitment, i, &shard).unwrap())
+        .collect::<Vec<_>>();
+
+    let decoded = S::decode(&config, &commitment, checked_shards.iter(), &STRATEGY).unwrap();
+    assert_eq!(&decoded, &data);
+}
+
+pub fn fuzz_phased<S: PhasedScheme>(input: FuzzInput) {
+    let FuzzInput {
+        recovery,
+        min,
+        to_use,
+        data,
+        shuffle,
+    } = input;
+
+    let config = Config {
+        minimum_shards: NZU16!(min),
+        extra_shards: NZU16!(recovery),
+    };
+    let (commitment, shards) = S::encode(&config, data.as_slice(), &STRATEGY).unwrap();
+    assert_eq!(shards.len(), (recovery + min) as usize);
+    let mut shards = (0u16..).zip(shards).collect::<Vec<_>>();
+    shuffle.shuffle(&mut shards);
+
     // From here on, we take the point of view of the last participant.
-    // (This is so that we can move their shard out of the vector easily).
+    // This lets us move our strong shard out directly while keeping the rest for forwarding.
     let (my_i, my_shard) = shards.pop().unwrap();
     let (my_checking_data, my_checked_shard, _) =
         S::weaken(&config, &commitment, my_i, my_shard).unwrap();
 
-    // Check to_use - 1 shards, then include our own checked shards.
+    // Check `to_use - 1` forwarded shards, then include our own checked shard.
     let checked_shards = shards
         .into_iter()
         .take((to_use - 1) as usize)
@@ -98,7 +127,7 @@ pub fn fuzz<S: Scheme>(input: FuzzInput) {
         &config,
         &commitment,
         my_checking_data,
-        &checked_shards,
+        checked_shards.iter(),
         &STRATEGY,
     )
     .unwrap();

coding/fuzz/src/lib.rs

@@ -4,9 +4,10 @@ use commonware_utils::{NZUsize, NZU16};
 use criterion::{criterion_main, BatchSize, Criterion};
 use rand::{RngCore, SeedableRng as _};
 use rand_chacha::ChaCha8Rng;
+use shard_selection::SELECTIONS;
 
-mod no_coding;
 mod reed_solomon;
+mod shard_selection;
 mod zoda;
 
 pub(crate) fn bench_encode_generic<S: Scheme>(name: &str, c: &mut Criterion) {
@@ -48,52 +49,6 @@ pub(crate) fn bench_encode_generic<S: Scheme>(name: &str, c: &mut Criterion) {
     }
 }
 
-#[derive(Clone, Copy)]
-pub(crate) enum ShardSelection {
-    Best,
-    Exception,
-    Worst,
-    Interleaved,
-}
-
-impl ShardSelection {
-    const fn label(self) -> &'static str {
-        match self {
-            Self::Best => "best",
-            Self::Exception => "exception",
-            Self::Worst => "worst",
-            Self::Interleaved => "interleaved",
-        }
-    }
-
-    fn indices(self, min: u16) -> Vec<u16> {
-        match self {
-            Self::Best => (0..min).collect(),
-            Self::Exception => (1..=min).collect(),
-            Self::Worst => (min..min + min).collect(),
-            Self::Interleaved => (0..min)
-                .map(|i| {
-                    let k = i / 2;
-                    // Alternate between original shard indices [0, min) and
-                    // recovery shard indices [min, 2 * min).
-                    if i % 2 == 0 {
-                        k
-                    } else {
-                        min + k
-                    }
-                })
-                .collect(),
-        }
-    }
-}
-
-const SELECTIONS: [ShardSelection; 4] = [
-    ShardSelection::Best,
-    ShardSelection::Exception,
-    ShardSelection::Worst,
-    ShardSelection::Interleaved,
-];
-
 pub(crate) fn bench_decode_generic<S: Scheme>(name: &str, c: &mut Criterion) {
     let mut rng = ChaCha8Rng::seed_from_u64(0);
     let cases = [20, 22, 23].map(|i| 2usize.pow(i));
@@ -120,74 +75,44 @@ pub(crate) fn bench_decode_generic<S: Scheme>(name: &str, c: &mut Criterion) {
                                     rng.fill_bytes(&mut data);
 
                                     // Encode data
-                                    let (commitment, mut shards) = if conc > 1 {
+                                    let (commitment, shards) = if conc > 1 {
                                         S::encode(&config, data.as_slice(), &strategy).unwrap()
                                     } else {
                                         S::encode(&config, data.as_slice(), &Sequential).unwrap()
                                     };
 
-                                    // Get my shard
-                                    let my_shard = shards.pop().unwrap();
                                     let indices = selection.indices(min);
-                                    let mut opt_shards: Vec<Option<_>> =
-                                        shards.into_iter().map(Some).collect();
-                                    let weak_shards: Vec<(u16, _)> = indices
+                                    let selected_shards: Vec<(u16, _)> = indices
                                         .iter()
                                         .map(|&i| {
-                                            let shard =
-                                                opt_shards[i as usize].take().unwrap();
-                                            let (_, _, weak_shard) =
-                                                S::weaken(&config, &commitment, i, shard)
-                                                    .unwrap();
-                                            (i, weak_shard)
+                                            (i, shards[i as usize].clone())
                                         })
                                         .collect();
 
-                                    (commitment, my_shard, weak_shards)
-                                },
-                                |(commitment, my_shard, weak_shards)| {
-                                    // Weaken my shard
-                                    let (checking_data, _, _) = S::weaken(
-                                        &config,
-                                        &commitment,
-                                        config.minimum_shards.get()
-                                            + config.extra_shards.get()
-                                            - 1,
-                                        my_shard,
-                                    )
-                                    .unwrap();
-
-                                    // Check shards
-                                    let checked_shards = weak_shards
-                                        .into_iter()
-                                        .map(|(idx, weak_shard)| {
-                                            S::check(
-                                                &config,
-                                                &commitment,
-                                                &checking_data,
-                                                idx,
-                                                weak_shard,
-                                            )
-                                            .unwrap()
+                                    let checked_shards: Vec<_> = selected_shards
+                                        .iter()
+                                        .map(|(idx, shard)| {
+                                            S::check(&config, &commitment, *idx, shard).unwrap()
                                         })
-                                        .collect::<Vec<_>>();
+                                        .collect();
 
+                                    (commitment, checked_shards)
+                                },
+                                |(commitment, checked_shards)| {
                                     // Decode data
                                     if conc > 1 {
                                         S::decode(
                                             &config,
                                             &commitment,
-                                            checking_data,
-                                            &checked_shards,
+                                            checked_shards.iter(),
                                             &strategy,
                                         )
-                                        .unwrap()
+                                            .unwrap()
                                     } else {
                                         S::decode(
                                             &config,
                                             &commitment,
-                                            checking_data,
-                                            &checked_shards,
+                                            checked_shards.iter(),
                                             &Sequential,
                                         )
                                         .unwrap()
@@ -203,4 +128,4 @@ pub(crate) fn bench_decode_generic<S: Scheme>(name: &str, c: &mut Criterion) {
     }
 }
 
-criterion_main!(reed_solomon::benches, no_coding::benches, zoda::benches);
+criterion_main!(reed_solomon::benches, zoda::benches);