remove legacy shreds from retransmit_stage.rs (#6025)

* remove legacy shreds from retransmit_stage.rs
* expand test cases to cover more niche scenarios
Thanks Brennan for help!

Author: alexpyattaev

turbine/src/retransmit_stage.rs

@@ -829,8 +829,9 @@ mod tests {
         super::*,
         rand::SeedableRng,
         rand_chacha::ChaChaRng,
-        solana_ledger::shred::{Shred, ShredFlags},
-        solana_sdk::signature::Keypair,
+        solana_entry::entry::create_ticks,
+        solana_ledger::shred::{ProcessShredsStats, ReedSolomonCache, Shredder},
+        solana_sdk::{hash::Hash, signature::Keypair},
     };
 
     fn get_keypair() -> Keypair {
@@ -845,74 +846,109 @@ mod tests {
     }
 
     #[test]
-    fn test_already_received() {
-        let slot = 1;
-        let index = 5;
-        let version = 0x40;
+    fn test_shred_deduper() {
         let keypair = get_keypair();
-        let mut shred = Shred::new_from_data(
-            slot,
-            index,
-            0,
-            &[],
-            ShredFlags::LAST_SHRED_IN_SLOT,
-            0,
-            version,
-            0,
-        );
-        shred.sign(&keypair);
+        let entries = create_ticks(10, 1, Hash::new_unique());
+        let rsc = ReedSolomonCache::default();
+        let make_shreds_for_slot = |slot, parent, code_index| {
+            let shredder = Shredder::new(slot, parent, 1, 0).unwrap();
+            shredder.entries_to_shreds(
+                &keypair,
+                &entries,
+                true,
+                // chained_merkle_root
+                Some(Hash::new_from_array(rand::thread_rng().gen())),
+                0,
+                code_index,
+                true,
+                &rsc,
+                &mut ProcessShredsStats::default(),
+            )
+        };
+
         let mut rng = ChaChaRng::from_seed([0xa5; 32]);
         let shred_deduper = ShredDeduper::<2>::new(&mut rng, /*num_bits:*/ 640_007);
-        // unique shred for (1, 5) should pass
-        assert!(!shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
-        // duplicate shred for (1, 5) blocked
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
-        let mut shred = Shred::new_from_data(
-            slot,
-            index,
-            2,
-            &[],
-            ShredFlags::LAST_SHRED_IN_SLOT,
-            0,
-            version,
-            0,
+
+        // make a set of shreds for slot 5 with parent slot 4
+        let (shreds_data_5_4, shreds_code_5_4) = make_shreds_for_slot(5, 4, 0);
+        // make a set of shreds for slot 5 with parent slot 3
+        let (shreds_data_5_3, _shreds_code_5_3) = make_shreds_for_slot(5, 3, 0);
+        // make a set of shreds for slot 5 with parent slot 2
+        let (shreds_data_5_2, _shreds_code_5_2) = make_shreds_for_slot(5, 2, 0);
+        // pick a shred for tests
+        let shred = shreds_data_5_4.last().unwrap().clone();
+        // unique shred should pass
+        assert!(
+            !shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT),
+            "First time shred X => Not dup because it is the only shred"
+        );
+        // duplicate shred blocked
+        assert!(
+            shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT),
+            "
+            Second time shred X => Dup because common header is duplicate
+            "
+        );
+        // Pick a shred with same index as `shred` but different parent offset
+        let shred_dup = shreds_data_5_3.last().unwrap().clone();
+        // first shred passed through
+        assert!(!shred_deduper.dedup(shred_dup.id(), shred_dup.payload(), MAX_DUPLICATE_COUNT),
+        "First time seeing shred X with differnt parent slot (3 instead of 4) => Not dup because common header is unique & shred ID only seen once"
         );
-        shred.sign(&keypair);
-        // first duplicate shred for (1, 5) passed
-        assert!(!shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
         // then blocked
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
-
-        let mut shred = Shred::new_from_data(
-            slot,
-            index,
-            8,
-            &[],
-            ShredFlags::LAST_SHRED_IN_SLOT,
-            0,
-            version,
-            0,
+        assert!(shred_deduper.dedup(shred_dup.id(), shred_dup.payload(), MAX_DUPLICATE_COUNT),
+        "Second time seeing shred X with parent slot 3 => Dup because common header is not unique & shred ID seen twice"
+        );
+
+        let shred_dup2 = shreds_data_5_2.last().unwrap().clone();
+
+        assert!(shred_deduper.dedup(shred_dup2.id(), shred_dup2.payload(), MAX_DUPLICATE_COUNT),
+            "First time seeing shred X with parent slot 2 => Dup because common header is unique but shred ID seen twice already"
+        );
+
+        /* Coding shreds */
+
+        // Pick a coding shred at index 4 based off FEC set index 0
+        let shred = shreds_code_5_4[4].clone();
+        // Coding passes
+        assert!(
+            !shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT),
+            "
+           First time seeing coding shred Y => Not dup because common header & shred ID are unique"
         );
-        shred.sign(&keypair);
-        // 2nd duplicate shred for (1, 5) blocked
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
-
-        let shred = Shred::new_from_parity_shard(slot, index, &[], 0, 1, 1, 0, version);
-        // Coding at (1, 5) passes
-        assert!(!shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
         // then blocked
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
+        assert!(
+            shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT),
+            "
+            Second time seeing coding shred Y => Dup because common header is dup
+            "
+        );
 
-        let shred = Shred::new_from_parity_shard(slot, index, &[], 2, 1, 1, 0, version);
-        // 2nd unique coding at (1, 5) passes
-        assert!(!shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
-        // same again is blocked
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
+        // Make a coding shred at index 4 based off FEC set index 2
+        let (_, shreds_code_invalid) = make_shreds_for_slot(5, 4, 2);
 
-        let shred = Shred::new_from_parity_shard(slot, index, &[], 3, 1, 1, 0, version);
-        // Another unique coding at (1, 5) always blocked
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
-        assert!(shred_deduper.dedup(shred.id(), shred.payload(), MAX_DUPLICATE_COUNT));
+        let shred_inv_code_1 = shreds_code_invalid[2].clone();
+        assert_eq!(
+            shred.index(),
+            shred_inv_code_1.index(),
+            "we want a shred with same index but different FEC set index"
+        );
+        // 2nd unique coding passes
+        assert!(!shred_deduper.dedup(shred_inv_code_1.id(), shred_inv_code_1.payload(), MAX_DUPLICATE_COUNT),
+            "First time seeing shred Y w/ changed header (FEC Set index 2) => Not dup because common header is unique & shred ID only seen once");
+        // same again is blocked
+        assert!(shred_deduper.dedup(shred_inv_code_1.id(), shred_inv_code_1.payload(), MAX_DUPLICATE_COUNT),"
+           Second time seeing shred Y w/ changed header (FEC Set index 2) => Dup because common header is not unique & shred ID seen twice ");
+        // Make a coding shred at index 4 based off FEC set index 3
+        let (_, shreds_code_invalid) = make_shreds_for_slot(5, 4, 3);
+
+        let shred_inv_code_2 = shreds_code_invalid[1].clone();
+        assert_eq!(
+            shred.index(),
+            shred_inv_code_2.index(),
+            "we want a shred with same index but different FEC set index"
+        );
+        assert!(shred_deduper.dedup(shred_inv_code_2.id(), shred_inv_code_2.payload(), MAX_DUPLICATE_COUNT),"
+           First time seeing shred Y w/ changed header (FEC Set index 3)=>Dup because common header is unique but shred ID seen twice already");
     }
 }