TLV PoC

gossip/src/contact_info.rs

@@ -1,6 +1,11 @@
 pub use solana_client::connection_cache::Protocol;
 use {
-    crate::{crds_data::MAX_WALLCLOCK, legacy_contact_info::LegacyContactInfo},
+    crate::{
+        crds_data::MAX_WALLCLOCK,
+        define_tlv_enum,
+        legacy_contact_info::LegacyContactInfo,
+        tlv::{self, TlvRecord},
+    },
     assert_matches::{assert_matches, debug_assert_matches},
     serde::{Deserialize, Deserializer, Serialize},
     solana_pubkey::Pubkey,
@@ -113,8 +118,19 @@ struct SocketEntry {
     offset: u16, // Port offset with respect to the previous entry.
 }
 
-#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
-enum Extension {}
+define_tlv_enum!(
+    /// TLV encoded Extensions in ContactInfo messages
+    ///
+    /// On the wire each record is: [type: u8][len: varint][bytes]
+    /// Extensions with unknown types are skipped by tlv::parse,
+    /// so new types can be added without breaking legacy code,
+    /// and support by all clients is not required.
+    ///
+    /// Always add new TLV records to the end of this enum.
+    /// Never reorder or reuse a type.
+    /// Ensure new type collisions do not happen.
+    pub(crate) enum Extension {}
+);
 
 // As part of deserialization, self.addrs and self.sockets should be cross
 // verified and self.cache needs to be populated. This type serves as a
@@ -132,8 +148,9 @@ struct ContactInfoLite {
     addrs: Vec<IpAddr>,
     #[serde(with = "short_vec")]
     sockets: Vec<SocketEntry>,
+    #[allow(dead_code)]
     #[serde(with = "short_vec")]
-    extensions: Vec<Extension>,
+    extensions: Vec<TlvRecord>,
 }
 
 macro_rules! get_socket {
@@ -220,7 +237,7 @@ impl ContactInfo {
             version: solana_version::Version::default(),
             addrs: Vec::<IpAddr>::default(),
             sockets: Vec::<SocketEntry>::default(),
-            extensions: Vec::<Extension>::default(),
+            extensions: Vec::default(),
             cache: EMPTY_SOCKET_ADDR_CACHE,
         }
     }
@@ -549,7 +566,7 @@ impl TryFrom<ContactInfoLite> for ContactInfo {
             version,
             addrs,
             sockets,
-            extensions,
+            extensions: tlv::parse(&extensions),
             cache: EMPTY_SOCKET_ADDR_CACHE,
         };
         // Populate node.cache.

gossip/src/lib.rs

@@ -22,6 +22,8 @@ pub mod epoch_specs;
 pub mod gossip_error;
 pub mod gossip_service;
 #[macro_use]
+mod tlv;
+#[macro_use]
 mod legacy_contact_info;
 pub mod ping_pong;
 mod protocol;

gossip/src/tlv.rs

@@ -0,0 +1,159 @@
+use {
+    serde::{Deserialize, Serialize},
+    solana_short_vec as short_vec,
+};
+
+/// Type-Length-Value encoding wrapper for bincode
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+pub(crate) struct TlvRecord {
+    // type
+    pub(crate) typ: u8,
+    // length and serialized bytes of the value
+    #[serde(with = "short_vec")]
+    pub(crate) bytes: Vec<u8>,
+}
+
+/// Macro that provides a quick and easy way to define TLV compatible enums
+#[macro_export]
+macro_rules! define_tlv_enum {
+    (
+        $(#[$meta:meta])*
+        $vis:vis enum $enum_name:ident {
+            $($typ:literal => $variant:ident($inner:ty)),* $(,)?
+        }
+    ) => {
+        // add the doc-comment if present
+        $(#[$meta])*
+        // define the enum itself
+        #[derive(Debug, Clone, Eq, PartialEq)]
+        $vis enum $enum_name {
+            $(
+                $variant($inner),
+            )*
+        }
+
+        // Serialize enum by first converting into TlvRecord, and then serializing that
+        impl serde::Serialize for $enum_name {
+            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+            where
+                S: serde::Serializer,
+            {
+                let tlv_rec = TlvRecord::try_from(self).map_err(|e| serde::ser::Error::custom(e))?;
+                tlv_rec.serialize(serializer)
+            }
+        }
+
+        // define conversion from TLV wire format
+        impl TryFrom<&TlvRecord> for $enum_name {
+            type Error = bincode::Error;
+            fn try_from(value: &TlvRecord) -> Result<Self, Self::Error> {
+                use serde::de::Error;
+                match value.typ {
+                    $(
+                        $typ => Ok(Self::$variant(bincode::deserialize::<$inner>(&value.bytes)?)),
+                    )*
+                    _ => Err(bincode::Error::custom("Invalid type")),
+                }
+            }
+        }
+        // define conversion into TLV wire format
+        impl TryFrom<&$enum_name> for TlvRecord {
+            type Error = bincode::Error;
+            fn try_from(value: &$enum_name) -> Result<Self, Self::Error> {
+                use serde::ser::Error;
+                match value {
+                    $(
+                        $enum_name::$variant(inner) => Ok(TlvRecord {
+                            typ: $typ,
+                            bytes: bincode::serialize(inner)?,
+                        }),
+                    )*
+                    #[allow(unreachable_patterns)]
+                    _ => Err(bincode::Error::custom("Unsupported enum variant")),
+                }
+            }
+        }
+    };
+}
+
+/// Parses a slice of serialized TLV records into a provided type. Unsupported
+/// TLV records are ignored.
+pub(crate) fn parse<'a, T: TryFrom<&'a TlvRecord>>(entries: &'a [TlvRecord]) -> Vec<T> {
+    entries.iter().filter_map(|v| T::try_from(v).ok()).collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{define_tlv_enum, tlv::TlvRecord};
+
+    define_tlv_enum! (pub(crate) enum ExtensionNew {
+        1=>Test(u64),
+        2=>LegacyString(String),
+        3=>NewString(String),
+    });
+
+    define_tlv_enum! ( pub(crate) enum ExtensionLegacy {
+        1=>Test(u64),
+        2=>LegacyString(String),
+    });
+
+    /// Test that TLV encoded data is backwards-compatible,
+    /// i.e. that new TLV data can be decoded by a new
+    /// receiver where possible, and skipped otherwise
+    #[test]
+    fn test_tlv_backwards_compat() {
+        let new_tlv_data = vec![
+            ExtensionNew::Test(42),
+            ExtensionNew::NewString(String::from("bla")),
+        ];
+
+        let new_bytes = bincode::serialize(&new_tlv_data).unwrap();
+        let tlv_vec: Vec<TlvRecord> = bincode::deserialize(&new_bytes).unwrap();
+        // check that both TLV are encoded correctly
+        let new: Vec<ExtensionNew> = crate::tlv::parse(&tlv_vec);
+        assert!(matches!(new[0], ExtensionNew::Test(42)));
+        if let ExtensionNew::NewString(s) = &new[1] {
+            assert_eq!(s, "bla");
+        } else {
+            panic!("Wrong deserialization")
+        };
+        // Make sure legacy recover works correctly
+        let legacy: Vec<ExtensionLegacy> = crate::tlv::parse(&tlv_vec);
+        assert!(matches!(legacy[0], ExtensionLegacy::Test(42)));
+        assert_eq!(
+            legacy.len(),
+            1,
+            "Legacy parser should only recover  1 entry"
+        )
+    }
+
+    /// Test that TLV encoded data is forwards-compatible,
+    /// i.e. that legacy TLV data can be decoded by a new
+    /// receiver
+    #[test]
+    fn test_tlv_forward_compat() {
+        let legacy_tlv_data = vec![
+            ExtensionLegacy::Test(42),
+            ExtensionLegacy::LegacyString(String::from("foo")),
+        ];
+        let legacy_bytes = bincode::serialize(&legacy_tlv_data).unwrap();
+
+        let tlv_vec: Vec<TlvRecord> = bincode::deserialize(&legacy_bytes).unwrap();
+        // Just in case make sure that legacy data is serialized correctly
+        let legacy: Vec<ExtensionLegacy> = crate::tlv::parse(&tlv_vec);
+        assert!(matches!(legacy[0], ExtensionLegacy::Test(42)));
+        if let ExtensionLegacy::LegacyString(s) = &legacy[1] {
+            assert_eq!(s, "foo");
+        } else {
+            panic!("Wrong deserialization")
+        };
+        // Parse the same bytes using new parser
+        let new: Vec<ExtensionNew> = crate::tlv::parse(&tlv_vec);
+        assert!(matches!(new[0], ExtensionNew::Test(42)));
+        if let ExtensionNew::LegacyString(s) = &new[1] {
+            assert_eq!(s, "foo");
+        } else {
+            panic!("Wrong deserialization")
+        };
+    }
+}