diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 4ab7f913595..5abd9e7360d 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -144,6 +144,7 @@ pub async fn create_api_server( let enr = EnrBuilder::new("v4").build(&enr_key).unwrap(); let network_globals = Arc::new(NetworkGlobals::new( enr.clone(), + enr_key, meta_data, vec![], false, diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 388790568f0..fbfc624f130 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -8,6 +8,7 @@ pub mod enr_ext; // Allow external use of the lighthouse ENR builder use crate::service::TARGET_SUBNET_PEERS; +use crate::types::mutable_enr::EnrPort; use crate::{error, Enr, NetworkConfig, NetworkGlobals, Subnet, SubnetDiscovery}; use crate::{metrics, ClearDialError}; use discv5::{enr::NodeId, Discv5, Discv5Event}; @@ -37,7 +38,7 @@ use slog::{crit, debug, error, info, trace, warn}; use ssz::Encode; use std::{ collections::{HashMap, VecDeque}, - net::{IpAddr, SocketAddr}, + net::IpAddr, path::Path, pin::Pin, sync::Arc, @@ -48,6 +49,7 @@ use tokio::sync::mpsc; use types::{EnrForkId, EthSpec}; mod subnet_predicate; +use super::types::mutable_enr::MutableEnr; pub use subnet_predicate::subnet_predicate; /// Local ENR storage filename. @@ -172,7 +174,7 @@ pub struct Discovery { discv5: Discv5, /// A collection of network constants that can be read from other threads. - network_globals: Arc>, + pub network_globals: Arc>, /// Indicates if we are actively searching for peers. We only allow a single FindPeers query at /// a time, regardless of the query concurrency. @@ -201,7 +203,6 @@ pub struct Discovery { impl Discovery { /// NOTE: Creating discovery requires running within a tokio execution environment. pub async fn new( - local_key: Keypair, config: &NetworkConfig, network_globals: Arc>, log: &slog::Logger, @@ -213,19 +214,20 @@ impl Discovery { None => String::from(""), }; - let local_enr = network_globals.local_enr.read().clone(); - let local_node_id = local_enr.node_id(); + let local_enr: MutableEnr = network_globals.local_enr.clone(); + let local_node_id = local_enr.enr().node_id(); - info!(log, "ENR Initialised"; "enr" => local_enr.to_base64(), "seq" => local_enr.seq(), "id"=> %local_enr.node_id(), - "ip4" => ?local_enr.ip4(), "udp4"=> ?local_enr.udp4(), "tcp4" => ?local_enr.tcp4(), "tcp6" => ?local_enr.tcp6(), "udp6" => ?local_enr.udp6(), - "quic4" => ?local_enr.quic4(), "quic6" => ?local_enr.quic6() + info!(log, "ENR Initialised"; "enr" => local_enr.enr().to_base64(), "seq" => local_enr.enr().seq(), "id"=> %local_enr.enr().node_id(), + "ip4" => ?local_enr.enr().ip4(), "udp4"=> ?local_enr.enr().udp4(), "tcp4" => ?local_enr.enr().tcp4(), "tcp6" => ?local_enr.enr().tcp6(), "udp6" => ?local_enr.enr().udp6(), + "quic4" => ?local_enr.enr().quic4(), "quic6" => ?local_enr.enr().quic6() ); - // convert the keypair into an ENR key - let enr_key: CombinedKey = CombinedKey::from_libp2p(local_key)?; - - let mut discv5 = Discv5::new(local_enr, enr_key, config.discv5_config.clone()) - .map_err(|e| format!("Discv5 service failed. Error: {:?}", e))?; + let mut discv5 = Discv5::new( + local_enr.enr().clone(), + local_enr.enr_key(), + config.discv5_config.clone(), + ) + .map_err(|e| format!("Discv5 service failed. Error: {:?}", e))?; // Add bootnodes to routing table for bootnode_enr in config.boot_nodes_enr.clone() { @@ -397,54 +399,6 @@ impl Discovery { self.discv5.table_entries_enr() } - /// Updates the local ENR TCP port. - /// There currently isn't a case to update the address here. We opt for discovery to - /// automatically update the external address. - /// - /// If the external address needs to be modified, use `update_enr_udp_socket. - pub fn update_enr_tcp_port(&mut self, port: u16) -> Result<(), String> { - self.discv5 - .enr_insert("tcp", &port) - .map_err(|e| format!("{:?}", e))?; - - // replace the global version - *self.network_globals.local_enr.write() = self.discv5.local_enr(); - // persist modified enr to disk - enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); - Ok(()) - } - - // TODO: Group these functions here once the ENR is shared across discv5 and lighthouse and - // Lighthouse can modify the ENR directly. - // This currently doesn't support ipv6. All of these functions should be removed and - // addressed properly in the following issue. - // https://github.com/sigp/lighthouse/issues/4706 - pub fn update_enr_quic_port(&mut self, port: u16) -> Result<(), String> { - self.discv5 - .enr_insert("quic", &port) - .map_err(|e| format!("{:?}", e))?; - - // replace the global version - *self.network_globals.local_enr.write() = self.discv5.local_enr(); - // persist modified enr to disk - enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); - Ok(()) - } - - /// Updates the local ENR UDP socket. - /// - /// This is with caution. Discovery should automatically maintain this. This should only be - /// used when automatic discovery is disabled. - pub fn update_enr_udp_socket(&mut self, socket_addr: SocketAddr) -> Result<(), String> { - const IS_TCP: bool = false; - if self.discv5.update_local_enr_socket(socket_addr, IS_TCP) { - // persist modified enr to disk - enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); - } - *self.network_globals.local_enr.write() = self.discv5.local_enr(); - Ok(()) - } - /// Adds/Removes a subnet from the ENR attnets/syncnets Bitfield pub fn update_enr_bitfield(&mut self, subnet: Subnet, value: bool) -> Result<(), String> { let local_enr = self.discv5.local_enr(); @@ -520,7 +474,7 @@ impl Discovery { } // replace the global version - *self.network_globals.local_enr.write() = self.discv5.local_enr(); + *self.network_globals.local_enr.enr.write() = self.discv5.local_enr(); // persist modified enr to disk enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); @@ -555,7 +509,7 @@ impl Discovery { }); // replace the global version with discovery version - *self.network_globals.local_enr.write() = self.discv5.local_enr(); + *self.network_globals.local_enr.enr.write() = self.discv5.local_enr(); // persist modified enr to disk enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); @@ -1025,7 +979,7 @@ impl NetworkBehaviour for Discovery { let enr = self.discv5.local_enr(); enr::save_enr_to_disk(Path::new(&self.enr_dir), &enr, &self.log); // update network globals - *self.network_globals.local_enr.write() = enr; + *self.network_globals.local_enr.enr.write() = enr; // A new UDP socket has been detected. // NOTE: We assume libp2p itself can keep track of IP changes and we do // not inform it about IP changes found via discovery. @@ -1062,7 +1016,9 @@ impl NetworkBehaviour for Discovery { return; } - self.update_enr_tcp_port(port) + self.network_globals + .local_enr + .update_port(EnrPort::Tcp4(port)) } (Some(Protocol::Udp(port)), Some(Protocol::QuicV1)) => { if !self.update_ports.quic4 { @@ -1070,7 +1026,9 @@ impl NetworkBehaviour for Discovery { return; } - self.update_enr_quic_port(port) + self.network_globals + .local_enr + .update_port(EnrPort::Udp4(port)) } _ => { debug!(self.log, "Encountered unacceptable multiaddr for listening (unsupported transport)"; "addr" => ?addr); @@ -1084,7 +1042,9 @@ impl NetworkBehaviour for Discovery { return; } - self.update_enr_tcp_port(port) + self.network_globals + .local_enr + .update_port(EnrPort::Tcp6(port)) } (Some(Protocol::Udp(port)), Some(Protocol::QuicV1)) => { if !self.update_ports.quic6 { @@ -1092,7 +1052,9 @@ impl NetworkBehaviour for Discovery { return; } - self.update_enr_quic_port(port) + self.network_globals + .local_enr + .update_port(EnrPort::Udp6(port)) } _ => { debug!(self.log, "Encountered unacceptable multiaddr for listening (unsupported transport)"; "addr" => ?addr); @@ -1182,6 +1144,7 @@ mod tests { let log = build_log(slog::Level::Debug, false); let globals = NetworkGlobals::new( enr, + enr_key, MetaData::V2(MetaDataV2 { seq_number: 0, attnets: Default::default(), diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 0617c8fa372..6aafa7390c9 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -248,7 +248,7 @@ impl PeerManager { } // Check NAT if metrics are enabled - if self.network_globals.local_enr.read().udp4().is_some() { + if self.network_globals.local_enr.enr().udp4().is_some() { metrics::check_nat(); } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index e70cda69756..c0eaf3de9e9 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -2,7 +2,8 @@ use self::behaviour::Behaviour; use self::gossip_cache::GossipCache; use crate::config::{gossipsub_config, GossipsubConfigParams, NetworkLoad}; use crate::discovery::{ - subnet_predicate, DiscoveredPeers, Discovery, FIND_NODE_QUERY_CLOSEST_PEERS, + enr_ext::CombinedKeyExt, subnet_predicate, DiscoveredPeers, Discovery, + FIND_NODE_QUERY_CLOSEST_PEERS, }; use crate::peer_manager::{ config::Config as PeerManagerCfg, peerdb::score::PeerAction, peerdb::score::ReportSource, @@ -21,6 +22,7 @@ use crate::EnrExt; use crate::Eth2Enr; use crate::{error, metrics, Enr, NetworkGlobals, PubsubMessage, TopicHash}; use api_types::{PeerRequestId, Request, RequestId, Response}; +use discv5::enr::CombinedKey; use futures::stream::StreamExt; use gossipsub_scoring_parameters::{lighthouse_gossip_thresholds, PeerScoreSettings}; use libp2p::bandwidth::BandwidthSinks; @@ -157,6 +159,7 @@ impl Network { let meta_data = utils::load_or_build_metadata(&config.network_dir, &log); let globals = NetworkGlobals::new( enr, + CombinedKey::from_libp2p(local_keypair.clone())?, meta_data, config .trusted_peers @@ -279,18 +282,15 @@ impl Network { let discovery = { // Build and start the discovery sub-behaviour - let mut discovery = Discovery::new( - local_keypair.clone(), - &config, - network_globals.clone(), - &log, - ) - .await?; + let mut discovery = Discovery::new(&config, network_globals.clone(), &log).await?; // start searching for peers discovery.discover_peers(FIND_NODE_QUERY_CLOSEST_PEERS); discovery }; + /* write back the ENR that discovery has built */ + *network_globals.local_enr.enr.write() = discovery.local_enr(); + let identify = { let local_public_key = local_keypair.public(); let identify_config = if config.private { diff --git a/beacon_node/lighthouse_network/src/types/globals.rs b/beacon_node/lighthouse_network/src/types/globals.rs index b2b605e8aec..4774161b42a 100644 --- a/beacon_node/lighthouse_network/src/types/globals.rs +++ b/beacon_node/lighthouse_network/src/types/globals.rs @@ -1,17 +1,19 @@ //! A collection of variables that are accessible outside of the network thread itself. +use super::mutable_enr::MutableEnr; use crate::peer_manager::peerdb::PeerDB; use crate::rpc::{MetaData, MetaDataV2}; use crate::types::{BackFillState, SyncState}; use crate::Client; use crate::EnrExt; use crate::{Enr, GossipTopic, Multiaddr, PeerId}; +use discv5::enr::CombinedKey; use parking_lot::RwLock; use std::collections::HashSet; use types::EthSpec; pub struct NetworkGlobals { /// The current local ENR. - pub local_enr: RwLock, + pub local_enr: MutableEnr, /// The local peer_id. pub peer_id: RwLock, /// Listening multiaddrs. @@ -31,13 +33,14 @@ pub struct NetworkGlobals { impl NetworkGlobals { pub fn new( enr: Enr, + enr_key: CombinedKey, local_metadata: MetaData, trusted_peers: Vec, disable_peer_scoring: bool, log: &slog::Logger, ) -> Self { NetworkGlobals { - local_enr: RwLock::new(enr.clone()), + local_enr: MutableEnr::new(enr.clone(), enr_key), peer_id: RwLock::new(enr.peer_id()), listen_multiaddrs: RwLock::new(Vec::new()), local_metadata: RwLock::new(local_metadata), @@ -51,7 +54,7 @@ impl NetworkGlobals { /// Returns the local ENR from the underlying Discv5 behaviour that external peers may connect /// to. pub fn local_enr(&self) -> Enr { - self.local_enr.read().clone() + self.local_enr.enr().clone() } /// Returns the local libp2p PeerID. @@ -121,6 +124,7 @@ impl NetworkGlobals { let enr = discv5::enr::EnrBuilder::new("v4").build(&enr_key).unwrap(); NetworkGlobals::new( enr, + enr_key, MetaData::V2(MetaDataV2 { seq_number: 0, attnets: Default::default(), diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index e7457f25dac..1be6c7fd822 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -1,5 +1,6 @@ pub mod error; mod globals; +pub mod mutable_enr; mod pubsub; mod subnet; mod sync_state; diff --git a/beacon_node/lighthouse_network/src/types/mutable_enr.rs b/beacon_node/lighthouse_network/src/types/mutable_enr.rs new file mode 100644 index 00000000000..ac507766a47 --- /dev/null +++ b/beacon_node/lighthouse_network/src/types/mutable_enr.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use discv5::enr::{Enr, EnrError}; +use libp2p::bytes::Bytes; +use parking_lot::RwLock; + +use crate::discovery::enr::CombinedKey; + +/// Represents a port advertised via an ENR +/// +/// The ENR specification allows extension of the keys stored in any given ENR +/// as well as specific keys for TCP and UDP ports over both IPv4 and IPv6. This +/// type extends these to add the QUIC transport. +pub enum EnrPort { + Tcp4(u16), + Tcp6(u16), + Udp4(u16), + Udp6(u16), + Quic4(u16), + Quic6(u16), +} + +/// Represents a mutable instance of an ENR +/// +/// Any mutation of an ENR necessitates re-signing the new ENR, so this type +/// contains both the ENR itself as well as its corresponding signing key. +#[derive(Clone)] +pub struct MutableEnr { + /// Shared ENR + pub enr: Arc>>, + /// Signing key for ENR modifications + enr_key: Arc, +} + +impl MutableEnr { + pub fn new(enr: Enr, enr_key: CombinedKey) -> Self { + Self { + enr: Arc::new(RwLock::new(enr)), + enr_key: Arc::new(enr_key), + } + } + + pub fn enr(&self) -> Enr { + self.enr.read().clone() + } + + pub fn enr_key(&self) -> CombinedKey { + Arc::into_inner(self.enr_key.clone()).unwrap() + } + + /// Insert an arbitrary key-value pair into the ENR + pub fn insert(&self, key: impl AsRef<[u8]>, value: &[u8]) -> Result, EnrError> { + self.enr.write().insert(key, &value, &self.enr_key) + } + + /// Update the specified port being advertised via this ENR + /// + /// If the port is already being advertised in this ENR, it is overwritten. Otherwise, it is added as a new key-value pair. + pub fn update_port(&self, port: EnrPort) -> Result, EnrError> { + match port { + EnrPort::Tcp4(p) => self.insert("tcp4", p.to_be_bytes().as_ref()), + EnrPort::Tcp6(p) => self.insert("tcp6", p.to_be_bytes().as_ref()), + EnrPort::Udp4(p) => self.insert("udp4", p.to_be_bytes().as_ref()), + EnrPort::Udp6(p) => self.insert("udp6", p.to_be_bytes().as_ref()), + EnrPort::Quic4(p) => self.insert("quic4", p.to_be_bytes().as_ref()), + EnrPort::Quic6(p) => self.insert("quic6", p.to_be_bytes().as_ref()), + } + } +} diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index aa92e0afdab..15b2ff65496 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -16,6 +16,7 @@ use futures::future::OptionFuture; use futures::prelude::*; use futures::StreamExt; use lighthouse_network::service::Network; +use lighthouse_network::types::mutable_enr::EnrPort; use lighthouse_network::types::GossipKind; use lighthouse_network::{prometheus_client::registry::Registry, MessageAcceptance}; use lighthouse_network::{ @@ -640,14 +641,26 @@ impl NetworkService { self.upnp_mappings = mappings; // If there is an external TCP port update, modify our local ENR. if let Some(tcp_port) = self.upnp_mappings.tcp_port { - if let Err(e) = self.libp2p.discovery_mut().update_enr_tcp_port(tcp_port) { - warn!(self.log, "Failed to update ENR"; "error" => e); + if let Err(e) = self + .libp2p + .discovery_mut() + .network_globals + .local_enr + .update_port(EnrPort::Tcp4(tcp_port)) + { + warn!(self.log, "Failed to update ENR"; "error" => ?e); } } // If there is an external QUIC port update, modify our local ENR. if let Some(quic_port) = self.upnp_mappings.udp_quic_port { - if let Err(e) = self.libp2p.discovery_mut().update_enr_quic_port(quic_port) { - warn!(self.log, "Failed to update ENR"; "error" => e); + if let Err(e) = self + .libp2p + .discovery_mut() + .network_globals + .local_enr + .update_port(EnrPort::Quic4(quic_port)) + { + warn!(self.log, "Failed to update ENR"; "error" => ?e); } } }