From eee0b0db757b9077b37daa15bc6daf2211d50247 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 18 Dec 2023 22:45:12 +1100 Subject: [PATCH 1/9] Build and publish column sidecars. Add stubs for gossip. --- beacon_node/beacon_chain/src/beacon_chain.rs | 17 +- .../beacon_chain/src/blob_verification.rs | 37 +++- beacon_node/beacon_chain/src/errors.rs | 1 + beacon_node/beacon_chain/src/metrics.rs | 12 ++ beacon_node/beacon_processor/src/lib.rs | 29 +++- beacon_node/beacon_processor/src/metrics.rs | 5 + beacon_node/http_api/src/publish_blocks.rs | 20 ++- .../src/service/gossip_cache.rs | 1 + .../lighthouse_network/src/types/pubsub.rs | 38 +++- .../lighthouse_network/src/types/topics.rs | 13 ++ beacon_node/network/src/metrics.rs | 23 +++ .../gossip_methods.rs | 87 +++++++++- .../src/network_beacon_processor/mod.rs | 30 ++++ beacon_node/network/src/router.rs | 14 ++ consensus/types/src/blob_sample.rs | 164 ++++++++++++++++++ consensus/types/src/eth_spec.rs | 45 ++++- consensus/types/src/lib.rs | 2 + 17 files changed, 522 insertions(+), 16 deletions(-) create mode 100644 consensus/types/src/blob_sample.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 83b593f9bc9..af9799cb13a 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -7,7 +7,9 @@ use crate::attester_cache::{AttesterCache, AttesterCacheKey}; use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache}; use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; -use crate::blob_verification::{GossipBlobError, GossipVerifiedBlob}; +use crate::blob_verification::{ + GossipBlobError, GossipVerifiedBlob, GossipVerifiedBlobColumnSidecar, +}; use crate::block_times_cache::BlockTimesCache; use crate::block_verification::POS_PANDA_BANNER; use crate::block_verification::{ @@ -2052,6 +2054,19 @@ impl BeaconChain { }) } + pub fn verify_blob_column_sidecar_for_gossip( + self: &Arc, + blob_column_sidecar: Arc>, + subnet_id: u64, + ) -> Result, GossipBlobError> { + metrics::inc_counter(&metrics::BLOBS_COLUMN_SIDECAR_PROCESSING_REQUESTS); + let _timer = metrics::start_timer(&metrics::BLOBS_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); + GossipVerifiedBlobColumnSidecar::new(blob_column_sidecar, subnet_id, self).map(|v| { + metrics::inc_counter(&metrics::BLOB_COLUMNS_SIDECAR_PROCESSING_SUCCESSES); + v + }) + } + pub fn verify_blob_sidecar_for_gossip( self: &Arc, blob_sidecar: Arc>, diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index cc087e74a07..7ebdbacf18c 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -17,7 +17,8 @@ use ssz_types::VariableList; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; use types::{ - BeaconStateError, BlobSidecar, CloneConfig, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, + BeaconStateError, BlobColumnSidecar, BlobSidecar, CloneConfig, EthSpec, Hash256, + SignedBeaconBlockHeader, Slot, }; /// An error occurred while validating a gossip blob. @@ -184,6 +185,29 @@ pub type GossipVerifiedBlobList = VariableList< <::EthSpec as EthSpec>::MaxBlobsPerBlock, >; +#[derive(Debug)] +pub struct GossipVerifiedBlobColumnSidecar { + blob_column_sidecar: Arc>, +} + +impl GossipVerifiedBlobColumnSidecar { + pub fn new( + column_sidecar: Arc>, + subnet_id: u64, + chain: &BeaconChain, + ) -> Result> { + let header = column_sidecar.signed_block_header.clone(); + // We only process slashing info if the gossip verification failed + // since we do not process the blob any further in that case. + validate_blob_column_sidecar_for_gossip(column_sidecar, subnet_id, chain).map_err(|e| { + process_block_slash_info::<_, GossipBlobError>( + chain, + BlockSlashInfo::from_early_error_blob(header, e), + ) + }) + } +} + /// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on /// the p2p network. #[derive(Debug)] @@ -637,6 +661,17 @@ pub fn validate_blob_sidecar_for_gossip( }) } +pub fn validate_blob_column_sidecar_for_gossip( + blob_column_sidecar: Arc>, + _subnet: u64, + _chain: &BeaconChain, +) -> Result, GossipBlobError> { + // TODO(das): validate + Ok(GossipVerifiedBlobColumnSidecar { + blob_column_sidecar: blob_column_sidecar.clone(), + }) +} + /// Returns the canonical root of the given `blob`. /// /// Use this function to ensure that we report the blob hashing time Prometheus metric. diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 9c1ba06f853..f9313b3ec8c 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -220,6 +220,7 @@ pub enum BeaconChainError { InconsistentFork(InconsistentFork), ProposerHeadForkChoiceError(fork_choice::Error), UnableToPublish, + UnableToBuildBlobColumnSidecar(String), AvailabilityCheckError(AvailabilityCheckError), LightClientError(LightClientError), UnsupportedFork, diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index ad095b37b51..7f8382147b8 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1014,14 +1014,26 @@ lazy_static! { "beacon_blobs_sidecar_processing_requests_total", "Count of all blob sidecars submitted for processing" ); + pub static ref BLOBS_COLUMN_SIDECAR_PROCESSING_REQUESTS: Result = try_create_int_counter( + "beacon_blobs_column_sidecar_processing_requests_total", + "Count of all blob column sidecars submitted for processing" + ); pub static ref BLOBS_SIDECAR_PROCESSING_SUCCESSES: Result = try_create_int_counter( "beacon_blobs_sidecar_processing_successes_total", "Number of blob sidecars verified for gossip" ); + pub static ref BLOB_COLUMNS_SIDECAR_PROCESSING_SUCCESSES: Result = try_create_int_counter( + "beacon_blobs_column_sidecar_processing_successes_total", + "Number of blob column sidecars verified for gossip" + ); pub static ref BLOBS_SIDECAR_GOSSIP_VERIFICATION_TIMES: Result = try_create_histogram( "beacon_blobs_sidecar_gossip_verification_seconds", "Full runtime of blob sidecars gossip verification" ); + pub static ref BLOBS_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: Result = try_create_histogram( + "beacon_blobs_column_sidecar_gossip_verification_seconds", + "Full runtime of blob column sidecars gossip verification" + ); pub static ref BLOB_SIDECAR_INCLUSION_PROOF_VERIFICATION: Result = try_create_histogram( "blob_sidecar_inclusion_proof_verification_seconds", "Time taken to verify blob sidecar inclusion proof" diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 1c675d280f8..9a92b87ef1a 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -110,6 +110,10 @@ const MAX_GOSSIP_BLOCK_QUEUE_LEN: usize = 1_024; /// will be stored before we start dropping them. const MAX_GOSSIP_BLOB_QUEUE_LEN: usize = 1_024; +/// The maximum number of queued `BlobColumnSidecar` objects received on gossip that +/// will be stored before we start dropping them. +const MAX_GOSSIP_BLOB_COL_QUEUE_LEN: usize = 1_024; + /// The maximum number of queued `SignedBeaconBlock` objects received prior to their slot (but /// within acceptable clock disparity) that will be queued before we start dropping them. const MAX_DELAYED_BLOCK_QUEUE_LEN: usize = 1_024; @@ -226,6 +230,7 @@ pub const GOSSIP_AGGREGATE: &str = "gossip_aggregate"; pub const GOSSIP_AGGREGATE_BATCH: &str = "gossip_aggregate_batch"; pub const GOSSIP_BLOCK: &str = "gossip_block"; pub const GOSSIP_BLOBS_SIDECAR: &str = "gossip_blobs_sidecar"; +pub const GOSSIP_BLOBS_COLUMN_SIDECAR: &str = "gossip_blobs_column_sidecar"; pub const DELAYED_IMPORT_BLOCK: &str = "delayed_import_block"; pub const GOSSIP_VOLUNTARY_EXIT: &str = "gossip_voluntary_exit"; pub const GOSSIP_PROPOSER_SLASHING: &str = "gossip_proposer_slashing"; @@ -592,6 +597,7 @@ pub enum Work { }, GossipBlock(AsyncFn), GossipBlobSidecar(AsyncFn), + GossipBlobColumnSidecar(AsyncFn), DelayedImportBlock { beacon_block_slot: Slot, beacon_block_root: Hash256, @@ -642,6 +648,7 @@ impl Work { Work::GossipAggregateBatch { .. } => GOSSIP_AGGREGATE_BATCH, Work::GossipBlock(_) => GOSSIP_BLOCK, Work::GossipBlobSidecar(_) => GOSSIP_BLOBS_SIDECAR, + Work::GossipBlobColumnSidecar(_) => GOSSIP_BLOBS_COLUMN_SIDECAR, Work::DelayedImportBlock { .. } => DELAYED_IMPORT_BLOCK, Work::GossipVoluntaryExit(_) => GOSSIP_VOLUNTARY_EXIT, Work::GossipProposerSlashing(_) => GOSSIP_PROPOSER_SLASHING, @@ -811,6 +818,7 @@ impl BeaconProcessor { let mut backfill_chain_segment = FifoQueue::new(MAX_CHAIN_SEGMENT_QUEUE_LEN); let mut gossip_block_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN); let mut gossip_blob_queue = FifoQueue::new(MAX_GOSSIP_BLOB_QUEUE_LEN); + let mut gossip_blob_column_queue = FifoQueue::new(MAX_GOSSIP_BLOB_COL_QUEUE_LEN); let mut delayed_block_queue = FifoQueue::new(MAX_DELAYED_BLOCK_QUEUE_LEN); let mut status_queue = FifoQueue::new(MAX_STATUS_QUEUE_LEN); @@ -966,6 +974,8 @@ impl BeaconProcessor { self.spawn_worker(item, idle_tx); } else if let Some(item) = gossip_blob_queue.pop() { self.spawn_worker(item, idle_tx); + } else if let Some(item) = gossip_blob_column_queue.pop() { + self.spawn_worker(item, idle_tx); // Check the priority 0 API requests after blocks and blobs, but before attestations. } else if let Some(item) = api_request_p0_queue.pop() { self.spawn_worker(item, idle_tx); @@ -1208,6 +1218,9 @@ impl BeaconProcessor { Work::GossipBlobSidecar { .. } => { gossip_blob_queue.push(work, work_id, &self.log) } + Work::GossipBlobColumnSidecar { .. } => { + gossip_blob_column_queue.push(work, work_id, &self.log) + } Work::DelayedImportBlock { .. } => { delayed_block_queue.push(work, work_id, &self.log) } @@ -1304,7 +1317,11 @@ impl BeaconProcessor { ); metrics::set_gauge( &metrics::BEACON_PROCESSOR_GOSSIP_BLOB_QUEUE_TOTAL, - gossip_block_queue.len() as i64, + gossip_blob_queue.len() as i64, + ); + metrics::set_gauge( + &metrics::BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_QUEUE_TOTAL, + gossip_blob_column_queue.len() as i64, ); metrics::set_gauge( &metrics::BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL, @@ -1457,11 +1474,11 @@ impl BeaconProcessor { task_spawner.spawn_async(process_fn) } Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), - Work::GossipBlock(work) | Work::GossipBlobSidecar(work) => { - task_spawner.spawn_async(async move { - work.await; - }) - } + Work::GossipBlock(work) + | Work::GossipBlobSidecar(work) + | Work::GossipBlobColumnSidecar(work) => task_spawner.spawn_async(async move { + work.await; + }), Work::BlobsByRangeRequest(process_fn) | Work::BlobsByRootsRequest(process_fn) => { task_spawner.spawn_blocking(process_fn) } diff --git a/beacon_node/beacon_processor/src/metrics.rs b/beacon_node/beacon_processor/src/metrics.rs index 9082a7d4742..8879a4ed2ee 100644 --- a/beacon_node/beacon_processor/src/metrics.rs +++ b/beacon_node/beacon_processor/src/metrics.rs @@ -51,6 +51,11 @@ lazy_static::lazy_static! { "beacon_processor_gossip_blob_queue_total", "Count of blocks from gossip waiting to be verified." ); + // Gossip blob column sidecars. + pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_gossip_blob_column_queue_total", + "Count of blob column sidecars from gossip waiting to be verified." + ); // Gossip Exits. pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result = try_create_int_gauge( "beacon_processor_exit_queue_total", diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 432d91b7234..d7aca6a76bf 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -19,9 +19,9 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, BlobSidecarList, EthSpec, ExecPayload, ExecutionBlockHash, - ForkName, FullPayload, FullPayloadMerge, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, - VariableList, + AbstractExecPayload, BeaconBlockRef, BlobColumnSidecar, BlobSidecarList, EthSpec, ExecPayload, + ExecutionBlockHash, ForkName, FullPayload, FullPayloadMerge, Hash256, SignedBeaconBlock, + SignedBlindedBeaconBlock, VariableList, }; use warp::http::StatusCode; use warp::{reply::Response, Rejection, Reply}; @@ -88,6 +88,20 @@ pub async fn publish_block { let mut pubsub_messages = vec![PubsubMessage::BeaconBlock(block.clone())]; if let Some(blob_sidecars) = blobs_opt { + // Build and publish column sidecars + let col_sidecars = BlobColumnSidecar::random_from_blob_sidecars(&blob_sidecars) + .map_err(|e| { + BlockError::BeaconChainError( + BeaconChainError::UnableToBuildBlobColumnSidecar(e), + ) + })?; + for (col_index, col_sidecar) in col_sidecars.into_iter().enumerate() { + pubsub_messages.push(PubsubMessage::BlobColumnSidecar(Box::new(( + (col_index as u64).into(), + Arc::new(col_sidecar), + )))); + } + // Publish blob sidecars for (blob_index, blob) in blob_sidecars.into_iter().enumerate() { pubsub_messages.push(PubsubMessage::BlobSidecar(Box::new(( blob_index as u64, diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 5dc0d29ff5b..407fc16d529 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -194,6 +194,7 @@ impl GossipCache { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, GossipKind::BlobSidecar(_) => self.blob_sidecar, + GossipKind::BlobColumnSidecar(_) => self.blob_sidecar, GossipKind::BeaconAggregateAndProof => self.aggregates, GossipKind::Attestation(_) => self.attestation, GossipKind::VoluntaryExit => self.voluntary_exit, diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 60fe3748265..44d7b30110e 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -9,7 +9,7 @@ use std::boxed::Box; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::{ - Attestation, AttesterSlashing, BlobSidecar, EthSpec, ForkContext, ForkName, + Attestation, AttesterSlashing, BlobColumnSidecar, BlobSidecar, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockMerge, @@ -23,6 +23,8 @@ pub enum PubsubMessage { BeaconBlock(Arc>), /// Gossipsub message providing notification of a [`BlobSidecar`] along with the subnet id where it was received. BlobSidecar(Box<(u64, Arc>)>), + /// Gossipsub message providing notification of a [`BlobColumnSidecar`] along with the subnet id where it was received. + BlobColumnSidecar(Box<(SubnetId, Arc>)>), /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a raw un-aggregated attestation with its shard id. @@ -119,6 +121,9 @@ impl PubsubMessage { PubsubMessage::BlobSidecar(blob_sidecar_data) => { GossipKind::BlobSidecar(blob_sidecar_data.0) } + PubsubMessage::BlobColumnSidecar(column_sidecar_data) => { + GossipKind::BlobColumnSidecar(column_sidecar_data.0) + } PubsubMessage::AggregateAndProofAttestation(_) => GossipKind::BeaconAggregateAndProof, PubsubMessage::Attestation(attestation_data) => { GossipKind::Attestation(attestation_data.0) @@ -226,6 +231,30 @@ impl PubsubMessage { )), } } + GossipKind::BlobColumnSidecar(subnet_id) => { + match fork_context.from_context_bytes(gossip_topic.fork_digest) { + Some(ForkName::Deneb) => { + let col_sidecar = Arc::new( + BlobColumnSidecar::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ); + Ok(PubsubMessage::BlobColumnSidecar(Box::new(( + *subnet_id, + col_sidecar, + )))) + } + Some( + ForkName::Base + | ForkName::Altair + | ForkName::Merge + | ForkName::Capella, + ) + | None => Err(format!( + "blob_column_sidecar topic invalid for given fork digest {:?}", + gossip_topic.fork_digest + )), + } + } GossipKind::VoluntaryExit => { let voluntary_exit = SignedVoluntaryExit::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?; @@ -295,6 +324,7 @@ impl PubsubMessage { match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), + PubsubMessage::BlobColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), @@ -324,6 +354,12 @@ impl std::fmt::Display for PubsubMessage { data.1.slot(), data.1.index, ), + PubsubMessage::BlobColumnSidecar(data) => write!( + f, + "BlobColumnSidecar: slot: {}, column index: {}", + data.1.slot(), + data.1.index, + ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, "Aggregate and Proof: slot: {}, index: {}, aggregator_index: {}", diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index e7e771e1adc..aa548e4a2dd 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -15,6 +15,7 @@ pub const BEACON_BLOCK_TOPIC: &str = "beacon_block"; pub const BEACON_AGGREGATE_AND_PROOF_TOPIC: &str = "beacon_aggregate_and_proof"; pub const BEACON_ATTESTATION_PREFIX: &str = "beacon_attestation_"; pub const BLOB_SIDECAR_PREFIX: &str = "blob_sidecar_"; +pub const BLOB_COLUMN_SIDECAR_PREFIX: &str = "blob_column_sidecar_"; pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit"; pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing"; pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing"; @@ -98,6 +99,8 @@ pub enum GossipKind { BeaconAggregateAndProof, /// Topic for publishing BlobSidecars. BlobSidecar(u64), + /// Topic for publishing BlobColumnSidecars. + BlobColumnSidecar(SubnetId), /// Topic for publishing raw attestations on a particular subnet. #[strum(serialize = "beacon_attestation")] Attestation(SubnetId), @@ -130,6 +133,9 @@ impl std::fmt::Display for GossipKind { GossipKind::BlobSidecar(blob_index) => { write!(f, "{}{}", BLOB_SIDECAR_PREFIX, blob_index) } + GossipKind::BlobColumnSidecar(column_index) => { + write!(f, "{}{}", BLOB_COLUMN_SIDECAR_PREFIX, column_index) + } x => f.write_str(x.as_ref()), } } @@ -255,6 +261,9 @@ impl std::fmt::Display for GossipTopic { GossipKind::BlobSidecar(blob_index) => { format!("{}{}", BLOB_SIDECAR_PREFIX, blob_index) } + GossipKind::BlobColumnSidecar(index) => { + format!("{}{}", BLOB_COLUMN_SIDECAR_PREFIX, *index) + } GossipKind::BlsToExecutionChange => BLS_TO_EXECUTION_CHANGE_TOPIC.into(), GossipKind::LightClientFinalityUpdate => LIGHT_CLIENT_FINALITY_UPDATE.into(), GossipKind::LightClientOptimisticUpdate => LIGHT_CLIENT_OPTIMISTIC_UPDATE.into(), @@ -298,6 +307,10 @@ fn subnet_topic_index(topic: &str) -> Option { ))); } else if let Some(index) = topic.strip_prefix(BLOB_SIDECAR_PREFIX) { return Some(GossipKind::BlobSidecar(index.parse::().ok()?)); + } else if let Some(index) = topic.strip_prefix(BLOB_COLUMN_SIDECAR_PREFIX) { + return Some(GossipKind::BlobColumnSidecar(SubnetId::new( + index.parse::().ok()?, + ))); } None } diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 0509ed1ea7d..56573e27624 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -71,6 +71,10 @@ lazy_static! { "beacon_processor_gossip_blob_verified_total", "Total number of gossip blob verified for propagation." ); + pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_SIDECAR_VERIFIED_TOTAL: Result = try_create_int_counter( + "beacon_processor_gossip_blob_column_verified_total", + "Total number of gossip blob column sidecar verified for propagation." + ); // Gossip Exits. pub static ref BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_exit_verified_total", @@ -283,6 +287,12 @@ lazy_static! { // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] decimal_buckets(-3,-1) ); + pub static ref BEACON_BLOB_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: Result = try_create_histogram_with_buckets( + "beacon_blob_column_gossip_propagation_verification_delay_time", + "Duration between when the blob column sidecar is received over gossip and when it is verified for propagation.", + // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] + decimal_buckets(-3,-1) + ); pub static ref BEACON_BLOB_GOSSIP_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( "beacon_blob_gossip_slot_start_delay_time", "Duration between when the blob is received over gossip and the start of the slot it belongs to.", @@ -292,6 +302,15 @@ lazy_static! { // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] //decimal_buckets(-1,2) ); + pub static ref BEACON_BLOB_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( + "beacon_blob_column_gossip_slot_start_delay_time", + "Duration between when the blob column sidecar is received over gossip and the start of the slot it belongs to.", + // Create a custom bucket list for greater granularity in block delay + Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) + // NOTE: Previous values, which we may want to switch back to. + // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] + //decimal_buckets(-1,2) + ); pub static ref BEACON_BLOB_RPC_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( "beacon_blob_rpc_slot_start_delay_time", "Duration between when a blob is received over rpc and the start of the slot it belongs to.", @@ -306,6 +325,10 @@ lazy_static! { "beacon_blob_last_delay", "Keeps track of the last blob's delay from the start of the slot" ); + pub static ref BEACON_BLOB_COLUMN_LAST_DELAY: Result = try_create_int_gauge( + "beacon_blob_column_last_delay", + "Keeps track of the last blob column sidecar's delay from the start of the slot" + ); pub static ref BEACON_BLOB_GOSSIP_ARRIVED_LATE_TOTAL: Result = try_create_int_counter( "beacon_blob_gossip_arrived_late_total", diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 9d9b196e9be..b3538b545f5 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -4,7 +4,9 @@ use crate::{ service::NetworkMessage, sync::SyncMessage, }; -use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; +use beacon_chain::blob_verification::{ + GossipBlobError, GossipVerifiedBlob, GossipVerifiedBlobColumnSidecar, +}; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::store::Error; use beacon_chain::{ @@ -31,8 +33,8 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; use types::{ - Attestation, AttesterSlashing, BlobSidecar, EthSpec, Hash256, IndexedAttestation, - LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, + Attestation, AttesterSlashing, BlobColumnSidecar, BlobSidecar, EthSpec, Hash256, + IndexedAttestation, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, @@ -599,6 +601,75 @@ impl NetworkBeaconProcessor { } } + pub async fn process_gossip_blob_column_sidecar( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + _peer_client: Client, + subnet_id: SubnetId, + column_sidecar: Arc>, + seen_duration: Duration, + ) { + let slot = column_sidecar.slot(); + let root = column_sidecar.block_root(); + let index = *subnet_id; + let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); + // Log metrics to track delay from other nodes on the network. + metrics::observe_duration( + &metrics::BEACON_BLOB_COLUMN_GOSSIP_SLOT_START_DELAY_TIME, + delay, + ); + metrics::set_gauge( + &metrics::BEACON_BLOB_COLUMN_LAST_DELAY, + delay.as_millis() as i64, + ); + match self + .chain + .verify_blob_column_sidecar_for_gossip(column_sidecar, index) + { + Ok(gossip_verified_blob_column) => { + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_SIDECAR_VERIFIED_TOTAL, + ); + + debug!( + self.log, + "Successfully verified gossip blob column sidecar"; + "slot" => %slot, + "root" => %root, + "index" => %index, + ); + + self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept); + + // Log metrics to keep track of propagation delay times. + if let Some(duration) = SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|now| now.checked_sub(seen_duration)) + { + metrics::observe_duration( + &metrics::BEACON_BLOB_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME, + duration, + ); + } + self.process_gossip_verified_blob_column( + peer_id, + gossip_verified_blob_column, + seen_duration, + ) + .await + } + Err(err) => { + error!( + self.log, + "Internal error when verifying blob column sidecar"; + "error" => ?err, + ) + } + } + } + #[allow(clippy::too_many_arguments)] pub async fn process_gossip_blob( self: &Arc, @@ -796,6 +867,16 @@ impl NetworkBeaconProcessor { } } + pub async fn process_gossip_verified_blob_column( + self: &Arc, + _peer_id: PeerId, + _verified_blob: GossipVerifiedBlobColumnSidecar, + // This value is not used presently, but it might come in handy for debugging. + _seen_duration: Duration, + ) { + todo!() + } + /// Process the beacon block received from the gossip network and: /// /// - If it passes gossip propagation criteria, tell the network thread to forward it. diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 67fc2fabb1e..4e5bdf3e695 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -229,6 +229,36 @@ impl NetworkBeaconProcessor { }) } + /// Create a new `Work` event for some blob column sidecar. + pub fn send_gossip_blob_column_sidecar( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + subnet_id: SubnetId, + column_sidecar: Arc>, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = async move { + processor + .process_gossip_blob_column_sidecar( + message_id, + peer_id, + peer_client, + subnet_id, + column_sidecar, + seen_timestamp, + ) + .await + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipBlobColumnSidecar(Box::pin(process_fn)), + }) + } + /// Create a new `Work` event for some sync committee signature. pub fn send_gossip_sync_signature( self: &Arc, diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index f56a3b7445e..e0ddb4fcc36 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -308,6 +308,20 @@ impl Router { ), ) } + PubsubMessage::BlobColumnSidecar(data) => { + let (subnet_id, column_sidecar) = *data; + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_blob_column_sidecar( + message_id, + peer_id, + self.network_globals.client(&peer_id), + subnet_id, + column_sidecar, + timestamp_now(), + ), + ) + } PubsubMessage::VoluntaryExit(exit) => { debug!(self.log, "Received a voluntary exit"; "peer_id" => %peer_id); self.handle_beacon_processor_send_result( diff --git a/consensus/types/src/blob_sample.rs b/consensus/types/src/blob_sample.rs new file mode 100644 index 00000000000..7e215c6b245 --- /dev/null +++ b/consensus/types/src/blob_sample.rs @@ -0,0 +1,164 @@ +use crate::beacon_block_body::KzgCommitments; +use crate::test_utils::TestRandom; +use crate::{BlobSidecarList, EthSpec, Hash256, SignedBeaconBlockHeader, Slot}; +use derivative::Derivative; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use ssz_types::FixedVector; +use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::TreeHash; + +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + Derivative, + arbitrary::Arbitrary, +)] +#[serde(bound = "T: EthSpec")] +#[arbitrary(bound = "T: EthSpec")] +#[derivative(PartialEq, Eq, Hash(bound = "T: EthSpec"))] +pub struct BlobColumnSidecar { + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, + #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")] + pub data: FixedVector, + pub signed_block_header: SignedBeaconBlockHeader, + pub kzg_commitments: KzgCommitments, + pub kzg_commitments_inclusion_proof: + FixedVector, + // List of cell proofs proving each column sample is part of the extended blob. + // pub cell_proofs: Vec, +} + +impl BlobColumnSidecar { + pub fn random_from_blob_sidecars( + blob_sidecars: &BlobSidecarList, + ) -> Result>, String> { + if blob_sidecars.is_empty() { + return Ok(vec![]); + } + + let first_blob_sidecar = blob_sidecars.first().ok_or("should exist")?; + let slot = first_blob_sidecar.slot(); + + // Proof for kzg commitments in `BeaconBlockBody` + let body_proof_start = first_blob_sidecar + .kzg_commitment_inclusion_proof + .len() + .saturating_sub(T::all_kzg_commitments_inclusion_proof_depth()); + let kzg_commitments_inclusion_proof: FixedVector< + Hash256, + T::AllKzgCommitmentsInclusionProofDepth, + > = first_blob_sidecar.kzg_commitment_inclusion_proof.to_vec()[body_proof_start..] + .to_vec() + .into(); + + let mut rng = StdRng::seed_from_u64(slot.as_u64()); + + (0..T::number_of_blob_columns()) + .map(|col_index| { + let index = col_index as u64; + let mut data = vec![0u8; T::bytes_per_column_sample()]; + // Prefix with column index + let prefix = index.to_le_bytes(); + data[..prefix.len()].copy_from_slice(&prefix); + // Fill the rest of the array with random values + rng.fill(&mut data[prefix.len()..]); + + Ok(BlobColumnSidecar { + index, + data: FixedVector::new(Vec::from(data)).map_err(|e| format!("{e:?}"))?, + signed_block_header: first_blob_sidecar.signed_block_header.clone(), + kzg_commitments: blob_sidecars + .iter() + .map(|b| b.kzg_commitment) + .collect::>() + .into(), + kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), + }) + }) + .collect::, _>>() + } + + pub fn slot(&self) -> Slot { + self.signed_block_header.message.slot + } + + pub fn block_root(&self) -> Hash256 { + self.signed_block_header.message.tree_hash_root() + } +} + +#[cfg(test)] +mod test { + use crate::beacon_block::EmptyBlock; + use crate::beacon_block_body::KzgCommitments; + use crate::eth_spec::EthSpec; + use crate::{ + BeaconBlock, BeaconBlockDeneb, Blob, BlobColumnSidecar, BlobSidecar, BlobSidecarList, + ChainSpec, MainnetEthSpec, SignedBeaconBlock, + }; + use bls::Signature; + use kzg::{KzgCommitment, KzgProof}; + use std::sync::Arc; + + #[test] + fn test_random_from_blob_sidecars() { + type E = MainnetEthSpec; + let num_of_blobs = 6; + let spec = E::default_spec(); + let blob_sidecars: BlobSidecarList = create_test_blob_sidecars(num_of_blobs, &spec); + + let column_sidecars = BlobColumnSidecar::random_from_blob_sidecars(&blob_sidecars).unwrap(); + + assert_eq!(column_sidecars.len(), E::number_of_blob_columns()); + + for (idx, col_sidecar) in column_sidecars.iter().enumerate() { + assert_eq!(col_sidecar.index, idx as u64); + assert_eq!(col_sidecar.kzg_commitments.len(), num_of_blobs); + // ensure column sidecars are prefixed with column index (for verification purpose in prototype only) + let prefix_len = 8; // column index (u64) is stored as the first 8 bytes + let col_index_prefix = + u64::from_le_bytes(col_sidecar.data[0..prefix_len].try_into().unwrap()); + assert_eq!(col_index_prefix, idx as u64) + } + } + + fn create_test_blob_sidecars( + num_of_blobs: usize, + spec: &ChainSpec, + ) -> BlobSidecarList { + let mut block = BeaconBlock::Deneb(BeaconBlockDeneb::empty(&spec)); + let mut body = block.body_mut(); + let blob_kzg_commitments = body.blob_kzg_commitments_mut().unwrap(); + *blob_kzg_commitments = + KzgCommitments::::new(vec![KzgCommitment::empty_for_testing(); num_of_blobs]) + .unwrap(); + + let signed_block = SignedBeaconBlock::from_block(block, Signature::empty()); + + (0..num_of_blobs) + .into_iter() + .map(|index| { + BlobSidecar::new( + index, + Blob::::default(), + &signed_block, + KzgProof::empty(), + ) + .map(Arc::new) + }) + .collect::, _>>() + .unwrap() + .into() + } +} diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 17baad9c4c7..b7e028678fb 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -4,7 +4,7 @@ use safe_arith::SafeArith; use serde::{Deserialize, Serialize}; use ssz_types::typenum::{ bit::B0, UInt, Unsigned, U0, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, U16, - U16777216, U2, U2048, U256, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, U8192, + U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, U8192, }; use ssz_types::typenum::{U17, U9}; use std::fmt::{self, Debug}; @@ -111,6 +111,13 @@ pub trait EthSpec: type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq; type KzgCommitmentInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; + /* + * New in PeerDAS + */ + type NumberOfBlobColumns: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type BytesPerColumnSample: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type BytesPerRowSample: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type AllKzgCommitmentsInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) */ @@ -277,6 +284,22 @@ pub trait EthSpec: fn kzg_proof_inclusion_proof_depth() -> usize { Self::KzgCommitmentInclusionProofDepth::to_usize() } + + fn number_of_blob_columns() -> usize { + Self::NumberOfBlobColumns::to_usize() + } + + fn bytes_per_column_sample() -> usize { + Self::BytesPerColumnSample::to_usize() + } + + fn bytes_per_row_sample() -> usize { + Self::BytesPerRowSample::to_usize() + } + + fn all_kzg_commitments_inclusion_proof_depth() -> usize { + Self::AllKzgCommitmentsInclusionProofDepth::to_usize() + } } /// Macro to inherit some type values from another EthSpec. @@ -322,6 +345,16 @@ impl EthSpec for MainnetEthSpec { type FieldElementsPerBlob = U4096; type BytesPerBlob = U131072; type KzgCommitmentInclusionProofDepth = U17; + type NumberOfBlobColumns = U128; + // Column samples are entire columns in 1D DAS. + // data size = row_size * num_of_rows / num_of_columns + // 256kb * 32 / 128 = 64kb + type BytesPerColumnSample = U65536; + // A 1D extended blob / row. + // data size = blob_size * 2 + // 128kb * 2 = 256kb + type BytesPerRowSample = U262144; + type AllKzgCommitmentsInclusionProofDepth = U4; // inclusion of the whole list of commitments type SyncSubcommitteeSize = U128; // 512 committee size / 4 sync committee subnet count type MaxPendingAttestations = U4096; // 128 max attestations * 32 slots per epoch type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch @@ -356,6 +389,11 @@ impl EthSpec for MinimalEthSpec { type BytesPerBlob = U131072; type MaxBlobCommitmentsPerBlock = U16; type KzgCommitmentInclusionProofDepth = U9; + // DAS spec values copied from `MainnetEthSpec` + type NumberOfBlobColumns = U128; + type BytesPerColumnSample = U65536; + type BytesPerRowSample = U262144; + type AllKzgCommitmentsInclusionProofDepth = U4; params_from_eth_spec!(MainnetEthSpec { JustificationBitsLength, @@ -430,6 +468,11 @@ impl EthSpec for GnosisEthSpec { type BytesPerFieldElement = U32; type BytesPerBlob = U131072; type KzgCommitmentInclusionProofDepth = U17; + // DAS spec values copied from `MainnetEthSpec` + type NumberOfBlobColumns = U128; + type BytesPerColumnSample = U65536; + type BytesPerRowSample = U262144; + type AllKzgCommitmentsInclusionProofDepth = U4; fn default_spec() -> ChainSpec { ChainSpec::gnosis() diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 52be2fff46a..6fa57a055de 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,6 +98,7 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; +pub mod blob_sample; pub mod blob_sidecar; pub mod light_client_header; pub mod non_zero_usize; @@ -120,6 +121,7 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; +pub use crate::blob_sample::BlobColumnSidecar; pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList, BlobsList}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; From e340737e9d0ccece246a0cb72e6aa76fc647a06f Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 20 Dec 2023 17:45:13 +1100 Subject: [PATCH 2/9] Add blob column subnets --- beacon_node/lighthouse_network/src/service/mod.rs | 5 ++++- beacon_node/lighthouse_network/src/service/utils.rs | 4 ++++ beacon_node/lighthouse_network/src/types/subnet.rs | 3 +++ beacon_node/lighthouse_network/src/types/topics.rs | 6 ++++-- consensus/types/src/blob_sample.rs | 2 ++ consensus/types/src/consts.rs | 2 ++ consensus/types/src/subnet_id.rs | 3 ++- 7 files changed, 21 insertions(+), 4 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index d15bde37d90..88c6bf817e5 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -41,7 +41,8 @@ use std::{ }; use types::ForkName; use types::{ - consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, consts::deneb::BLOB_SIDECAR_SUBNET_COUNT, + consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, + consts::deneb::{BLOB_COLUMN_SUBNET_COUNT, BLOB_SIDECAR_SUBNET_COUNT}, EnrForkId, EthSpec, ForkContext, Slot, SubnetId, }; use utils::{build_transport, strip_peer_id, Context as ServiceContext, MAX_CONNECTIONS_PER_PEER}; @@ -227,6 +228,7 @@ impl Network { let max_topics = ctx.chain_spec.attestation_subnet_count as usize + SYNC_COMMITTEE_SUBNET_COUNT as usize + BLOB_SIDECAR_SUBNET_COUNT as usize + + BLOB_COLUMN_SUBNET_COUNT as usize + BASE_CORE_TOPICS.len() + ALTAIR_CORE_TOPICS.len() + CAPELLA_CORE_TOPICS.len() @@ -240,6 +242,7 @@ impl Network { ctx.chain_spec.attestation_subnet_count, SYNC_COMMITTEE_SUBNET_COUNT, BLOB_SIDECAR_SUBNET_COUNT, + BLOB_COLUMN_SUBNET_COUNT, ), // during a fork we subscribe to both the old and new topics max_subscribed_topics: max_topics * 4, diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 5fe5946ce29..88638a495ce 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -233,6 +233,7 @@ pub(crate) fn create_whitelist_filter( attestation_subnet_count: u64, sync_committee_subnet_count: u64, blob_sidecar_subnet_count: u64, + blob_column_subnet_count: u64, ) -> gossipsub::WhitelistSubscriptionFilter { let mut possible_hashes = HashSet::new(); for fork_digest in possible_fork_digests { @@ -261,6 +262,9 @@ pub(crate) fn create_whitelist_filter( for id in 0..blob_sidecar_subnet_count { add(BlobSidecar(id)); } + for id in 0..blob_column_subnet_count { + add(BlobColumnSidecar(SubnetId::new(id))); + } } gossipsub::WhitelistSubscriptionFilter(possible_hashes) } diff --git a/beacon_node/lighthouse_network/src/types/subnet.rs b/beacon_node/lighthouse_network/src/types/subnet.rs index 50d28542bec..4522f30fe70 100644 --- a/beacon_node/lighthouse_network/src/types/subnet.rs +++ b/beacon_node/lighthouse_network/src/types/subnet.rs @@ -12,6 +12,9 @@ pub enum Subnet { Attestation(SubnetId), /// Represents a gossipsub sync committee subnet and the metadata `syncnets` field. SyncCommittee(SyncSubnetId), + // Represents a gossipsub blob column subnet and the metadata `blbcolnets` field. + // TODO(das) + // BlobColumn(SubnetId), } /// A subnet to discover peers on along with the instant after which it's no longer useful. diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index aa548e4a2dd..bc32c6ae5f0 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -1,7 +1,7 @@ use libp2p::gossipsub::{IdentTopic as Topic, TopicHash}; use serde::{Deserialize, Serialize}; use strum::AsRefStr; -use types::consts::deneb::BLOB_SIDECAR_SUBNET_COUNT; +use types::consts::deneb::{BLOB_COLUMN_SUBNET_COUNT, BLOB_SIDECAR_SUBNET_COUNT}; use types::{EthSpec, ForkName, SubnetId, SyncSubnetId}; use crate::Subnet; @@ -134,7 +134,7 @@ impl std::fmt::Display for GossipKind { write!(f, "{}{}", BLOB_SIDECAR_PREFIX, blob_index) } GossipKind::BlobColumnSidecar(column_index) => { - write!(f, "{}{}", BLOB_COLUMN_SIDECAR_PREFIX, column_index) + write!(f, "{}{}", BLOB_COLUMN_SIDECAR_PREFIX, **column_index) } x => f.write_str(x.as_ref()), } @@ -223,6 +223,8 @@ impl GossipTopic { match self.kind() { GossipKind::Attestation(subnet_id) => Some(Subnet::Attestation(*subnet_id)), GossipKind::SyncCommitteeMessage(subnet_id) => Some(Subnet::SyncCommittee(*subnet_id)), + // TODO(das) + // GossipKind::BlobColumnSidecar(subnet_id) => Some(Subnet::BlobColumn(*subnet_id)), _ => None, } } diff --git a/consensus/types/src/blob_sample.rs b/consensus/types/src/blob_sample.rs index 7e215c6b245..b75b81e5b6f 100644 --- a/consensus/types/src/blob_sample.rs +++ b/consensus/types/src/blob_sample.rs @@ -32,7 +32,9 @@ pub struct BlobColumnSidecar { #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")] pub data: FixedVector, pub signed_block_header: SignedBeaconBlockHeader, + /// All of the KZG commitments associated with the block. pub kzg_commitments: KzgCommitments, + /// An inclusion proof, proving the inclusion of `blob_kzg_commitments` in `BeaconBlockBody`. pub kzg_commitments_inclusion_proof: FixedVector, // List of cell proofs proving each column sample is part of the extended blob. diff --git a/consensus/types/src/consts.rs b/consensus/types/src/consts.rs index f93c75ee8d8..bcf4b6fbc61 100644 --- a/consensus/types/src/consts.rs +++ b/consensus/types/src/consts.rs @@ -29,4 +29,6 @@ pub mod deneb { pub const BLOB_SIDECAR_SUBNET_COUNT: u64 = 6; pub const MAX_BLOBS_PER_BLOCK: u64 = BLOB_SIDECAR_SUBNET_COUNT; pub const MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: Epoch = Epoch::new(4096); + /// DAS constants + pub const BLOB_COLUMN_SUBNET_COUNT: u64 = 128; } diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 2752e31092e..c9927408065 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -5,7 +5,8 @@ use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; use swap_or_not_shuffle::compute_shuffled_index; -const MAX_SUBNET_ID: usize = 64; +// TODO(das): do we reuse this for das subnets? +const MAX_SUBNET_ID: usize = 128; lazy_static! { static ref SUBNET_ID_TO_STRING: Vec = { From 22bc8b0def23e2b6e13e5ef5ca28492d36e0f9b2 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 21 Dec 2023 02:26:01 +1100 Subject: [PATCH 3/9] Add `BlobColumnSubnetId` and initial compute subnet logic. --- .../lighthouse_network/src/service/utils.rs | 6 +- .../lighthouse_network/src/types/pubsub.rs | 14 +- .../lighthouse_network/src/types/subnet.rs | 2 +- .../lighthouse_network/src/types/topics.rs | 8 +- .../gossip_methods.rs | 8 +- .../src/network_beacon_processor/mod.rs | 2 +- ...{blob_sample.rs => blob_column_sidecar.rs} | 19 ++- consensus/types/src/blob_column_subnet_id.rs | 151 ++++++++++++++++++ consensus/types/src/chain_spec.rs | 15 +- consensus/types/src/lib.rs | 6 +- consensus/types/src/subnet_id.rs | 3 +- 11 files changed, 204 insertions(+), 30 deletions(-) rename consensus/types/src/{blob_sample.rs => blob_column_sidecar.rs} (90%) create mode 100644 consensus/types/src/blob_column_subnet_id.rs diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 88638a495ce..57dfb3d7738 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -20,7 +20,9 @@ use std::io::prelude::*; use std::path::Path; use std::sync::Arc; use std::time::Duration; -use types::{ChainSpec, EnrForkId, EthSpec, ForkContext, SubnetId, SyncSubnetId}; +use types::{ + BlobColumnSubnetId, ChainSpec, EnrForkId, EthSpec, ForkContext, SubnetId, SyncSubnetId, +}; pub const NETWORK_KEY_FILENAME: &str = "key"; /// The maximum simultaneous libp2p connections per peer. @@ -263,7 +265,7 @@ pub(crate) fn create_whitelist_filter( add(BlobSidecar(id)); } for id in 0..blob_column_subnet_count { - add(BlobColumnSidecar(SubnetId::new(id))); + add(BlobColumnSidecar(BlobColumnSubnetId::new(id))); } } gossipsub::WhitelistSubscriptionFilter(possible_hashes) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 44d7b30110e..20cb549a41a 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -9,12 +9,12 @@ use std::boxed::Box; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::{ - Attestation, AttesterSlashing, BlobColumnSidecar, BlobSidecar, EthSpec, ForkContext, ForkName, - LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, - SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, - SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockMerge, - SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, - SyncCommitteeMessage, SyncSubnetId, + Attestation, AttesterSlashing, BlobColumnSidecar, BlobColumnSubnetId, BlobSidecar, EthSpec, + ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, + ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, + SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, + SignedBeaconBlockMerge, SignedBlsToExecutionChange, SignedContributionAndProof, + SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; #[derive(Debug, Clone, PartialEq)] @@ -24,7 +24,7 @@ pub enum PubsubMessage { /// Gossipsub message providing notification of a [`BlobSidecar`] along with the subnet id where it was received. BlobSidecar(Box<(u64, Arc>)>), /// Gossipsub message providing notification of a [`BlobColumnSidecar`] along with the subnet id where it was received. - BlobColumnSidecar(Box<(SubnetId, Arc>)>), + BlobColumnSidecar(Box<(BlobColumnSubnetId, Arc>)>), /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a raw un-aggregated attestation with its shard id. diff --git a/beacon_node/lighthouse_network/src/types/subnet.rs b/beacon_node/lighthouse_network/src/types/subnet.rs index 4522f30fe70..fd91a35f11f 100644 --- a/beacon_node/lighthouse_network/src/types/subnet.rs +++ b/beacon_node/lighthouse_network/src/types/subnet.rs @@ -14,7 +14,7 @@ pub enum Subnet { SyncCommittee(SyncSubnetId), // Represents a gossipsub blob column subnet and the metadata `blbcolnets` field. // TODO(das) - // BlobColumn(SubnetId), + // BlobColumn(BlobColumnSubnetId), } /// A subnet to discover peers on along with the instant after which it's no longer useful. diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index bc32c6ae5f0..757215e53b2 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -1,8 +1,8 @@ use libp2p::gossipsub::{IdentTopic as Topic, TopicHash}; use serde::{Deserialize, Serialize}; use strum::AsRefStr; -use types::consts::deneb::{BLOB_COLUMN_SUBNET_COUNT, BLOB_SIDECAR_SUBNET_COUNT}; -use types::{EthSpec, ForkName, SubnetId, SyncSubnetId}; +use types::consts::deneb::BLOB_SIDECAR_SUBNET_COUNT; +use types::{BlobColumnSubnetId, EthSpec, ForkName, SubnetId, SyncSubnetId}; use crate::Subnet; @@ -100,7 +100,7 @@ pub enum GossipKind { /// Topic for publishing BlobSidecars. BlobSidecar(u64), /// Topic for publishing BlobColumnSidecars. - BlobColumnSidecar(SubnetId), + BlobColumnSidecar(BlobColumnSubnetId), /// Topic for publishing raw attestations on a particular subnet. #[strum(serialize = "beacon_attestation")] Attestation(SubnetId), @@ -310,7 +310,7 @@ fn subnet_topic_index(topic: &str) -> Option { } else if let Some(index) = topic.strip_prefix(BLOB_SIDECAR_PREFIX) { return Some(GossipKind::BlobSidecar(index.parse::().ok()?)); } else if let Some(index) = topic.strip_prefix(BLOB_COLUMN_SIDECAR_PREFIX) { - return Some(GossipKind::BlobColumnSidecar(SubnetId::new( + return Some(GossipKind::BlobColumnSidecar(BlobColumnSubnetId::new( index.parse::().ok()?, ))); } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index b3538b545f5..97a59267bf5 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -33,9 +33,9 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; use types::{ - Attestation, AttesterSlashing, BlobColumnSidecar, BlobSidecar, EthSpec, Hash256, - IndexedAttestation, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, - SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, + Attestation, AttesterSlashing, BlobColumnSidecar, BlobColumnSubnetId, BlobSidecar, EthSpec, + Hash256, IndexedAttestation, LightClientFinalityUpdate, LightClientOptimisticUpdate, + ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; @@ -606,7 +606,7 @@ impl NetworkBeaconProcessor { message_id: MessageId, peer_id: PeerId, _peer_client: Client, - subnet_id: SubnetId, + subnet_id: BlobColumnSubnetId, column_sidecar: Arc>, seen_duration: Duration, ) { diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 4e5bdf3e695..89082f65810 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -235,7 +235,7 @@ impl NetworkBeaconProcessor { message_id: MessageId, peer_id: PeerId, peer_client: Client, - subnet_id: SubnetId, + subnet_id: BlobColumnSubnetId, column_sidecar: Arc>, seen_timestamp: Duration, ) -> Result<(), Error> { diff --git a/consensus/types/src/blob_sample.rs b/consensus/types/src/blob_column_sidecar.rs similarity index 90% rename from consensus/types/src/blob_sample.rs rename to consensus/types/src/blob_column_sidecar.rs index b75b81e5b6f..2bd1ba80a08 100644 --- a/consensus/types/src/blob_sample.rs +++ b/consensus/types/src/blob_column_sidecar.rs @@ -60,7 +60,10 @@ impl BlobColumnSidecar { let kzg_commitments_inclusion_proof: FixedVector< Hash256, T::AllKzgCommitmentsInclusionProofDepth, - > = first_blob_sidecar.kzg_commitment_inclusion_proof.to_vec()[body_proof_start..] + > = first_blob_sidecar + .kzg_commitment_inclusion_proof + .get(body_proof_start..) + .ok_or("kzg_commitment_inclusion_proof index out of bounds")? .to_vec() .into(); @@ -72,13 +75,18 @@ impl BlobColumnSidecar { let mut data = vec![0u8; T::bytes_per_column_sample()]; // Prefix with column index let prefix = index.to_le_bytes(); - data[..prefix.len()].copy_from_slice(&prefix); + data.get_mut(..prefix.len()) + .ok_or("blob column index out of bounds")? + .copy_from_slice(&prefix); // Fill the rest of the array with random values - rng.fill(&mut data[prefix.len()..]); + rng.fill( + data.get_mut(prefix.len()..) + .ok_or("blob column index out of bounds")?, + ); Ok(BlobColumnSidecar { index, - data: FixedVector::new(Vec::from(data)).map_err(|e| format!("{e:?}"))?, + data: FixedVector::new(data).map_err(|e| format!("{e:?}"))?, signed_block_header: first_blob_sidecar.signed_block_header.clone(), kzg_commitments: blob_sidecars .iter() @@ -139,7 +147,7 @@ mod test { num_of_blobs: usize, spec: &ChainSpec, ) -> BlobSidecarList { - let mut block = BeaconBlock::Deneb(BeaconBlockDeneb::empty(&spec)); + let mut block = BeaconBlock::Deneb(BeaconBlockDeneb::empty(spec)); let mut body = block.body_mut(); let blob_kzg_commitments = body.blob_kzg_commitments_mut().unwrap(); *blob_kzg_commitments = @@ -149,7 +157,6 @@ mod test { let signed_block = SignedBeaconBlock::from_block(block, Signature::empty()); (0..num_of_blobs) - .into_iter() .map(|index| { BlobSidecar::new( index, diff --git a/consensus/types/src/blob_column_subnet_id.rs b/consensus/types/src/blob_column_subnet_id.rs new file mode 100644 index 00000000000..0d4c7388ad4 --- /dev/null +++ b/consensus/types/src/blob_column_subnet_id.rs @@ -0,0 +1,151 @@ +//! Identifies each blob column subnet by an integer identifier. +use crate::consts::deneb::BLOB_COLUMN_SUBNET_COUNT; +use crate::{ChainSpec, EthSpec}; +use ethereum_types::U256; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; +use std::ops::{Deref, DerefMut}; + +lazy_static! { + static ref BLOB_COLUMN_SUBNET_ID_TO_STRING: Vec = { + let mut v = Vec::with_capacity(BLOB_COLUMN_SUBNET_COUNT as usize); + + for i in 0..BLOB_COLUMN_SUBNET_COUNT { + v.push(i.to_string()); + } + v + }; +} + +#[derive(arbitrary::Arbitrary, Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct BlobColumnSubnetId(#[serde(with = "serde_utils::quoted_u64")] u64); + +pub fn blob_column_subnet_id_to_string(i: u64) -> &'static str { + if i < BLOB_COLUMN_SUBNET_COUNT { + BLOB_COLUMN_SUBNET_ID_TO_STRING + .get(i as usize) + .expect("index below BLOB_COLUMN_SUBNET_COUNT") + } else { + "blob column subnet id out of range" + } +} + +impl BlobColumnSubnetId { + pub fn new(id: u64) -> Self { + id.into() + } + + #[allow(clippy::arithmetic_side_effects)] + /// Compute required subnets to subscribe to given the node id. + /// TODO(das): Add epoch + pub fn compute_subnets_for_blob_column( + node_id: U256, + spec: &ChainSpec, + ) -> impl Iterator { + let num_of_columns = T::number_of_blob_columns() as u64; + (0..spec.blob_custody_requirement) + .map(move |i| { + let node_offset = (node_id % U256::from(num_of_columns)).as_u64(); + node_offset.saturating_add(i) % num_of_columns + }) + .map(BlobColumnSubnetId::new) + } +} + +impl Display for BlobColumnSubnetId { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{}", self.0) + } +} + +impl Deref for BlobColumnSubnetId { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for BlobColumnSubnetId { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From for BlobColumnSubnetId { + fn from(x: u64) -> Self { + Self(x) + } +} + +impl Into for BlobColumnSubnetId { + fn into(self) -> u64 { + self.0 + } +} + +impl Into for &BlobColumnSubnetId { + fn into(self) -> u64 { + self.0 + } +} + +impl AsRef for BlobColumnSubnetId { + fn as_ref(&self) -> &str { + blob_column_subnet_id_to_string(self.0) + } +} + +#[cfg(test)] +mod test { + use crate::blob_column_subnet_id::BlobColumnSubnetId; + use crate::ChainSpec; + + #[test] + fn test_compute_subnets_for_blob_column() { + let node_ids = [ + "0", + "88752428858350697756262172400162263450541348766581994718383409852729519486397", + "18732750322395381632951253735273868184515463718109267674920115648614659369468", + "27726842142488109545414954493849224833670205008410190955613662332153332462900", + "39755236029158558527862903296867805548949739810920318269566095185775868999998", + "31899136003441886988955119620035330314647133604576220223892254902004850516297", + "58579998103852084482416614330746509727562027284701078483890722833654510444626", + "28248042035542126088870192155378394518950310811868093527036637864276176517397", + "60930578857433095740782970114409273483106482059893286066493409689627770333527", + "103822458477361691467064888613019442068586830412598673713899771287914656699997", + ] + .into_iter() + .map(|v| ethereum_types::U256::from_dec_str(v).unwrap()) + .collect::>(); + + let expected_subnets = vec![ + vec![0, 1], + vec![61, 62], + vec![124, 125], + vec![52, 53], + vec![62, 63], + vec![73, 74], + vec![82, 83], + vec![21, 22], + vec![87, 88], + vec![93, 94], + ]; + + let spec = ChainSpec::mainnet(); + + for x in 0..node_ids.len() { + let computed_subnets = BlobColumnSubnetId::compute_subnets_for_blob_column::< + crate::MainnetEthSpec, + >(node_ids[x], &spec); + + assert_eq!( + expected_subnets[x], + computed_subnets + .map(BlobColumnSubnetId::into) + .collect::>() + ); + } + } +} diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 784d98c1397..f4706bca709 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -167,6 +167,11 @@ pub struct ChainSpec { pub deneb_fork_version: [u8; 4], pub deneb_fork_epoch: Option, + /* + * DAS params + */ + pub blob_custody_requirement: u64, + /* * Networking */ @@ -643,6 +648,11 @@ impl ChainSpec { deneb_fork_version: [0x04, 0x00, 0x00, 0x00], deneb_fork_epoch: None, + /* + * DAS params + */ + blob_custody_requirement: 2, + /* * Network specific */ @@ -886,7 +896,10 @@ impl ChainSpec { */ deneb_fork_version: [0x04, 0x00, 0x00, 0x64], deneb_fork_epoch: None, - + /* + * DAS params + */ + blob_custody_requirement: 2, /* * Network specific */ diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 6fa57a055de..479978627ea 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,7 +98,8 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; -pub mod blob_sample; +pub mod blob_column_sidecar; +pub mod blob_column_subnet_id; pub mod blob_sidecar; pub mod light_client_header; pub mod non_zero_usize; @@ -121,7 +122,8 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blob_sample::BlobColumnSidecar; +pub use crate::blob_column_sidecar::BlobColumnSidecar; +pub use crate::blob_column_subnet_id::BlobColumnSubnetId; pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList, BlobsList}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index c9927408065..2752e31092e 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -5,8 +5,7 @@ use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; use swap_or_not_shuffle::compute_shuffled_index; -// TODO(das): do we reuse this for das subnets? -const MAX_SUBNET_ID: usize = 128; +const MAX_SUBNET_ID: usize = 64; lazy_static! { static ref SUBNET_ID_TO_STRING: Vec = { From 8e865004da6ad6c1ed548e3b1b814bf36e2d7c4f Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 22 Dec 2023 11:42:34 +1100 Subject: [PATCH 4/9] Subscribe to blob column subnets. --- beacon_node/beacon_chain/src/beacon_chain.rs | 14 ++++++ .../beacon_chain/src/blob_verification.rs | 6 ++- .../lighthouse_network/src/discovery/mod.rs | 3 ++ .../src/discovery/subnet_predicate.rs | 2 + .../src/peer_manager/mod.rs | 4 ++ .../src/peer_manager/peerdb/peer_info.rs | 2 + .../lighthouse_network/src/types/subnet.rs | 7 ++- .../lighthouse_network/src/types/topics.rs | 4 +- .../gossip_methods.rs | 4 +- beacon_node/network/src/service.rs | 44 ++++++++++++++++++- consensus/types/src/blob_column_subnet_id.rs | 2 +- 11 files changed, 80 insertions(+), 12 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index af9799cb13a..0bba3d48a78 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2882,6 +2882,20 @@ impl BeaconChain { self.remove_notified(&block_root, r) } + pub fn process_gossip_blob_column( + self: &Arc, + blob_column: GossipVerifiedBlobColumnSidecar, + ) { + let blob_column = blob_column.as_blob_column(); + // TODO(das) send to DA checker + info!( + self.log, + "Processed gossip blob column"; + "index" => blob_column.index, + "slot" => blob_column.slot().as_u64() + ); + } + /// Cache the blobs in the processing cache, process it, then evict it from the cache if it was /// imported or errors. pub async fn process_rpc_blobs( diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 7ebdbacf18c..625f7d5e7d6 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -206,6 +206,10 @@ impl GossipVerifiedBlobColumnSidecar { ) }) } + + pub fn as_blob_column(&self) -> &Arc> { + &self.blob_column_sidecar + } } /// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on @@ -666,7 +670,7 @@ pub fn validate_blob_column_sidecar_for_gossip( _subnet: u64, _chain: &BeaconChain, ) -> Result, GossipBlobError> { - // TODO(das): validate + // TODO(das): validate kzg commitments, cell proofs etc Ok(GossipVerifiedBlobColumnSidecar { blob_column_sidecar: blob_column_sidecar.clone(), }) diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index fbe686aa2a6..a1d4f68fcf1 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -521,6 +521,8 @@ impl Discovery { ) .map_err(|e| format!("{:?}", e))?; } + // TODO(das) discovery to be implemented at a later phase. Initially we just use a large peer count. + Subnet::BlobColumn(_) => return Ok(()), } // replace the global version @@ -840,6 +842,7 @@ impl Discovery { let query_str = match query.subnet { Subnet::Attestation(_) => "attestation", Subnet::SyncCommittee(_) => "sync_committee", + Subnet::BlobColumn(_) => "blob_column", }; if let Some(v) = metrics::get_int_counter( diff --git a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs index f79ff8daf69..b3c2d78331d 100644 --- a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs +++ b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs @@ -33,6 +33,8 @@ where Subnet::SyncCommittee(s) => sync_committee_bitfield .as_ref() .map_or(false, |b| b.get(*s.deref() as usize).unwrap_or(false)), + // TODO(das) discovery to be implemented at a later phase. Initially we just use a large peer count. + Subnet::BlobColumn(_) => false, }); if !predicate { diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 3459548ec31..c110e5db385 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -1065,6 +1065,10 @@ impl PeerManager { .or_default() .insert(id); } + // TODO(das) to be implemented. We're not pruning blob column peers yet + // because blob column topics are subscribed as core topics until we + // implement recomputing blob column subnets. + Subnet::BlobColumn(_) => {} } } } diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs index 44c54511ddc..0324bb6cdfb 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs @@ -94,6 +94,8 @@ impl PeerInfo { .syncnets() .map_or(false, |s| s.get(**id as usize).unwrap_or(false)) } + // TODO(das) Add blob column nets bitfield + Subnet::BlobColumn(_) => return false, } } false diff --git a/beacon_node/lighthouse_network/src/types/subnet.rs b/beacon_node/lighthouse_network/src/types/subnet.rs index fd91a35f11f..6fcf4f50fe0 100644 --- a/beacon_node/lighthouse_network/src/types/subnet.rs +++ b/beacon_node/lighthouse_network/src/types/subnet.rs @@ -1,6 +1,6 @@ use serde::Serialize; use std::time::Instant; -use types::{SubnetId, SyncSubnetId}; +use types::{BlobColumnSubnetId, SubnetId, SyncSubnetId}; /// Represents a subnet on an attestation or sync committee `SubnetId`. /// @@ -12,9 +12,8 @@ pub enum Subnet { Attestation(SubnetId), /// Represents a gossipsub sync committee subnet and the metadata `syncnets` field. SyncCommittee(SyncSubnetId), - // Represents a gossipsub blob column subnet and the metadata `blbcolnets` field. - // TODO(das) - // BlobColumn(BlobColumnSubnetId), + /// Represents a gossipsub blob column subnet and the metadata `blbcolnets` field. + BlobColumn(BlobColumnSubnetId), } /// A subnet to discover peers on along with the instant after which it's no longer useful. diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 757215e53b2..f630f2865e7 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -223,8 +223,7 @@ impl GossipTopic { match self.kind() { GossipKind::Attestation(subnet_id) => Some(Subnet::Attestation(*subnet_id)), GossipKind::SyncCommitteeMessage(subnet_id) => Some(Subnet::SyncCommittee(*subnet_id)), - // TODO(das) - // GossipKind::BlobColumnSidecar(subnet_id) => Some(Subnet::BlobColumn(*subnet_id)), + GossipKind::BlobColumnSidecar(subnet_id) => Some(Subnet::BlobColumn(*subnet_id)), _ => None, } } @@ -286,6 +285,7 @@ impl From for GossipKind { match subnet_id { Subnet::Attestation(s) => GossipKind::Attestation(s), Subnet::SyncCommittee(s) => GossipKind::SyncCommitteeMessage(s), + Subnet::BlobColumn(s) => GossipKind::BlobColumnSidecar(s), } } } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 97a59267bf5..307dbd5f92e 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -870,11 +870,11 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_verified_blob_column( self: &Arc, _peer_id: PeerId, - _verified_blob: GossipVerifiedBlobColumnSidecar, + verified_blob_column: GossipVerifiedBlobColumnSidecar, // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { - todo!() + self.chain.process_gossip_blob_column(verified_blob_column); } /// Process the beacon block received from the gossip network and: diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 17760cef592..ddc67156107 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -34,8 +34,8 @@ use task_executor::ShutdownReason; use tokio::sync::mpsc; use tokio::time::Sleep; use types::{ - ChainSpec, EthSpec, ForkContext, Slot, SubnetId, SyncCommitteeSubscription, SyncSubnetId, - Unsigned, ValidatorSubscription, + BlobColumnSubnetId, ChainSpec, EthSpec, ForkContext, Slot, SubnetId, SyncCommitteeSubscription, + SyncSubnetId, Unsigned, ValidatorSubscription, }; mod tests; @@ -752,6 +752,29 @@ impl NetworkService { } } + if !self.subscribe_all_subnets { + for column_subnet in + BlobColumnSubnetId::compute_subnets_for_blob_column::( + self.network_globals.local_enr().node_id().raw().into(), + &self.beacon_chain.spec, + ) + { + for fork_digest in self.required_gossip_fork_digests() { + let gossip_kind = Subnet::BlobColumn(column_subnet).into(); + let topic = GossipTopic::new( + gossip_kind, + GossipEncoding::default(), + fork_digest, + ); + if self.libp2p.subscribe(topic.clone()) { + subscribed_topics.push(topic); + } else { + warn!(self.log, "Could not subscribe to topic"; "topic" => %topic); + } + } + } + } + // If we are to subscribe to all subnets we do it here if self.subscribe_all_subnets { for subnet_id in 0..<::EthSpec as EthSpec>::SubnetBitfieldLength::to_u64() { @@ -785,6 +808,23 @@ impl NetworkService { } } } + // Subscribe to all blob column subnets + for column_subnet in 0..T::EthSpec::number_of_blob_columns() as u64 { + for fork_digest in self.required_gossip_fork_digests() { + let gossip_kind = + Subnet::BlobColumn(BlobColumnSubnetId::new(column_subnet)).into(); + let topic = GossipTopic::new( + gossip_kind, + GossipEncoding::default(), + fork_digest, + ); + if self.libp2p.subscribe(topic.clone()) { + subscribed_topics.push(topic); + } else { + warn!(self.log, "Could not subscribe to topic"; "topic" => %topic); + } + } + } } if !subscribed_topics.is_empty() { diff --git a/consensus/types/src/blob_column_subnet_id.rs b/consensus/types/src/blob_column_subnet_id.rs index 0d4c7388ad4..1c62c3d629b 100644 --- a/consensus/types/src/blob_column_subnet_id.rs +++ b/consensus/types/src/blob_column_subnet_id.rs @@ -38,7 +38,7 @@ impl BlobColumnSubnetId { #[allow(clippy::arithmetic_side_effects)] /// Compute required subnets to subscribe to given the node id. - /// TODO(das): Add epoch + /// TODO(das): Add epoch param pub fn compute_subnets_for_blob_column( node_id: U256, spec: &ChainSpec, From 985c98c1cc2df717a026bca1de9e985cf91af3eb Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 22 Dec 2023 17:23:00 +1100 Subject: [PATCH 5/9] Introduce `BLOB_COLUMN_SUBNET_COUNT` based on DAS configuration parameter changes. --- beacon_node/http_api/src/publish_blocks.rs | 15 ++++-- .../gossip_methods.rs | 4 +- beacon_node/network/src/service.rs | 2 +- consensus/types/src/blob_column_sidecar.rs | 4 +- consensus/types/src/blob_column_subnet_id.rs | 49 ++++++++++++++----- consensus/types/src/chain_spec.rs | 4 +- consensus/types/src/consts.rs | 2 +- consensus/types/src/eth_spec.rs | 20 +++++--- 8 files changed, 69 insertions(+), 31 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index d7aca6a76bf..7ba66a9b299 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -19,9 +19,9 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, BlobColumnSidecar, BlobSidecarList, EthSpec, ExecPayload, - ExecutionBlockHash, ForkName, FullPayload, FullPayloadMerge, Hash256, SignedBeaconBlock, - SignedBlindedBeaconBlock, VariableList, + AbstractExecPayload, BeaconBlockRef, BlobColumnSidecar, BlobColumnSubnetId, BlobSidecarList, + EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, FullPayloadMerge, Hash256, + SignedBeaconBlock, SignedBlindedBeaconBlock, VariableList, }; use warp::http::StatusCode; use warp::{reply::Response, Rejection, Reply}; @@ -97,7 +97,14 @@ pub async fn publish_block(col_index) + .map_err(|e| { + BlockError::BeaconChainError( + BeaconChainError::UnableToBuildBlobColumnSidecar(format!( + "{e:?}" + )), + ) + })?, Arc::new(col_sidecar), )))); } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 307dbd5f92e..f88375c550a 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -612,7 +612,7 @@ impl NetworkBeaconProcessor { ) { let slot = column_sidecar.slot(); let root = column_sidecar.block_root(); - let index = *subnet_id; + let index = column_sidecar.index; let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); // Log metrics to track delay from other nodes on the network. metrics::observe_duration( @@ -625,7 +625,7 @@ impl NetworkBeaconProcessor { ); match self .chain - .verify_blob_column_sidecar_for_gossip(column_sidecar, index) + .verify_blob_column_sidecar_for_gossip(column_sidecar, *subnet_id) { Ok(gossip_verified_blob_column) => { metrics::inc_counter( diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index ddc67156107..d6efbda73f7 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -809,7 +809,7 @@ impl NetworkService { } } // Subscribe to all blob column subnets - for column_subnet in 0..T::EthSpec::number_of_blob_columns() as u64 { + for column_subnet in 0..T::EthSpec::blob_column_subnet_count() as u64 { for fork_digest in self.required_gossip_fork_digests() { let gossip_kind = Subnet::BlobColumn(BlobColumnSubnetId::new(column_subnet)).into(); diff --git a/consensus/types/src/blob_column_sidecar.rs b/consensus/types/src/blob_column_sidecar.rs index 2bd1ba80a08..3f49172d0c2 100644 --- a/consensus/types/src/blob_column_sidecar.rs +++ b/consensus/types/src/blob_column_sidecar.rs @@ -69,7 +69,7 @@ impl BlobColumnSidecar { let mut rng = StdRng::seed_from_u64(slot.as_u64()); - (0..T::number_of_blob_columns()) + (0..T::blob_column_count()) .map(|col_index| { let index = col_index as u64; let mut data = vec![0u8; T::bytes_per_column_sample()]; @@ -130,7 +130,7 @@ mod test { let column_sidecars = BlobColumnSidecar::random_from_blob_sidecars(&blob_sidecars).unwrap(); - assert_eq!(column_sidecars.len(), E::number_of_blob_columns()); + assert_eq!(column_sidecars.len(), E::blob_column_count()); for (idx, col_sidecar) in column_sidecars.iter().enumerate() { assert_eq!(col_sidecar.index, idx as u64); diff --git a/consensus/types/src/blob_column_subnet_id.rs b/consensus/types/src/blob_column_subnet_id.rs index 1c62c3d629b..cd465fef951 100644 --- a/consensus/types/src/blob_column_subnet_id.rs +++ b/consensus/types/src/blob_column_subnet_id.rs @@ -2,6 +2,7 @@ use crate::consts::deneb::BLOB_COLUMN_SUBNET_COUNT; use crate::{ChainSpec, EthSpec}; use ethereum_types::U256; +use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; use std::ops::{Deref, DerefMut}; @@ -36,6 +37,16 @@ impl BlobColumnSubnetId { id.into() } + pub fn try_from_column_index(column_index: usize) -> Result { + let cols_per_subnet = T::blob_column_count().safe_div(T::blob_column_subnet_count())?; + let subnet_id = column_index.safe_div(cols_per_subnet)?; + if subnet_id < T::blob_column_subnet_count() { + Ok((subnet_id as u64).into()) + } else { + Err(Error::InvalidColumn(column_index)) + } + } + #[allow(clippy::arithmetic_side_effects)] /// Compute required subnets to subscribe to given the node id. /// TODO(das): Add epoch param @@ -43,11 +54,11 @@ impl BlobColumnSubnetId { node_id: U256, spec: &ChainSpec, ) -> impl Iterator { - let num_of_columns = T::number_of_blob_columns() as u64; + let num_of_column_subnets = T::blob_column_subnet_count() as u64; (0..spec.blob_custody_requirement) .map(move |i| { - let node_offset = (node_id % U256::from(num_of_columns)).as_u64(); - node_offset.saturating_add(i) % num_of_columns + let node_offset = (node_id % U256::from(num_of_column_subnets)).as_u64(); + node_offset.saturating_add(i) % num_of_column_subnets }) .map(BlobColumnSubnetId::new) } @@ -97,6 +108,18 @@ impl AsRef for BlobColumnSubnetId { } } +#[derive(Debug)] +pub enum Error { + ArithError(ArithError), + InvalidColumn(usize), +} + +impl From for Error { + fn from(e: ArithError) -> Self { + Error::ArithError(e) + } +} + #[cfg(test)] mod test { use crate::blob_column_subnet_id::BlobColumnSubnetId; @@ -121,16 +144,16 @@ mod test { .collect::>(); let expected_subnets = vec![ - vec![0, 1], - vec![61, 62], - vec![124, 125], - vec![52, 53], - vec![62, 63], - vec![73, 74], - vec![82, 83], - vec![21, 22], - vec![87, 88], - vec![93, 94], + vec![0], + vec![29], + vec![28], + vec![20], + vec![30], + vec![9], + vec![18], + vec![21], + vec![23], + vec![29], ]; let spec = ChainSpec::mainnet(); diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index f4706bca709..22d50413fc0 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -651,7 +651,7 @@ impl ChainSpec { /* * DAS params */ - blob_custody_requirement: 2, + blob_custody_requirement: 1, /* * Network specific @@ -899,7 +899,7 @@ impl ChainSpec { /* * DAS params */ - blob_custody_requirement: 2, + blob_custody_requirement: 1, /* * Network specific */ diff --git a/consensus/types/src/consts.rs b/consensus/types/src/consts.rs index bcf4b6fbc61..493e901d924 100644 --- a/consensus/types/src/consts.rs +++ b/consensus/types/src/consts.rs @@ -30,5 +30,5 @@ pub mod deneb { pub const MAX_BLOBS_PER_BLOCK: u64 = BLOB_SIDECAR_SUBNET_COUNT; pub const MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: Epoch = Epoch::new(4096); /// DAS constants - pub const BLOB_COLUMN_SUBNET_COUNT: u64 = 128; + pub const BLOB_COLUMN_SUBNET_COUNT: u64 = 32; } diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index b7e028678fb..a4f2cbd9c6b 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -114,7 +114,8 @@ pub trait EthSpec: /* * New in PeerDAS */ - type NumberOfBlobColumns: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type BlobColumnSubnetCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type BlobColumnCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerColumnSample: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerRowSample: Unsigned + Clone + Sync + Send + Debug + PartialEq; type AllKzgCommitmentsInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; @@ -285,8 +286,12 @@ pub trait EthSpec: Self::KzgCommitmentInclusionProofDepth::to_usize() } - fn number_of_blob_columns() -> usize { - Self::NumberOfBlobColumns::to_usize() + fn blob_column_count() -> usize { + Self::BlobColumnCount::to_usize() + } + + fn blob_column_subnet_count() -> usize { + Self::BlobColumnSubnetCount::to_usize() } fn bytes_per_column_sample() -> usize { @@ -345,7 +350,8 @@ impl EthSpec for MainnetEthSpec { type FieldElementsPerBlob = U4096; type BytesPerBlob = U131072; type KzgCommitmentInclusionProofDepth = U17; - type NumberOfBlobColumns = U128; + type BlobColumnSubnetCount = U32; + type BlobColumnCount = U128; // Column samples are entire columns in 1D DAS. // data size = row_size * num_of_rows / num_of_columns // 256kb * 32 / 128 = 64kb @@ -390,7 +396,8 @@ impl EthSpec for MinimalEthSpec { type MaxBlobCommitmentsPerBlock = U16; type KzgCommitmentInclusionProofDepth = U9; // DAS spec values copied from `MainnetEthSpec` - type NumberOfBlobColumns = U128; + type BlobColumnSubnetCount = U32; + type BlobColumnCount = U128; type BytesPerColumnSample = U65536; type BytesPerRowSample = U262144; type AllKzgCommitmentsInclusionProofDepth = U4; @@ -469,7 +476,8 @@ impl EthSpec for GnosisEthSpec { type BytesPerBlob = U131072; type KzgCommitmentInclusionProofDepth = U17; // DAS spec values copied from `MainnetEthSpec` - type NumberOfBlobColumns = U128; + type BlobColumnSubnetCount = U32; + type BlobColumnCount = U128; type BytesPerColumnSample = U65536; type BytesPerRowSample = U262144; type AllKzgCommitmentsInclusionProofDepth = U4; From d6d6a64893d0620271933572fe2a9397727d135e Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 3 Jan 2024 14:10:28 +1100 Subject: [PATCH 6/9] Fix column sidecar type to use `VariableList` for data. --- consensus/types/src/blob_column_sidecar.rs | 12 +++--- consensus/types/src/blob_column_subnet_id.rs | 1 + consensus/types/src/eth_spec.rs | 39 +++++++++++--------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/consensus/types/src/blob_column_sidecar.rs b/consensus/types/src/blob_column_sidecar.rs index 3f49172d0c2..ecc7bd54146 100644 --- a/consensus/types/src/blob_column_sidecar.rs +++ b/consensus/types/src/blob_column_sidecar.rs @@ -6,7 +6,7 @@ use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use ssz_types::FixedVector; +use ssz_types::{FixedVector, VariableList}; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -29,8 +29,8 @@ use tree_hash_derive::TreeHash; pub struct BlobColumnSidecar { #[serde(with = "serde_utils::quoted_u64")] pub index: u64, - #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")] - pub data: FixedVector, + #[serde(with = "ssz_types::serde_utils::hex_var_list")] + pub data: VariableList, pub signed_block_header: SignedBeaconBlockHeader, /// All of the KZG commitments associated with the block. pub kzg_commitments: KzgCommitments, @@ -68,11 +68,13 @@ impl BlobColumnSidecar { .into(); let mut rng = StdRng::seed_from_u64(slot.as_u64()); + let num_of_blobs = blob_sidecars.len(); + let bytes_per_column = T::bytes_per_extended_blob() * num_of_blobs / T::blob_column_count(); (0..T::blob_column_count()) .map(|col_index| { let index = col_index as u64; - let mut data = vec![0u8; T::bytes_per_column_sample()]; + let mut data = vec![0u8; bytes_per_column]; // Prefix with column index let prefix = index.to_le_bytes(); data.get_mut(..prefix.len()) @@ -86,7 +88,7 @@ impl BlobColumnSidecar { Ok(BlobColumnSidecar { index, - data: FixedVector::new(data).map_err(|e| format!("{e:?}"))?, + data: VariableList::new(data).map_err(|e| format!("{e:?}"))?, signed_block_header: first_blob_sidecar.signed_block_header.clone(), kzg_commitments: blob_sidecars .iter() diff --git a/consensus/types/src/blob_column_subnet_id.rs b/consensus/types/src/blob_column_subnet_id.rs index cd465fef951..ce02c7ba30d 100644 --- a/consensus/types/src/blob_column_subnet_id.rs +++ b/consensus/types/src/blob_column_subnet_id.rs @@ -50,6 +50,7 @@ impl BlobColumnSubnetId { #[allow(clippy::arithmetic_side_effects)] /// Compute required subnets to subscribe to given the node id. /// TODO(das): Add epoch param + /// TODO(das): Add num of subnets (from ENR) pub fn compute_subnets_for_blob_column( node_id: U256, spec: &ChainSpec, diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index a4f2cbd9c6b..d1300a2e3fc 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -116,8 +116,7 @@ pub trait EthSpec: */ type BlobColumnSubnetCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BlobColumnCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; - type BytesPerColumnSample: Unsigned + Clone + Sync + Send + Debug + PartialEq; - type BytesPerRowSample: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxBytesPerColumn: Unsigned + Clone + Sync + Send + Debug + PartialEq; type AllKzgCommitmentsInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) @@ -142,6 +141,11 @@ pub trait EthSpec: /// Must be set to `BytesPerFieldElement * FieldElementsPerBlob`. type BytesPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; + /// The total length of an extended blob in bytes. + /// + /// Must be set to `BytesPerBlob * 2`. + type BytesPerExtendedBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; + fn default_spec() -> ChainSpec; fn spec_name() -> EthSpecId; @@ -281,6 +285,12 @@ pub trait EthSpec: fn bytes_per_blob() -> usize { Self::BytesPerBlob::to_usize() } + + /// Returns the `BYTES_PER_EXTENDED_BLOB` constant for this specification. + fn bytes_per_extended_blob() -> usize { + Self::BytesPerExtendedBlob::to_usize() + } + /// Returns the `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH` preset for this specification. fn kzg_proof_inclusion_proof_depth() -> usize { Self::KzgCommitmentInclusionProofDepth::to_usize() @@ -294,12 +304,8 @@ pub trait EthSpec: Self::BlobColumnSubnetCount::to_usize() } - fn bytes_per_column_sample() -> usize { - Self::BytesPerColumnSample::to_usize() - } - - fn bytes_per_row_sample() -> usize { - Self::BytesPerRowSample::to_usize() + fn max_bytes_per_column() -> usize { + Self::MaxBytesPerColumn::to_usize() } fn all_kzg_commitments_inclusion_proof_depth() -> usize { @@ -349,17 +355,14 @@ impl EthSpec for MainnetEthSpec { type BytesPerFieldElement = U32; type FieldElementsPerBlob = U4096; type BytesPerBlob = U131072; + type BytesPerExtendedBlob = U262144; type KzgCommitmentInclusionProofDepth = U17; type BlobColumnSubnetCount = U32; type BlobColumnCount = U128; // Column samples are entire columns in 1D DAS. - // data size = row_size * num_of_rows / num_of_columns + // max data size = extended_blob_bytes * max_blobs_per_block / num_of_columns // 256kb * 32 / 128 = 64kb - type BytesPerColumnSample = U65536; - // A 1D extended blob / row. - // data size = blob_size * 2 - // 128kb * 2 = 256kb - type BytesPerRowSample = U262144; + type MaxBytesPerColumn = U65536; type AllKzgCommitmentsInclusionProofDepth = U4; // inclusion of the whole list of commitments type SyncSubcommitteeSize = U128; // 512 committee size / 4 sync committee subnet count type MaxPendingAttestations = U4096; // 128 max attestations * 32 slots per epoch @@ -393,13 +396,13 @@ impl EthSpec for MinimalEthSpec { type MaxWithdrawalsPerPayload = U4; type FieldElementsPerBlob = U4096; type BytesPerBlob = U131072; + type BytesPerExtendedBlob = U262144; type MaxBlobCommitmentsPerBlock = U16; type KzgCommitmentInclusionProofDepth = U9; // DAS spec values copied from `MainnetEthSpec` type BlobColumnSubnetCount = U32; type BlobColumnCount = U128; - type BytesPerColumnSample = U65536; - type BytesPerRowSample = U262144; + type MaxBytesPerColumn = U65536; type AllKzgCommitmentsInclusionProofDepth = U4; params_from_eth_spec!(MainnetEthSpec { @@ -474,12 +477,12 @@ impl EthSpec for GnosisEthSpec { type FieldElementsPerBlob = U4096; type BytesPerFieldElement = U32; type BytesPerBlob = U131072; + type BytesPerExtendedBlob = U262144; type KzgCommitmentInclusionProofDepth = U17; // DAS spec values copied from `MainnetEthSpec` type BlobColumnSubnetCount = U32; type BlobColumnCount = U128; - type BytesPerColumnSample = U65536; - type BytesPerRowSample = U262144; + type MaxBytesPerColumn = U65536; type AllKzgCommitmentsInclusionProofDepth = U4; fn default_spec() -> ChainSpec { From ed6fa22b962df6f5ca49af0781dae1097347b051 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 9 Jan 2024 16:51:38 +1100 Subject: [PATCH 7/9] Fix lint errors. --- beacon_node/http_api/src/publish_blocks.rs | 2 +- consensus/types/src/blob_column_sidecar.rs | 41 ++++++++++++++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index efc8be03ca2..5f8d09d2110 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -92,7 +92,7 @@ pub async fn publish_block { impl BlobColumnSidecar { pub fn random_from_blob_sidecars( blob_sidecars: &BlobSidecarList, - ) -> Result>, String> { + ) -> Result>, BlobColumnSidecarError> { if blob_sidecars.is_empty() { return Ok(vec![]); } - let first_blob_sidecar = blob_sidecars.first().ok_or("should exist")?; + let first_blob_sidecar = blob_sidecars + .first() + .ok_or(BlobColumnSidecarError::MissingBlobSidecars)?; let slot = first_blob_sidecar.slot(); // Proof for kzg commitments in `BeaconBlockBody` @@ -63,13 +67,15 @@ impl BlobColumnSidecar { > = first_blob_sidecar .kzg_commitment_inclusion_proof .get(body_proof_start..) - .ok_or("kzg_commitment_inclusion_proof index out of bounds")? + .ok_or(BlobColumnSidecarError::KzgCommitmentInclusionProofOutOfBounds)? .to_vec() .into(); let mut rng = StdRng::seed_from_u64(slot.as_u64()); let num_of_blobs = blob_sidecars.len(); - let bytes_per_column = T::bytes_per_extended_blob() * num_of_blobs / T::blob_column_count(); + let bytes_per_column = T::bytes_per_extended_blob() + .safe_div(T::blob_column_count())? + .safe_mul(num_of_blobs)?; (0..T::blob_column_count()) .map(|col_index| { @@ -78,17 +84,17 @@ impl BlobColumnSidecar { // Prefix with column index let prefix = index.to_le_bytes(); data.get_mut(..prefix.len()) - .ok_or("blob column index out of bounds")? + .ok_or(BlobColumnSidecarError::BlobColumnIndexOutOfBounds)? .copy_from_slice(&prefix); // Fill the rest of the array with random values rng.fill( data.get_mut(prefix.len()..) - .ok_or("blob column index out of bounds")?, + .ok_or(BlobColumnSidecarError::BlobColumnIndexOutOfBounds)?, ); Ok(BlobColumnSidecar { index, - data: VariableList::new(data).map_err(|e| format!("{e:?}"))?, + data: VariableList::new(data)?, signed_block_header: first_blob_sidecar.signed_block_header.clone(), kzg_commitments: blob_sidecars .iter() @@ -110,6 +116,27 @@ impl BlobColumnSidecar { } } +#[derive(Debug)] +pub enum BlobColumnSidecarError { + ArithError(ArithError), + MissingBlobSidecars, + KzgCommitmentInclusionProofOutOfBounds, + BlobColumnIndexOutOfBounds, + SszError(SszError), +} + +impl From for BlobColumnSidecarError { + fn from(e: ArithError) -> Self { + Self::ArithError(e) + } +} + +impl From for BlobColumnSidecarError { + fn from(e: SszError) -> Self { + Self::SszError(e) + } +} + #[cfg(test)] mod test { use crate::beacon_block::EmptyBlock; From b9a970bf09669c1c49006c0cc5250ceabc061728 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 2 Feb 2024 22:45:48 +1100 Subject: [PATCH 8/9] Update types and naming to latest consensus-spec #3574. --- beacon_node/beacon_chain/src/beacon_chain.rs | 26 ++-- .../beacon_chain/src/blob_verification.rs | 26 ++-- beacon_node/beacon_chain/src/errors.rs | 2 +- beacon_node/beacon_chain/src/metrics.rs | 10 +- beacon_node/beacon_processor/src/lib.rs | 22 ++-- beacon_node/beacon_processor/src/metrics.rs | 8 +- beacon_node/http_api/src/publish_blocks.rs | 24 ++-- .../lighthouse_network/src/discovery/mod.rs | 4 +- .../src/discovery/subnet_predicate.rs | 2 +- .../src/peer_manager/mod.rs | 8 +- .../src/peer_manager/peerdb/peer_info.rs | 4 +- .../src/service/gossip_cache.rs | 2 +- .../lighthouse_network/src/service/mod.rs | 4 +- .../lighthouse_network/src/service/utils.rs | 8 +- .../lighthouse_network/src/types/pubsub.rs | 24 ++-- .../lighthouse_network/src/types/subnet.rs | 6 +- .../lighthouse_network/src/types/topics.rs | 24 ++-- beacon_node/network/src/metrics.rs | 24 ++-- .../gossip_methods.rs | 36 +++--- .../src/network_beacon_processor/mod.rs | 12 +- beacon_node/network/src/router.rs | 4 +- beacon_node/network/src/service.rs | 12 +- consensus/types/src/chain_spec.rs | 24 ++-- ...lumn_sidecar.rs => data_column_sidecar.rs} | 112 ++++++++++-------- ..._subnet_id.rs => data_column_subnet_id.rs} | 71 +++++------ consensus/types/src/eth_spec.rs | 60 +++++----- consensus/types/src/lib.rs | 8 +- consensus/types/src/preset.rs | 11 ++ 28 files changed, 296 insertions(+), 282 deletions(-) rename consensus/types/src/{blob_column_sidecar.rs => data_column_sidecar.rs} (62%) rename consensus/types/src/{blob_column_subnet_id.rs => data_column_subnet_id.rs} (62%) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f99ab80045e..f67129b3299 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -8,7 +8,7 @@ use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache} use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_verification::{ - GossipBlobError, GossipVerifiedBlob, GossipVerifiedBlobColumnSidecar, + GossipBlobError, GossipVerifiedBlob, GossipVerifiedDataColumnSidecar, }; use crate::block_times_cache::BlockTimesCache; use crate::block_verification::POS_PANDA_BANNER; @@ -2057,15 +2057,15 @@ impl BeaconChain { }) } - pub fn verify_blob_column_sidecar_for_gossip( + pub fn verify_data_column_sidecar_for_gossip( self: &Arc, - blob_column_sidecar: Arc>, + data_column_sidecar: Arc>, subnet_id: u64, - ) -> Result, GossipBlobError> { + ) -> Result, GossipBlobError> { metrics::inc_counter(&metrics::BLOBS_COLUMN_SIDECAR_PROCESSING_REQUESTS); - let _timer = metrics::start_timer(&metrics::BLOBS_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); - GossipVerifiedBlobColumnSidecar::new(blob_column_sidecar, subnet_id, self).map(|v| { - metrics::inc_counter(&metrics::BLOB_COLUMNS_SIDECAR_PROCESSING_SUCCESSES); + let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); + GossipVerifiedDataColumnSidecar::new(data_column_sidecar, subnet_id, self).map(|v| { + metrics::inc_counter(&metrics::DATA_COLUMNS_SIDECAR_PROCESSING_SUCCESSES); v }) } @@ -2885,17 +2885,17 @@ impl BeaconChain { self.remove_notified(&block_root, r) } - pub fn process_gossip_blob_column( + pub fn process_gossip_data_column( self: &Arc, - blob_column: GossipVerifiedBlobColumnSidecar, + gossip_verified_data_column: GossipVerifiedDataColumnSidecar, ) { - let blob_column = blob_column.as_blob_column(); + let data_column = gossip_verified_data_column.as_data_column(); // TODO(das) send to DA checker info!( self.log, - "Processed gossip blob column"; - "index" => blob_column.index, - "slot" => blob_column.slot().as_u64() + "Processed gossip data column"; + "index" => data_column.index, + "slot" => data_column.slot().as_u64() ); } diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index f2434363e8c..ffc64f9d10e 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -17,7 +17,7 @@ use ssz_types::VariableList; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; use types::{ - BeaconStateError, BlobColumnSidecar, BlobSidecar, CloneConfig, EthSpec, Hash256, + BeaconStateError, BlobSidecar, CloneConfig, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, }; @@ -186,20 +186,20 @@ pub type GossipVerifiedBlobList = VariableList< >; #[derive(Debug)] -pub struct GossipVerifiedBlobColumnSidecar { - blob_column_sidecar: Arc>, +pub struct GossipVerifiedDataColumnSidecar { + data_column_sidecar: Arc>, } -impl GossipVerifiedBlobColumnSidecar { +impl GossipVerifiedDataColumnSidecar { pub fn new( - column_sidecar: Arc>, + column_sidecar: Arc>, subnet_id: u64, chain: &BeaconChain, ) -> Result> { let header = column_sidecar.signed_block_header.clone(); // We only process slashing info if the gossip verification failed // since we do not process the blob any further in that case. - validate_blob_column_sidecar_for_gossip(column_sidecar, subnet_id, chain).map_err(|e| { + validate_data_column_sidecar_for_gossip(column_sidecar, subnet_id, chain).map_err(|e| { process_block_slash_info::<_, GossipBlobError>( chain, BlockSlashInfo::from_early_error_blob(header, e), @@ -207,8 +207,8 @@ impl GossipVerifiedBlobColumnSidecar { }) } - pub fn as_blob_column(&self) -> &Arc> { - &self.blob_column_sidecar + pub fn as_data_column(&self) -> &Arc> { + &self.data_column_sidecar } } @@ -675,14 +675,14 @@ pub fn validate_blob_sidecar_for_gossip( }) } -pub fn validate_blob_column_sidecar_for_gossip( - blob_column_sidecar: Arc>, +pub fn validate_data_column_sidecar_for_gossip( + data_column_sidecar: Arc>, _subnet: u64, _chain: &BeaconChain, -) -> Result, GossipBlobError> { +) -> Result, GossipBlobError> { // TODO(das): validate kzg commitments, cell proofs etc - Ok(GossipVerifiedBlobColumnSidecar { - blob_column_sidecar: blob_column_sidecar.clone(), + Ok(GossipVerifiedDataColumnSidecar { + data_column_sidecar: data_column_sidecar.clone(), }) } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index f9313b3ec8c..92ab669126b 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -220,7 +220,7 @@ pub enum BeaconChainError { InconsistentFork(InconsistentFork), ProposerHeadForkChoiceError(fork_choice::Error), UnableToPublish, - UnableToBuildBlobColumnSidecar(String), + UnableToBuildDataColumnSidecar(String), AvailabilityCheckError(AvailabilityCheckError), LightClientError(LightClientError), UnsupportedFork, diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 7f8382147b8..4a2187be610 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1016,23 +1016,23 @@ lazy_static! { ); pub static ref BLOBS_COLUMN_SIDECAR_PROCESSING_REQUESTS: Result = try_create_int_counter( "beacon_blobs_column_sidecar_processing_requests_total", - "Count of all blob column sidecars submitted for processing" + "Count of all data column sidecars submitted for processing" ); pub static ref BLOBS_SIDECAR_PROCESSING_SUCCESSES: Result = try_create_int_counter( "beacon_blobs_sidecar_processing_successes_total", "Number of blob sidecars verified for gossip" ); - pub static ref BLOB_COLUMNS_SIDECAR_PROCESSING_SUCCESSES: Result = try_create_int_counter( + pub static ref DATA_COLUMNS_SIDECAR_PROCESSING_SUCCESSES: Result = try_create_int_counter( "beacon_blobs_column_sidecar_processing_successes_total", - "Number of blob column sidecars verified for gossip" + "Number of data column sidecars verified for gossip" ); pub static ref BLOBS_SIDECAR_GOSSIP_VERIFICATION_TIMES: Result = try_create_histogram( "beacon_blobs_sidecar_gossip_verification_seconds", "Full runtime of blob sidecars gossip verification" ); - pub static ref BLOBS_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: Result = try_create_histogram( + pub static ref DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: Result = try_create_histogram( "beacon_blobs_column_sidecar_gossip_verification_seconds", - "Full runtime of blob column sidecars gossip verification" + "Full runtime of data column sidecars gossip verification" ); pub static ref BLOB_SIDECAR_INCLUSION_PROOF_VERIFICATION: Result = try_create_histogram( "blob_sidecar_inclusion_proof_verification_seconds", diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index e91196c9a4a..8b1c127d300 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -109,9 +109,9 @@ const MAX_GOSSIP_BLOCK_QUEUE_LEN: usize = 1_024; /// will be stored before we start dropping them. const MAX_GOSSIP_BLOB_QUEUE_LEN: usize = 1_024; -/// The maximum number of queued `BlobColumnSidecar` objects received on gossip that +/// The maximum number of queued `DataColumnSidecar` objects received on gossip that /// will be stored before we start dropping them. -const MAX_GOSSIP_BLOB_COL_QUEUE_LEN: usize = 1_024; +const MAX_GOSSIP_DATA_COL_QUEUE_LEN: usize = 1_024; /// The maximum number of queued `SignedBeaconBlock` objects received prior to their slot (but /// within acceptable clock disparity) that will be queued before we start dropping them. @@ -595,7 +595,7 @@ pub enum Work { }, GossipBlock(AsyncFn), GossipBlobSidecar(AsyncFn), - GossipBlobColumnSidecar(AsyncFn), + GossipDataColumnSidecar(AsyncFn), DelayedImportBlock { beacon_block_slot: Slot, beacon_block_root: Hash256, @@ -646,7 +646,7 @@ impl Work { Work::GossipAggregateBatch { .. } => GOSSIP_AGGREGATE_BATCH, Work::GossipBlock(_) => GOSSIP_BLOCK, Work::GossipBlobSidecar(_) => GOSSIP_BLOBS_SIDECAR, - Work::GossipBlobColumnSidecar(_) => GOSSIP_BLOBS_COLUMN_SIDECAR, + Work::GossipDataColumnSidecar(_) => GOSSIP_BLOBS_COLUMN_SIDECAR, Work::DelayedImportBlock { .. } => DELAYED_IMPORT_BLOCK, Work::GossipVoluntaryExit(_) => GOSSIP_VOLUNTARY_EXIT, Work::GossipProposerSlashing(_) => GOSSIP_PROPOSER_SLASHING, @@ -816,7 +816,7 @@ impl BeaconProcessor { let mut backfill_chain_segment = FifoQueue::new(MAX_CHAIN_SEGMENT_QUEUE_LEN); let mut gossip_block_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN); let mut gossip_blob_queue = FifoQueue::new(MAX_GOSSIP_BLOB_QUEUE_LEN); - let mut gossip_blob_column_queue = FifoQueue::new(MAX_GOSSIP_BLOB_COL_QUEUE_LEN); + let mut gossip_data_column_queue = FifoQueue::new(MAX_GOSSIP_DATA_COL_QUEUE_LEN); let mut delayed_block_queue = FifoQueue::new(MAX_DELAYED_BLOCK_QUEUE_LEN); let mut status_queue = FifoQueue::new(MAX_STATUS_QUEUE_LEN); @@ -972,7 +972,7 @@ impl BeaconProcessor { self.spawn_worker(item, idle_tx); } else if let Some(item) = gossip_blob_queue.pop() { self.spawn_worker(item, idle_tx); - } else if let Some(item) = gossip_blob_column_queue.pop() { + } else if let Some(item) = gossip_data_column_queue.pop() { self.spawn_worker(item, idle_tx); // Check the priority 0 API requests after blocks and blobs, but before attestations. } else if let Some(item) = api_request_p0_queue.pop() { @@ -1216,8 +1216,8 @@ impl BeaconProcessor { Work::GossipBlobSidecar { .. } => { gossip_blob_queue.push(work, work_id, &self.log) } - Work::GossipBlobColumnSidecar { .. } => { - gossip_blob_column_queue.push(work, work_id, &self.log) + Work::GossipDataColumnSidecar { .. } => { + gossip_data_column_queue.push(work, work_id, &self.log) } Work::DelayedImportBlock { .. } => { delayed_block_queue.push(work, work_id, &self.log) @@ -1318,8 +1318,8 @@ impl BeaconProcessor { gossip_blob_queue.len() as i64, ); metrics::set_gauge( - &metrics::BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_QUEUE_TOTAL, - gossip_blob_column_queue.len() as i64, + &metrics::BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_QUEUE_TOTAL, + gossip_data_column_queue.len() as i64, ); metrics::set_gauge( &metrics::BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL, @@ -1474,7 +1474,7 @@ impl BeaconProcessor { Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), Work::GossipBlock(work) | Work::GossipBlobSidecar(work) - | Work::GossipBlobColumnSidecar(work) => task_spawner.spawn_async(async move { + | Work::GossipDataColumnSidecar(work) => task_spawner.spawn_async(async move { work.await; }), Work::BlobsByRangeRequest(process_fn) | Work::BlobsByRootsRequest(process_fn) => { diff --git a/beacon_node/beacon_processor/src/metrics.rs b/beacon_node/beacon_processor/src/metrics.rs index 02dc40a6b1e..bcd422b357d 100644 --- a/beacon_node/beacon_processor/src/metrics.rs +++ b/beacon_node/beacon_processor/src/metrics.rs @@ -51,10 +51,10 @@ lazy_static::lazy_static! { "beacon_processor_gossip_blob_queue_total", "Count of blobs from gossip waiting to be verified." ); - // Gossip blob column sidecars. - pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_gossip_blob_column_queue_total", - "Count of blob column sidecars from gossip waiting to be verified." + // Gossip data column sidecars. + pub static ref BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_gossip_data_column_queue_total", + "Count of data column sidecars from gossip waiting to be verified." ); // Gossip Exits. pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result = try_create_int_gauge( diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 5f8d09d2110..75755adb83b 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -19,7 +19,7 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, BlobColumnSidecar, BlobColumnSubnetId, BlobSidecarList, + AbstractExecPayload, BeaconBlockRef, BlobSidecarList, DataColumnSidecar, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, FullPayloadMerge, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, VariableList, }; @@ -89,22 +89,22 @@ pub async fn publish_block(col_index) - .map_err(|e| { - BlockError::BeaconChainError( - BeaconChainError::UnableToBuildBlobColumnSidecar(format!( - "{e:?}" - )), - ) - })?, + pubsub_messages.push(PubsubMessage::DataColumnSidecar(Box::new(( + DataColumnSubnetId::from_column_index::(col_index), + // .map_err(|e| { + // BlockError::BeaconChainError( + // BeaconChainError::UnableToBuildDataColumnSidecar(format!( + // "{e:?}" + // )), + // ) + // })?, Arc::new(col_sidecar), )))); } diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index a1d4f68fcf1..109abe10c4b 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -522,7 +522,7 @@ impl Discovery { .map_err(|e| format!("{:?}", e))?; } // TODO(das) discovery to be implemented at a later phase. Initially we just use a large peer count. - Subnet::BlobColumn(_) => return Ok(()), + Subnet::DataColumn(_) => return Ok(()), } // replace the global version @@ -842,7 +842,7 @@ impl Discovery { let query_str = match query.subnet { Subnet::Attestation(_) => "attestation", Subnet::SyncCommittee(_) => "sync_committee", - Subnet::BlobColumn(_) => "blob_column", + Subnet::DataColumn(_) => "data_column", }; if let Some(v) = metrics::get_int_counter( diff --git a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs index b3c2d78331d..0b35465233a 100644 --- a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs +++ b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs @@ -34,7 +34,7 @@ where .as_ref() .map_or(false, |b| b.get(*s.deref() as usize).unwrap_or(false)), // TODO(das) discovery to be implemented at a later phase. Initially we just use a large peer count. - Subnet::BlobColumn(_) => false, + Subnet::DataColumn(_) => false, }); if !predicate { diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index c110e5db385..0531017bce1 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -1065,10 +1065,10 @@ impl PeerManager { .or_default() .insert(id); } - // TODO(das) to be implemented. We're not pruning blob column peers yet - // because blob column topics are subscribed as core topics until we - // implement recomputing blob column subnets. - Subnet::BlobColumn(_) => {} + // TODO(das) to be implemented. We're not pruning data column peers yet + // because data column topics are subscribed as core topics until we + // implement recomputing data column subnets. + Subnet::DataColumn(_) => {} } } } diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs index 0324bb6cdfb..6e0c00e42b8 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs @@ -94,8 +94,8 @@ impl PeerInfo { .syncnets() .map_or(false, |s| s.get(**id as usize).unwrap_or(false)) } - // TODO(das) Add blob column nets bitfield - Subnet::BlobColumn(_) => return false, + // TODO(das) Add data column nets bitfield + Subnet::DataColumn(_) => return false, } } false diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 407fc16d529..d0927283da7 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -194,7 +194,7 @@ impl GossipCache { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, GossipKind::BlobSidecar(_) => self.blob_sidecar, - GossipKind::BlobColumnSidecar(_) => self.blob_sidecar, + GossipKind::DataColumnSidecar(_) => self.blob_sidecar, GossipKind::BeaconAggregateAndProof => self.aggregates, GossipKind::Attestation(_) => self.attestation, GossipKind::VoluntaryExit => self.voluntary_exit, diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 56ad6482a60..c2b00617cd5 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -226,7 +226,7 @@ impl Network { let max_topics = ctx.chain_spec.attestation_subnet_count as usize + SYNC_COMMITTEE_SUBNET_COUNT as usize + ctx.chain_spec.blob_sidecar_subnet_count as usize - + ctx.chain_spec.blob_column_sidecar_subnet_count as usize + + ctx.chain_spec.data_column_sidecar_subnet_count as usize + BASE_CORE_TOPICS.len() + ALTAIR_CORE_TOPICS.len() + CAPELLA_CORE_TOPICS.len() @@ -240,7 +240,7 @@ impl Network { ctx.chain_spec.attestation_subnet_count, SYNC_COMMITTEE_SUBNET_COUNT, ctx.chain_spec.blob_sidecar_subnet_count, - ctx.chain_spec.blob_column_sidecar_subnet_count, + ctx.chain_spec.data_column_sidecar_subnet_count, ), // during a fork we subscribe to both the old and new topics max_subscribed_topics: max_topics * 4, diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 57dfb3d7738..bbe9059a41e 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -21,7 +21,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; use types::{ - BlobColumnSubnetId, ChainSpec, EnrForkId, EthSpec, ForkContext, SubnetId, SyncSubnetId, + ChainSpec, DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, SubnetId, SyncSubnetId, }; pub const NETWORK_KEY_FILENAME: &str = "key"; @@ -235,7 +235,7 @@ pub(crate) fn create_whitelist_filter( attestation_subnet_count: u64, sync_committee_subnet_count: u64, blob_sidecar_subnet_count: u64, - blob_column_subnet_count: u64, + data_column_subnet_count: u64, ) -> gossipsub::WhitelistSubscriptionFilter { let mut possible_hashes = HashSet::new(); for fork_digest in possible_fork_digests { @@ -264,8 +264,8 @@ pub(crate) fn create_whitelist_filter( for id in 0..blob_sidecar_subnet_count { add(BlobSidecar(id)); } - for id in 0..blob_column_subnet_count { - add(BlobColumnSidecar(BlobColumnSubnetId::new(id))); + for id in 0..data_column_subnet_count { + add(DataColumnSidecar(DataColumnSubnetId::new(id))); } } gossipsub::WhitelistSubscriptionFilter(possible_hashes) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 20cb549a41a..e9bccf59617 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -9,7 +9,7 @@ use std::boxed::Box; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::{ - Attestation, AttesterSlashing, BlobColumnSidecar, BlobColumnSubnetId, BlobSidecar, EthSpec, + Attestation, AttesterSlashing, BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, @@ -23,8 +23,8 @@ pub enum PubsubMessage { BeaconBlock(Arc>), /// Gossipsub message providing notification of a [`BlobSidecar`] along with the subnet id where it was received. BlobSidecar(Box<(u64, Arc>)>), - /// Gossipsub message providing notification of a [`BlobColumnSidecar`] along with the subnet id where it was received. - BlobColumnSidecar(Box<(BlobColumnSubnetId, Arc>)>), + /// Gossipsub message providing notification of a [`DataColumnSidecar`] along with the subnet id where it was received. + DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a raw un-aggregated attestation with its shard id. @@ -121,8 +121,8 @@ impl PubsubMessage { PubsubMessage::BlobSidecar(blob_sidecar_data) => { GossipKind::BlobSidecar(blob_sidecar_data.0) } - PubsubMessage::BlobColumnSidecar(column_sidecar_data) => { - GossipKind::BlobColumnSidecar(column_sidecar_data.0) + PubsubMessage::DataColumnSidecar(column_sidecar_data) => { + GossipKind::DataColumnSidecar(column_sidecar_data.0) } PubsubMessage::AggregateAndProofAttestation(_) => GossipKind::BeaconAggregateAndProof, PubsubMessage::Attestation(attestation_data) => { @@ -231,14 +231,14 @@ impl PubsubMessage { )), } } - GossipKind::BlobColumnSidecar(subnet_id) => { + GossipKind::DataColumnSidecar(subnet_id) => { match fork_context.from_context_bytes(gossip_topic.fork_digest) { Some(ForkName::Deneb) => { let col_sidecar = Arc::new( - BlobColumnSidecar::from_ssz_bytes(data) + DataColumnSidecar::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ); - Ok(PubsubMessage::BlobColumnSidecar(Box::new(( + Ok(PubsubMessage::DataColumnSidecar(Box::new(( *subnet_id, col_sidecar, )))) @@ -250,7 +250,7 @@ impl PubsubMessage { | ForkName::Capella, ) | None => Err(format!( - "blob_column_sidecar topic invalid for given fork digest {:?}", + "data_column_sidecar topic invalid for given fork digest {:?}", gossip_topic.fork_digest )), } @@ -324,7 +324,7 @@ impl PubsubMessage { match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), - PubsubMessage::BlobColumnSidecar(data) => data.1.as_ssz_bytes(), + PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), @@ -354,9 +354,9 @@ impl std::fmt::Display for PubsubMessage { data.1.slot(), data.1.index, ), - PubsubMessage::BlobColumnSidecar(data) => write!( + PubsubMessage::DataColumnSidecar(data) => write!( f, - "BlobColumnSidecar: slot: {}, column index: {}", + "DataColumnSidecar: slot: {}, column index: {}", data.1.slot(), data.1.index, ), diff --git a/beacon_node/lighthouse_network/src/types/subnet.rs b/beacon_node/lighthouse_network/src/types/subnet.rs index 6fcf4f50fe0..e814feefc70 100644 --- a/beacon_node/lighthouse_network/src/types/subnet.rs +++ b/beacon_node/lighthouse_network/src/types/subnet.rs @@ -1,6 +1,6 @@ use serde::Serialize; use std::time::Instant; -use types::{BlobColumnSubnetId, SubnetId, SyncSubnetId}; +use types::{DataColumnSubnetId, SubnetId, SyncSubnetId}; /// Represents a subnet on an attestation or sync committee `SubnetId`. /// @@ -12,8 +12,8 @@ pub enum Subnet { Attestation(SubnetId), /// Represents a gossipsub sync committee subnet and the metadata `syncnets` field. SyncCommittee(SyncSubnetId), - /// Represents a gossipsub blob column subnet and the metadata `blbcolnets` field. - BlobColumn(BlobColumnSubnetId), + /// Represents a gossipsub data column subnet and the metadata `blbcolnets` field. + DataColumn(DataColumnSubnetId), } /// A subnet to discover peers on along with the instant after which it's no longer useful. diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 0f003b56b71..dc1479e9f6a 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -1,7 +1,7 @@ use libp2p::gossipsub::{IdentTopic as Topic, TopicHash}; use serde::{Deserialize, Serialize}; use strum::AsRefStr; -use types::{BlobColumnSubnetId, ChainSpec, EthSpec, ForkName, SubnetId, SyncSubnetId}; +use types::{ChainSpec, DataColumnSubnetId, EthSpec, ForkName, SubnetId, SyncSubnetId}; use crate::Subnet; @@ -14,7 +14,7 @@ pub const BEACON_BLOCK_TOPIC: &str = "beacon_block"; pub const BEACON_AGGREGATE_AND_PROOF_TOPIC: &str = "beacon_aggregate_and_proof"; pub const BEACON_ATTESTATION_PREFIX: &str = "beacon_attestation_"; pub const BLOB_SIDECAR_PREFIX: &str = "blob_sidecar_"; -pub const BLOB_COLUMN_SIDECAR_PREFIX: &str = "blob_column_sidecar_"; +pub const DATA_COLUMN_SIDECAR_PREFIX: &str = "data_column_sidecar_"; pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit"; pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing"; pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing"; @@ -101,8 +101,8 @@ pub enum GossipKind { BeaconAggregateAndProof, /// Topic for publishing BlobSidecars. BlobSidecar(u64), - /// Topic for publishing BlobColumnSidecars. - BlobColumnSidecar(BlobColumnSubnetId), + /// Topic for publishing DataColumnSidecars. + DataColumnSidecar(DataColumnSubnetId), /// Topic for publishing raw attestations on a particular subnet. #[strum(serialize = "beacon_attestation")] Attestation(SubnetId), @@ -135,8 +135,8 @@ impl std::fmt::Display for GossipKind { GossipKind::BlobSidecar(blob_index) => { write!(f, "{}{}", BLOB_SIDECAR_PREFIX, blob_index) } - GossipKind::BlobColumnSidecar(column_index) => { - write!(f, "{}{}", BLOB_COLUMN_SIDECAR_PREFIX, **column_index) + GossipKind::DataColumnSidecar(column_index) => { + write!(f, "{}{}", DATA_COLUMN_SIDECAR_PREFIX, **column_index) } x => f.write_str(x.as_ref()), } @@ -225,7 +225,7 @@ impl GossipTopic { match self.kind() { GossipKind::Attestation(subnet_id) => Some(Subnet::Attestation(*subnet_id)), GossipKind::SyncCommitteeMessage(subnet_id) => Some(Subnet::SyncCommittee(*subnet_id)), - GossipKind::BlobColumnSidecar(subnet_id) => Some(Subnet::BlobColumn(*subnet_id)), + GossipKind::DataColumnSidecar(subnet_id) => Some(Subnet::DataColumn(*subnet_id)), _ => None, } } @@ -264,8 +264,8 @@ impl std::fmt::Display for GossipTopic { GossipKind::BlobSidecar(blob_index) => { format!("{}{}", BLOB_SIDECAR_PREFIX, blob_index) } - GossipKind::BlobColumnSidecar(index) => { - format!("{}{}", BLOB_COLUMN_SIDECAR_PREFIX, *index) + GossipKind::DataColumnSidecar(index) => { + format!("{}{}", DATA_COLUMN_SIDECAR_PREFIX, *index) } GossipKind::BlsToExecutionChange => BLS_TO_EXECUTION_CHANGE_TOPIC.into(), GossipKind::LightClientFinalityUpdate => LIGHT_CLIENT_FINALITY_UPDATE.into(), @@ -287,7 +287,7 @@ impl From for GossipKind { match subnet_id { Subnet::Attestation(s) => GossipKind::Attestation(s), Subnet::SyncCommittee(s) => GossipKind::SyncCommitteeMessage(s), - Subnet::BlobColumn(s) => GossipKind::BlobColumnSidecar(s), + Subnet::DataColumn(s) => GossipKind::DataColumnSidecar(s), } } } @@ -311,8 +311,8 @@ fn subnet_topic_index(topic: &str) -> Option { ))); } else if let Some(index) = topic.strip_prefix(BLOB_SIDECAR_PREFIX) { return Some(GossipKind::BlobSidecar(index.parse::().ok()?)); - } else if let Some(index) = topic.strip_prefix(BLOB_COLUMN_SIDECAR_PREFIX) { - return Some(GossipKind::BlobColumnSidecar(BlobColumnSubnetId::new( + } else if let Some(index) = topic.strip_prefix(DATA_COLUMN_SIDECAR_PREFIX) { + return Some(GossipKind::DataColumnSidecar(DataColumnSubnetId::new( index.parse::().ok()?, ))); } diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 56573e27624..11e02f5f3e7 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -71,9 +71,9 @@ lazy_static! { "beacon_processor_gossip_blob_verified_total", "Total number of gossip blob verified for propagation." ); - pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_SIDECAR_VERIFIED_TOTAL: Result = try_create_int_counter( - "beacon_processor_gossip_blob_column_verified_total", - "Total number of gossip blob column sidecar verified for propagation." + pub static ref BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: Result = try_create_int_counter( + "beacon_processor_gossip_data_column_verified_total", + "Total number of gossip data column sidecar verified for propagation." ); // Gossip Exits. pub static ref BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: Result = try_create_int_counter( @@ -287,9 +287,9 @@ lazy_static! { // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] decimal_buckets(-3,-1) ); - pub static ref BEACON_BLOB_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_blob_column_gossip_propagation_verification_delay_time", - "Duration between when the blob column sidecar is received over gossip and when it is verified for propagation.", + pub static ref BEACON_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: Result = try_create_histogram_with_buckets( + "beacon_data_column_gossip_propagation_verification_delay_time", + "Duration between when the data column sidecar is received over gossip and when it is verified for propagation.", // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] decimal_buckets(-3,-1) ); @@ -302,9 +302,9 @@ lazy_static! { // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] //decimal_buckets(-1,2) ); - pub static ref BEACON_BLOB_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_blob_column_gossip_slot_start_delay_time", - "Duration between when the blob column sidecar is received over gossip and the start of the slot it belongs to.", + pub static ref BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( + "beacon_data_column_gossip_slot_start_delay_time", + "Duration between when the data column sidecar is received over gossip and the start of the slot it belongs to.", // Create a custom bucket list for greater granularity in block delay Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) // NOTE: Previous values, which we may want to switch back to. @@ -325,9 +325,9 @@ lazy_static! { "beacon_blob_last_delay", "Keeps track of the last blob's delay from the start of the slot" ); - pub static ref BEACON_BLOB_COLUMN_LAST_DELAY: Result = try_create_int_gauge( - "beacon_blob_column_last_delay", - "Keeps track of the last blob column sidecar's delay from the start of the slot" + pub static ref BEACON_DATA_COLUMN_LAST_DELAY: Result = try_create_int_gauge( + "beacon_data_column_last_delay", + "Keeps track of the last data column sidecar's delay from the start of the slot" ); pub static ref BEACON_BLOB_GOSSIP_ARRIVED_LATE_TOTAL: Result = try_create_int_counter( diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index f88375c550a..2362b3d672f 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -5,7 +5,7 @@ use crate::{ sync::SyncMessage, }; use beacon_chain::blob_verification::{ - GossipBlobError, GossipVerifiedBlob, GossipVerifiedBlobColumnSidecar, + GossipBlobError, GossipVerifiedBlob, GossipVerifiedDataColumnSidecar, }; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::store::Error; @@ -33,7 +33,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; use types::{ - Attestation, AttesterSlashing, BlobColumnSidecar, BlobColumnSubnetId, BlobSidecar, EthSpec, + Attestation, AttesterSlashing, BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, IndexedAttestation, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId, SyncCommitteeMessage, @@ -601,13 +601,13 @@ impl NetworkBeaconProcessor { } } - pub async fn process_gossip_blob_column_sidecar( + pub async fn process_gossip_data_column_sidecar( self: &Arc, message_id: MessageId, peer_id: PeerId, _peer_client: Client, - subnet_id: BlobColumnSubnetId, - column_sidecar: Arc>, + subnet_id: DataColumnSubnetId, + column_sidecar: Arc>, seen_duration: Duration, ) { let slot = column_sidecar.slot(); @@ -616,25 +616,25 @@ impl NetworkBeaconProcessor { let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); // Log metrics to track delay from other nodes on the network. metrics::observe_duration( - &metrics::BEACON_BLOB_COLUMN_GOSSIP_SLOT_START_DELAY_TIME, + &metrics::BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME, delay, ); metrics::set_gauge( - &metrics::BEACON_BLOB_COLUMN_LAST_DELAY, + &metrics::BEACON_DATA_COLUMN_LAST_DELAY, delay.as_millis() as i64, ); match self .chain - .verify_blob_column_sidecar_for_gossip(column_sidecar, *subnet_id) + .verify_data_column_sidecar_for_gossip(column_sidecar, *subnet_id) { - Ok(gossip_verified_blob_column) => { + Ok(gossip_verified_data_column) => { metrics::inc_counter( - &metrics::BEACON_PROCESSOR_GOSSIP_BLOB_COLUMN_SIDECAR_VERIFIED_TOTAL, + &metrics::BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL, ); debug!( self.log, - "Successfully verified gossip blob column sidecar"; + "Successfully verified gossip data column sidecar"; "slot" => %slot, "root" => %root, "index" => %index, @@ -649,13 +649,13 @@ impl NetworkBeaconProcessor { .and_then(|now| now.checked_sub(seen_duration)) { metrics::observe_duration( - &metrics::BEACON_BLOB_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME, + &metrics::BEACON_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME, duration, ); } - self.process_gossip_verified_blob_column( + self.process_gossip_verified_data_column( peer_id, - gossip_verified_blob_column, + gossip_verified_data_column, seen_duration, ) .await @@ -663,7 +663,7 @@ impl NetworkBeaconProcessor { Err(err) => { error!( self.log, - "Internal error when verifying blob column sidecar"; + "Internal error when verifying data column sidecar"; "error" => ?err, ) } @@ -867,14 +867,14 @@ impl NetworkBeaconProcessor { } } - pub async fn process_gossip_verified_blob_column( + pub async fn process_gossip_verified_data_column( self: &Arc, _peer_id: PeerId, - verified_blob_column: GossipVerifiedBlobColumnSidecar, + verified_data_column: GossipVerifiedDataColumnSidecar, // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { - self.chain.process_gossip_blob_column(verified_blob_column); + self.chain.process_gossip_data_column(verified_data_column); } /// Process the beacon block received from the gossip network and: diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 89082f65810..42cab254412 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -229,20 +229,20 @@ impl NetworkBeaconProcessor { }) } - /// Create a new `Work` event for some blob column sidecar. - pub fn send_gossip_blob_column_sidecar( + /// Create a new `Work` event for some data column sidecar. + pub fn send_gossip_data_column_sidecar( self: &Arc, message_id: MessageId, peer_id: PeerId, peer_client: Client, - subnet_id: BlobColumnSubnetId, - column_sidecar: Arc>, + subnet_id: DataColumnSubnetId, + column_sidecar: Arc>, seen_timestamp: Duration, ) -> Result<(), Error> { let processor = self.clone(); let process_fn = async move { processor - .process_gossip_blob_column_sidecar( + .process_gossip_data_column_sidecar( message_id, peer_id, peer_client, @@ -255,7 +255,7 @@ impl NetworkBeaconProcessor { self.try_send(BeaconWorkEvent { drop_during_sync: false, - work: Work::GossipBlobColumnSidecar(Box::pin(process_fn)), + work: Work::GossipDataColumnSidecar(Box::pin(process_fn)), }) } diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index e0ddb4fcc36..924e9355d8c 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -308,11 +308,11 @@ impl Router { ), ) } - PubsubMessage::BlobColumnSidecar(data) => { + PubsubMessage::DataColumnSidecar(data) => { let (subnet_id, column_sidecar) = *data; self.handle_beacon_processor_send_result( self.network_beacon_processor - .send_gossip_blob_column_sidecar( + .send_gossip_data_column_sidecar( message_id, peer_id, self.network_globals.client(&peer_id), diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index ee07672e853..2186f8ac896 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -34,7 +34,7 @@ use task_executor::ShutdownReason; use tokio::sync::mpsc; use tokio::time::Sleep; use types::{ - BlobColumnSubnetId, ChainSpec, EthSpec, ForkContext, Slot, SubnetId, SyncCommitteeSubscription, + ChainSpec, DataColumnSubnetId, EthSpec, ForkContext, Slot, SubnetId, SyncCommitteeSubscription, SyncSubnetId, Unsigned, ValidatorSubscription, }; @@ -755,13 +755,13 @@ impl NetworkService { if !self.subscribe_all_subnets { for column_subnet in - BlobColumnSubnetId::compute_subnets_for_blob_column::( + DataColumnSubnetId::compute_subnets_for_data_column::( self.network_globals.local_enr().node_id().raw().into(), &self.beacon_chain.spec, ) { for fork_digest in self.required_gossip_fork_digests() { - let gossip_kind = Subnet::BlobColumn(column_subnet).into(); + let gossip_kind = Subnet::DataColumn(column_subnet).into(); let topic = GossipTopic::new( gossip_kind, GossipEncoding::default(), @@ -809,11 +809,11 @@ impl NetworkService { } } } - // Subscribe to all blob column subnets - for column_subnet in 0..T::EthSpec::blob_column_subnet_count() as u64 { + // Subscribe to all data column subnets + for column_subnet in 0..T::EthSpec::data_column_subnet_count() as u64 { for fork_digest in self.required_gossip_fork_digests() { let gossip_kind = - Subnet::BlobColumn(BlobColumnSubnetId::new(column_subnet)).into(); + Subnet::DataColumn(DataColumnSubnetId::new(column_subnet)).into(); let topic = GossipTopic::new( gossip_kind, GossipEncoding::default(), diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index c1ceec313e6..109d706b800 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -171,7 +171,7 @@ pub struct ChainSpec { /* * DAS params */ - pub blob_custody_requirement: u64, + pub custody_requirement: u64, /* * Networking @@ -202,7 +202,7 @@ pub struct ChainSpec { pub max_request_blob_sidecars: u64, pub min_epochs_for_blob_sidecars_requests: u64, pub blob_sidecar_subnet_count: u64, - pub blob_column_sidecar_subnet_count: u64, + pub data_column_sidecar_subnet_count: u64, /* * Networking Derived @@ -690,7 +690,7 @@ impl ChainSpec { /* * DAS params */ - blob_custody_requirement: 1, + custody_requirement: 1, /* * Network specific @@ -721,7 +721,7 @@ impl ChainSpec { max_request_blob_sidecars: default_max_request_blob_sidecars(), min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(), blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), - blob_column_sidecar_subnet_count: default_blob_column_sidecar_subnet_count(), + data_column_sidecar_subnet_count: default_data_column_sidecar_subnet_count(), /* * Derived Deneb Specific @@ -956,7 +956,7 @@ impl ChainSpec { /* * DAS params */ - blob_custody_requirement: 1, + custody_requirement: 1, /* * Network specific */ @@ -986,7 +986,7 @@ impl ChainSpec { max_request_blob_sidecars: default_max_request_blob_sidecars(), min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(), blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), - blob_column_sidecar_subnet_count: default_blob_column_sidecar_subnet_count(), + data_column_sidecar_subnet_count: default_data_column_sidecar_subnet_count(), /* * Derived Deneb Specific @@ -1167,9 +1167,9 @@ pub struct Config { #[serde(default = "default_blob_sidecar_subnet_count")] #[serde(with = "serde_utils::quoted_u64")] blob_sidecar_subnet_count: u64, - #[serde(default = "default_blob_column_sidecar_subnet_count")] + #[serde(default = "default_data_column_sidecar_subnet_count")] #[serde(with = "serde_utils::quoted_u64")] - blob_column_sidecar_subnet_count: u64, + data_column_sidecar_subnet_count: u64, } fn default_bellatrix_fork_version() -> [u8; 4] { @@ -1275,7 +1275,7 @@ const fn default_blob_sidecar_subnet_count() -> u64 { 6 } -const fn default_blob_column_sidecar_subnet_count() -> u64 { +const fn default_data_column_sidecar_subnet_count() -> u64 { 32 } @@ -1441,7 +1441,7 @@ impl Config { max_request_blob_sidecars: spec.max_request_blob_sidecars, min_epochs_for_blob_sidecars_requests: spec.min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count: spec.blob_sidecar_subnet_count, - blob_column_sidecar_subnet_count: spec.blob_column_sidecar_subnet_count, + data_column_sidecar_subnet_count: spec.data_column_sidecar_subnet_count, } } @@ -1506,7 +1506,7 @@ impl Config { max_request_blob_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, - blob_column_sidecar_subnet_count, + data_column_sidecar_subnet_count, } = self; if preset_base != T::spec_name().to_string().as_str() { @@ -1564,7 +1564,7 @@ impl Config { max_request_blob_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, - blob_column_sidecar_subnet_count, + data_column_sidecar_subnet_count, // We need to re-derive any values that might have changed in the config. max_blocks_by_root_request: max_blocks_by_root_request_common(max_request_blocks), diff --git a/consensus/types/src/blob_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs similarity index 62% rename from consensus/types/src/blob_column_sidecar.rs rename to consensus/types/src/data_column_sidecar.rs index ccf2909e77e..310c13a5e94 100644 --- a/consensus/types/src/blob_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,10 +1,10 @@ use crate::beacon_block_body::KzgCommitments; use crate::test_utils::TestRandom; -use crate::{BlobSidecarList, EthSpec, Hash256, SignedBeaconBlockHeader, Slot}; +use crate::{BlobSidecarList, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot}; use derivative::Derivative; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; -use safe_arith::{ArithError, SafeArith}; +use safe_arith::ArithError; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::Error as SszError; @@ -13,6 +13,10 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +pub type ColumnIndex = u64; +pub type Cell = FixedVector::FieldElementsPerCell>; +pub type DataColumn = VariableList, ::MaxBlobsPerBlock>; + #[derive( Debug, Clone, @@ -28,85 +32,95 @@ use tree_hash_derive::TreeHash; #[serde(bound = "T: EthSpec")] #[arbitrary(bound = "T: EthSpec")] #[derivative(PartialEq, Eq, Hash(bound = "T: EthSpec"))] -pub struct BlobColumnSidecar { +pub struct DataColumnSidecar { #[serde(with = "serde_utils::quoted_u64")] - pub index: u64, - #[serde(with = "ssz_types::serde_utils::hex_var_list")] - pub data: VariableList, - pub signed_block_header: SignedBeaconBlockHeader, - /// All of the KZG commitments associated with the block. + pub index: ColumnIndex, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + pub column: DataColumn, + /// All of the KZG commitments and proofs associated with the block, used for verifying sample cells. pub kzg_commitments: KzgCommitments, + pub kzg_proofs: KzgProofs, + pub signed_block_header: SignedBeaconBlockHeader, /// An inclusion proof, proving the inclusion of `blob_kzg_commitments` in `BeaconBlockBody`. - pub kzg_commitments_inclusion_proof: - FixedVector, - // List of cell proofs proving each column sample is part of the extended blob. - // pub cell_proofs: Vec, + pub kzg_commitments_inclusion_proof: FixedVector, } -impl BlobColumnSidecar { +impl DataColumnSidecar { pub fn random_from_blob_sidecars( blob_sidecars: &BlobSidecarList, - ) -> Result>, BlobColumnSidecarError> { + ) -> Result>, DataColumnSidecarError> { if blob_sidecars.is_empty() { return Ok(vec![]); } let first_blob_sidecar = blob_sidecars .first() - .ok_or(BlobColumnSidecarError::MissingBlobSidecars)?; + .ok_or(DataColumnSidecarError::MissingBlobSidecars)?; let slot = first_blob_sidecar.slot(); // Proof for kzg commitments in `BeaconBlockBody` let body_proof_start = first_blob_sidecar .kzg_commitment_inclusion_proof .len() - .saturating_sub(T::all_kzg_commitments_inclusion_proof_depth()); + .saturating_sub(T::kzg_commitments_inclusion_proof_depth()); let kzg_commitments_inclusion_proof: FixedVector< Hash256, - T::AllKzgCommitmentsInclusionProofDepth, + T::KzgCommitmentsInclusionProofDepth, > = first_blob_sidecar .kzg_commitment_inclusion_proof .get(body_proof_start..) - .ok_or(BlobColumnSidecarError::KzgCommitmentInclusionProofOutOfBounds)? + .ok_or(DataColumnSidecarError::KzgCommitmentInclusionProofOutOfBounds)? .to_vec() .into(); let mut rng = StdRng::seed_from_u64(slot.as_u64()); let num_of_blobs = blob_sidecars.len(); - let bytes_per_column = T::bytes_per_extended_blob() - .safe_div(T::blob_column_count())? - .safe_mul(num_of_blobs)?; - (0..T::blob_column_count()) + (0..T::number_of_columns()) .map(|col_index| { - let index = col_index as u64; - let mut data = vec![0u8; bytes_per_column]; - // Prefix with column index - let prefix = index.to_le_bytes(); - data.get_mut(..prefix.len()) - .ok_or(BlobColumnSidecarError::BlobColumnIndexOutOfBounds)? - .copy_from_slice(&prefix); - // Fill the rest of the array with random values - rng.fill( - data.get_mut(prefix.len()..) - .ok_or(BlobColumnSidecarError::BlobColumnIndexOutOfBounds)?, - ); - - Ok(BlobColumnSidecar { - index, - data: VariableList::new(data)?, - signed_block_header: first_blob_sidecar.signed_block_header.clone(), + Ok(DataColumnSidecar { + index: col_index as u64, + column: Self::generate_column_data(&mut rng, num_of_blobs, col_index)?, kzg_commitments: blob_sidecars .iter() .map(|b| b.kzg_commitment) .collect::>() .into(), + kzg_proofs: blob_sidecars + .iter() + .map(|b| b.kzg_proof) + .collect::>() + .into(), + signed_block_header: first_blob_sidecar.signed_block_header.clone(), kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), }) }) .collect::, _>>() } + fn generate_column_data( + rng: &mut StdRng, + num_of_blobs: usize, + index: usize, + ) -> Result, DataColumnSidecarError> { + let mut dummy_cell_data = Cell::::default(); + // Prefix with column index + let prefix = index.to_le_bytes(); + dummy_cell_data + .get_mut(..prefix.len()) + .ok_or(DataColumnSidecarError::DataColumnIndexOutOfBounds)? + .copy_from_slice(&prefix); + // Fill the rest of the vec with random values + rng.fill( + dummy_cell_data + .get_mut(prefix.len()..) + .ok_or(DataColumnSidecarError::DataColumnIndexOutOfBounds)?, + ); + + let column = DataColumn::::new(vec![dummy_cell_data; num_of_blobs])?; + Ok(column) + } + pub fn slot(&self) -> Slot { self.signed_block_header.message.slot } @@ -117,21 +131,21 @@ impl BlobColumnSidecar { } #[derive(Debug)] -pub enum BlobColumnSidecarError { +pub enum DataColumnSidecarError { ArithError(ArithError), MissingBlobSidecars, KzgCommitmentInclusionProofOutOfBounds, - BlobColumnIndexOutOfBounds, + DataColumnIndexOutOfBounds, SszError(SszError), } -impl From for BlobColumnSidecarError { +impl From for DataColumnSidecarError { fn from(e: ArithError) -> Self { Self::ArithError(e) } } -impl From for BlobColumnSidecarError { +impl From for DataColumnSidecarError { fn from(e: SszError) -> Self { Self::SszError(e) } @@ -143,8 +157,8 @@ mod test { use crate::beacon_block_body::KzgCommitments; use crate::eth_spec::EthSpec; use crate::{ - BeaconBlock, BeaconBlockDeneb, Blob, BlobColumnSidecar, BlobSidecar, BlobSidecarList, - ChainSpec, MainnetEthSpec, SignedBeaconBlock, + BeaconBlock, BeaconBlockDeneb, Blob, BlobSidecar, BlobSidecarList, ChainSpec, + DataColumnSidecar, MainnetEthSpec, SignedBeaconBlock, }; use bls::Signature; use kzg::{KzgCommitment, KzgProof}; @@ -157,17 +171,17 @@ mod test { let spec = E::default_spec(); let blob_sidecars: BlobSidecarList = create_test_blob_sidecars(num_of_blobs, &spec); - let column_sidecars = BlobColumnSidecar::random_from_blob_sidecars(&blob_sidecars).unwrap(); + let column_sidecars = DataColumnSidecar::random_from_blob_sidecars(&blob_sidecars).unwrap(); - assert_eq!(column_sidecars.len(), E::blob_column_count()); + assert_eq!(column_sidecars.len(), E::number_of_columns()); for (idx, col_sidecar) in column_sidecars.iter().enumerate() { assert_eq!(col_sidecar.index, idx as u64); assert_eq!(col_sidecar.kzg_commitments.len(), num_of_blobs); // ensure column sidecars are prefixed with column index (for verification purpose in prototype only) let prefix_len = 8; // column index (u64) is stored as the first 8 bytes - let col_index_prefix = - u64::from_le_bytes(col_sidecar.data[0..prefix_len].try_into().unwrap()); + let cell = col_sidecar.column.first().unwrap(); + let col_index_prefix = u64::from_le_bytes(cell[0..prefix_len].try_into().unwrap()); assert_eq!(col_index_prefix, idx as u64) } } diff --git a/consensus/types/src/blob_column_subnet_id.rs b/consensus/types/src/data_column_subnet_id.rs similarity index 62% rename from consensus/types/src/blob_column_subnet_id.rs rename to consensus/types/src/data_column_subnet_id.rs index 4599e421490..e8325a12e54 100644 --- a/consensus/types/src/blob_column_subnet_id.rs +++ b/consensus/types/src/data_column_subnet_id.rs @@ -1,18 +1,18 @@ -//! Identifies each blob column subnet by an integer identifier. +//! Identifies each data column subnet by an integer identifier. use crate::{ChainSpec, EthSpec}; use ethereum_types::U256; -use safe_arith::{ArithError, SafeArith}; +use safe_arith::ArithError; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; use std::ops::{Deref, DerefMut}; -const BLOB_COLUMN_SUBNET_COUNT: u64 = 64; +const DATA_COLUMN_SUBNET_COUNT: u64 = 64; lazy_static! { - static ref BLOB_COLUMN_SUBNET_ID_TO_STRING: Vec = { - let mut v = Vec::with_capacity(BLOB_COLUMN_SUBNET_COUNT as usize); + static ref DATA_COLUMN_SUBNET_ID_TO_STRING: Vec = { + let mut v = Vec::with_capacity(DATA_COLUMN_SUBNET_COUNT as usize); - for i in 0..BLOB_COLUMN_SUBNET_COUNT { + for i in 0..DATA_COLUMN_SUBNET_COUNT { v.push(i.to_string()); } v @@ -21,58 +21,52 @@ lazy_static! { #[derive(arbitrary::Arbitrary, Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(transparent)] -pub struct BlobColumnSubnetId(#[serde(with = "serde_utils::quoted_u64")] u64); +pub struct DataColumnSubnetId(#[serde(with = "serde_utils::quoted_u64")] u64); -pub fn blob_column_subnet_id_to_string(i: u64) -> &'static str { - if i < BLOB_COLUMN_SUBNET_COUNT { - BLOB_COLUMN_SUBNET_ID_TO_STRING +pub fn data_column_subnet_id_to_string(i: u64) -> &'static str { + if i < DATA_COLUMN_SUBNET_COUNT { + DATA_COLUMN_SUBNET_ID_TO_STRING .get(i as usize) - .expect("index below BLOB_COLUMN_SUBNET_COUNT") + .expect("index below DATA_COLUMN_SUBNET_COUNT") } else { - "blob column subnet id out of range" + "data column subnet id out of range" } } -impl BlobColumnSubnetId { +impl DataColumnSubnetId { pub fn new(id: u64) -> Self { id.into() } - pub fn try_from_column_index(column_index: usize) -> Result { - let cols_per_subnet = T::blob_column_count().safe_div(T::blob_column_subnet_count())?; - let subnet_id = column_index.safe_div(cols_per_subnet)?; - if subnet_id < T::blob_column_subnet_count() { - Ok((subnet_id as u64).into()) - } else { - Err(Error::InvalidColumn(column_index)) - } + pub fn from_column_index(column_index: usize) -> Self { + DataColumnSubnetId((column_index % T::data_column_subnet_count()) as u64) } #[allow(clippy::arithmetic_side_effects)] /// Compute required subnets to subscribe to given the node id. /// TODO(das): Add epoch param /// TODO(das): Add num of subnets (from ENR) - pub fn compute_subnets_for_blob_column( + pub fn compute_subnets_for_data_column( node_id: U256, spec: &ChainSpec, - ) -> impl Iterator { - let num_of_column_subnets = T::blob_column_subnet_count() as u64; - (0..spec.blob_custody_requirement) + ) -> impl Iterator { + let num_of_column_subnets = T::data_column_subnet_count() as u64; + (0..spec.custody_requirement) .map(move |i| { let node_offset = (node_id % U256::from(num_of_column_subnets)).as_u64(); node_offset.saturating_add(i) % num_of_column_subnets }) - .map(BlobColumnSubnetId::new) + .map(DataColumnSubnetId::new) } } -impl Display for BlobColumnSubnetId { +impl Display for DataColumnSubnetId { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { write!(f, "{}", self.0) } } -impl Deref for BlobColumnSubnetId { +impl Deref for DataColumnSubnetId { type Target = u64; fn deref(&self) -> &Self::Target { @@ -80,40 +74,39 @@ impl Deref for BlobColumnSubnetId { } } -impl DerefMut for BlobColumnSubnetId { +impl DerefMut for DataColumnSubnetId { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } } -impl From for BlobColumnSubnetId { +impl From for DataColumnSubnetId { fn from(x: u64) -> Self { Self(x) } } -impl Into for BlobColumnSubnetId { +impl Into for DataColumnSubnetId { fn into(self) -> u64 { self.0 } } -impl Into for &BlobColumnSubnetId { +impl Into for &DataColumnSubnetId { fn into(self) -> u64 { self.0 } } -impl AsRef for BlobColumnSubnetId { +impl AsRef for DataColumnSubnetId { fn as_ref(&self) -> &str { - blob_column_subnet_id_to_string(self.0) + data_column_subnet_id_to_string(self.0) } } #[derive(Debug)] pub enum Error { ArithError(ArithError), - InvalidColumn(usize), } impl From for Error { @@ -124,11 +117,11 @@ impl From for Error { #[cfg(test)] mod test { - use crate::blob_column_subnet_id::BlobColumnSubnetId; + use crate::data_column_subnet_id::DataColumnSubnetId; use crate::ChainSpec; #[test] - fn test_compute_subnets_for_blob_column() { + fn test_compute_subnets_for_data_column() { let node_ids = [ "0", "88752428858350697756262172400162263450541348766581994718383409852729519486397", @@ -161,14 +154,14 @@ mod test { let spec = ChainSpec::mainnet(); for x in 0..node_ids.len() { - let computed_subnets = BlobColumnSubnetId::compute_subnets_for_blob_column::< + let computed_subnets = DataColumnSubnetId::compute_subnets_for_data_column::< crate::MainnetEthSpec, >(node_ids[x], &spec); assert_eq!( expected_subnets[x], computed_subnets - .map(BlobColumnSubnetId::into) + .map(DataColumnSubnetId::into) .collect::>() ); } diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index d1300a2e3fc..00bf41e8a73 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -4,7 +4,7 @@ use safe_arith::SafeArith; use serde::{Deserialize, Serialize}; use ssz_types::typenum::{ bit::B0, UInt, Unsigned, U0, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, U16, - U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, U8192, + U16777216, U2, U2048, U256, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, U8192, }; use ssz_types::typenum::{U17, U9}; use std::fmt::{self, Debug}; @@ -109,15 +109,16 @@ pub trait EthSpec: type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin; type MaxBlobCommitmentsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin; type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type FieldElementsPerCell: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq; type KzgCommitmentInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * New in PeerDAS */ - type BlobColumnSubnetCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; - type BlobColumnCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type DataColumnSubnetCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type DataColumnCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxBytesPerColumn: Unsigned + Clone + Sync + Send + Debug + PartialEq; - type AllKzgCommitmentsInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type KzgCommitmentsInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) */ @@ -141,11 +142,6 @@ pub trait EthSpec: /// Must be set to `BytesPerFieldElement * FieldElementsPerBlob`. type BytesPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; - /// The total length of an extended blob in bytes. - /// - /// Must be set to `BytesPerBlob * 2`. - type BytesPerExtendedBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; - fn default_spec() -> ChainSpec; fn spec_name() -> EthSpecId; @@ -281,35 +277,35 @@ pub trait EthSpec: Self::FieldElementsPerBlob::to_usize() } + /// Returns the `FIELD_ELEMENTS_PER_CELL` constant for this specification. + fn field_elements_per_cell() -> usize { + Self::FieldElementsPerCell::to_usize() + } + /// Returns the `BYTES_PER_BLOB` constant for this specification. fn bytes_per_blob() -> usize { Self::BytesPerBlob::to_usize() } - /// Returns the `BYTES_PER_EXTENDED_BLOB` constant for this specification. - fn bytes_per_extended_blob() -> usize { - Self::BytesPerExtendedBlob::to_usize() - } - /// Returns the `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH` preset for this specification. fn kzg_proof_inclusion_proof_depth() -> usize { Self::KzgCommitmentInclusionProofDepth::to_usize() } - fn blob_column_count() -> usize { - Self::BlobColumnCount::to_usize() + fn number_of_columns() -> usize { + Self::DataColumnCount::to_usize() } - fn blob_column_subnet_count() -> usize { - Self::BlobColumnSubnetCount::to_usize() + fn data_column_subnet_count() -> usize { + Self::DataColumnSubnetCount::to_usize() } fn max_bytes_per_column() -> usize { Self::MaxBytesPerColumn::to_usize() } - fn all_kzg_commitments_inclusion_proof_depth() -> usize { - Self::AllKzgCommitmentsInclusionProofDepth::to_usize() + fn kzg_commitments_inclusion_proof_depth() -> usize { + Self::KzgCommitmentsInclusionProofDepth::to_usize() } } @@ -354,16 +350,16 @@ impl EthSpec for MainnetEthSpec { type MaxBlobCommitmentsPerBlock = U4096; type BytesPerFieldElement = U32; type FieldElementsPerBlob = U4096; + type FieldElementsPerCell = U64; type BytesPerBlob = U131072; - type BytesPerExtendedBlob = U262144; type KzgCommitmentInclusionProofDepth = U17; - type BlobColumnSubnetCount = U32; - type BlobColumnCount = U128; + type DataColumnSubnetCount = U32; + type DataColumnCount = U128; // Column samples are entire columns in 1D DAS. // max data size = extended_blob_bytes * max_blobs_per_block / num_of_columns // 256kb * 32 / 128 = 64kb type MaxBytesPerColumn = U65536; - type AllKzgCommitmentsInclusionProofDepth = U4; // inclusion of the whole list of commitments + type KzgCommitmentsInclusionProofDepth = U4; // inclusion of the whole list of commitments type SyncSubcommitteeSize = U128; // 512 committee size / 4 sync committee subnet count type MaxPendingAttestations = U4096; // 128 max attestations * 32 slots per epoch type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch @@ -395,15 +391,15 @@ impl EthSpec for MinimalEthSpec { type SlotsPerEth1VotingPeriod = U32; // 4 epochs * 8 slots per epoch type MaxWithdrawalsPerPayload = U4; type FieldElementsPerBlob = U4096; + type FieldElementsPerCell = U64; type BytesPerBlob = U131072; - type BytesPerExtendedBlob = U262144; type MaxBlobCommitmentsPerBlock = U16; type KzgCommitmentInclusionProofDepth = U9; // DAS spec values copied from `MainnetEthSpec` - type BlobColumnSubnetCount = U32; - type BlobColumnCount = U128; + type DataColumnSubnetCount = U32; + type DataColumnCount = U128; type MaxBytesPerColumn = U65536; - type AllKzgCommitmentsInclusionProofDepth = U4; + type KzgCommitmentsInclusionProofDepth = U4; params_from_eth_spec!(MainnetEthSpec { JustificationBitsLength, @@ -475,15 +471,15 @@ impl EthSpec for GnosisEthSpec { type MaxBlobsPerBlock = U6; type MaxBlobCommitmentsPerBlock = U4096; type FieldElementsPerBlob = U4096; + type FieldElementsPerCell = U64; type BytesPerFieldElement = U32; type BytesPerBlob = U131072; - type BytesPerExtendedBlob = U262144; type KzgCommitmentInclusionProofDepth = U17; // DAS spec values copied from `MainnetEthSpec` - type BlobColumnSubnetCount = U32; - type BlobColumnCount = U128; + type DataColumnSubnetCount = U32; + type DataColumnCount = U128; type MaxBytesPerColumn = U65536; - type AllKzgCommitmentsInclusionProofDepth = U4; + type KzgCommitmentsInclusionProofDepth = U4; fn default_spec() -> ChainSpec { ChainSpec::gnosis() diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 40292aff056..3c3e18d9297 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,9 +98,9 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; -pub mod blob_column_sidecar; -pub mod blob_column_subnet_id; pub mod blob_sidecar; +pub mod data_column_sidecar; +pub mod data_column_subnet_id; pub mod light_client_header; pub mod non_zero_usize; pub mod runtime_var_list; @@ -123,14 +123,14 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blob_column_sidecar::BlobColumnSidecar; -pub use crate::blob_column_subnet_id::BlobColumnSubnetId; pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList, BlobsList}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; pub use crate::config_and_preset::{ConfigAndPreset, ConfigAndPresetCapella, ConfigAndPresetDeneb}; pub use crate::contribution_and_proof::ContributionAndProof; +pub use crate::data_column_sidecar::DataColumnSidecar; +pub use crate::data_column_subnet_id::DataColumnSubnetId; pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; pub use crate::deposit_message::DepositMessage; diff --git a/consensus/types/src/preset.rs b/consensus/types/src/preset.rs index 63a372ea1c9..fe8cc94f818 100644 --- a/consensus/types/src/preset.rs +++ b/consensus/types/src/preset.rs @@ -214,6 +214,13 @@ pub struct DenebPreset { pub max_blob_commitments_per_block: u64, #[serde(with = "serde_utils::quoted_u64")] pub field_elements_per_blob: u64, + // EIP-7594 DAS presets - to be moved to the next fork + #[serde(with = "serde_utils::quoted_u64")] + pub field_elements_per_cell: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub kzg_commitments_inclusion_proof_depth: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub number_of_columns: u64, } impl DenebPreset { @@ -222,6 +229,10 @@ impl DenebPreset { max_blobs_per_block: T::max_blobs_per_block() as u64, max_blob_commitments_per_block: T::max_blob_commitments_per_block() as u64, field_elements_per_blob: T::field_elements_per_blob() as u64, + field_elements_per_cell: T::field_elements_per_cell() as u64, + kzg_commitments_inclusion_proof_depth: T::kzg_commitments_inclusion_proof_depth() + as u64, + number_of_columns: T::number_of_columns() as u64, } } } From 31cde6911a3c5906a476fe63aab05bb8ec3fa9b9 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 2 Feb 2024 23:58:58 +1100 Subject: [PATCH 9/9] Fix test and some cleanups. --- beacon_node/beacon_chain/src/errors.rs | 2 +- beacon_node/http_api/src/publish_blocks.rs | 19 ++++++++----------- consensus/types/presets/gnosis/deneb.yaml | 9 +++++++++ consensus/types/presets/mainnet/deneb.yaml | 9 +++++++++ consensus/types/presets/minimal/deneb.yaml | 9 +++++++++ consensus/types/src/data_column_subnet_id.rs | 7 ++++--- 6 files changed, 40 insertions(+), 15 deletions(-) diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 92ab669126b..010488a558b 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -220,7 +220,7 @@ pub enum BeaconChainError { InconsistentFork(InconsistentFork), ProposerHeadForkChoiceError(fork_choice::Error), UnableToPublish, - UnableToBuildDataColumnSidecar(String), + UnableToBuildColumnSidecar(String), AvailabilityCheckError(AvailabilityCheckError), LightClientError(LightClientError), UnsupportedFork, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 75755adb83b..75df401ea3d 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -91,20 +91,17 @@ pub async fn publish_block(col_index) + .map_err(|e| { + BeaconChainError::UnableToBuildColumnSidecar(format!("{e:?}")) + })?; pubsub_messages.push(PubsubMessage::DataColumnSidecar(Box::new(( - DataColumnSubnetId::from_column_index::(col_index), - // .map_err(|e| { - // BlockError::BeaconChainError( - // BeaconChainError::UnableToBuildDataColumnSidecar(format!( - // "{e:?}" - // )), - // ) - // })?, + subnet_id, Arc::new(col_sidecar), )))); } diff --git a/consensus/types/presets/gnosis/deneb.yaml b/consensus/types/presets/gnosis/deneb.yaml index d2d7d0abed3..bef51470e89 100644 --- a/consensus/types/presets/gnosis/deneb.yaml +++ b/consensus/types/presets/gnosis/deneb.yaml @@ -12,3 +12,12 @@ MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 MAX_BLOBS_PER_BLOCK: 6 # `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 + +# EIP-7594 (temporary in Deneb for the purpose of prototyping) +# --------------------------------------------------------------- +# `uint64(2**6)` (= 64) +FIELD_ELEMENTS_PER_CELL: 64 +# uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) +KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 4 +# `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128) +NUMBER_OF_COLUMNS: 128 \ No newline at end of file diff --git a/consensus/types/presets/mainnet/deneb.yaml b/consensus/types/presets/mainnet/deneb.yaml index 6d2fb4abde9..0b2f28853ec 100644 --- a/consensus/types/presets/mainnet/deneb.yaml +++ b/consensus/types/presets/mainnet/deneb.yaml @@ -10,3 +10,12 @@ MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 MAX_BLOBS_PER_BLOCK: 6 # `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 + +# EIP-7594 (temporary in Deneb for the purpose of prototyping) +# --------------------------------------------------------------- +# `uint64(2**6)` (= 64) +FIELD_ELEMENTS_PER_CELL: 64 +# uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) +KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 4 +# `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128) +NUMBER_OF_COLUMNS: 128 \ No newline at end of file diff --git a/consensus/types/presets/minimal/deneb.yaml b/consensus/types/presets/minimal/deneb.yaml index be2b9fadfa5..8bb3e0b66bd 100644 --- a/consensus/types/presets/minimal/deneb.yaml +++ b/consensus/types/presets/minimal/deneb.yaml @@ -10,3 +10,12 @@ MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 MAX_BLOBS_PER_BLOCK: 6 # [customized] `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9 + +# EIP-7594 (temporary in Deneb for the purpose of prototyping) +# --------------------------------------------------------------- +# `uint64(2**6)` (= 64) +FIELD_ELEMENTS_PER_CELL: 64 +# uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) +KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 4 +# `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128) +NUMBER_OF_COLUMNS: 128 \ No newline at end of file diff --git a/consensus/types/src/data_column_subnet_id.rs b/consensus/types/src/data_column_subnet_id.rs index e8325a12e54..5a42b323895 100644 --- a/consensus/types/src/data_column_subnet_id.rs +++ b/consensus/types/src/data_column_subnet_id.rs @@ -1,7 +1,7 @@ //! Identifies each data column subnet by an integer identifier. use crate::{ChainSpec, EthSpec}; use ethereum_types::U256; -use safe_arith::ArithError; +use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; use std::ops::{Deref, DerefMut}; @@ -38,8 +38,9 @@ impl DataColumnSubnetId { id.into() } - pub fn from_column_index(column_index: usize) -> Self { - DataColumnSubnetId((column_index % T::data_column_subnet_count()) as u64) + pub fn try_from_column_index(column_index: usize) -> Result { + let id = column_index.safe_rem(T::data_column_subnet_count())? as u64; + Ok(id.into()) } #[allow(clippy::arithmetic_side_effects)]