diff --git a/Cargo.lock b/Cargo.lock index 27f775b2d0d..e9962f5f39f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1233,7 +1233,6 @@ dependencies = [ "int_to_bytes", "itertools 0.10.5", "kzg", - "lighthouse_tracing", "lighthouse_version", "logging", "lru 0.12.5", @@ -4169,7 +4168,6 @@ dependencies = [ "health_metrics", "hex", "lighthouse_network", - "lighthouse_tracing", "lighthouse_version", "logging", "lru 0.12.5", @@ -5383,7 +5381,6 @@ dependencies = [ "futures", "initialized_validators", "lighthouse_network", - "lighthouse_tracing", "lighthouse_version", "logging", "malloc_utils", @@ -5404,6 +5401,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "tracing_samplers", "types", "validator_client", "validator_dir", @@ -5466,10 +5464,6 @@ dependencies = [ "unsigned-varint 0.8.0", ] -[[package]] -name = "lighthouse_tracing" -version = "0.1.0" - [[package]] name = "lighthouse_validator_store" version = "0.1.0" @@ -6149,7 +6143,6 @@ dependencies = [ "kzg", "libp2p-gossipsub", "lighthouse_network", - "lighthouse_tracing", "logging", "lru_cache", "matches", @@ -9323,6 +9316,14 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing_samplers" +version = "0.1.0" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", +] + [[package]] name = "tree_hash" version = "0.12.0" diff --git a/Cargo.toml b/Cargo.toml index 441490ee1b9..b90603f2d0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ members = [ "beacon_node/http_api", "beacon_node/http_metrics", "beacon_node/lighthouse_network", - "beacon_node/lighthouse_tracing", "beacon_node/network", "beacon_node/operation_pool", "beacon_node/store", @@ -44,6 +43,7 @@ members = [ "common/target_check", "common/task_executor", "common/test_random_derive", + "common/tracing_samplers", "common/validator_dir", "common/warp_utils", "common/workspace_members", @@ -170,7 +170,6 @@ itertools = "0.10" kzg = { path = "crypto/kzg" } libsecp256k1 = "0.7" lighthouse_network = { path = "beacon_node/lighthouse_network" } -lighthouse_tracing = { path = "beacon_node/lighthouse_tracing" } lighthouse_validator_store = { path = "validator_client/lighthouse_validator_store" } lighthouse_version = { path = "common/lighthouse_version" } lockfile = { path = "common/lockfile" } @@ -257,6 +256,7 @@ tracing-core = "0.1" tracing-log = "0.2" tracing-opentelemetry = "0.31.0" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +tracing_samplers = { path = "common/tracing_samplers" } tree_hash = "0.12.0" tree_hash_derive = "0.12.0" typenum = "1" diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 734cfdf32bb..5e1c41b8302 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -34,7 +34,6 @@ hex = { workspace = true } int_to_bytes = { workspace = true } itertools = { workspace = true } kzg = { workspace = true } -lighthouse_tracing = { workspace = true } lighthouse_version = { workspace = true } logging = { workspace = true } lru = { workspace = true } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a6be3dde8cb..4edef95381c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -91,7 +91,6 @@ use futures::channel::mpsc::Sender; use itertools::Itertools; use itertools::process_results; use kzg::Kzg; -use lighthouse_tracing::SPAN_PRODUCE_UNAGGREGATED_ATTESTATION; use logging::crit; use operation_pool::{ CompactAttestationRef, OperationPool, PersistedOperationPool, ReceivedPreCapella, @@ -1844,7 +1843,7 @@ impl BeaconChain { /// ## Errors /// /// May return an error if the `request_slot` is too far behind the head state. - #[instrument(name = SPAN_PRODUCE_UNAGGREGATED_ATTESTATION, skip_all, fields(%request_slot, %request_index), level = "debug")] + #[instrument(name = "lh_produce_unaggregated_attestation", skip_all, fields(%request_slot, %request_index), level = "debug")] pub fn produce_unaggregated_attestation( &self, request_slot: Slot, diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 76c08c5e39c..f18fdb25f69 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -47,7 +47,7 @@ use fork_choice::{ ResetPayloadStatuses, }; use itertools::process_results; -use lighthouse_tracing::SPAN_RECOMPUTE_HEAD; + use logging::crit; use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard}; use slot_clock::SlotClock; @@ -514,7 +514,7 @@ impl BeaconChain { /// can't abort block import because an error is returned here. pub async fn recompute_head_at_slot(self: &Arc, current_slot: Slot) { let span = info_span!( - SPAN_RECOMPUTE_HEAD, + "lh_recompute_head_at_slot", slot = %current_slot ); diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 776fb50f619..0c954d1a73f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -9,7 +9,6 @@ use crate::block_verification_types::{ use crate::data_availability_checker::{Availability, AvailabilityCheckError}; use crate::data_column_verification::KzgVerifiedCustodyDataColumn; use crate::{BeaconChainTypes, BlockProcessStatus}; -use lighthouse_tracing::SPAN_PENDING_COMPONENTS; use lru::LruCache; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use ssz_types::{RuntimeFixedVector, RuntimeVariableList}; @@ -334,7 +333,7 @@ impl PendingComponents { /// Returns an empty `PendingComponents` object with the given block root. pub fn empty(block_root: Hash256, max_len: usize) -> Self { - let span = debug_span!(parent: None, SPAN_PENDING_COMPONENTS, %block_root); + let span = debug_span!(parent: None, "lh_pending_components", %block_root); let _guard = span.clone().entered(); Self { block_root, diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 571dab10273..6211ac67265 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -23,7 +23,6 @@ futures = { workspace = true } health_metrics = { workspace = true } hex = { workspace = true } lighthouse_network = { workspace = true } -lighthouse_tracing = { workspace = true } lighthouse_version = { workspace = true } logging = { workspace = true } lru = { workspace = true } diff --git a/beacon_node/http_api/src/produce_block.rs b/beacon_node/http_api/src/produce_block.rs index 3bd0cec7e33..41478aac696 100644 --- a/beacon_node/http_api/src/produce_block.rs +++ b/beacon_node/http_api/src/produce_block.rs @@ -12,7 +12,6 @@ use beacon_chain::{ }; use eth2::beacon_response::ForkVersionedResponse; use eth2::types::{self as api_types, ProduceBlockV3Metadata, SkipRandaoVerification}; -use lighthouse_tracing::{SPAN_PRODUCE_BLOCK_V2, SPAN_PRODUCE_BLOCK_V3}; use ssz::Encode; use std::sync::Arc; use tracing::instrument; @@ -45,7 +44,7 @@ pub fn get_randao_verification( } #[instrument( - name = SPAN_PRODUCE_BLOCK_V3, + name = "lh_produce_block_v3", skip_all, fields(%slot) )] @@ -169,7 +168,7 @@ pub async fn produce_blinded_block_v2( } #[instrument( - name = SPAN_PRODUCE_BLOCK_V2, + name = "lh_produce_block_v2", skip_all, fields(%slot) )] diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b54c071eb80..99a00f5f97b 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -19,7 +19,6 @@ use eth2::{ use execution_layer::{ProvenancedPayload, SubmitBlindedBlockResponse}; use futures::TryFutureExt; use lighthouse_network::PubsubMessage; -use lighthouse_tracing::SPAN_PUBLISH_BLOCK; use network::NetworkMessage; use rand::prelude::SliceRandom; use slot_clock::SlotClock; @@ -79,7 +78,7 @@ impl ProvenancedBlock> /// Handles a request from the HTTP API for full blocks. #[allow(clippy::too_many_arguments)] #[instrument( - name = SPAN_PUBLISH_BLOCK, + name = "lh_publish_block", level = "info", skip_all, fields(block_root = field::Empty, ?validation_level, block_slot = field::Empty, provenance = field::Empty) diff --git a/beacon_node/lighthouse_tracing/Cargo.toml b/beacon_node/lighthouse_tracing/Cargo.toml deleted file mode 100644 index cd71c202531..00000000000 --- a/beacon_node/lighthouse_tracing/Cargo.toml +++ /dev/null @@ -1,4 +0,0 @@ -[package] -name = "lighthouse_tracing" -version = "0.1.0" -edition = { workspace = true } diff --git a/beacon_node/lighthouse_tracing/src/lib.rs b/beacon_node/lighthouse_tracing/src/lib.rs deleted file mode 100644 index 094ad042fc6..00000000000 --- a/beacon_node/lighthouse_tracing/src/lib.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! This module contains root span identifiers for key code paths in the beacon node. -//! -//! TODO: These span identifiers will be used to implement selective tracing export (to be implemented), -//! where only the listed root spans and their descendants will be exported to the tracing backend. - -/// Root span names for block production and publishing -pub const SPAN_PRODUCE_BLOCK_V2: &str = "produce_block_v2"; -pub const SPAN_PRODUCE_BLOCK_V3: &str = "produce_block_v3"; -pub const SPAN_PUBLISH_BLOCK: &str = "publish_block"; - -/// Root span names for attestation production -pub const SPAN_PRODUCE_UNAGGREGATED_ATTESTATION: &str = "produce_unaggregated_attestation"; - -/// Data Availability checker span identifiers -pub const SPAN_PENDING_COMPONENTS: &str = "pending_components"; - -/// Gossip methods root spans -pub const SPAN_PROCESS_GOSSIP_DATA_COLUMN: &str = "process_gossip_data_column"; -pub const SPAN_PROCESS_GOSSIP_BLOB: &str = "process_gossip_blob"; -pub const SPAN_PROCESS_GOSSIP_BLOCK: &str = "process_gossip_block"; - -/// Sync methods root spans -pub const SPAN_SYNCING_CHAIN: &str = "syncing_chain"; -pub const SPAN_OUTGOING_RANGE_REQUEST: &str = "outgoing_range_request"; -pub const SPAN_SINGLE_BLOCK_LOOKUP: &str = "single_block_lookup"; -pub const SPAN_OUTGOING_BLOCK_BY_ROOT_REQUEST: &str = "outgoing_block_by_root_request"; -pub const SPAN_OUTGOING_CUSTODY_REQUEST: &str = "outgoing_custody_request"; -pub const SPAN_PROCESS_RPC_BLOCK: &str = "process_rpc_block"; -pub const SPAN_PROCESS_RPC_BLOBS: &str = "process_rpc_blobs"; -pub const SPAN_PROCESS_RPC_CUSTODY_COLUMNS: &str = "process_rpc_custody_columns"; -pub const SPAN_PROCESS_CHAIN_SEGMENT: &str = "process_chain_segment"; -pub const SPAN_CUSTODY_BACKFILL_SYNC_BATCH_REQUEST: &str = "custody_backfill_sync_batch_request"; -pub const SPAN_PROCESS_CHAIN_SEGMENT_BACKFILL: &str = "process_chain_segment_backfill"; -pub const SPAN_CUSTODY_BACKFILL_SYNC_IMPORT_COLUMNS: &str = "custody_backfill_sync_import_columns"; - -/// Fork choice root spans -pub const SPAN_RECOMPUTE_HEAD: &str = "recompute_head_at_slot"; - -/// RPC methods root spans -pub const SPAN_HANDLE_BLOCKS_BY_RANGE_REQUEST: &str = "handle_blocks_by_range_request"; -pub const SPAN_HANDLE_BLOBS_BY_RANGE_REQUEST: &str = "handle_blobs_by_range_request"; -pub const SPAN_HANDLE_DATA_COLUMNS_BY_RANGE_REQUEST: &str = "handle_data_columns_by_range_request"; -pub const SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST: &str = "handle_blocks_by_root_request"; -pub const SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST: &str = "handle_blobs_by_root_request"; -pub const SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST: &str = "handle_data_columns_by_root_request"; -pub const SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE: &str = "handle_light_client_updates_by_range"; -pub const SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP: &str = "handle_light_client_bootstrap"; -pub const SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE: &str = - "handle_light_client_optimistic_update"; -pub const SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE: &str = "handle_light_client_finality_update"; - -/// List of all root span names that are allowed to be exported to the tracing backend. -/// Only these spans and their descendants will be processed to reduce noise from -/// uninstrumented code paths. New root spans must be added to this list to be traced. -pub const LH_BN_ROOT_SPAN_NAMES: &[&str] = &[ - SPAN_PRODUCE_BLOCK_V2, - SPAN_PRODUCE_BLOCK_V3, - SPAN_PUBLISH_BLOCK, - SPAN_PENDING_COMPONENTS, - SPAN_PROCESS_GOSSIP_DATA_COLUMN, - SPAN_PROCESS_GOSSIP_BLOB, - SPAN_PROCESS_GOSSIP_BLOCK, - SPAN_SYNCING_CHAIN, - SPAN_OUTGOING_RANGE_REQUEST, - SPAN_SINGLE_BLOCK_LOOKUP, - SPAN_PROCESS_RPC_BLOCK, - SPAN_PROCESS_RPC_BLOBS, - SPAN_PROCESS_RPC_CUSTODY_COLUMNS, - SPAN_PROCESS_CHAIN_SEGMENT, - SPAN_PROCESS_CHAIN_SEGMENT_BACKFILL, - SPAN_HANDLE_BLOCKS_BY_RANGE_REQUEST, - SPAN_HANDLE_BLOBS_BY_RANGE_REQUEST, - SPAN_HANDLE_DATA_COLUMNS_BY_RANGE_REQUEST, - SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST, - SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST, - SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, - SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, - SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, - SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE, - SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE, - SPAN_CUSTODY_BACKFILL_SYNC_BATCH_REQUEST, - SPAN_CUSTODY_BACKFILL_SYNC_IMPORT_COLUMNS, -]; diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index bf261965760..1ed470721b1 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -29,7 +29,6 @@ hex = { workspace = true } igd-next = { version = "0.16", features = ["aio_tokio"] } itertools = { workspace = true } lighthouse_network = { workspace = true } -lighthouse_tracing = { workspace = true } logging = { workspace = true } lru_cache = { workspace = true } metrics = { workspace = true } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index eb70147c6ef..8a54c854516 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -21,9 +21,6 @@ use beacon_chain::{ }; use beacon_processor::{Work, WorkEvent}; use lighthouse_network::{Client, MessageAcceptance, MessageId, PeerAction, PeerId, ReportSource}; -use lighthouse_tracing::{ - SPAN_PROCESS_GOSSIP_BLOB, SPAN_PROCESS_GOSSIP_BLOCK, SPAN_PROCESS_GOSSIP_DATA_COLUMN, -}; use logging::crit; use operation_pool::ReceivedPreCapella; use slot_clock::SlotClock; @@ -605,7 +602,7 @@ impl NetworkBeaconProcessor { } #[instrument( - name = SPAN_PROCESS_GOSSIP_DATA_COLUMN, + name = "lh_process_gossip_data_column", parent = None, level = "debug", skip_all, @@ -769,7 +766,7 @@ impl NetworkBeaconProcessor { #[allow(clippy::too_many_arguments)] #[instrument( - name = SPAN_PROCESS_GOSSIP_BLOB, + name = "lh_process_gossip_blob", parent = None, level = "debug", skip_all, @@ -1135,7 +1132,7 @@ impl NetworkBeaconProcessor { /// Raises a log if there are errors. #[allow(clippy::too_many_arguments)] #[instrument( - name = SPAN_PROCESS_GOSSIP_BLOCK, + name = "lh_process_gossip_block", parent = None, level = "debug", skip_all, diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index ac24b648e05..e51a1948c9a 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -10,13 +10,6 @@ use lighthouse_network::rpc::methods::{ }; use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, ReportSource, Response, SyncInfo}; -use lighthouse_tracing::{ - SPAN_HANDLE_BLOBS_BY_RANGE_REQUEST, SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST, - SPAN_HANDLE_BLOCKS_BY_RANGE_REQUEST, SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST, - SPAN_HANDLE_DATA_COLUMNS_BY_RANGE_REQUEST, SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, - SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE, - SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE, SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, -}; use methods::LightClientUpdatesByRangeRequest; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet, hash_map::Entry}; @@ -163,7 +156,7 @@ impl NetworkBeaconProcessor { /// Handle a `BlocksByRoot` request from the peer. #[instrument( - name = SPAN_HANDLE_BLOCKS_BY_ROOT_REQUEST, + name = "lh_handle_blocks_by_root_request", parent = None, level = "debug", skip_all, @@ -263,7 +256,7 @@ impl NetworkBeaconProcessor { /// Handle a `BlobsByRoot` request from the peer. #[instrument( - name = SPAN_HANDLE_BLOBS_BY_ROOT_REQUEST, + name = "lh_handle_blobs_by_root_request", parent = None, level = "debug", skip_all, @@ -392,7 +385,7 @@ impl NetworkBeaconProcessor { /// Handle a `DataColumnsByRoot` request from the peer. #[instrument( - name = SPAN_HANDLE_DATA_COLUMNS_BY_ROOT_REQUEST, + name = "lh_handle_data_columns_by_root_request", parent = None, level = "debug", skip_all, @@ -485,7 +478,7 @@ impl NetworkBeaconProcessor { } #[instrument( - name = SPAN_HANDLE_LIGHT_CLIENT_UPDATES_BY_RANGE, + name = "lh_handle_light_client_updates_by_range", parent = None, level = "debug", skip_all, @@ -586,7 +579,7 @@ impl NetworkBeaconProcessor { /// Handle a `LightClientBootstrap` request from the peer. #[instrument( - name = SPAN_HANDLE_LIGHT_CLIENT_BOOTSTRAP, + name = "lh_handle_light_client_bootstrap", parent = None, level = "debug", skip_all, @@ -626,7 +619,7 @@ impl NetworkBeaconProcessor { /// Handle a `LightClientOptimisticUpdate` request from the peer. #[instrument( - name = SPAN_HANDLE_LIGHT_CLIENT_OPTIMISTIC_UPDATE, + name = "lh_handle_light_client_optimistic_update", parent = None, level = "debug", skip_all, @@ -660,7 +653,7 @@ impl NetworkBeaconProcessor { /// Handle a `LightClientFinalityUpdate` request from the peer. #[instrument( - name = SPAN_HANDLE_LIGHT_CLIENT_FINALITY_UPDATE, + name = "lh_handle_light_client_finality_update", parent = None, level = "debug", skip_all, @@ -694,7 +687,7 @@ impl NetworkBeaconProcessor { /// Handle a `BlocksByRange` request from the peer. #[instrument( - name = SPAN_HANDLE_BLOCKS_BY_RANGE_REQUEST, + name = "lh_handle_blocks_by_range_request", parent = None, level = "debug", skip_all, @@ -990,7 +983,7 @@ impl NetworkBeaconProcessor { /// Handle a `BlobsByRange` request from the peer. #[instrument( - name = SPAN_HANDLE_BLOBS_BY_RANGE_REQUEST, + name = "lh_handle_blobs_by_range_request", parent = None, skip_all, level = "debug", @@ -1155,7 +1148,7 @@ impl NetworkBeaconProcessor { /// Handle a `DataColumnsByRange` request from the peer. #[instrument( - name = SPAN_HANDLE_DATA_COLUMNS_BY_RANGE_REQUEST, + name = "lh_handle_data_columns_by_range_request", parent = None, skip_all, level = "debug", diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index e49ae134fe4..91115a10a5d 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -21,11 +21,6 @@ use beacon_processor::{ use beacon_processor::{Work, WorkEvent}; use lighthouse_network::PeerAction; use lighthouse_network::service::api_types::CustodyBackfillBatchId; -use lighthouse_tracing::{ - SPAN_CUSTODY_BACKFILL_SYNC_IMPORT_COLUMNS, SPAN_PROCESS_CHAIN_SEGMENT, - SPAN_PROCESS_CHAIN_SEGMENT_BACKFILL, SPAN_PROCESS_RPC_BLOBS, SPAN_PROCESS_RPC_BLOCK, - SPAN_PROCESS_RPC_CUSTODY_COLUMNS, -}; use logging::crit; use std::sync::Arc; use std::time::Duration; @@ -107,7 +102,7 @@ impl NetworkBeaconProcessor { /// Attempt to process a block received from a direct RPC request. #[allow(clippy::too_many_arguments)] #[instrument( - name = SPAN_PROCESS_RPC_BLOCK, + name = "lh_process_rpc_block", parent = None, level = "debug", skip_all, @@ -261,7 +256,7 @@ impl NetworkBeaconProcessor { /// Attempt to process a list of blobs received from a direct RPC request. #[instrument( - name = SPAN_PROCESS_RPC_BLOBS, + name = "lh_process_rpc_blobs", parent = None, level = "debug", skip_all, @@ -349,7 +344,7 @@ impl NetworkBeaconProcessor { } #[instrument( - name = SPAN_PROCESS_RPC_CUSTODY_COLUMNS, + name = "lh_process_rpc_custody_columns", parent = None, level = "debug", skip_all, @@ -429,7 +424,7 @@ impl NetworkBeaconProcessor { expected_cgc: u64, ) { let _guard = debug_span!( - SPAN_CUSTODY_BACKFILL_SYNC_IMPORT_COLUMNS, + "lh_custody_backfill_sync_import_columns", epoch = %batch_id.epoch, columns_received_count = downloaded_columns.len() ) @@ -524,7 +519,7 @@ impl NetworkBeaconProcessor { /// Attempt to import the chain segment (`blocks`) to the beacon chain, informing the sync /// thread if more blocks are needed to process it. #[instrument( - name = SPAN_PROCESS_CHAIN_SEGMENT, + name = "lh_process_chain_segment", parent = None, level = "debug", skip_all, @@ -605,7 +600,7 @@ impl NetworkBeaconProcessor { /// Attempt to import the chain segment (`blocks`) to the beacon chain, informing the sync /// thread if more blocks are needed to process it. #[instrument( - name = SPAN_PROCESS_CHAIN_SEGMENT_BACKFILL, + name = "lh_process_chain_segment_backfill", parent = None, level = "debug", skip_all, diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 46897b2283b..e6423e3ae0b 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -7,7 +7,6 @@ use crate::sync::network_context::{ use beacon_chain::{BeaconChainTypes, BlockProcessStatus}; use educe::Educe; use lighthouse_network::service::api_types::Id; -use lighthouse_tracing::SPAN_SINGLE_BLOCK_LOOKUP; use parking_lot::RwLock; use std::collections::HashSet; use std::fmt::Debug; @@ -93,7 +92,7 @@ impl SingleBlockLookup { awaiting_parent: Option, ) -> Self { let lookup_span = debug_span!( - SPAN_SINGLE_BLOCK_LOOKUP, + "lh_single_block_lookup", block_root = %requested_block_root, id = id, ); diff --git a/beacon_node/network/src/sync/custody_backfill_sync/mod.rs b/beacon_node/network/src/sync/custody_backfill_sync/mod.rs index bb2c6799f1d..fa8b70c8b46 100644 --- a/beacon_node/network/src/sync/custody_backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/custody_backfill_sync/mod.rs @@ -10,7 +10,6 @@ use lighthouse_network::{ service::api_types::{CustodyBackFillBatchRequestId, CustodyBackfillBatchId}, types::CustodyBackFillState, }; -use lighthouse_tracing::SPAN_CUSTODY_BACKFILL_SYNC_BATCH_REQUEST; use logging::crit; use std::hash::{DefaultHasher, Hash, Hasher}; use tracing::{debug, error, info, info_span, warn}; @@ -1004,7 +1003,7 @@ impl CustodyBackFillSync { network: &mut SyncNetworkContext, batch_id: BatchId, ) -> Result<(), CustodyBackfillError> { - let span = info_span!(SPAN_CUSTODY_BACKFILL_SYNC_BATCH_REQUEST); + let span = info_span!("lh_custody_backfill_sync_batch_request"); let _enter = span.enter(); if let Some(batch) = self.batches.get_mut(&batch_id) { diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 2e0c56db23f..496ba7235cb 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -31,7 +31,6 @@ use lighthouse_network::service::api_types::{ DataColumnsByRootRequester, Id, SingleLookupReqId, SyncRequestId, }; use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource}; -use lighthouse_tracing::{SPAN_OUTGOING_BLOCK_BY_ROOT_REQUEST, SPAN_OUTGOING_RANGE_REQUEST}; use parking_lot::RwLock; pub use requests::LookupVerifyError; use requests::{ @@ -546,7 +545,7 @@ impl SyncNetworkContext { ) -> Result { let range_request_span = debug_span!( parent: None, - SPAN_OUTGOING_RANGE_REQUEST, + "lh_outgoing_range_request", range_req_id = %requester, block_peers = block_peers.len(), column_peers = column_peers.len() @@ -908,7 +907,7 @@ impl SyncNetworkContext { let request_span = debug_span!( parent: Span::current(), - SPAN_OUTGOING_BLOCK_BY_ROOT_REQUEST, + "lh_outgoing_block_by_root_request", %block_root, ); self.blocks_by_root_requests.insert( diff --git a/beacon_node/network/src/sync/network_context/custody.rs b/beacon_node/network/src/sync/network_context/custody.rs index 71e002cc422..3f4c8bd73fb 100644 --- a/beacon_node/network/src/sync/network_context/custody.rs +++ b/beacon_node/network/src/sync/network_context/custody.rs @@ -6,7 +6,6 @@ use beacon_chain::validator_monitor::timestamp_now; use fnv::FnvHashMap; use lighthouse_network::PeerId; use lighthouse_network::service::api_types::{CustodyId, DataColumnsByRootRequester}; -use lighthouse_tracing::SPAN_OUTGOING_CUSTODY_REQUEST; use parking_lot::RwLock; use std::collections::HashSet; use std::hash::{BuildHasher, RandomState}; @@ -69,7 +68,7 @@ impl ActiveCustodyRequest { ) -> Self { let span = debug_span!( parent: Span::current(), - SPAN_OUTGOING_CUSTODY_REQUEST, + "lh_outgoing_custody_request", %block_root, ); Self { diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index a3f2b798b1e..d67d6468a92 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -12,7 +12,6 @@ use beacon_chain::BeaconChainTypes; use beacon_chain::block_verification_types::RpcBlock; use lighthouse_network::service::api_types::Id; use lighthouse_network::{PeerAction, PeerId}; -use lighthouse_tracing::SPAN_SYNCING_CHAIN; use logging::crit; use std::collections::{BTreeMap, HashSet, btree_map::Entry}; use std::hash::{Hash, Hasher}; @@ -161,7 +160,7 @@ pub enum ChainSyncingState { impl SyncingChain { #[allow(clippy::too_many_arguments)] #[instrument( - name = SPAN_SYNCING_CHAIN, + name = "lh_syncing_chain", parent = None, level="debug", skip_all, diff --git a/common/tracing_samplers/Cargo.toml b/common/tracing_samplers/Cargo.toml new file mode 100644 index 00000000000..76afc10ee71 --- /dev/null +++ b/common/tracing_samplers/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "tracing_samplers" +version = "0.1.0" +edition = { workspace = true } + +[dependencies] +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } diff --git a/common/tracing_samplers/src/lib.rs b/common/tracing_samplers/src/lib.rs new file mode 100644 index 00000000000..e34518ac4a1 --- /dev/null +++ b/common/tracing_samplers/src/lib.rs @@ -0,0 +1,78 @@ +//! OpenTelemetry samplers for filtering traces. + +use opentelemetry::Context; +use opentelemetry::trace::{Link, SamplingDecision, SamplingResult, SpanKind, TraceState}; +use opentelemetry_sdk::trace::ShouldSample; + +/// A sampler that only samples spans whose names start with a given prefix. +/// +/// This sampler is designed to be used with `Sampler::ParentBased`, which will: +/// - Use this sampler's decision for root spans (no parent) +/// - Automatically inherit the parent's sampling decision for child spans +/// +/// This ensures that only traces starting from known instrumented code paths are exported, +/// reducing noise from partially instrumented code paths. +#[derive(Debug, Clone)] +pub struct PrefixBasedSampler { + prefix: &'static str, +} + +impl PrefixBasedSampler { + pub fn new(prefix: &'static str) -> Self { + Self { prefix } + } +} + +impl ShouldSample for PrefixBasedSampler { + fn should_sample( + &self, + _parent_context: Option<&Context>, + _trace_id: opentelemetry::trace::TraceId, + name: &str, + _span_kind: &SpanKind, + _attributes: &[opentelemetry::KeyValue], + _links: &[Link], + ) -> SamplingResult { + if name.starts_with(self.prefix) { + SamplingResult { + decision: SamplingDecision::RecordAndSample, + attributes: Vec::new(), + trace_state: TraceState::default(), + } + } else { + SamplingResult { + decision: SamplingDecision::Drop, + attributes: Vec::new(), + trace_state: TraceState::default(), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::trace::TraceId; + + #[test] + fn prefix_based_sampler_filters_by_prefix() { + let sampler = PrefixBasedSampler::new("test_"); + let trace_id = TraceId::from_hex("0123456789abcdef0123456789abcdef").unwrap(); + + // Spans with prefix should be sampled + let result = sampler.should_sample( + None, + trace_id, + "test_my_span", + &SpanKind::Internal, + &[], + &[], + ); + assert!(matches!(result.decision, SamplingDecision::RecordAndSample)); + + // Spans without prefix should be dropped + let result = + sampler.should_sample(None, trace_id, "other_span", &SpanKind::Internal, &[], &[]); + assert!(matches!(result.decision, SamplingDecision::Drop)); + } +} diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index ebe00c9be59..000c6fd0da4 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -53,7 +53,6 @@ environment = { workspace = true } eth2_network_config = { workspace = true } ethereum_hashing = { workspace = true } futures = { workspace = true } -lighthouse_tracing = { workspace = true } lighthouse_version = { workspace = true } logging = { workspace = true } metrics = { workspace = true } @@ -70,6 +69,7 @@ task_executor = { workspace = true } tracing = { workspace = true } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true } +tracing_samplers = { workspace = true } types = { workspace = true } validator_client = { workspace = true } validator_manager = { path = "../validator_manager" } diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 2592d2e6042..53946ed2730 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -29,6 +29,7 @@ use std::process::exit; use std::sync::LazyLock; use task_executor::ShutdownReason; use tracing::{Level, info}; +use tracing_samplers::PrefixBasedSampler; use tracing_subscriber::{Layer, filter::EnvFilter, layer::SubscriberExt, util::SubscriberInitExt}; use types::{EthSpec, EthSpecId}; use validator_client::ProductionValidatorClient; @@ -675,7 +676,15 @@ fn run( _ => "lighthouse".to_string(), }); + // Use ParentBased sampler with PrefixBasedSampler to only export traces + // that start from known instrumented code paths (lh_ prefix). Child spans + // automatically inherit their parent's sampling decision. + let sampler = opentelemetry_sdk::trace::Sampler::ParentBased(Box::new( + PrefixBasedSampler::new("lh_"), + )); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_sampler(sampler) .with_batch_exporter(exporter) .with_resource( opentelemetry_sdk::Resource::builder()