Implement custom OpenTelemetry sampler to filter uninstrumented traces (#8647)

Co-Authored-By: Jimmy Chen <jchen.tc@gmail.com>

Author: jimmygchen

Cargo.lock

@@ -11,7 +11,6 @@ members = [
     "beacon_node/http_api",
     "beacon_node/http_metrics",
     "beacon_node/lighthouse_network",
-    "beacon_node/lighthouse_tracing",
     "beacon_node/network",
     "beacon_node/operation_pool",
     "beacon_node/store",
@@ -44,6 +43,7 @@ members = [
     "common/target_check",
     "common/task_executor",
     "common/test_random_derive",
+    "common/tracing_samplers",
     "common/validator_dir",
     "common/warp_utils",
     "common/workspace_members",
@@ -182,7 +182,6 @@ libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", default-features =
 ] }
 libsecp256k1 = "0.7"
 lighthouse_network = { path = "beacon_node/lighthouse_network" }
-lighthouse_tracing = { path = "beacon_node/lighthouse_tracing" }
 lighthouse_validator_store = { path = "validator_client/lighthouse_validator_store" }
 lighthouse_version = { path = "common/lighthouse_version" }
 lockfile = { path = "common/lockfile" }
@@ -269,6 +268,7 @@ tracing-core = "0.1"
 tracing-log = "0.2"
 tracing-opentelemetry = "0.31.0"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+tracing_samplers = { path = "common/tracing_samplers" }
 tree_hash = "0.12.0"
 tree_hash_derive = "0.12.0"
 typenum = "1"

Cargo.toml

@@ -34,7 +34,6 @@ hex = { workspace = true }
 int_to_bytes = { workspace = true }
 itertools = { workspace = true }
 kzg = { workspace = true }
-lighthouse_tracing = { workspace = true }
 lighthouse_version = { workspace = true }
 logging = { workspace = true }
 lru = { workspace = true }

beacon_node/beacon_chain/Cargo.toml

@@ -91,7 +91,6 @@ use futures::channel::mpsc::Sender;
 use itertools::Itertools;
 use itertools::process_results;
 use kzg::Kzg;
-use lighthouse_tracing::SPAN_PRODUCE_UNAGGREGATED_ATTESTATION;
 use logging::crit;
 use operation_pool::{
     CompactAttestationRef, OperationPool, PersistedOperationPool, ReceivedPreCapella,
@@ -1843,7 +1842,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// ## Errors
     ///
     /// May return an error if the `request_slot` is too far behind the head state.
-    #[instrument(name = SPAN_PRODUCE_UNAGGREGATED_ATTESTATION, skip_all, fields(%request_slot, %request_index), level = "debug")]
+    #[instrument(name = "lh_produce_unaggregated_attestation", skip_all, fields(%request_slot, %request_index), level = "debug")]
     pub fn produce_unaggregated_attestation(
         &self,
         request_slot: Slot,

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -47,7 +47,7 @@ use fork_choice::{
     ResetPayloadStatuses,
 };
 use itertools::process_results;
-use lighthouse_tracing::SPAN_RECOMPUTE_HEAD;
+
 use logging::crit;
 use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard};
 use slot_clock::SlotClock;
@@ -514,7 +514,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// can't abort block import because an error is returned here.
     pub async fn recompute_head_at_slot(self: &Arc<Self>, current_slot: Slot) {
         let span = info_span!(
-            SPAN_RECOMPUTE_HEAD,
+            "lh_recompute_head_at_slot",
             slot = %current_slot
         );
 

beacon_node/beacon_chain/src/canonical_head.rs

@@ -9,7 +9,6 @@ use crate::block_verification_types::{
 use crate::data_availability_checker::{Availability, AvailabilityCheckError};
 use crate::data_column_verification::KzgVerifiedCustodyDataColumn;
 use crate::{BeaconChainTypes, BlockProcessStatus};
-use lighthouse_tracing::SPAN_PENDING_COMPONENTS;
 use lru::LruCache;
 use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
 use ssz_types::{RuntimeFixedVector, RuntimeVariableList};
@@ -334,7 +333,7 @@ impl<E: EthSpec> PendingComponents<E> {
 
     /// Returns an empty `PendingComponents` object with the given block root.
     pub fn empty(block_root: Hash256, max_len: usize) -> Self {
-        let span = debug_span!(parent: None, SPAN_PENDING_COMPONENTS, %block_root);
+        let span = debug_span!(parent: None, "lh_pending_components", %block_root);
         let _guard = span.clone().entered();
         Self {
             block_root,

beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

@@ -23,7 +23,6 @@ futures = { workspace = true }
 health_metrics = { workspace = true }
 hex = { workspace = true }
 lighthouse_network = { workspace = true }
-lighthouse_tracing = { workspace = true }
 lighthouse_version = { workspace = true }
 logging = { workspace = true }
 lru = { workspace = true }

beacon_node/http_api/Cargo.toml

@@ -12,7 +12,6 @@ use beacon_chain::{
 };
 use eth2::beacon_response::ForkVersionedResponse;
 use eth2::types::{self as api_types, ProduceBlockV3Metadata, SkipRandaoVerification};
-use lighthouse_tracing::{SPAN_PRODUCE_BLOCK_V2, SPAN_PRODUCE_BLOCK_V3};
 use ssz::Encode;
 use std::sync::Arc;
 use tracing::instrument;
@@ -45,7 +44,7 @@ pub fn get_randao_verification(
 }
 
 #[instrument(
-    name = SPAN_PRODUCE_BLOCK_V3,
+    name = "lh_produce_block_v3",
     skip_all,
     fields(%slot)
 )]
@@ -169,7 +168,7 @@ pub async fn produce_blinded_block_v2<T: BeaconChainTypes>(
 }
 
 #[instrument(
-    name = SPAN_PRODUCE_BLOCK_V2,
+    name = "lh_produce_block_v2",
     skip_all,
     fields(%slot)
 )]

beacon_node/http_api/src/produce_block.rs

@@ -19,7 +19,6 @@ use eth2::{
 use execution_layer::{ProvenancedPayload, SubmitBlindedBlockResponse};
 use futures::TryFutureExt;
 use lighthouse_network::PubsubMessage;
-use lighthouse_tracing::SPAN_PUBLISH_BLOCK;
 use network::NetworkMessage;
 use rand::prelude::SliceRandom;
 use slot_clock::SlotClock;
@@ -79,7 +78,7 @@ impl<T: BeaconChainTypes> ProvenancedBlock<T, Arc<SignedBeaconBlock<T::EthSpec>>
 /// Handles a request from the HTTP API for full blocks.
 #[allow(clippy::too_many_arguments)]
 #[instrument(
-    name = SPAN_PUBLISH_BLOCK,
+    name = "lh_publish_block",
     level = "info",
     skip_all,
     fields(block_root = field::Empty, ?validation_level, block_slot = field::Empty, provenance = field::Empty)