feat: otel instrumentation configuration (#1127)

* Introduced the OpenTelemetry configuration for self-instrumentation
* Refactored the previous logging config and moved it to the new instrumentation module
---------

Co-authored-by: Christian Felipe Álvarez <calvarez@newrelic.com>

Author: alvarocabanas

Cargo.lock

@@ -1279,11 +1279,36 @@ Distributed under the following license(s):
 Distributed under the following license(s):
 
 * MIT
+* Apache-2.0
+## opentelemetry <https://crates.io/crates/opentelemetry>
+
+Distributed under the following license(s):
+
+* Apache-2.0
+## opentelemetry-http <https://crates.io/crates/opentelemetry-http>
+
+Distributed under the following license(s):
+
+* Apache-2.0
+## opentelemetry-otlp <https://crates.io/crates/opentelemetry-otlp>
+
+Distributed under the following license(s):
+
+* Apache-2.0
+## opentelemetry-proto <https://crates.io/crates/opentelemetry-proto>
+
+Distributed under the following license(s):
+
 * Apache-2.0
 ## opentelemetry-semantic-conventions <https://crates.io/crates/opentelemetry-semantic-conventions>
 
 Distributed under the following license(s):
 
+* Apache-2.0
+## opentelemetry_sdk <https://crates.io/crates/opentelemetry_sdk>
+
+Distributed under the following license(s):
+
 * Apache-2.0
 ## ordered-float <https://crates.io/crates/ordered-float>
 
@@ -1464,6 +1489,13 @@ Distributed under the following license(s):
 
 * MIT
 * Apache-2.0
+## r-efi <https://crates.io/crates/r-efi>
+
+Distributed under the following license(s):
+
+* MIT
+* Apache-2.0
+* LGPL-2.1-or-later
 ## rand <https://crates.io/crates/rand>
 
 Distributed under the following license(s):
@@ -1934,6 +1966,11 @@ Distributed under the following license(s):
 
 Distributed under the following license(s):
 
+* MIT
+## tracing-opentelemetry <https://crates.io/crates/tracing-opentelemetry>
+
+Distributed under the following license(s):
+
 * MIT
 ## tracing-subscriber <https://crates.io/crates/tracing-subscriber>
 
@@ -2090,6 +2127,12 @@ Distributed under the following license(s):
 
 Distributed under the following license(s):
 
+* MIT
+* Apache-2.0
+## web-time <https://crates.io/crates/web-time>
+
+Distributed under the following license(s):
+
 * MIT
 * Apache-2.0
 ## webpki <https://crates.io/crates/webpki>
@@ -2270,13 +2313,6 @@ Distributed under the following license(s):
 
 Distributed under the following license(s):
 
-* BSD-2-Clause
-* Apache-2.0
-* MIT
-## zerocopy-derive <https://crates.io/crates/zerocopy-derive>
-
-Distributed under the following license(s):
-
 * BSD-2-Clause
 * Apache-2.0
 * MIT

THIRD_PARTY_NOTICES.md

@@ -17,7 +17,7 @@ serde = { workspace = true, features = ["derive"] }
 thiserror = { workspace = true }
 nix = { workspace = true, features = ["signal", "user", "hostname"] }
 tracing-subscriber = { workspace = true, features = ["env-filter", "chrono"] }
-tracing = { workspace = true }
+tracing = { workspace = true, features = ["attributes"] }
 ctrlc = { version = "3.4.5", features = ["termination"] }
 serde_yaml = { workspace = true }
 regex = { workspace = true }
@@ -67,6 +67,17 @@ rustls = { version = "0.23.23", features = ["ring"] }
 webpki = { version = "0.22.4", features = ["alloc"] }
 x509-parser = "0.17.0"
 ring = "0.17.13"
+bytes="1.10.1"
+opentelemetry = { version = "0.28.0", features = [
+  "trace",
+] }
+opentelemetry_sdk = { version = "0.28.0", features = [
+  "trace",
+] }
+tracing-opentelemetry = "0.29.0"
+opentelemetry-otlp = { version = "0.28.0", features = ["reqwest-rustls"] }
+opentelemetry-http = { version = "0.28.0"}
+async-trait = "0.1.86"
 
 [dev-dependencies]
 assert_cmd = { workspace = true }

agent-control/Cargo.toml

@@ -1,12 +1,14 @@
 use super::agent_id::AgentID;
 use super::http_server::config::ServerConfig;
-use crate::agent_type::agent_type_id::AgentTypeID;
 use crate::http::config::ProxyConfig;
-use crate::logging::config::LoggingConfig;
+use crate::instrumentation::config::logs::config::LoggingConfig;
 use crate::opamp::auth::config::AuthConfig;
 use crate::opamp::remote_config::validators::signature::validator::SignatureValidatorConfig;
 use crate::opamp::remote_config::RemoteConfigError;
 use crate::values::yaml_config::YAMLConfig;
+use crate::{
+    agent_type::agent_type_id::AgentTypeID, instrumentation::config::InstrumentationConfig,
+};
 use http::HeaderMap;
 #[cfg(feature = "k8s")]
 use kube::api::TypeMeta;
@@ -42,6 +44,9 @@ pub struct AgentControlConfig {
 
     #[serde(default)]
     pub proxy: ProxyConfig,
+
+    #[serde(default)]
+    pub self_instrumentation: InstrumentationConfig,
 }
 
 #[derive(Error, Debug)]
@@ -238,7 +243,7 @@ pub(crate) mod tests {
 
     use std::path::PathBuf;
 
-    use crate::logging::{
+    use crate::instrumentation::config::logs::{
         file_logging::{FileLoggingConfig, LogFilePath},
         format::{LoggingFormat, TimestampFormat},
     };

agent-control/src/agent_control/config.rs

@@ -10,16 +10,16 @@ use newrelic_agent_control::config_migrate::migration::defaults::NEWRELIC_INFRA_
 use newrelic_agent_control::config_migrate::migration::migrator::{ConfigMigrator, MigratorError};
 use newrelic_agent_control::config_migrate::migration::persister::legacy_config_renamer::LegacyConfigRenamer;
 use newrelic_agent_control::config_migrate::migration::persister::values_persister_file::ValuesPersisterFile;
-use newrelic_agent_control::logging::config::LoggingConfig;
+use newrelic_agent_control::instrumentation::tracing::{try_init_tracing, TracingConfig};
 use newrelic_agent_control::values::file::YAMLConfigRepositoryFile;
 use std::error::Error;
 use std::path::PathBuf;
 use std::sync::Arc;
 use tracing::{debug, info, warn};
 
 fn main() -> Result<(), Box<dyn Error>> {
-    // init logging singleton
-    LoggingConfig::default().try_init(PathBuf::from(AGENT_CONTROL_LOG_DIR))?;
+    let tracing_config = TracingConfig::from_logging_path(PathBuf::from(AGENT_CONTROL_LOG_DIR));
+    let _tracer = try_init_tracing(tracing_config);
 
     info!("Starting config conversion tool...");
 

agent-control/src/bin/config_migrate.rs

@@ -11,7 +11,7 @@ use newrelic_agent_control::cli::{AgentControlCliConfig, Cli, CliCommand};
 use newrelic_agent_control::event::channel::{pub_sub, EventPublisher};
 use newrelic_agent_control::event::ApplicationEvent;
 use newrelic_agent_control::http::tls::install_rustls_default_crypto_provider;
-use newrelic_agent_control::logging::config::FileLoggerGuard;
+use newrelic_agent_control::instrumentation::tracing::TracingGuardBox;
 use std::error::Error;
 use std::process::ExitCode;
 use tracing::{error, info, trace};
@@ -29,9 +29,9 @@ fn main() -> ExitCode {
         return ExitCode::FAILURE;
     };
 
-    let agent_control_config = match cli_command {
+    let (agent_control_config, tracer) = match cli_command {
         // Agent Control command call instructs normal operation. Continue with required data.
-        CliCommand::InitAgentControl(cli) => cli,
+        CliCommand::InitAgentControl(cli, tracer) => (cli, tracer),
 
         // Agent Control command call was a "one-shot" operation. Exit successfully after performing.
         CliCommand::OneShot(op) => {
@@ -40,7 +40,7 @@ fn main() -> ExitCode {
         }
     };
 
-    match _main(agent_control_config) {
+    match _main(agent_control_config, tracer) {
         Err(e) => {
             error!("The agent control main process exited with an error: {e}");
             ExitCode::FAILURE
@@ -62,11 +62,10 @@ fn main() -> ExitCode {
 /// Could not read Agent Control config from /invalid/path: error loading the agent control config: \`error retrieving config: \`missing field \`agents\`\`\`
 /// Error: ConfigRead(LoadConfigError(ConfigError(missing field \`agents\`)))
 /// ```
-fn _main(agent_control_config: AgentControlCliConfig) -> Result<(), Box<dyn Error>> {
-    // Acquire the file logger guard (if any) for the whole duration of the program
-    // Needed for remaining usages of `tracing` macros in `main`.
-    let _guard: FileLoggerGuard = agent_control_config.file_logger_guard;
-
+fn _main(
+    agent_control_config: AgentControlCliConfig,
+    _tracer: Vec<TracingGuardBox>, // Needs to take ownership of the tracer as it can be shutdown on drop
+) -> Result<(), Box<dyn Error>> {
     #[cfg(all(unix, feature = "onhost"))]
     if !nix::unistd::Uid::effective().is_root() {
         return Err("Program must run as root".into());
@@ -89,6 +88,7 @@ fn _main(agent_control_config: AgentControlCliConfig) -> Result<(), Box<dyn Erro
     AgentControlRunner::new(agent_control_config.run_config, application_event_consumer)?.run()?;
 
     info!("exiting gracefully");
+
     Ok(())
 }
 

agent-control/src/bin/main.rs

@@ -6,14 +6,16 @@
 mod one_shot_operation;
 #[cfg(debug_assertions)]
 use crate::agent_control::run::set_debug_dirs;
+use crate::instrumentation::tracing::{
+    try_init_tracing, TracingConfig, TracingError, TracingGuardBox,
+};
 use crate::opamp::client_builder::DEFAULT_POLL_INTERVAL;
 use crate::values::file::YAMLConfigRepositoryFile;
 use crate::{
     agent_control::{
         config_storer::{loader_storer::AgentControlConfigLoader, store::AgentControlConfigStore},
         run::{AgentControlRunConfig, BasePaths},
     },
-    logging::config::{FileLoggerGuard, LoggingError},
     utils::binary_metadata::binary_metadata,
 };
 use clap::Parser;
@@ -26,17 +28,15 @@ use tracing::info;
 pub struct AgentControlCliConfig {
     /// The configuration to run the agent control
     pub run_config: AgentControlRunConfig,
-    /// Structure that keeps the file logging active
-    pub file_logger_guard: FileLoggerGuard,
 }
 
 /// All possible errors that can happen while running the CLI.
 #[derive(Debug, Error)]
 pub enum CliError {
-    /// The logging could not be initialized
-    #[error("Could not initialize logging: `{0}`")]
-    LoggingInit(#[from] LoggingError),
-    /// The K8s config is missing
+    /// Could not initialize tracer
+    #[error("Could not initialize tracer: `{0}`")]
+    TracerError(#[from] TracingError),
+    /// K8s config is missing
     #[error("k8s config missing while running on k8s ")]
     K8sConfig(),
     /// The config could not be read
@@ -50,7 +50,7 @@ pub enum CliError {
 /// What action was requested from the CLI?
 pub enum CliCommand {
     /// Normal operation requested. Get the required config and continue.
-    InitAgentControl(AgentControlCliConfig),
+    InitAgentControl(AgentControlCliConfig, Vec<TracingGuardBox>),
     /// Do an "one-shot" operation and exit successfully.
     /// In the future, many different operations could be added here.
     OneShot(OneShotCommand),
@@ -119,9 +119,20 @@ impl Cli {
                 )
             })?;
 
-        let file_logger_guard = agent_control_config
-            .log
-            .try_init(base_paths.log_dir.clone())?;
+        let proxy = agent_control_config
+            .proxy
+            .try_with_url_from_env()
+            .map_err(|err| CliError::InvalidConfig(err.to_string()))?;
+
+        let tracing_config = TracingConfig::from_logging_path(base_paths.log_dir.clone())
+            .with_logging_config(agent_control_config.log)
+            .with_instrumentation_config(
+                agent_control_config
+                    .self_instrumentation
+                    .with_proxy_config(proxy.clone()),
+            );
+        let tracer = try_init_tracing(tracing_config)?;
+
         info!("{}", binary_metadata());
         info!(
             "Starting NewRelic Agent Control with config folder '{}'",
@@ -130,10 +141,6 @@ impl Cli {
 
         let opamp = agent_control_config.fleet_control;
         let http_server = agent_control_config.server;
-        let proxy = agent_control_config
-            .proxy
-            .try_with_url_from_env()
-            .map_err(|err| CliError::InvalidConfig(err.to_string()))?;
 
         let run_config = AgentControlRunConfig {
             opamp,
@@ -151,12 +158,9 @@ impl Cli {
             garbage_collector_interval: DEFAULT_POLL_INTERVAL - std::time::Duration::from_secs(5),
         };
 
-        let cli_config = AgentControlCliConfig {
-            run_config,
-            file_logger_guard,
-        };
+        let cli_config = AgentControlCliConfig { run_config };
 
-        Ok(CliCommand::InitAgentControl(cli_config))
+        Ok(CliCommand::InitAgentControl(cli_config, tracer))
     }
 
     fn print_version(&self) -> bool {

agent-control/src/cli.rs

@@ -1,11 +1,15 @@
 //! # Helpers to build a reqwest blocking client and handle responses and handle responses
+//!
 use crate::http::config::HttpConfig;
+use async_trait::async_trait;
+use bytes::Bytes;
 use http::Response as HttpResponse;
 use http::{Request, Response};
 use nix::NixPath;
 use nr_auth::http_client::HttpClient as OauthHttpClient;
 use nr_auth::http_client::HttpClientError as OauthHttpClientError;
 use opamp_client::http::HttpClientError as OpampHttpClientError;
+use opentelemetry_http::HttpError;
 use reqwest::tls::TlsInfo;
 use reqwest::{
     blocking::{Client, Response as BlockingResponse},
@@ -131,6 +135,24 @@ impl From<HttpResponseError> for OauthHttpClientError {
     }
 }
 
+// Implements opentelemetry_http HttpClient so it can be injected to an opentelemetry_otlp exporter
+#[async_trait]
+impl opentelemetry_http::HttpClient for HttpClient {
+    async fn send_bytes(&self, request: Request<Bytes>) -> Result<Response<Bytes>, HttpError> {
+        let (parts, body) = request.into_parts();
+        let vec_body = Vec::from(body);
+        let req_vec = Request::from_parts(parts, vec_body);
+
+        let response = self.send(req_vec)?;
+
+        let (parts, body) = response.into_parts();
+        let bytes_body = Bytes::from(body);
+        let resp_vec = Response::from_parts(parts, bytes_body);
+
+        Ok(resp_vec)
+    }
+}
+
 /// Helper to build a [HttpResponse<Vec<u8>>] from a reqwest's blocking response.
 /// It includes status, version and body. Headers are not included but they could be added if needed.
 fn try_build_response(res: BlockingResponse) -> Result<HttpResponse<Vec<u8>>, HttpResponseError> {

agent-control/src/http/client.rs

@@ -0,0 +1,3 @@
+pub mod config;
+pub mod tracing;
+pub mod tracing_layers;