feat: improved logging support

Author: jacobtread

Cargo.lock

@@ -64,7 +64,6 @@ bytes-utils = "0.1.4"
 
 # Logging
 tracing = "0.1.44"
-tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }
 
 # Serialization and JSON
 serde = { version = "1.0.228", features = ["derive"] }

Cargo.toml

@@ -45,7 +45,6 @@ serde_with.workspace = true
 
 # Logging
 tracing.workspace = true
-tracing-subscriber.workspace = true
 
 bytes.workspace = true
 

packages/docbox-http/Cargo.toml

@@ -39,7 +39,7 @@ thiserror.workspace = true
 
 # Logging
 tracing.workspace = true
-tracing-subscriber.workspace = true
+tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
 
 # Sentry
 sentry = { version = "=0.46.1", default-features = false, features = [

packages/docbox/Cargo.toml

@@ -0,0 +1,27 @@
+use thiserror::Error;
+
+use crate::logging::{
+    fmt::{LoggingFormatConfig, LoggingFormatConfigError},
+    sentry::SentryLoggingConfig,
+};
+
+/// Configuration for logging
+#[derive(Default)]
+pub struct LoggingConfig {
+    pub format: LoggingFormatConfig,
+    pub sentry: SentryLoggingConfig,
+}
+
+#[derive(Debug, Error)]
+pub enum LoggingConfigError {
+    #[error(transparent)]
+    LoggingFormatConfig(#[from] LoggingFormatConfigError),
+}
+
+impl LoggingConfig {
+    pub fn from_env() -> Result<Self, LoggingConfigError> {
+        let format = LoggingFormatConfig::from_env()?;
+        let sentry = SentryLoggingConfig::from_env()?;
+        Ok(Self { format, sentry })
+    }
+}

packages/docbox/src/logging.rs

@@ -0,0 +1,130 @@
+use std::str::{FromStr, ParseBoolError};
+
+use thiserror::Error;
+use tracing::Subscriber;
+use tracing_subscriber::{EnvFilter, Layer, fmt, registry::LookupSpan};
+
+/// Logging format to use
+#[derive(Debug, Default)]
+pub enum LoggingFormat {
+    Text,
+    #[default]
+    Json,
+}
+
+#[derive(Debug, Error)]
+#[error("unknown logging format: {0}")]
+pub struct UnknownLoggingFormat(String);
+
+impl FromStr for LoggingFormat {
+    type Err = UnknownLoggingFormat;
+
+    fn from_str(value: &str) -> Result<Self, Self::Err> {
+        match value {
+            "text" => Ok(LoggingFormat::Text),
+            "json" => Ok(LoggingFormat::Json),
+            value => Err(UnknownLoggingFormat(value.to_string())),
+        }
+    }
+}
+
+impl LoggingFormat {
+    pub fn from_env() -> Result<LoggingFormat, UnknownLoggingFormat> {
+        let format = match std::env::var("DOCBOX_LOGGING_FORMAT") {
+            Ok(value) => value,
+            Err(_) => return Ok(LoggingFormat::default()),
+        };
+
+        format.parse()
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct LoggingFormatConfig {
+    pub format: LoggingFormat,
+    pub allow_noisy: bool,
+}
+
+#[derive(Debug, Error)]
+pub enum LoggingFormatConfigError {
+    #[error(transparent)]
+    LoggingFormat(#[from] UnknownLoggingFormat),
+    #[error("failed to parse DOCBOX_LOGGING_ALLOW_NOISY")]
+    InvalidAllowNoisy(ParseBoolError),
+}
+
+impl LoggingFormatConfig {
+    pub fn from_env() -> Result<Self, LoggingFormatConfigError> {
+        let format = LoggingFormat::from_env()?;
+        let allow_noisy = std::env::var("DOCBOX_LOGGING_ALLOW_NOISY")
+            .ok()
+            .map(|value| value.parse::<bool>())
+            .transpose()
+            .map_err(LoggingFormatConfigError::InvalidAllowNoisy)?
+            .unwrap_or_default();
+
+        Ok(Self {
+            format,
+            allow_noisy,
+        })
+    }
+}
+
+/// Create a formatting layer from the provided format config
+pub fn fmt_layer<S>(config: LoggingFormatConfig) -> Box<dyn Layer<S> + Send + Sync + 'static>
+where
+    S: Subscriber + for<'a> LookupSpan<'a>,
+{
+    match config.format {
+        LoggingFormat::Text => text_fmt_layer().boxed(),
+        LoggingFormat::Json => json_fmt_layer().boxed(),
+    }
+}
+
+/// Layer the outputs content using a text based formatting
+pub fn text_fmt_layer<S>() -> fmt::Layer<S> {
+    fmt::layer()
+        // Display source code file paths
+        .with_file(true)
+        // Display source code line numbers
+        .with_line_number(true)
+        // Don't display the event's target (module path)
+        .with_target(false)
+}
+
+/// Layer that outputs content using a JSON formatting
+pub fn json_fmt_layer<S>()
+-> fmt::Layer<S, fmt::format::JsonFields, fmt::format::Format<fmt::format::Json>> {
+    fmt::layer::<S>()
+        .json()
+        .with_span_list(true)
+        // Display source code file paths
+        .with_file(true)
+        // Display source code line numbers
+        .with_line_number(true)
+        // Don't display the event's target (module path)
+        .with_target(false)
+}
+
+pub fn filter_layer(allow_noisy: bool) -> EnvFilter {
+    if allow_noisy {
+        return EnvFilter::from_default_env();
+    }
+
+    EnvFilter::from_default_env()
+        // Increase logging requirements for noisy dependencies
+        .add_directive(
+            "aws_sdk_secretsmanager=info"
+                .parse()
+                .expect("directive was invalid"),
+        )
+        .add_directive("aws_runtime=info".parse().expect("directive was invalid"))
+        .add_directive(
+            "aws_smithy_runtime=info"
+                .parse()
+                .expect("directive was invalid"),
+        )
+        .add_directive("hyper_util=info".parse().expect("directive was invalid"))
+        .add_directive("aws_sdk_sqs=info".parse().expect("directive was invalid"))
+        .add_directive("h2=info".parse().expect("directive was invalid"))
+}

packages/docbox/src/logging/config.rs

@@ -0,0 +1,40 @@
+use std::error::Error;
+
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+
+use crate::logging::{
+    config::LoggingConfig,
+    fmt::{filter_layer, fmt_layer},
+    sentry::sentry_layer,
+};
+
+pub mod config;
+pub mod fmt;
+pub mod sentry;
+
+/// Guards that must be held for active loggers
+#[derive(Default)]
+pub struct LoggingGuards {
+    sentry: Option<::sentry::ClientInitGuard>,
+}
+
+pub fn init_logging(config: LoggingConfig) -> Result<LoggingGuards, Box<dyn Error>> {
+    let mut guards = LoggingGuards::default();
+
+    let filter_layer = filter_layer(config.format.allow_noisy);
+
+    let mut sentry = None;
+
+    if let Some((sentry_layer, client_init_guard)) = sentry_layer(config.sentry) {
+        guards.sentry = Some(client_init_guard);
+        sentry = Some(sentry_layer);
+    }
+
+    tracing_subscriber::registry()
+        .with(fmt_layer(config.format))
+        .with(sentry)
+        .with(filter_layer)
+        .init();
+
+    Ok(guards)
+}

packages/docbox/src/logging/fmt.rs

@@ -0,0 +1,57 @@
+use sentry_tracing::SentryLayer;
+use tracing::{Level, Subscriber};
+use tracing_subscriber::registry::LookupSpan;
+
+use crate::logging::config::LoggingConfigError;
+
+/// Configuration for sentry logging
+#[derive(Default)]
+pub struct SentryLoggingConfig {
+    pub dsn: Option<String>,
+}
+
+impl SentryLoggingConfig {
+    pub fn from_env() -> Result<Self, LoggingConfigError> {
+        let dsn = std::env::var("SENTRY_DSN")
+            .ok()
+            .or(std::env::var("DOCBOX_SENTRY_DSN").ok());
+
+        Ok(Self { dsn })
+    }
+}
+
+pub fn sentry_layer<S>(
+    config: SentryLoggingConfig,
+) -> Option<(SentryLayer<S>, sentry::ClientInitGuard)>
+where
+    S: Subscriber + for<'a> LookupSpan<'a>,
+{
+    let dsn = config.dsn?;
+    let options = sentry::ClientOptions {
+        release: sentry::release_name!(),
+        ..Default::default()
+    };
+    let client_init_guard = sentry::init((dsn, options));
+
+    let sentry_layer = sentry_tracing::layer()
+        .event_filter(|event| {
+            match event.level() {
+                &Level::ERROR => {
+                    // Ignore errors emitted from the docbox_web_scraper when emitting
+                    // errors to sentry (These are errors caused by the upstream site)
+                    if let Some(module_path) = event.module_path()
+                        && module_path.starts_with("docbox_web_scraper")
+                    {
+                        return sentry_tracing::EventFilter::Ignore;
+                    }
+
+                    sentry_tracing::EventFilter::Event
+                }
+                &Level::WARN | &Level::INFO => sentry_tracing::EventFilter::Breadcrumb,
+                &Level::DEBUG | &Level::TRACE => sentry_tracing::EventFilter::Ignore,
+            }
+        })
+        .enable_span_attributes();
+
+    Some((sentry_layer, client_init_guard))
+}