Merge pull request #14700 from Turbo87/add-otel-metrics

Add `opentelemetry` metrics support

Author: Turbo87

Cargo.lock

@@ -98,6 +98,7 @@ crates_io_env_vars = { path = "crates/crates_io_env_vars" }
 crates_io_fastly = { path = "crates/crates_io_fastly" }
 crates_io_github = { path = "crates/crates_io_github" }
 crates_io_github_app = { path = "crates/crates_io_github_app" }
+crates_io_heroku = { path = "crates/crates_io_heroku" }
 crates_io_index = { path = "crates/crates_io_index" }
 crates_io_linecount = { path = "crates/crates_io_linecount" }
 crates_io_markdown = { path = "crates/crates_io_markdown" }
@@ -139,6 +140,9 @@ moka = { version = "=0.12.16", default-features = false, features = ["future"] }
 native-tls = "=0.2.18"
 oauth2 = { version = "=5.0.0", default-features = false }
 object_store = { version = "=0.14.1", features = ["aws"] }
+opentelemetry = { version = "=0.32.0", default-features = false, features = ["metrics"] }
+opentelemetry-otlp = { version = "=0.32.0", default-features = false, features = ["gzip-http", "http-proto", "internal-logs", "metrics", "reqwest-blocking-client", "zstd-http"] }
+opentelemetry_sdk = { version = "=0.32.1", default-features = false, features = ["metrics"] }
 p256 = "=0.14.0"
 parking_lot = "=0.12.5"
 paste = "=1.0.15"
@@ -171,6 +175,7 @@ typomania = { version = "=0.2.0", default-features = false }
 url = "=2.5.8"
 utoipa = { version = "=5.5.0", features = ["chrono"] }
 utoipa-axum = "=0.2.0"
+uuid = { version = "=1.26.1", features = ["v4"] }
 
 [dev-dependencies]
 bytes = "=1.12.1"
@@ -188,6 +193,7 @@ diesel = { version = "=2.3.13", features = ["r2d2"] }
 googletest = "=0.14.3"
 insta = { version = "=1.48.0", features = ["glob", "json", "redactions"] }
 jsonwebtoken = { version = "=11.0.0", features = ["aws_lc_rs"] }
+opentelemetry_sdk = { version = "=0.32.1", default-features = false, features = ["testing"] }
 quoted_printable = "=0.5.2"
 sentry = { version = "=0.49.2", features = ["test"] }
 tokio = "=1.53.1"

Cargo.toml

@@ -2,6 +2,48 @@
 
 use crates_io_env_vars::var;
 
+/// Returns whether the process is running on Heroku, as indicated by `HEROKU`.
+pub fn is_heroku() -> anyhow::Result<bool> {
+    Ok(var("HEROKU")?.is_some())
+}
+
+/// Returns the dyno name from `DYNO`, or `None` when unset.
+///
+/// An example value may be: `"web.1"`.
+pub fn dyno() -> anyhow::Result<Option<String>> {
+    var("DYNO")
+}
+
+/// Returns the dyno UUID from `HEROKU_DYNO_ID`, or `None` when unset.
+///
+/// An example value may be: `"1vac4117-c29f-4312-521e-ba4d8638c1ac"`.
+pub fn dyno_id() -> anyhow::Result<Option<String>> {
+    var("HEROKU_DYNO_ID")
+}
+
+/// Returns the release identifier from `HEROKU_RELEASE_VERSION`, or `None` when
+/// unset.
+///
+/// An example value may be: `"v42"`.
+pub fn release_version() -> anyhow::Result<Option<String>> {
+    var("HEROKU_RELEASE_VERSION")
+}
+
+/// Returns the application UUID from `HEROKU_APP_ID`, or `None` when unset.
+///
+/// An example value may be: `"9daa2797-e49b-4624-932f-ec3f9688e3da"`.
+pub fn app_id() -> anyhow::Result<Option<String>> {
+    var("HEROKU_APP_ID")
+}
+
+/// Returns the release timestamp from `HEROKU_RELEASE_CREATED_AT`, or `None`
+/// when unset.
+///
+/// An example value may be: `"2015-04-02T18:00:42Z"`.
+pub fn release_created_at() -> anyhow::Result<Option<String>> {
+    var("HEROKU_RELEASE_CREATED_AT")
+}
+
 /// Returns the Git SHA of the currently deployed commit.
 ///
 /// This function tries `HEROKU_BUILD_COMMIT` first (the current standard),

crates/crates_io_heroku/src/lib.rs

@@ -16,6 +16,8 @@ use crates_io::Emails;
 use crates_io::cloudfront::CloudFront;
 use crates_io::config::SharedConfig;
 use crates_io::db;
+use crates_io::metrics::consts::METER_NAME;
+use crates_io::metrics::{WorkerMetrics, meter_provider};
 use crates_io::ssh;
 use crates_io::storage::Storage;
 use crates_io::worker::{RunnerExt, WorkerContext};
@@ -56,6 +58,10 @@ pub fn run() -> anyhow::Result<()> {
     // increase the statement timeout a bit…
     config.db.primary.statement_timeout = Duration::from_secs(4 * 60 * 60);
 
+    let meter_provider = meter_provider(&config);
+    let meter = meter_provider.meter(METER_NAME);
+    let worker_metrics = WorkerMetrics::new(&meter);
+
     let runtime = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()
@@ -79,7 +85,9 @@ pub fn run() -> anyhow::Result<()> {
 
     let user_agent = crates_io_version::user_agent();
     let http_client = Client::builder().user_agent(user_agent).build()?;
-    let datadog = config.datadog.client(http_client.clone()).map(Arc::new);
+    let datadog = (!config.metrics.otlp_enabled)
+        .then(|| config.datadog.client(http_client.clone()).map(Arc::new))
+        .flatten();
 
     let cloudfront = CloudFront::from_environment();
     let storage = Arc::new(Storage::from_config(&config.storage));
@@ -103,9 +111,11 @@ pub fn run() -> anyhow::Result<()> {
     let sync_github_app = build_sync_github_app()?;
 
     let deadpool = db::create_pool(&config.db.primary);
+    worker_metrics.track_db_pool("worker", &deadpool);
 
     let ctx = WorkerContext::builder()
         .config(Arc::new(config))
+        .metrics(worker_metrics)
         .repository_config(repository_config)
         .maybe_cloudfront(cloudfront)
         .maybe_fastly(fastly)
@@ -143,7 +153,12 @@ pub fn run() -> anyhow::Result<()> {
 
     runtime.block_on(async {
         let handle = runner.start();
-        crates_io::metrics::datadog::spawn(&ctx.config, ctx.deadpool.clone(), datadog);
+        crates_io::metrics::datadog::spawn(
+            &ctx.config,
+            ctx.deadpool.clone(),
+            ctx.metrics.clone(),
+            datadog,
+        );
 
         info!("Runner booted, running jobs");
         handle.wait_for_shutdown().await

src/bin/crates-io/background_worker.rs

@@ -1,6 +1,8 @@
 use crates_io::config::SharedConfig;
+use crates_io::metrics::consts::METER_NAME;
+use crates_io::metrics::{LogEncoder, ServerMetrics};
 use crates_io::middleware::normalize_path::normalize_path;
-use crates_io::{Emails, ServerContext, metrics::LogEncoder};
+use crates_io::{Emails, ServerContext};
 use std::{sync::Arc, time::Duration};
 
 use axum::ServiceExt;
@@ -25,6 +27,8 @@ pub fn run() -> anyhow::Result<()> {
     let _span = info_span!("server.run");
 
     let config = SharedConfig::from_environment()?;
+    let meter_provider = crates_io::metrics::meter_provider(&config);
+    let meter = meter_provider.meter(METER_NAME);
 
     let emails = Emails::from_environment(&config);
 
@@ -41,10 +45,16 @@ pub fn run() -> anyhow::Result<()> {
         .trustpub_providers(&list("TRUSTPUB_PROVIDERS")?)
         .emails(emails)
         .storage_from_config(&config.storage)
+        .metrics(ServerMetrics::new(&meter))
         .rate_limiter_from_config(config.rate_limits.actions.clone())
         .config(Arc::new(config))
         .build();
 
+    ctx.metrics.track_db_pool("primary", &ctx.primary_database);
+    if let Some(pool) = &ctx.replica_database {
+        ctx.metrics.track_db_pool("replica", pool);
+    }
+
     // Start the background thread periodically logging instance metrics.
     log_instance_metrics_thread(ctx.clone());