Apply histogram bucket overrides consistently (#884)

Author: Sushisource

core/src/telemetry/otel.rs

@@ -52,13 +52,22 @@ fn histo_view(
 }
 
 pub(super) fn augment_meter_provider_with_defaults(
-    mpb: MeterProviderBuilder,
+    mut mpb: MeterProviderBuilder,
     global_tags: &HashMap<String, String>,
     use_seconds: bool,
     bucket_overrides: HistogramBucketOverrides,
 ) -> opentelemetry::metrics::Result<MeterProviderBuilder> {
-    // Some histograms are actually gauges, but we have to use histograms otherwise they forget
-    // their value between collections since we don't use callbacks.
+    for (name, buckets) in bucket_overrides.overrides {
+        mpb = mpb.with_view(new_view(
+            Instrument::new().name(format!("*{name}")),
+            opentelemetry_sdk::metrics::Stream::new().aggregation(
+                Aggregation::ExplicitBucketHistogram {
+                    boundaries: buckets,
+                    record_min_max: true,
+                },
+            ),
+        )?)
+    }
     let mut mpb = mpb
         .with_view(histo_view(
             WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME,
@@ -84,17 +93,6 @@ pub(super) fn augment_meter_provider_with_defaults(
             ACTIVITY_EXEC_LATENCY_HISTOGRAM_NAME,
             use_seconds,
         )?);
-    for (name, buckets) in bucket_overrides.overrides {
-        mpb = mpb.with_view(new_view(
-            Instrument::new().name(format!("*{name}")),
-            opentelemetry_sdk::metrics::Stream::new().aggregation(
-                Aggregation::ExplicitBucketHistogram {
-                    boundaries: buckets,
-                    record_min_max: true,
-                },
-            ),
-        )?)
-    }
     // Fallback default
     mpb = mpb.with_view(new_view(
         {

tests/integ_tests/metrics_tests.rs

@@ -18,7 +18,10 @@ use temporal_sdk::{
 };
 use temporal_sdk_core::{
     CoreRuntime, TokioRuntimeBuilder, init_worker,
-    telemetry::{build_otlp_metric_exporter, start_prometheus_metric_exporter},
+    telemetry::{
+        WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME, build_otlp_metric_exporter,
+        start_prometheus_metric_exporter,
+    },
 };
 use temporal_sdk_core_api::{
     Worker,
@@ -585,6 +588,16 @@ async fn latency_metrics(
             .socket_addr(ANY_PORT.parse().unwrap())
             .use_seconds_for_durations(use_seconds_latency)
             .unit_suffix(show_units)
+            .histogram_bucket_overrides(HistogramBucketOverrides {
+                overrides: {
+                    let mut hm = HashMap::new();
+                    hm.insert(
+                        WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME.to_string(),
+                        vec![1337.0],
+                    );
+                    hm
+                },
+            })
             .build()
             .unwrap(),
     ));
@@ -620,6 +633,12 @@ async fn latency_metrics(
         assert!(matching_line.contains("le=\"100\""));
     }
 
+    let matching_line = body
+        .lines()
+        .find(|l| l.starts_with("temporal_workflow_task_execution_latency"))
+        .unwrap();
+    assert!(matching_line.contains("le=\"1337\""));
+
     // Ensure poll metrics show up as long polls properly
     let matching_lines = body
         .lines()