Merge pull request #845 from NVIDIA/release/0.8

Forward-merge release/0.8 into main

Author: GPUtester

crates/core/src/observability/otel.rs

@@ -1485,16 +1485,19 @@ impl OtelEventProcessor {
         event: &Event,
         protected_keys: &HashSet<String>,
     ) {
-        let issues = promote_event_metadata_attributes(
+        let mut issues = promote_event_metadata_attributes(
             attributes,
             event,
             &self.promote_metadata_prefixes,
             protected_keys,
         );
 
+        issues.sort_by(|left, right| left.key.cmp(&right.key));
         for issue in issues {
+            let diagnostic_code =
+                format!("otel.metadata_promotion_value_unsupported.{}", issue.key);
             let diagnostic_count = self.runtime_diagnostics.record(
-                "otel.metadata_promotion_value_unsupported",
+                diagnostic_code,
                 format!(
                     "OpenTelemetry metadata attribute {:?} was not promoted: {}",
                     issue.key, issue.reason

crates/core/src/observability/otel_signal.rs

@@ -57,7 +57,7 @@ impl OpenTelemetryRuntimeDiagnostics {
 
 #[derive(Debug, Default)]
 struct RuntimeDiagnosticState {
-    diagnostics: BTreeMap<&'static str, OpenTelemetryRuntimeDiagnostic>,
+    diagnostics: BTreeMap<String, OpenTelemetryRuntimeDiagnostic>,
 }
 
 /// Shared runtime-diagnostic recorder for one independently owned OTLP subscriber.
@@ -75,21 +75,22 @@ impl SignalRuntimeDiagnostics {
         }
     }
 
-    pub(super) fn record(&self, code: &'static str, message: String, count: u64) -> u64 {
+    pub(super) fn record(&self, code: impl Into<String>, message: String, count: u64) -> u64 {
+        let code = code.into();
         let count = count.max(1);
         let mut state = self
             .state
             .lock()
             .unwrap_or_else(|poisoned| poisoned.into_inner());
-        let total = if let Some(diagnostic) = state.diagnostics.get_mut(code) {
+        let total = if let Some(diagnostic) = state.diagnostics.get_mut(&code) {
             diagnostic.message = truncate_runtime_diagnostic_message(message.clone());
             diagnostic.count = diagnostic.count.saturating_add(count);
             diagnostic.count
         } else if state.diagnostics.len() < MAX_RUNTIME_DIAGNOSTICS {
             state.diagnostics.insert(
-                code,
+                code.clone(),
                 OpenTelemetryRuntimeDiagnostic {
-                    code: code.to_string(),
+                    code: code.clone(),
                     message: truncate_runtime_diagnostic_message(message.clone()),
                     count,
                 },
@@ -100,7 +101,7 @@ impl SignalRuntimeDiagnostics {
         };
         drop(state);
 
-        record_signal_runtime_diagnostic(code, self.plugin_field.clone(), message, count);
+        record_signal_runtime_diagnostic(&code, self.plugin_field.clone(), message, count);
         total
     }
 

crates/core/tests/unit/observability/otel_tests.rs

@@ -709,13 +709,81 @@ fn omits_scope_metadata_when_final_value_is_unsupported_across_trace_projections
 
         let diagnostics = runtime_diagnostics.snapshot();
         let diagnostic = diagnostics
-            .get("otel.metadata_promotion_value_unsupported")
+            .get("otel.metadata_promotion_value_unsupported.nv.source")
             .expect("unsupported final metadata diagnostic");
         assert_eq!(diagnostic.count, 1);
         assert!(diagnostic.message.contains("nv.source"));
     }
 }
 
+#[test]
+fn reports_each_unsupported_metadata_key_with_a_deterministic_diagnostic_code() {
+    let (provider, exporter) = make_provider();
+    let runtime_diagnostics = SignalRuntimeDiagnostics::new(None);
+    let mut processor =
+        OtelEventProcessor::new_with_mark_projection_and_exclusions_and_mappings_and_runtime_diagnostics(
+            provider,
+            "unsupported-metadata-promotion-keys-test".into(),
+            OpenTelemetryType::Full,
+            MarkProjection::default(),
+            default_mark_exclude_names(),
+            Vec::new(),
+            vec!["tenant.".to_string()],
+            runtime_diagnostics.clone(),
+        );
+    let uuid = Uuid::now_v7();
+    let unsupported_metadata = json!({
+        "tenant.plan": {"name": "enterprise"},
+        "tenant.flags": null,
+        "tenant.tags": [["nested"]],
+        "tenant.mixed": [1, "string"],
+    });
+
+    processor.process(&make_start_event_with_metadata(
+        uuid,
+        None,
+        "unsupported-metadata-promotion-keys-scope",
+        unsupported_metadata.clone(),
+    ));
+    processor.process(&make_end_event_with_metadata(
+        uuid,
+        None,
+        "unsupported-metadata-promotion-keys-scope",
+        ScopeType::Agent,
+        unsupported_metadata,
+    ));
+    processor.force_flush().unwrap();
+
+    let spans = exporter.get_finished_spans().unwrap();
+    assert_eq!(spans.len(), 1);
+    let span_attributes = attr_map(&spans[0].attributes);
+    for key in ["tenant.flags", "tenant.mixed", "tenant.plan", "tenant.tags"] {
+        assert!(!span_attributes.contains_key(key));
+    }
+
+    let diagnostics = runtime_diagnostics.snapshot();
+    let expected_keys = ["tenant.flags", "tenant.mixed", "tenant.plan", "tenant.tags"];
+    assert_eq!(
+        diagnostics
+            .entries()
+            .iter()
+            .map(|diagnostic| diagnostic.code.as_str())
+            .collect::<Vec<_>>(),
+        expected_keys
+            .iter()
+            .map(|key| format!("otel.metadata_promotion_value_unsupported.{key}"))
+            .collect::<Vec<_>>()
+    );
+    for key in expected_keys {
+        let code = format!("otel.metadata_promotion_value_unsupported.{key}");
+        let diagnostic = diagnostics
+            .get(&code)
+            .expect("metadata-key-specific promotion diagnostic");
+        assert_eq!(diagnostic.count, 2);
+        assert!(diagnostic.message.contains(key));
+    }
+}
+
 #[test]
 fn promotes_start_only_scope_metadata_across_trace_projections() {
     for otel_type in [

docs/configure-plugins/observability/opentelemetry.mdx

@@ -179,8 +179,13 @@ Marks.
 
 Promotion supports strings, booleans, signed 64-bit integers, floating-point
 numbers, empty arrays, and homogeneous arrays of those primitive types. Relay
-omits rejected values and records a bounded runtime diagnostic containing the
-key and reason, but it does not record the rejected value or stop trace export.
+omits rejected values and records one bounded runtime diagnostic per rejected
+key. The diagnostic code is
+`otel.metadata_promotion_value_unsupported.<metadata-key>`, its message contains
+the key and rejection reason, and its count is the number of occurrences for
+that key. Relay does not record the rejected value or stop trace export. Match
+the `otel.metadata_promotion_value_unsupported.` prefix to monitor all rejected
+metadata keys.
 
 Projection-owned attributes take precedence over promoted metadata with the
 same key. For `full` and `openinference`, configured attribute-mapping aliases