feat(tracing): export OTLP trace metrics

ddtrace/internal/native/_native.pyi

@@ -648,6 +648,9 @@ class TraceExporterBuilder:
         :param process_tags: Comma-separated list of key:value process tags (e.g., "key1:val1,key2:val2").
         """
         ...
+    def set_tracer_tags(self, tracer_tags: list[str]) -> TraceExporterBuilder:
+        """Set tracer tags on the OTLP metrics resource."""
+        ...
     def set_tracer_version(self, version: str) -> TraceExporterBuilder:
         """
         Set the tracer version of the TraceExporter.
@@ -716,6 +719,10 @@ class TraceExporterBuilder:
         :param bucket_size_ns: The size of stats bucket in nanoseconds.
         """
 
+    def set_additional_metric_tag_keys(self, tag_keys: list[str]) -> TraceExporterBuilder:
+        """Set span tag keys included in computed stats."""
+        ...
+
     def enable_client_side_stats_obfuscation(self) -> TraceExporterBuilder:
         """
         Obfuscate client side stats buckets in the client instead of in the agent.

ddtrace/internal/settings/_config.py

@@ -658,6 +658,11 @@ def __init__(self) -> None:
             "DD_TRACE_STATS_COMPUTATION_ENABLED", trace_compute_stats_default, asbool
         )
         self._otel_stats_computation_enabled = _get_config("OTEL_TRACES_SPAN_METRICS_ENABLED", None, asbool)
+        self._trace_stats_additional_tags = _get_config(
+            "DD_TRACE_STATS_ADDITIONAL_TAGS",
+            [],
+            lambda value: [tag.strip() for tag in value.split(",") if tag.strip()],
+        )
         self._client_side_stats_obfuscation = _get_config(
             "_DD_TRACE_STATS_COMPUTATION_EXPERIMENTAL_CLIENT_OBFUSCATION_ENABLED", True, asbool
         )

ddtrace/internal/settings/_supported_configurations.py

@@ -668,6 +668,7 @@
         "DD_TRACE_SQLITE_ENABLED",
         "DD_TRACE_STARLETTE_ENABLED",
         "DD_TRACE_STARTUP_LOGS",
+        "DD_TRACE_STATS_ADDITIONAL_TAGS",
         "DD_TRACE_STATS_COMPUTATION_ENABLED",
         "DD_TRACE_STRUCTLOG_ENABLED",
         "DD_TRACE_SUBPROCESS_ENABLED",

ddtrace/internal/writer/writer.py

@@ -69,6 +69,7 @@
 log = get_logger(__name__)
 
 LOG_ERR_INTERVAL = 60
+_OTLP_TRACER_TAG_RESERVED_KEYS = frozenset({"service", "env", "version", "runtime_id", "runtime-id"})
 
 
 def _safelog(log_func: Callable[..., None], msg: str, *args, **kwargs) -> None:
@@ -761,6 +762,16 @@ def _build_base_exporter_builder(
         builder.set_app_version(config.version)
     if test_session_token is not None:
         builder.set_test_session_token(test_session_token)
+    if otlp_metrics_enabled:
+        tracer_tags = [
+            f"{key}:{value}"
+            for key, value in sorted(config.tags.items())
+            if key.lower() not in _OTLP_TRACER_TAG_RESERVED_KEYS and value
+        ]
+        if tracer_tags:
+            builder.set_tracer_tags(tracer_tags)
+    if config._trace_stats_additional_tags:
+        builder.set_additional_metric_tag_keys(config._trace_stats_additional_tags)
     # OTLP trace metrics require the native concentrator regardless of DD_TRACE_STATS_COMPUTATION_ENABLED.
     if otlp_metrics_enabled or (compute_stats_enabled and not stats_opt_out):
         if otlp_metrics_enabled:

releasenotes/notes/add-otlp-trace-metrics-83cad1e1cdcc65c7.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    tracing: Adds support for exporting Datadog trace metrics when OTLP trace export is enabled. Set ``OTEL_TRACES_SPAN_METRICS_ENABLED=true`` to enable it.
+  - |
+    tracing: Adds support for ``DD_TRACE_STATS_ADDITIONAL_TAGS``, which adds configured span tags to native v0.6 trace stats and OTLP trace metrics.

src/native/data_pipeline/mod.rs

@@ -80,6 +80,14 @@ impl TraceExporterBuilderPy {
         Ok(slf.into())
     }
 
+    fn set_tracer_tags(
+        mut slf: PyRefMut<'_, Self>,
+        tracer_tags: Vec<String>,
+    ) -> PyResult<Py<Self>> {
+        slf.try_as_mut()?.set_tracer_tags(tracer_tags);
+        Ok(slf.into())
+    }
+
     fn set_tracer_version(mut slf: PyRefMut<'_, Self>, version: &'_ str) -> PyResult<Py<Self>> {
         slf.try_as_mut()?.set_tracer_version(version);
         Ok(slf.into())
@@ -152,6 +160,14 @@ impl TraceExporterBuilderPy {
         Ok(slf.into())
     }
 
+    fn set_additional_metric_tag_keys(
+        mut slf: PyRefMut<'_, Self>,
+        tag_keys: Vec<String>,
+    ) -> PyResult<Py<Self>> {
+        slf.try_as_mut()?.set_additional_metric_tag_keys(tag_keys);
+        Ok(slf.into())
+    }
+
     fn enable_client_side_stats_obfuscation(mut slf: PyRefMut<'_, Self>) -> PyResult<Py<Self>> {
         slf.try_as_mut()?.enable_client_side_stats_obfuscation();
         Ok(slf.into())

supported-configurations.json

@@ -5007,6 +5007,14 @@
         "default": "false"
       }
     ],
+    "DD_TRACE_STATS_ADDITIONAL_TAGS": [
+      {
+        "implementation": "B",
+        "type": "string",
+        "default": null,
+        "experimental": true
+      }
+    ],
     "DD_TRACE_STATS_COMPUTATION_ENABLED": [
       {
         "implementation": "C",

tests/integration/test_trace_stats.py

@@ -1,5 +1,9 @@
+import base64
+import contextlib
 import functools
+import uuid
 
+import msgpack
 import pytest
 
 from ddtrace.constants import _SPAN_MEASURED_KEY
@@ -11,6 +15,31 @@
 pytestmark = pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent")
 
 
+@contextlib.contextmanager
+def _trace_stats_context(generate_stats):
+    from ddtrace import tracer
+    from tests.utils import TestAgentClient
+
+    token = uuid.uuid4().hex
+    tracer._span_aggregator.writer.set_test_session_token(token)
+    agent_url = tracer.agent_trace_url
+    assert agent_url is not None
+    client = TestAgentClient(agent_url, token)
+    connection = client.create_connection()
+    connection.request("GET", f"/test/session/start?test_session_token={token}")
+    assert connection.getresponse().status == 200
+    connection.close()
+
+    try:
+        generate_stats(tracer)
+        tracer.shutdown()
+        requests = [request for request in client.requests() if "/v0.6/stats" in request["url"]]
+        payloads = [msgpack.unpackb(base64.b64decode(request["body"]), raw=False) for request in requests]
+        yield payloads
+    finally:
+        client.clear()
+
+
 @pytest.fixture
 def stats_tracer(tracer):
     # Recreate tracer with stats enabled
@@ -71,6 +100,26 @@ def test_stats_30(send_once_stats_tracer):
             pass
 
 
+@pytest.mark.subprocess(
+    env={
+        "DD_TRACE_STATS_ADDITIONAL_TAGS": "customer.tier,region",
+        "DD_TRACE_STATS_COMPUTATION_ENABLED": "true",
+    }
+)
+def test_stats_additional_tags_v06_payload():
+    from tests.integration.test_trace_stats import _trace_stats_context
+
+    def generate_stats(tracer):
+        with tracer.trace("additional-tags", service="test") as span:
+            span._set_attribute("customer.tier", "gold")
+            span._set_attribute("region", "us-east-1")
+
+    with _trace_stats_context(generate_stats) as payloads:
+        groups = [group for payload in payloads for bucket in payload["Stats"] for group in bucket["Stats"]]
+        group = next(group for group in groups if group["Name"] == "additional-tags")
+        assert group["AdditionalMetricTags"] == ["customer.tier:gold", "region:us-east-1"]
+
+
 @pytest.mark.snapshot()
 def test_stats_errors(send_once_stats_tracer):
     for i in range(30):

tests/tracer/test_writer.py

@@ -1227,6 +1227,37 @@ def test_writer_telemetry_enabled_on_linux(
                 mock_builder.enable_telemetry.assert_not_called()
 
 
+@pytest.mark.subprocess(
+    env={
+        "DD_TAGS": "team:apm,tier:backend,service:ignored,env:ignored,version:ignored,runtime_id:ignored",
+        "DD_TRACE_STATS_ADDITIONAL_TAGS": "customer.tier,region",
+    }
+)
+def test_otlp_metric_tags_configured():
+    from unittest import mock
+
+    from ddtrace.internal import native
+    from ddtrace.internal.writer.writer import _build_base_exporter_builder
+
+    mock_builder = mock.Mock()
+    for method_name in [
+        "set_url",
+        "set_language",
+        "set_language_version",
+        "set_language_interpreter",
+        "set_tracer_version",
+        "set_git_commit_sha",
+        "set_client_computed_top_level",
+    ]:
+        getattr(mock_builder, method_name).return_value = mock_builder
+
+    with mock.patch.object(native, "TraceExporterBuilder", return_value=mock_builder):
+        _build_base_exporter_builder("http://localhost:8126", None, False, False, True)
+
+    mock_builder.set_tracer_tags.assert_called_once_with(["team:apm", "tier:backend"])
+    mock_builder.set_additional_metric_tag_keys.assert_called_once_with(["customer.tier", "region"])
+
+
 class TestSafelog:
     """Tests for the _safelog function that handles closed I/O streams gracefully."""