fix(otlp): Ensure all OTLP spans get the full 128-bit trace IDs (#8618)

Previously the upper 64 bits (present in the '_dd.p.tid' tag) were only copied onto the first span of the trace chunk, but not the rest.

* Move the conversion to 128-bit hex trace ID to the id.js file

* Apply PR feedback to inline 128-bit trace ID for B3 propagation

* Apply PR feedback to inline 128-bit trace ID for DatadogSpanContext.toTraceId()

* Apply PR feedback to improve performance of the 128-bit trace ID lookup in #transformScopeSpans by relying on implementation detail that the '_dd.p.tid' trace tag is only applied to the first span

Author: zacharycmontoya

packages/dd-trace/src/id.js

@@ -78,6 +78,21 @@ class Identifier {
     return this.toString()
   }
 
+  /**
+   * Returns the full hex trace ID. When this is a 64-bit identifier and `traceIdHigh`
+   * is provided, prepends it to form the 128-bit trace ID. Otherwise returns
+   * only this identifier's hex representation.
+   *
+   * @param {string | undefined} traceIdHigh - 16-char hex of the upper 64 bits, or undefined
+   * @returns {string}
+   */
+  toTraceIdHex (traceIdHigh) {
+    if (traceIdHigh && this.#buffer.length <= 8) {
+      return traceIdHigh + this.toString(16)
+    }
+    return this.toString(16)
+  }
+
   /**
    * @param {Identifier} other
    * @returns {boolean}

packages/dd-trace/src/opentelemetry/trace/otlp_transformer.js

@@ -16,6 +16,11 @@ const SPAN_KIND_CONSUMER = protoSpanKind.values.SPAN_KIND_CONSUMER
 // Cached zero Identifier used to detect zero IDs without re-allocating per span.
 const ZERO_ID = id('0')
 
+// DD propagation tag carrying the upper 64 bits of a 128-bit trace ID as 16 hex chars.
+// span_format.js#extractChunkTags only copies this onto the first-in-chunk span, so the
+// transformer scans the batch to find it and applies it to every span's traceId.
+const TRACE_ID_128 = '_dd.p.tid'
+
 /**
  * @typedef {import('../../id').Identifier} Identifier
  *
@@ -65,6 +70,7 @@ const STATUS_CODE_ERROR = 2
 const EXCLUDED_META_KEYS = new Set([
   '_dd.span_links',
   'span.kind',
+  TRACE_ID_128,
 ])
 
 /**
@@ -113,6 +119,18 @@ class OtlpTraceTransformer extends OtlpTransformerBase {
    * @returns {object[]} Array of scope span objects
    */
   #transformScopeSpans (spans) {
+    let traceKey
+    let traceIdHigh
+    const otlpSpans = spans.map((span) => {
+      // `_dd.p.tid` lives only on the first-in-chunk span of each trace.
+      // Reset at each trace boundary for batching of multiple traces.
+      const key = span.trace_id.toString(16)
+      if (key !== traceKey) {
+        traceKey = key
+        traceIdHigh = span.meta?.[TRACE_ID_128]?.toLowerCase()
+      }
+      return this.#transformSpan(span, traceIdHigh)
+    })
     return [{
       scope: {
         name: 'dd-trace-js',
@@ -121,22 +139,23 @@ class OtlpTraceTransformer extends OtlpTransformerBase {
         droppedAttributesCount: 0,
       },
       schemaUrl: '',
-      spans: spans.map(span => this.#transformSpan(span)),
+      spans: otlpSpans,
     }]
   }
 
   /**
    * Transforms a single DD-formatted span to an OTLP Span object.
    *
    * @param {DDFormattedSpan} span - DD-formatted span to transform
+   * @param {string | undefined} traceIdHigh - 16-char hex of the upper 64 bits of the trace ID
    * @returns {object} OTLP Span object
    */
-  #transformSpan (span) {
+  #transformSpan (span, traceIdHigh) {
     const parentId = span.parent_id
     const links = this.#extractLinks(span.meta?.['_dd.span_links'])
 
     return {
-      traceId: this.#idToBytes(span.trace_id, 16),
+      traceId: span.trace_id.toTraceIdHex(traceIdHigh).padStart(32, '0'),
       spanId: this.#idToBytes(span.span_id, 8),
       parentSpanId: (parentId && !parentId.equals(ZERO_ID)) ? this.#idToBytes(parentId, 8) : undefined,
       name: span.resource,

packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -284,7 +284,7 @@ class TextMapPropagator {
       (DD_MAJOR < 6 && this._hasPropagationStyle('inject', 'b3'))
     if (!hasB3multi) return
 
-    carrier[b3TraceKey] = this._getB3TraceId(spanContext)
+    carrier[b3TraceKey] = spanContext._traceId.toTraceIdHex(spanContext._trace.tags['_dd.p.tid'])
     carrier[b3SpanKey] = spanContext._spanId.toString(16)
     carrier[b3SampledKey] = spanContext._sampling.priority >= AUTO_KEEP ? '1' : '0'
 
@@ -303,7 +303,7 @@ class TextMapPropagator {
       (DD_MAJOR >= 6 && this._hasPropagationStyle('inject', 'b3'))
     if (!hasB3SingleHeader) return
 
-    const traceId = this._getB3TraceId(spanContext)
+    const traceId = spanContext._traceId.toTraceIdHex(spanContext._trace.tags['_dd.p.tid'])
     const spanId = spanContext._spanId.toString(16)
     const sampled = spanContext._sampling.priority >= AUTO_KEEP ? '1' : '0'
 
@@ -859,14 +859,6 @@ class TextMapPropagator {
     }
   }
 
-  _getB3TraceId (spanContext) {
-    if (spanContext._traceId.toBuffer().length <= 8 && spanContext._trace.tags['_dd.p.tid']) {
-      return spanContext._trace.tags['_dd.p.tid'] + spanContext._traceId.toString(16)
-    }
-
-    return spanContext._traceId.toString(16)
-  }
-
   /**
    * @param {number} traceparentSampled
    * @param {number|undefined} tracestateSamplingPriority

packages/dd-trace/src/opentracing/span_context.js

@@ -46,9 +46,7 @@ class DatadogSpanContext {
 
   toTraceId (get128bitId = false) {
     if (get128bitId) {
-      return this._traceId.toBuffer().length <= 8 && this._trace.tags[TRACE_ID_128]
-        ? this._trace.tags[TRACE_ID_128] + this._traceId.toString(16).padStart(16, '0')
-        : this._traceId.toString(16).padStart(32, '0')
+      return this._traceId.toTraceIdHex(this._trace.tags[TRACE_ID_128]).padStart(32, '0')
     }
     return this._traceId.toString(10)
   }

packages/dd-trace/test/opentelemetry/traces.spec.js

@@ -462,6 +462,98 @@ describe('OpenTelemetry Traces', () => {
         ['/api/first', '/api/second']
       )
     })
+
+    describe('128-bit trace ID handling', () => {
+      // DD splits 128-bit trace IDs: low 64 bits live on the span Identifier,
+      // upper 64 bits live in trace-level tags as `_dd.p.tid` (16 hex chars).
+      // span_format.js#extractChunkTags only copies trace-level tags onto the
+      // first-in-chunk span, so the transformer has to look across the batch
+      // to find `_dd.p.tid` and then apply it to every span.
+
+      it('reconstructs the full 128-bit traceId for every span in a batch from _dd.p.tid', () => {
+        const transformer = new OtlpTraceTransformer({})
+        const lowHex = 'abcdef0123456789'
+        const tidHigh = '1234567890abcdef'
+        const traceIdLow = id(lowHex)
+
+        const firstSpan = createMockSpan({
+          trace_id: traceIdLow,
+          meta: { 'span.kind': 'internal', '_dd.p.tid': tidHigh },
+        })
+        const secondSpan = createMockSpan({
+          trace_id: traceIdLow,
+          span_id: id('bbbbbbbbbbbbbbbb'),
+          meta: { 'span.kind': 'internal' },
+        })
+        const thirdSpan = createMockSpan({
+          trace_id: traceIdLow,
+          span_id: id('cccccccccccccccc'),
+          meta: { 'span.kind': 'internal' },
+        })
+
+        const decoded = decodePayload(transformer.transformSpans([firstSpan, secondSpan, thirdSpan]))
+        const otlpSpans = decoded.resourceSpans[0].scopeSpans[0].spans
+
+        const expectedTraceId = tidHigh + lowHex
+        for (const otlpSpan of otlpSpans) {
+          assert.strictEqual(otlpSpan.traceId, expectedTraceId)
+        }
+      })
+
+      it('drops _dd.p.tid from OTLP attributes once consumed into traceId', () => {
+        const transformer = new OtlpTraceTransformer({})
+        const span = createMockSpan({
+          trace_id: id('abcdef0123456789'),
+          meta: { 'span.kind': 'internal', '_dd.p.tid': '1234567890abcdef' },
+        })
+
+        const decoded = decodePayload(transformer.transformSpans([span]))
+        const attrs = extractAttrs(decoded.resourceSpans[0].scopeSpans[0].spans[0].attributes)
+
+        assert.strictEqual(attrs['_dd.p.tid'], undefined)
+      })
+
+      it('zero-pads traceId to 32 hex chars when no _dd.p.tid is present', () => {
+        const transformer = new OtlpTraceTransformer({})
+        const span = createMockSpan({
+          trace_id: id('abcdef0123456789'),
+          meta: { 'span.kind': 'internal' },
+        })
+
+        const decoded = decodePayload(transformer.transformSpans([span]))
+        const otlpSpan = decoded.resourceSpans[0].scopeSpans[0].spans[0]
+
+        assert.strictEqual(otlpSpan.traceId, '0000000000000000abcdef0123456789')
+      })
+
+      it('lowercases an uppercase _dd.p.tid so the OTLP traceId is canonical lowercase hex', () => {
+        const transformer = new OtlpTraceTransformer({})
+        const lowHex = 'abcdef0123456789'
+        const span = createMockSpan({
+          trace_id: id(lowHex),
+          meta: { 'span.kind': 'internal', '_dd.p.tid': '1234567890ABCDEF' },
+        })
+
+        const decoded = decodePayload(transformer.transformSpans([span]))
+        const otlpSpan = decoded.resourceSpans[0].scopeSpans[0].spans[0]
+
+        assert.strictEqual(otlpSpan.traceId, '1234567890abcdef' + lowHex)
+      })
+
+      it('uses a full 16-byte trace_id buffer directly without consulting _dd.p.tid', () => {
+        const transformer = new OtlpTraceTransformer({})
+        const unusedTidHigh = '1000000000000000'
+        const span = createMockSpan({
+          trace_id: id('1234567890abcdef1234567890abcdef'),
+          meta: { 'span.kind': 'internal', '_dd.p.tid': unusedTidHigh },
+        })
+
+        const decoded = decodePayload(transformer.transformSpans([span]))
+        const otlpSpan = decoded.resourceSpans[0].scopeSpans[0].spans[0]
+
+        assert.strictEqual(otlpSpan.traceId, '1234567890abcdef1234567890abcdef')
+      })
+    })
   })
 
   describe('Exporter', () => {