Publish OTEP-4947 process-context metadata via process discovery

Sets up the OTEP-4719 process context so an out-of-process reader can decode
the on-the-wire records the thread-context writer emits. The metadata is
published through libdatadog-nodejs's process-discovery napi crate: bumped
here to 0.12.0, which exposes the ThreadLocalMetadata substruct with the full
'threadlocal.*' block (attribute key map, schema-version string, and extra
KeyValues for reader-side layout constants).

The pieces:

- Add getThreadLocalMetadata() in otel-thread-ctx.js. Pulls the process-context
  snapshot from @datadog/pprof (its otelThreadCtx.getProcessContextAttributes
  is the source of truth for the schema-version string and V8 layout constants
  the reader needs) and reshapes it into the napi ThreadLocalMetadata form:
  { attributeKeys, schemaVersion, extraAttributes: [{ key, intValue|stringValue }] }.
  Returns undefined when @datadog/pprof isn't installed or doesn't expose the
  helper.

- Wire tracer_metadata.js to pass the substruct (or undefined) as the last
  positional arg to processDiscovery.TracerMetadata(...), replacing the flat
  threadlocalAttributeKeys array. Gated on the same DD_TRACE_OTEL_CTX_ENABLED
  flag that activates the writer.

- Bump the @DataDog/libdatadog optionalDependency from 0.10.0 to 0.12.0.

Author: szegedi

packages/dd-trace/src/otel-thread-ctx.js

@@ -44,12 +44,26 @@ const {
 // convention (mirrors libdatadog's libdd-otel-thread-ctx, where
 // `local_root_span_id` is always the first entry in
 // `threadlocal.attribute_key_map`), encoded as a 16-character lowercase
-// hex string. Endpoint, thread name, and thread id follow.
+// hex string. Endpoint, thread name, and thread id follow. Adding more
+// means assigning the next index and updating ATTRIBUTE_KEYS
+// accordingly.
 const LOCAL_ROOT_SPAN_ID_IDX = 0
 const ENDPOINT_IDX = 1
 const THREAD_NAME_IDX = 2
 const THREAD_ID_IDX = 3
 
+// The dd-trace-js-supplied subset of the OTEP-4719 attribute_key_map
+// (the implicit `datadog.local_root_span_id` at wire index 0 is
+// prepended by libdatadog when it publishes the process context, so it
+// is NOT listed here). Index N here corresponds to wire key index N+1.
+// Kept in sync with the positional indices above.
+// Also see https://docs.google.com/document/d/1IwjjVJzEChcFPcnVV2N5Kkjg-4_Q4v4Q3ojpxntbdvY/edit?pli=1&tab=t.efaosgjya44c#bookmark=id.700gvw31vb7h
+const ATTRIBUTE_KEYS = [
+  'datadog.trace_endpoint',
+  'datadog.thread_name',
+  'datadog.thread_id',
+]
+
 // Stable per-thread values baked into every record. Same shape as the
 // profiler's `eventLoopThreadName` in profiling/profilers/shared.js.
 const THREAD_NAME = (isMainThread ? 'Main' : `Worker #${threadId}`) + ' Event Loop'
@@ -225,4 +239,51 @@ function start () {
   return true
 }
 
-module.exports = { start }
+// Snapshot of the OTEP-4719 process-context attributes describing this
+// writer's on-the-wire record schema — schema-version string, the caller-side
+// attribute key map, and the V8 layout constants a reader needs to walk from
+// our discovery TLS symbol into the record. Returned in the shape libdatadog's
+// napi ThreadLocalMetadata expects:
+//
+//   { attributeKeys, schemaVersion, extraAttributes: [{ key, intValue|stringValue }] }
+//
+// Returns undefined if @datadog/pprof isn't installed or doesn't expose the
+// otelThreadCtx.getProcessContextAttributes helper; callers should treat that
+// as "no threadlocal block" (equivalent to the flag being off).
+function getThreadLocalMetadata () {
+  let pprofMod
+  try {
+    pprofMod = require('@datadog/pprof')
+  } catch (e) {
+    log.warn('OTEP-4947 thread context: @datadog/pprof unavailable', e)
+    return
+  }
+  const ns = pprofMod.otelThreadCtx
+  if (!ns || typeof ns.getProcessContextAttributes !== 'function') {
+    log.warn(
+      'OTEP-4947 thread context: installed @datadog/pprof does not expose getProcessContextAttributes'
+    )
+    return
+  }
+  const pca = ns.getProcessContextAttributes(ATTRIBUTE_KEYS)
+  const extraAttributes = []
+  for (const [key, value] of Object.entries(pca)) {
+    if (key === 'threadlocal.schema_version' || key === 'threadlocal.attribute_key_map') continue
+    if (typeof value === 'number' && Number.isInteger(value)) {
+      extraAttributes.push({ key, intValue: value })
+    } else if (typeof value === 'string') {
+      extraAttributes.push({ key, stringValue: value })
+    } else {
+      throw new TypeError(
+        `OTEP-4947 process-context attribute ${JSON.stringify(key)} has unsupported value type: ${typeof value}`
+      )
+    }
+  }
+  return {
+    attributeKeys: [...pca['threadlocal.attribute_key_map']],
+    schemaVersion: pca['threadlocal.schema_version'],
+    extraAttributes,
+  }
+}
+
+module.exports = { start, ATTRIBUTE_KEYS, getThreadLocalMetadata }

packages/dd-trace/src/tracer_metadata.js

@@ -18,6 +18,18 @@ function storeConfig (config) {
       ? (processTags.serialized || null)
       : null
 
+    // OTEP-4947 thread-context writer metadata, published as part of the
+    // OTel process context so an out-of-process reader can decode the
+    // on-wire record: the attribute key map (libdatadog prepends the
+    // implicit `datadog.local_root_span_id` at wire index 0, so we only
+    // supply our own additional keys), the schema-version string, and
+    // the V8 layout constants the reader needs. Gated on the same config
+    // flag that activates the writer; undefined when off or when
+    // @datadog/pprof isn't installed to provide the values.
+    const threadlocalMetadata = config.DD_TRACE_OTEL_CTX_ENABLED
+      ? require('./otel-thread-ctx').getThreadLocalMetadata()
+      : undefined
+
     const metadata = new processDiscovery.TracerMetadata(
       config.tags['runtime-id'],
       tracerVersion,
@@ -26,7 +38,8 @@ function storeConfig (config) {
       config.env || null,
       config.version || null,
       processTagsSerialized,
-      containerId || null
+      containerId || null,
+      threadlocalMetadata
     )
 
     return processDiscovery.storeMetadata(metadata)

packages/dd-trace/test/otel-thread-ctx.spec.js

@@ -321,4 +321,69 @@ describe('otel-thread-ctx', () => {
       assert.equal(constructedContexts.length, 0)
     })
   })
+
+  describe('getThreadLocalMetadata()', () => {
+    it('returns the process-context snapshot in libdatadog-nodejs shape', () => {
+      pprofStub.otelThreadCtx.getProcessContextAttributes = sinon.stub().returns({
+        'threadlocal.schema_version': 'nodejs_v1_dev',
+        'threadlocal.attribute_key_map': ['datadog.trace_endpoint', 'datadog.thread_name'],
+        'threadlocal.wrapped_object_offset': 24,
+        'threadlocal.tagged_size': 8,
+      })
+      const m = loadModule()
+      const md = m.getThreadLocalMetadata()
+      sinon.assert.calledOnceWithExactly(
+        pprofStub.otelThreadCtx.getProcessContextAttributes,
+        m.ATTRIBUTE_KEYS
+      )
+      assert.deepEqual(md.attributeKeys, ['datadog.trace_endpoint', 'datadog.thread_name'])
+      assert.equal(md.schemaVersion, 'nodejs_v1_dev')
+      // Order isn't guaranteed since Object.entries is object-key ordered.
+      const byKey = Object.fromEntries(md.extraAttributes.map(a => [a.key, a]))
+      assert.deepEqual(byKey['threadlocal.wrapped_object_offset'],
+        { key: 'threadlocal.wrapped_object_offset', intValue: 24 })
+      assert.deepEqual(byKey['threadlocal.tagged_size'],
+        { key: 'threadlocal.tagged_size', intValue: 8 })
+    })
+
+    it('returns undefined when @datadog/pprof is not installed', () => {
+      const m = loadModule({ pprof: { '@noCallThru': true } })
+      assert.equal(m.getThreadLocalMetadata(), undefined)
+      sinon.assert.calledWithMatch(log.warn, /pprof unavailable|does not expose/)
+    })
+
+    it('returns undefined when otelThreadCtx.getProcessContextAttributes is missing', () => {
+      const m = loadModule({
+        pprof: {
+          '@noCallThru': true,
+          otelThreadCtx: { /* no getProcessContextAttributes */ },
+        },
+      })
+      assert.equal(m.getThreadLocalMetadata(), undefined)
+      sinon.assert.calledWithMatch(log.warn, /does not expose getProcessContextAttributes/)
+    })
+
+    it('encodes string-valued extra attributes as stringValue', () => {
+      pprofStub.otelThreadCtx.getProcessContextAttributes = sinon.stub().returns({
+        'threadlocal.schema_version': 'nodejs_v1_dev',
+        'threadlocal.attribute_key_map': [],
+        'threadlocal.runtime.name': 'nodejs',
+      })
+      const m = loadModule()
+      const md = m.getThreadLocalMetadata()
+      assert.deepEqual(md.extraAttributes, [
+        { key: 'threadlocal.runtime.name', stringValue: 'nodejs' },
+      ])
+    })
+
+    it('throws on an extra attribute with an unsupported value type', () => {
+      pprofStub.otelThreadCtx.getProcessContextAttributes = sinon.stub().returns({
+        'threadlocal.schema_version': 'nodejs_v1_dev',
+        'threadlocal.attribute_key_map': [],
+        'threadlocal.weird': true, // booleans aren't wired through yet
+      })
+      const m = loadModule()
+      assert.throws(() => m.getThreadLocalMetadata(), /unsupported value type/)
+    })
+  })
 })

packages/dd-trace/test/tracer_metadata.spec.js

@@ -13,6 +13,7 @@ describe('tracer_metadata', () => {
   let libdatadogStub
   let dockerStub
   let processTagsStub
+  let otelThreadCtxStub
 
   const baseConfig = {
     tags: { 'runtime-id': 'test-runtime-id' },
@@ -21,6 +22,7 @@ describe('tracer_metadata', () => {
     env: 'test-env',
     version: '1.0.0',
     DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED: false,
+    DD_TRACE_OTEL_CTX_ENABLED: false,
   }
 
   beforeEach(() => {
@@ -38,11 +40,21 @@ describe('tracer_metadata', () => {
 
     dockerStub = { containerId: undefined }
     processTagsStub = { serialized: 'tag1:val1,tag2:val2' }
+    otelThreadCtxStub = {
+      getThreadLocalMetadata: sinon.stub().returns({
+        attributeKeys: ['datadog.thread_name'],
+        schemaVersion: 'nodejs_v1_dev',
+        extraAttributes: [
+          { key: 'threadlocal.wrapped_object_offset', intValue: 24 },
+        ],
+      }),
+    }
 
     storeConfig = proxyquire('../src/tracer_metadata', {
       '@datadog/libdatadog': libdatadogStub,
       './exporters/common/docker': dockerStub,
       './process-tags': processTagsStub,
+      './otel-thread-ctx': otelThreadCtxStub,
     })
   })
 
@@ -96,6 +108,33 @@ describe('tracer_metadata', () => {
     assert.strictEqual(args[7], null)
   })
 
+  it('passes undefined for threadlocal_metadata when DD_TRACE_OTEL_CTX_ENABLED is false', () => {
+    storeConfig(baseConfig)
+    const args = TracerMetadataStub.firstCall.args
+    assert.strictEqual(args[8], undefined)
+    sinon.assert.notCalled(otelThreadCtxStub.getThreadLocalMetadata)
+  })
+
+  it('passes the OTEP-4947 process-context snapshot when DD_TRACE_OTEL_CTX_ENABLED is true', () => {
+    storeConfig({ ...baseConfig, DD_TRACE_OTEL_CTX_ENABLED: true })
+    const args = TracerMetadataStub.firstCall.args
+    sinon.assert.calledOnce(otelThreadCtxStub.getThreadLocalMetadata)
+    assert.deepStrictEqual(args[8], {
+      attributeKeys: ['datadog.thread_name'],
+      schemaVersion: 'nodejs_v1_dev',
+      extraAttributes: [
+        { key: 'threadlocal.wrapped_object_offset', intValue: 24 },
+      ],
+    })
+  })
+
+  it('passes undefined when getThreadLocalMetadata is unavailable (e.g. no pprof)', () => {
+    otelThreadCtxStub.getThreadLocalMetadata.returns(undefined)
+    storeConfig({ ...baseConfig, DD_TRACE_OTEL_CTX_ENABLED: true })
+    const args = TracerMetadataStub.firstCall.args
+    assert.strictEqual(args[8], undefined)
+  })
+
   it('passes null for service when config.service is falsy', () => {
     storeConfig({ ...baseConfig, service: undefined })