diff --git a/packages/datadog-instrumentations/test/helpers/azure-trace-context.spec.js b/packages/datadog-instrumentations/test/helpers/azure-trace-context.spec.js new file mode 100644 index 00000000000..4bba525f3e6 --- /dev/null +++ b/packages/datadog-instrumentations/test/helpers/azure-trace-context.spec.js @@ -0,0 +1,184 @@ +'use strict' + +const assert = require('node:assert/strict') +const os = require('os') +const path = require('path') + +const { describe, it } = require('mocha') + +const { + buildSpanParentContext, + buildSpanParentContextAsync, + carrierFromTraceContext, + extractContext, + getInstanceId, + getInvocationContext, + runWithTraceContext, +} = require('../../src/helpers/azure-trace-context') + +describe('azure-trace-context', () => { + describe('carrierFromTraceContext', () => { + it('returns null when traceContext is missing', () => { + assert.equal(carrierFromTraceContext(undefined), null) + }) + + it('maps traceParent and traceState to W3C carrier keys', () => { + assert.deepEqual( + carrierFromTraceContext({ + traceParent: '00-abc-def-01', + traceState: 'dd=s:1', + }), + { + traceparent: '00-abc-def-01', + tracestate: 'dd=s:1', + }, + ) + }) + + it('returns null when no W3C fields are present', () => { + assert.equal(carrierFromTraceContext({}), null) + }) + }) + + describe('getInvocationContext', () => { + it('reads HTTP invocation context from the second argument', () => { + const ctx = { traceContext: { traceParent: '00-a-b-01' } } + assert.equal(getInvocationContext([{}, ctx], 'http'), ctx) + }) + + it('reads durable orchestration context from the first argument', () => { + const ctx = { df: { isReplaying: false } } + assert.equal(getInvocationContext([ctx], 'durable-orchestration'), ctx) + }) + + it('reads durable activity context from any argument with traceContext', () => { + const ctx = { traceContext: { traceParent: '00-a-b-01' } } + assert.equal(getInvocationContext([ctx], 'durable-activity'), ctx) + assert.equal(getInvocationContext(['input', ctx], 'durable-activity'), ctx) + }) + }) + + describe('getInstanceId', () => { + it('reads the durable instance id from traceContext attributes', () => { + assert.equal(getInstanceId({ + traceContext: { + attributes: { + 'durabletask.task.instance_id': 'abc123', + }, + }, + }), 'abc123') + }) + + it('falls back to the legacy DurableFunctionsInstanceId attribute', () => { + assert.equal(getInstanceId({ + traceContext: { + attributes: { + DurableFunctionsInstanceId: 'legacy-id', + }, + }, + }), 'legacy-id') + }) + }) + + describe('buildSpanParentContext', () => { + it('extracts HTTP parent context from the invocation context', () => { + const ctx = { traceContext: { traceParent: '00-00000000000000000000000000000001-0000000000000004-01' } } + const parentContext = buildSpanParentContext([{}, ctx], 'http') + assert.ok(parentContext) + }) + + it('reads generic orchestration context from the second argument', () => { + const ctx = { traceContext: { traceParent: '00-a-b-01' } } + assert.equal(getInvocationContext([{}, ctx], 'orchestration-generic'), ctx) + }) + + it('parents activity spans to the in-flight orchestration span', () => { + const api = require('@opentelemetry/api') + const { + registerOrchestrationSpan, + unregisterOrchestrationSpan, + } = require('../../src/helpers/otel-orchestration-registry') + + const orchestrationSpan = { + spanContext () { + return { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + traceFlags: 1, + } + }, + } + + registerOrchestrationSpan('abc123', orchestrationSpan) + + const activityContext = { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000003-00', + attributes: { + 'durabletask.task.instance_id': 'abc123', + }, + }, + } + + const parentContext = buildSpanParentContext(['input', activityContext], 'durable-activity') + const span = api.trace.getSpan(parentContext) + + assert.equal(span, orchestrationSpan) + unregisterOrchestrationSpan('abc123') + }) + }) + + describe('buildSpanParentContextAsync', () => { + it('parents async activity spans to orchestration metadata from the store', async () => { + const storeDir = path.join( + os.tmpdir(), + `dd-orch-async-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR = storeDir + + const { publishOrchestrationMetaSync } = require('../../src/helpers/otel-orchestration-store') + publishOrchestrationMetaSync('async-inst', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + startTime: Date.now(), + status: 'open', + }) + + const activityContext = { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000003-01', + attributes: { + 'durabletask.task.instance_id': 'async-inst', + }, + }, + } + + const parentContext = await buildSpanParentContextAsync( + ['input', activityContext], + 'durable-activity', + ) + assert.ok(parentContext) + }) + }) + + describe('runWithTraceContext', () => { + it('runs the callback when traceContext is missing', () => { + assert.equal(runWithTraceContext(undefined, () => 42), 42) + }) + + it('runs the callback when traceContext is present', () => { + const result = runWithTraceContext( + { traceParent: '00-00000000000000000000000000000000-0000000000000000-01' }, + () => 'ok', + ) + assert.equal(result, 'ok') + }) + }) + + describe('extractContext', () => { + it('returns root context when traceContext is missing', () => { + const root = extractContext(undefined) + assert.ok(root) + }) + }) +}) diff --git a/packages/datadog-instrumentations/test/helpers/otel-azure-handlers.spec.js b/packages/datadog-instrumentations/test/helpers/otel-azure-handlers.spec.js new file mode 100644 index 00000000000..1a0cc0c6879 --- /dev/null +++ b/packages/datadog-instrumentations/test/helpers/otel-azure-handlers.spec.js @@ -0,0 +1,407 @@ +'use strict' + +const assert = require('node:assert/strict') +const os = require('os') +const path = require('path') + +const api = require('@opentelemetry/api') +const { afterEach, before, beforeEach, describe, it } = require('mocha') + +const { patchApp: patchAzureFunctionsApp } = require('../../src/otel-azure-functions') +const { patchApp: patchDurableFunctionsApp } = require('../../src/otel-azure-durable-functions') + +function initTracer () { + process.env.DD_TRACE_OTEL_ENABLED = 'true' + process.env.DD_TRACE_AZURE_DURABLE_FUNCTIONS_ENABLED = 'false' + + const ddtrace = require('../../../dd-trace') + if (!global.__otelAzureHandlersTracerInitialized) { + ddtrace.init({ plugins: false, sampleRate: 1 }) + new ddtrace.TracerProvider().register() + global.__otelAzureHandlersTracerInitialized = true + } +} + +function createAzureFunctionsApp () { + let handler + const app = {} + for (const method of ['deleteRequest', 'http', 'get', 'patch', 'post', 'put', 'generic']) { + app[method] = function (name, arg) { + if (typeof arg === 'function') { + handler = arg + } else if (arg?.handler) { + handler = arg.handler + } + return arg + } + } + patchAzureFunctionsApp(app) + return { + app, + getHandler: () => handler, + } +} + +function createDurableFunctionsApp () { + let handler + const app = { + entity (name, arg) { + if (typeof arg === 'function') { + handler = arg + } else if (arg?.handler) { + handler = arg.handler + } + return arg + }, + activity (name, arg) { + if (arg?.handler) { + handler = arg.handler + } + return arg + }, + orchestration (name, arg) { + handler = arg + return arg + }, + } + patchDurableFunctionsApp(app) + return { + app, + getHandler: () => handler, + } +} + +function invocationContext (instanceId, overrides = {}) { + return { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000003-01', + attributes: { + 'durabletask.task.instance_id': instanceId, + ...overrides.attributes, + }, + ...overrides.traceContext, + }, + df: { isReplaying: false, ...overrides.df }, + ...overrides, + } +} + +describe('otel-azure handler instrumentation', () => { + let previousStoreDir + + before(() => { + initTracer() + }) + + beforeEach(() => { + previousStoreDir = process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR + process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR = path.join( + os.tmpdir(), + `dd-orch-handler-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + }) + + afterEach(() => { + if (previousStoreDir === undefined) { + delete process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR + } else { + process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR = previousStoreDir + } + }) + + describe('HTTP handlers', () => { + it('traces handlers registered as objects', async () => { + const { app, getHandler } = createAzureFunctionsApp() + + app.http('StartParty', { + handler: async () => 'started', + }) + + const result = await getHandler()({}, { + traceContext: { traceParent: '00-00000000000000000000000000000001-0000000000000004-01' }, + }) + assert.equal(result, 'started') + }) + + it('traces handlers registered as functions', async () => { + const { app, getHandler } = createAzureFunctionsApp() + + app.post('StartParty', async () => 'posted') + + const result = await getHandler()({}, { + traceContext: { traceParent: '00-00000000000000000000000000000001-0000000000000004-01' }, + }) + assert.equal(result, 'posted') + }) + + it('records errors on failed HTTP handlers', async () => { + const { app, getHandler } = createAzureFunctionsApp() + + app.http('FailParty', { + handler: async () => { + throw new Error('http failed') + }, + }) + + await assert.rejects( + () => getHandler()({}, { traceContext: { traceParent: '00-a-b-01' } }), + /http failed/, + ) + }) + }) + + describe('generic orchestration handlers', () => { + it('skips tracing during replay turns', async () => { + const { app, getHandler } = createAzureFunctionsApp() + + app.generic('GenericOrch', { + trigger: { type: 'orchestrationTrigger' }, + handler: async () => 'replayed', + }) + + const result = await getHandler()( + { isReplaying: true }, + invocationContext('replay-inst'), + ) + assert.equal(result, 'replayed') + }) + + it('exports the orchestration span when the runtime reports completion', async () => { + const { app, getHandler } = createAzureFunctionsApp() + + app.generic('GenericOrch', { + trigger: { type: 'orchestrationTrigger' }, + handler: async () => 'done', + }) + + const ctx = invocationContext('generic-complete', { + traceContext: { + attributes: { + 'durabletask.task.instance_id': 'generic-complete', + DurableFunctionsRuntimeStatus: 'Completed', + }, + }, + }) + + const result = await getHandler()({ isReplaying: false }, ctx) + assert.equal(result, 'done') + }) + + it('exports the orchestration span when the handler fails', async () => { + const { app, getHandler } = createAzureFunctionsApp() + + app.generic('GenericOrch', { + trigger: { type: 'orchestrationTrigger' }, + handler: async () => { + throw new Error('generic failed') + }, + }) + + await assert.rejects( + () => getHandler()({ isReplaying: false }, invocationContext('generic-fail')), + /generic failed/, + ) + }) + }) + + describe('durable entity handlers', () => { + it('traces entity handlers registered as functions', () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.entity('Counter', () => 1) + + assert.equal(getHandler()(invocationContext('entity-fn')), 1) + }) + + it('traces entity handlers registered as objects', () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.entity('Counter', { + handler: function increment () { + return 2 + }, + }) + + assert.equal(getHandler()(invocationContext('entity-obj')), 2) + }) + }) + + describe('durable activity handlers', () => { + it('traces synchronous activity handlers', () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.activity('Prepare', { + handler: () => 'ready', + }) + + assert.equal( + getHandler()('input', invocationContext('activity-sync')), + 'ready', + ) + }) + + it('traces async activity handlers', async () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.activity('Bake', { + handler: async () => 'baked', + }) + + const result = await getHandler()('input', invocationContext('activity-async')) + assert.equal(result, 'baked') + }) + + it('records errors on failed activity handlers', () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.activity('Fail', { + handler: () => { + throw new Error('activity failed') + }, + }) + + assert.throws( + () => getHandler()('input', invocationContext('activity-error')), + /activity failed/, + ) + }) + }) + + describe('durable orchestration handlers', () => { + it('exports one orchestration span across generator turns', () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.orchestration('Party', function * () { + yield 'prepare' + return 'done' + }) + + const gen = getHandler()(invocationContext('orch-complete')) + assert.deepEqual(gen.next(), { value: 'prepare', done: false }) + assert.deepEqual(gen.next('input'), { value: 'done', done: true }) + }) + + it('exports the orchestration span when the generator throws', () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.orchestration('Party', function * () { + yield + throw new Error('orchestration failed') + }) + + const gen = getHandler()(invocationContext('orch-error')) + gen.next() + assert.throws(() => gen.next(), /orchestration failed/) + }) + + it('skips metadata updates during replay turns', () => { + const { app, getHandler } = createDurableFunctionsApp() + + app.orchestration('Party', function * () { + yield 'replay' + return 'done' + }) + + const gen = getHandler()(invocationContext('orch-replay', { + df: { isReplaying: true }, + })) + assert.deepEqual(gen.next(), { value: 'replay', done: false }) + assert.deepEqual(gen.next(), { value: 'done', done: true }) + }) + }) +}) + +describe('otel-orchestration-http-link', () => { + const { + applyHttpParentToMeta, + patchDurableClient, + peekHttpParentForInstance, + peekPendingHttpParent, + publishHttpParentMeta, + publishPendingHttpParent, + resolveHttpParentForOrchestration, + traceIdsEquivalent, + } = require('../../src/helpers/otel-orchestration-http-link') + + before(() => { + initTracer() + }) + + beforeEach(() => { + process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR = path.join( + os.tmpdir(), + `dd-orch-link-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + }) + + it('matches trace ids by their lower 64 bits', () => { + assert.equal( + traceIdsEquivalent( + '00000000000000000000000000000001', + '0000000000000001', + ), + true, + ) + }) + + it('resolves pending HTTP parents by trace id', () => { + publishPendingHttpParent({ + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + }) + + assert.deepEqual( + peekPendingHttpParent('0000000000000001'), + { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + }, + ) + }) + + it('resolves HTTP parents stored by instance id', () => { + publishHttpParentMeta('inst-1', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + }) + + assert.deepEqual(peekHttpParentForInstance('inst-1').spanId, '0000000000000004') + assert.deepEqual( + resolveHttpParentForOrchestration('inst-1', { + traceParent: '00-00000000000000000000000000000001-0000000000000099-01', + }).spanId, + '0000000000000004', + ) + }) + + it('returns the original metadata when no HTTP parent is available', () => { + const meta = { traceId: '1', spanId: '2', parentId: '3' } + assert.equal(applyHttpParentToMeta(meta, undefined), meta) + }) + + it('seeds orchestration metadata from startNew while an HTTP span is active', async () => { + const { readOrchestrationSpanMetaSync } = require('../../src/helpers/otel-orchestration-store') + const tracer = api.trace.getTracer('test') + const parentSpan = tracer.startSpan('http StartParty') + const ctx = api.trace.setSpan(api.context.active(), parentSpan) + + class DurableClient { + async startNew () { + return 'start-new-inst' + } + } + + patchDurableClient(DurableClient) + + await api.context.with(ctx, async () => { + const instanceId = await new DurableClient().startNew('PartyOrchestration') + assert.equal(instanceId, 'start-new-inst') + }) + + parentSpan.end() + + const seeded = readOrchestrationSpanMetaSync('start-new-inst') + assert.equal(seeded.parentId.length, 16) + assert.equal(seeded.pendingStart, true) + }) +}) diff --git a/packages/datadog-instrumentations/test/helpers/otel-azure.spec.js b/packages/datadog-instrumentations/test/helpers/otel-azure.spec.js new file mode 100644 index 00000000000..e91c598064e --- /dev/null +++ b/packages/datadog-instrumentations/test/helpers/otel-azure.spec.js @@ -0,0 +1,207 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { describe, it } = require('mocha') +const proxyquire = require('proxyquire').noPreserveCache() + +describe('otel-azure-enabled', () => { + function loadWithEnv (env) { + const values = { + DD_TRACE_OTEL_ENABLED: env.ddTraceOtelEnabled, + OTEL_SDK_DISABLED: env.otelSdkDisabled, + DD_TRACE_AZURE_DURABLE_FUNCTIONS_ENABLED: env.nativeDurableEnabled, + } + return proxyquire('../../src/helpers/otel-azure-enabled', { + '../../../dd-trace/src/config/helper': { + getValueFromEnvSources: (name) => values[name], + }, + }) + } + + it('disables when DD_TRACE_OTEL_ENABLED is false', () => { + const { isOtelAzureInstrumentationEnabled } = loadWithEnv({ ddTraceOtelEnabled: false }) + assert.equal(isOtelAzureInstrumentationEnabled(), false) + }) + + it('disables when OTEL_SDK_DISABLED is true', () => { + const { isOtelAzureInstrumentationEnabled } = loadWithEnv({ otelSdkDisabled: true }) + assert.equal(isOtelAzureInstrumentationEnabled(), false) + }) + + it('disables when native durable plugin is enabled', () => { + const { isOtelAzureInstrumentationEnabled } = loadWithEnv({ + ddTraceOtelEnabled: true, + nativeDurableEnabled: true, + }) + assert.equal(isOtelAzureInstrumentationEnabled(), false) + }) + + it('enables when OTel is on and native durable plugin is disabled', () => { + const { isOtelAzureInstrumentationEnabled } = loadWithEnv({ + ddTraceOtelEnabled: true, + nativeDurableEnabled: false, + }) + assert.equal(isOtelAzureInstrumentationEnabled(), true) + }) + + it('enables when OTEL_SDK_DISABLED=false opts in and native durable is disabled', () => { + const { isOtelAzureInstrumentationEnabled } = loadWithEnv({ + otelSdkDisabled: false, + nativeDurableEnabled: false, + }) + assert.equal(isOtelAzureInstrumentationEnabled(), true) + }) + + it('reports isOtelSdkEnabled directly when DD_TRACE_OTEL_ENABLED opts in', () => { + const { isOtelSdkEnabled } = loadWithEnv({ ddTraceOtelEnabled: true }) + assert.equal(isOtelSdkEnabled(), true) + }) + + it('stays disabled by default when neither OTel opt-in is set', () => { + const { isOtelSdkEnabled, isOtelAzureInstrumentationEnabled } = loadWithEnv({}) + assert.equal(isOtelSdkEnabled(), false) + assert.equal(isOtelAzureInstrumentationEnabled(), false) + }) +}) + +function tracingChannelStub () { + return { + subscribe () {}, + unsubscribe () {}, + hasSubscribers: false, + } +} + +describe('azure-functions OTel gating', () => { + function loadHook (enabled) { + let patchAppCalled = false + const app = {} + for (const method of [ + 'deleteRequest', 'http', 'get', 'patch', 'post', 'put', + 'serviceBusQueue', 'serviceBusTopic', 'eventHub', 'cosmosDB', + ]) { + app[method] = function () {} + } + + proxyquire('../../src/azure-functions', { + './helpers/otel-azure-enabled': { + isOtelAzureInstrumentationEnabled: () => enabled, + }, + './otel-azure-functions': { + patchApp: (patchedApp) => { + patchAppCalled = true + assert.equal(patchedApp, app) + }, + }, + './helpers/instrument': { + addHook: (opts, fn) => { + if (opts.name === '@azure/functions') fn({ app }) + }, + }, + 'dc-polyfill': { tracingChannel: tracingChannelStub }, + '../../datadog-shimmer': require('../../../datadog-shimmer'), + }) + + return patchAppCalled + } + + it('patches OTel handlers when instrumentation is enabled', () => { + assert.equal(loadHook(true), true) + }) + + it('skips OTel handlers when instrumentation is disabled', () => { + assert.equal(loadHook(false), false) + }) +}) + +describe('azure-durable-functions OTel gating', () => { + function loadHooks (enabled) { + let patchAppCalled = false + let patchClientCalled = false + const app = { + entity () {}, + activity () {}, + orchestration () {}, + } + + proxyquire('../../src/azure-durable-functions', { + './helpers/otel-azure-enabled': { + isOtelAzureInstrumentationEnabled: () => enabled, + }, + './otel-azure-durable-functions': { + patchApp: () => { patchAppCalled = true }, + }, + './helpers/otel-orchestration-http-link': { + patchDurableClient: () => { patchClientCalled = true }, + }, + './helpers/instrument': { + addHook: (opts, fn) => { + if (opts.name === 'durable-functions' && !opts.file) fn({ app }) + if (opts.file === 'lib/src/durableClient/DurableClient.js') { + fn({ DurableClient: { prototype: { startNew () {} } } }) + } + }, + }, + 'dc-polyfill': { tracingChannel: tracingChannelStub }, + '../../datadog-shimmer': require('../../../datadog-shimmer'), + }) + + return { patchAppCalled, patchClientCalled } + } + + it('patches OTel handlers and DurableClient when instrumentation is enabled', () => { + const { patchAppCalled, patchClientCalled } = loadHooks(true) + assert.equal(patchAppCalled, true) + assert.equal(patchClientCalled, true) + }) + + it('skips OTel hooks when instrumentation is disabled', () => { + const { patchAppCalled, patchClientCalled } = loadHooks(false) + assert.equal(patchAppCalled, false) + assert.equal(patchClientCalled, false) + }) +}) + +describe('otel-azure-functions', () => { + it('wraps HTTP and generic registration methods', () => { + // The key is the path as the module under test requires it; the value is + // resolved from this spec's own directory. + const { patchApp } = proxyquire('../../src/otel-azure-functions', { + '../../datadog-shimmer': require('../../../datadog-shimmer'), + }) + const app = { + deleteRequest (name, arg) { return arg }, + http (name, arg) { return arg }, + get (name, arg) { return arg }, + patch (name, arg) { return arg }, + post (name, arg) { return arg }, + put (name, arg) { return arg }, + generic (name, options) { return options }, + } + + patchApp(app) + + assert.notEqual(app.http, undefined) + assert.notEqual(app.generic, undefined) + }) +}) + +describe('otel-azure-durable-functions', () => { + it('wraps orchestration, activity, and entity registration methods', () => { + const { patchApp } = proxyquire('../../src/otel-azure-durable-functions', { + '../../datadog-shimmer': require('../../../datadog-shimmer'), + }) + const app = { + entity (name, arg) { return arg }, + activity (name, options) { return options }, + orchestration (name, handler) { return handler }, + } + + patchApp(app) + + assert.notEqual(app.entity, undefined) + assert.notEqual(app.activity, undefined) + assert.notEqual(app.orchestration, undefined) + }) +}) diff --git a/packages/datadog-instrumentations/test/helpers/otel-orchestration-export.spec.js b/packages/datadog-instrumentations/test/helpers/otel-orchestration-export.spec.js new file mode 100644 index 00000000000..1ec489b55dd --- /dev/null +++ b/packages/datadog-instrumentations/test/helpers/otel-orchestration-export.spec.js @@ -0,0 +1,79 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { describe, it } = require('mocha') + +const { + createOrchestrationMetaFromHttpParent, + resolveOrchestrationSpanBounds, +} = require('../../src/helpers/otel-orchestration-export') +const { + stampOrchestrationStartTime, +} = require('../../src/helpers/otel-orchestration-store') + +describe('otel orchestration span timing', () => { + describe('createOrchestrationMetaFromHttpParent', () => { + it('anchors start time to startNew instead of deferring it', () => { + const meta = createOrchestrationMetaFromHttpParent('inst', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + }, 'PizzaOrderOrchestration', 150) + + assert.equal(meta.startTime, 150) + assert.equal(meta.pendingStart, undefined) + }) + }) + + describe('stampOrchestrationStartTime', () => { + it('preserves an existing start time', () => { + const seeded = { + traceId: 'abc', + spanId: 'def', + startTime: 150, + } + + const stamped = stampOrchestrationStartTime(seeded, 'PizzaOrderOrchestration') + assert.strictEqual(stamped.startTime, 150) + }) + + it('stamps only when start time is missing', () => { + const seeded = { + traceId: 'abc', + spanId: 'def', + } + + const first = stampOrchestrationStartTime(seeded, 'PizzaOrderOrchestration') + assert.ok(first.startTime) + + const second = stampOrchestrationStartTime(first, 'PizzaOrderOrchestration') + assert.strictEqual(second.startTime, first.startTime) + }) + }) + + describe('resolveOrchestrationSpanBounds', () => { + it('spans the full instance window from startNew through completion', () => { + const bounds = resolveOrchestrationSpanBounds({ startTime: 100 }, 350) + + assert.strictEqual(bounds.startTime, 100) + assert.strictEqual(bounds.endTime, 350) + }) + + it('pulls the start time earlier when activities ran first', () => { + const bounds = resolveOrchestrationSpanBounds({ + startTime: 250, + earliestChildStartTime: 180, + }, 350) + + assert.strictEqual(bounds.startTime, 180) + assert.strictEqual(bounds.endTime, 350) + }) + + it('clamps start time when it would exceed the end time', () => { + const bounds = resolveOrchestrationSpanBounds({ startTime: 400 }, 350) + + assert.strictEqual(bounds.startTime, 350) + assert.strictEqual(bounds.endTime, 350) + }) + }) +}) diff --git a/packages/datadog-instrumentations/test/helpers/otel-orchestration-store.spec.js b/packages/datadog-instrumentations/test/helpers/otel-orchestration-store.spec.js new file mode 100644 index 00000000000..9457fca12a4 --- /dev/null +++ b/packages/datadog-instrumentations/test/helpers/otel-orchestration-store.spec.js @@ -0,0 +1,448 @@ +'use strict' + +const Module = require('module') +const assert = require('node:assert/strict') +const fs = require('fs') +const os = require('os') +const path = require('path') + +const { describe, it, beforeEach, afterEach } = require('mocha') + +const storePath = require.resolve('../../src/helpers/otel-orchestration-store') + +const { + appendOrchestrationSpanToTraceState, + parseOrchestrationMetaFromTraceContext, + traceContextFromMeta, +} = require('../../src/helpers/otel-orchestration-meta') +const { + completeOrchestrationSpan, + ensureOrchestrationMeta, + injectOrchestrationMetaIntoTraceState, + publishOrchestrationMetaSync, + publishOrchestrationSpanMetaSync, + readOrchestrationSpanMetaSync, + reconcileOrchestrationHttpParent, + seedOrchestrationMetaFromHttpParent, +} = require('../../src/helpers/otel-orchestration-store') +const { + createOrchestrationMeta, + createOrchestrationMetaFromHttpParent, + exportOrchestrationSpanFromMeta, + getParentFromTraceContext, +} = require('../../src/helpers/otel-orchestration-export') +const { publishHttpParentMeta } = require('../../src/helpers/otel-orchestration-http-link') +const { buildSpanParentContext } = require('../../src/helpers/azure-trace-context') + +describe('otel-orchestration-meta', () => { + it('builds traceparent from orchestration metadata', () => { + assert.deepEqual( + traceContextFromMeta({ + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + }), + { + traceParent: '00-00000000000000000000000000000001-0000000000000002-01', + }, + ) + }) + + it('parses orchestration span id from tracestate', () => { + assert.deepEqual( + parseOrchestrationMetaFromTraceContext({ + traceParent: '00-00000000000000000000000000000001-0000000000000003-00', + traceState: 'dd=s:1,dd=o:0000000000000002', + }), + { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + }, + ) + }) + + it('appends orchestration span id to tracestate', () => { + assert.equal( + appendOrchestrationSpanToTraceState('dd=s:1', '0000000000000002'), + 'dd=s:1,dd=o:0000000000000002', + ) + }) +}) + +describe('otel-orchestration-store', () => { + let previousStoreDir + + beforeEach(() => { + previousStoreDir = process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR + process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR = path.join( + os.tmpdir(), + `dd-orch-test-${Date.now()}-${Math.random().toString(16).slice(2)}`, + ) + }) + + afterEach(() => { + if (previousStoreDir === undefined) { + delete process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR + } else { + process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR = previousStoreDir + } + }) + + it('writes and reads orchestration metadata from the shared store', () => { + publishOrchestrationSpanMetaSync('abc123', { + _ddSpan: { + context () { + return { + _traceId: '00000000000000000000000000000001', + _spanId: '0000000000000002', + } + }, + }, + }) + + assert.deepEqual( + readOrchestrationSpanMetaSync('abc123'), + { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + startTime: readOrchestrationSpanMetaSync('abc123').startTime, + status: 'open', + }, + ) + assert.ok(fs.existsSync(path.join(process.env.DD_TRACE_AZURE_ORCHESTRATION_STORE_DIR, 'abc123.json'))) + }) + + it('creates orchestration metadata once per instance', () => { + const invocationContext = { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000003-00', + }, + } + + const first = ensureOrchestrationMeta('abc123', invocationContext, 'PizzaOrderOrchestration') + const second = ensureOrchestrationMeta('abc123', invocationContext, 'PizzaOrderOrchestration') + + assert.equal(first.spanId, second.spanId) + assert.equal(first.traceId, '00000000000000000000000000000001') + assert.equal(first.status, 'open') + }) + + it('parents orchestration metadata to the HTTP span that started the instance', () => { + publishHttpParentMeta('abc123', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + }) + + const meta = createOrchestrationMeta('abc123', { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000099-00', + }, + }, 'PizzaOrderOrchestration') + + assert.equal(meta.parentId, '0000000000000004') + assert.equal(meta.traceId, '00000000000000000000000000000001') + }) + + it('reconciles orchestration metadata when the HTTP parent arrives after instance creation', () => { + publishOrchestrationMetaSync('abc123', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + parentId: '0000000000000099', + startTime: Date.now(), + status: 'open', + }) + + const updated = reconcileOrchestrationHttpParent('abc123', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + }) + + assert.equal(updated.parentId, '0000000000000004') + assert.equal(updated.httpParentSpanId, '0000000000000004') + assert.equal(readOrchestrationSpanMetaSync('abc123').parentId, '0000000000000004') + }) + + it('seeds orchestration metadata from the HTTP parent at startNew time', () => { + const meta = seedOrchestrationMetaFromHttpParent('seed-new', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + }, 'PizzaPartyOrchestration') + + assert.equal(meta.traceId, '00000000000000000000000000000001') + assert.equal(meta.parentId, '0000000000000004') + assert.equal(meta.spanId.length, 16) + assert.ok(meta.startTime) + assert.equal(meta.pendingStart, undefined) + // A worker with no in-process state must still read the HTTP parent. + assert.equal(readOrchestrationSpanMetaSync('seed-new').parentId, '0000000000000004') + }) + + it('keeps the seeded HTTP parent when the orchestration starts on another worker', () => { + seedOrchestrationMetaFromHttpParent('seed-other-worker', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + }, 'PizzaPartyOrchestration') + + const seeded = readOrchestrationSpanMetaSync('seed-other-worker') + + // Azure hands the orchestration its own unrelated trace context. + const meta = ensureOrchestrationMeta('seed-other-worker', { + traceContext: { + traceParent: '00-99999999999999999999999999999999-0000000000000099-01', + }, + }, 'PizzaPartyOrchestration') + + assert.equal(meta.parentId, '0000000000000004') + assert.equal(meta.traceId, '00000000000000000000000000000001') + assert.equal(meta.spanId, seeded.spanId) + assert.strictEqual(meta.startTime, seeded.startTime) + }) + + it('does not seed twice for the same instance', () => { + const httpParent = { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000004', + } + + const first = seedOrchestrationMetaFromHttpParent('seed-once', httpParent, 'PizzaPartyOrchestration') + const second = seedOrchestrationMetaFromHttpParent('seed-once', httpParent, 'PizzaPartyOrchestration') + + assert.equal(first.spanId, second.spanId) + }) + + it('prefers the stored HTTP parent over the tracestate marker', () => { + publishOrchestrationMetaSync('abc123', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + parentId: '0000000000000004', + httpParentSpanId: '0000000000000004', + startTime: Date.now(), + status: 'open', + functionName: 'PizzaPartyOrchestration', + }) + + const merged = readOrchestrationSpanMetaSync('abc123', { + traceParent: '00-00000000000000000000000000000001-0000000000000099-00', + traceState: 'dd=s:1,dd=o:0000000000000002', + }) + + assert.equal(merged.parentId, '0000000000000004') + assert.equal(merged.httpParentSpanId, '0000000000000004') + assert.equal(merged.spanId, '0000000000000002') + }) + + it('exports one orchestration span on completion', () => { + process.env.DD_TRACE_OTEL_ENABLED = 'true' + process.env.DD_TRACE_AZURE_DURABLE_FUNCTIONS_ENABLED = 'false' + + const ddtrace = require('../../../dd-trace') + ddtrace.init({ plugins: false, sampleRate: 1 }) + new ddtrace.TracerProvider().register() + + const invocationContext = { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000003-00', + }, + } + + const meta = ensureOrchestrationMeta('abc123', invocationContext, 'PizzaOrderOrchestration') + const complete = () => completeOrchestrationSpan( + '@azure/durable-functions', 'abc123', invocationContext, 'PizzaOrderOrchestration' + ) + assert.equal(complete(), true) + assert.equal(complete(), false) + assert.equal(readOrchestrationSpanMetaSync('abc123'), undefined) + assert.equal(meta.spanId.length, 16) + }) + + it('parents activity spans to orchestration metadata from the shared store', () => { + process.env.DD_TRACE_OTEL_ENABLED = 'true' + process.env.DD_TRACE_AZURE_DURABLE_FUNCTIONS_ENABLED = 'false' + + const ddtrace = require('../../../dd-trace') + ddtrace.init({ plugins: false, sampleRate: 1 }) + new ddtrace.TracerProvider().register() + + publishOrchestrationSpanMetaSync('abc123', { + _ddSpan: { + context () { + return { + _traceId: '00000000000000000000000000000001', + _spanId: '0000000000000002', + } + }, + }, + }) + + const activityContext = { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000003-00', + attributes: { + 'durabletask.task.instance_id': 'abc123', + }, + }, + } + + const api = require('@opentelemetry/api') + const parentContext = buildSpanParentContext(['input', activityContext], 'durable-activity') + const actSpan = api.trace.getTracer('test').startSpan('durable-activity Test', {}, parentContext) + assert.equal(actSpan._ddSpan.context()._parentId.toString(16).padStart(16, '0'), '0000000000000002') + actSpan.end() + }) + + it('injects orchestration span ids into tracestate', () => { + assert.deepEqual( + injectOrchestrationMetaIntoTraceState( + { traceState: 'dd=s:1' }, + { spanId: '0000000000000002' }, + ), + { traceState: 'dd=s:1,dd=o:0000000000000002' }, + ) + }) + + it('persists orchestration metadata to azure table storage', async () => { + const previousStorage = process.env.AzureWebJobsStorage + process.env.AzureWebJobsStorage = 'UseDevelopmentStorage=true' + + let upsertCalled = false + delete require.cache[storePath] + const originalRequire = Module.prototype.require + Module.prototype.require = function (id) { + if (id === '@azure/data-tables') { + return { + TableClient: { + fromConnectionString (connectionString) { + assert.match(connectionString, /127\.0\.0\.1:10002/) + return { + createTable: async () => {}, + upsertEntity: async () => { upsertCalled = true }, + } + }, + }, + } + } + return originalRequire.apply(this, arguments) + } + + try { + const { publishOrchestrationMetaSync } = require('../../src/helpers/otel-orchestration-store') + publishOrchestrationMetaSync('table-write', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + startTime: Date.now(), + status: 'open', + }) + await new Promise(resolve => setImmediate(resolve)) + assert.equal(upsertCalled, true) + } finally { + Module.prototype.require = originalRequire + delete require.cache[storePath] + if (previousStorage === undefined) { + delete process.env.AzureWebJobsStorage + } else { + process.env.AzureWebJobsStorage = previousStorage + } + } + }) + + it('reads orchestration metadata from azure table storage', async () => { + const previousStorage = process.env.AzureWebJobsStorage + process.env.AzureWebJobsStorage = 'UseDevelopmentStorage=true' + + delete require.cache[storePath] + const originalRequire = Module.prototype.require + Module.prototype.require = function (id) { + if (id === '@azure/data-tables') { + return { + TableClient: { + fromConnectionString () { + return { + createTable: async () => { + const error = new Error('exists') + error.statusCode = 409 + throw error + }, + getEntity: async () => ({ + traceId: '00000000000000000000000000000001', + spanId: '0000000000000005', + parentId: '0000000000000004', + status: 'open', + startTime: Date.now(), + }), + } + }, + }, + } + } + return originalRequire.apply(this, arguments) + } + + try { + const { readOrchestrationSpanMetaAsync } = require('../../src/helpers/otel-orchestration-store') + const meta = await readOrchestrationSpanMetaAsync('table-read') + assert.equal(meta.spanId, '0000000000000005') + assert.equal(meta.parentId, '0000000000000004') + } finally { + Module.prototype.require = originalRequire + delete require.cache[storePath] + if (previousStorage === undefined) { + delete process.env.AzureWebJobsStorage + } else { + process.env.AzureWebJobsStorage = previousStorage + } + } + }) + + it('exports orchestration spans with errors', () => { + process.env.DD_TRACE_OTEL_ENABLED = 'true' + process.env.DD_TRACE_AZURE_DURABLE_FUNCTIONS_ENABLED = 'false' + + const ddtrace = require('../../../dd-trace') + ddtrace.init({ plugins: false, sampleRate: 1 }) + new ddtrace.TracerProvider().register() + + const invocationContext = { + traceContext: { + traceParent: '00-00000000000000000000000000000001-0000000000000003-00', + }, + } + + ensureOrchestrationMeta('error-inst', invocationContext, 'PizzaOrderOrchestration') + assert.equal( + completeOrchestrationSpan( + '@azure/durable-functions', + 'error-inst', + invocationContext, + 'PizzaOrderOrchestration', + new Error('failed'), + ), + true, + ) + }) +}) + +describe('otel-orchestration-export', () => { + it('ignores invalid traceparent headers', () => { + assert.equal(getParentFromTraceContext({ traceParent: 'invalid' }), undefined) + }) + + it('ignores HTTP parent metadata without span ids', () => { + assert.equal(createOrchestrationMetaFromHttpParent('inst', {}, 'Orch'), undefined) + }) + + it('exports orchestration spans directly from metadata', () => { + process.env.DD_TRACE_OTEL_ENABLED = 'true' + process.env.DD_TRACE_AZURE_DURABLE_FUNCTIONS_ENABLED = 'false' + + const ddtrace = require('../../../dd-trace') + ddtrace.init({ plugins: false, sampleRate: 1 }) + new ddtrace.TracerProvider().register() + + assert.equal(exportOrchestrationSpanFromMeta('@azure/durable-functions', { + traceId: '00000000000000000000000000000001', + spanId: '0000000000000002', + parentId: '0000000000000004', + functionName: 'DirectExport', + startTime: Date.now(), + }), true) + }) +}) diff --git a/packages/dd-trace/test/plugins/agent.js b/packages/dd-trace/test/plugins/agent.js index 182939f2c33..50c4fa09bcd 100644 --- a/packages/dd-trace/test/plugins/agent.js +++ b/packages/dd-trace/test/plugins/agent.js @@ -77,6 +77,8 @@ const TRACKED_NON_PREFIX_ENV_NAMES = new Set([ 'WEBSITE_OWNER_NAME', 'WEBSITE_OS', 'WEBSITE_RESOURCE_GROUP', + // durable orchestration span store (shared table client, cached at first read) + 'AzureWebJobsStorage', // CI-visibility runner detection (test plugins, ci-visibility exporters) 'CUCUMBER_WORKER_ID', 'JEST_WORKER_ID',