diff --git a/index.d.ts b/index.d.ts index 76a3cdc56bf..3906b29fea7 100644 --- a/index.d.ts +++ b/index.d.ts @@ -4297,6 +4297,12 @@ declare namespace tracer { * Each definition requires a `name` and optionally accepts `description`, `schema`, and `version`. * */ toolDefinitions?: ToolDefinition[] + + /** + * An Agent object identifying the versioned agent this span represents. + * Only used on `agent` spans. + */ + agent?: Agent } interface AnnotationContextOptions { @@ -4321,6 +4327,24 @@ declare namespace tracer { * A Prompt object that represents the prompt used for an LLM call. Only used on `llm` spans. */ prompt?: Prompt, + + /** + * An Agent object identifying the versioned agent running in this context. Sets the + * `agent_version` tag on `agent` spans created within the context; other span kinds + * are unaffected. + */ + agent?: Agent, + } + + /** + * An Agent object that identifies a versioned agent. + */ + interface Agent { + /** + * User tag for the version of the agent. Set as an `agent_version` tag on the agent span + * only, never on its children. + */ + version?: string, } interface RoutingContextOptions { @@ -4378,6 +4402,12 @@ declare namespace tracer { * If not provided for LLM or embedding spans, a default value of 'custom' will be set. */ modelProvider?: string, + + /** + * The version of this agent. Set as an `agent_version` tag on this span, and not on its + * child spans. Only used on `agent` spans. + */ + version?: string, } interface LLMObsNamedSpanOptions extends LLMObsSpanOptions { diff --git a/index.d.v5.ts b/index.d.v5.ts index 34fa62e7646..1e3054f967b 100644 --- a/index.d.v5.ts +++ b/index.d.v5.ts @@ -4496,6 +4496,12 @@ declare namespace tracer { * Each definition requires a `name` and optionally accepts `description`, `schema`, and `version`. * */ toolDefinitions?: ToolDefinition[] + + /** + * An Agent object identifying the versioned agent this span represents. + * Only used on `agent` spans. + */ + agent?: Agent } interface AnnotationContextOptions { @@ -4520,6 +4526,24 @@ declare namespace tracer { * A Prompt object that represents the prompt used for an LLM call. Only used on `llm` spans. */ prompt?: Prompt, + + /** + * An Agent object identifying the versioned agent running in this context. Sets the + * `agent_version` tag on `agent` spans created within the context; other span kinds + * are unaffected. + */ + agent?: Agent, + } + + /** + * An Agent object that identifies a versioned agent. + */ + interface Agent { + /** + * User tag for the version of the agent. Set as an `agent_version` tag on the agent span + * only, never on its children. + */ + version?: string, } interface RoutingContextOptions { @@ -4577,6 +4601,12 @@ declare namespace tracer { * If not provided for LLM or embedding spans, a default value of 'custom' will be set. */ modelProvider?: string, + + /** + * The version of this agent. Set as an `agent_version` tag on this span, and not on its + * child spans. Only used on `agent` spans. + */ + version?: string, } interface LLMObsNamedSpanOptions extends LLMObsSpanOptions { diff --git a/packages/dd-trace/src/llmobs/constants/tags.js b/packages/dd-trace/src/llmobs/constants/tags.js index ed37a1222e1..272a0f0bdc9 100644 --- a/packages/dd-trace/src/llmobs/constants/tags.js +++ b/packages/dd-trace/src/llmobs/constants/tags.js @@ -28,6 +28,10 @@ module.exports = { SAMPLING_DECISION_SAMPLED: '1', SAMPLING_DECISION_DROPPED: '0', TAGS: '_ml_obs.tags', + // Holds the version a user supplied, until the span kind is known at finish. + AGENT_VERSION: '_ml_obs.agent_version', + // Agent tracking tag. Set on agent spans only, at span finish. + AGENT_VERSION_TAG_KEY: 'agent_version', NAME: '_ml_obs.name', TRACE_ID: '_ml_obs.trace_id', PROPAGATED_TRACE_ID_KEY: '_dd.p.llmobs_trace_id', diff --git a/packages/dd-trace/src/llmobs/sdk.js b/packages/dd-trace/src/llmobs/sdk.js index a20111ae06a..30ce8591c8d 100644 --- a/packages/dd-trace/src/llmobs/sdk.js +++ b/packages/dd-trace/src/llmobs/sdk.js @@ -143,7 +143,7 @@ class LLMObs extends NoopLLMObs { const { spanOptions, ...llmobsOptions - } = this.#extractOptions(options) + } = this.#extractOptions(options, kind) if (fn.length > 1) { return this._tracer.trace(name, spanOptions, (span, cb) => @@ -173,7 +173,7 @@ class LLMObs extends NoopLLMObs { const { spanOptions, ...llmobsOptions - } = this.#extractOptions(options) + } = this.#extractOptions(options, kind) const llmobs = this @@ -282,7 +282,7 @@ class LLMObs extends NoopLLMObs { throw new Error('LLMObs span must have a span kind specified') } - const { inputData, outputData, metadata, metrics, tags, prompt, costTags, toolDefinitions } = options + const { inputData, outputData, metadata, metrics, tags, prompt, costTags, toolDefinitions, agent } = options if (inputData || outputData) { if (spanKind === 'llm') { @@ -315,6 +315,9 @@ class LLMObs extends NoopLLMObs { if (toolDefinitions != null) { this._tagger.tagToolDefinitions(span, toolDefinitions) } + if (agent?.version != null) { + this._tagger.tagAgentVersion(span, agent.version) + } } catch (e) { if (e.ddErrorTag) { err = e.ddErrorTag @@ -671,21 +674,28 @@ class LLMObs extends NoopLLMObs { } } - #extractOptions (options) { + #extractOptions (options, kind) { const { modelName, modelProvider, sessionId, mlApp, + version, _decorator, ...spanOptions } = options + if (version != null && kind !== 'agent') { + logger.warn(`[LLM Observability] The "version" option is only supported on agent spans. Ignoring it for "${ + kind}" spans.`) + } + return { mlApp, modelName, modelProvider, sessionId, + version, _decorator, spanOptions, } diff --git a/packages/dd-trace/src/llmobs/span_processor.js b/packages/dd-trace/src/llmobs/span_processor.js index bddf24a2377..3b122622f3e 100644 --- a/packages/dd-trace/src/llmobs/span_processor.js +++ b/packages/dd-trace/src/llmobs/span_processor.js @@ -14,6 +14,8 @@ const { MODEL_NAME, MODEL_PROVIDER, METADATA, + AGENT_VERSION, + AGENT_VERSION_TAG_KEY, COST_TAGS, TOOL_DEFINITIONS, INPUT_MESSAGES, @@ -212,7 +214,7 @@ class LLMObsSpanProcessor { const name = mlObsTags[NAME] || span._name - const tags = this.#getTags(span, mlApp, sessionId, error) + const tags = this.#getTags(span, mlApp, sessionId, error, spanKind) llmObsSpan._tags = tags const processedSpan = this.#runProcessor(llmObsSpan) @@ -332,7 +334,7 @@ class LLMObsSpanProcessor { return metadata._dd } - #getTags (span, mlApp, sessionId, error) { + #getTags (span, mlApp, sessionId, error, spanKind) { let tags = { ...this.#config.parsedDdTags, version: this.#config.version, @@ -356,6 +358,14 @@ class LLMObsSpanProcessor { const existingTags = LLMObsTagger.tagMap.get(span)?.[TAGS] || {} if (existingTags) tags = { ...tags, ...existingTags } + // An agent version can be supplied before the kind is settled (annotation context spans a whole + // block, and integrations may still call `changeKind`), so it is only promoted to a tag here, + // once the final kind is known. Only agent spans carry it, never their children. + if (spanKind === 'agent') { + const agentVersion = LLMObsTagger.tagMap.get(span)?.[AGENT_VERSION] + if (agentVersion) tags[AGENT_VERSION_TAG_KEY] = agentVersion + } + return tags } diff --git a/packages/dd-trace/src/llmobs/tagger.js b/packages/dd-trace/src/llmobs/tagger.js index c5fa928b3e4..7e0fa6bdbcf 100644 --- a/packages/dd-trace/src/llmobs/tagger.js +++ b/packages/dd-trace/src/llmobs/tagger.js @@ -22,6 +22,7 @@ const { INPUT_MESSAGES, OUTPUT_MESSAGES, TAGS, + AGENT_VERSION, NAME, PARENT_AGENT_NAME, PARENT_AGENT_SPAN_ID, @@ -119,6 +120,7 @@ class LLMObsTagger { kind, name, integration, + version, _decorator, } = {}) { if (!this.#config.llmobs.DD_LLMOBS_ENABLED) return @@ -200,6 +202,12 @@ class LLMObsTagger { const annotationContextPrompt = annotationContext?.prompt if (annotationContextPrompt) this.tagPrompt(span, annotationContextPrompt) + // Stashed rather than tagged: the kind here can still be changed before finish, and + // only agent spans carry the tag. The span processor materializes it. + // Annotation context wins over the span's own option, matching `name` above. + const agentVersion = annotationContext?.agent?.version ?? version + if (agentVersion) this.tagAgentVersion(span, agentVersion) + const routing = storage.getStore()?.routingContext if (routing) { this._setTag(span, ROUTING_API_KEY, routing.apiKey) @@ -361,6 +369,20 @@ class LLMObsTagger { } } + /** + * Stashes a user-supplied agent version on the span. The span processor promotes it to an + * `agent_version` span tag at finish, but only if the span's final kind is `agent`. + * @param {import('../opentracing/span')} span + * @param {string} version + */ + tagAgentVersion (span, version) { + if (typeof version !== 'string' || !version) { + this.#handleFailure('Agent version must be a non-empty string.', 'invalid_agent_version') + return + } + this._setTag(span, AGENT_VERSION, version) + } + /** * Validates and tags cost tag keys on an LLMObs span. Cost tag references are validated against * the span's already-applied tags, which are read from the registry. diff --git a/packages/dd-trace/test/llmobs/sdk/integration.spec.js b/packages/dd-trace/test/llmobs/sdk/integration.spec.js index a7bd6bd64ef..5b9add1e6f8 100644 --- a/packages/dd-trace/test/llmobs/sdk/integration.spec.js +++ b/packages/dd-trace/test/llmobs/sdk/integration.spec.js @@ -647,4 +647,50 @@ describe('end to end sdk integration tests', () => { assert.equal(llmobsSpans[0].tags.includes('prompt_tracking_instrumentation_method:annotated'), true) }) }) + + describe('agent versioning', () => { + it('tags an agent span from the span options', async () => { + llmobs.trace({ kind: 'agent', name: 'myAgent', version: 'v3' }, () => {}) + + const { llmobsSpans } = await getEvents() + assert.equal(llmobsSpans.length, 1) + assert.equal(getTag(llmobsSpans[0], 'agent_version'), 'v3') + }) + + it('tags an agent span via annotate', async () => { + llmobs.trace({ kind: 'agent', name: 'myAgent' }, () => { + llmobs.annotate({ agent: { version: 'v3' } }) + }) + + const { llmobsSpans } = await getEvents() + assert.equal(llmobsSpans.length, 1) + assert.equal(getTag(llmobsSpans[0], 'agent_version'), 'v3') + }) + + it('tags only the agent span within an annotation context, not its children', async () => { + llmobs.annotationContext({ agent: { version: 'v3' } }, () => { + llmobs.trace({ kind: 'agent', name: 'myAgent' }, () => { + llmobs.trace({ kind: 'tool', name: 'myTool' }, () => {}) + }) + }) + + const { llmobsSpans } = await getEvents(2) + assert.equal(llmobsSpans.length, 2) + + const agentSpan = llmobsSpans.find(span => span.meta['span.kind'] === 'agent') + const toolSpan = llmobsSpans.find(span => span.meta['span.kind'] === 'tool') + assert.equal(getTag(agentSpan, 'agent_version'), 'v3') + assert.equal(getTag(toolSpan, 'agent_version'), undefined) + }) + + it('does not tag a non-agent span', async () => { + llmobs.trace({ kind: 'workflow', name: 'myWorkflow', version: 'v3' }, () => { + llmobs.annotate({ agent: { version: 'v3' } }) + }) + + const { llmobsSpans } = await getEvents() + assert.equal(llmobsSpans.length, 1) + assert.equal(getTag(llmobsSpans[0], 'agent_version'), undefined) + }) + }) }) diff --git a/packages/dd-trace/test/llmobs/tagger.spec.js b/packages/dd-trace/test/llmobs/tagger.spec.js index cfdf53cf821..5b684dab581 100644 --- a/packages/dd-trace/test/llmobs/tagger.spec.js +++ b/packages/dd-trace/test/llmobs/tagger.spec.js @@ -629,6 +629,26 @@ describe('tagger', () => { }) }) + describe('tagAgentVersion', () => { + it('stashes the version under the internal key', () => { + tagger._register(span) + tagger.tagAgentVersion(span, 'v3') + assertObjectContains(Tagger.tagMap.get(span), { + '_ml_obs.agent_version': 'v3', + }) + }) + + it('throws for a non string version', () => { + tagger._register(span) + assert.throws(() => tagger.tagAgentVersion(span, 3), { message: /Agent version must be a non-empty string/ }) + }) + + it('throws for an empty string version', () => { + tagger._register(span) + assert.throws(() => tagger.tagAgentVersion(span, ''), { message: /Agent version must be a non-empty string/ }) + }) + }) + describe('tagSpanTags', () => { it('sets tags on a span', () => { const tags = { foo: 'bar' }