feat(llmobs): add prompt tracking telemetry tags for OpenAI (#7106)

* feat(llmobs): add prompt tracking telemetry tags for OpenAI

- Add prompt_tracking_source:auto tag for auto-instrumented prompts
- Add prompt_multimodal:true tag for prompts with image/file inputs
- Add hasMultimodalInputs helper function in utils
- Note: LangChain prompt tracking deferred to separate PR

* docs :docstrings

* refactor(llmobs): rename prompt tracking source tag to instrumentation method

- Update prompt tracking source to prompt_tracking_instrumentation_method in OpenAI plugin
- Adjust related documentation and test assertions to reflect the new naming convention

* refactor tagging logic

* refactor constant logic

* re-organize constants

Author: PROFeNoM

packages/dd-trace/src/llmobs/constants/tags.js

@@ -36,5 +36,11 @@ module.exports = {
   CACHE_WRITE_INPUT_TOKENS_METRIC_KEY: 'cache_write_input_tokens',
   REASONING_OUTPUT_TOKENS_METRIC_KEY: 'reasoning_output_tokens',
 
-  DROPPED_IO_COLLECTION_ERROR: 'dropped_io'
+  DROPPED_IO_COLLECTION_ERROR: 'dropped_io',
+
+  PROMPT_TRACKING_INSTRUMENTATION_METHOD: 'prompt_tracking_instrumentation_method',
+  PROMPT_MULTIMODAL: 'prompt_multimodal',
+  INSTRUMENTATION_METHOD_AUTO: 'auto',
+  INSTRUMENTATION_METHOD_ANNOTATED: 'annotated',
+  INSTRUMENTATION_METHOD_UNKNOWN: 'unknown'
 }

packages/dd-trace/src/llmobs/plugins/openai/constants.js

@@ -0,0 +1,16 @@
+'use strict'
+
+const INPUT_TYPE_IMAGE = 'input_image'
+const INPUT_TYPE_FILE = 'input_file'
+const INPUT_TYPE_TEXT = 'input_text'
+
+const IMAGE_FALLBACK = '[image]'
+const FILE_FALLBACK = '[file]'
+
+module.exports = {
+  INPUT_TYPE_IMAGE,
+  INPUT_TYPE_FILE,
+  INPUT_TYPE_TEXT,
+  IMAGE_FALLBACK,
+  FILE_FALLBACK
+}

packages/dd-trace/src/llmobs/plugins/openai/index.js

@@ -1,7 +1,17 @@
 'use strict'
 
 const LLMObsPlugin = require('../base')
-const { extractChatTemplateFromInstructions, normalizePromptVariables, extractTextFromContentItem } = require('./utils')
+const {
+  extractChatTemplateFromInstructions,
+  normalizePromptVariables,
+  extractTextFromContentItem,
+  hasMultimodalInputs
+} = require('./utils')
+const {
+  PROMPT_TRACKING_INSTRUMENTATION_METHOD,
+  PROMPT_MULTIMODAL,
+  INSTRUMENTATION_METHOD_AUTO
+} = require('../../constants/tags')
 
 const allowedParamKeys = new Set([
   'max_output_tokens',
@@ -422,6 +432,11 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
           variables: normalizedVariables,
           chat_template: chatTemplate
         })
+        const tags = { [PROMPT_TRACKING_INSTRUMENTATION_METHOD]: INSTRUMENTATION_METHOD_AUTO }
+        if (hasMultimodalInputs(inputs.prompt.variables)) {
+          tags[PROMPT_MULTIMODAL] = 'true'
+        }
+        this._tagger.tagSpanTags(span, tags)
       }
     }
 

packages/dd-trace/src/llmobs/plugins/openai/utils.js

@@ -1,7 +1,11 @@
 'use strict'
 
-const IMAGE_FALLBACK = '[image]'
-const FILE_FALLBACK = '[file]'
+const {
+  INPUT_TYPE_IMAGE,
+  INPUT_TYPE_FILE,
+  IMAGE_FALLBACK,
+  FILE_FALLBACK
+} = require('./constants')
 
 const REGEX_SPECIAL_CHARS = /[.*+?^${}()|[\]\\]/g
 
@@ -77,11 +81,11 @@ function extractTextFromContentItem (contentItem) {
   }
 
   // For image/file items, extract the actual reference value
-  if (contentItem.type === 'input_image') {
+  if (contentItem.type === INPUT_TYPE_IMAGE) {
     return contentItem.image_url || contentItem.file_id || IMAGE_FALLBACK
   }
 
-  if (contentItem.type === 'input_file') {
+  if (contentItem.type === INPUT_TYPE_FILE) {
     return contentItem.file_id || contentItem.file_url || contentItem.filename || FILE_FALLBACK
   }
 
@@ -107,8 +111,16 @@ function normalizePromptVariables (variables) {
   )
 }
 
+function hasMultimodalInputs (variables) {
+  if (!variables) return false
+  return Object.values(variables).some(value =>
+    value?.type === INPUT_TYPE_IMAGE || value?.type === INPUT_TYPE_FILE
+  )
+}
+
 module.exports = {
   extractChatTemplateFromInstructions,
   normalizePromptVariables,
-  extractTextFromContentItem
+  extractTextFromContentItem,
+  hasMultimodalInputs
 }

packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js

@@ -919,7 +919,7 @@ describe('integrations', () => {
                 'Image reference 2: file-BCuhT1HQ24kmtsuuzF1mh2\n\n' +
                 'Please provide a comprehensive analysis.'
             }
-          ])
+          ], { promptMultimodal: true })
         })
 
         it('submits a response span with prompt tracking - mixed input types (url preserved)', async function () {
@@ -969,7 +969,7 @@ describe('integrations', () => {
                 'Image reference 2: file-BCuhT1HQ24kmtsuuzF1mh2\n\n' +
                 'Please provide a comprehensive analysis.'
             }
-          ])
+          ], { promptMultimodal: true })
         })
       })
 

packages/dd-trace/test/llmobs/util.js

@@ -366,8 +366,18 @@ function getLlmObsSpansFromRequests (llmobsSpanEventsRequests) {
  * @param {object} spanEvent - The LLMObs span event to verify
  * @param {object} expectedPrompt - Expected prompt metadata (id, version, variables, chat_template)
  * @param {Array<{role: string, content: string}>} expectedInputMessages - Expected input messages
+ * @param {object} options - Optional configuration
+ * @param {string} options.promptTrackingInstrumentationMethod - Expected prompt tracking instrumentation method
+ * ('auto' or 'manual'), defaults to 'auto'
+ * @param {boolean} options.promptMultimodal - Whether prompt contains multimodal inputs,
+ *   defaults to false
  */
-function assertPromptTracking (spanEvent, expectedPrompt, expectedInputMessages) {
+function assertPromptTracking (
+  spanEvent,
+  expectedPrompt,
+  expectedInputMessages,
+  { promptTrackingInstrumentationMethod = 'auto', promptMultimodal = false } = {}
+) {
   // Verify input messages are captured from instructions
   assert(spanEvent.meta.input.messages, 'Input messages should be present')
   assert(Array.isArray(spanEvent.meta.input.messages), 'Input messages should be an array')
@@ -385,6 +395,13 @@ function assertPromptTracking (spanEvent, expectedPrompt, expectedInputMessages)
   assert.strictEqual(prompt.version, expectedPrompt.version)
   assert.deepStrictEqual(prompt.variables, expectedPrompt.variables)
   assert.deepStrictEqual(prompt.chat_template, expectedPrompt.chat_template)
+
+  // Verify tags
+  assert(spanEvent.tags, 'Span event should include tags')
+  assert(spanEvent.tags.includes(`prompt_tracking_instrumentation_method:${promptTrackingInstrumentationMethod}`))
+  if (promptMultimodal) {
+    assert(spanEvent.tags.includes('prompt_multimodal:true'))
+  }
 }
 
 module.exports = {