Skip to content

Commit ec5cbc1

Browse files
XG-xinrochdev
XG-xin
authored and
rochdev
committed
feat(llmobs): add reasoning token metrics in openai plugin (#7026)
1 parent 3574815 commit ec5cbc1

6 files changed

Lines changed: 218 additions & 12 deletions

File tree

‎packages/dd-trace/src/llmobs/constants/tags.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ module.exports = {
3434
TOTAL_TOKENS_METRIC_KEY: 'total_tokens',
3535
CACHE_READ_INPUT_TOKENS_METRIC_KEY: 'cache_read_input_tokens',
3636
CACHE_WRITE_INPUT_TOKENS_METRIC_KEY: 'cache_write_input_tokens',
37+
REASONING_OUTPUT_TOKENS_METRIC_KEY: 'reasoning_output_tokens',
3738

3839
DROPPED_IO_COLLECTION_ERROR: 'dropped_io'
3940
}

‎packages/dd-trace/src/llmobs/plugins/openai/index.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
116116
metrics.cacheReadTokens = cacheReadTokens
117117
}
118118
}
119+
// Reasoning tokens - Responses API returns `output_tokens_details`, `completion_tokens_details`
120+
const reasoningOutputObject = tokenUsage.output_tokens_details ?? tokenUsage.completion_tokens_details
121+
const reasoningOutputTokens = reasoningOutputObject?.reasoning_tokens ?? 0
122+
if (reasoningOutputTokens !== undefined) metrics.reasoningOutputTokens = reasoningOutputTokens
119123
}
120124

121125
return metrics
@@ -429,9 +433,6 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
429433
if (response.tool_choice !== undefined) outputMetadata.tool_choice = response.tool_choice
430434
if (response.truncation !== undefined) outputMetadata.truncation = response.truncation
431435
if (response.text !== undefined) outputMetadata.text = response.text
432-
if (response.usage?.output_tokens_details?.reasoning_tokens !== undefined) {
433-
outputMetadata.reasoning_tokens = response.usage.output_tokens_details.reasoning_tokens
434-
}
435436

436437
this._tagger.tagMetadata(span, outputMetadata) // update the metadata with the output metadata
437438
}

‎packages/dd-trace/src/llmobs/tagger.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ const {
2525
INPUT_TOKENS_METRIC_KEY,
2626
OUTPUT_TOKENS_METRIC_KEY,
2727
TOTAL_TOKENS_METRIC_KEY,
28+
REASONING_OUTPUT_TOKENS_METRIC_KEY,
2829
INTEGRATION,
2930
DECORATOR,
3031
PROPAGATED_ML_APP_KEY
@@ -164,6 +165,9 @@ class LLMObsTagger {
164165
case 'cacheWriteTokens':
165166
processedKey = CACHE_WRITE_INPUT_TOKENS_METRIC_KEY
166167
break
168+
case 'reasoningOutputTokens':
169+
processedKey = REASONING_OUTPUT_TOKENS_METRIC_KEY
170+
break
167171
}
168172

169173
if (typeof value === 'number') {

‎packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_01aba65f.yaml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
interactions:
2+
- request:
3+
body: '{"model":"gpt-5-mini","input":"Solve this step by step: What is 15 * 24?","max_output_tokens":500,"stream":false}'
4+
headers:
5+
? !!python/object/apply:multidict._multidict.istr
6+
- Accept
7+
: - application/json
8+
? !!python/object/apply:multidict._multidict.istr
9+
- Accept-Encoding
10+
: - gzip, deflate
11+
? !!python/object/apply:multidict._multidict.istr
12+
- Accept-Language
13+
: - '*'
14+
? !!python/object/apply:multidict._multidict.istr
15+
- Connection
16+
: - keep-alive
17+
Content-Length:
18+
- '113'
19+
? !!python/object/apply:multidict._multidict.istr
20+
- Content-Type
21+
: - application/json
22+
? !!python/object/apply:multidict._multidict.istr
23+
- User-Agent
24+
: - OpenAI/JS 6.8.1
25+
? !!python/object/apply:multidict._multidict.istr
26+
- X-Stainless-Arch
27+
: - arm64
28+
? !!python/object/apply:multidict._multidict.istr
29+
- X-Stainless-Lang
30+
: - js
31+
? !!python/object/apply:multidict._multidict.istr
32+
- X-Stainless-OS
33+
: - MacOS
34+
? !!python/object/apply:multidict._multidict.istr
35+
- X-Stainless-Package-Version
36+
: - 6.8.1
37+
? !!python/object/apply:multidict._multidict.istr
38+
- X-Stainless-Retry-Count
39+
: - '0'
40+
? !!python/object/apply:multidict._multidict.istr
41+
- X-Stainless-Runtime
42+
: - node
43+
? !!python/object/apply:multidict._multidict.istr
44+
- X-Stainless-Runtime-Version
45+
: - v22.21.1
46+
? !!python/object/apply:multidict._multidict.istr
47+
- sec-fetch-mode
48+
: - cors
49+
method: POST
50+
uri: https://api.openai.com/v1/responses
51+
response:
52+
body:
53+
string: "{\n \"id\": \"resp_0e28e5318868c83a0169308b613bdc819184e3bca92ada65e6\",\n
54+
\ \"object\": \"response\",\n \"created_at\": 1764789089,\n \"status\":
55+
\"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\":
56+
\"developer\"\n },\n \"error\": null,\n \"incomplete_details\": null,\n
57+
\ \"instructions\": null,\n \"max_output_tokens\": 500,\n \"max_tool_calls\":
58+
null,\n \"model\": \"gpt-5-mini-2025-08-07\",\n \"output\": [\n {\n \"id\":
59+
\"rs_0e28e5318868c83a0169308b617fa08191b6726155d23d010d\",\n \"type\":
60+
\"reasoning\",\n \"summary\": []\n },\n {\n \"id\": \"msg_0e28e5318868c83a0169308b63623c819198d6e2c9b0eb26b0\",\n
61+
\ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\":
62+
[\n {\n \"type\": \"output_text\",\n \"annotations\":
63+
[],\n \"logprobs\": [],\n \"text\": \"Method 1 \\u2014 distributive
64+
property:\\n15 \\u00d7 24 = 15 \\u00d7 (20 + 4)\\n= 15 \\u00d7 20 + 15 \\u00d7
65+
4\\n= 300 + 60\\n= 360\\n\\nMethod 2 \\u2014 split the other way:\\n24 \\u00d7
66+
15 = 24 \\u00d7 (10 + 5)\\n= 24 \\u00d7 10 + 24 \\u00d7 5\\n= 240 + 120\\n=
67+
360\\n\\nAnswer: 360.\"\n }\n ],\n \"role\": \"assistant\"\n
68+
\ }\n ],\n \"parallel_tool_calls\": true,\n \"previous_response_id\":
69+
null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": null,\n
70+
\ \"reasoning\": {\n \"effort\": \"medium\",\n \"summary\": null\n },\n
71+
\ \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\":
72+
false,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\":
73+
\"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\":
74+
\"auto\",\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\":
75+
\"disabled\",\n \"usage\": {\n \"input_tokens\": 20,\n \"input_tokens_details\":
76+
{\n \"cached_tokens\": 0\n },\n \"output_tokens\": 232,\n \"output_tokens_details\":
77+
{\n \"reasoning_tokens\": 128\n },\n \"total_tokens\": 252\n },\n
78+
\ \"user\": null,\n \"metadata\": {}\n}"
79+
headers:
80+
CF-RAY:
81+
- 9a855ebe0d40cb6a-BOS
82+
Connection:
83+
- keep-alive
84+
Content-Encoding:
85+
- gzip
86+
Content-Type:
87+
- application/json
88+
Date:
89+
- Wed, 03 Dec 2025 19:11:33 GMT
90+
Server:
91+
- cloudflare
92+
Set-Cookie:
93+
- __cf_bm=tGToqitV1eFrOrI1DCsm6GfeeW0ajMWiagCcgm7EZ84-1764789093-1.0.1.1-d.Bwuc72eKGJd7bZL0hQRAOgBqhp15Rk0H9FzUo.0s8hqVVPBeKHE39I5EaaTi2YZdtQyCFyHmd3iILpZJKskSfEdJ3MtfEHr_4Ktk7yDw8;
94+
path=/; expires=Wed, 03-Dec-25 19:41:33 GMT; domain=.api.openai.com; HttpOnly;
95+
Secure; SameSite=None
96+
- _cfuvid=NO0A21ruRpV_Vd_9ghyhNxI_FfJlolr0EqD7FHB3tOs-1764789093127-0.0.1.1-604800000;
97+
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
98+
Strict-Transport-Security:
99+
- max-age=31536000; includeSubDomains; preload
100+
Transfer-Encoding:
101+
- chunked
102+
X-Content-Type-Options:
103+
- nosniff
104+
alt-svc:
105+
- h3=":443"; ma=86400
106+
cf-cache-status:
107+
- DYNAMIC
108+
openai-organization:
109+
- datadog-staging
110+
openai-processing-ms:
111+
- '3895'
112+
openai-project:
113+
- proj_gt6TQZPRbZfoY2J9AQlEJMpd
114+
openai-version:
115+
- '2020-10-01'
116+
x-envoy-upstream-service-time:
117+
- '3900'
118+
x-ratelimit-limit-requests:
119+
- '30000'
120+
x-ratelimit-limit-tokens:
121+
- '180000000'
122+
x-ratelimit-remaining-requests:
123+
- '29999'
124+
x-ratelimit-remaining-tokens:
125+
- '179999985'
126+
x-ratelimit-reset-requests:
127+
- 2ms
128+
x-ratelimit-reset-tokens:
129+
- 0s
130+
x-request-id:
131+
- req_d13dea53ffc94a628dfa63719a5bcbb9
132+
status:
133+
code: 200
134+
message: OK
135+
version: 1

‎packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@ describe('integrations', () => {
6161
metrics: {
6262
input_tokens: MOCK_NUMBER,
6363
output_tokens: MOCK_NUMBER,
64-
total_tokens: MOCK_NUMBER
64+
total_tokens: MOCK_NUMBER,
65+
reasoning_output_tokens: 0
6566
},
6667
modelName: 'gpt-3.5-turbo-instruct:20230824-v2',
6768
modelProvider: 'openai',
@@ -113,6 +114,7 @@ describe('integrations', () => {
113114
],
114115
metrics: {
115116
cache_read_input_tokens: 0,
117+
reasoning_output_tokens: 0,
116118
input_tokens: MOCK_NUMBER,
117119
output_tokens: MOCK_NUMBER,
118120
total_tokens: MOCK_NUMBER
@@ -146,7 +148,9 @@ describe('integrations', () => {
146148
{ text: 'hello world' }
147149
],
148150
outputValue: '[1 embedding(s) returned]',
149-
metrics: { input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER },
151+
metrics: {
152+
input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER, reasoning_output_tokens: 0
153+
},
150154
modelName: 'text-embedding-ada-002-v2',
151155
modelProvider: 'openai',
152156
metadata: { encoding_format: 'base64' },

‎packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js‎

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,9 @@ describe('integrations', () => {
8181
outputMessages: [
8282
{ content: MOCK_STRING, role: '' }
8383
],
84-
metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER },
84+
metrics: {
85+
input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER, reasoning_output_tokens: 0
86+
},
8587
modelName: 'gpt-3.5-turbo-instruct:20230824-v2',
8688
modelProvider: 'openai',
8789
metadata: {
@@ -128,6 +130,7 @@ describe('integrations', () => {
128130
],
129131
metrics: {
130132
cache_read_input_tokens: 0,
133+
reasoning_output_tokens: 0,
131134
input_tokens: MOCK_NUMBER,
132135
output_tokens: MOCK_NUMBER,
133136
total_tokens: MOCK_NUMBER
@@ -161,7 +164,9 @@ describe('integrations', () => {
161164
{ text: 'hello world' }
162165
],
163166
outputValue: '[1 embedding(s) returned]',
164-
metrics: { input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER },
167+
metrics: {
168+
input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER, reasoning_output_tokens: 0
169+
},
165170
modelName: 'text-embedding-ada-002-v2',
166171
modelProvider: 'openai',
167172
metadata: { encoding_format: 'base64' },
@@ -220,6 +225,7 @@ describe('integrations', () => {
220225
tags: { ml_app: 'test', integration: 'openai' },
221226
metrics: {
222227
cache_read_input_tokens: 0,
228+
reasoning_output_tokens: 0,
223229
input_tokens: MOCK_NUMBER,
224230
output_tokens: MOCK_NUMBER,
225231
total_tokens: MOCK_NUMBER
@@ -268,7 +274,12 @@ describe('integrations', () => {
268274
outputMessages: [
269275
{ content: '\n\nHello! How can I assist you?', role: '' }
270276
],
271-
metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER },
277+
metrics: {
278+
input_tokens: MOCK_NUMBER,
279+
output_tokens: MOCK_NUMBER,
280+
total_tokens: MOCK_NUMBER,
281+
reasoning_output_tokens: 0
282+
},
272283
modelName: 'gpt-3.5-turbo-instruct:20230824-v2',
273284
modelProvider: 'openai',
274285
metadata: {
@@ -329,6 +340,7 @@ describe('integrations', () => {
329340
],
330341
metrics: {
331342
cache_read_input_tokens: 0,
343+
reasoning_output_tokens: 0,
332344
input_tokens: MOCK_NUMBER,
333345
output_tokens: MOCK_NUMBER,
334346
total_tokens: MOCK_NUMBER
@@ -413,6 +425,7 @@ describe('integrations', () => {
413425
tags: { ml_app: 'test', integration: 'openai' },
414426
metrics: {
415427
cache_read_input_tokens: 0,
428+
reasoning_output_tokens: 0,
416429
input_tokens: MOCK_NUMBER,
417430
output_tokens: MOCK_NUMBER,
418431
total_tokens: MOCK_NUMBER
@@ -599,6 +612,7 @@ describe('integrations', () => {
599612
],
600613
metrics: {
601614
cache_read_input_tokens: 0,
615+
reasoning_output_tokens: 0,
602616
input_tokens: 1221,
603617
output_tokens: 100,
604618
total_tokens: 1321
@@ -647,6 +661,7 @@ describe('integrations', () => {
647661
output_tokens: 100,
648662
total_tokens: 1320,
649663
cache_read_input_tokens: 1152,
664+
reasoning_output_tokens: 0
650665
},
651666
modelName: 'gpt-4o-2024-08-06',
652667
modelProvider: 'openai',
@@ -689,7 +704,8 @@ describe('integrations', () => {
689704
input_tokens: MOCK_NUMBER,
690705
output_tokens: MOCK_NUMBER,
691706
total_tokens: MOCK_NUMBER,
692-
cache_read_input_tokens: 0
707+
cache_read_input_tokens: 0,
708+
reasoning_output_tokens: 0
693709
},
694710
modelName: 'gpt-4o-mini-2024-07-18',
695711
modelProvider: 'openai',
@@ -700,7 +716,6 @@ describe('integrations', () => {
700716
tool_choice: 'auto',
701717
truncation: 'disabled',
702718
text: { format: { type: 'text' }, verbosity: 'medium' },
703-
reasoning_tokens: 0,
704719
stream: false
705720
},
706721
tags: { ml_app: 'test', integration: 'openai' }
@@ -739,7 +754,8 @@ describe('integrations', () => {
739754
input_tokens: MOCK_NUMBER,
740755
output_tokens: MOCK_NUMBER,
741756
total_tokens: MOCK_NUMBER,
742-
cache_read_input_tokens: 0
757+
cache_read_input_tokens: 0,
758+
reasoning_output_tokens: 0
743759
},
744760
modelName: 'gpt-4o-mini-2024-07-18',
745761
modelProvider: 'openai',
@@ -750,7 +766,6 @@ describe('integrations', () => {
750766
tool_choice: 'auto',
751767
truncation: 'disabled',
752768
text: { format: { type: 'text' }, verbosity: 'medium' },
753-
reasoning_tokens: 0,
754769
stream: true
755770
},
756771
tags: { ml_app: 'test', integration: 'openai' }
@@ -957,6 +972,52 @@ describe('integrations', () => {
957972
])
958973
})
959974
})
975+
976+
it('submits a response span with reasoning tokens', async function () {
977+
if (semifies(realVersion, '<4.87.0')) {
978+
this.skip()
979+
}
980+
981+
await openai.responses.create({
982+
model: 'gpt-5-mini',
983+
input: 'Solve this step by step: What is 15 * 24?',
984+
max_output_tokens: 500,
985+
stream: false
986+
})
987+
988+
const { apmSpans, llmobsSpans } = await getEvents()
989+
assertLlmObsSpanEvent(llmobsSpans[0], {
990+
span: apmSpans[0],
991+
spanKind: 'llm',
992+
name: 'OpenAI.createResponse',
993+
inputMessages: [
994+
{ role: 'user', content: 'Solve this step by step: What is 15 * 24?' }
995+
],
996+
outputMessages: [
997+
{ role: 'reasoning', content: MOCK_STRING },
998+
{ role: 'assistant', content: MOCK_STRING }
999+
],
1000+
metrics: {
1001+
input_tokens: MOCK_NUMBER,
1002+
output_tokens: MOCK_NUMBER,
1003+
total_tokens: MOCK_NUMBER,
1004+
cache_read_input_tokens: MOCK_NUMBER,
1005+
reasoning_output_tokens: 128
1006+
},
1007+
modelName: 'gpt-5-mini-2025-08-07',
1008+
modelProvider: 'openai',
1009+
metadata: {
1010+
max_output_tokens: 500,
1011+
top_p: 1,
1012+
temperature: 1,
1013+
tool_choice: 'auto',
1014+
truncation: 'disabled',
1015+
text: { format: { type: 'text' }, verbosity: 'medium' },
1016+
stream: false
1017+
},
1018+
tags: { ml_app: 'test', integration: 'openai' }
1019+
})
1020+
})
9601021
})
9611022
})
9621023
})

0 commit comments

Comments
 (0)