feat: Store and display reasoning tokens separately (#5429)

* feat: Store and display reasoning tokens separately

* rename migration file

* change row to be non nullable

Author: H2Shami

bifrost/lib/clients/jawnTypes/private.ts

@@ -2042,6 +2042,8 @@ Json: JsonObject;
       /** Format: double */
       completion_tokens: number | null;
       /** Format: double */
+      reasoning_tokens: number | null;
+      /** Format: double */
       prompt_audio_tokens: number | null;
       /** Format: double */
       completion_audio_tokens: number | null;

bifrost/lib/clients/jawnTypes/public.ts

@@ -2117,6 +2117,8 @@ Json: JsonObject;
       /** Format: double */
       completion_tokens: number | null;
       /** Format: double */
+      reasoning_tokens: number | null;
+      /** Format: double */
       prompt_audio_tokens: number | null;
       /** Format: double */
       completion_audio_tokens: number | null;

clickhouse/migrations/schema_78_reasoning_tokens.sql

@@ -0,0 +1,2 @@
+ALTER TABLE request_response_rmt
+ADD COLUMN reasoning_tokens Int64 DEFAULT 0;

docs/swagger.json

@@ -5189,6 +5189,11 @@
 						"format": "double",
 						"nullable": true
 					},
+					"reasoning_tokens": {
+						"type": "number",
+						"format": "double",
+						"nullable": true
+					},
 					"prompt_audio_tokens": {
 						"type": "number",
 						"format": "double",
@@ -5327,6 +5332,7 @@
 					"prompt_cache_write_tokens",
 					"prompt_cache_read_tokens",
 					"completion_tokens",
+					"reasoning_tokens",
 					"prompt_audio_tokens",
 					"completion_audio_tokens",
 					"cost",

helicone-mcp/src/types/public.ts

@@ -2117,6 +2117,8 @@ Json: JsonObject;
       /** Format: double */
       completion_tokens: number | null;
       /** Format: double */
+      reasoning_tokens: number | null;
+      /** Format: double */
       prompt_audio_tokens: number | null;
       /** Format: double */
       completion_audio_tokens: number | null;

packages/llm-mapper/types.ts

@@ -244,6 +244,7 @@ type HeliconeMetadata = {
   totalTokens: number | null;
   promptTokens: number | null;
   completionTokens: number | null;
+  reasoningTokens: number | null;
   promptCacheWriteTokens: number | null;
   promptCacheReadTokens: number | null;
   latency: number | null;
@@ -366,6 +367,7 @@ export interface HeliconeRequest {
   prompt_cache_write_tokens: number | null;
   prompt_cache_read_tokens: number | null;
   completion_tokens: number | null;
+  reasoning_tokens: number | null;
   prompt_audio_tokens: number | null;
   completion_audio_tokens: number | null;
   cost: number | null;

packages/llm-mapper/utils/getMappedContent.ts

@@ -80,6 +80,7 @@ const metaDataFromHeliconeRequest = (
     createdAt: heliconeRequest.request_created_at,
     path: heliconeRequest.request_path,
     completionTokens: heliconeRequest.completion_tokens,
+    reasoningTokens: heliconeRequest.reasoning_tokens,
     promptCacheWriteTokens: heliconeRequest.prompt_cache_write_tokens,
     promptCacheReadTokens: heliconeRequest.prompt_cache_read_tokens,
     promptTokens: heliconeRequest.prompt_tokens,

valhalla/jawn/src/controllers/public/__tests__/requestController.test.ts

@@ -60,6 +60,7 @@ function makeHeliconeRequest(overrides: Partial<HeliconeRequest>): HeliconeReque
     prompt_cache_write_tokens: 0,
     prompt_cache_read_tokens: 0,
     completion_tokens: 0,
+    reasoning_tokens: 0,
     prompt_audio_tokens: 0,
     completion_audio_tokens: 0,
     cost: 0,

valhalla/jawn/src/lib/db/ClickhouseWrapper.ts

@@ -307,6 +307,7 @@ export interface RequestResponseRMT {
   prompt_cache_read_tokens: number;
   prompt_audio_tokens: number;
   completion_audio_tokens: number;
+  reasoning_tokens: number;
   model: string;
   ai_gateway_body_mapping: string;
   request_id: string;

valhalla/jawn/src/lib/handlers/HandlerContext.ts

@@ -194,6 +194,7 @@ export const toHeliconeRequest = (context: HandlerContext): HeliconeRequest => {
   );
   const promptAudioTokens = getPromptAudioTokens(modelUsage, legacyUsage);
   const completionAudioTokens = legacyUsage.completionAudioTokens ?? null;
+  const reasoningTokens = getReasoningTokens(modelUsage);
 
   return {
     cost: context.costBreakdown?.totalCost ?? legacyUsage.cost ?? null,
@@ -230,6 +231,7 @@ export const toHeliconeRequest = (context: HandlerContext): HeliconeRequest => {
     prompt_cache_read_tokens: isCacheHit ? 0 : promptCacheReadTokens,
     prompt_audio_tokens: isCacheHit ? 0 : promptAudioTokens,
     completion_audio_tokens: isCacheHit ? 0 : completionAudioTokens,
+    reasoning_tokens: isCacheHit ? 0 : reasoningTokens,
 
     /// NOTE: Unfortunately our codebase is running two prompts systems in parallel.
     // This used to track the legacy feature, but its now the new one.
@@ -286,9 +288,9 @@ export function getCompletionTokens(
       (modelUsage.image?.output ?? 0) +
       (modelUsage.video?.output ?? 0) +
       (modelUsage.file?.output ?? 0);
-    const thinking = modelUsage.thinking ?? 0;
-    if (modelUsage.output > 0 || modalityOutput > 0 || thinking > 0) {
-      return modelUsage.output + modalityOutput + thinking;
+    // Note: reasoning/thinking tokens are now tracked separately and NOT included in completion tokens
+    if (modelUsage.output > 0 || modalityOutput > 0) {
+      return modelUsage.output + modalityOutput;
     }
   }
   return legacyUsage.completionTokens ?? null;
@@ -300,9 +302,16 @@ function getTotalTokens(
 ): number | null {
   const promptTokens = getPromptTokens(modelUsage, legacyUsage);
   const completionTokens = getCompletionTokens(modelUsage, legacyUsage);
+  const reasoningTokens = getReasoningTokens(modelUsage);
 
-  if (promptTokens !== null || completionTokens !== null) {
-    return (promptTokens ?? 0) + (completionTokens ?? 0);
+  if (
+    promptTokens !== null ||
+    completionTokens !== null ||
+    reasoningTokens !== null
+  ) {
+    return (
+      (promptTokens ?? 0) + (completionTokens ?? 0) + (reasoningTokens ?? 0)
+    );
   }
   return legacyUsage.totalTokens ?? null;
 }
@@ -357,3 +366,12 @@ export function getCompletionAudioTokens(
   }
   return legacyUsage.completionAudioTokens ?? null;
 }
+
+export function getReasoningTokens(
+  modelUsage: ModelUsage | undefined
+): number | null {
+  if (modelUsage?.thinking !== undefined && modelUsage.thinking > 0) {
+    return modelUsage.thinking;
+  }
+  return null;
+}