feat: [ENG-2928] Tracking usage for new model registry (#4676)

* new usage processor for new model registry

* cleaner type

* type get usage processor

* fix

* update snapshots and add usage processing tests

* cost calculation functions with tests

* remoe comments

* fix test

* store and pull attempt

* nits

* nit comment

* rename

* throw on failed usage parsing on successful gateway attempt

* indent

* safer timeout on reading raw response

* rm comment

* refactor to only pull response body once

Author: connortbot

packages/__tests__/cost/__snapshots__/usageProcessor.test.ts.snap

@@ -0,0 +1,23 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`OpenAIUsageProcessor usage processing snapshot 1`] = `
+{
+  "cached-response": {
+    "data": {
+      "cacheDetails": {
+        "cachedInput": 1152,
+      },
+      "input": 96,
+      "output": 10,
+    },
+    "error": null,
+  },
+  "stream-response": {
+    "data": {
+      "input": 1248,
+      "output": 10,
+    },
+    "error": null,
+  },
+}
+`;

packages/__tests__/cost/modelCostFromRegistry.test.ts

@@ -0,0 +1,176 @@
+import type { ModelUsage } from "../../cost/usage/types";
+import type { ModelProviderName } from "../../cost/models/providers";
+import { modelCostBreakdownFromRegistry } from "../../cost/costCalc";
+
+describe("modelCostBreakdownFromRegistry", () => {
+  it("should calculate cost for basic GPT-4o usage", () => {
+    const modelUsage: ModelUsage = {
+      input: 1000,
+      output: 500,
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "gpt-4o",
+      provider: "openai" as ModelProviderName,
+    });
+
+    expect(breakdown).not.toBeNull();
+    if (breakdown) {
+      // GPT-4o pricing: $0.0025 per 1K input, $0.01 per 1K output
+      // Expected: 1000 * 0.0025/1000 + 500 * 0.01/1000
+      // = 0.0025 + 0.005 = 0.0075
+      expect(breakdown.totalCost).toBe(0.0075);
+    }
+  });
+
+  it("should calculate cost for Claude with cache", () => {
+    const modelUsage: ModelUsage = {
+      input: 1500,
+      output: 1000,
+      cacheDetails: {
+        cachedInput: 500,
+        write5m: 100,
+        write1h: 50,
+      },
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "claude-3.5-sonnet-v2",
+      provider: "anthropic" as ModelProviderName,
+    });
+
+    expect(breakdown).not.toBeNull();
+    if (breakdown) {
+      // Claude pricing: $0.003 per 1K input, $0.015 per 1K output
+      // Cache multipliers: cachedInput: 0.1, write5m: 1.25, write1h: 2.0
+      // Expected calculation:
+      // - Regular input: 1500 * 0.003/1000 = 0.0045
+      // - Cached input: 500 * 0.003/1000 * 0.1 = 0.00015
+      // - Cache write 5m: 100 * 0.003/1000 * 1.25 = 0.000375
+      // - Cache write 1h: 50 * 0.003/1000 * 2.0 = 0.0003
+      // - Output: 1000 * 0.015/1000 = 0.015
+      // Total: 0.0045 + 0.00015 + 0.000375 + 0.0003 + 0.015 = 0.020325
+      expect(breakdown.totalCost).toBeCloseTo(0.020325, 10);
+    }
+  });
+
+  it("should return null for non-existent model", () => {
+    const modelUsage: ModelUsage = {
+      input: 100,
+      output: 50,
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "non-existent-model",
+      provider: "unknown" as ModelProviderName,
+    });
+
+    expect(breakdown).toBeNull();
+  });
+
+  it("should return 0 for empty usage", () => {
+    const modelUsage: ModelUsage = {
+      input: 0,
+      output: 0,
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "gpt-4o",
+      provider: "openai" as ModelProviderName,
+    });
+
+    expect(breakdown).not.toBeNull();
+    if (breakdown) {
+      expect(breakdown.totalCost).toBe(0);
+    }
+  });
+
+  it("should calculate cost breakdown correctly", () => {
+    const modelUsage: ModelUsage = {
+      input: 800,
+      output: 500,
+      cacheDetails: {
+        cachedInput: 200,
+      },
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "gpt-4o",
+      provider: "openai" as ModelProviderName,
+    });
+
+    expect(breakdown).not.toBeNull();
+    if (breakdown) {
+      // GPT-4o pricing: $0.0025 per 1K input, $0.01 per 1K output
+      // Cache multiplier for cached input: 0.5
+      expect(breakdown.inputCost).toBe(800 * 0.0025 / 1000);
+      expect(breakdown.cachedInputCost).toBe(200 * 0.0025 / 1000 * 0.5);
+      expect(breakdown.outputCost).toBe(500 * 0.01 / 1000);
+      expect(breakdown.totalCost).toBe(breakdown.inputCost + breakdown.cachedInputCost + breakdown.outputCost);
+    }
+  });
+
+  it("should handle audio tokens for Gemini", () => {
+    const modelUsage: ModelUsage = {
+      input: 1000,
+      output: 500,
+      audio: 200,
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "gemini-2.5-flash",
+      provider: "google-ai-studio" as ModelProviderName,
+    });
+
+    expect(breakdown).not.toBeNull();
+    if (breakdown) {
+      expect(breakdown.audioCost).toBeGreaterThan(0);
+      expect(breakdown.totalCost).toBeGreaterThan(0);
+    }
+  });
+
+  it("should handle web search for Grok", () => {
+    const modelUsage: ModelUsage = {
+      input: 1000,
+      output: 500,
+      web_search: 5,
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "grok-3",
+      provider: "xai" as ModelProviderName,
+    });
+
+    expect(breakdown).not.toBeNull();
+    if (breakdown) {
+      expect(breakdown.webSearchCost).toBe(5 * 0.025);
+    }
+  });
+
+  it("should handle images for Gemini", () => {
+    const modelUsage: ModelUsage = {
+      input: 500,
+      output: 200,
+      image: 3,
+    };
+
+    const breakdown = modelCostBreakdownFromRegistry({
+      modelUsage,
+      model: "gemini-2.5-flash",
+      provider: "vertex" as ModelProviderName,
+    });
+
+    expect(breakdown).not.toBeNull();
+    if (breakdown) {
+      // Gemini image price: $0.001238 per image
+      expect(breakdown.imageCost).toBe(3 * 0.001238);
+    }
+  });
+});

packages/__tests__/cost/testData/gpt4o-response-cached.snapshot

@@ -0,0 +1,37 @@
+{
+  "id": "chatcmpl-CDYL3CqkVL6FBFfhB2tTsnMgwShUR",
+  "object": "chat.completion",
+  "created": 1757346297,
+  "model": "gpt-4o-2024-08-06",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Hello! How can I assist you today?",
+        "refusal": null,
+        "annotations": []
+      },
+      "logprobs": null,
+      "finish_reason": "stop"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 1248,
+    "completion_tokens": 10,
+    "total_tokens": 1258,
+    "prompt_tokens_details": {
+      "cached_tokens": 1152,
+      "audio_tokens": 0
+    },
+    "completion_tokens_details": {
+      "reasoning_tokens": 0,
+      "audio_tokens": 0,
+      "accepted_prediction_tokens": 0,
+      "rejected_prediction_tokens": 0
+    }
+  },
+  "service_tier": "default",
+  "system_fingerprint": "fp_f33640a400"
+}
+  

packages/__tests__/cost/testData/gpt4o-stream-response.snapshot

@@ -0,0 +1,25 @@
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mOg9lfxO2hD6ra"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"or0Gvzs1Kvx"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"M0FpDo6EbYZ5Orf"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"FBWYXZkuD1dS"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"58OQJ4RLM1cD"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ROvCbFebzDtaxg"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mS05oYfyl"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"CCV0ABejZdq1"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"VWsCuMl6YH"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"KSh4A88SQR5QntO"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"BOeJlkS3pX"}
+
+data: {"id":"chatcmpl-CDYHKJaUtey5GfGKB4TquZ09VuER2","object":"chat.completion.chunk","created":1757346066,"model":"gpt-4o-2024-08-06","service_tier":"default","system_fingerprint":"fp_f33640a400","choices":[],"usage":{"prompt_tokens":1248,"completion_tokens":10,"total_tokens":1258,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"UJyZNZ13R3y"}
+
+data: [DONE]

packages/__tests__/cost/usageProcessor.test.ts

@@ -0,0 +1,88 @@
+import { describe, it, expect } from "@jest/globals";
+import { OpenAIUsageProcessor } from "@helicone-package/cost/usage/openAIUsageProcessor";
+import { getUsageProcessor } from "@helicone-package/cost/usage/getUsageProcessor";
+import * as fs from "fs";
+import * as path from "path";
+
+describe("getUsageProcessor", () => {
+  it("should return OpenAIUsageProcessor for openai provider", () => {
+    const processor = getUsageProcessor("openai");
+    expect(processor).toBeInstanceOf(OpenAIUsageProcessor);
+  });
+
+  it("should throw error for unsupported provider", () => {
+    expect(() => {
+      getUsageProcessor("unsupported-provider" as any);
+    }).toThrow("Usage processor not found for provider: unsupported-provider");
+  });
+});
+
+describe("OpenAIUsageProcessor", () => {
+  const processor = new OpenAIUsageProcessor();
+
+  it("should parse real GPT-4o response with cached tokens", async () => {
+    const responseData = fs.readFileSync(
+      path.join(__dirname, "testData", "gpt4o-response-cached.txt"),
+      "utf-8"
+    );
+
+    const result = await processor.parse({ 
+      responseBody: responseData, 
+      isStream: false 
+    });
+
+    expect(result.error).toBeNull();
+    expect(result.data).toEqual({
+      input: 96,
+      output: 10,
+      cacheDetails: {
+        cachedInput: 1152
+      }
+    });
+  });
+
+  it("should parse real GPT-4o stream response", async () => {
+    const streamData = fs.readFileSync(
+      path.join(__dirname, "testData", "gpt4o-stream-response.txt"),
+      "utf-8"
+    );
+
+    const result = await processor.parse({ 
+      responseBody: streamData, 
+      isStream: true 
+    });
+
+    expect(result.error).toBeNull();
+    expect(result.data).toEqual({
+      input: 1248,
+      output: 10
+    });
+  });
+
+  it("usage processing snapshot", async () => {
+    const testCases = [
+      {
+        name: "cached-response",
+        data: fs.readFileSync(path.join(__dirname, "testData", "gpt4o-response-cached.txt"), "utf-8"),
+        isStream: false
+      },
+      {
+        name: "stream-response", 
+        data: fs.readFileSync(path.join(__dirname, "testData", "gpt4o-stream-response.txt"), "utf-8"),
+        isStream: true
+      }
+    ];
+
+    const results: Record<string, any> = {};
+    
+    for (const testCase of testCases) {
+      const result = await processor.parse({
+        responseBody: testCase.data,
+        isStream: testCase.isStream
+      });
+      results[testCase.name] = result;
+    }
+
+    expect(results).toMatchSnapshot();
+  });
+}); 

packages/cost/costCalc.ts

@@ -1,7 +1,17 @@
 import { costOfPrompt } from "./index";
+import type { ModelUsage } from "./usage/types";
+import type { ModelProviderName } from "./models/providers";
+import { calculateModelCostBreakdown, CostBreakdown } from "./models/calculate-cost";
 
+// since costs in clickhouse are multiplied by the multiplier
+// divide to get real cost in USD in dollars
 export const COST_PRECISION_MULTIPLIER = 1_000_000_000;
 
+/**
+ * LEGACY: Calculate model cost using the old cost registry format
+ * This function uses the legacy cost registry in /providers/mappings
+ * @deprecated Use modelCostFromRegistry for new implementations
+ */
 export function modelCost(
   params: {
     provider: string;
@@ -37,3 +47,19 @@ export function modelCost(
     }) ?? 0
   );
 }
+
+export function modelCostBreakdownFromRegistry(params: {
+  modelUsage: ModelUsage;
+  provider: ModelProviderName;
+  model: string;
+  requestCount?: number;
+}): CostBreakdown | null {
+  const breakdown = calculateModelCostBreakdown({
+    modelUsage: params.modelUsage,
+    model: params.model,
+    provider: params.provider,
+    requestCount: params.requestCount,
+  });
+  
+  return breakdown;
+}