diff --git a/.changeset/cached-input-tokens.md b/.changeset/cached-input-tokens.md new file mode 100644 index 0000000..8e3138d --- /dev/null +++ b/.changeset/cached-input-tokens.md @@ -0,0 +1,5 @@ +--- +"@upstash/box": minor +--- + +Expose cached input tokens in run usage. `run.cost` now includes `cachedInputTokens` (prompt-cache hits, billed at a discounted rate), the streaming `finish` chunk's `usage` carries `cachedInputTokens`, and `RunMetadata`/`BoxRunData` include the backend's `cached_input_tokens` field. diff --git a/packages/sdk/src/__tests__/box-agent-run.test.ts b/packages/sdk/src/__tests__/box-agent-run.test.ts index 6e96d84..4b6eb05 100644 --- a/packages/sdk/src/__tests__/box-agent-run.test.ts +++ b/packages/sdk/src/__tests__/box-agent-run.test.ts @@ -58,6 +58,30 @@ describe("box.agent.run", () => { expect(run.cost.totalUsd).toBe(0); }); + it("populates run.cost.cachedInputTokens from done event, defaulting to 0 when absent", async () => { + const { box, fetchMock } = await createTestBox(); + + fetchMock.mockResolvedValueOnce( + mockSSEResponse([ + { event: "run_start", data: { run_id: "r1" } }, + { event: "done", data: { input_tokens: 100, output_tokens: 50, cached_input_tokens: 80 } }, + ]), + ); + + const run = await box.agent.run({ prompt: "test" }); + expect(run.cost.cachedInputTokens).toBe(80); + + fetchMock.mockResolvedValueOnce( + mockSSEResponse([ + { event: "run_start", data: { run_id: "r2" } }, + { event: "done", data: { input_tokens: 10, output_tokens: 5 } }, + ]), + ); + + const run2 = await box.agent.run({ prompt: "test" }); + expect(run2.cost.cachedInputTokens).toBe(0); + }); + it("calls onToolUse callback", async () => { const { box, fetchMock } = await createTestBox(); const tools: Array<{ toolCallId?: string; name: string; input: Record }> = []; @@ -487,6 +511,29 @@ describe("box.agent.stream", () => { expect(run.result).toBe("Hello world"); }); + it("includes cachedInputTokens in the finish chunk usage", async () => { + const { box, fetchMock } = await createTestBox(); + + fetchMock.mockResolvedValueOnce( + mockSSEResponse([ + { event: "run_start", data: { run_id: "r1" } }, + { event: "text", data: { text: "hi" } }, + { event: "done", data: { input_tokens: 100, output_tokens: 50, cached_input_tokens: 80 } }, + ]), + ); + + const run = await box.agent.stream({ prompt: "test" }); + const chunks: Chunk[] = []; + for await (const chunk of run) { + chunks.push(chunk); + } + const finish = chunks.find( + (c): c is Extract => c.type === "finish", + ); + expect(finish?.usage).toEqual({ inputTokens: 100, outputTokens: 50, cachedInputTokens: 80 }); + expect(run.cost.cachedInputTokens).toBe(80); + }); + it("yields tool-call chunks and calls onToolUse", async () => { const { box, fetchMock } = await createTestBox(); const tools: Array<{ toolCallId?: string; name: string; input: Record }> = []; diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 44bffdc..e1f2920 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -196,6 +196,7 @@ export class Run { private _exitCode: number | null = null; private _inputTokens = 0; private _outputTokens = 0; + private _cachedInputTokens = 0; private _totalUsd = 0; private _computeMs = 0; private _box: Box; @@ -246,6 +247,7 @@ export class Run { return { inputTokens: this._inputTokens, outputTokens: this._outputTokens, + cachedInputTokens: this._cachedInputTokens, computeMs: this._computeMs || Date.now() - this._startTime, totalUsd: this._totalUsd, }; @@ -288,6 +290,7 @@ export class Run { exitCode?: number; inputTokens?: number; outputTokens?: number; + cachedInputTokens?: number; totalUsd?: number; computeMs?: number; abortController?: AbortController; @@ -299,6 +302,7 @@ export class Run { if (data.exitCode !== undefined) run._exitCode = data.exitCode; if (data.inputTokens !== undefined) run._inputTokens = data.inputTokens; if (data.outputTokens !== undefined) run._outputTokens = data.outputTokens; + if (data.cachedInputTokens !== undefined) run._cachedInputTokens = data.cachedInputTokens; if (data.totalUsd !== undefined) run._totalUsd = data.totalUsd; if (data.computeMs !== undefined) run._computeMs = data.computeMs; if (data.abortController !== undefined) run._abortController = data.abortController; @@ -1064,6 +1068,7 @@ export class Box { Run._update(run, { inputTokens: parsed.input_tokens ?? 0, outputTokens: parsed.output_tokens ?? 0, + cachedInputTokens: parsed.cached_input_tokens ?? 0, totalUsd: parsed.total_cost_usd ?? 0, }); if (parsed.output) rawOutput = parsed.output; @@ -1241,6 +1246,7 @@ export class Box { Run._update(run, { inputTokens: parsed.input_tokens ?? 0, outputTokens: parsed.output_tokens ?? 0, + cachedInputTokens: parsed.cached_input_tokens ?? 0, totalUsd: parsed.total_cost_usd ?? 0, }); if (parsed.output) rawOutput = parsed.output; @@ -1250,6 +1256,7 @@ export class Box { usage: { inputTokens: parsed.input_tokens ?? 0, outputTokens: parsed.output_tokens ?? 0, + cachedInputTokens: parsed.cached_input_tokens ?? 0, }, sessionId: parsed.session_id ?? "", }; diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 7741c9e..cdfadf8 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -557,6 +557,7 @@ export type Chunk = usage: { inputTokens: number; outputTokens: number; + cachedInputTokens: number; }; sessionId: string; } @@ -634,6 +635,8 @@ export interface RunCost { inputTokens: number; /** Output tokens consumed */ outputTokens: number; + /** Input tokens served from the provider's prompt cache (subset of inputTokens, billed at a discounted rate) */ + cachedInputTokens: number; /** Wall-clock execution time in milliseconds */ computeMs: number; /** Total cost in USD */ @@ -801,6 +804,7 @@ export interface RunResult { export interface RunMetadata { input_tokens?: number; output_tokens?: number; + cached_input_tokens?: number; } export interface ExecResult { @@ -885,6 +889,7 @@ export type BoxRunData = { output?: string; input_tokens: number; output_tokens: number; + cached_input_tokens?: number; cost_usd: number; duration_ms: number; cpu_ns?: number;