Normalize Gemini token usage for cost tracking (#516)

## Summary

- Normalize Gemini CLI token usage into the same keys used by usage cost
aggregation.
- Include Gemini thoughts in output token usage and cached tokens in
cache-read usage.
- Add a regression test covering the normalized Gemini token usage
shape.

Fixes #515

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>

Author: nergal-perm

internal/db/db.go

@@ -27,9 +27,16 @@ import (
 // trigger a non-destructive re-sync (mtime reset + skip cache
 // clear) so existing session data is preserved.
 //
-// Bumped to 27: Piebald parser now persists normalized per-message
+// Bumped to 28: Gemini parser now persists normalized
+// (Anthropic-style) per-message token_usage JSON instead of the raw
+// tokens object, and rolls thoughts tokens into OutputTokens so
+// per-message and session output totals match the cost JSON.
+// Existing Gemini rows need re-parsing so usage and cost reports
+// reflect the new shape and include thoughts tokens.
+//
+// (27: Piebald parser now persists normalized per-message
 // token_usage JSON. Existing Piebald rows need re-parsing so Usage
-// reports can include older Piebald sessions.
+// reports can include older Piebald sessions.)
 //
 // (26: Claude parser now (a) links Task / Agent tool
 // calls to child subagent sessions via toolUseResult.agentId
@@ -81,7 +88,7 @@ import (
 //
 // (17: Codex <skill> template filtering.)
 // (16: <turn_aborted> system messages.)
-const dataVersion = 27
+const dataVersion = 28
 
 const tokenCoverageRepairStatsKey = "token_coverage_repair_v1"
 

internal/parser/gemini.go

@@ -15,9 +15,10 @@ import (
 
 // geminiTokens holds token usage counts from a Gemini message.
 type geminiTokens struct {
-	Input  int
-	Output int
-	Cached int
+	Input    int
+	Output   int
+	Cached   int
+	Thoughts int
 }
 
 // extractGeminiTokens reads the tokens object from a Gemini
@@ -28,12 +29,30 @@ func extractGeminiTokens(msg gjson.Result) geminiTokens {
 		return geminiTokens{}
 	}
 	return geminiTokens{
-		Input:  int(tok.Get("input").Int()),
-		Output: int(tok.Get("output").Int()),
-		Cached: int(tok.Get("cached").Int()),
+		Input:    int(tok.Get("input").Int()),
+		Output:   int(tok.Get("output").Int()),
+		Cached:   int(tok.Get("cached").Int()),
+		Thoughts: int(tok.Get("thoughts").Int()),
 	}
 }
 
+// normalizedGeminiTokenUsage maps Gemini's token counts onto the
+// Anthropic-style shape used by usage and cost queries. Thoughts
+// tokens are billed at the output rate, so they fold into
+// output_tokens here.
+func normalizedGeminiTokenUsage(tok geminiTokens) json.RawMessage {
+	payload := map[string]int{
+		"input_tokens":            tok.Input,
+		"output_tokens":           tok.Output + tok.Thoughts,
+		"cache_read_input_tokens": tok.Cached,
+	}
+	raw, err := json.Marshal(payload)
+	if err != nil {
+		return nil
+	}
+	return raw
+}
+
 // ParseGeminiSession parses a Gemini CLI session JSON file.
 // Unlike Claude/Codex JSONL, each Gemini file is a single JSON
 // document containing all messages.
@@ -233,7 +252,7 @@ func parseGeminiMessage(
 	var tokenUsage json.RawMessage
 	tokResult := msg.Get("tokens")
 	if tokResult.Exists() {
-		tokenUsage = json.RawMessage(tokResult.Raw)
+		tokenUsage = normalizedGeminiTokenUsage(tok)
 	}
 	return ParsedMessage{
 		Ordinal:       ordinal,
@@ -248,10 +267,11 @@ func parseGeminiMessage(
 		Model:         msg.Get("model").String(),
 		TokenUsage:    tokenUsage,
 		ContextTokens: tok.Input + tok.Cached,
-		OutputTokens:  tok.Output,
+		OutputTokens:  tok.Output + tok.Thoughts,
 		HasContextTokens: tokResult.Get("input").Exists() ||
 			tokResult.Get("cached").Exists(),
-		HasOutputTokens:    tokResult.Get("output").Exists(),
+		HasOutputTokens: tokResult.Get("output").Exists() ||
+			tokResult.Get("thoughts").Exists(),
 		tokenPresenceKnown: true,
 	}, true
 }

internal/parser/gemini_parser_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/tidwall/gjson"
 	"github.com/wesm/agentsview/internal/testjsonl"
 )
 
@@ -279,9 +280,10 @@ func TestParseGeminiSession_TokenUsage(t *testing.T) {
 		assert.False(t, msgs[0].HasOutputTokens)
 		assert.Empty(t, msgs[0].TokenUsage)
 
-		// First assistant message (a1): input=1500, cached=100, output=200
+		// First assistant message (a1): input=1500, cached=100,
+		// output=200, thoughts=50
 		assert.Equal(t, 1600, msgs[1].ContextTokens)
-		assert.Equal(t, 200, msgs[1].OutputTokens)
+		assert.Equal(t, 250, msgs[1].OutputTokens)
 		assert.True(t, msgs[1].HasContextTokens)
 		assert.True(t, msgs[1].HasOutputTokens)
 		assert.NotEmpty(t, msgs[1].TokenUsage)
@@ -292,15 +294,16 @@ func TestParseGeminiSession_TokenUsage(t *testing.T) {
 		assert.False(t, msgs[2].HasContextTokens)
 		assert.False(t, msgs[2].HasOutputTokens)
 
-		// Second assistant message (a2): input=2000, cached=50, output=300
+		// Second assistant message (a2): input=2000, cached=50,
+		// output=300, thoughts=100
 		assert.Equal(t, 2050, msgs[3].ContextTokens)
-		assert.Equal(t, 300, msgs[3].OutputTokens)
+		assert.Equal(t, 400, msgs[3].OutputTokens)
 		assert.True(t, msgs[3].HasContextTokens)
 		assert.True(t, msgs[3].HasOutputTokens)
 		assert.NotEmpty(t, msgs[3].TokenUsage)
 
 		// Session totals
-		assert.Equal(t, 500, sess.TotalOutputTokens)
+		assert.Equal(t, 650, sess.TotalOutputTokens)
 		assert.Equal(t, 2050, sess.PeakContextTokens)
 		assert.True(t, sess.HasTotalOutputTokens)
 		assert.True(t, sess.HasPeakContextTokens)
@@ -355,11 +358,18 @@ func TestParseGeminiSession_TokenUsage(t *testing.T) {
 
 		require.Equal(t, 2, len(msgs))
 		assert.Equal(t, 5200, msgs[1].ContextTokens)
-		assert.Equal(t, 800, msgs[1].OutputTokens)
+		assert.Equal(t, 900, msgs[1].OutputTokens)
 		assert.True(t, msgs[1].HasContextTokens)
 		assert.True(t, msgs[1].HasOutputTokens)
 		assert.NotEmpty(t, msgs[1].TokenUsage)
-		assert.Equal(t, 800, sess.TotalOutputTokens)
+		assert.Equal(t, int64(5000),
+			gjson.GetBytes(msgs[1].TokenUsage, "input_tokens").Int())
+		assert.Equal(t, int64(900),
+			gjson.GetBytes(msgs[1].TokenUsage, "output_tokens").Int())
+		assert.Equal(t, int64(200),
+			gjson.GetBytes(msgs[1].TokenUsage,
+				"cache_read_input_tokens").Int())
+		assert.Equal(t, 900, sess.TotalOutputTokens)
 		assert.Equal(t, 5200, sess.PeakContextTokens)
 		assert.True(t, sess.HasTotalOutputTokens)
 		assert.True(t, sess.HasPeakContextTokens)