Skip to content

Commit 1af0b9e

Browse files
authored
Merge pull request #17 from comet-ml/jacques/cc-context-attribution-fixes
Expand context attribution: MCP, /context fetch, per-span snapshots
2 parents f63cfdc + e66cad3 commit 1af0b9e

26 files changed

Lines changed: 2878 additions & 122 deletions

bin/opik-logger-darwin-amd64

1.32 MB
Binary file not shown.

bin/opik-logger-darwin-arm64

974 KB
Binary file not shown.

bin/opik-logger-linux-amd64

1.5 MB
Binary file not shown.

bin/opik-logger-windows-amd64.exe

1.57 MB
Binary file not shown.

src/attribution.go

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,28 @@ func ExtractAttribution(entries []TranscriptEntry) *Attribution {
2222
func BuildSkillsSnapshot(allEntries []TranscriptEntry) map[string]interface{} {
2323
listing := extractSkillEvents(allEntries)
2424
loaded := extractLoadedSkills(allEntries)
25-
menuTokens := extractSkillMenuTokens(allEntries)
26-
if len(listing) == 0 && len(loaded) == 0 && menuTokens == 0 {
25+
attachmentTokens := extractSkillMenuTokens(allEntries)
26+
if len(listing) == 0 && len(loaded) == 0 && attachmentTokens == 0 {
2727
return nil
2828
}
2929

30+
// menu_tokens is the always-on cost of the skill menu: summed per skill
31+
// from each skill's name + frontmatter description (what /context's
32+
// /skills view attributes per row). Bundled (in-binary) skills can't be
33+
// read from disk, so they contribute 0 — bundled_count surfaces how much
34+
// of /context's number we structurally cannot reach.
3035
available := make([]map[string]interface{}, 0, len(listing))
36+
menuTokens := 0
37+
bundledCount := 0
3138
for _, s := range listing {
39+
menuTokens += s.MenuTokens
40+
if s.Source == "bundled" {
41+
bundledCount++
42+
}
3243
e := map[string]interface{}{
33-
"name": s.Name,
34-
"source": s.Source,
44+
"name": s.Name,
45+
"source": s.Source,
46+
"menu_tokens": s.MenuTokens,
3547
}
3648
if s.SHA256 != "" {
3749
e["sha256"] = s.SHA256
@@ -56,6 +68,10 @@ func BuildSkillsSnapshot(allEntries []TranscriptEntry) map[string]interface{} {
5668
if s.Path != "" {
5769
e["path"] = s.Path
5870
}
71+
if s.CatalogBodyTokens > 0 {
72+
e["catalog_body_tokens"] = s.CatalogBodyTokens
73+
e["catalog_source"] = s.CatalogSource
74+
}
5975
loadedOut = append(loadedOut, e)
6076
}
6177

@@ -66,6 +82,12 @@ func BuildSkillsSnapshot(allEntries []TranscriptEntry) map[string]interface{} {
6682
"loaded_tokens": loadedTokens,
6783
"available_count": len(available),
6884
"loaded_count": len(loadedOut),
85+
"bundled_count": bundledCount,
86+
// The token estimate of the raw skill_listing attachment in the
87+
// transcript, kept for cross-checking. It differs from menu_tokens:
88+
// the attachment is a partial artifact, whereas menu_tokens is
89+
// reconstructed per skill from on-disk frontmatter.
90+
"menu_tokens_attachment": attachmentTokens,
6991
},
7092
"available": available,
7193
"loaded": loadedOut,
@@ -91,9 +113,20 @@ func tokEstimate(s string) int {
91113
// "json" → 2.8 (tool_use input, MCP payload)
92114
// "deferred_tools_payload" → 2.5 (pure JSON list of names)
93115
// "tool_result" → 3.0 (mixed text + JSON-ish output)
116+
// "skill_listing_menu" → 3.0 (per-skill `- name: description` block —
117+
// derived from /context's published
118+
// per-skill tokens against the actual
119+
// attachment text: 6368 chars / 2116 tokens)
120+
// "agent_frontmatter" → 3.1 (YAML name/desc + example blocks; verified
121+
// vs /context's "Custom agents" rows)
94122
// "skill_body" → 3.5 (markdown w/ code blocks)
123+
// "memory_file" → 2.4 (.claude/rules/**/*.md + auto-mem MEMORY.md;
124+
// derived from /context's "Memory files"
125+
// rows — denser than skill bodies because
126+
// these files lean on brackets/dashes/code
127+
// conventions that tokenize as separate
128+
// short tokens)
95129
// "assistant_text" → 3.9 (prose with occasional code)
96-
// "skill_listing_menu" → 3.9 (name + short description lines)
97130
// "prose" → 3.9
98131
// "user_prompt" → 4.3 (natural English from a user)
99132
// "" / unknown → 3.6 (overall calibrated median)
@@ -130,11 +163,15 @@ func charsPerToken(s, contentType string) float64 {
130163
return 2.8
131164
case "deferred_tools_payload":
132165
return 2.5
133-
case "tool_result":
166+
case "memory_file":
167+
return 2.4
168+
case "tool_result", "skill_listing_menu":
134169
return 3.0
170+
case "agent_frontmatter":
171+
return 3.1
135172
case "skill_body":
136173
return 3.5
137-
case "assistant_text", "prose", "skill_listing_menu":
174+
case "assistant_text", "prose":
138175
return 3.9
139176
case "user_prompt":
140177
return 4.3

src/cc_builtin.go

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
package main
2+
3+
import (
4+
"strconv"
5+
"strings"
6+
)
7+
8+
// ccBuiltinConstants holds the always-on costs Claude Code charges every
9+
// turn that we structurally cannot read from the transcript:
10+
//
11+
// - SystemPromptTokens the bundled default system prompt
12+
// - SystemToolsTokens full JSON schemas for the default tools
13+
// (Read, Edit, Bash, …) — not the names alone
14+
// - SystemToolsDeferredTokens the catalog of deferred tool definitions
15+
// (Cron*, Task*, Web*, Monitor, …) plus any
16+
// schemas Claude Code injects on demand
17+
//
18+
// These are taken from `/context` for a known CC version. They drift with
19+
// each binary release, so the table below should grow over time.
20+
type ccBuiltinConstants struct {
21+
SystemPromptTokens int
22+
SystemToolsTokens int
23+
SystemToolsDeferredTokens int
24+
}
25+
26+
// ccBuiltinByVersion is a small versioned table — keys are exact CC
27+
// CLI versions (the `version` field stamped on every transcript entry).
28+
// When the binary changes its bundled prompt or tool catalog, /context's
29+
// numbers shift; add an entry here with the new ones.
30+
//
31+
// Source: `claude -p "/context"` in a project with NO MCPs connected, on
32+
// the listed CC version + Opus model. The two figures together cover
33+
// >65% of /context's accounted-for tokens.
34+
var ccBuiltinByVersion = map[string]ccBuiltinConstants{
35+
"2.1.150": {
36+
SystemPromptTokens: 8000,
37+
SystemToolsTokens: 17600,
38+
SystemToolsDeferredTokens: 19200,
39+
},
40+
}
41+
42+
// ccBuiltinFor returns the constants for the given CC version. Falls
43+
// through to the highest known version (sorted lexically — fine while
44+
// the table is small) when an exact match isn't found; returns the zero
45+
// value when the table is empty. Callers should check `version != ""` on
46+
// the returned snapshot to distinguish "estimated" from "unknown".
47+
func ccBuiltinFor(version string) (ccBuiltinConstants, string) {
48+
if c, ok := ccBuiltinByVersion[version]; ok {
49+
return c, version
50+
}
51+
// Patch-version fallback: 2.1.151 → use the closest known 2.1.* row.
52+
// Major.Minor match is a reasonable proxy because the system prompt
53+
// and tool catalog rarely change inside a minor.
54+
if best, key := closestKnownVersion(version); key != "" {
55+
return best, key
56+
}
57+
return ccBuiltinConstants{}, ""
58+
}
59+
60+
// closestKnownVersion returns the highest known version that shares the
61+
// same major.minor as want. Returns ("", "") if no such version exists.
62+
func closestKnownVersion(want string) (ccBuiltinConstants, string) {
63+
if want == "" {
64+
return ccBuiltinConstants{}, ""
65+
}
66+
wantMaj, wantMin, _ := splitSemver(want)
67+
if wantMaj < 0 {
68+
return ccBuiltinConstants{}, ""
69+
}
70+
bestKey := ""
71+
var bestVal ccBuiltinConstants
72+
bestPatch := -1
73+
for k, v := range ccBuiltinByVersion {
74+
maj, min, patch := splitSemver(k)
75+
if maj != wantMaj || min != wantMin {
76+
continue
77+
}
78+
if patch > bestPatch {
79+
bestPatch = patch
80+
bestKey = k
81+
bestVal = v
82+
}
83+
}
84+
return bestVal, bestKey
85+
}
86+
87+
func splitSemver(v string) (maj, min, patch int) {
88+
maj, min, patch = -1, -1, -1
89+
parts := strings.SplitN(v, ".", 3)
90+
if len(parts) < 2 {
91+
return
92+
}
93+
var err error
94+
if maj, err = strconv.Atoi(parts[0]); err != nil {
95+
maj = -1
96+
return
97+
}
98+
if min, err = strconv.Atoi(parts[1]); err != nil {
99+
min = -1
100+
return
101+
}
102+
if len(parts) == 3 {
103+
patch, _ = strconv.Atoi(parts[2])
104+
}
105+
return
106+
}
107+
108+
// findCCVersion returns the first non-empty `version` field stamped on
109+
// any transcript entry. Claude Code puts the CLI version on every user
110+
// and assistant entry, so any pass-through tells us the binary that wrote
111+
// the session.
112+
func findCCVersion(entries []TranscriptEntry) string {
113+
for _, e := range entries {
114+
if e.Version != "" {
115+
return e.Version
116+
}
117+
}
118+
return ""
119+
}
120+
121+
// extractCCBuiltinSnapshot returns the `cc.cc_builtin` block —
122+
// approximated, version-keyed costs for the bundled system prompt and
123+
// tool catalog. Marked `estimated: true` so dashboards can distinguish
124+
// these from transcript-derived numbers. Returns nil when the version
125+
// is unknown (better to under-report than ship made-up numbers).
126+
//
127+
// total_tokens splits the same way /context does: always_on is what
128+
// ships in the request envelope every turn (and bills against
129+
// input_tokens / cache_*); deferred_tools is what WOULD cost if loaded
130+
// via ToolSearch, but normally doesn't ship at all. Summing total +
131+
// deferred would over-count by the deferred bucket — same trap we
132+
// addressed in buildContextSnapshot.
133+
func extractCCBuiltinSnapshot(entries []TranscriptEntry) map[string]interface{} {
134+
version := findCCVersion(entries)
135+
consts, matched := ccBuiltinFor(version)
136+
if matched == "" {
137+
return nil
138+
}
139+
alwaysOn := consts.SystemPromptTokens + consts.SystemToolsTokens
140+
return map[string]interface{}{
141+
"summary": map[string]interface{}{
142+
"total_tokens": alwaysOn, // ← matches /context and API billing
143+
"deferred_tokens": consts.SystemToolsDeferredTokens,
144+
"estimated": true,
145+
"cc_version": version,
146+
"matched_table": matched,
147+
"system_prompt": consts.SystemPromptTokens,
148+
"system_tools": consts.SystemToolsTokens,
149+
"deferred_tools": consts.SystemToolsDeferredTokens,
150+
},
151+
}
152+
}

src/cc_builtin_test.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package main
2+
3+
import "testing"
4+
5+
func TestFindCCVersion(t *testing.T) {
6+
entries := []TranscriptEntry{
7+
{Type: "queue-operation"},
8+
{Type: "user", Version: "2.1.150"},
9+
{Type: "assistant", Version: "2.1.150"},
10+
}
11+
if got := findCCVersion(entries); got != "2.1.150" {
12+
t.Errorf("findCCVersion = %q, want 2.1.150", got)
13+
}
14+
if got := findCCVersion(nil); got != "" {
15+
t.Errorf("findCCVersion(nil) = %q, want empty", got)
16+
}
17+
}
18+
19+
func TestCCBuiltinFor(t *testing.T) {
20+
// Exact match.
21+
c, key := ccBuiltinFor("2.1.150")
22+
if key != "2.1.150" || c.SystemPromptTokens == 0 {
23+
t.Errorf("exact: got key=%q tokens=%+v", key, c)
24+
}
25+
// Patch fallback inside the same major.minor.
26+
c, key = ccBuiltinFor("2.1.151")
27+
if key == "" || c.SystemPromptTokens == 0 {
28+
t.Errorf("patch fallback: got key=%q tokens=%+v", key, c)
29+
}
30+
// Unknown major.minor → no fallback.
31+
c, key = ccBuiltinFor("3.0.0")
32+
if key != "" || c.SystemPromptTokens != 0 {
33+
t.Errorf("unknown major: expected zero, got key=%q tokens=%+v", key, c)
34+
}
35+
// Empty version.
36+
c, key = ccBuiltinFor("")
37+
if key != "" {
38+
t.Errorf("empty version: expected miss, got %q", key)
39+
}
40+
}
41+
42+
func TestExtractCCBuiltinSnapshotShape(t *testing.T) {
43+
snap := extractCCBuiltinSnapshot([]TranscriptEntry{{Version: "2.1.150"}})
44+
if snap == nil {
45+
t.Fatal("expected snapshot for known version")
46+
}
47+
sum := snap["summary"].(map[string]interface{})
48+
if est, _ := sum["estimated"].(bool); !est {
49+
t.Error("expected estimated:true")
50+
}
51+
if v, _ := sum["cc_version"].(string); v != "2.1.150" {
52+
t.Errorf("cc_version = %q, want 2.1.150", v)
53+
}
54+
// total_tokens excludes deferred (matches /context's visible total
55+
// and API billing). For 2.1.150: system_prompt + system_tools.
56+
total, _ := sum["total_tokens"].(int)
57+
sp, _ := sum["system_prompt"].(int)
58+
st, _ := sum["system_tools"].(int)
59+
if total != sp+st {
60+
t.Errorf("total_tokens = %d, want system_prompt(%d) + system_tools(%d) = %d", total, sp, st, sp+st)
61+
}
62+
deferred, _ := sum["deferred_tokens"].(int)
63+
df, _ := sum["deferred_tools"].(int)
64+
if deferred != df {
65+
t.Errorf("deferred_tokens = %d, want deferred_tools = %d", deferred, df)
66+
}
67+
if extractCCBuiltinSnapshot(nil) != nil {
68+
t.Error("nil entries should yield nil snapshot")
69+
}
70+
if extractCCBuiltinSnapshot([]TranscriptEntry{{Version: "99.99.99"}}) != nil {
71+
t.Error("unknown version should yield nil snapshot")
72+
}
73+
}

0 commit comments

Comments
 (0)