feat(metrics): per-call-site cache hit rate tracking (#60)

Siddhant-K-code · ona-agent · web-flow · commit 7ca5e8a6b74f · 2026-05-02T19:59:16.000+05:30
* feat(metrics): per-call-site cache hit rate tracking (#47) Add CallSiteTracker to pkg/metrics. Records Anthropic API usage per call site and exposes HitRate, WriteEfficiency, RequestHitRate. - Record(callSite, UsageRecord): accumulates token counters and request counts; marks CacheHitRequests when cache_read_input_tokens > 0 - Stats(callSite): snapshot for a single call site - AllStats(): all call sites sorted by hit rate ascending (worst first) - Reset / ResetAll for test isolation - Summary(): human-readable table of call site, hit%, efficiency, reqs Thread-safe via sync.RWMutex. Co-authored-by: Ona <no-reply@ona.com> * docs: document per-call-site hit rate tracking in README (#47) Co-authored-by: Ona <no-reply@ona.com> --------- Co-authored-by: Ona <no-reply@ona.com>
diff --git a/README.md b/README.md
@@ -583,6 +583,30 @@ Record Anthropic API usage with `metrics.RecordCacheUsage(UsageRecord{...})` aft
 | `distill_cache_hit_rate` | Gauge | Rolling hit rate: `cache_read / (cache_read + cache_creation + input)` |
 | `distill_cache_write_efficiency` | Gauge | Reads/writes ratio — values < 1.0 mean cache writes that expire before being read |
 
+**Per-call-site hit rate tracking**
+
+`CallSiteTracker` records Anthropic API usage per call site and surfaces the worst performers first:
+
+```go
+tracker := metrics.NewCallSiteTracker()
+
+// After each Anthropic API call:
+tracker.Record("agent/planner.go:84", metrics.UsageRecord{
+    CacheCreationInputTokens: resp.Usage.CacheCreationInputTokens,
+    CacheReadInputTokens:     resp.Usage.CacheReadInputTokens,
+    InputTokens:              resp.Usage.InputTokens,
+})
+
+// Inspect
+s := tracker.Stats("agent/planner.go:84")
+fmt.Printf("hit rate: %.0f%%  efficiency: %.1fx\n", s.HitRate()*100, s.WriteEfficiency())
+
+// All call sites, worst hit rate first
+for _, s := range tracker.AllStats() {
+    fmt.Printf("%-40s %.0f%%\n", s.CallSite, s.HitRate()*100)
+}
+```
+
 **Cache boundary metrics** (populated by the session boundary manager)
 
 | Metric | Type | Description |
@@ -874,6 +898,7 @@ Distill is evolving from a dedup utility into a context intelligence layer. Here
 | **Memory decay lifecycle events** | [#54](https://github.com/Siddhant-K-code/distill/issues/54) | Shipped | `DecayWorker` emits `EventCompressed` and `EventEvicted` on each transition. `RecallResult` includes a `CacheBoundaryHint` for high-relevance entries. |
 | **Cache-aware dedup** | [#50](https://github.com/Siddhant-K-code/distill/issues/50) | Shipped | `preserve_cache_prefix` option freezes chunks before the last `cache_control` marker so dedup cannot reorder them. Prefix hash and token count reported in stats. |
 | **Prefix stability validator** | [#48](https://github.com/Siddhant-K-code/distill/issues/48) | Shipped | `StabilityValidator` tracks prefix hashes across requests and detects dynamic content (timestamps, request IDs, UUIDs) bleeding into cached prefixes. |
+| **Per-call-site hit rate tracking** | [#47](https://github.com/Siddhant-K-code/distill/issues/47) | Shipped | `CallSiteTracker` records Anthropic cache usage per call site; `AllStats()` returns worst performers first. |
 
 ### Code Intelligence
 
diff --git a/pkg/metrics/callsite.go b/pkg/metrics/callsite.go
@@ -0,0 +1,171 @@
+package metrics
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// CallSiteRecord holds cumulative cache usage for a single call site.
+type CallSiteRecord struct {
+	CallSite string
+
+	// Token counters from Anthropic API usage blocks.
+	CacheCreationTokens int64
+	CacheReadTokens     int64
+	UncachedInputTokens int64
+	OutputTokens        int64
+
+	// Request counts.
+	TotalRequests int64
+	// CacheHitRequests is the number of requests where cache_read_input_tokens > 0.
+	CacheHitRequests int64
+
+	FirstSeen time.Time
+	LastSeen  time.Time
+}
+
+// HitRate returns cache_read / (cache_read + cache_creation + uncached_input).
+// Returns 0 when no tokens have been recorded.
+func (r *CallSiteRecord) HitRate() float64 {
+	total := r.CacheReadTokens + r.CacheCreationTokens + r.UncachedInputTokens
+	if total == 0 {
+		return 0
+	}
+	return float64(r.CacheReadTokens) / float64(total)
+}
+
+// WriteEfficiency returns cache_read / cache_creation.
+// Returns 0 when no cache writes have been recorded.
+func (r *CallSiteRecord) WriteEfficiency() float64 {
+	if r.CacheCreationTokens == 0 {
+		return 0
+	}
+	return float64(r.CacheReadTokens) / float64(r.CacheCreationTokens)
+}
+
+// RequestHitRate returns the fraction of requests that had at least one
+// cache read token (i.e. the cache was warm for that request).
+func (r *CallSiteRecord) RequestHitRate() float64 {
+	if r.TotalRequests == 0 {
+		return 0
+	}
+	return float64(r.CacheHitRequests) / float64(r.TotalRequests)
+}
+
+// CallSiteTracker records per-call-site Anthropic cache usage and exposes
+// aggregated hit rate statistics. It is safe for concurrent use.
+//
+// Typical usage:
+//
+//	tracker := metrics.NewCallSiteTracker()
+//	// after each Anthropic API call:
+//	tracker.Record("agent/planner.go:84", metrics.UsageRecord{
+//	    CacheCreationInputTokens: resp.Usage.CacheCreationInputTokens,
+//	    CacheReadInputTokens:     resp.Usage.CacheReadInputTokens,
+//	    InputTokens:              resp.Usage.InputTokens,
+//	    OutputTokens:             resp.Usage.OutputTokens,
+//	})
+//	stats := tracker.Stats("agent/planner.go:84")
+//	fmt.Printf("hit rate: %.0f%%\n", stats.HitRate()*100)
+type CallSiteTracker struct {
+	mu      sync.RWMutex
+	records map[string]*CallSiteRecord
+}
+
+// NewCallSiteTracker creates a new tracker.
+func NewCallSiteTracker() *CallSiteTracker {
+	return &CallSiteTracker{
+		records: make(map[string]*CallSiteRecord),
+	}
+}
+
+// Record adds a usage observation for callSite.
+func (t *CallSiteTracker) Record(callSite string, u UsageRecord) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	rec, ok := t.records[callSite]
+	if !ok {
+		rec = &CallSiteRecord{
+			CallSite:  callSite,
+			FirstSeen: time.Now(),
+		}
+		t.records[callSite] = rec
+	}
+
+	rec.CacheCreationTokens += int64(u.CacheCreationInputTokens)
+	rec.CacheReadTokens += int64(u.CacheReadInputTokens)
+	rec.UncachedInputTokens += int64(u.InputTokens)
+	rec.OutputTokens += int64(u.OutputTokens)
+	rec.TotalRequests++
+	if u.CacheReadInputTokens > 0 {
+		rec.CacheHitRequests++
+	}
+	rec.LastSeen = time.Now()
+}
+
+// Stats returns a snapshot of the record for callSite, or nil if not found.
+func (t *CallSiteTracker) Stats(callSite string) *CallSiteRecord {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	r := t.records[callSite]
+	if r == nil {
+		return nil
+	}
+	cp := *r
+	return &cp
+}
+
+// AllStats returns snapshots of all recorded call sites, sorted by hit rate
+// ascending (worst performers first).
+func (t *CallSiteTracker) AllStats() []*CallSiteRecord {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	out := make([]*CallSiteRecord, 0, len(t.records))
+	for _, r := range t.records {
+		cp := *r
+		out = append(out, &cp)
+	}
+
+	// Sort worst hit rate first so callers can surface actionable items.
+	for i := 1; i < len(out); i++ {
+		for j := i; j > 0 && out[j].HitRate() < out[j-1].HitRate(); j-- {
+			out[j], out[j-1] = out[j-1], out[j]
+		}
+	}
+	return out
+}
+
+// Reset clears all observations for callSite.
+func (t *CallSiteTracker) Reset(callSite string) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	delete(t.records, callSite)
+}
+
+// ResetAll clears all observations.
+func (t *CallSiteTracker) ResetAll() {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.records = make(map[string]*CallSiteRecord)
+}
+
+// Summary returns a human-readable summary of all call sites.
+func (t *CallSiteTracker) Summary() string {
+	stats := t.AllStats()
+	if len(stats) == 0 {
+		return "no call sites recorded"
+	}
+	out := fmt.Sprintf("%-40s %8s %8s %8s\n", "call site", "hit%", "eff", "reqs")
+	for _, s := range stats {
+		out += fmt.Sprintf("%-40s %7.0f%% %7.1fx %8d\n",
+			s.CallSite,
+			s.HitRate()*100,
+			s.WriteEfficiency(),
+			s.TotalRequests,
+		)
+	}
+	return out
+}
diff --git a/pkg/metrics/callsite_test.go b/pkg/metrics/callsite_test.go
@@ -0,0 +1,113 @@
+package metrics
+
+import (
+	"testing"
+)
+
+func TestCallSiteTracker_HitRate(t *testing.T) {
+	tr := NewCallSiteTracker()
+
+	// First request: cache write (no reads yet).
+	tr.Record("agent.go:42", UsageRecord{
+		CacheCreationInputTokens: 8000,
+		InputTokens:              200,
+	})
+
+	s := tr.Stats("agent.go:42")
+	if s == nil {
+		t.Fatal("expected stats, got nil")
+	}
+	if s.HitRate() != 0 {
+		t.Errorf("expected 0 hit rate after write-only request, got %f", s.HitRate())
+	}
+	if s.TotalRequests != 1 {
+		t.Errorf("expected 1 request, got %d", s.TotalRequests)
+	}
+
+	// Second request: cache read.
+	tr.Record("agent.go:42", UsageRecord{
+		CacheReadInputTokens: 8000,
+	})
+
+	s = tr.Stats("agent.go:42")
+	// hit rate = 8000 / (8000 + 8000 + 200) = ~0.49
+	if s.HitRate() <= 0 {
+		t.Errorf("expected positive hit rate after cache read, got %f", s.HitRate())
+	}
+	if s.CacheHitRequests != 1 {
+		t.Errorf("expected 1 cache hit request, got %d", s.CacheHitRequests)
+	}
+	if s.RequestHitRate() != 0.5 {
+		t.Errorf("expected 0.5 request hit rate, got %f", s.RequestHitRate())
+	}
+}
+
+func TestCallSiteTracker_WriteEfficiency(t *testing.T) {
+	tr := NewCallSiteTracker()
+
+	tr.Record("planner.go:84", UsageRecord{CacheCreationInputTokens: 4000})
+	tr.Record("planner.go:84", UsageRecord{CacheReadInputTokens: 4000})
+	tr.Record("planner.go:84", UsageRecord{CacheReadInputTokens: 4000})
+
+	s := tr.Stats("planner.go:84")
+	// efficiency = 8000 / 4000 = 2.0
+	if s.WriteEfficiency() != 2.0 {
+		t.Errorf("expected write efficiency 2.0, got %f", s.WriteEfficiency())
+	}
+}
+
+func TestCallSiteTracker_AllStats_SortedByHitRate(t *testing.T) {
+	tr := NewCallSiteTracker()
+
+	// good: 100% hit rate
+	tr.Record("good.go:1", UsageRecord{CacheReadInputTokens: 1000})
+	// bad: 0% hit rate
+	tr.Record("bad.go:1", UsageRecord{CacheCreationInputTokens: 1000})
+
+	all := tr.AllStats()
+	if len(all) != 2 {
+		t.Fatalf("expected 2 records, got %d", len(all))
+	}
+	// Worst first.
+	if all[0].CallSite != "bad.go:1" {
+		t.Errorf("expected bad.go:1 first (worst hit rate), got %s", all[0].CallSite)
+	}
+}
+
+func TestCallSiteTracker_Reset(t *testing.T) {
+	tr := NewCallSiteTracker()
+	tr.Record("x.go:1", UsageRecord{InputTokens: 100})
+	tr.Reset("x.go:1")
+	if tr.Stats("x.go:1") != nil {
+		t.Error("expected nil after reset")
+	}
+}
+
+func TestCallSiteTracker_ResetAll(t *testing.T) {
+	tr := NewCallSiteTracker()
+	tr.Record("a.go:1", UsageRecord{InputTokens: 100})
+	tr.Record("b.go:1", UsageRecord{InputTokens: 100})
+	tr.ResetAll()
+	if len(tr.AllStats()) != 0 {
+		t.Error("expected empty after ResetAll")
+	}
+}
+
+func TestCallSiteTracker_Summary(t *testing.T) {
+	tr := NewCallSiteTracker()
+	tr.Record("agent.go:42", UsageRecord{
+		CacheCreationInputTokens: 4000,
+		CacheReadInputTokens:     4000,
+	})
+	s := tr.Summary()
+	if s == "" || s == "no call sites recorded" {
+		t.Error("expected non-empty summary")
+	}
+}
+
+func TestCallSiteTracker_NilStats(t *testing.T) {
+	tr := NewCallSiteTracker()
+	if tr.Stats("nonexistent.go:1") != nil {
+		t.Error("expected nil for unknown call site")
+	}
+}
