Skip to content

Commit 4a77ce3

Browse files
pgermishuyspgermishuys
committed
feat: add quality scoring, model tracking, and per-session cost attribution to metrics
Introduce a composite quality score (adherence, task completion, efficiency), per-session token breakdowns with model attribution, and cost tracking. Update MetricsReport to replace placeholder quality/gaps fields with structured QualityReport, modelsUsed, totalCost, and sessionBreakdown. Wire up model capture via plugin interface hook.
1 parent c4bbca2 commit 4a77ce3

15 files changed

Lines changed: 1071 additions & 39 deletions

‎src/features/analytics/format-metrics.test.ts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,4 +188,128 @@ describe("formatMetricsMarkdown", () => {
188188
expect(result).toContain("2m")
189189
expect(result).not.toContain("2m 0s")
190190
})
191+
192+
it("shows models row when modelsUsed is non-empty", () => {
193+
const result = formatMetricsMarkdown([makeReport({ modelsUsed: ["claude-opus-4", "gpt-4o"] })], [])
194+
expect(result).toContain("Models")
195+
expect(result).toContain("claude-opus-4")
196+
expect(result).toContain("gpt-4o")
197+
})
198+
199+
it("omits models row when modelsUsed is absent", () => {
200+
const result = formatMetricsMarkdown([makeReport()], [])
201+
expect(result).not.toContain("| Models |")
202+
})
203+
204+
it("shows total cost row when totalCost is defined and > 0", () => {
205+
const result = formatMetricsMarkdown([makeReport({ totalCost: 1.23 })], [])
206+
expect(result).toContain("Total Cost")
207+
expect(result).toContain("$1.23")
208+
})
209+
210+
it("omits total cost row when totalCost is absent", () => {
211+
const result = formatMetricsMarkdown([makeReport()], [])
212+
expect(result).not.toContain("Total Cost")
213+
})
214+
215+
it("shows quality score section when quality is present", () => {
216+
const quality = {
217+
composite: 0.78,
218+
components: {
219+
adherenceCoverage: 0.85,
220+
adherencePrecision: 0.90,
221+
taskCompletion: 0.75,
222+
efficiency: 0.62,
223+
},
224+
efficiencyData: { totalTokens: 100_000, totalTasks: 2, tokensPerTask: 50_000 },
225+
}
226+
const result = formatMetricsMarkdown([makeReport({ quality })], [])
227+
expect(result).toContain("Quality Score")
228+
expect(result).toContain("78%")
229+
expect(result).toContain("Adherence Coverage")
230+
expect(result).toContain("85%")
231+
expect(result).toContain("Adherence Precision")
232+
expect(result).toContain("90%")
233+
expect(result).toContain("Task Completion")
234+
expect(result).toContain("75%")
235+
expect(result).toContain("Efficiency")
236+
expect(result).toContain("62%")
237+
})
238+
239+
it("omits quality section when quality is absent (backward compatible)", () => {
240+
const result = formatMetricsMarkdown([makeReport()], [])
241+
expect(result).not.toContain("Quality Score")
242+
})
243+
244+
it("shows session breakdown when sessionBreakdown is present", () => {
245+
const sessionBreakdown = [
246+
{
247+
sessionId: "abc12345678",
248+
model: "claude-opus-4",
249+
agentName: "Loom",
250+
tokens: { input: 10_000, output: 5_000, reasoning: 2_000, cacheRead: 0, cacheWrite: 0 },
251+
cost: 0.55,
252+
durationMs: 330_000,
253+
},
254+
]
255+
const result = formatMetricsMarkdown([makeReport({ sessionBreakdown })], [])
256+
expect(result).toContain("Session Breakdown")
257+
expect(result).toContain("abc12345")
258+
expect(result).toContain("Loom")
259+
expect(result).toContain("17,000") // 10k+5k+2k = 17k total tokens
260+
expect(result).toContain("claude-opus-4")
261+
expect(result).toContain("$0.55")
262+
expect(result).toContain("5m 30s")
263+
})
264+
265+
it("omits session breakdown when sessionBreakdown is absent", () => {
266+
const result = formatMetricsMarkdown([makeReport()], [])
267+
expect(result).not.toContain("Session Breakdown")
268+
})
269+
270+
it("shows model attribution when multiple models and sessionBreakdown present", () => {
271+
const sessionBreakdown = [
272+
{
273+
sessionId: "s1",
274+
model: "claude-opus-4",
275+
agentName: "Loom",
276+
tokens: { input: 30_000, output: 10_000, reasoning: 5_000, cacheRead: 0, cacheWrite: 0 },
277+
cost: 0.85,
278+
durationMs: 300_000,
279+
},
280+
{
281+
sessionId: "s2",
282+
model: "gpt-4o",
283+
agentName: "Tapestry",
284+
tokens: { input: 10_000, output: 2_000, reasoning: 0, cacheRead: 0, cacheWrite: 0 },
285+
cost: 0.23,
286+
durationMs: 200_000,
287+
},
288+
]
289+
const result = formatMetricsMarkdown([makeReport({
290+
modelsUsed: ["claude-opus-4", "gpt-4o"],
291+
sessionBreakdown,
292+
})], [])
293+
expect(result).toContain("Model Attribution")
294+
expect(result).toContain("claude-opus-4")
295+
expect(result).toContain("gpt-4o")
296+
})
297+
298+
it("omits model attribution for single-model plans", () => {
299+
const sessionBreakdown = [
300+
{
301+
sessionId: "s1",
302+
model: "claude-opus-4",
303+
agentName: "Loom",
304+
tokens: { input: 10_000, output: 5_000, reasoning: 0, cacheRead: 0, cacheWrite: 0 },
305+
cost: 0.55,
306+
durationMs: 300_000,
307+
},
308+
]
309+
const result = formatMetricsMarkdown([makeReport({
310+
modelsUsed: ["claude-opus-4"],
311+
sessionBreakdown,
312+
})], [])
313+
expect(result).not.toContain("Model Attribution")
314+
})
191315
})

‎src/features/analytics/format-metrics.ts‎

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,13 @@ function formatDuration(ms: number): string {
1818
return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`
1919
}
2020

21+
/**
22+
* Format a dollar cost as $X.XX
23+
*/
24+
function formatCost(n: number): string {
25+
return `$${n.toFixed(2)}`
26+
}
27+
2128
/**
2229
* Format an ISO date string as a short human-readable date.
2330
*/
@@ -30,6 +37,13 @@ function formatDate(iso: string): string {
3037
}
3138
}
3239

40+
/**
41+
* Format a percentage (0-1 value) as an integer percent string (e.g., 0.782 → "78%").
42+
*/
43+
function formatPct(v: number): string {
44+
return `${Math.round(v * 100)}%`
45+
}
46+
3347
/**
3448
* Format a single MetricsReport as a markdown section.
3549
*/
@@ -41,8 +55,8 @@ function formatReport(report: MetricsReport): string {
4155
lines.push("")
4256
lines.push("| Metric | Value |")
4357
lines.push("|--------|-------|")
44-
lines.push(`| Coverage | ${Math.round(report.adherence.coverage * 100)}% |`)
45-
lines.push(`| Precision | ${Math.round(report.adherence.precision * 100)}% |`)
58+
lines.push(`| Coverage | ${formatPct(report.adherence.coverage)} |`)
59+
lines.push(`| Precision | ${formatPct(report.adherence.precision)} |`)
4660
lines.push(`| Sessions | ${report.sessionCount} |`)
4761
lines.push(`| Duration | ${formatDuration(report.durationMs)} |`)
4862
lines.push(`| Input Tokens | ${formatNumber(report.tokenUsage.input)} |`)
@@ -56,6 +70,26 @@ function formatReport(report: MetricsReport): string {
5670
lines.push(`| Cache Write | ${formatNumber(report.tokenUsage.cacheWrite)} |`)
5771
}
5872

73+
// Models used
74+
if (report.modelsUsed && report.modelsUsed.length > 0) {
75+
lines.push(`| Models | ${report.modelsUsed.join(", ")} |`)
76+
}
77+
78+
// Total cost
79+
if (report.totalCost !== undefined && report.totalCost > 0) {
80+
lines.push(`| Total Cost | ${formatCost(report.totalCost)} |`)
81+
}
82+
83+
// Quality score section
84+
if (report.quality) {
85+
const q = report.quality
86+
lines.push(`| Quality Score | ${formatPct(q.composite)} |`)
87+
lines.push(`| ├ Adherence Coverage | ${formatPct(q.components.adherenceCoverage)} |`)
88+
lines.push(`| ├ Adherence Precision | ${formatPct(q.components.adherencePrecision)} |`)
89+
lines.push(`| ├ Task Completion | ${formatPct(q.components.taskCompletion)} |`)
90+
lines.push(`| └ Efficiency | ${formatPct(q.components.efficiency)} |`)
91+
}
92+
5993
if (report.adherence.unplannedChanges.length > 0) {
6094
lines.push("")
6195
lines.push(`**Unplanned Changes**: ${report.adherence.unplannedChanges.map((f) => `\`${f}\``).join(", ")}`)
@@ -66,6 +100,46 @@ function formatReport(report: MetricsReport): string {
66100
lines.push(`**Missed Files**: ${report.adherence.missedFiles.map((f) => `\`${f}\``).join(", ")}`)
67101
}
68102

103+
// Model attribution (when multiple models used)
104+
if (report.sessionBreakdown && report.modelsUsed && report.modelsUsed.length > 1) {
105+
// Build per-model summary from session breakdown
106+
const modelTotals = new Map<string, { tokens: number; cost: number }>()
107+
for (const s of report.sessionBreakdown) {
108+
const key = s.model ?? "(unknown)"
109+
const t = s.tokens.input + s.tokens.output + s.tokens.reasoning
110+
const c = s.cost ?? 0
111+
const existing = modelTotals.get(key)
112+
if (existing) {
113+
existing.tokens += t
114+
existing.cost += c
115+
} else {
116+
modelTotals.set(key, { tokens: t, cost: c })
117+
}
118+
}
119+
const attribution = Array.from(modelTotals.entries())
120+
.filter(([k]) => k !== "(unknown)")
121+
.map(([model, data]) => `${formatNumber(data.tokens)} tokens on ${model} (${formatCost(data.cost)})`)
122+
if (attribution.length > 0) {
123+
lines.push("")
124+
lines.push(`**Model Attribution**: ${attribution.join(", ")}`)
125+
}
126+
}
127+
128+
// Session breakdown
129+
if (report.sessionBreakdown && report.sessionBreakdown.length > 0) {
130+
lines.push("")
131+
lines.push("**Session Breakdown**:")
132+
for (const s of report.sessionBreakdown) {
133+
const id = s.sessionId.length > 8 ? s.sessionId.slice(0, 8) : s.sessionId
134+
const agent = s.agentName ?? "(unknown)"
135+
const totalTokens = s.tokens.input + s.tokens.output + s.tokens.reasoning
136+
const model = s.model ? `, ${s.model}` : ""
137+
const cost = s.cost !== undefined && s.cost > 0 ? `, ${formatCost(s.cost)}` : ""
138+
const dur = formatDuration(s.durationMs)
139+
lines.push(`- \`${id}\` ${agent} — ${formatNumber(totalTokens)} tokens${model}${cost}, ${dur}`)
140+
}
141+
}
142+
69143
return lines.join("\n")
70144
}
71145

@@ -193,3 +267,4 @@ export function formatMetricsMarkdown(
193267

194268
return lines.join("\n")
195269
}
270+

‎src/features/analytics/generate-metrics-report.test.ts‎

Lines changed: 125 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@ function makeSummary(sessionId: string, durationMs: number = 300_000): SessionSu
3030
}
3131
}
3232

33+
function makeSummaryWithModel(sessionId: string, model: string, durationMs = 300_000): SessionSummary {
34+
return {
35+
...makeSummary(sessionId, durationMs),
36+
model,
37+
totalCost: 0.05,
38+
}
39+
}
40+
3341
function createPlanFile(dir: string, content: string): string {
3442
const plansDir = join(dir, ".weave", "plans")
3543
mkdirSync(plansDir, { recursive: true })
@@ -80,8 +88,7 @@ describe("generateMetricsReport", () => {
8088
expect(report!.tokenUsage.input).toBe(2000) // 1000 * 2 sessions
8189
expect(report!.tokenUsage.output).toBe(1000)
8290
expect(report!.durationMs).toBe(300_000) // 120k + 180k
83-
expect(report!.quality).toBeUndefined()
84-
expect(report!.gaps).toBeUndefined()
91+
expect(report!.quality).toBeDefined()
8592
})
8693

8794
it("calculates adherence with vacuous coverage when no start_sha", () => {
@@ -198,4 +205,120 @@ describe("generateMetricsReport", () => {
198205
expect(report!.generatedAt >= before).toBe(true)
199206
expect(report!.generatedAt <= after).toBe(true)
200207
})
208+
209+
it("populates modelsUsed from session model fields", () => {
210+
const planPath = createPlanFile(tempDir, `# Plan\n\n## TODOs\n\n- [x] 1. **Task**\n **Files**: src/a.ts\n`)
211+
appendSessionSummary(tempDir, makeSummaryWithModel("s1", "claude-opus-4"))
212+
appendSessionSummary(tempDir, makeSummaryWithModel("s2", "claude-sonnet-4-20250514"))
213+
appendSessionSummary(tempDir, makeSummaryWithModel("s3", "claude-opus-4")) // duplicate model
214+
215+
const state: WorkState = {
216+
active_plan: planPath,
217+
started_at: "2026-01-01T00:00:00.000Z",
218+
session_ids: ["s1", "s2", "s3"],
219+
plan_name: "test-plan",
220+
}
221+
222+
const report = generateMetricsReport(tempDir, state)
223+
expect(report).not.toBeNull()
224+
expect(report!.modelsUsed).toBeDefined()
225+
expect(report!.modelsUsed!.length).toBe(2)
226+
expect(report!.modelsUsed).toContain("claude-opus-4")
227+
expect(report!.modelsUsed).toContain("claude-sonnet-4-20250514")
228+
})
229+
230+
it("omits modelsUsed when sessions have no model field", () => {
231+
const planPath = createPlanFile(tempDir, `# Plan\n\n## TODOs\n\n- [ ] 1. **Task**\n **Files**: src/a.ts\n`)
232+
appendSessionSummary(tempDir, makeSummary("s1"))
233+
234+
const state: WorkState = {
235+
active_plan: planPath,
236+
started_at: "2026-01-01T00:00:00.000Z",
237+
session_ids: ["s1"],
238+
plan_name: "test-plan",
239+
}
240+
241+
const report = generateMetricsReport(tempDir, state)
242+
expect(report).not.toBeNull()
243+
expect(report!.modelsUsed).toBeUndefined()
244+
})
245+
246+
it("populates totalCost from session costs", () => {
247+
const planPath = createPlanFile(tempDir, `# Plan\n\n## TODOs\n\n- [ ] 1. **Task**\n **Files**: src/a.ts\n`)
248+
appendSessionSummary(tempDir, makeSummaryWithModel("s1", "claude-opus-4"))
249+
appendSessionSummary(tempDir, makeSummaryWithModel("s2", "claude-sonnet-4-20250514"))
250+
251+
const state: WorkState = {
252+
active_plan: planPath,
253+
started_at: "2026-01-01T00:00:00.000Z",
254+
session_ids: ["s1", "s2"],
255+
plan_name: "test-plan",
256+
}
257+
258+
const report = generateMetricsReport(tempDir, state)
259+
expect(report).not.toBeNull()
260+
expect(report!.totalCost).toBeCloseTo(0.10, 5)
261+
})
262+
263+
it("populates sessionBreakdown with per-session detail", () => {
264+
const planPath = createPlanFile(tempDir, `# Plan\n\n## TODOs\n\n- [ ] 1. **Task**\n **Files**: src/a.ts\n`)
265+
appendSessionSummary(tempDir, makeSummaryWithModel("s1", "claude-opus-4", 120_000))
266+
appendSessionSummary(tempDir, makeSummaryWithModel("s2", "claude-sonnet-4-20250514", 180_000))
267+
268+
const state: WorkState = {
269+
active_plan: planPath,
270+
started_at: "2026-01-01T00:00:00.000Z",
271+
session_ids: ["s1", "s2"],
272+
plan_name: "test-plan",
273+
}
274+
275+
const report = generateMetricsReport(tempDir, state)
276+
expect(report).not.toBeNull()
277+
expect(report!.sessionBreakdown).toBeDefined()
278+
expect(report!.sessionBreakdown!.length).toBe(2)
279+
const s1 = report!.sessionBreakdown!.find((s) => s.sessionId === "s1")
280+
expect(s1).toBeDefined()
281+
expect(s1!.model).toBe("claude-opus-4")
282+
expect(s1!.durationMs).toBe(120_000)
283+
})
284+
285+
it("populates quality with valid QualityReport structure for completed plan", () => {
286+
const planPath = createPlanFile(tempDir, `# Plan\n\n## TODOs\n\n- [x] 1. **Task one**\n **Files**: src/a.ts\n- [x] 2. **Task two**\n **Files**: src/b.ts\n`)
287+
appendSessionSummary(tempDir, makeSummary("s1"))
288+
289+
const state: WorkState = {
290+
active_plan: planPath,
291+
started_at: "2026-01-01T00:00:00.000Z",
292+
session_ids: ["s1"],
293+
plan_name: "test-plan",
294+
}
295+
296+
const report = generateMetricsReport(tempDir, state)
297+
expect(report).not.toBeNull()
298+
expect(report!.quality).toBeDefined()
299+
expect(report!.quality!.composite).toBeGreaterThanOrEqual(0)
300+
expect(report!.quality!.composite).toBeLessThanOrEqual(1)
301+
expect(report!.quality!.components.adherenceCoverage).toBeGreaterThanOrEqual(0)
302+
expect(report!.quality!.components.taskCompletion).toBe(1) // all tasks done
303+
expect(report!.quality!.efficiencyData.totalTasks).toBe(2)
304+
})
305+
306+
it("gaps field no longer exists on MetricsReport", () => {
307+
// Verify at compile time — if gaps existed on the type, this would need updating
308+
const planPath = createPlanFile(tempDir, `# Plan\n\n## TODOs\n\n- [ ] 1. **Task**\n **Files**: src/a.ts\n`)
309+
appendSessionSummary(tempDir, makeSummary("s1"))
310+
311+
const state: WorkState = {
312+
active_plan: planPath,
313+
started_at: "2026-01-01T00:00:00.000Z",
314+
session_ids: ["s1"],
315+
plan_name: "test-plan",
316+
}
317+
318+
const report = generateMetricsReport(tempDir, state)
319+
expect(report).not.toBeNull()
320+
// TypeScript compilation will fail if 'gaps' property exists on MetricsReport — this confirms it was removed
321+
const keys = Object.keys(report!)
322+
expect(keys).not.toContain("gaps")
323+
})
201324
})

0 commit comments

Comments
 (0)