Skip to content

Commit 9591729

Browse files
dzarlaxdzarlax
committed
Show model usage telemetry in admin UI
1 parent bcfaffc commit 9591729

8 files changed

Lines changed: 187 additions & 42 deletions

File tree

‎internal/adminapi/adminapi_test.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,42 @@ type fakeConfigurableProvider struct {
6868
caps llm.Capabilities
6969
}
7070

71+
type fakeUsageStore struct {
72+
byModel []llm.UsageModelRow
73+
}
74+
75+
func (f fakeUsageStore) PutUsage(context.Context, llm.UsageLog) (int64, error) {
76+
return 0, nil
77+
}
78+
79+
func (f fakeUsageStore) UpdateAssistantMessageID(context.Context, int64, int64) error {
80+
return nil
81+
}
82+
83+
func (f fakeUsageStore) UpdateTurnLatencyMs(context.Context, int64, int) error {
84+
return nil
85+
}
86+
87+
func (f fakeUsageStore) UsageTotals(context.Context, time.Time) (llm.UsageTotals, error) {
88+
return llm.UsageTotals{}, nil
89+
}
90+
91+
func (f fakeUsageStore) UsageByDay(context.Context, time.Time) ([]llm.UsageDayBucket, error) {
92+
return nil, nil
93+
}
94+
95+
func (f fakeUsageStore) UsageByModel(context.Context, time.Time, int) ([]llm.UsageModelRow, error) {
96+
return f.byModel, nil
97+
}
98+
99+
func (f fakeUsageStore) UsageByRole(context.Context, time.Time) ([]llm.UsageRoleRow, error) {
100+
return nil, nil
101+
}
102+
103+
func (f fakeUsageStore) ExpensiveTurns(context.Context, time.Time, int) ([]llm.ExpensiveTurn, error) {
104+
return nil, nil
105+
}
106+
71107
func (f *fakeConfigurableProvider) Chat(context.Context, []llm.Message, string, []llm.Tool) (llm.Response, error) {
72108
return llm.Response{}, nil
73109
}
@@ -264,6 +300,40 @@ func TestTGAdminModelsReturnsSearchResults(t *testing.T) {
264300
}
265301
}
266302

303+
func TestTGAdminModelsIncludesUsageTelemetry(t *testing.T) {
304+
s, _ := newTGModelTestServer(t)
305+
s.cfgRef.Telegram.BotToken = "123:abc"
306+
s.cfgRef.Telegram.OwnerChatID = 42
307+
s.usageStore = fakeUsageStore{byModel: []llm.UsageModelRow{{
308+
Provider: "openrouter",
309+
ModelID: "qwen/qwen3.5-plus",
310+
Calls: 12,
311+
CostUSD: 0.0345,
312+
AvgLatencyMS: 830,
313+
ErrorCount: 3,
314+
}}}
315+
316+
req := httptest.NewRequest(http.MethodGet, "/tg-admin/api/models?role=default&q=plus", nil)
317+
req.Header.Set("X-Telegram-Init-Data", signedTGInitData(t, "123:abc", 42, time.Now()))
318+
rec := httptest.NewRecorder()
319+
s.handleTGAdminRouter(rec, req)
320+
321+
if rec.Code != http.StatusOK {
322+
t.Fatalf("status = %d, want %d, body: %s", rec.Code, http.StatusOK, rec.Body.String())
323+
}
324+
var payload tgAdminModelsResponse
325+
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
326+
t.Fatal(err)
327+
}
328+
if len(payload.Models) != 1 {
329+
t.Fatalf("unexpected models: %+v", payload.Models)
330+
}
331+
tel := payload.Models[0].Telemetry
332+
if tel.Calls != 12 || tel.AvgLatencyMS != 830 || tel.ErrorRatePct != 25 || tel.WindowDays != 7 {
333+
t.Fatalf("unexpected telemetry: %+v", tel)
334+
}
335+
}
336+
267337
func TestTGAdminModelsMarksFreeSearchResultsUnverified(t *testing.T) {
268338
s, _ := newTGModelTestServer(t)
269339
s.cfgRef.Telegram.BotToken = "123:abc"

‎internal/adminapi/handlers.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,15 @@ type uiModel struct {
5353
Policy string
5454
Reasons []string
5555
Warnings []string
56+
Telemetry modelTelemetry
57+
}
58+
59+
type modelTelemetry struct {
60+
Calls int `json:"calls"`
61+
AvgLatencyMS int `json:"avg_latency_ms"`
62+
ErrorRatePct float64 `json:"error_rate_pct"`
63+
CostUSD float64 `json:"cost_usd"`
64+
WindowDays int `json:"window_days"`
5665
}
5766

5867
type uiSlotInfo struct {
@@ -318,6 +327,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
318327
}
319328
}
320329
models := applyPreset(allCaps, aaModels, preset, visionFallbackPrompt)
330+
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
321331
filters := uiFilters{
322332
ActivePreset: preset,
323333
PresetDescription: p.Description,
@@ -427,6 +437,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
427437
annotateModelForRole(&m, "", "catalog")
428438
models = append(models, m)
429439
}
440+
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
430441
asc := sortDir == "asc"
431442
sort.Slice(models, func(i, j int) bool {
432443
var less bool
@@ -492,6 +503,42 @@ func effectiveOrNominal(m uiModel) float64 {
492503
return m.PromptPrice
493504
}
494505

506+
func (s *Server) loadModelTelemetry(ctx context.Context, provider string) map[string]modelTelemetry {
507+
if s.usageStore == nil {
508+
return nil
509+
}
510+
rows, err := s.usageStore.UsageByModel(ctx, time.Now().Add(-7*24*time.Hour), 1000)
511+
if err != nil {
512+
s.logger.Warn("model telemetry load failed", "err", err)
513+
return nil
514+
}
515+
out := make(map[string]modelTelemetry, len(rows))
516+
for _, row := range rows {
517+
if row.Provider != provider || row.Calls <= 0 {
518+
continue
519+
}
520+
out[row.ModelID] = modelTelemetry{
521+
Calls: row.Calls,
522+
AvgLatencyMS: int(row.AvgLatencyMS + 0.5),
523+
ErrorRatePct: 100 * float64(row.ErrorCount) / float64(row.Calls),
524+
CostUSD: row.CostUSD,
525+
WindowDays: 7,
526+
}
527+
}
528+
return out
529+
}
530+
531+
func attachModelTelemetry(models []uiModel, telemetry map[string]modelTelemetry) {
532+
if len(models) == 0 || len(telemetry) == 0 {
533+
return
534+
}
535+
for i := range models {
536+
if t, ok := telemetry[models[i].ID]; ok {
537+
models[i].Telemetry = t
538+
}
539+
}
540+
}
541+
495542
// enrichFromAA populates the AA-derived fields on a uiModel from the matched
496543
// AAModelInfo record. Used by both the preset path and the browse path so
497544
// they share one formula for Think time and Markup %.

‎internal/adminapi/templates/partials_models_row.html‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@
1414
{{range .Warnings}}<span>{{.}}</span> {{end}}
1515
</div>
1616
{{end}}
17+
{{if gt .Telemetry.Calls 0}}
18+
<div style="margin-top:4px;color:var(--text-secondary);font-size:0.78rem;">
19+
7d usage:
20+
{{.Telemetry.Calls}} calls
21+
· {{.Telemetry.AvgLatencyMS}} ms avg
22+
· {{printf "%.0f%%" .Telemetry.ErrorRatePct}} errors
23+
· {{printf "$%.4f" .Telemetry.CostUSD}}
24+
</div>
25+
{{end}}
1726
<div class="assign-buttons">
1827
{{$prov := $.CatalogProvider}}
1928
{{range $.Slots}}

‎internal/adminapi/templates/tg_admin.html‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -614,6 +614,16 @@ <h2 id="sheet-title">Choose model</h2>
614614
warnings.textContent = model.warnings.slice(0, 2).join(" · ");
615615
body.appendChild(warnings);
616616
}
617+
if (model.telemetry && model.telemetry.calls) {
618+
var telemetry = document.createElement("div");
619+
telemetry.className = model.telemetry.error_rate_pct > 0 ? "model-card__warn" : "model-card__note";
620+
telemetry.textContent = model.telemetry.window_days + "d usage · " +
621+
int(model.telemetry.calls) + " calls · " +
622+
int(model.telemetry.avg_latency_ms) + " ms avg · " +
623+
Math.round(model.telemetry.error_rate_pct) + "% errors · " +
624+
money(model.telemetry.cost_usd);
625+
body.appendChild(telemetry);
626+
}
617627
if (model.check_status) {
618628
var check = document.createElement("div");
619629
check.className = model.check_status === "free_verified" ? "model-card__note" : "model-card__warn";

‎internal/adminapi/tgadmin.go‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -58,30 +58,31 @@ type tgAdminRoute struct {
5858
}
5959

6060
type tgAdminModel struct {
61-
ID string `json:"id"`
62-
Provider string `json:"provider"`
63-
PromptPrice float64 `json:"prompt_price"`
64-
CompletionPrice float64 `json:"completion_price"`
65-
ContextLength int `json:"context_length"`
66-
Vision bool `json:"vision"`
67-
Tools bool `json:"tools"`
68-
Reasoning bool `json:"reasoning"`
69-
Free bool `json:"free"`
70-
Score float64 `json:"score"`
71-
AgenticIndex float64 `json:"agentic_index"`
72-
SpeedTPS float64 `json:"speed_tps"`
73-
TTFT float64 `json:"ttft"`
74-
ValuePerDollar float64 `json:"value_per_dollar"`
75-
Recommended bool `json:"recommended"`
76-
Current bool `json:"current"`
77-
Source string `json:"source,omitempty"`
78-
Policy string `json:"policy,omitempty"`
79-
Reasons []string `json:"reasons,omitempty"`
80-
Warnings []string `json:"warnings,omitempty"`
81-
CheckStatus string `json:"check_status,omitempty"`
82-
CheckedAt string `json:"checked_at,omitempty"`
83-
CheckLatencyMS int64 `json:"check_latency_ms,omitempty"`
84-
CheckError string `json:"check_error,omitempty"`
61+
ID string `json:"id"`
62+
Provider string `json:"provider"`
63+
PromptPrice float64 `json:"prompt_price"`
64+
CompletionPrice float64 `json:"completion_price"`
65+
ContextLength int `json:"context_length"`
66+
Vision bool `json:"vision"`
67+
Tools bool `json:"tools"`
68+
Reasoning bool `json:"reasoning"`
69+
Free bool `json:"free"`
70+
Score float64 `json:"score"`
71+
AgenticIndex float64 `json:"agentic_index"`
72+
SpeedTPS float64 `json:"speed_tps"`
73+
TTFT float64 `json:"ttft"`
74+
ValuePerDollar float64 `json:"value_per_dollar"`
75+
Recommended bool `json:"recommended"`
76+
Current bool `json:"current"`
77+
Source string `json:"source,omitempty"`
78+
Policy string `json:"policy,omitempty"`
79+
Reasons []string `json:"reasons,omitempty"`
80+
Warnings []string `json:"warnings,omitempty"`
81+
Telemetry modelTelemetry `json:"telemetry,omitempty"`
82+
CheckStatus string `json:"check_status,omitempty"`
83+
CheckedAt string `json:"checked_at,omitempty"`
84+
CheckLatencyMS int64 `json:"check_latency_ms,omitempty"`
85+
CheckError string `json:"check_error,omitempty"`
8586
}
8687

8788
type tgAdminModelsResponse struct {
@@ -431,6 +432,7 @@ func (s *Server) buildTGAdminModels(ctx context.Context, role, provider, query s
431432
if len(models) == 0 {
432433
models = tgAdminBrowseModels(allCaps, aaModels, query, role)
433434
}
435+
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, provider))
434436
models = filterBlockedFreeModels(models, provider, checks)
435437
resp.Recommended = recommendedMode
436438
if len(models) > limit {
@@ -596,6 +598,7 @@ func tgModelFromUI(provider string, m uiModel, current string, checks map[string
596598
Policy: m.Policy,
597599
Reasons: m.Reasons,
598600
Warnings: m.Warnings,
601+
Telemetry: m.Telemetry,
599602
CheckStatus: check.Status,
600603
CheckedAt: check.CheckedAt,
601604
CheckLatencyMS: check.LatencyMS,

‎internal/llm/usage.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@ type UsageLog struct {
1919
ChatID int64
2020
PromptTokens int
2121
CompletionTokens int
22-
CachedPromptTokens int // subset of PromptTokens that hit provider's prompt cache (Anthropic/OpenAI)
23-
ReasoningTokens int // for thinking-enabled models: tokens spent on internal reasoning (billed as completion but semantically distinct)
22+
CachedPromptTokens int // subset of PromptTokens that hit provider's prompt cache (Anthropic/OpenAI)
23+
ReasoningTokens int // for thinking-enabled models: tokens spent on internal reasoning (billed as completion but semantically distinct)
2424
Cost float64 // USD, authoritative per-request cost reported by the provider (OpenRouter's usage.cost). 0 when the provider doesn't surface this.
2525
LatencyMs int
26-
TurnLatencyMs int // end-to-end turn time (user msg → final reply); only set on the last call of the turn
26+
TurnLatencyMs int // end-to-end turn time (user msg → final reply); only set on the last call of the turn
2727
Success bool
2828
ErrorClass string // "" / "rate_limit" / "5xx" / "network" / "timeout" / "other"
2929
RequestID string // provider's request id (e.g. OpenRouter gen-xxxxx) — useful for cross-referencing with provider dashboards
@@ -92,6 +92,8 @@ type UsageModelRow struct {
9292
PromptTokens int
9393
CompletionTokens int
9494
CostUSD float64
95+
AvgLatencyMS float64
96+
ErrorCount int
9597
}
9698

9799
// UsageRoleRow aggregates per routing role.

‎internal/store/postgres.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -644,7 +644,7 @@ func (p *Postgres) SearchAllSessions(chatID int64, queryEmb []float32, topK int,
644644
}
645645

646646
type turn struct {
647-
date time.Time
647+
date time.Time
648648
userText string
649649
botText string
650650
userEmb []float32
@@ -850,7 +850,9 @@ func (p *Postgres) UsageByModel(ctx context.Context, since time.Time, limit int)
850850
COUNT(*),
851851
COALESCE(SUM(prompt_tokens), 0),
852852
COALESCE(SUM(completion_tokens), 0),
853-
COALESCE(SUM(cost_usd), 0) AS cost
853+
COALESCE(SUM(cost_usd), 0) AS cost,
854+
COALESCE(AVG(NULLIF(latency_ms, 0)), 0) AS avg_latency,
855+
COALESCE(SUM(CASE WHEN success THEN 0 ELSE 1 END), 0) AS errors
854856
FROM usage_log WHERE ts >= $1
855857
GROUP BY provider, model_id
856858
ORDER BY cost DESC, 3 DESC
@@ -862,7 +864,7 @@ func (p *Postgres) UsageByModel(ctx context.Context, since time.Time, limit int)
862864
var out []llm.UsageModelRow
863865
for rows.Next() {
864866
var r llm.UsageModelRow
865-
if err := rows.Scan(&r.Provider, &r.ModelID, &r.Calls, &r.PromptTokens, &r.CompletionTokens, &r.CostUSD); err != nil {
867+
if err := rows.Scan(&r.Provider, &r.ModelID, &r.Calls, &r.PromptTokens, &r.CompletionTokens, &r.CostUSD, &r.AvgLatencyMS, &r.ErrorCount); err != nil {
866868
return nil, err
867869
}
868870
out = append(out, r)

‎internal/store/sqlite.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,11 @@ func NewSQLite(path string) (*SQLite, error) {
110110
return nil, fmt.Errorf("init schema: %w", err)
111111
}
112112
// Migrations for existing databases
113-
db.Exec(`ALTER TABLE messages ADD COLUMN parts TEXT`) //nolint:errcheck
114-
db.Exec(`ALTER TABLE messages ADD COLUMN embedding BLOB`) //nolint:errcheck
115-
db.Exec(`ALTER TABLE model_capabilities ADD COLUMN score REAL NOT NULL DEFAULT 0`) //nolint:errcheck
116-
db.Exec(`ALTER TABLE usage_log ADD COLUMN cost_usd REAL NOT NULL DEFAULT 0`) //nolint:errcheck
117-
db.Exec(`ALTER TABLE usage_log ADD COLUMN turn_latency_ms INTEGER NOT NULL DEFAULT 0`) //nolint:errcheck
113+
db.Exec(`ALTER TABLE messages ADD COLUMN parts TEXT`) //nolint:errcheck
114+
db.Exec(`ALTER TABLE messages ADD COLUMN embedding BLOB`) //nolint:errcheck
115+
db.Exec(`ALTER TABLE model_capabilities ADD COLUMN score REAL NOT NULL DEFAULT 0`) //nolint:errcheck
116+
db.Exec(`ALTER TABLE usage_log ADD COLUMN cost_usd REAL NOT NULL DEFAULT 0`) //nolint:errcheck
117+
db.Exec(`ALTER TABLE usage_log ADD COLUMN turn_latency_ms INTEGER NOT NULL DEFAULT 0`) //nolint:errcheck
118118
return &SQLite{db: db}, nil
119119
}
120120

@@ -550,11 +550,11 @@ func (s *SQLite) SearchAllSessions(chatID int64, queryEmb []float32, topK int, m
550550

551551
// Group into turns: user message followed by the next assistant response.
552552
type turn struct {
553-
date time.Time
554-
userText string
555-
botText string
556-
userEmb []float32
557-
score float64
553+
date time.Time
554+
userText string
555+
botText string
556+
userEmb []float32
557+
score float64
558558
}
559559
var turns []turn
560560
for i := 0; i < len(all); i++ {
@@ -920,7 +920,9 @@ func (s *SQLite) UsageByModel(ctx context.Context, since time.Time, limit int) (
920920
COUNT(*),
921921
COALESCE(SUM(prompt_tokens), 0),
922922
COALESCE(SUM(completion_tokens), 0),
923-
COALESCE(SUM(cost_usd), 0) AS cost
923+
COALESCE(SUM(cost_usd), 0) AS cost,
924+
COALESCE(AVG(NULLIF(latency_ms, 0)), 0) AS avg_latency,
925+
COALESCE(SUM(CASE WHEN success THEN 0 ELSE 1 END), 0) AS errors
924926
FROM usage_log WHERE ts >= ?
925927
GROUP BY provider, model_id
926928
ORDER BY cost DESC, 3 DESC
@@ -932,7 +934,7 @@ func (s *SQLite) UsageByModel(ctx context.Context, since time.Time, limit int) (
932934
var out []llm.UsageModelRow
933935
for rows.Next() {
934936
var r llm.UsageModelRow
935-
if err := rows.Scan(&r.Provider, &r.ModelID, &r.Calls, &r.PromptTokens, &r.CompletionTokens, &r.CostUSD); err != nil {
937+
if err := rows.Scan(&r.Provider, &r.ModelID, &r.Calls, &r.PromptTokens, &r.CompletionTokens, &r.CostUSD, &r.AvgLatencyMS, &r.ErrorCount); err != nil {
936938
return nil, err
937939
}
938940
out = append(out, r)

0 commit comments

Comments
 (0)