Skip to content

Commit 8bffe09

Browse files
Alexey Panfilovclaude
andcommitted
feat(adminapi): Pareto-frontier Suggest engine + markup/think/value cols
Replaces the per-role "filters + single sort" recommender with a Pareto frontier over (role-specific quality, price), then sorts the frontier by quality desc. Every model shown is a valid trade-off: none is strictly dominated (worse AND more expensive) by another. Per-role axes: - simple/default/complex: (AA Agentic Index, prompt price) - multimodal/compaction: (AA Intelligence Index, prompt or completion price) - classifier: (1/TTFT, prompt price) when speed data present, else (intel, price) Knee-point detection surfaces "Best value: X (N% quality @ M% price)" in the banner when a non-top frontier model maximizes quality/price under a 50% quality floor. UI adds three columns derived from AA data: - Value (quality/$) - role-specific in Suggest, agentic/$ in browse - Think (TTFA - TTFT) - reasoner thinking time before answer begins - Markup (OR vs AA blended 3:1) - surfaces whether OR routes a cheaper provider (negative %) or marks the model up (positive %) Also fixes a latent bug where artificial_analysis_agentic_index was never parsed from the /api/v2/data/llms/models response - our presets rely on AgenticIndex but a live fetch would have zeroed it for every model. Adds median_time_to_first_answer_token and price_1m_blended_3_to_1 parsing alongside the fix. Tightens preset filters to remove data-coverage artefacts: - Models with quality=0 are excluded from the frontier (no untested models reaching the top purely by price-floor coincidence) - simple cap tightened to $0.2/M so it genuinely differs from default - classifier excludes :free variants (rate-limited on OR) - compaction relaxed to ctx ≥ 64k / completion ≤ $2 (was 128k / $1, which left only 2 candidates) - All non-classifier presets require ctx ≥ 32k (32k was missing before) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 3a687c1 commit 8bffe09

6 files changed

Lines changed: 425 additions & 134 deletions

File tree

internal/adminapi/handlers.go

Lines changed: 77 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package adminapi
22

33
import (
44
"context"
5+
"fmt"
56
"net/http"
67
"sort"
78
"strings"
@@ -38,6 +39,10 @@ type uiModel struct {
3839
AgenticIndex float64 // AA Agentic Index
3940
SpeedTPS float64 // median output tokens/sec
4041
TTFT float64 // median time-to-first-token, seconds
42+
ThinkTime float64 // TTFA - TTFT — time spent thinking before answer starts (reasoners only)
43+
AAPriceBlended float64 // AA's reference blended 3:1 input/output price (USD / 1M)
44+
MarkupPct float64 // (OR blended - AA blended) / AA blended × 100 — positive means OR charges more
45+
ValuePerDollar float64 // quality / prompt price (role-specific in preset path; agent/$ in browse path)
4146
}
4247

4348
type uiRouting struct {
@@ -53,6 +58,8 @@ type uiFilters struct {
5358
Reasoning bool
5459
ActivePreset string // role name when a preset is applied; empty otherwise
5560
PresetDescription string // human-readable summary for the banner
61+
ValueLeaderID string // model id of the knee-point value leader (preset path only)
62+
ValueLeaderHint string // e.g. "85% quality @ 30% price" (preset path only)
5663
Sort string // active sort column: "prompt", "completion", "score", "context", "id"
5764
SortDir string // "asc" or "desc"
5865
}
@@ -237,18 +244,37 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
237244
// Preset path — pre-filter + pre-sort via the role's preset. Checkbox
238245
// filters are ignored on this path: the preset is a complete override.
239246
if p, ok := rolePresets[preset]; ok {
247+
models := applyPreset(allCaps, aaModels, preset)
248+
filters := uiFilters{
249+
ActivePreset: preset,
250+
PresetDescription: p.Description,
251+
Tools: requiresTools(preset),
252+
Vision: preset == "multimodal",
253+
Reasoning: preset == "complex",
254+
}
255+
// Knee-point value leader: best quality/price among frontier models
256+
// with quality ≥ 50% of top. Skipped when the top model is already
257+
// the value leader.
258+
axes := p.Axes
259+
if axes == nil && preset == "classifier" {
260+
axes = classifierAxes(models)
261+
}
262+
if axes != nil {
263+
if vl := valueLeader(models, axes, 0.5); vl != nil {
264+
topQ, topP := axes(models[0])
265+
vQ, vP := axes(*vl)
266+
if topQ > 0 && topP > 0 {
267+
filters.ValueLeaderID = vl.ID
268+
filters.ValueLeaderHint = fmt.Sprintf("%.0f%% quality @ %.0f%% price",
269+
100*vQ/topQ, 100*vP/topP)
270+
}
271+
}
272+
}
240273
return indexData{
241274
Routing: s.buildRouting(),
242275
Slots: s.openRouterSlots(),
243-
Models: applyPreset(allCaps, preset),
244-
Filters: uiFilters{
245-
ActivePreset: preset,
246-
PresetDescription: p.Description,
247-
// reflect preset intent in the checkboxes so user can tell what's on
248-
Tools: requiresTools(preset),
249-
Vision: preset == "multimodal",
250-
Reasoning: preset == "complex",
251-
},
276+
Models: models,
277+
Filters: filters,
252278
}
253279
}
254280

@@ -309,10 +335,18 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
309335
}
310336
if aaModels != nil {
311337
if info := llm.LookupAAInfo(id, aaModels); info != nil {
312-
m.CodingIndex = info.CodingIndex
313-
m.AgenticIndex = info.AgenticIndex
314-
m.SpeedTPS = info.SpeedTPS
315-
m.TTFT = info.TTFT
338+
enrichFromAA(&m, *info)
339+
}
340+
}
341+
// Generic value metric for browse path: agentic index per $1/M prompt
342+
// tokens, falling back to intelligence index when agentic is absent.
343+
if m.PromptPrice > 0 {
344+
q := m.AgenticIndex
345+
if q == 0 {
346+
q = m.Score
347+
}
348+
if q > 0 {
349+
m.ValuePerDollar = q / m.PromptPrice
316350
}
317351
}
318352
models = append(models, m)
@@ -333,6 +367,12 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
333367
less = models[i].SpeedTPS < models[j].SpeedTPS
334368
case "ttft":
335369
less = models[i].TTFT < models[j].TTFT
370+
case "think":
371+
less = models[i].ThinkTime < models[j].ThinkTime
372+
case "markup":
373+
less = models[i].MarkupPct < models[j].MarkupPct
374+
case "value":
375+
less = models[i].ValuePerDollar < models[j].ValuePerDollar
336376
case "context":
337377
less = models[i].ContextLength < models[j].ContextLength
338378
case "id":
@@ -357,6 +397,30 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
357397
}
358398
}
359399

400+
// orBlendedPrice mirrors AA's 3:1 input/output weighting so prices are
401+
// directly comparable across sources.
402+
func orBlendedPrice(promptPrice, completionPrice float64) float64 {
403+
return (3*promptPrice + completionPrice) / 4
404+
}
405+
406+
// enrichFromAA populates the AA-derived fields on a uiModel from the matched
407+
// AAModelInfo record. Used by both the preset path and the browse path so
408+
// they share one formula for Think time and Markup %.
409+
func enrichFromAA(m *uiModel, aa llm.AAModelInfo) {
410+
m.CodingIndex = aa.CodingIndex
411+
m.AgenticIndex = aa.AgenticIndex
412+
m.SpeedTPS = aa.SpeedTPS
413+
m.TTFT = aa.TTFT
414+
if aa.TTFA > 0 && aa.TTFT > 0 && aa.TTFA >= aa.TTFT {
415+
m.ThinkTime = aa.TTFA - aa.TTFT
416+
}
417+
m.AAPriceBlended = aa.PriceBlended
418+
if aa.PriceBlended > 0 && m.PromptPrice > 0 {
419+
orBlended := orBlendedPrice(m.PromptPrice, m.CompletionPrice)
420+
m.MarkupPct = (orBlended - aa.PriceBlended) / aa.PriceBlended * 100
421+
}
422+
}
423+
360424
// requiresTools returns true when the preset's filter requires tool-calling.
361425
func requiresTools(preset string) bool {
362426
switch preset {

0 commit comments

Comments
 (0)