Skip to content

Commit 84af71e

Browse files
dzarlaxdzarlax
committed
improve model recommendation ranking
1 parent 9f5d392 commit 84af71e

11 files changed

Lines changed: 337 additions & 53 deletions

‎internal/adminapi/handlers.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ type uiModel struct {
5252
Recommended bool
5353
Source string
5454
Policy string
55+
Section string
5556
OverrideNote string
5657
Reasons []string
5758
Warnings []string
@@ -90,11 +91,19 @@ type uiFilters struct {
9091
SortDir string // "asc" or "desc"
9192
}
9293

94+
type uiModelSection struct {
95+
Key string
96+
Title string
97+
Description string
98+
Models []uiModel
99+
}
100+
93101
type indexData struct {
94102
ActiveTab string // "routing" or "analytics" — drives tab highlight in layout
95103
Routing uiRouting
96104
Slots []uiSlot // slots backing the currently-browsed provider (for per-model assign buttons)
97105
Models []uiModel
106+
ModelSections []uiModelSection
98107
Filters uiFilters
99108
CatalogProvider string // "openrouter" | "gemini" — which catalog is shown in the models browser
100109
}
@@ -383,6 +392,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
383392
models = appendAllowedOverrideCandidates(models, allCaps, aaModels, catalogProv, preset, overrides)
384393
models = filterModelOverrides(models, catalogProv, overrides, true)
385394
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
395+
sections := buildModelSections(models, true)
386396
filters := uiFilters{
387397
ActivePreset: preset,
388398
PresetDescription: p.Description,
@@ -410,6 +420,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
410420
Routing: s.buildRouting(),
411421
Slots: s.allAssignableSlots(),
412422
Models: models,
423+
ModelSections: sections,
413424
Filters: filters,
414425
CatalogProvider: catalogProv,
415426
}
@@ -556,6 +567,83 @@ func effectiveOrNominal(m uiModel) float64 {
556567
return m.PromptPrice
557568
}
558569

570+
func buildModelSections(models []uiModel, presetMode bool) []uiModelSection {
571+
if !presetMode || len(models) == 0 {
572+
return nil
573+
}
574+
order := []uiModelSection{
575+
{Key: "recommended", Title: "Recommended", Description: "Stable, role-fit models backed by benchmark data."},
576+
{Key: "interesting", Title: "Interesting", Description: "Plausible alternatives and non-obvious trade-offs worth reviewing."},
577+
{Key: "untested", Title: "Untested", Description: "Capability-compatible models with missing quality data or local evidence."},
578+
{Key: "blocked", Title: "Blocked", Description: "Filtered or manually denied models, shown only when present."},
579+
}
580+
byKey := make(map[string][]uiModel, len(order))
581+
for _, m := range models {
582+
key := modelSectionKey(m)
583+
byKey[key] = append(byKey[key], m)
584+
}
585+
out := make([]uiModelSection, 0, len(order))
586+
for _, section := range order {
587+
section.Models = applySectionDiversity(byKey[section.Key], 3)
588+
if len(section.Models) == 0 {
589+
continue
590+
}
591+
out = append(out, section)
592+
}
593+
return out
594+
}
595+
596+
func modelSectionKey(m uiModel) string {
597+
switch {
598+
case m.Policy == "manual_deny" || m.Policy == "free_blocked":
599+
return "blocked"
600+
case m.Section != "":
601+
return m.Section
602+
case m.Recommended:
603+
return "recommended"
604+
case m.Source == "untested" || m.Policy == "free_unverified":
605+
return "untested"
606+
case m.Source == "near_frontier" || m.Source == "manual" || m.Policy == "manual_allow":
607+
return "interesting"
608+
default:
609+
return "interesting"
610+
}
611+
}
612+
613+
func applySectionDiversity(models []uiModel, perFamily int) []uiModel {
614+
if perFamily <= 0 || len(models) == 0 {
615+
return models
616+
}
617+
out := make([]uiModel, 0, len(models))
618+
counts := map[string]int{}
619+
for _, m := range models {
620+
if m.Policy == "manual_allow" {
621+
out = append(out, m)
622+
counts[modelFamilyKey(m.ID)]++
623+
continue
624+
}
625+
family := modelFamilyKey(m.ID)
626+
if counts[family] >= perFamily {
627+
continue
628+
}
629+
counts[family]++
630+
out = append(out, m)
631+
}
632+
return out
633+
}
634+
635+
func modelFamilyKey(id string) string {
636+
provider, slug, ok := strings.Cut(id, "/")
637+
if !ok {
638+
return id
639+
}
640+
first := slug
641+
if i := strings.IndexAny(slug, "-:"); i >= 0 {
642+
first = slug[:i]
643+
}
644+
return provider + "/" + first
645+
}
646+
559647
func (s *Server) loadModelTelemetry(ctx context.Context, provider string) map[string]modelTelemetry {
560648
if s.usageStore == nil {
561649
return nil

‎internal/adminapi/model_overrides.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ func appendAllowedOverrideCandidates(models []uiModel, allCaps map[string]llm.Ca
126126
Free: c.Free(),
127127
Score: c.Score,
128128
Recommended: false,
129+
Section: "interesting",
129130
}
130131
if aaModels != nil {
131132
if info := llm.LookupAAInfo(modelID, aaModels); info != nil {

‎internal/adminapi/recommend.go‎

Lines changed: 83 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,69 @@ type rolePreset struct {
7575
// (1 - imageShare) * candidate.prompt + imageShare * multimodal_slot.prompt
7676
const imageShare = 0.10
7777

78-
// bestAgentic — use AA Agentic Index when available; fall back to Intelligence
79-
// Index for untested models (scaled down to de-rank vs. tested models).
80-
func bestAgentic(m uiModel) float64 {
78+
func roleQuality(m uiModel, role string) float64 {
79+
switch role {
80+
case "simple", "classifier":
81+
if m.TTFT > 0 {
82+
return inverseTTFT(m)
83+
}
84+
if m.SpeedTPS > 0 {
85+
return m.SpeedTPS / 100
86+
}
87+
return maxPositive(m.AgenticIndex, m.Score, m.CodingIndex)
88+
case "default", "complex":
89+
return maxPositive(m.AgenticIndex, m.CodingIndex, m.Score)
90+
case "multimodal", "compaction":
91+
return maxPositive(m.Score, m.CodingIndex)
92+
default:
93+
return maxPositive(m.AgenticIndex, m.Score, m.CodingIndex)
94+
}
95+
}
96+
97+
func roleQualityLabel(m uiModel, role string) string {
98+
switch role {
99+
case "simple", "classifier":
100+
if m.TTFT > 0 {
101+
return fmt.Sprintf("AA TTFT %.2fs", m.TTFT)
102+
}
103+
if m.SpeedTPS > 0 {
104+
return fmt.Sprintf("AA speed %.0f t/s", m.SpeedTPS)
105+
}
106+
case "default", "complex":
107+
if m.AgenticIndex > 0 && m.AgenticIndex >= m.CodingIndex && m.AgenticIndex >= m.Score {
108+
return fmt.Sprintf("AA agentic %.0f", m.AgenticIndex)
109+
}
110+
if m.CodingIndex > 0 && m.CodingIndex >= m.Score {
111+
return fmt.Sprintf("AA coding %.0f", m.CodingIndex)
112+
}
113+
case "multimodal", "compaction":
114+
if m.Score > 0 && m.Score >= m.CodingIndex {
115+
return fmt.Sprintf("AA intelligence %.0f", m.Score)
116+
}
117+
if m.CodingIndex > 0 {
118+
return fmt.Sprintf("AA coding %.0f", m.CodingIndex)
119+
}
120+
}
81121
if m.AgenticIndex > 0 {
82-
return m.AgenticIndex
122+
return fmt.Sprintf("AA agentic %.0f", m.AgenticIndex)
123+
}
124+
if m.Score > 0 {
125+
return fmt.Sprintf("AA intelligence %.0f", m.Score)
126+
}
127+
if m.CodingIndex > 0 {
128+
return fmt.Sprintf("AA coding %.0f", m.CodingIndex)
129+
}
130+
return ""
131+
}
132+
133+
func maxPositive(values ...float64) float64 {
134+
best := 0.0
135+
for _, v := range values {
136+
if v > best {
137+
best = v
138+
}
83139
}
84-
return m.Score
140+
return best
85141
}
86142

87143
// effectivePromptOf returns the cost-adjusted prompt price. If the model is
@@ -121,7 +177,7 @@ func inverseTTFT(m uiModel) float64 {
121177

122178
var rolePresets = map[string]rolePreset{
123179
"simple": {
124-
Description: "tools + multilingual, ≤ $0.2/M prompt, ctx ≥ 32k. Pareto frontier on (1/TTFT, blended cost) when speed data is available, else (AA Agentic Index, blended cost).",
180+
Description: "tools + multilingual, ≤ $0.2/M prompt, ctx ≥ 32k. Pareto frontier on role quality (TTFT, throughput, then AA quality) versus blended cost.",
125181
Filter: func(c llm.Capabilities, id string, aa llm.AAModelInfo) bool {
126182
return multilingualRegex.MatchString(id) &&
127183
!excludedVendorsRegex.MatchString(id) &&
@@ -138,7 +194,7 @@ var rolePresets = map[string]rolePreset{
138194
},
139195

140196
"default": {
141-
Description: "workhorse tools + multilingual, excludes L1/classifier-sized models, ≤ $2/M prompt, ctx ≥ 32k. Pareto frontier on (AA Agentic Index, blended cost).",
197+
Description: "workhorse tools + multilingual, excludes L1/classifier-sized models, ≤ $2/M prompt, ctx ≥ 32k. Pareto frontier on role quality (AA Agentic/Coding/Intelligence) versus blended cost.",
142198
Filter: func(c llm.Capabilities, id string, aa llm.AAModelInfo) bool {
143199
return multilingualRegex.MatchString(id) &&
144200
!excludedVendorsRegex.MatchString(id) &&
@@ -151,11 +207,11 @@ var rolePresets = map[string]rolePreset{
151207
c.ContextLength >= 32000 &&
152208
c.PromptPrice > 0 && c.PromptPrice <= 2.0
153209
},
154-
Axes: func(m uiModel) (float64, float64) { return bestAgentic(m), effectiveBlendedPriceOf(m) },
210+
Axes: func(m uiModel) (float64, float64) { return roleQuality(m, "default"), effectiveBlendedPriceOf(m) },
155211
},
156212

157213
"complex": {
158-
Description: "frontier reasoners (thinking/r1/qwq/grok) with tools + multilingual, ≤ $5/M prompt, ctx ≥ 64k. Pareto frontier on (AA Agentic Index, blended cost). Claude via bridge is preferred when configured.",
214+
Description: "frontier reasoners (thinking/r1/qwq/grok) with tools + multilingual, ≤ $5/M prompt, ctx ≥ 64k. Pareto frontier on role quality (AA Agentic/Coding/Intelligence) versus blended cost. Claude via bridge is preferred when configured.",
159215
Filter: func(c llm.Capabilities, id string, aa llm.AAModelInfo) bool {
160216
return multilingualRegex.MatchString(id) &&
161217
!excludedVendorsRegex.MatchString(id) &&
@@ -168,11 +224,11 @@ var rolePresets = map[string]rolePreset{
168224
c.ContextLength >= 64000 &&
169225
c.PromptPrice > 0 && c.PromptPrice <= 5.0
170226
},
171-
Axes: func(m uiModel) (float64, float64) { return bestAgentic(m), effectiveBlendedPriceOf(m) },
227+
Axes: func(m uiModel) (float64, float64) { return roleQuality(m, "complex"), effectiveBlendedPriceOf(m) },
172228
},
173229

174230
"multimodal": {
175-
Description: "vision + tools + multilingual, ≤ $2/M prompt, ctx ≥ 32k. Pareto frontier on (AA Intelligence Index, blended cost).",
231+
Description: "vision + tools + multilingual, ≤ $2/M prompt, ctx ≥ 32k. Pareto frontier on role quality (AA Intelligence/Coding) versus blended cost.",
176232
Filter: func(c llm.Capabilities, id string, aa llm.AAModelInfo) bool {
177233
return multilingualRegex.MatchString(id) &&
178234
!excludedVendorsRegex.MatchString(id) &&
@@ -182,11 +238,11 @@ var rolePresets = map[string]rolePreset{
182238
c.ContextLength >= 32000 &&
183239
c.PromptPrice > 0 && c.PromptPrice <= 2.0
184240
},
185-
Axes: func(m uiModel) (float64, float64) { return m.Score, effectiveBlendedPriceOf(m) },
241+
Axes: func(m uiModel) (float64, float64) { return roleQuality(m, "multimodal"), effectiveBlendedPriceOf(m) },
186242
},
187243

188244
"compaction": {
189-
Description: "multilingual, ctx ≥ 64k (long history in, short summary out), completion ≤ $2/M. Pareto frontier on (AA Intelligence Index, completion price).",
245+
Description: "multilingual, ctx ≥ 64k (long history in, short summary out), completion ≤ $2/M. Pareto frontier on role quality (AA Intelligence/Coding) versus completion price.",
190246
Filter: func(c llm.Capabilities, id string, aa llm.AAModelInfo) bool {
191247
return multilingualRegex.MatchString(id) &&
192248
!excludedVendorsRegex.MatchString(id) &&
@@ -197,11 +253,11 @@ var rolePresets = map[string]rolePreset{
197253
c.ContextLength >= 64000 &&
198254
c.CompletionPrice > 0 && c.CompletionPrice <= 2.0
199255
},
200-
Axes: func(m uiModel) (float64, float64) { return m.Score, m.CompletionPrice },
256+
Axes: func(m uiModel) (float64, float64) { return roleQuality(m, "compaction"), m.CompletionPrice },
201257
},
202258

203259
"classifier": {
204-
Description: "≤ $0.1/M prompt, multilingual, verified paid path for automatic recommendations. Free variants are shown separately until checked. Pareto frontier on (1/TTFT, prompt price) when speed data is available, else (AA Intelligence Index, prompt price). Local Ollama stays the primary recommendation.",
260+
Description: "≤ $0.1/M prompt, multilingual, verified paid path for automatic recommendations. Free variants are shown separately until checked. Pareto frontier on role quality (TTFT, throughput, then AA quality) versus prompt price. Local Ollama stays the primary recommendation.",
205261
Filter: func(c llm.Capabilities, id string, aa llm.AAModelInfo) bool {
206262
return multilingualRegex.MatchString(id) &&
207263
!excludedVendorsRegex.MatchString(id) &&
@@ -227,28 +283,16 @@ func isUnstableVariant(modelID string) bool {
227283
return unstableVariantRegex.MatchString(modelID)
228284
}
229285

230-
// classifierAxes picks (1/TTFT, price) when at least one candidate has TTFT
231-
// data, otherwise falls back to (Score, price). Mixing two quality scales in
232-
// the same Pareto frontier would be meaningless, so we pick one globally.
286+
// classifierAxes ranks tiny prompt-only models by latency first, then speed or
287+
// available AA quality when latency is absent.
233288
func classifierAxes(candidates []uiModel) paretoAxes {
234-
for _, m := range candidates {
235-
if m.TTFT > 0 {
236-
return func(m uiModel) (float64, float64) { return inverseTTFT(m), m.PromptPrice }
237-
}
238-
}
239-
return func(m uiModel) (float64, float64) { return m.Score, m.PromptPrice }
289+
return func(m uiModel) (float64, float64) { return roleQuality(m, "classifier"), m.PromptPrice }
240290
}
241291

242-
// simpleAxes keeps L1 recommendations distinct from default: when AA speed
243-
// data exists, startup latency is the quality axis. Without TTFT data, fall
244-
// back to agentic quality so new/untested models are not promoted blindly.
292+
// simpleAxes keeps L1 recommendations distinct from default: latency and
293+
// throughput are stronger signals than benchmark quality for this role.
245294
func simpleAxes(candidates []uiModel) paretoAxes {
246-
for _, m := range candidates {
247-
if m.TTFT > 0 {
248-
return func(m uiModel) (float64, float64) { return inverseTTFT(m), effectiveBlendedPriceOf(m) }
249-
}
250-
}
251-
return func(m uiModel) (float64, float64) { return bestAgentic(m), effectiveBlendedPriceOf(m) }
295+
return func(m uiModel) (float64, float64) { return roleQuality(m, "simple"), effectiveBlendedPriceOf(m) }
252296
}
253297

254298
func axesForPreset(role string, preset rolePreset, candidates []uiModel) paretoAxes {
@@ -326,6 +370,7 @@ func appendNearFrontierAlternatives(frontier, candidates []uiModel, axes paretoA
326370
continue
327371
}
328372
m.Recommended = false
373+
m.Section = "interesting"
329374
if p > 0 {
330375
m.ValuePerDollar = q / p
331376
}
@@ -368,6 +413,7 @@ func appendUntestedAlternatives(models, candidates []uiModel, axes paretoAxes, r
368413
continue
369414
}
370415
m.Recommended = false
416+
m.Section = "untested"
371417
annotateModelForRole(&m, role, "untested")
372418
untested = append(untested, m)
373419
}
@@ -450,6 +496,7 @@ func applyPreset(all map[string]llm.Capabilities, aaModels map[string]llm.AAMode
450496
frontier[i].ValuePerDollar = q / p
451497
}
452498
frontier[i].Recommended = true
499+
frontier[i].Section = "recommended"
453500
annotateModelForRole(&frontier[i], role, "preset")
454501
}
455502
return appendNearFrontierAlternatives(frontier, candidates, axes, role, 4)
@@ -486,12 +533,10 @@ func annotateModelForRole(m *uiModel, role, source string) {
486533
if m.ContextLength > 0 {
487534
reasons = append(reasons, fmt.Sprintf("ctx %s", shortContextLabel(m.ContextLength)))
488535
}
489-
if m.AgenticIndex > 0 {
490-
reasons = append(reasons, fmt.Sprintf("AA agentic %.0f", m.AgenticIndex))
491-
} else if m.Score > 0 {
492-
reasons = append(reasons, fmt.Sprintf("AA intelligence %.0f", m.Score))
536+
if label := roleQualityLabel(*m, role); label != "" {
537+
reasons = append(reasons, label)
493538
} else {
494-
warnings = append(warnings, "no AA benchmark data")
539+
warnings = append(warnings, "no role benchmark data")
495540
}
496541
if m.ValuePerDollar > 0 {
497542
reasons = append(reasons, fmt.Sprintf("value %.0f/$", m.ValuePerDollar))

‎internal/adminapi/recommend_inspection_test.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,23 +62,27 @@ func TestPresetInspection(t *testing.T) {
6262
vl.ID, 100*vQ/topQ, 100*vP/topP))
6363
}
6464

65-
lines = append(lines, fmt.Sprintf("%-52s %7s %7s %4s %6s %6s %6s %7s %8s", "model", "prompt$", "eff$", "V", "agent", "TTFT", "think", "markup", "value"))
65+
lines = append(lines, fmt.Sprintf("%-52s %7s %7s %4s %-18s %6s %6s %7s %8s", "model", "prompt$", "eff$", "V", "quality", "TTFT", "think", "markup", "value"))
6666
for _, m := range results {
6767
vis := " "
6868
if m.Vision {
6969
vis = "✓"
7070
}
71+
qualityLabel := roleQualityLabel(m, role)
72+
if qualityLabel == "" {
73+
qualityLabel = "-"
74+
}
7175
eff := m.PromptPrice
7276
if m.EffectivePrompt > 0 {
7377
eff = m.EffectivePrompt
7478
}
7579
lines = append(lines, fmt.Sprintf(
76-
"%-52s %7.3f %7.3f %4s %6.1f %6.2f %6.1f %+6.0f%% %8.0f",
80+
"%-52s %7.3f %7.3f %4s %-18s %6.2f %6.1f %+6.0f%% %8.0f",
7781
trunc(m.ID, 52),
7882
m.PromptPrice,
7983
eff,
8084
vis,
81-
m.AgenticIndex,
85+
trunc(qualityLabel, 18),
8286
m.TTFT,
8387
m.ThinkTime,
8488
m.MarkupPct,

0 commit comments

Comments
 (0)