Skip to content

Commit 9105976

Browse files
dzarlaxdzarlax
committed
add openrouter market ranking signals
1 parent 84af71e commit 9105976

7 files changed

Lines changed: 435 additions & 30 deletions

File tree

‎internal/adminapi/handlers.go‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ type uiModel struct {
5757
Reasons []string
5858
Warnings []string
5959
Telemetry modelTelemetry
60+
Market llm.OpenRouterMarketSignal
6061
}
6162

6263
type modelTelemetry struct {
@@ -330,6 +331,16 @@ func (s *Server) handleRefresh(w http.ResponseWriter, r *http.Request) {
330331
}
331332
}
332333
}
334+
if s.settings != nil {
335+
if market, marketErr := llm.FetchOpenRouterRankings(ctx); marketErr != nil {
336+
s.logger.Warn("openrouter rankings refresh failed", "err", marketErr)
337+
} else {
338+
s.logger.Info("openrouter rankings refreshed", "models", len(market))
339+
if storeErr := llm.StoreOpenRouterRankingsCache(ctx, s.settings, market); storeErr != nil {
340+
s.logger.Warn("openrouter rankings cache store failed", "err", storeErr)
341+
}
342+
}
343+
}
333344

334345
if s.capStore != nil {
335346
for id, c := range caps {
@@ -370,6 +381,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
370381
aaModels = cache.Models
371382
}
372383
}
384+
marketSignals := s.loadMarketSignals(ctx5, catalogProv)
373385
overrides := loadModelOverrides(ctx5, s.settings)
374386

375387
// Preset path — pre-filter + pre-sort via the role's preset. Checkbox
@@ -391,6 +403,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
391403
models := applyPreset(allCaps, aaModels, preset, visionFallbackPrompt)
392404
models = appendAllowedOverrideCandidates(models, allCaps, aaModels, catalogProv, preset, overrides)
393405
models = filterModelOverrides(models, catalogProv, overrides, true)
406+
attachMarketSignals(models, marketSignals)
394407
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
395408
sections := buildModelSections(models, true)
396409
filters := uiFilters{
@@ -436,7 +449,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
436449
}
437450
sortDir := q.Get("dir")
438451
if sortDir != "asc" && sortDir != "desc" {
439-
if sortCol == "score" || sortCol == "context" {
452+
if sortCol == "score" || sortCol == "context" || sortCol == "market" {
440453
sortDir = "desc"
441454
} else {
442455
sortDir = "asc"
@@ -501,6 +514,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
501514
models = append(models, m)
502515
}
503516
models = filterModelOverrides(models, catalogProv, overrides, f.Search == "")
517+
attachMarketSignals(models, marketSignals)
504518
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
505519
asc := sortDir == "asc"
506520
sort.Slice(models, func(i, j int) bool {
@@ -526,6 +540,8 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
526540
less = effectiveOrNominal(models[i]) < effectiveOrNominal(models[j])
527541
case "value":
528542
less = models[i].ValuePerDollar < models[j].ValuePerDollar
543+
case "market":
544+
less = marketSortValue(models[i].Market) < marketSortValue(models[j].Market)
529545
case "context":
530546
less = models[i].ContextLength < models[j].ContextLength
531547
case "id":
@@ -680,6 +696,60 @@ func attachModelTelemetry(models []uiModel, telemetry map[string]modelTelemetry)
680696
}
681697
}
682698

699+
func (s *Server) loadMarketSignals(ctx context.Context, provider string) map[string]llm.OpenRouterMarketSignal {
700+
if provider != "openrouter" || s.settings == nil {
701+
return nil
702+
}
703+
cache, err := llm.LoadOpenRouterRankingsCache(ctx, s.settings)
704+
if err != nil {
705+
s.logger.Warn("openrouter rankings cache load failed", "err", err)
706+
return nil
707+
}
708+
if cache == nil {
709+
return nil
710+
}
711+
return cache.Models
712+
}
713+
714+
func attachMarketSignals(models []uiModel, signals map[string]llm.OpenRouterMarketSignal) {
715+
if len(models) == 0 || len(signals) == 0 {
716+
return
717+
}
718+
for i := range models {
719+
sig, ok := signals[models[i].ID]
720+
if !ok {
721+
continue
722+
}
723+
models[i].Market = sig
724+
if label := marketSignalLabel(sig); label != "" {
725+
models[i].Reasons = uniqueStrings(append(models[i].Reasons, label))
726+
}
727+
}
728+
}
729+
730+
func marketSignalLabel(sig llm.OpenRouterMarketSignal) string {
731+
if sig.Rank > 0 {
732+
return fmt.Sprintf("OR market #%d", sig.Rank)
733+
}
734+
if sig.Share > 0 {
735+
return fmt.Sprintf("OR market %.1f%%", sig.Share)
736+
}
737+
if sig.Score > 0 {
738+
return "OR market signal"
739+
}
740+
return ""
741+
}
742+
743+
func marketSortValue(sig llm.OpenRouterMarketSignal) float64 {
744+
if sig.Rank > 0 {
745+
return 1_000_000 - float64(sig.Rank)
746+
}
747+
if sig.Share > 0 {
748+
return sig.Share
749+
}
750+
return sig.Score
751+
}
752+
683753
// enrichFromAA populates the AA-derived fields on a uiModel from the matched
684754
// AAModelInfo record. Used by both the preset path and the browse path so
685755
// they share one formula for Think time and Markup %.

‎internal/adminapi/templates/partials_models_browser.html‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@
113113
<th hx-get="/models" hx-target="#models-content" hx-swap="outerHTML" hx-include="#models-filters"
114114
hx-vals='{"sort":"think","dir":"{{if and (eq $s "think") (eq $d "asc")}}desc{{else}}asc{{end}}"}'
115115
style="cursor:pointer" title="TTFA - TTFT: seconds spent streaming thinking trace before the answer begins. 0 for non-reasoners.">Think{{if eq $s "think"}} {{if eq $d "asc"}}↑{{else}}↓{{end}}{{end}}</th>
116+
<th hx-get="/models" hx-target="#models-content" hx-swap="outerHTML" hx-include="#models-filters"
117+
hx-vals='{"sort":"market","dir":"{{if and (eq $s "market") (eq $d "desc")}}asc{{else}}desc{{end}}"}'
118+
style="cursor:pointer" title="OpenRouter rankings market signal. Advisory usage/popularity signal only; not used as a quality benchmark.">Market{{if eq $s "market"}} {{if eq $d "asc"}}↑{{else}}↓{{end}}{{end}}</th>
116119
<th hx-get="/models" hx-target="#models-content" hx-swap="outerHTML" hx-include="#models-filters"
117120
hx-vals='{"sort":"value","dir":"{{if and (eq $s "value") (eq $d "desc")}}asc{{else}}desc{{end}}"}'
118121
style="cursor:pointer" title="Quality per $ (higher = more bang per buck). Role-specific in Suggest mode (uses role's Pareto axes); generic agentic/prompt in browse mode.">Value{{if eq $s "value"}} {{if eq $d "asc"}}↑{{else}}↓{{end}}{{end}}</th>

‎internal/adminapi/templates/partials_models_row.html‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
{{range .ModelSections}}
44
<tbody class="models-section" data-section="{{.Key}}">
55
<tr class="models-section__header">
6-
<td colspan="14">
6+
<td colspan="15">
77
<strong>{{.Title}}</strong>
88
<span style="color:var(--text-secondary);margin-left:0.5rem;">{{len .Models}} models</span>
99
<div style="color:var(--text-secondary);font-size:0.82rem;margin-top:0.2rem;">{{.Description}}</div>
@@ -19,7 +19,7 @@
1919
{{range .Models}}
2020
{{template "model_row" dict "Model" . "Root" $}}
2121
{{else}}
22-
<tr><td colspan="14" style="text-align:center;padding:2rem;color:var(--text-secondary);">No models match the filters.</td></tr>
22+
<tr><td colspan="15" style="text-align:center;padding:2rem;color:var(--text-secondary);">No models match the filters.</td></tr>
2323
{{end}}
2424
</tbody>
2525
{{end}}
@@ -105,6 +105,7 @@
105105
<td class="price-cell">{{if gt .SpeedTPS 0.0}}{{printf "%.0f" .SpeedTPS}}{{else}}<span style="color:var(--text-secondary)">—</span>{{end}}</td>
106106
<td class="price-cell">{{if gt .TTFT 0.0}}{{printf "%.2f" .TTFT}}{{else}}<span style="color:var(--text-secondary)">—</span>{{end}}</td>
107107
<td class="price-cell" title="TTFA - TTFT: reasoner's thinking time before answer starts.">{{if gt .ThinkTime 0.0}}{{printf "%.1f" .ThinkTime}}{{else}}<span style="color:var(--text-secondary)">—</span>{{end}}</td>
108+
<td class="price-cell" title="OpenRouter rankings market signal. Advisory only; not used as quality.">{{if gt .Market.Rank 0}}#{{.Market.Rank}}{{else if gt .Market.Share 0.0}}{{printf "%.1f%%" .Market.Share}}{{else if gt .Market.Score 0.0}}✓{{else}}<span style="color:var(--text-secondary)">—</span>{{end}}</td>
108109
<td class="price-cell" title="Quality per $ — role-specific in Suggest, agentic/prompt otherwise.">{{if gt .ValuePerDollar 0.0}}{{printf "%.0f" .ValuePerDollar}}{{else}}<span style="color:var(--text-secondary)">—</span>{{end}}</td>
109110
</tr>
110111
{{end}}

‎internal/adminapi/templates/tg_admin.html‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,8 @@ <h2 id="sheet-title">Choose model</h2>
593593
model.tools ? "tools" : "",
594594
model.vision ? "vision" : "",
595595
model.reasoning ? "reasoning" : "",
596+
model.market && model.market.rank ? "market #" + model.market.rank : "",
597+
model.market && !model.market.rank && (model.market.share || model.market.score) ? "market" : "",
596598
model.agentic_index ? "agentic " + Math.round(model.agentic_index) : ""
597599
].filter(Boolean).forEach(function (v) {
598600
var chip = document.createElement("span");

‎internal/adminapi/tgadmin.go‎

Lines changed: 31 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -58,33 +58,34 @@ type tgAdminRoute struct {
5858
}
5959

6060
type tgAdminModel struct {
61-
ID string `json:"id"`
62-
Provider string `json:"provider"`
63-
PromptPrice float64 `json:"prompt_price"`
64-
CompletionPrice float64 `json:"completion_price"`
65-
ContextLength int `json:"context_length"`
66-
Vision bool `json:"vision"`
67-
Tools bool `json:"tools"`
68-
Reasoning bool `json:"reasoning"`
69-
Free bool `json:"free"`
70-
Score float64 `json:"score"`
71-
AgenticIndex float64 `json:"agentic_index"`
72-
SpeedTPS float64 `json:"speed_tps"`
73-
TTFT float64 `json:"ttft"`
74-
ValuePerDollar float64 `json:"value_per_dollar"`
75-
Recommended bool `json:"recommended"`
76-
Current bool `json:"current"`
77-
Source string `json:"source,omitempty"`
78-
Policy string `json:"policy,omitempty"`
79-
Section string `json:"section,omitempty"`
80-
OverrideNote string `json:"override_note,omitempty"`
81-
Reasons []string `json:"reasons,omitempty"`
82-
Warnings []string `json:"warnings,omitempty"`
83-
Telemetry modelTelemetry `json:"telemetry,omitempty"`
84-
CheckStatus string `json:"check_status,omitempty"`
85-
CheckedAt string `json:"checked_at,omitempty"`
86-
CheckLatencyMS int64 `json:"check_latency_ms,omitempty"`
87-
CheckError string `json:"check_error,omitempty"`
61+
ID string `json:"id"`
62+
Provider string `json:"provider"`
63+
PromptPrice float64 `json:"prompt_price"`
64+
CompletionPrice float64 `json:"completion_price"`
65+
ContextLength int `json:"context_length"`
66+
Vision bool `json:"vision"`
67+
Tools bool `json:"tools"`
68+
Reasoning bool `json:"reasoning"`
69+
Free bool `json:"free"`
70+
Score float64 `json:"score"`
71+
AgenticIndex float64 `json:"agentic_index"`
72+
SpeedTPS float64 `json:"speed_tps"`
73+
TTFT float64 `json:"ttft"`
74+
ValuePerDollar float64 `json:"value_per_dollar"`
75+
Recommended bool `json:"recommended"`
76+
Current bool `json:"current"`
77+
Source string `json:"source,omitempty"`
78+
Policy string `json:"policy,omitempty"`
79+
Section string `json:"section,omitempty"`
80+
OverrideNote string `json:"override_note,omitempty"`
81+
Reasons []string `json:"reasons,omitempty"`
82+
Warnings []string `json:"warnings,omitempty"`
83+
Telemetry modelTelemetry `json:"telemetry,omitempty"`
84+
Market llm.OpenRouterMarketSignal `json:"market,omitempty"`
85+
CheckStatus string `json:"check_status,omitempty"`
86+
CheckedAt string `json:"checked_at,omitempty"`
87+
CheckLatencyMS int64 `json:"check_latency_ms,omitempty"`
88+
CheckError string `json:"check_error,omitempty"`
8889
}
8990

9091
type tgAdminModelsResponse struct {
@@ -409,6 +410,7 @@ func (s *Server) buildTGAdminModels(ctx context.Context, role, provider, query s
409410
aaModels = cache.Models
410411
}
411412
}
413+
marketSignals := s.loadMarketSignals(ctx5, provider)
412414

413415
var models []uiModel
414416
recommendedMode := false
@@ -436,6 +438,7 @@ func (s *Server) buildTGAdminModels(ctx context.Context, role, provider, query s
436438
if len(models) == 0 {
437439
models = tgAdminBrowseModels(allCaps, aaModels, query, role)
438440
}
441+
attachMarketSignals(models, marketSignals)
439442
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, provider))
440443
models = filterModelOverrides(models, provider, overrides, true)
441444
models = filterBlockedFreeModels(models, provider, checks)
@@ -607,6 +610,7 @@ func tgModelFromUI(provider string, m uiModel, current string, checks map[string
607610
Reasons: m.Reasons,
608611
Warnings: m.Warnings,
609612
Telemetry: m.Telemetry,
613+
Market: m.Market,
610614
CheckStatus: check.Status,
611615
CheckedAt: check.CheckedAt,
612616
CheckLatencyMS: check.LatencyMS,

0 commit comments

Comments
 (0)