Skip to content

Commit 8be9018

Browse files
committed
Add manual model recommendation overrides
1 parent 57f402d commit 8be9018

6 files changed

Lines changed: 302 additions & 0 deletions

File tree

internal/adminapi/adminapi_test.go

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,72 @@ func TestTGAdminRecommendedModelsHideBlockedFreeCandidates(t *testing.T) {
467467
}
468468
}
469469

470+
func TestTGAdminModelsHideManualDeniedModels(t *testing.T) {
471+
s, _ := newTGModelTestServer(t)
472+
s.cfgRef.Telegram.BotToken = "123:abc"
473+
s.cfgRef.Telegram.OwnerChatID = 42
474+
overrides := map[string]modelOverride{
475+
modelCheckKey("openrouter", "qwen/qwen3.5-plus"): {State: "deny", Note: "too costly"},
476+
}
477+
data, err := json.Marshal(overrides)
478+
if err != nil {
479+
t.Fatal(err)
480+
}
481+
s.settings = &fakeSettingsStore{values: map[string]string{settingKeyModelOverrides: string(data)}}
482+
483+
req := httptest.NewRequest(http.MethodGet, "/tg-admin/api/models?role=default&q=plus", nil)
484+
req.Header.Set("X-Telegram-Init-Data", signedTGInitData(t, "123:abc", 42, time.Now()))
485+
rec := httptest.NewRecorder()
486+
s.handleTGAdminRouter(rec, req)
487+
488+
if rec.Code != http.StatusOK {
489+
t.Fatalf("status = %d, want %d, body: %s", rec.Code, http.StatusOK, rec.Body.String())
490+
}
491+
var payload tgAdminModelsResponse
492+
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
493+
t.Fatal(err)
494+
}
495+
if len(payload.Models) != 0 {
496+
t.Fatalf("manual denied model should be hidden: %+v", payload.Models)
497+
}
498+
}
499+
500+
func TestTGAdminRecommendedModelsIncludeManualAllowedModel(t *testing.T) {
501+
s, _ := newTGModelTestServer(t)
502+
s.cfgRef.Telegram.BotToken = "123:abc"
503+
s.cfgRef.Telegram.OwnerChatID = 42
504+
overrides := map[string]modelOverride{
505+
modelCheckKey("openrouter", "qwen/qwen3.5-plus"): {State: "allow", Note: "known good"},
506+
}
507+
data, err := json.Marshal(overrides)
508+
if err != nil {
509+
t.Fatal(err)
510+
}
511+
s.settings = &fakeSettingsStore{values: map[string]string{settingKeyModelOverrides: string(data)}}
512+
513+
req := httptest.NewRequest(http.MethodGet, "/tg-admin/api/models?role=simple", nil)
514+
req.Header.Set("X-Telegram-Init-Data", signedTGInitData(t, "123:abc", 42, time.Now()))
515+
rec := httptest.NewRecorder()
516+
s.handleTGAdminRouter(rec, req)
517+
518+
if rec.Code != http.StatusOK {
519+
t.Fatalf("status = %d, want %d, body: %s", rec.Code, http.StatusOK, rec.Body.String())
520+
}
521+
var payload tgAdminModelsResponse
522+
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
523+
t.Fatal(err)
524+
}
525+
for _, model := range payload.Models {
526+
if model.ID == "qwen/qwen3.5-plus" {
527+
if model.Policy != "manual_allow" || model.OverrideNote != "known good" {
528+
t.Fatalf("unexpected manual allow model: %+v", model)
529+
}
530+
return
531+
}
532+
}
533+
t.Fatalf("manual allowed model missing: %+v", payload.Models)
534+
}
535+
470536
func TestTGAdminModelCheckPersistsFreeModelStatus(t *testing.T) {
471537
s, _ := newTGModelTestServer(t)
472538
s.cfgRef.Telegram.BotToken = "123:abc"

internal/adminapi/handlers.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"net/http"
7+
"net/url"
78
"os"
89
"sort"
910
"strings"
@@ -51,6 +52,7 @@ type uiModel struct {
5152
Recommended bool
5253
Source string
5354
Policy string
55+
OverrideNote string
5456
Reasons []string
5557
Warnings []string
5658
Telemetry modelTelemetry
@@ -131,6 +133,56 @@ func (s *Server) handleModels(w http.ResponseWriter, r *http.Request) {
131133
}
132134
}
133135

136+
func (s *Server) handleModelOverride(w http.ResponseWriter, r *http.Request) {
137+
if r.Method != http.MethodPost {
138+
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
139+
return
140+
}
141+
if s.settings == nil {
142+
http.Error(w, "settings store not available", http.StatusServiceUnavailable)
143+
return
144+
}
145+
if err := r.ParseForm(); err != nil {
146+
http.Error(w, "parse form", http.StatusBadRequest)
147+
return
148+
}
149+
provider := strings.TrimSpace(r.FormValue("provider"))
150+
if provider == "" {
151+
provider = "openrouter"
152+
}
153+
modelID := strings.TrimSpace(r.FormValue("model_id"))
154+
if modelID == "" {
155+
http.Error(w, "model_id required", http.StatusBadRequest)
156+
return
157+
}
158+
state := strings.TrimSpace(r.FormValue("state"))
159+
if state != "" && state != "allow" && state != "deny" {
160+
http.Error(w, "invalid state", http.StatusBadRequest)
161+
return
162+
}
163+
note := strings.TrimSpace(r.FormValue("note"))
164+
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
165+
defer cancel()
166+
if err := saveModelOverride(ctx, s.settings, provider, modelID, state, note); err != nil {
167+
http.Error(w, err.Error(), http.StatusServiceUnavailable)
168+
return
169+
}
170+
171+
next := r.Clone(r.Context())
172+
next.URL = cloneURL(r.URL)
173+
next.URL.RawQuery = r.Form.Encode()
174+
data := s.buildIndexData(next)
175+
w.Header().Set("Content-Type", "text/html; charset=utf-8")
176+
if err := render(w, viewModelsContent, data); err != nil {
177+
s.logger.Error("render models after override", "err", err)
178+
}
179+
}
180+
181+
func cloneURL(u *url.URL) *url.URL {
182+
v := *u
183+
return &v
184+
}
185+
134186
func (s *Server) handleRouting(w http.ResponseWriter, r *http.Request) {
135187
data := s.buildRouting()
136188
w.Header().Set("Content-Type", "text/html; charset=utf-8")
@@ -309,6 +361,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
309361
aaModels = cache.Models
310362
}
311363
}
364+
overrides := loadModelOverrides(ctx5, s.settings)
312365

313366
// Preset path — pre-filter + pre-sort via the role's preset. Checkbox
314367
// filters are ignored on this path: the preset is a complete override.
@@ -327,6 +380,8 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
327380
}
328381
}
329382
models := applyPreset(allCaps, aaModels, preset, visionFallbackPrompt)
383+
models = appendAllowedOverrideCandidates(models, allCaps, aaModels, catalogProv, preset, overrides)
384+
models = filterModelOverrides(models, catalogProv, overrides, true)
330385
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
331386
filters := uiFilters{
332387
ActivePreset: preset,
@@ -437,6 +492,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
437492
annotateModelForRole(&m, "", "catalog")
438493
models = append(models, m)
439494
}
495+
models = filterModelOverrides(models, catalogProv, overrides, f.Search == "")
440496
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
441497
asc := sortDir == "asc"
442498
sort.Slice(models, func(i, j int) bool {
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
package adminapi
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
8+
"telegram-agent/internal/llm"
9+
)
10+
11+
const settingKeyModelOverrides = "recommendation.overrides"
12+
13+
type modelOverride struct {
14+
State string `json:"state"`
15+
Note string `json:"note,omitempty"`
16+
}
17+
18+
func loadModelOverrides(ctx context.Context, settings llm.SettingsStore) map[string]modelOverride {
19+
if settings == nil {
20+
return nil
21+
}
22+
raw, ok, err := settings.GetSetting(ctx, settingKeyModelOverrides)
23+
if err != nil || !ok || raw == "" {
24+
return nil
25+
}
26+
out := map[string]modelOverride{}
27+
if err := json.Unmarshal([]byte(raw), &out); err != nil {
28+
return nil
29+
}
30+
return out
31+
}
32+
33+
func saveModelOverride(ctx context.Context, settings llm.SettingsStore, provider, modelID, state, note string) error {
34+
if settings == nil {
35+
return fmt.Errorf("settings store unavailable")
36+
}
37+
overrides := loadModelOverrides(ctx, settings)
38+
if overrides == nil {
39+
overrides = map[string]modelOverride{}
40+
}
41+
key := modelCheckKey(provider, modelID)
42+
if state == "" {
43+
delete(overrides, key)
44+
} else {
45+
overrides[key] = modelOverride{State: state, Note: note}
46+
}
47+
data, err := json.Marshal(overrides)
48+
if err != nil {
49+
return err
50+
}
51+
return settings.PutSetting(ctx, settingKeyModelOverrides, string(data))
52+
}
53+
54+
func applyModelOverride(provider string, m *uiModel, overrides map[string]modelOverride) bool {
55+
if len(overrides) == 0 {
56+
return true
57+
}
58+
override := overrides[modelCheckKey(provider, m.ID)]
59+
switch override.State {
60+
case "deny":
61+
m.Policy = "manual_deny"
62+
m.OverrideNote = override.Note
63+
m.Warnings = append(m.Warnings, "manual deny")
64+
if override.Note != "" {
65+
m.Warnings = append(m.Warnings, override.Note)
66+
}
67+
case "allow":
68+
m.Policy = "manual_allow"
69+
m.OverrideNote = override.Note
70+
m.Source = "manual"
71+
m.Reasons = append(m.Reasons, "manual allow")
72+
if override.Note != "" {
73+
m.Reasons = append(m.Reasons, override.Note)
74+
}
75+
}
76+
return true
77+
}
78+
79+
func filterModelOverrides(models []uiModel, provider string, overrides map[string]modelOverride, hideDenied bool) []uiModel {
80+
if len(models) == 0 || len(overrides) == 0 {
81+
return models
82+
}
83+
out := models[:0]
84+
for _, m := range models {
85+
applyModelOverride(provider, &m, overrides)
86+
if hideDenied && m.Policy == "manual_deny" {
87+
continue
88+
}
89+
out = append(out, m)
90+
}
91+
return out
92+
}
93+
94+
func appendAllowedOverrideCandidates(models []uiModel, allCaps map[string]llm.Capabilities, aaModels map[string]llm.AAModelInfo, provider, role string, overrides map[string]modelOverride) []uiModel {
95+
if len(overrides) == 0 {
96+
return models
97+
}
98+
seen := make(map[string]bool, len(models))
99+
for _, m := range models {
100+
seen[m.ID] = true
101+
}
102+
for id, override := range overrides {
103+
if override.State != "allow" {
104+
continue
105+
}
106+
prefix := provider + "|"
107+
if len(id) <= len(prefix) || id[:len(prefix)] != prefix {
108+
continue
109+
}
110+
modelID := id[len(prefix):]
111+
if seen[modelID] {
112+
continue
113+
}
114+
c, ok := allCaps[modelID]
115+
if !ok {
116+
continue
117+
}
118+
m := uiModel{
119+
ID: modelID,
120+
PromptPrice: c.PromptPrice,
121+
CompletionPrice: c.CompletionPrice,
122+
ContextLength: c.ContextLength,
123+
Vision: c.Vision,
124+
Tools: c.Tools,
125+
Reasoning: c.Reasoning,
126+
Free: c.Free(),
127+
Score: c.Score,
128+
Recommended: false,
129+
}
130+
if aaModels != nil {
131+
if info := llm.LookupAAInfo(modelID, aaModels); info != nil {
132+
enrichFromAA(&m, *info)
133+
}
134+
}
135+
if m.PromptPrice > 0 {
136+
q := m.AgenticIndex
137+
if q == 0 {
138+
q = m.Score
139+
}
140+
if q > 0 {
141+
m.ValuePerDollar = q / m.PromptPrice
142+
}
143+
}
144+
annotateModelForRole(&m, role, "manual")
145+
applyModelOverride(provider, &m, overrides)
146+
models = append(models, m)
147+
seen[modelID] = true
148+
}
149+
return models
150+
}

internal/adminapi/server.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ func (s *Server) registerRoutes(mux *http.ServeMux) {
141141
// Authenticated routes.
142142
authed := s.requireAuth
143143
mux.Handle("/", authed(http.HandlerFunc(s.handleIndex)))
144+
mux.Handle("/models/override", authed(http.HandlerFunc(s.handleModelOverride)))
144145
mux.Handle("/models", authed(http.HandlerFunc(s.handleModels)))
145146
mux.Handle("/routing", authed(http.HandlerFunc(s.handleRouting)))
146147
mux.Handle("/slots/", authed(http.HandlerFunc(s.handleSlotAssign))) // POST /slots/{slot}/assign

internal/adminapi/templates/partials_models_row.html

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,30 @@
2323
· {{printf "$%.4f" .Telemetry.CostUSD}}
2424
</div>
2525
{{end}}
26+
<div style="display:flex;gap:0.35rem;align-items:center;margin-top:0.45rem;flex-wrap:wrap;">
27+
<input class="form-input" name="note" type="text" value="{{.OverrideNote}}" placeholder="Override note"
28+
style="width:12rem;min-width:9rem;font-size:0.8rem;">
29+
<button class="btn btn--small btn--ghost"
30+
hx-post="/models/override"
31+
hx-include="#models-filters, closest tr"
32+
hx-vals='{"model_id":"{{$modelID}}","provider":"{{$.CatalogProvider}}","state":"allow"}'
33+
hx-target="#models-content"
34+
hx-swap="outerHTML">Allow</button>
35+
<button class="btn btn--small btn--ghost"
36+
hx-post="/models/override"
37+
hx-include="#models-filters, closest tr"
38+
hx-vals='{"model_id":"{{$modelID}}","provider":"{{$.CatalogProvider}}","state":"deny"}'
39+
hx-target="#models-content"
40+
hx-swap="outerHTML">Deny</button>
41+
{{if or (eq .Policy "manual_allow") (eq .Policy "manual_deny")}}
42+
<button class="btn btn--small btn--ghost"
43+
hx-post="/models/override"
44+
hx-include="#models-filters, closest tr"
45+
hx-vals='{"model_id":"{{$modelID}}","provider":"{{$.CatalogProvider}}","state":""}'
46+
hx-target="#models-content"
47+
hx-swap="outerHTML">Clear</button>
48+
{{end}}
49+
</div>
2650
<div class="assign-buttons">
2751
{{$prov := $.CatalogProvider}}
2852
{{range $.Slots}}

internal/adminapi/tgadmin.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ type tgAdminModel struct {
7676
Current bool `json:"current"`
7777
Source string `json:"source,omitempty"`
7878
Policy string `json:"policy,omitempty"`
79+
OverrideNote string `json:"override_note,omitempty"`
7980
Reasons []string `json:"reasons,omitempty"`
8081
Warnings []string `json:"warnings,omitempty"`
8182
Telemetry modelTelemetry `json:"telemetry,omitempty"`
@@ -411,6 +412,7 @@ func (s *Server) buildTGAdminModels(ctx context.Context, role, provider, query s
411412
var models []uiModel
412413
recommendedMode := false
413414
checks := s.loadModelChecks(ctx5)
415+
overrides := loadModelOverrides(ctx5, s.settings)
414416
if query == "" && provider == "openrouter" {
415417
var visionFallbackPrompt float64
416418
cfg := s.router.GetConfig()
@@ -427,12 +429,14 @@ func (s *Server) buildTGAdminModels(ctx context.Context, role, provider, query s
427429
resp.Description = p.Description
428430
}
429431
models = appendFreeCandidates(models, tgAdminBrowseModels(allCaps, aaModels, ":free", role), role)
432+
models = appendAllowedOverrideCandidates(models, allCaps, aaModels, provider, role, overrides)
430433
recommendedMode = hasRecommendedModel(models)
431434
}
432435
if len(models) == 0 {
433436
models = tgAdminBrowseModels(allCaps, aaModels, query, role)
434437
}
435438
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, provider))
439+
models = filterModelOverrides(models, provider, overrides, true)
436440
models = filterBlockedFreeModels(models, provider, checks)
437441
resp.Recommended = recommendedMode
438442
if len(models) > limit {
@@ -596,6 +600,7 @@ func tgModelFromUI(provider string, m uiModel, current string, checks map[string
596600
Current: m.ID == current,
597601
Source: m.Source,
598602
Policy: m.Policy,
603+
OverrideNote: m.OverrideNote,
599604
Reasons: m.Reasons,
600605
Warnings: m.Warnings,
601606
Telemetry: m.Telemetry,

0 commit comments

Comments
 (0)