Skip to content

Commit f86f1af

Browse files
dzarlaxdzarlax
authored
Add model metadata and free checks to catalog
Summary: - cache OpenRouter model names and descriptions in capability stores - switch the default web model catalog to compact rows - add web free-model checks and block unverified free model assignment Tests: - go test -count=1 ./internal/llm - go test -count=1 ./internal/store - go test -count=1 ./internal/adminapi - go test -count=1 ./... - go build ./cmd/agent
1 parent d0360c7 commit f86f1af

14 files changed

Lines changed: 371 additions & 58 deletions

‎internal/adminapi/adminapi_test.go‎

Lines changed: 132 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,8 @@ func TestModelsBrowserRendersCardAndTableModes(t *testing.T) {
183183
Slots: []uiSlot{{Name: "default-or", ModelID: "qwen/qwen3.5-plus"}},
184184
Models: []uiModel{{
185185
ID: "qwen/qwen3.5-plus",
186+
Name: "Qwen 3.5 Plus",
187+
Description: "Balanced multilingual model for tool-heavy assistant workflows.",
186188
PromptPrice: 1.0,
187189
CompletionPrice: 3.0,
188190
ContextLength: 128000,
@@ -193,19 +195,22 @@ func TestModelsBrowserRendersCardAndTableModes(t *testing.T) {
193195
CatalogProvider: "openrouter",
194196
}
195197

196-
t.Run("cards default", func(t *testing.T) {
198+
t.Run("compact default", func(t *testing.T) {
197199
data := base
198-
data.Filters.View = "cards"
200+
data.Filters.View = "compact"
199201
var buf bytes.Buffer
200202
if err := render(&buf, viewModelsBrowser, data); err != nil {
201203
t.Fatal(err)
202204
}
203205
html := buf.String()
204-
if !strings.Contains(html, "catalog-grid") || !strings.Contains(html, "catalog-card") {
205-
t.Fatalf("card view missing catalog cards: %s", html)
206+
if !strings.Contains(html, "catalog-list") || !strings.Contains(html, "catalog-row") {
207+
t.Fatalf("compact view missing catalog rows: %s", html)
208+
}
209+
if !strings.Contains(html, "Balanced multilingual model") {
210+
t.Fatalf("compact view missing model description: %s", html)
206211
}
207212
if strings.Contains(html, "catalog-audit-table") {
208-
t.Fatalf("card view should not render audit table")
213+
t.Fatalf("compact view should not render audit table")
209214
}
210215
})
211216

@@ -220,14 +225,14 @@ func TestModelsBrowserRendersCardAndTableModes(t *testing.T) {
220225
if !strings.Contains(html, "catalog-audit-table") {
221226
t.Fatalf("table view missing audit table: %s", html)
222227
}
223-
if strings.Contains(html, "catalog-grid") || strings.Contains(html, "catalog-card") {
224-
t.Fatalf("table view should not render card layout")
228+
if strings.Contains(html, "catalog-list") || strings.Contains(html, "catalog-row") {
229+
t.Fatalf("table view should not render compact layout")
225230
}
226231
})
227232

228233
t.Run("preset state is preserved except clear", func(t *testing.T) {
229234
data := base
230-
data.Filters.View = "cards"
235+
data.Filters.View = "compact"
231236
data.Filters.ActivePreset = "default"
232237
var buf bytes.Buffer
233238
if err := render(&buf, viewModelsBrowser, data); err != nil {
@@ -237,12 +242,130 @@ func TestModelsBrowserRendersCardAndTableModes(t *testing.T) {
237242
if !strings.Contains(html, `name="preset" value="default"`) {
238243
t.Fatalf("active preset should be submitted with filters and view toggle: %s", html)
239244
}
240-
if !strings.Contains(html, `hx-get="/models?provider=openrouter&view=cards&full=1"`) {
245+
if !strings.Contains(html, `hx-get="/models?provider=openrouter&view=compact&full=1"`) {
241246
t.Fatalf("clear preset should refresh browser without preset: %s", html)
242247
}
243248
})
244249
}
245250

251+
func TestModelsBrowserRendersFreeCheckAction(t *testing.T) {
252+
data := indexData{
253+
Slots: []uiSlot{{Name: "default-or", ModelID: "qwen/qwen3.5-plus"}},
254+
Models: []uiModel{{
255+
ID: "qwen/qwen3.5-flash:free",
256+
Description: "Free endpoint for quick routing checks.",
257+
PromptPrice: 0,
258+
CompletionPrice: 0,
259+
ContextLength: 64000,
260+
Tools: true,
261+
Free: true,
262+
Policy: "free_degraded",
263+
CheckStatus: "free_degraded",
264+
CheckLatencyMS: 321,
265+
StatusLabel: "Untested",
266+
PolicyLabel: "Free degraded",
267+
PrimaryReason: "Free endpoint degraded",
268+
}},
269+
Filters: uiFilters{View: "compact"},
270+
CatalogProvider: "openrouter",
271+
}
272+
var buf bytes.Buffer
273+
if err := render(&buf, viewModelsBrowser, data); err != nil {
274+
t.Fatal(err)
275+
}
276+
html := buf.String()
277+
if !strings.Contains(html, `hx-post="/models/check"`) || !strings.Contains(html, "Check free") {
278+
t.Fatalf("free check action missing: %s", html)
279+
}
280+
if !strings.Contains(html, "Free model must pass check before routing") {
281+
t.Fatalf("unverified free model should not render active assign buttons: %s", html)
282+
}
283+
if !strings.Contains(html, "free_degraded") || !strings.Contains(html, "321 ms") {
284+
t.Fatalf("check status not rendered: %s", html)
285+
}
286+
}
287+
288+
func TestWebModelCheckPersistsFreeModelStatus(t *testing.T) {
289+
s, _ := newTGModelTestServer(t)
290+
settings := &fakeSettingsStore{values: map[string]string{}}
291+
s.settings = settings
292+
s.modelProbe = func(context.Context, string, string, llm.Capabilities) modelCheckStatus {
293+
return modelCheckStatus{
294+
Status: "free_verified",
295+
CheckedAt: "2026-06-15T12:00:00Z",
296+
LatencyMS: 88,
297+
}
298+
}
299+
300+
form := url.Values{}
301+
form.Set("provider", "openrouter")
302+
form.Set("model_id", "qwen/qwen3.5-flash:free")
303+
form.Set("view", "compact")
304+
req := httptest.NewRequest(http.MethodPost, "/models/check", strings.NewReader(form.Encode()))
305+
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
306+
rec := httptest.NewRecorder()
307+
s.handleModelCheck(rec, req)
308+
309+
if rec.Code != http.StatusOK {
310+
t.Fatalf("status = %d, want %d, body: %s", rec.Code, http.StatusOK, rec.Body.String())
311+
}
312+
checks := s.loadModelChecks(context.Background())
313+
got := checks[modelCheckKey("openrouter", "qwen/qwen3.5-flash:free")]
314+
if got.Status != "free_verified" || got.LatencyMS != 88 {
315+
t.Fatalf("status not persisted: %+v", checks)
316+
}
317+
if !strings.Contains(rec.Body.String(), "free_verified") {
318+
t.Fatalf("response should render updated check status: %s", rec.Body.String())
319+
}
320+
}
321+
322+
func TestWebSlotAssignRejectsUnverifiedFreeModel(t *testing.T) {
323+
s, provider := newTGModelTestServer(t)
324+
s.settings = &fakeSettingsStore{values: map[string]string{}}
325+
326+
form := url.Values{}
327+
form.Set("provider", "openrouter")
328+
form.Set("model_id", "qwen/qwen3.5-flash:free")
329+
req := httptest.NewRequest(http.MethodPost, "/slots/default-or/assign", strings.NewReader(form.Encode()))
330+
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
331+
rec := httptest.NewRecorder()
332+
s.handleSlotAssign(rec, req)
333+
334+
if rec.Code != http.StatusBadRequest {
335+
t.Fatalf("status = %d, want %d, body: %s", rec.Code, http.StatusBadRequest, rec.Body.String())
336+
}
337+
if provider.CurrentModel() == "qwen/qwen3.5-flash:free" {
338+
t.Fatalf("unverified free model should not be assigned")
339+
}
340+
}
341+
342+
func TestWebSlotAssignAllowsVerifiedFreeModel(t *testing.T) {
343+
s, provider := newTGModelTestServer(t)
344+
checks := map[string]modelCheckStatus{
345+
modelCheckKey("openrouter", "qwen/qwen3.5-flash:free"): {Status: "free_verified", CheckedAt: time.Now().Format(time.RFC3339)},
346+
}
347+
data, err := json.Marshal(checks)
348+
if err != nil {
349+
t.Fatal(err)
350+
}
351+
s.settings = &fakeSettingsStore{values: map[string]string{settingKeyModelChecks: string(data)}}
352+
353+
form := url.Values{}
354+
form.Set("provider", "openrouter")
355+
form.Set("model_id", "qwen/qwen3.5-flash:free")
356+
req := httptest.NewRequest(http.MethodPost, "/slots/default-or/assign", strings.NewReader(form.Encode()))
357+
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
358+
rec := httptest.NewRecorder()
359+
s.handleSlotAssign(rec, req)
360+
361+
if rec.Code != http.StatusOK {
362+
t.Fatalf("status = %d, want %d, body: %s", rec.Code, http.StatusOK, rec.Body.String())
363+
}
364+
if provider.CurrentModel() != "qwen/qwen3.5-flash:free" {
365+
t.Fatalf("model = %q, want free model", provider.CurrentModel())
366+
}
367+
}
368+
246369
func signedTGInitData(t *testing.T, token string, userID int64, authTime time.Time) string {
247370
t.Helper()
248371
userJSON, err := json.Marshal(tgAdminUser{ID: userID, FirstName: "Alex"})

‎internal/adminapi/handlers.go‎

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ type uiSlot struct {
3232

3333
type uiModel struct {
3434
ID string
35+
Name string
36+
Description string
3537
PromptPrice float64
3638
CompletionPrice float64
3739
ContextLength int
@@ -58,6 +60,10 @@ type uiModel struct {
5860
Warnings []string
5961
Telemetry modelTelemetry
6062
Market llm.OpenRouterMarketSignal
63+
CheckStatus string
64+
CheckedAt string
65+
CheckLatencyMS int64
66+
CheckError string
6167
StatusLabel string
6268
PolicyLabel string
6369
PrimaryReason string
@@ -94,7 +100,7 @@ type uiFilters struct {
94100
ValueLeaderHint string // e.g. "85% quality @ 30% price" (preset path only)
95101
Sort string // active sort column: "prompt", "completion", "score", "context", "id"
96102
SortDir string // "asc" or "desc"
97-
View string // "cards" (default) or "table"
103+
View string // "compact" (default) or "table"
98104
}
99105

100106
type uiModelSection struct {
@@ -148,6 +154,52 @@ func (s *Server) handleModels(w http.ResponseWriter, r *http.Request) {
148154
}
149155
}
150156

157+
func (s *Server) handleModelCheck(w http.ResponseWriter, r *http.Request) {
158+
if r.Method != http.MethodPost {
159+
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
160+
return
161+
}
162+
if s.settings == nil {
163+
http.Error(w, "settings store not available", http.StatusServiceUnavailable)
164+
return
165+
}
166+
if err := r.ParseForm(); err != nil {
167+
http.Error(w, "parse form", http.StatusBadRequest)
168+
return
169+
}
170+
provider := strings.TrimSpace(r.FormValue("provider"))
171+
if provider == "" {
172+
provider = "openrouter"
173+
}
174+
modelID := strings.TrimSpace(r.FormValue("model_id"))
175+
if modelID == "" {
176+
http.Error(w, "model_id required", http.StatusBadRequest)
177+
return
178+
}
179+
if provider != "openrouter" || !isFreeVariant(modelID) {
180+
http.Error(w, "only free OpenRouter models can be checked here", http.StatusBadRequest)
181+
return
182+
}
183+
184+
caps := s.lookupCapsFor(r.Context(), provider, modelID)
185+
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
186+
defer cancel()
187+
status := s.probeModel(ctx, provider, modelID, caps)
188+
if err := s.saveModelCheck(ctx, provider, modelID, status); err != nil {
189+
http.Error(w, err.Error(), http.StatusServiceUnavailable)
190+
return
191+
}
192+
193+
next := r.Clone(r.Context())
194+
next.URL = cloneURL(r.URL)
195+
next.URL.RawQuery = r.Form.Encode()
196+
data := s.buildIndexData(next)
197+
w.Header().Set("Content-Type", "text/html; charset=utf-8")
198+
if err := render(w, viewModelsContent, data); err != nil {
199+
s.logger.Error("render models after check", "err", err)
200+
}
201+
}
202+
151203
func (s *Server) handleModelOverride(w http.ResponseWriter, r *http.Request) {
152204
if r.Method != http.MethodPost {
153205
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
@@ -239,6 +291,13 @@ func (s *Server) handleSlotAssign(w http.ResponseWriter, r *http.Request) {
239291
providerType = mc.Provider
240292
}
241293
}
294+
if providerType == "openrouter" && isFreeVariant(modelID) {
295+
check := s.modelCheckStatus(r.Context(), providerType, modelID)
296+
if check.Status != "free_verified" {
297+
http.Error(w, "free model must pass check before routing", http.StatusBadRequest)
298+
return
299+
}
300+
}
242301
caps := s.lookupCapsFor(r.Context(), providerType, modelID)
243302
if err := s.router.SetProviderModel(slot, providerType, modelID, caps); err != nil {
244303
s.logger.Warn("slot assign failed", "slot", slot, "model", modelID, "err", err)
@@ -372,7 +431,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
372431
}
373432
viewMode := q.Get("view")
374433
if viewMode != "table" {
375-
viewMode = "cards"
434+
viewMode = "compact"
376435
}
377436

378437
ctx5, cancel5 := context.WithTimeout(r.Context(), 5*time.Second)
@@ -414,6 +473,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
414473
models = filterModelOverrides(models, catalogProv, overrides, true)
415474
attachMarketSignals(models, marketSignals)
416475
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
476+
attachModelChecks(models, catalogProv, s.loadModelChecks(ctx5))
417477
decorateModelDisplay(models, preset)
418478
sections := buildModelSections(models, true)
419479
filters := uiFilters{
@@ -497,6 +557,8 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
497557
}
498558
m := uiModel{
499559
ID: id,
560+
Name: c.Name,
561+
Description: c.Description,
500562
PromptPrice: c.PromptPrice,
501563
CompletionPrice: c.CompletionPrice,
502564
ContextLength: c.ContextLength,
@@ -528,6 +590,7 @@ func (s *Server) buildIndexData(r *http.Request) indexData {
528590
models = filterModelOverrides(models, catalogProv, overrides, f.Search == "")
529591
attachMarketSignals(models, marketSignals)
530592
attachModelTelemetry(models, s.loadModelTelemetry(ctx5, catalogProv))
593+
attachModelChecks(models, catalogProv, s.loadModelChecks(ctx5))
531594
decorateModelDisplay(models, "")
532595
asc := sortDir == "asc"
533596
sort.Slice(models, func(i, j int) bool {
@@ -709,6 +772,43 @@ func attachModelTelemetry(models []uiModel, telemetry map[string]modelTelemetry)
709772
}
710773
}
711774

775+
func attachModelChecks(models []uiModel, provider string, checks map[string]modelCheckStatus) {
776+
if len(models) == 0 || len(checks) == 0 {
777+
return
778+
}
779+
for i := range models {
780+
if !models[i].Free {
781+
continue
782+
}
783+
check := checks[modelCheckKey(provider, models[i].ID)]
784+
if check.Status == "" {
785+
continue
786+
}
787+
models[i].CheckStatus = check.Status
788+
models[i].CheckedAt = check.CheckedAt
789+
models[i].CheckLatencyMS = check.LatencyMS
790+
models[i].CheckError = check.Error
791+
if check.Status != "free_unverified" {
792+
models[i].Policy = check.Status
793+
models[i].Warnings = withoutWarning(models[i].Warnings, "free model: validate availability before routing")
794+
}
795+
if check.Error != "" {
796+
models[i].Warnings = append(models[i].Warnings, check.Error)
797+
}
798+
}
799+
}
800+
801+
func withoutWarning(warnings []string, remove string) []string {
802+
out := warnings[:0]
803+
for _, warning := range warnings {
804+
if warning == remove {
805+
continue
806+
}
807+
out = append(out, warning)
808+
}
809+
return out
810+
}
811+
712812
func (s *Server) loadMarketSignals(ctx context.Context, provider string) map[string]llm.OpenRouterMarketSignal {
713813
if provider != "openrouter" || s.settings == nil {
714814
return nil

‎internal/adminapi/model_overrides.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ func appendAllowedOverrideCandidates(models []uiModel, allCaps map[string]llm.Ca
117117
}
118118
m := uiModel{
119119
ID: modelID,
120+
Name: c.Name,
121+
Description: c.Description,
120122
PromptPrice: c.PromptPrice,
121123
CompletionPrice: c.CompletionPrice,
122124
ContextLength: c.ContextLength,

‎internal/adminapi/recommend.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,8 @@ func applyPreset(all map[string]llm.Capabilities, aaModels map[string]llm.AAMode
459459
}
460460
m := uiModel{
461461
ID: id,
462+
Name: c.Name,
463+
Description: c.Description,
462464
PromptPrice: c.PromptPrice,
463465
CompletionPrice: c.CompletionPrice,
464466
ContextLength: c.ContextLength,

‎internal/adminapi/server.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ func (s *Server) registerRoutes(mux *http.ServeMux) {
142142
authed := s.requireAuth
143143
mux.Handle("/", authed(http.HandlerFunc(s.handleIndex)))
144144
mux.Handle("/models/override", authed(http.HandlerFunc(s.handleModelOverride)))
145+
mux.Handle("/models/check", authed(http.HandlerFunc(s.handleModelCheck)))
145146
mux.Handle("/models", authed(http.HandlerFunc(s.handleModels)))
146147
mux.Handle("/routing", authed(http.HandlerFunc(s.handleRouting)))
147148
mux.Handle("/slots/", authed(http.HandlerFunc(s.handleSlotAssign))) // POST /slots/{slot}/assign

0 commit comments

Comments
 (0)