Skip to content

Commit 0f0742b

Browse files
dzarlaxdzarlax
committed
Expose free model check controls
1 parent 6b05946 commit 0f0742b

5 files changed

Lines changed: 271 additions & 13 deletions

File tree

‎internal/adminapi/adminapi_test.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,72 @@ func TestFreeModelCheckCandidatesSkipFreshChecks(t *testing.T) {
420420
}
421421
}
422422

423+
func TestSettingSpecsIncludeModelCheckControls(t *testing.T) {
424+
s := newTestServer(t)
425+
keys := map[string]bool{}
426+
for _, spec := range s.settingSpecs() {
427+
keys[spec.Key] = true
428+
}
429+
for _, key := range []string{
430+
SettingKeyModelCheckIntervalHours,
431+
SettingKeyModelCheckInitialDelayMins,
432+
SettingKeyModelCheckStaleHours,
433+
SettingKeyModelCheckBatchLimit,
434+
} {
435+
if !keys[key] {
436+
t.Fatalf("setting %s is not exposed", key)
437+
}
438+
}
439+
}
440+
441+
func TestTGAdminSummaryIncludesModelCheckCounts(t *testing.T) {
442+
s, _ := newTGModelTestServer(t)
443+
checks := map[string]modelCheckStatus{
444+
modelCheckKey("openrouter", "verified:free"): {Status: "free_verified", CheckedAt: time.Now().Format(time.RFC3339)},
445+
modelCheckKey("openrouter", "blocked:free"): {Status: "free_blocked", CheckedAt: time.Now().Format(time.RFC3339)},
446+
}
447+
data, err := json.Marshal(checks)
448+
if err != nil {
449+
t.Fatal(err)
450+
}
451+
s.settings = &fakeSettingsStore{values: map[string]string{settingKeyModelChecks: string(data)}}
452+
453+
summary := s.buildTGAdminSummary(context.Background())
454+
if summary.ModelChecks.Total != 2 || summary.ModelChecks.Verified != 1 || summary.ModelChecks.Blocked != 1 {
455+
t.Fatalf("unexpected model check summary: %+v", summary.ModelChecks)
456+
}
457+
}
458+
459+
func TestTGAdminRunModelChecksRunsSweep(t *testing.T) {
460+
s, _ := newTGModelTestServer(t)
461+
s.cfgRef.Telegram.BotToken = "123:abc"
462+
s.cfgRef.Telegram.OwnerChatID = 42
463+
s.settings = &fakeSettingsStore{values: map[string]string{}}
464+
s.modelProbe = func(context.Context, string, string, llm.Capabilities) modelCheckStatus {
465+
return modelCheckStatus{Status: "free_verified", CheckedAt: time.Now().Format(time.RFC3339), LatencyMS: 10}
466+
}
467+
468+
req := httptest.NewRequest(http.MethodPost, "/tg-admin/api/model-checks/run", nil)
469+
req.Header.Set("X-Telegram-Init-Data", signedTGInitData(t, "123:abc", 42, time.Now()))
470+
rec := httptest.NewRecorder()
471+
s.handleTGAdminRouter(rec, req)
472+
473+
if rec.Code != http.StatusOK {
474+
t.Fatalf("status = %d, want %d, body: %s", rec.Code, http.StatusOK, rec.Body.String())
475+
}
476+
var payload modelCheckSweepResult
477+
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
478+
t.Fatal(err)
479+
}
480+
if payload.Checked != 1 || payload.Statuses["free_verified"] != 1 {
481+
t.Fatalf("unexpected sweep result: %+v", payload)
482+
}
483+
check := s.modelCheckStatus(context.Background(), "openrouter", "qwen/qwen3.5-flash:free")
484+
if check.Status != "free_verified" {
485+
t.Fatalf("check not persisted: %+v", check)
486+
}
487+
}
488+
423489
func TestTGAdminModelSetUpdatesCurrentRoleSlot(t *testing.T) {
424490
s, provider := newTGModelTestServer(t)
425491
s.cfgRef.Telegram.BotToken = "123:abc"

‎internal/adminapi/handlers.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1021,6 +1021,34 @@ func (s *Server) settingSpecs() []settingSpec {
10211021
Default: boolDefault(s.cfgRef.AdminAPI.TrustForwardAuth),
10221022
InputType: "text",
10231023
},
1024+
{
1025+
Key: SettingKeyModelCheckIntervalHours,
1026+
Label: "Free model check interval (hours)",
1027+
Description: "How often the background scheduler checks stale free OpenRouter models. Applies on the next scheduler cycle.",
1028+
Default: fmt.Sprintf("%d", defaultModelCheckIntervalHours),
1029+
InputType: "number",
1030+
},
1031+
{
1032+
Key: SettingKeyModelCheckInitialDelayMins,
1033+
Label: "Free model check initial delay (minutes)",
1034+
Description: "Delay after admin API start before the first background free-model sweep.",
1035+
Default: fmt.Sprintf("%d", defaultModelCheckInitialDelayMins),
1036+
InputType: "number",
1037+
},
1038+
{
1039+
Key: SettingKeyModelCheckStaleHours,
1040+
Label: "Free model check TTL (hours)",
1041+
Description: "A free-model check older than this is considered stale and eligible for recheck.",
1042+
Default: fmt.Sprintf("%d", defaultModelCheckStaleHours),
1043+
InputType: "number",
1044+
},
1045+
{
1046+
Key: SettingKeyModelCheckBatchLimit,
1047+
Label: "Free model check batch limit",
1048+
Description: "Maximum number of stale free models to check per scheduler run.",
1049+
Default: fmt.Sprintf("%d", defaultModelCheckBatchLimit),
1050+
InputType: "number",
1051+
},
10241052
}
10251053
if s.settings != nil {
10261054
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)

‎internal/adminapi/model_checks.go‎

Lines changed: 127 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,17 @@ import (
1414
const settingKeyModelChecks = "recommendation.model_checks"
1515

1616
const (
17-
modelCheckInterval = 12 * time.Hour
18-
modelCheckInitialDelay = 5 * time.Minute
19-
modelCheckStaleAfter = 24 * time.Hour
20-
modelCheckBatchLimit = 5
17+
SettingKeyModelCheckIntervalHours = "recommendation.model_check_interval_hours"
18+
SettingKeyModelCheckInitialDelayMins = "recommendation.model_check_initial_delay_minutes"
19+
SettingKeyModelCheckStaleHours = "recommendation.model_check_stale_hours"
20+
SettingKeyModelCheckBatchLimit = "recommendation.model_check_batch_limit"
21+
)
22+
23+
const (
24+
defaultModelCheckIntervalHours = 12
25+
defaultModelCheckInitialDelayMins = 5
26+
defaultModelCheckStaleHours = 24
27+
defaultModelCheckBatchLimit = 5
2128
)
2229

2330
type modelCheckStatus struct {
@@ -39,6 +46,30 @@ type modelCheckResponse struct {
3946
Status modelCheckStatus `json:"status"`
4047
}
4148

49+
type modelCheckSummary struct {
50+
Total int `json:"total"`
51+
Verified int `json:"verified"`
52+
Degraded int `json:"degraded"`
53+
Blocked int `json:"blocked"`
54+
Unverified int `json:"unverified"`
55+
OldestCheck string `json:"oldest_check,omitempty"`
56+
NewestCheck string `json:"newest_check,omitempty"`
57+
}
58+
59+
type modelCheckSweepResult struct {
60+
Checked int `json:"checked"`
61+
Remaining int `json:"remaining"`
62+
Statuses map[string]int `json:"statuses"`
63+
Models []modelCheckModel `json:"models,omitempty"`
64+
}
65+
66+
type modelCheckModel struct {
67+
ModelID string `json:"model_id"`
68+
Status string `json:"status"`
69+
LatencyMS int64 `json:"latency_ms,omitempty"`
70+
Error string `json:"error,omitempty"`
71+
}
72+
4273
type modelProbeFunc func(ctx context.Context, providerType, modelID string, caps llm.Capabilities) modelCheckStatus
4374

4475
func modelCheckKey(provider, modelID string) string {
@@ -82,6 +113,38 @@ func (s *Server) saveModelCheck(ctx context.Context, provider, modelID string, s
82113
return s.settings.PutSetting(ctx, settingKeyModelChecks, string(data))
83114
}
84115

116+
func (s *Server) modelCheckInterval() time.Duration {
117+
return time.Duration(s.intSetting(SettingKeyModelCheckIntervalHours, defaultModelCheckIntervalHours, 1, 168)) * time.Hour
118+
}
119+
120+
func (s *Server) modelCheckInitialDelay() time.Duration {
121+
return time.Duration(s.intSetting(SettingKeyModelCheckInitialDelayMins, defaultModelCheckInitialDelayMins, 0, 1440)) * time.Minute
122+
}
123+
124+
func (s *Server) modelCheckStaleAfter() time.Duration {
125+
return time.Duration(s.intSetting(SettingKeyModelCheckStaleHours, defaultModelCheckStaleHours, 1, 720)) * time.Hour
126+
}
127+
128+
func (s *Server) modelCheckBatchLimit() int {
129+
return s.intSetting(SettingKeyModelCheckBatchLimit, defaultModelCheckBatchLimit, 1, 50)
130+
}
131+
132+
func (s *Server) intSetting(key string, def, min, max int) int {
133+
if s.settings == nil {
134+
return def
135+
}
136+
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
137+
defer cancel()
138+
v := llm.GetIntSetting(ctx, s.settings, key, def)
139+
if v < min {
140+
return min
141+
}
142+
if v > max {
143+
return max
144+
}
145+
return v
146+
}
147+
85148
func (s *Server) probeModel(ctx context.Context, providerType, modelID string, caps llm.Capabilities) modelCheckStatus {
86149
if s.modelProbe != nil {
87150
return s.modelProbe(ctx, providerType, modelID, caps)
@@ -149,34 +212,41 @@ func (s *Server) startModelCheckScheduler() {
149212
}
150213

151214
func (s *Server) modelCheckScheduler(ctx context.Context) {
152-
timer := time.NewTimer(modelCheckInitialDelay)
215+
timer := time.NewTimer(s.modelCheckInitialDelay())
153216
defer timer.Stop()
154217
for {
155218
select {
156219
case <-ctx.Done():
157220
return
158221
case <-timer.C:
159222
s.runModelCheckSweep(ctx)
160-
timer.Reset(modelCheckInterval)
223+
timer.Reset(s.modelCheckInterval())
161224
}
162225
}
163226
}
164227

165-
func (s *Server) runModelCheckSweep(ctx context.Context) {
228+
func (s *Server) runModelCheckSweep(ctx context.Context) modelCheckSweepResult {
229+
return s.runModelCheckSweepProvider(ctx, "openrouter")
230+
}
231+
232+
func (s *Server) runModelCheckSweepProvider(ctx context.Context, provider string) modelCheckSweepResult {
166233
sweepCtx, cancel := context.WithTimeout(ctx, 3*time.Minute)
167234
defer cancel()
168-
candidates := s.freeModelCheckCandidates(sweepCtx, "openrouter")
235+
candidates := s.freeModelCheckCandidates(sweepCtx, provider)
236+
result := modelCheckSweepResult{Statuses: map[string]int{}}
169237
if len(candidates) == 0 {
170-
return
238+
return result
171239
}
172-
if len(candidates) > modelCheckBatchLimit {
173-
candidates = candidates[:modelCheckBatchLimit]
240+
limit := s.modelCheckBatchLimit()
241+
if len(candidates) > limit {
242+
result.Remaining = len(candidates) - limit
243+
candidates = candidates[:limit]
174244
}
175245
s.logger.Info("free model check sweep started", "count", len(candidates))
176246
for _, candidate := range candidates {
177247
select {
178248
case <-sweepCtx.Done():
179-
return
249+
return result
180250
default:
181251
}
182252
checkCtx, checkCancel := context.WithTimeout(sweepCtx, 30*time.Second)
@@ -186,8 +256,17 @@ func (s *Server) runModelCheckSweep(ctx context.Context) {
186256
s.logger.Warn("free model check save failed", "model", candidate.modelID, "err", err)
187257
continue
188258
}
259+
result.Checked++
260+
result.Statuses[status.Status]++
261+
result.Models = append(result.Models, modelCheckModel{
262+
ModelID: candidate.modelID,
263+
Status: status.Status,
264+
LatencyMS: status.LatencyMS,
265+
Error: status.Error,
266+
})
189267
s.logger.Info("free model checked", "model", candidate.modelID, "status", status.Status, "latency_ms", status.LatencyMS)
190268
}
269+
return result
191270
}
192271

193272
type freeModelCandidate struct {
@@ -212,7 +291,7 @@ func (s *Server) freeModelCheckCandidates(ctx context.Context, provider string)
212291
}
213292
check := checks[modelCheckKey(provider, id)]
214293
checkedAt := parseCheckTime(check.CheckedAt)
215-
if !checkedAt.IsZero() && now.Sub(checkedAt) < modelCheckStaleAfter {
294+
if !checkedAt.IsZero() && now.Sub(checkedAt) < s.modelCheckStaleAfter() {
216295
continue
217296
}
218297
out = append(out, freeModelCandidate{
@@ -234,6 +313,41 @@ func (s *Server) freeModelCheckCandidates(ctx context.Context, provider string)
234313
return out
235314
}
236315

316+
func (s *Server) buildModelCheckSummary(ctx context.Context) modelCheckSummary {
317+
checks := s.loadModelChecks(ctx)
318+
out := modelCheckSummary{Total: len(checks)}
319+
var oldest, newest time.Time
320+
for _, check := range checks {
321+
switch check.Status {
322+
case "free_verified":
323+
out.Verified++
324+
case "free_degraded":
325+
out.Degraded++
326+
case "free_blocked":
327+
out.Blocked++
328+
default:
329+
out.Unverified++
330+
}
331+
checkedAt := parseCheckTime(check.CheckedAt)
332+
if checkedAt.IsZero() {
333+
continue
334+
}
335+
if oldest.IsZero() || checkedAt.Before(oldest) {
336+
oldest = checkedAt
337+
}
338+
if newest.IsZero() || checkedAt.After(newest) {
339+
newest = checkedAt
340+
}
341+
}
342+
if !oldest.IsZero() {
343+
out.OldestCheck = oldest.Format(time.RFC3339)
344+
}
345+
if !newest.IsZero() {
346+
out.NewestCheck = newest.Format(time.RFC3339)
347+
}
348+
return out
349+
}
350+
237351
func parseCheckTime(raw string) time.Time {
238352
if raw == "" {
239353
return time.Time{}

‎internal/adminapi/templates/tg_admin.html‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,8 +372,13 @@ <h2>Operations</h2>
372372
<button type="button" class="ghost" id="show-tools">Tools</button>
373373
<button type="button" class="secondary" id="compact-history">Compact</button>
374374
<button type="button" class="secondary" id="reload-mcp">Reload MCP</button>
375+
<button type="button" class="secondary" id="run-model-checks">Check models</button>
375376
<button type="button" class="danger" id="clear-context">Clear context</button>
376377
</div>
378+
<div class="panel-row">
379+
<div class="label">Free model checks</div>
380+
<div class="value" id="model-checks">-</div>
381+
</div>
377382
<div class="panel-list" id="stats-panel"></div>
378383
<div class="panel-list" id="tools-panel"></div>
379384
</article>
@@ -726,6 +731,12 @@ <h2 id="sheet-title">Choose model</h2>
726731
text("cost-24h", money(summary.usage && summary.usage.cost_24h));
727732
text("tokens-24h", int(summary.usage && summary.usage.tokens_24h));
728733
text("calls-7d", int(summary.usage && summary.usage.calls_7d));
734+
var checks = summary.model_checks || {};
735+
text("model-checks",
736+
int(checks.verified) + " verified · " +
737+
int(checks.degraded) + " degraded · " +
738+
int(checks.blocked) + " blocked · " +
739+
int(checks.unverified) + " unverified");
729740
renderRouting(summary.routing);
730741
renderMCP(summary.mcp);
731742
setHealth("online", "ok");
@@ -834,6 +845,25 @@ <h2 id="sheet-title">Choose model</h2>
834845
});
835846
}
836847

848+
function runModelChecks() {
849+
if (state.busy) return;
850+
confirmAction("Run stale free model checks now?", function () {
851+
state.busy = true;
852+
text("ops-state", "Checking models");
853+
request("/tg-admin/api/model-checks/run", { method: "POST" })
854+
.then(function (payload) {
855+
text("ops-state", "Checked " + int(payload.checked) + " models");
856+
return load();
857+
})
858+
.catch(function (err) {
859+
text("ops-state", err.message);
860+
})
861+
.finally(function () {
862+
state.busy = false;
863+
});
864+
});
865+
}
866+
837867
byId("refresh").addEventListener("click", load);
838868
byId("show-stats").addEventListener("click", loadStats);
839869
byId("show-tools").addEventListener("click", loadTools);
@@ -843,6 +873,7 @@ <h2 id="sheet-title">Choose model</h2>
843873
byId("reload-mcp").addEventListener("click", function () {
844874
runOperation("mcp_reload", "Reload MCP servers now?");
845875
});
876+
byId("run-model-checks").addEventListener("click", runModelChecks);
846877
byId("clear-context").addEventListener("click", function () {
847878
runOperation("clear", "Clear the current conversation context? This cannot be undone.");
848879
});

0 commit comments

Comments
 (0)