Add observable model operations UI

Author: dzarlax

internal/adminapi/adminapi_test.go

@@ -20,6 +20,7 @@ import (
 
 	"telegram-agent/internal/agent"
 	"telegram-agent/internal/config"
+	"telegram-agent/internal/evalpack"
 	"telegram-agent/internal/llm"
 	"telegram-agent/internal/store"
 )
@@ -161,7 +162,30 @@ func TestTemplatesParse(t *testing.T) {
 		viewIndex:         data,
 		viewRouting:       data.Routing, // routing view takes uiRouting directly
 		viewModelsBrowser: data,
-		viewTGAdmin:       tgAdminData{FullAdminURL: "https://assistant.example/admin"},
+		viewEvals: modelOpsData{
+			ActiveTab: "evals",
+			Checks: []modelCheckStatus{{
+				Status:    "free_verified",
+				Provider:  "openrouter",
+				ModelID:   "qwen/qwen3.5-flash:free",
+				CheckedAt: "2026-06-16T10:00:00Z",
+			}},
+			Evals: []modelEvalStatus{{
+				Status:     "passed",
+				Provider:   "openrouter",
+				ModelID:    "qwen/qwen3.5-flash:free",
+				CheckedAt:  "2026-06-16T10:00:00Z",
+				FinishedAt: "2026-06-16T10:00:00Z",
+				Passed:     1,
+				Results:    []evalpack.Result{{ID: "case-1", Category: "smoke", Passed: true}},
+			}},
+			Suite: evalpack.Suite{Version: 1, Name: "suite", Cases: []evalpack.Case{{
+				ID:       "case-1",
+				Category: "smoke",
+				Prompt:   "Say ok",
+			}}},
+		},
+		viewTGAdmin: tgAdminData{FullAdminURL: "https://assistant.example/admin"},
 	}
 	for v, d := range cases {
 		t.Run(v, func(t *testing.T) {
@@ -357,6 +381,121 @@ func TestModelEvalPaidGuard(t *testing.T) {
 	}
 }
 
+func TestEvalsPageRendersChecksReportsAndSuite(t *testing.T) {
+	data := modelOpsData{
+		ActiveTab: "evals",
+		Checks: []modelCheckStatus{{
+			Status:    "checking",
+			Provider:  "openrouter",
+			ModelID:   "qwen/qwen3.5-flash:free",
+			StartedAt: "2026-06-16T10:00:00Z",
+		}},
+		Evals: []modelEvalStatus{{
+			Status:     "failed",
+			Provider:   "openrouter",
+			ModelID:    "x-ai/grok-4.3",
+			Suite:      "personal-assistant-core-workloads",
+			StartedAt:  "2026-06-16T10:00:00Z",
+			FinishedAt: "2026-06-16T10:00:02Z",
+			Passed:     4,
+			Failed:     1,
+			Failures:   []string{"tool-web-fetch-intent: missing tool call"},
+			Results: []evalpack.Result{{
+				ID:             "tool-web-fetch-intent",
+				Category:       "web_fetch",
+				Passed:         false,
+				LatencyMS:      1200,
+				ContentPreview: "I cannot open links.",
+				Failures:       []string{"missing tool call \"web_fetch\""},
+			}},
+		}},
+		Suite: evalpack.Suite{Version: 1, Name: "personal-assistant-core-workloads", Cases: []evalpack.Case{{
+			ID:       "tool-web-fetch-intent",
+			Category: "web_fetch",
+			Prompt:   "Open https://example.com",
+			Expect:   evalpack.Expect{ToolCall: "web_fetch"},
+		}}},
+	}
+	var buf bytes.Buffer
+	if err := render(&buf, viewEvals, data); err != nil {
+		t.Fatal(err)
+	}
+	html := buf.String()
+	for _, want := range []string{"Model Operations", "checking", "x-ai/grok-4.3", "tool-web-fetch-intent", "Open https://example.com", "missing tool call"} {
+		if !strings.Contains(html, want) {
+			t.Fatalf("evals page missing %q: %s", want, html)
+		}
+	}
+}
+
+func TestModelChecksPersistCheckingBeforeProbe(t *testing.T) {
+	s, _ := newTGModelTestServer(t)
+	settings := &fakeSettingsStore{values: map[string]string{}}
+	s.settings = settings
+	s.modelProbe = func(ctx context.Context, provider, modelID string, _ llm.Capabilities) modelCheckStatus {
+		got := s.modelCheckStatus(ctx, provider, modelID)
+		if got.Status != "checking" || got.StartedAt == "" {
+			t.Fatalf("expected checking state before probe, got %+v", got)
+		}
+		return modelCheckStatus{Status: "free_verified", CheckedAt: time.Now().Format(time.RFC3339), LatencyMS: 22}
+	}
+
+	form := url.Values{}
+	form.Set("provider", "openrouter")
+	form.Set("model_id", "qwen/qwen3.5-flash:free")
+	req := httptest.NewRequest(http.MethodPost, "/models/check", strings.NewReader(form.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	rec := httptest.NewRecorder()
+	s.handleModelCheck(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, body=%s", rec.Code, rec.Body.String())
+	}
+	got := s.modelCheckStatus(context.Background(), "openrouter", "qwen/qwen3.5-flash:free")
+	if got.Status != "free_verified" || got.LatencyMS != 22 {
+		t.Fatalf("final check state not persisted: %+v", got)
+	}
+}
+
+func TestModelEvalLoadLegacyAndDetailedReports(t *testing.T) {
+	s := newTestServer(t)
+	legacy := map[string]modelEvalStatus{
+		modelCheckKey("openrouter", "legacy/model"): {
+			CheckedAt: "2026-06-16T10:00:00Z",
+			Passed:    4,
+			Failed:    1,
+			Failures:  []string{"legacy failure"},
+		},
+	}
+	data, err := json.Marshal(legacy)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s.settings = &fakeSettingsStore{values: map[string]string{settingKeyModelEvals: string(data)}}
+	evals := s.loadModelEvals(context.Background())
+	got := evals[modelCheckKey("openrouter", "legacy/model")]
+	if got.Status != "failed" || got.Provider != "openrouter" || got.ModelID != "legacy/model" {
+		t.Fatalf("legacy eval not normalized: %+v", got)
+	}
+
+	detailed := modelEvalStatus{
+		Status:    "passed",
+		Provider:  "openrouter",
+		ModelID:   "new/model",
+		Suite:     "suite",
+		StartedAt: "2026-06-16T10:01:00Z",
+		Passed:    1,
+		Results:   []evalpack.Result{{ID: "case-1", Category: "smoke", Passed: true, ContentPreview: "ok"}},
+	}
+	if err := s.saveModelEval(context.Background(), "openrouter", "new/model", detailed); err != nil {
+		t.Fatal(err)
+	}
+	evals = s.loadModelEvals(context.Background())
+	got = evals[modelCheckKey("openrouter", "new/model")]
+	if got.Status != "passed" || len(got.Results) != 1 || got.Results[0].ContentPreview != "ok" {
+		t.Fatalf("detailed eval not persisted: %+v", got)
+	}
+}
+
 func TestWebModelCheckPersistsFreeModelStatus(t *testing.T) {
 	s, _ := newTGModelTestServer(t)
 	settings := &fakeSettingsStore{values: map[string]string{}}

internal/adminapi/handlers.go

@@ -144,6 +144,17 @@ func (s *Server) handleAnalytics(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *Server) handleEvals(w http.ResponseWriter, r *http.Request) {
+	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
+	defer cancel()
+	data := s.buildModelOpsData(ctx)
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	if err := render(w, viewEvals, data); err != nil {
+		s.logger.Error("render evals", "err", err)
+		http.Error(w, "render error", http.StatusInternalServerError)
+	}
+}
+
 func (s *Server) handleModels(w http.ResponseWriter, r *http.Request) {
 	data := s.buildIndexData(r)
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
@@ -183,6 +194,15 @@ func (s *Server) handleModelCheck(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "only free OpenRouter models can be checked here", http.StatusBadRequest)
 		return
 	}
+	if !s.modelEvalMu.TryLock() {
+		http.Error(w, "another model operation is already running", http.StatusTooManyRequests)
+		return
+	}
+	defer s.modelEvalMu.Unlock()
+	if err := s.saveModelCheck(r.Context(), provider, modelID, checkingModelCheck(provider, modelID)); err != nil {
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+		return
+	}
 
 	caps := s.lookupCapsFor(r.Context(), provider, modelID)
 	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
@@ -842,7 +862,7 @@ func attachModelChecks(models []uiModel, provider string, checks map[string]mode
 		models[i].CheckedAt = check.CheckedAt
 		models[i].CheckLatencyMS = check.LatencyMS
 		models[i].CheckError = check.Error
-		if check.Status != "free_unverified" {
+		if check.Status != "free_unverified" && check.Status != "checking" {
 			models[i].Policy = check.Status
 			models[i].Warnings = withoutWarning(models[i].Warnings, "free model: validate availability before routing")
 		}

internal/adminapi/model_checks.go

@@ -29,7 +29,11 @@ const (
 
 type modelCheckStatus struct {
 	Status    string `json:"status"`
+	Provider  string `json:"provider,omitempty"`
+	ModelID   string `json:"model_id,omitempty"`
+	StartedAt string `json:"started_at,omitempty"`
 	CheckedAt string `json:"checked_at,omitempty"`
+	UpdatedAt string `json:"updated_at,omitempty"`
 	LatencyMS int64  `json:"latency_ms,omitempty"`
 	Error     string `json:"error,omitempty"`
 }
@@ -106,14 +110,25 @@ func (s *Server) saveModelCheck(ctx context.Context, provider, modelID string, s
 	if checks == nil {
 		checks = map[string]modelCheckStatus{}
 	}
-	checks[modelCheckKey(provider, modelID)] = status
+	checks[modelCheckKey(provider, modelID)] = normalizeSingleModelCheck(provider, modelID, status)
 	data, err := json.Marshal(checks)
 	if err != nil {
 		return err
 	}
 	return s.settings.PutSetting(ctx, settingKeyModelChecks, string(data))
 }
 
+func checkingModelCheck(provider, modelID string) modelCheckStatus {
+	now := time.Now().Format(time.RFC3339)
+	return modelCheckStatus{
+		Status:    "checking",
+		Provider:  provider,
+		ModelID:   modelID,
+		StartedAt: now,
+		UpdatedAt: now,
+	}
+}
+
 func (s *Server) modelCheckInterval() time.Duration {
 	return time.Duration(s.intSetting(SettingKeyModelCheckIntervalHours, defaultModelCheckIntervalHours, 1, 168)) * time.Hour
 }
@@ -221,11 +236,32 @@ func isBlockedModelCheckError(message string) bool {
 
 func normalizeModelChecks(checks map[string]modelCheckStatus) {
 	for key, check := range checks {
-		if check.Status == "free_degraded" && isBlockedModelCheckError(check.Error) {
-			check.Status = "free_blocked"
-			checks[key] = check
+		provider, modelID, _ := strings.Cut(key, "|")
+		checks[key] = normalizeSingleModelCheck(provider, modelID, check)
+	}
+}
+
+func normalizeSingleModelCheck(provider, modelID string, check modelCheckStatus) modelCheckStatus {
+	if check.Provider == "" {
+		check.Provider = provider
+	}
+	if check.ModelID == "" {
+		check.ModelID = modelID
+	}
+	if check.Status == "" && check.CheckedAt != "" {
+		if check.Error != "" {
+			check.Status = "free_degraded"
+		} else {
+			check.Status = "free_verified"
 		}
 	}
+	if check.Status == "free_degraded" && isBlockedModelCheckError(check.Error) {
+		check.Status = "free_blocked"
+	}
+	if check.UpdatedAt == "" {
+		check.UpdatedAt = check.CheckedAt
+	}
+	return check
 }
 
 func (s *Server) startModelCheckScheduler() {