|
| 1 | +package adminapi |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "encoding/json" |
| 6 | + "fmt" |
| 7 | + "net/http" |
| 8 | + "strings" |
| 9 | + "time" |
| 10 | + |
| 11 | + "telegram-agent/internal/evalpack" |
| 12 | + "telegram-agent/internal/llm" |
| 13 | +) |
| 14 | + |
| 15 | +const ( |
| 16 | + settingKeyModelEvals = "recommendation.model_evals" |
| 17 | + modelEvalDatasetPath = "evals/workload.json" |
| 18 | + modelEvalTimeout = 45 * time.Second |
| 19 | +) |
| 20 | + |
| 21 | +type modelEvalStatus struct { |
| 22 | + CheckedAt string `json:"checked_at,omitempty"` |
| 23 | + Passed int `json:"passed"` |
| 24 | + Failed int `json:"failed"` |
| 25 | + DurationMS int64 `json:"duration_ms"` |
| 26 | + Error string `json:"error,omitempty"` |
| 27 | + Failures []string `json:"failures,omitempty"` |
| 28 | +} |
| 29 | + |
| 30 | +func (s *Server) handleModelEval(w http.ResponseWriter, r *http.Request) { |
| 31 | + if r.Method != http.MethodPost { |
| 32 | + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) |
| 33 | + return |
| 34 | + } |
| 35 | + if s.settings == nil { |
| 36 | + http.Error(w, "settings store not available", http.StatusServiceUnavailable) |
| 37 | + return |
| 38 | + } |
| 39 | + if err := r.ParseForm(); err != nil { |
| 40 | + http.Error(w, "parse form", http.StatusBadRequest) |
| 41 | + return |
| 42 | + } |
| 43 | + provider := strings.TrimSpace(r.FormValue("provider")) |
| 44 | + if provider == "" { |
| 45 | + provider = "openrouter" |
| 46 | + } |
| 47 | + modelID := strings.TrimSpace(r.FormValue("model_id")) |
| 48 | + if modelID == "" { |
| 49 | + http.Error(w, "model_id required", http.StatusBadRequest) |
| 50 | + return |
| 51 | + } |
| 52 | + paid := isPaidModelEval(provider, modelID) |
| 53 | + if paid && r.FormValue("confirm_paid_eval") != "1" { |
| 54 | + http.Error(w, "paid/cloud eval requires confirmation", http.StatusBadRequest) |
| 55 | + return |
| 56 | + } |
| 57 | + |
| 58 | + if !s.modelEvalMu.TryLock() { |
| 59 | + http.Error(w, "another model eval is already running", http.StatusTooManyRequests) |
| 60 | + return |
| 61 | + } |
| 62 | + defer s.modelEvalMu.Unlock() |
| 63 | + |
| 64 | + caps := s.lookupCapsFor(r.Context(), provider, modelID) |
| 65 | + ctx, cancel := context.WithTimeout(r.Context(), modelEvalTimeout) |
| 66 | + defer cancel() |
| 67 | + status := s.runModelEval(ctx, provider, modelID, caps) |
| 68 | + if err := s.saveModelEval(ctx, provider, modelID, status); err != nil { |
| 69 | + http.Error(w, err.Error(), http.StatusServiceUnavailable) |
| 70 | + return |
| 71 | + } |
| 72 | + |
| 73 | + next := r.Clone(r.Context()) |
| 74 | + next.URL = cloneURL(r.URL) |
| 75 | + next.URL.RawQuery = r.Form.Encode() |
| 76 | + data := s.buildIndexData(next) |
| 77 | + w.Header().Set("Content-Type", "text/html; charset=utf-8") |
| 78 | + if err := render(w, viewModelsContent, data); err != nil { |
| 79 | + s.logger.Error("render models after eval", "err", err) |
| 80 | + } |
| 81 | +} |
| 82 | + |
| 83 | +func isPaidModelEval(provider, modelID string) bool { |
| 84 | + provider = strings.ToLower(strings.TrimSpace(provider)) |
| 85 | + switch provider { |
| 86 | + case "local", "ollama": |
| 87 | + return false |
| 88 | + case "openrouter": |
| 89 | + return !isFreeVariant(modelID) |
| 90 | + default: |
| 91 | + return true |
| 92 | + } |
| 93 | +} |
| 94 | + |
| 95 | +func (s *Server) runModelEval(ctx context.Context, providerType, modelID string, caps llm.Capabilities) modelEvalStatus { |
| 96 | + if s.cfgRef == nil { |
| 97 | + return failedModelEval("missing runtime config") |
| 98 | + } |
| 99 | + suite, err := evalpack.LoadSuite(modelEvalDatasetPath) |
| 100 | + if err != nil { |
| 101 | + return failedModelEval(err.Error()) |
| 102 | + } |
| 103 | + factories := llm.BuildBackendFactories(s.cfgRef) |
| 104 | + factory := factories[providerType] |
| 105 | + if factory == nil { |
| 106 | + return failedModelEval("provider is not eval-capable from current config") |
| 107 | + } |
| 108 | + provider, err := factory(modelID, caps) |
| 109 | + if err != nil { |
| 110 | + return failedModelEval(err.Error()) |
| 111 | + } |
| 112 | + report := evalpack.RunSuite(ctx, provider, suite, evalpack.Options{Timeout: 12 * time.Second}) |
| 113 | + status := modelEvalStatus{ |
| 114 | + CheckedAt: time.Now().Format(time.RFC3339), |
| 115 | + Passed: report.Passed, |
| 116 | + Failed: report.Failed, |
| 117 | + DurationMS: report.DurationMS, |
| 118 | + } |
| 119 | + for _, result := range report.Results { |
| 120 | + if result.Passed { |
| 121 | + continue |
| 122 | + } |
| 123 | + if result.Error != "" { |
| 124 | + status.Failures = append(status.Failures, fmt.Sprintf("%s: %s", result.ID, result.Error)) |
| 125 | + } |
| 126 | + for _, failure := range result.Failures { |
| 127 | + status.Failures = append(status.Failures, fmt.Sprintf("%s: %s", result.ID, failure)) |
| 128 | + } |
| 129 | + } |
| 130 | + if report.Failed > 0 && len(status.Failures) == 0 { |
| 131 | + status.Failures = append(status.Failures, "one or more cases failed") |
| 132 | + } |
| 133 | + if len(status.Failures) > 5 { |
| 134 | + status.Failures = status.Failures[:5] |
| 135 | + } |
| 136 | + return status |
| 137 | +} |
| 138 | + |
| 139 | +func failedModelEval(message string) modelEvalStatus { |
| 140 | + return modelEvalStatus{ |
| 141 | + CheckedAt: time.Now().Format(time.RFC3339), |
| 142 | + Failed: 1, |
| 143 | + Error: message, |
| 144 | + Failures: []string{message}, |
| 145 | + } |
| 146 | +} |
| 147 | + |
| 148 | +func (s *Server) loadModelEvals(ctx context.Context) map[string]modelEvalStatus { |
| 149 | + if s.settings == nil { |
| 150 | + return nil |
| 151 | + } |
| 152 | + raw, ok, err := s.settings.GetSetting(ctx, settingKeyModelEvals) |
| 153 | + if err != nil || !ok || strings.TrimSpace(raw) == "" { |
| 154 | + return nil |
| 155 | + } |
| 156 | + out := map[string]modelEvalStatus{} |
| 157 | + if err := json.Unmarshal([]byte(raw), &out); err != nil { |
| 158 | + s.logger.Warn("model eval cache parse failed", "err", err) |
| 159 | + return nil |
| 160 | + } |
| 161 | + return out |
| 162 | +} |
| 163 | + |
| 164 | +func (s *Server) saveModelEval(ctx context.Context, provider, modelID string, status modelEvalStatus) error { |
| 165 | + if s.settings == nil { |
| 166 | + return fmt.Errorf("settings store unavailable") |
| 167 | + } |
| 168 | + evals := s.loadModelEvals(ctx) |
| 169 | + if evals == nil { |
| 170 | + evals = map[string]modelEvalStatus{} |
| 171 | + } |
| 172 | + evals[modelCheckKey(provider, modelID)] = status |
| 173 | + data, err := json.Marshal(evals) |
| 174 | + if err != nil { |
| 175 | + return err |
| 176 | + } |
| 177 | + return s.settings.PutSetting(ctx, settingKeyModelEvals, string(data)) |
| 178 | +} |
0 commit comments