Skip to content

Commit 25baf0b

Browse files
dzarlaxdzarlax
authored
Add manual workload eval runner
1 parent 8de9eb0 commit 25baf0b

6 files changed

Lines changed: 597 additions & 0 deletions

File tree

‎cmd/model-eval/main.go‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"flag"
7+
"fmt"
8+
"os"
9+
"path/filepath"
10+
"strings"
11+
"time"
12+
13+
"telegram-agent/internal/config"
14+
"telegram-agent/internal/evalpack"
15+
"telegram-agent/internal/llm"
16+
)
17+
18+
func main() {
19+
var (
20+
dataset = flag.String("dataset", "evals/workload.json", "path to eval suite JSON")
21+
provider = flag.String("provider", "openrouter", "provider type: openrouter, gemini, ollama, local")
22+
model = flag.String("model", "", "model id to evaluate")
23+
apiKey = flag.String("api-key", "", "provider API key; defaults to provider env var")
24+
baseURL = flag.String("base-url", "", "provider base URL override")
25+
maxTokens = flag.Int("max-tokens", 2048, "max output tokens per case")
26+
timeout = flag.Duration("timeout", 60*time.Second, "timeout per case")
27+
format = flag.String("format", "markdown", "output format: markdown or json")
28+
outPath = flag.String("out", "", "optional output file")
29+
allowPaid = flag.Bool("allow-paid", false, "allow evaluating a paid/cloud model; default only allows local/ollama and OpenRouter :free models")
30+
)
31+
flag.Parse()
32+
33+
if strings.TrimSpace(*model) == "" {
34+
fatalf("-model is required")
35+
}
36+
if err := enforceCostGuard(*provider, *model, *allowPaid); err != nil {
37+
fatalf("%v", err)
38+
}
39+
suite, err := evalpack.LoadSuite(*dataset)
40+
if err != nil {
41+
fatalf("load dataset: %v", err)
42+
}
43+
p, err := buildProvider(*provider, *model, *apiKey, *baseURL, *maxTokens)
44+
if err != nil {
45+
fatalf("provider: %v", err)
46+
}
47+
48+
report := evalpack.RunSuite(context.Background(), p, suite, evalpack.Options{Timeout: *timeout})
49+
output, err := renderReport(report, *format)
50+
if err != nil {
51+
fatalf("render report: %v", err)
52+
}
53+
if *outPath == "" {
54+
fmt.Print(output)
55+
} else {
56+
if err := os.MkdirAll(filepath.Dir(*outPath), 0755); err != nil && filepath.Dir(*outPath) != "." {
57+
fatalf("create output dir: %v", err)
58+
}
59+
if err := os.WriteFile(*outPath, []byte(output), 0644); err != nil {
60+
fatalf("write output: %v", err)
61+
}
62+
fmt.Printf("wrote %s\n", *outPath)
63+
}
64+
if report.Failed > 0 {
65+
os.Exit(2)
66+
}
67+
}
68+
69+
func enforceCostGuard(provider, model string, allowPaid bool) error {
70+
if allowPaid {
71+
return nil
72+
}
73+
provider = strings.ToLower(strings.TrimSpace(provider))
74+
switch provider {
75+
case "ollama", "local":
76+
return nil
77+
case "openrouter":
78+
if strings.HasSuffix(model, ":free") {
79+
return nil
80+
}
81+
return fmt.Errorf("refusing to evaluate paid OpenRouter model %q without --allow-paid", model)
82+
default:
83+
return fmt.Errorf("refusing to evaluate cloud provider %q without --allow-paid", provider)
84+
}
85+
}
86+
87+
func buildProvider(provider, model, apiKey, baseURL string, maxTokens int) (llm.Provider, error) {
88+
provider = strings.ToLower(strings.TrimSpace(provider))
89+
if apiKey == "" {
90+
apiKey = defaultAPIKey(provider)
91+
}
92+
cfg := llm.ApplyProviderDefaults(config.ModelConfig{
93+
Provider: provider,
94+
Model: model,
95+
APIKey: apiKey,
96+
BaseURL: baseURL,
97+
MaxTokens: maxTokens,
98+
})
99+
switch provider {
100+
case "openrouter":
101+
return llm.NewOpenRouter(cfg)
102+
case "gemini":
103+
return llm.NewGeminiNative(cfg)
104+
case "ollama":
105+
return llm.NewOllama(cfg)
106+
case "local":
107+
return llm.NewLocal(cfg)
108+
default:
109+
return nil, fmt.Errorf("unsupported provider %q", provider)
110+
}
111+
}
112+
113+
func defaultAPIKey(provider string) string {
114+
switch provider {
115+
case "openrouter":
116+
return os.Getenv("OPENROUTER_API_KEY")
117+
case "gemini":
118+
return os.Getenv("GEMINI_API_KEY")
119+
case "ollama":
120+
return os.Getenv("OLLAMA_API_KEY")
121+
default:
122+
return ""
123+
}
124+
}
125+
126+
func renderReport(report evalpack.Report, format string) (string, error) {
127+
switch strings.ToLower(format) {
128+
case "json":
129+
data, err := json.MarshalIndent(report, "", " ")
130+
if err != nil {
131+
return "", err
132+
}
133+
return string(data) + "\n", nil
134+
case "markdown", "md":
135+
return renderMarkdown(report), nil
136+
default:
137+
return "", fmt.Errorf("unsupported format %q", format)
138+
}
139+
}
140+
141+
func renderMarkdown(report evalpack.Report) string {
142+
var b strings.Builder
143+
fmt.Fprintf(&b, "# Model Eval: %s\n\n", report.Model)
144+
fmt.Fprintf(&b, "- Suite: `%s`\n", report.Suite)
145+
fmt.Fprintf(&b, "- Started: `%s`\n", report.StartedAt.Format(time.RFC3339))
146+
fmt.Fprintf(&b, "- Duration: `%d ms`\n", report.DurationMS)
147+
fmt.Fprintf(&b, "- Passed: `%d`\n", report.Passed)
148+
fmt.Fprintf(&b, "- Failed: `%d`\n\n", report.Failed)
149+
for _, r := range report.Results {
150+
status := "PASS"
151+
if !r.Passed {
152+
status = "FAIL"
153+
}
154+
fmt.Fprintf(&b, "## %s `%s`\n\n", status, r.ID)
155+
fmt.Fprintf(&b, "- Category: `%s`\n", r.Category)
156+
fmt.Fprintf(&b, "- Latency: `%d ms`\n", r.LatencyMS)
157+
if len(r.ToolCalls) > 0 {
158+
fmt.Fprintf(&b, "- Tool calls: `%s`\n", strings.Join(r.ToolCalls, "`, `"))
159+
}
160+
if r.Error != "" {
161+
fmt.Fprintf(&b, "- Error: `%s`\n", r.Error)
162+
}
163+
if len(r.Failures) > 0 {
164+
fmt.Fprintf(&b, "- Failures: `%s`\n", strings.Join(r.Failures, "`, `"))
165+
}
166+
if r.ContentPreview != "" {
167+
fmt.Fprintf(&b, "\n%s\n", fenced(r.ContentPreview))
168+
}
169+
b.WriteString("\n")
170+
}
171+
return b.String()
172+
}
173+
174+
func fenced(s string) string {
175+
return "```text\n" + strings.TrimSpace(s) + "\n```"
176+
}
177+
178+
func fatalf(format string, args ...any) {
179+
fmt.Fprintf(os.Stderr, format+"\n", args...)
180+
os.Exit(1)
181+
}

‎cmd/model-eval/main_test.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package main
2+
3+
import "testing"
4+
5+
func TestEnforceCostGuardAllowsFreeAndLocalByDefault(t *testing.T) {
6+
if err := enforceCostGuard("openrouter", "google/gemma-3-27b-it:free", false); err != nil {
7+
t.Fatalf("free OpenRouter model should pass: %v", err)
8+
}
9+
if err := enforceCostGuard("ollama", "qwen3:8b", false); err != nil {
10+
t.Fatalf("ollama model should pass: %v", err)
11+
}
12+
if err := enforceCostGuard("local", "local-model", false); err != nil {
13+
t.Fatalf("local model should pass: %v", err)
14+
}
15+
}
16+
17+
func TestEnforceCostGuardBlocksPaidCloudByDefault(t *testing.T) {
18+
if err := enforceCostGuard("openrouter", "x-ai/grok-4.3", false); err == nil {
19+
t.Fatal("paid OpenRouter model should require --allow-paid")
20+
}
21+
if err := enforceCostGuard("gemini", "gemini-2.5-flash", false); err == nil {
22+
t.Fatal("cloud provider should require --allow-paid")
23+
}
24+
}
25+
26+
func TestEnforceCostGuardAllowsPaidWhenExplicit(t *testing.T) {
27+
if err := enforceCostGuard("openrouter", "x-ai/grok-4.3", true); err != nil {
28+
t.Fatalf("explicit paid run should pass: %v", err)
29+
}
30+
}

‎evals/README.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Model evals
2+
3+
Manual-only workload checks for candidate models.
4+
5+
These evals are intentionally not wired into CI, schedulers, or background jobs.
6+
The default runner refuses cloud paid models unless `--allow-paid` is passed.
7+
OpenRouter models are allowed by default only when the model id ends with
8+
`:free`; local and Ollama models are allowed by default.
9+
10+
Examples:
11+
12+
```bash
13+
go run ./cmd/model-eval -provider openrouter -model google/gemma-3-27b-it:free
14+
go run ./cmd/model-eval -provider ollama -model qwen3:8b -base-url http://localhost:11434
15+
go run ./cmd/model-eval -provider openrouter -model x-ai/grok-4.3 --allow-paid -out eval-results/grok-4.3.md
16+
```
17+
18+
Use paid runs sparingly and only for explicit model selection decisions.

‎evals/workload.json‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
{
2+
"version": 1,
3+
"name": "personal-assistant-core-workloads",
4+
"cases": [
5+
{
6+
"id": "ru-chat-concise",
7+
"category": "russian_chat",
8+
"system_prompt": "Reply in Russian. Be concise and practical.",
9+
"prompt": "Пользователь спрашивает: \"Что лучше сделать сначала: проверить логи сервиса или сразу перезапустить контейнер?\" Ответь коротко, как инженер.",
10+
"expect": {
11+
"must_contain": ["логи"],
12+
"must_not_contain": ["As an AI"],
13+
"no_tool_call": true
14+
}
15+
},
16+
{
17+
"id": "tool-calendar-intent",
18+
"category": "tool_use",
19+
"system_prompt": "Use tools when the user asks for data that only a tool can know.",
20+
"prompt": "Проверь мой календарь на завтра утром и скажи, есть ли свободное окно на 30 минут.",
21+
"tools": [
22+
{
23+
"name": "calendar_list_events",
24+
"description": "List calendar events for a date range.",
25+
"input_schema": {
26+
"type": "object",
27+
"properties": {
28+
"date_from": {"type": "string"},
29+
"date_to": {"type": "string"}
30+
},
31+
"required": ["date_from", "date_to"]
32+
}
33+
}
34+
],
35+
"expect": {
36+
"tool_call": "calendar_list_events"
37+
}
38+
},
39+
{
40+
"id": "tool-web-fetch-intent",
41+
"category": "web_fetch",
42+
"system_prompt": "Use tools when the user asks you to inspect a URL. Do not invent page contents.",
43+
"prompt": "Открой https://example.com/release-notes и кратко перечисли изменения в последнем релизе.",
44+
"tools": [
45+
{
46+
"name": "web_fetch",
47+
"description": "Fetch a URL and return readable page text.",
48+
"input_schema": {
49+
"type": "object",
50+
"properties": {
51+
"url": {"type": "string"}
52+
},
53+
"required": ["url"]
54+
}
55+
}
56+
],
57+
"expect": {
58+
"tool_call": "web_fetch"
59+
}
60+
},
61+
{
62+
"id": "compaction-summary",
63+
"category": "compaction",
64+
"system_prompt": "Summarize the conversation state for future turns. Preserve decisions, unresolved tasks, and concrete identifiers.",
65+
"prompt": "Сожми историю: решили вынести ARM docker build в manual workflow; сделали compact model catalog; free модели проверяем перед назначением; остался watchlist и eval pack. Нужно продолжить без потери контекста.",
66+
"expect": {
67+
"must_contain": ["ARM", "watchlist", "eval"],
68+
"no_tool_call": true
69+
}
70+
},
71+
{
72+
"id": "admin-debug-next-step",
73+
"category": "admin_debug",
74+
"system_prompt": "Act as a pragmatic senior engineer. Give the next diagnostic step first.",
75+
"prompt": "После деплоя админка открывается, но список моделей пустой. Что проверишь первым?",
76+
"expect": {
77+
"must_contain": ["capabilities"],
78+
"must_not_contain": ["перезапусти все"],
79+
"no_tool_call": true
80+
}
81+
}
82+
]
83+
}

0 commit comments

Comments
 (0)