Add manual workload eval runner

cmd/model-eval/main.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"telegram-agent/internal/config"
+	"telegram-agent/internal/evalpack"
+	"telegram-agent/internal/llm"
+)
+
+func main() {
+	var (
+		dataset   = flag.String("dataset", "evals/workload.json", "path to eval suite JSON")
+		provider  = flag.String("provider", "openrouter", "provider type: openrouter, gemini, ollama, local")
+		model     = flag.String("model", "", "model id to evaluate")
+		apiKey    = flag.String("api-key", "", "provider API key; defaults to provider env var")
+		baseURL   = flag.String("base-url", "", "provider base URL override")
+		maxTokens = flag.Int("max-tokens", 2048, "max output tokens per case")
+		timeout   = flag.Duration("timeout", 60*time.Second, "timeout per case")
+		format    = flag.String("format", "markdown", "output format: markdown or json")
+		outPath   = flag.String("out", "", "optional output file")
+		allowPaid = flag.Bool("allow-paid", false, "allow evaluating a paid/cloud model; default only allows local/ollama and OpenRouter :free models")
+	)
+	flag.Parse()
+
+	if strings.TrimSpace(*model) == "" {
+		fatalf("-model is required")
+	}
+	if err := enforceCostGuard(*provider, *model, *allowPaid); err != nil {
+		fatalf("%v", err)
+	}
+	suite, err := evalpack.LoadSuite(*dataset)
+	if err != nil {
+		fatalf("load dataset: %v", err)
+	}
+	p, err := buildProvider(*provider, *model, *apiKey, *baseURL, *maxTokens)
+	if err != nil {
+		fatalf("provider: %v", err)
+	}
+
+	report := evalpack.RunSuite(context.Background(), p, suite, evalpack.Options{Timeout: *timeout})
+	output, err := renderReport(report, *format)
+	if err != nil {
+		fatalf("render report: %v", err)
+	}
+	if *outPath == "" {
+		fmt.Print(output)
+	} else {
+		if err := os.MkdirAll(filepath.Dir(*outPath), 0755); err != nil && filepath.Dir(*outPath) != "." {
+			fatalf("create output dir: %v", err)
+		}
+		if err := os.WriteFile(*outPath, []byte(output), 0644); err != nil {
+			fatalf("write output: %v", err)
+		}
+		fmt.Printf("wrote %s\n", *outPath)
+	}
+	if report.Failed > 0 {
+		os.Exit(2)
+	}
+}
+
+func enforceCostGuard(provider, model string, allowPaid bool) error {
+	if allowPaid {
+		return nil
+	}
+	provider = strings.ToLower(strings.TrimSpace(provider))
+	switch provider {
+	case "ollama", "local":
+		return nil
+	case "openrouter":
+		if strings.HasSuffix(model, ":free") {
+			return nil
+		}
+		return fmt.Errorf("refusing to evaluate paid OpenRouter model %q without --allow-paid", model)
+	default:
+		return fmt.Errorf("refusing to evaluate cloud provider %q without --allow-paid", provider)
+	}
+}
+
+func buildProvider(provider, model, apiKey, baseURL string, maxTokens int) (llm.Provider, error) {
+	provider = strings.ToLower(strings.TrimSpace(provider))
+	if apiKey == "" {
+		apiKey = defaultAPIKey(provider)
+	}
+	cfg := llm.ApplyProviderDefaults(config.ModelConfig{
+		Provider:  provider,
+		Model:     model,
+		APIKey:    apiKey,
+		BaseURL:   baseURL,
+		MaxTokens: maxTokens,
+	})
+	switch provider {
+	case "openrouter":
+		return llm.NewOpenRouter(cfg)
+	case "gemini":
+		return llm.NewGeminiNative(cfg)
+	case "ollama":
+		return llm.NewOllama(cfg)
+	case "local":
+		return llm.NewLocal(cfg)
+	default:
+		return nil, fmt.Errorf("unsupported provider %q", provider)
+	}
+}
+
+func defaultAPIKey(provider string) string {
+	switch provider {
+	case "openrouter":
+		return os.Getenv("OPENROUTER_API_KEY")
+	case "gemini":
+		return os.Getenv("GEMINI_API_KEY")
+	case "ollama":
+		return os.Getenv("OLLAMA_API_KEY")
+	default:
+		return ""
+	}
+}
+
+func renderReport(report evalpack.Report, format string) (string, error) {
+	switch strings.ToLower(format) {
+	case "json":
+		data, err := json.MarshalIndent(report, "", "  ")
+		if err != nil {
+			return "", err
+		}
+		return string(data) + "\n", nil
+	case "markdown", "md":
+		return renderMarkdown(report), nil
+	default:
+		return "", fmt.Errorf("unsupported format %q", format)
+	}
+}
+
+func renderMarkdown(report evalpack.Report) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "# Model Eval: %s\n\n", report.Model)
+	fmt.Fprintf(&b, "- Suite: `%s`\n", report.Suite)
+	fmt.Fprintf(&b, "- Started: `%s`\n", report.StartedAt.Format(time.RFC3339))
+	fmt.Fprintf(&b, "- Duration: `%d ms`\n", report.DurationMS)
+	fmt.Fprintf(&b, "- Passed: `%d`\n", report.Passed)
+	fmt.Fprintf(&b, "- Failed: `%d`\n\n", report.Failed)
+	for _, r := range report.Results {
+		status := "PASS"
+		if !r.Passed {
+			status = "FAIL"
+		}
+		fmt.Fprintf(&b, "## %s `%s`\n\n", status, r.ID)
+		fmt.Fprintf(&b, "- Category: `%s`\n", r.Category)
+		fmt.Fprintf(&b, "- Latency: `%d ms`\n", r.LatencyMS)
+		if len(r.ToolCalls) > 0 {
+			fmt.Fprintf(&b, "- Tool calls: `%s`\n", strings.Join(r.ToolCalls, "`, `"))
+		}
+		if r.Error != "" {
+			fmt.Fprintf(&b, "- Error: `%s`\n", r.Error)
+		}
+		if len(r.Failures) > 0 {
+			fmt.Fprintf(&b, "- Failures: `%s`\n", strings.Join(r.Failures, "`, `"))
+		}
+		if r.ContentPreview != "" {
+			fmt.Fprintf(&b, "\n%s\n", fenced(r.ContentPreview))
+		}
+		b.WriteString("\n")
+	}
+	return b.String()
+}
+
+func fenced(s string) string {
+	return "```text\n" + strings.TrimSpace(s) + "\n```"
+}
+
+func fatalf(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+	os.Exit(1)
+}

cmd/model-eval/main_test.go

@@ -0,0 +1,30 @@
+package main
+
+import "testing"
+
+func TestEnforceCostGuardAllowsFreeAndLocalByDefault(t *testing.T) {
+	if err := enforceCostGuard("openrouter", "google/gemma-3-27b-it:free", false); err != nil {
+		t.Fatalf("free OpenRouter model should pass: %v", err)
+	}
+	if err := enforceCostGuard("ollama", "qwen3:8b", false); err != nil {
+		t.Fatalf("ollama model should pass: %v", err)
+	}
+	if err := enforceCostGuard("local", "local-model", false); err != nil {
+		t.Fatalf("local model should pass: %v", err)
+	}
+}
+
+func TestEnforceCostGuardBlocksPaidCloudByDefault(t *testing.T) {
+	if err := enforceCostGuard("openrouter", "x-ai/grok-4.3", false); err == nil {
+		t.Fatal("paid OpenRouter model should require --allow-paid")
+	}
+	if err := enforceCostGuard("gemini", "gemini-2.5-flash", false); err == nil {
+		t.Fatal("cloud provider should require --allow-paid")
+	}
+}
+
+func TestEnforceCostGuardAllowsPaidWhenExplicit(t *testing.T) {
+	if err := enforceCostGuard("openrouter", "x-ai/grok-4.3", true); err != nil {
+		t.Fatalf("explicit paid run should pass: %v", err)
+	}
+}

evals/README.md

@@ -0,0 +1,18 @@
+# Model evals
+
+Manual-only workload checks for candidate models.
+
+These evals are intentionally not wired into CI, schedulers, or background jobs.
+The default runner refuses cloud paid models unless `--allow-paid` is passed.
+OpenRouter models are allowed by default only when the model id ends with
+`:free`; local and Ollama models are allowed by default.
+
+Examples:
+
+```bash
+go run ./cmd/model-eval -provider openrouter -model google/gemma-3-27b-it:free
+go run ./cmd/model-eval -provider ollama -model qwen3:8b -base-url http://localhost:11434
+go run ./cmd/model-eval -provider openrouter -model x-ai/grok-4.3 --allow-paid -out eval-results/grok-4.3.md
+```
+
+Use paid runs sparingly and only for explicit model selection decisions.

evals/workload.json

@@ -0,0 +1,83 @@
+{
+  "version": 1,
+  "name": "personal-assistant-core-workloads",
+  "cases": [
+    {
+      "id": "ru-chat-concise",
+      "category": "russian_chat",
+      "system_prompt": "Reply in Russian. Be concise and practical.",
+      "prompt": "Пользователь спрашивает: \"Что лучше сделать сначала: проверить логи сервиса или сразу перезапустить контейнер?\" Ответь коротко, как инженер.",
+      "expect": {
+        "must_contain": ["логи"],
+        "must_not_contain": ["As an AI"],
+        "no_tool_call": true
+      }
+    },
+    {
+      "id": "tool-calendar-intent",
+      "category": "tool_use",
+      "system_prompt": "Use tools when the user asks for data that only a tool can know.",
+      "prompt": "Проверь мой календарь на завтра утром и скажи, есть ли свободное окно на 30 минут.",
+      "tools": [
+        {
+          "name": "calendar_list_events",
+          "description": "List calendar events for a date range.",
+          "input_schema": {
+            "type": "object",
+            "properties": {
+              "date_from": {"type": "string"},
+              "date_to": {"type": "string"}
+            },
+            "required": ["date_from", "date_to"]
+          }
+        }
+      ],
+      "expect": {
+        "tool_call": "calendar_list_events"
+      }
+    },
+    {
+      "id": "tool-web-fetch-intent",
+      "category": "web_fetch",
+      "system_prompt": "Use tools when the user asks you to inspect a URL. Do not invent page contents.",
+      "prompt": "Открой https://example.com/release-notes и кратко перечисли изменения в последнем релизе.",
+      "tools": [
+        {
+          "name": "web_fetch",
+          "description": "Fetch a URL and return readable page text.",
+          "input_schema": {
+            "type": "object",
+            "properties": {
+              "url": {"type": "string"}
+            },
+            "required": ["url"]
+          }
+        }
+      ],
+      "expect": {
+        "tool_call": "web_fetch"
+      }
+    },
+    {
+      "id": "compaction-summary",
+      "category": "compaction",
+      "system_prompt": "Summarize the conversation state for future turns. Preserve decisions, unresolved tasks, and concrete identifiers.",
+      "prompt": "Сожми историю: решили вынести ARM docker build в manual workflow; сделали compact model catalog; free модели проверяем перед назначением; остался watchlist и eval pack. Нужно продолжить без потери контекста.",
+      "expect": {
+        "must_contain": ["ARM", "watchlist", "eval"],
+        "no_tool_call": true
+      }
+    },
+    {
+      "id": "admin-debug-next-step",
+      "category": "admin_debug",
+      "system_prompt": "Act as a pragmatic senior engineer. Give the next diagnostic step first.",
+      "prompt": "После деплоя админка открывается, но список моделей пустой. Что проверишь первым?",
+      "expect": {
+        "must_contain": ["capabilities"],
+        "must_not_contain": ["перезапусти все"],
+        "no_tool_call": true
+      }
+    }
+  ]
+}