ci: Day-1 eval automation — frontier-release watcher + runbook issues + report generator

.github/workflows/day1-eval.yml

@@ -0,0 +1,44 @@
+name: day1-eval-watch
+
+on:
+  schedule:
+    - cron: "0 7 * * *" # daily 07:00 UTC
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  watch:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Collect previously reported models
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh issue list --repo "$GITHUB_REPOSITORY" --label day1-eval --state all \
+            --limit 200 --json body --jq '.[].body' > seen.txt || true
+
+      - name: Scan OpenRouter for new frontier models
+        run: |
+          python scripts/day1_watch.py --days 2 --out day1-candidates.md --seen-file seen.txt
+
+      - name: Open runbook issue
+        if: ${{ hashFiles('day1-candidates.md') != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh label create day1-eval --repo "$GITHUB_REPOSITORY" \
+            --description "Day-1 evaluation runbook for a newly released frontier model" \
+            --color 0E8A16 || true
+          gh issue create --repo "$GITHUB_REPOSITORY" \
+            --title "[day1-eval] New frontier model release — $(date -u +%Y-%m-%d)" \
+            --label day1-eval \
+            --body-file day1-candidates.md

scripts/day1_report.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Turn a batch-summary.json into a Day-1 results post draft.
+
+Reads the batch summary written by `clawbench-batch` (and, optionally, a
+rescore summary produced by `clawbench-rescore`) and prints a Markdown post
+draft: headline numbers, per-status breakdown, an X-thread skeleton, and a
+Chinese blurb. Purely local formatting — no network calls.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def pct(n: int, d: int) -> str:
+    return f"{100 * n / d:.1f}%" if d else "n/a"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("batch_summary", help="path to batch-summary.json")
+    ap.add_argument("--model", default=None, help="model name for the headline")
+    ap.add_argument(
+        "--rescore-summary",
+        default=None,
+        help="optional eval_results summary.json from clawbench-rescore",
+    )
+    args = ap.parse_args()
+
+    s = json.loads(Path(args.batch_summary).read_text(encoding="utf-8"))
+    jobs = s.get("jobs", [])
+    totals = s.get("totals", {})
+    model = args.model or (jobs[0]["model"] if jobs else "unknown-model")
+    n = len(jobs)
+    passed = totals.get("passed", 0)
+    errors = totals.get("error", 0)
+
+    reward_line = ""
+    if args.rescore_summary and Path(args.rescore_summary).exists():
+        r = json.loads(Path(args.rescore_summary).read_text(encoding="utf-8"))
+        # rescore summaries vary by rubric config; surface whatever is present
+        found = {
+            k: v
+            for k, v in r.items()
+            if isinstance(v, (int, float))
+            and ("reward" in k or "pass" in k or "judge" in k)
+        }
+        if found:
+            pretty = ", ".join(f"{k}={v}" for k, v in sorted(found.items()))
+            reward_line = f"- Judge (two-stage): {pretty}\n"
+
+    print(
+        f"""# Day-1 ClawBench results — {model}
+
+- Corpus: V2 ({n} tasks, live websites) · harness: hermes
+- **Intercepted: {passed}/{n} ({pct(passed, n)})** (Stage 1, deterministic)
+{reward_line}- Infra errors: {errors} (excluded runs are re-run before publishing)
+- Elapsed: {s.get("elapsed_seconds", "?")}s · concurrency {s.get("max_concurrent", "?")}
+
+Full five-layer traces will land in the public Trace dataset; leaderboard:
+https://claw-bench.com/leaderboard
+
+## X thread
+
+1/ {model} dropped — we ran it on ClawBench (everyday tasks on live websites)
+within a day. Result: {pct(passed, n)} of {n} tasks reached a valid final
+request. Details + traces below 🧵
+2/ What ClawBench measures: can an agent actually order food, book travel,
+apply for jobs on the real web — graded by request interception + LLM judge,
+not vibes.
+3/ [Insert 2-3 notable failures/successes from the traces]
+4/ How it compares: current top is claude-opus-4-7 at 54.6% intercepted /
+44.6% reward. Leaderboard: claw-bench.com/leaderboard
+5/ Everything is open: tasks, judge, and full five-layer traces of every run.
+Repo: github.com/TIGER-AI-Lab/ClawBench
+6/ Reproduce this row: `clawbench-reproduce --model {model}` — scores are
+stable within ±2 pp.
+
+## 中文短版(知乎/公众号)
+
+{model} 发布后 24 小时,我们在 ClawBench(真实网站上的日常任务基准)上完成了
+第三方评测:{n} 个任务中 {passed} 个到达有效最终请求({pct(passed, n)})。
+全部五层轨迹(录屏/截图/HTTP/动作/agent 消息)公开可查,欢迎复现:
+github.com/TIGER-AI-Lab/ClawBench
+"""
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/day1_watch.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Watch for newly released frontier models and open a Day-1 eval runbook issue.
+
+Polls the public OpenRouter model catalog (no key required) for models from
+frontier vendors created within the last --days window, drops anything already
+reported in a previous issue (--seen-file), and writes a ready-to-run Day-1
+evaluation checklist + vendor-outreach template to --out. The GitHub Actions
+workflow opens an issue from that file; the actual benchmark run happens on
+maintainer infrastructure, never in CI.
+
+Exit code is always 0; an empty/absent --out file means "nothing new".
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import urllib.request
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+CATALOG_URL = "https://openrouter.ai/api/v1/models"
+
+# Vendors whose new releases warrant a Day-1 ClawBench run.
+FRONTIER_PREFIXES = (
+    "anthropic/",
+    "openai/",
+    "google/",
+    "x-ai/",
+    "deepseek/",
+    "moonshotai/",
+    "z-ai/",
+    "minimax/",
+    "qwen/",
+    "meta-llama/",
+    "mistralai/",
+)
+
+RUNBOOK = """\
+### Runbook — {model_id}
+
+- [ ] Add `{model_id}` to `models/models.yaml` (OpenRouter route or native API)
+- [ ] Smoke: `clawbench-batch --models {model_id} --cases-suite v1-lite --all-cases --harness hermes --no-judge`
+- [ ] Full V2: `clawbench-batch --models {model_id} --cases-suite v2 --all-cases --harness hermes --no-judge --max-concurrent 3`
+- [ ] Score: `clawbench-rescore <output-dir> --judge-model deepseek-v4-pro --rubric both`
+- [ ] Add row to the leaderboard (`leaderboard/results.csv` PR) + claw-bench.com
+- [ ] Draft the results post: `python scripts/day1_report.py <output-dir>/batch-summary.json --model {model_id}`
+- [ ] Publish thread (X + 知乎/公众号) linking the leaderboard
+
+<details>
+<summary>Vendor outreach template</summary>
+
+> Subject: {model_short} on ClawBench — Day-1 third-party browser-agent results
+>
+> Hi — we run ClawBench (github.com/TIGER-AI-Lab/ClawBench, arXiv:2604.08523),
+> an open benchmark of everyday tasks on live websites. We evaluated
+> {model_short} within days of release; results and full five-layer traces are
+> public. Happy to coordinate on future releases (pre-release runs under NDA
+> possible) or have the row cited in your model card / technical report — Li
+> Auto's Mach-Mind-4-Flash report already reports ClawBench results.
+
+</details>
+"""
+
+
+def fetch_catalog() -> list[dict]:
+    req = urllib.request.Request(
+        CATALOG_URL, headers={"User-Agent": "clawbench-day1-watch"}
+    )
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return json.load(resp).get("data", [])
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--days", type=int, default=2, help="release window to report")
+    ap.add_argument("--out", default="day1-candidates.md")
+    ap.add_argument("--seen-file", default=None, help="text dump of previous issues")
+    args = ap.parse_args()
+
+    seen = ""
+    if args.seen_file and Path(args.seen_file).exists():
+        seen = Path(args.seen_file).read_text(encoding="utf-8")
+
+    cutoff = datetime.now(timezone.utc) - timedelta(days=args.days)
+    fresh = []
+    for m in fetch_catalog():
+        mid = m.get("id", "")
+        if not mid.startswith(FRONTIER_PREFIXES) or mid.endswith(":free"):
+            continue
+        created = m.get("created")
+        if not created or datetime.fromtimestamp(created, tz=timezone.utc) < cutoff:
+            continue
+        if re.search(re.escape(mid), seen):
+            continue
+        fresh.append(m)
+
+    if not fresh:
+        print("no new frontier models")
+        return 0
+
+    lines = [
+        "New frontier model release(s) detected on OpenRouter — candidates for a",
+        "**Day-1 ClawBench evaluation**. Runbook per model below; close as",
+        "not-planned for minor variants not worth a full run.",
+        "",
+        "cc @Perry2004",
+        "",
+    ]
+    for m in fresh:
+        mid = m["id"]
+        short = mid.split("/", 1)[-1]
+        when = datetime.fromtimestamp(m["created"], tz=timezone.utc).strftime(
+            "%Y-%m-%d"
+        )
+        lines.append(f"## `{mid}` (listed {when})")
+        lines.append("")
+        lines.append(RUNBOOK.format(model_id=mid, model_short=short))
+    Path(args.out).write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"{len(fresh)} candidate model(s) written to {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())