From d0b0cf7fb7cf94b7178031f539efebeb72feb7f4 Mon Sep 17 00:00:00 2001 From: reacher-z Date: Sun, 16 Aug 2026 13:04:39 -0700 Subject: [PATCH] =?UTF-8?q?ci:=20Day-1=20eval=20watch=20=E2=80=94=20new=20?= =?UTF-8?q?frontier-model=20releases=20open=20a=20runbook=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daily cron polls the public OpenRouter catalog for frontier-vendor models released in the last 2 days, dedups against previous day1-eval issues, and opens a runbook issue: models.yaml entry, v1-lite smoke, full V2 batch, rescore, leaderboard row, and a results-post draft via day1_report.py. Includes a vendor-outreach template per model. The benchmark run itself happens on maintainer infra, never in CI. --- .github/workflows/day1-eval.yml | 44 +++++++++++ scripts/day1_report.py | 93 +++++++++++++++++++++++ scripts/day1_watch.py | 127 ++++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 .github/workflows/day1-eval.yml create mode 100644 scripts/day1_report.py create mode 100644 scripts/day1_watch.py diff --git a/.github/workflows/day1-eval.yml b/.github/workflows/day1-eval.yml new file mode 100644 index 00000000..213acb94 --- /dev/null +++ b/.github/workflows/day1-eval.yml @@ -0,0 +1,44 @@ +name: day1-eval-watch + +on: + schedule: + - cron: "0 7 * * *" # daily 07:00 UTC + workflow_dispatch: {} + +permissions: + contents: read + issues: write + +jobs: + watch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Collect previously reported models + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue list --repo "$GITHUB_REPOSITORY" --label day1-eval --state all \ + --limit 200 --json body --jq '.[].body' > seen.txt || true + + - name: Scan OpenRouter for new frontier models + run: | + python scripts/day1_watch.py --days 2 --out day1-candidates.md --seen-file seen.txt + + - name: Open runbook issue + if: ${{ hashFiles('day1-candidates.md') != '' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh label create day1-eval --repo "$GITHUB_REPOSITORY" \ + --description "Day-1 evaluation runbook for a newly released frontier model" \ + --color 0E8A16 || true + gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "[day1-eval] New frontier model release — $(date -u +%Y-%m-%d)" \ + --label day1-eval \ + --body-file day1-candidates.md diff --git a/scripts/day1_report.py b/scripts/day1_report.py new file mode 100644 index 00000000..c90436c8 --- /dev/null +++ b/scripts/day1_report.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Turn a batch-summary.json into a Day-1 results post draft. + +Reads the batch summary written by `clawbench-batch` (and, optionally, a +rescore summary produced by `clawbench-rescore`) and prints a Markdown post +draft: headline numbers, per-status breakdown, an X-thread skeleton, and a +Chinese blurb. Purely local formatting — no network calls. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def pct(n: int, d: int) -> str: + return f"{100 * n / d:.1f}%" if d else "n/a" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("batch_summary", help="path to batch-summary.json") + ap.add_argument("--model", default=None, help="model name for the headline") + ap.add_argument( + "--rescore-summary", + default=None, + help="optional eval_results summary.json from clawbench-rescore", + ) + args = ap.parse_args() + + s = json.loads(Path(args.batch_summary).read_text(encoding="utf-8")) + jobs = s.get("jobs", []) + totals = s.get("totals", {}) + model = args.model or (jobs[0]["model"] if jobs else "unknown-model") + n = len(jobs) + passed = totals.get("passed", 0) + errors = totals.get("error", 0) + + reward_line = "" + if args.rescore_summary and Path(args.rescore_summary).exists(): + r = json.loads(Path(args.rescore_summary).read_text(encoding="utf-8")) + # rescore summaries vary by rubric config; surface whatever is present + found = { + k: v + for k, v in r.items() + if isinstance(v, (int, float)) + and ("reward" in k or "pass" in k or "judge" in k) + } + if found: + pretty = ", ".join(f"{k}={v}" for k, v in sorted(found.items())) + reward_line = f"- Judge (two-stage): {pretty}\n" + + print( + f"""# Day-1 ClawBench results — {model} + +- Corpus: V2 ({n} tasks, live websites) · harness: hermes +- **Intercepted: {passed}/{n} ({pct(passed, n)})** (Stage 1, deterministic) +{reward_line}- Infra errors: {errors} (excluded runs are re-run before publishing) +- Elapsed: {s.get("elapsed_seconds", "?")}s · concurrency {s.get("max_concurrent", "?")} + +Full five-layer traces will land in the public Trace dataset; leaderboard: +https://claw-bench.com/leaderboard + +## X thread + +1/ {model} dropped — we ran it on ClawBench (everyday tasks on live websites) +within a day. Result: {pct(passed, n)} of {n} tasks reached a valid final +request. Details + traces below 🧵 +2/ What ClawBench measures: can an agent actually order food, book travel, +apply for jobs on the real web — graded by request interception + LLM judge, +not vibes. +3/ [Insert 2-3 notable failures/successes from the traces] +4/ How it compares: current top is claude-opus-4-7 at 54.6% intercepted / +44.6% reward. Leaderboard: claw-bench.com/leaderboard +5/ Everything is open: tasks, judge, and full five-layer traces of every run. +Repo: github.com/TIGER-AI-Lab/ClawBench +6/ Reproduce this row: `clawbench-reproduce --model {model}` — scores are +stable within ±2 pp. + +## 中文短版(知乎/公众号) + +{model} 发布后 24 小时,我们在 ClawBench(真实网站上的日常任务基准)上完成了 +第三方评测:{n} 个任务中 {passed} 个到达有效最终请求({pct(passed, n)})。 +全部五层轨迹(录屏/截图/HTTP/动作/agent 消息)公开可查,欢迎复现: +github.com/TIGER-AI-Lab/ClawBench +""" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/day1_watch.py b/scripts/day1_watch.py new file mode 100644 index 00000000..0fe9ffcb --- /dev/null +++ b/scripts/day1_watch.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Watch for newly released frontier models and open a Day-1 eval runbook issue. + +Polls the public OpenRouter model catalog (no key required) for models from +frontier vendors created within the last --days window, drops anything already +reported in a previous issue (--seen-file), and writes a ready-to-run Day-1 +evaluation checklist + vendor-outreach template to --out. The GitHub Actions +workflow opens an issue from that file; the actual benchmark run happens on +maintainer infrastructure, never in CI. + +Exit code is always 0; an empty/absent --out file means "nothing new". +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import urllib.request +from datetime import datetime, timedelta, timezone +from pathlib import Path + +CATALOG_URL = "https://openrouter.ai/api/v1/models" + +# Vendors whose new releases warrant a Day-1 ClawBench run. +FRONTIER_PREFIXES = ( + "anthropic/", + "openai/", + "google/", + "x-ai/", + "deepseek/", + "moonshotai/", + "z-ai/", + "minimax/", + "qwen/", + "meta-llama/", + "mistralai/", +) + +RUNBOOK = """\ +### Runbook — {model_id} + +- [ ] Add `{model_id}` to `models/models.yaml` (OpenRouter route or native API) +- [ ] Smoke: `clawbench-batch --models {model_id} --cases-suite v1-lite --all-cases --harness hermes --no-judge` +- [ ] Full V2: `clawbench-batch --models {model_id} --cases-suite v2 --all-cases --harness hermes --no-judge --max-concurrent 3` +- [ ] Score: `clawbench-rescore --judge-model deepseek-v4-pro --rubric both` +- [ ] Add row to the leaderboard (`leaderboard/results.csv` PR) + claw-bench.com +- [ ] Draft the results post: `python scripts/day1_report.py /batch-summary.json --model {model_id}` +- [ ] Publish thread (X + 知乎/公众号) linking the leaderboard + +
+Vendor outreach template + +> Subject: {model_short} on ClawBench — Day-1 third-party browser-agent results +> +> Hi — we run ClawBench (github.com/TIGER-AI-Lab/ClawBench, arXiv:2604.08523), +> an open benchmark of everyday tasks on live websites. We evaluated +> {model_short} within days of release; results and full five-layer traces are +> public. Happy to coordinate on future releases (pre-release runs under NDA +> possible) or have the row cited in your model card / technical report — Li +> Auto's Mach-Mind-4-Flash report already reports ClawBench results. + +
+""" + + +def fetch_catalog() -> list[dict]: + req = urllib.request.Request( + CATALOG_URL, headers={"User-Agent": "clawbench-day1-watch"} + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.load(resp).get("data", []) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--days", type=int, default=2, help="release window to report") + ap.add_argument("--out", default="day1-candidates.md") + ap.add_argument("--seen-file", default=None, help="text dump of previous issues") + args = ap.parse_args() + + seen = "" + if args.seen_file and Path(args.seen_file).exists(): + seen = Path(args.seen_file).read_text(encoding="utf-8") + + cutoff = datetime.now(timezone.utc) - timedelta(days=args.days) + fresh = [] + for m in fetch_catalog(): + mid = m.get("id", "") + if not mid.startswith(FRONTIER_PREFIXES) or mid.endswith(":free"): + continue + created = m.get("created") + if not created or datetime.fromtimestamp(created, tz=timezone.utc) < cutoff: + continue + if re.search(re.escape(mid), seen): + continue + fresh.append(m) + + if not fresh: + print("no new frontier models") + return 0 + + lines = [ + "New frontier model release(s) detected on OpenRouter — candidates for a", + "**Day-1 ClawBench evaluation**. Runbook per model below; close as", + "not-planned for minor variants not worth a full run.", + "", + "cc @Perry2004", + "", + ] + for m in fresh: + mid = m["id"] + short = mid.split("/", 1)[-1] + when = datetime.fromtimestamp(m["created"], tz=timezone.utc).strftime( + "%Y-%m-%d" + ) + lines.append(f"## `{mid}` (listed {when})") + lines.append("") + lines.append(RUNBOOK.format(model_id=mid, model_short=short)) + Path(args.out).write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"{len(fresh)} candidate model(s) written to {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())