Repository navigation
Model Sync #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Model Sync | |
| # Weekly benchmark-driven model-pin sync for the wet live benchmark. | |
| # | |
| # Source of truth: the hull model-candidates publisher (weekly Mon 04:41 UTC | |
| # sticky PR committing data/model-candidates.json, schema_version 1). This | |
| # workflow runs after it and reads the profile "agentic" (task profile of the | |
| # wet MCP benchmark). Selection is deterministic: the first candidate with | |
| # weak_evidence == false AND pareto_rank == 0 is the proposed OpenRouter slug. | |
| # When no strong candidate exists, the sticky issue records the weak-board | |
| # state and no pin is proposed — the sync never guesses a model. | |
| # | |
| # When a strong candidate differs from the current pin (vars.OPENROUTER_MODEL, | |
| # else bench/model.txt), this workflow files or updates ONE sticky issue with | |
| # the proposed slug and the manual apply command. It NEVER opens a PR itself | |
| # (personal repos deny GITHUB_TOKEN pull-request creation — same constraint | |
| # that moved hull's publisher to a canonical data branch), never sets the repo | |
| # variable, and never edits code defaults. Moving the paid benchmark model | |
| # stays a human-approved change: run the printed gh variable set command, or | |
| # hand-write bench/model.txt and open the PR yourself. | |
| on: | |
| schedule: | |
| - cron: "11 7 * * 1" # weekly Mon 07:11 UTC — trails hull's 04:41 publish + merge window | |
| workflow_dispatch: | |
| permissions: | |
| issues: write # sticky dashboard issue | |
| concurrency: | |
| group: model-sync | |
| cancel-in-progress: false | |
| env: | |
| MODEL_CANDIDATES_URL: https://raw.githubusercontent.com/n24q02m/hull/data/model-candidates/data/model-candidates.json | |
| TASK_PROFILE: agentic | |
| PIN_FILE: bench/model.txt | |
| STICKY_TITLE: "[model-sync] weekly model pin" | |
| # Read through the vars context (no actions:read needed) so the pin | |
| # comparison mirrors exactly what benchmark.yml resolves. | |
| VAR_MODEL: ${{ vars.OPENROUTER_MODEL }} | |
| GH_TOKEN: ${{ github.token }} | |
| jobs: | |
| sync: | |
| name: Sync benchmark model pin | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout pin file state | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Fetch published candidates | |
| run: | | |
| code="$(curl -sS -o model-candidates.json -w '%{http_code}' "${MODEL_CANDIDATES_URL}")" || code=000 | |
| if [ "${code}" != "200" ]; then | |
| echo "::error::Fetching hull model-candidates.json returned HTTP ${code}. The snapshot is committed by hull's weekly model-candidates publisher (data/model-candidates branch); the data/model-candidates branch is canonical; until it is force-pushed the raw URL does not serve yet. Failing closed — the sync never guesses a model." | |
| exit 1 | |
| fi | |
| - name: Select strong candidate | |
| id: select | |
| env: | |
| TASK_PROFILE: agentic | |
| run: | | |
| python3 - <<'PY' >> "${GITHUB_OUTPUT}" | |
| import json | |
| import os | |
| data = json.load(open("model-candidates.json")) | |
| if data.get("schema_version") != 1: | |
| raise SystemExit(f"unsupported schema_version {data.get('schema_version')!r} — failing closed") | |
| task = data.get("tasks", {}).get(os.environ["TASK_PROFILE"]) | |
| if task is None: | |
| raise SystemExit(f"task profile {os.environ['TASK_PROFILE']!r} missing from snapshot — failing closed") | |
| candidates = task.get("candidates") or [] | |
| strong = [ | |
| c for c in candidates | |
| if c.get("pareto_rank") == 0 and c.get("weak_evidence") is False | |
| ] | |
| print(f"generated_at={data.get('generated_at', 'unknown')}") | |
| print("strong=" + "; ".join(c.get("or_slug") or c.get("litellm_id") or "?" for c in strong[:3])) | |
| if not strong: | |
| print("proposed=") | |
| print("proposed_slug=") | |
| else: | |
| print(f"proposed={strong[0]['litellm_id']}") | |
| print(f"proposed_slug={strong[0].get('or_slug') or strong[0]['litellm_id']}") | |
| PY | |
| - name: Resolve current effective pin | |
| id: current | |
| env: | |
| PROPOSED: ${{ steps.select.outputs.proposed }} | |
| run: | | |
| pin="" | |
| if [ -f "${PIN_FILE}" ]; then | |
| pin="$(tr -d ' \t\r\n' < "${PIN_FILE}")" | |
| fi | |
| if [ -n "${VAR_MODEL}" ]; then | |
| current="${VAR_MODEL}" | |
| source="vars.OPENROUTER_MODEL" | |
| elif [ -n "${pin}" ]; then | |
| current="${pin}" | |
| source="${PIN_FILE}" | |
| else | |
| current="" | |
| source="unset" | |
| fi | |
| if [ -z "${PROPOSED}" ]; then | |
| state="no-strong-candidate" | |
| elif [ "${current}" != "${PROPOSED}" ]; then | |
| state="pr-needed" | |
| else | |
| state="unchanged" | |
| fi | |
| printf 'current=%s\nsource=%s\nstate=%s\n' "${current}" "${source}" "${state}" >> "${GITHUB_OUTPUT}" | |
| - name: Locate sticky issue | |
| id: sticky | |
| env: | |
| REPO: ${{ github.repository }} | |
| run: | | |
| num="$(gh issue list --repo "${REPO}" --state open --limit 200 --json number,title \ | |
| | python3 -c 'import json,os,sys;t=os.environ["STICKY_TITLE"];items=[i for i in json.load(sys.stdin) if i["title"]==t];print(items[0]["number"] if items else "")')" | |
| printf 'number=%s\n' "${num}" >> "${GITHUB_OUTPUT}" | |
| - name: Create or patch sticky issue | |
| id: upsert | |
| env: | |
| REPO: ${{ github.repository }} | |
| PROPOSED: ${{ steps.select.outputs.proposed_slug }} | |
| GENERATED_AT: ${{ steps.select.outputs.generated_at }} | |
| STRONG: ${{ steps.select.outputs.strong }} | |
| CURRENT: ${{ steps.current.outputs.current }} | |
| CURRENT_SOURCE: ${{ steps.current.outputs.source }} | |
| STATE: ${{ steps.current.outputs.state }} | |
| STICKY_NUM: ${{ steps.sticky.outputs.number }} | |
| run: | | |
| export NOW="$(date -u '+%Y-%m-%d %H:%M UTC')" | |
| python3 - <<'PY' | |
| import os | |
| repo = os.environ["REPO"] | |
| proposed = os.environ["PROPOSED"] | |
| current = os.environ["CURRENT"] | |
| source = os.environ["CURRENT_SOURCE"] | |
| state = os.environ["STATE"] | |
| state_label = { | |
| "pr-needed": "**proposal — apply manually (see below)**", | |
| "unchanged": "unchanged", | |
| "no-strong-candidate": "**no strong candidate this week — no pin change proposed**", | |
| }[state] | |
| rows = [ | |
| ("last checked (UTC)", os.environ["NOW"]), | |
| ("source", f"hull `model-candidates.json`, profile agentic (generated_at {os.environ['GENERATED_AT']})"), | |
| ("proposed text-chain leader", f"`{proposed}`" if proposed else "(none — weak board)"), | |
| ("strong candidates (top)", os.environ["STRONG"] or "(none)"), | |
| ("current effective pin", f"`{current}` ({source})" if current else f"unset ({source})"), | |
| ] | |
| rows.append(("state", state_label)) | |
| table = "| field | value |\n| --- | --- |\n" + "\n".join( | |
| f"| {k} | {v} |" for k, v in rows | |
| ) | |
| with open("sticky.md", "w") as f: | |
| f.write( | |
| "## [model-sync] weekly model pin\n\n" | |
| + table | |
| + "\n\n" | |
| + "**Apply (manual — this bot cannot open PRs on a personal repo):** either set the repo variable directly\n" | |
| + f"`gh variable set OPENROUTER_MODEL --repo {repo} --body \"{proposed}\"`\n" | |
| + "or write that slug into `bench/model.txt` and open a `model-pin` PR yourself.\n\n" | |
| + "This workflow never sets the variable and never edits `bench/model.txt` — moving the paid benchmark model is a human-approved change.\n" | |
| ) | |
| PY | |
| if [ -z "${STICKY_NUM}" ]; then | |
| url="$(gh issue create --repo "${REPO}" --title "${STICKY_TITLE}" --body-file sticky.md)" | |
| NUM="${url##*/}" | |
| else | |
| gh issue edit "${STICKY_NUM}" --repo "${REPO}" --body-file sticky.md >/dev/null | |
| NUM="${STICKY_NUM}" | |
| fi | |
| echo "number=${NUM}" >> "${GITHUB_OUTPUT}" | |
| if [ "${STATE}" = "pr-needed" ]; then | |
| gh issue comment "${NUM}" --repo "${REPO}" \ | |
| --body "Proposal updated at ${NOW}: effective pin \`${CURRENT:-unset}\` (${CURRENT_SOURCE}) → proposed \`${PROPOSED}\`. Apply via \`gh variable set OPENROUTER_MODEL\` or a hand-written \`bench/model.txt\` PR." >/dev/null | |
| fi | |
| cat sticky.md >> "${GITHUB_STEP_SUMMARY}" |