chore: commit benchmark training updates

Author: Shaw

bun.lock

@@ -432,13 +432,30 @@ def _has_vision_language_bundle(tier: str = "eliza-1-9b") -> bool:
         manifest_payload = json.loads(manifest.read_text(encoding="utf-8"))
     except Exception:
         return False
-    runtime = manifest_payload.get("runtime") if isinstance(manifest_payload, dict) else None
-    if not isinstance(runtime, dict) or "dflash" not in runtime:
+    if not isinstance(manifest_payload, dict):
+        return False
+    runtime = manifest_payload.get("runtime")
+    kernels = manifest_payload.get("kernels")
+    files = manifest_payload.get("files")
+    has_dflash_runtime = isinstance(runtime, dict) and "dflash" in runtime
+    has_dflash_kernel = (
+        isinstance(kernels, dict)
+        and isinstance(kernels.get("required"), list)
+        and "dflash" in {str(item) for item in kernels["required"]}
+    )
+    has_dflash_file = (
+        isinstance(files, dict)
+        and isinstance(files.get("dflash"), list)
+        and any(isinstance(item, dict) and item.get("path") for item in files["dflash"])
+    )
+    if not (has_dflash_runtime or has_dflash_kernel or has_dflash_file):
         return False
     slug = tier.removeprefix("eliza-1-")
     text_candidates = [
         bundle / "text" / f"eliza-1-{slug}-64k.gguf",
         bundle / "text" / f"eliza-1-{slug}-32k.gguf",
+        bundle / "text" / f"eliza-1-{slug}-128k.gguf",
+        bundle / "text" / f"eliza-1-{slug}-256k.gguf",
         bundle / "text" / f"eliza-1-{slug}.gguf",
     ]
     vision = bundle / "vision" / f"mmproj-{slug}.gguf"

packages/benchmarks/orchestrator/adapters.py

@@ -714,6 +714,12 @@ def _publication_quarantine_reason(
         return "sample_task_set"
     if metrics.get("demo_mode") is True or metrics.get("demoMode") is True:
         return "demo_mode"
+    failed_scenarios = metrics.get("failed_scenarios")
+    if isinstance(failed_scenarios, (int, float)) and not isinstance(failed_scenarios, bool):
+        if failed_scenarios > 0:
+            return "failed_scenarios"
+    if metrics.get("interrupted") is True:
+        return "interrupted_run"
     return None
 
 

packages/benchmarks/orchestrator/runner.py

@@ -3167,6 +3167,39 @@ def test_vision_language_harness_runtime_requires_multimodal_model(
     assert orchestrator_adapters._has_vision_language_harness_runtime() is True
 
 
+def test_vision_language_bundle_accepts_current_manifest_schema(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    bundle = (
+        tmp_path
+        / ".eliza"
+        / "local-inference"
+        / "models"
+        / "eliza-1-9b.bundle"
+    )
+    (bundle / "text").mkdir(parents=True)
+    (bundle / "vision").mkdir()
+    (bundle / "dflash").mkdir()
+    (bundle / "text" / "eliza-1-9b-128k.gguf").write_text("text", encoding="utf-8")
+    (bundle / "vision" / "mmproj-9b.gguf").write_text("vision", encoding="utf-8")
+    (bundle / "dflash" / "drafter-9b.gguf").write_text("dflash", encoding="utf-8")
+    (bundle / "eliza-1.manifest.json").write_text(
+        json.dumps(
+            {
+                "id": "eliza-1-9b",
+                "runtime": {"dflash": {"enabled": True}},
+                "kernels": {"required": ["dflash"]},
+                "files": {"dflash": [{"path": "dflash/drafter-9b.gguf"}]},
+            }
+        ),
+        encoding="utf-8",
+    )
+    monkeypatch.setenv("ELIZA_STATE_DIR", str(tmp_path / ".eliza"))
+
+    assert orchestrator_adapters._has_vision_language_bundle("eliza-1-9b") is True
+
+
 def test_rlm_registry_forwards_model_to_root_and_subcall(tmp_path: Path) -> None:
     entry = {item.id: item for item in get_benchmark_registry(_workspace_root())}[
         "rlm_bench"

packages/benchmarks/orchestrator/tests/test_adapter_discovery.py

@@ -69,7 +69,22 @@
     "synth-346eec6c": 3,
     "synth-e772ae68": 1,
     "synth-6566db2a": 1,
-    "synth-7893eeb5": 1
+    "synth-7893eeb5": 1,
+    "synth-21a34891": 3,
+    "synth-b061248a": 3,
+    "synth-5f90212e": 1,
+    "synth-f5851987": 1,
+    "synth-bc69589c": 1,
+    "synth-f4caa2cf": 3,
+    "synth-c3511b12": 3,
+    "synth-9b2e3a4d": 1,
+    "synth-ee0956b5": 1,
+    "synth-37f41a4a": 1,
+    "synth-e07a70e4": 3,
+    "synth-9444efb1": 3,
+    "synth-2c42736e": 1,
+    "synth-fd34ff03": 1,
+    "synth-017755da": 1
   },
   "scores": {
     "synth-871fc8d4": [
@@ -337,37 +352,100 @@
     ],
     "synth-7893eeb5": [
       0.9
+    ],
+    "synth-21a34891": [
+      0.9,
+      0.9,
+      0.9
+    ],
+    "synth-b061248a": [
+      0.9,
+      0.9,
+      0.9
+    ],
+    "synth-5f90212e": [
+      0.5
+    ],
+    "synth-f5851987": [
+      0.7
+    ],
+    "synth-bc69589c": [
+      0.9
+    ],
+    "synth-f4caa2cf": [
+      0.9,
+      0.9,
+      0.9
+    ],
+    "synth-c3511b12": [
+      0.9,
+      0.9,
+      0.9
+    ],
+    "synth-9b2e3a4d": [
+      0.5
+    ],
+    "synth-ee0956b5": [
+      0.7
+    ],
+    "synth-37f41a4a": [
+      0.9
+    ],
+    "synth-e07a70e4": [
+      0.9,
+      0.9,
+      0.9
+    ],
+    "synth-9444efb1": [
+      0.9,
+      0.9,
+      0.9
+    ],
+    "synth-2c42736e": [
+      0.5
+    ],
+    "synth-fd34ff03": [
+      0.7
+    ],
+    "synth-017755da": [
+      0.9
     ]
   },
   "solved": [
+    "synth-0909a501",
+    "synth-63d72ad4",
+    "synth-f4caa2cf",
+    "synth-b061248a",
+    "synth-114dabcb",
+    "synth-22309d43",
+    "synth-9444efb1",
+    "synth-ea1adeb6",
+    "synth-7be2922d",
     "synth-3570ed6f",
     "synth-2bdf09a1",
-    "synth-67f61a80",
-    "synth-10466b18",
-    "synth-22309d43",
-    "synth-808575e8",
-    "synth-43c4906f",
     "synth-215e5aca",
-    "synth-06eb2699",
-    "synth-c5ff1314",
-    "synth-9b83cfc8",
-    "synth-2fdb5a5c",
     "synth-46111142",
-    "synth-10043ebf",
-    "synth-e82700bb",
+    "synth-c3511b12",
+    "synth-43c4906f",
     "synth-4ef1ca8f",
-    "synth-0909a501",
-    "synth-7be2922d",
+    "synth-10466b18",
+    "synth-e82700bb",
+    "synth-06eb2699",
+    "synth-147a7802",
     "synth-93db52f4",
-    "synth-5d4e07bc",
-    "synth-63d72ad4",
     "synth-871fc8d4",
-    "synth-111403d8",
-    "synth-114dabcb",
-    "synth-147a7802",
+    "synth-67f61a80",
     "synth-346eec6c",
+    "synth-c5ff1314",
+    "synth-5d4e07bc",
+    "synth-111403d8",
     "synth-36ceaf32",
-    "synth-ea1adeb6"
+    "synth-e07a70e4",
+    "synth-808575e8",
+    "synth-2fdb5a5c",
+    "synth-10043ebf",
+    "synth-9b83cfc8",
+    "synth-21a34891"
   ],
-  "last_updated": "2026-05-20T05:17:37.940076+00:00"
+  "last_updated": "2026-05-20T05:20:37.501685+00:00"
 }