dashboard

Author: michielkallenberg

cybench/runs/analysis/build_global_insights_dashboard.py

@@ -33,7 +33,7 @@ def write_insights_dashboard(
     *,
     output_root: Path,
     dest: Path,
-    version: int = 1,
+    version: int = 2,
 ) -> Path:
     payload = build_insights_payload(output_root, version=version)
     if payload["n_rows"] == 0:
@@ -62,7 +62,7 @@ def main() -> int:
         default=Path("/lustre/backup/SHARED/AIN/agml/CY-Bench-dashboard"),
         help="GitHub Pages clone root (used when --dest is omitted)",
     )
-    parser.add_argument("--version", type=int, default=1, help="Batch version tag (default: 1)")
+    parser.add_argument("--version", type=int, default=2, help="Batch version tag (default: 2)")
     args = parser.parse_args()
 
     dest = args.dest or (args.publish_root / "insights.html")

cybench/runs/analysis/build_model_family_radar_dashboard.py

@@ -34,7 +34,7 @@ def write_model_family_radar_dashboard(
     *,
     output_root: Path,
     dest: Path,
-    version: int = 1,
+    version: int = 2,
 ) -> Path:
     payload = build_radar_payload(output_root, version=version)
     if payload["n_rows"] == 0:
@@ -66,7 +66,7 @@ def main() -> int:
         default=Path("/lustre/backup/SHARED/AIN/agml/CY-Bench-dashboard"),
         help="GitHub Pages clone root (used when --dest is omitted)",
     )
-    parser.add_argument("--version", type=int, default=1, help="Batch version tag (default: 1)")
+    parser.add_argument("--version", type=int, default=2, help="Batch version tag (default: 2)")
     args = parser.parse_args()
 
     dest = args.dest or (args.publish_root / "model_families.html")

cybench/runs/analysis/dashboard_targets.yaml

@@ -9,7 +9,7 @@
 # Optional: uncomment ``include`` to pin a subset (also used by --mode planned).
 
 defaults:
-  version: 1
+  version: 2
   output_root: /lustre/backup/SHARED/AIN/agml/output
   repo_root: /lustre/backup/SHARED/AIN/agml/AgML-CY-Bench-AAAI
   publish_root: /lustre/backup/SHARED/AIN/agml/CY-Bench-dashboard

cybench/runs/analysis/global_insights_lib.py

@@ -14,13 +14,14 @@
 
 _BASELINE_MODELS = frozenset({"average", "averageyieldmodel", "average_yield"})
 
-# Lead time increases left → right (less season observed → more observed).
-BATCH_HORIZON_ORDER: tuple[str, ...] = ("qtr", "mid", "eos")
+# Lead time decreases left → right (less season observed → more observed).
+# Code: mid = 50% observed; qtr = cutoff at season_length/4 before EOS (~75% observed, 25% left); eos = ~100%.
+BATCH_HORIZON_ORDER: tuple[str, ...] = ("mid", "qtr", "eos")
 
 HORIZON_DISPLAY_LABELS: dict[str, str] = {
-    "qtr": "Quarter-season (~75%)",
-    "mid": "Mid-season (~50%)",
-    "eos": "End of season",
+    "mid": "Mid-season (~50% observed)",
+    "qtr": "Late season (~75% observed, 25% left)",
+    "eos": "End of season (~100% observed)",
 }
 
 

cybench/runs/analysis/publish_dashboard_bundle.py

@@ -73,8 +73,8 @@
     "eos": "End of season",
     "mid": "Mid-season",
     "mid_season": "Mid-season",
-    "qtr": "Quarter-season (75%)",
-    "quarter_season": "Quarter-season (75%)",
+    "qtr": "Late season (75% observed)",
+    "quarter_season": "Late season (75% observed)",
 }
 
 
@@ -123,13 +123,18 @@ def prune_obsolete_dashboard_dirs(
     GitHub Pages artifacts are limited to 1 GB; keeping only the latest version per
     country×horizon avoids duplicate v1+v2 asset trees (~30% of the bundle).
     """
+    publish_root = publish_root.resolve()
     by_key: dict[tuple[str, str], list[tuple[int, Path]]] = {}
+    n_dirs = 0
+    n_matched = 0
     for child in publish_root.iterdir():
         if not child.is_dir() or child.name == "assets":
             continue
+        n_dirs += 1
         parsed = parse_publish_slug(child.name)
         if parsed is None:
             continue
+        n_matched += 1
         cc, hz, ver = parsed
         by_key.setdefault((cc, hz), []).append((ver, child))
 
@@ -146,6 +151,23 @@ def prune_obsolete_dashboard_dirs(
                 else:
                     shutil.rmtree(path)
                     print(f"[OK] pruned {path.name}")
+
+    if not removed:
+        print(
+            f"[INFO] publish-root={publish_root} · "
+            f"{n_dirs} subdirs scanned · {n_matched} walk-forward slug(s) · "
+            f"{len(by_key)} country×horizon keys"
+        )
+        if n_matched == 0:
+            print(
+                "[WARN] No folders matching "
+                "'{cc}_walk_forward_{eos|mid|qtr}_vN'. "
+                "Use the CY-Bench-dashboard git clone as --publish-root "
+                "(e.g. /lustre/backup/SHARED/AIN/agml/CY-Bench-dashboard), "
+                "not the AgML-CY-Bench-AAAI source tree."
+            )
+        elif all(len(e) == 1 for e in by_key.values()):
+            print("[INFO] No obsolete versions: at most one vN per country×horizon.")
     return sorted(removed, key=lambda p: p.name)
 
 

cybench/runs/analysis/publish_pipeline_lib.py

@@ -46,7 +46,7 @@
 _BATCH_HORIZON_LABELS: dict[str, str] = {
     "eos": "end-of-season",
     "mid": "mid-season",
-    "qtr": "quarter-season (75%)",
+    "qtr": "late season (75% observed, 25% left)",
 }
 
 
@@ -479,13 +479,12 @@ def filter_publish_targets(
     countries: list[str] | None = None,
     horizons: list[str] | None = None,
     version: int | None = None,
-    keep_latest_version: bool = True,
+    keep_latest_version: bool = False,
 ) -> list[PublishTarget]:
     """Narrow targets by country, horizon (eos|mid|qtr), and optional batch version.
 
-    When ``version`` is omitted and ``keep_latest_version`` is true (default), only the
-    highest ``vN`` per (country, horizon) is kept — avoids publishing duplicate v1+v2
-    bundles that blow the GitHub Pages 1 GB artifact limit.
+    When ``keep_latest_version`` is true, only the highest ``vN`` per (country, horizon)
+    is kept — use for GitHub Pages publish to avoid duplicate v1+v2 bundles.
     """
     if countries:
         wanted = {c.upper() for c in countries}
@@ -949,7 +948,11 @@ def resolve_targets(
         targets = discover_baselines_batches(defaults.output_root, defaults=defaults)
 
     targets = filter_publish_targets(
-        targets, countries=countries, horizons=horizons, version=version
+        targets,
+        countries=countries,
+        horizons=horizons,
+        version=version,
+        keep_latest_version=version is None,
     )
 
     if mode == "all-available":

cybench/runs/viz/global_insights_template.html

@@ -412,9 +412,9 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
     }
 
     const HORIZON_LABELS = DATA.horizon_labels || {
-      qtr: "Quarter-season (~75%)",
-      mid: "Mid-season (~50%)",
-      eos: "End of season",
+      mid: "Mid-season (~50% observed)",
+      qtr: "Late season (~75% observed, 25% left)",
+      eos: "End of season (~100% observed)",
     };
     const AVAILABLE_HORIZONS = (DATA.available_horizons && DATA.available_horizons.length)
       ? DATA.available_horizons
@@ -889,7 +889,7 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
         svg += `<line x1="${x}" y1="${pad.t}" x2="${x}" y2="${height - pad.b}" stroke="#f0f1f3"/>`;
         svg += `<text x="${x}" y="${height - 28}" text-anchor="middle" font-size="11" fill="#1f2328">${hz.label || hz.id}</text>`;
       });
-      svg += `<text x="${pad.l + innerW / 2}" y="${height - 6}" text-anchor="middle" font-size="11" fill="#1f2328">Forecast horizon (more season observed →)</text>`;
+      svg += `<text x="${pad.l + innerW / 2}" y="${height - 6}" text-anchor="middle" font-size="11" fill="#1f2328">Season progress (more of season observed →)</text>`;
       svg += `<text x="14" y="${pad.t + innerH / 2}" text-anchor="middle" font-size="11" fill="#1f2328" transform="rotate(-90 14 ${pad.t + innerH / 2})">${yLabel}</text>`;
 
       for (const fam of slice.families) {

cybench/runs/viz/model_family_radar_template.html

@@ -389,7 +389,11 @@ <h2>World map</h2>
       render();
     });
 
-    const horizonLabels = DATA.horizon_labels || { eos: "End of season", mid: "Mid-season", qtr: "Quarter-season (~75%)" };
+    const horizonLabels = DATA.horizon_labels || {
+      eos: "End of season (~100% observed)",
+      mid: "Mid-season (~50% observed)",
+      qtr: "Late season (~75% observed, 25% left)",
+    };
     Object.keys(DATA.by_horizon).forEach(hz => {
       const opt = document.createElement("option");
       opt.value = hz;

tests/runs/test_global_insights.py

@@ -447,8 +447,8 @@ def test_build_insights_payload_structure(tmp_path: Path):
 def _three_horizon_fixture(tmp_path: Path) -> pd.DataFrame:
     """DE+FR with eos/mid/qtr; US only eos (excluded from curves)."""
     rows_by_hz = {
-        "qtr": {"ridge": 0.28, "trend": 0.32, "xgboost": 0.26},
-        "mid": {"ridge": 0.22, "trend": 0.30, "xgboost": 0.20},
+        "mid": {"ridge": 0.30, "trend": 0.34, "xgboost": 0.28},
+        "qtr": {"ridge": 0.22, "trend": 0.30, "xgboost": 0.20},
         "eos": {"ridge": 0.15, "trend": 0.28, "xgboost": 0.14},
     }
     for cc, hz in [("de", "eos"), ("de", "mid"), ("de", "qtr"), ("fr", "eos"), ("fr", "mid"), ("fr", "qtr")]:
@@ -527,15 +527,15 @@ def test_horizon_skill_curves_inner_join_countries():
         df = _three_horizon_fixture(Path(tmp))
         payload = build_horizon_skill_curves_payload(df)
         assert len(payload["horizons"]) == 3
-        assert [h["id"] for h in payload["horizons"]] == ["qtr", "mid", "eos"]
+        assert [h["id"] for h in payload["horizons"]] == ["mid", "qtr", "eos"]
         maize = payload["by_crop"]["maize"]
         assert maize["n_countries"] == 2
         assert set(maize["countries"]) == {"DE", "FR"}
         assert "US" in maize["excluded_countries"]
 
         xgb = next(f for f in maize["families"] if f["model"] == "xgboost")
         nrmse_by_hz = {p["horizon"]: p["median_nrmse"] for p in xgb["points"]}
-        assert nrmse_by_hz["qtr"] > nrmse_by_hz["mid"] > nrmse_by_hz["eos"]
+        assert nrmse_by_hz["mid"] > nrmse_by_hz["qtr"] > nrmse_by_hz["eos"]
 
 
 def test_build_insights_payload_includes_qtr_and_curves(tmp_path: Path):

tests/runs/test_publish_pipeline.py

@@ -135,7 +135,7 @@ def test_filter_publish_targets_by_version(tmp_path: Path):
     assert filtered[0].batch_name == "baselines_DE_eos_v2"
 
 
-def test_filter_publish_targets_keeps_latest_version_by_default(tmp_path: Path):
+def test_filter_publish_targets_keeps_latest_version_when_requested(tmp_path: Path):
     targets = [
         PublishTarget(country="DE", batch_horizon="eos", version=1, output_root=tmp_path),
         PublishTarget(country="DE", batch_horizon="eos", version=2, output_root=tmp_path),