radar love

Author: michielkallenberg

cybench/runs/analysis/global_insights_lib.py

@@ -28,17 +28,17 @@
     {
         "id": "spatial",
         "label": "Spatial",
-        "note": "Pearson r on regional means (aggregate across years).",
+        "note": "Median per-year Pearson r across regions (typical-year slice).",
         "metrics": (
-            {"id": "r", "column": "r_spatial_agg", "label": "r", "higher_better": True},
+            {"id": "r", "column": "r_spatial", "label": "r", "higher_better": True},
         ),
     },
     {
         "id": "temporal",
         "label": "Temporal",
-        "note": "Pearson r on yearly national means (aggregate across regions).",
+        "note": "Median per-region Pearson r across years (typical-region slice).",
         "metrics": (
-            {"id": "r", "column": "r_temporal_agg", "label": "r", "higher_better": True},
+            {"id": "r", "column": "r_temporal", "label": "r", "higher_better": True},
         ),
     },
     {

cybench/runs/analysis/model_family_radar_lib.py

@@ -74,9 +74,22 @@
 }
 
 RADAR_NORMALIZATION_NOTE = (
-    "Each axis is independently normalized to highlight the relative strengths of "
-    "each modeling paradigm (NRMSE on Overall, Pearson r on other views). "
-    "Absolute values are reported in the tables below."
+    "Relative: each axis is min–max normalized across the five family representatives "
+    "(best paradigm on that axis reaches the outer ring). "
+    "Absolute values are in the table below."
+)
+
+RADAR_ABSOLUTE_SCALES: dict[str, dict[str, Any]] = {
+    "nrmse": {"lo": 0.1, "hi": 0.30, "higher_better": False, "display": "NRMSE"},
+    "r_spatial": {"lo": 0.0, "hi": 1.0, "higher_better": True, "display": "r"},
+    "r_temporal": {"lo": 0.0, "hi": 1.0, "higher_better": True, "display": "r"},
+    "r_res": {"lo": 0.0, "hi": 1.0, "higher_better": True, "display": "r"},
+}
+
+RADAR_ABSOLUTE_NOTE = (
+    "Absolute: fixed scales per axis — NRMSE 0.10 (outer, best) to 0.30 (center, worst); "
+    "Pearson r axes 0 (center) to 1 (outer). Values outside the range are clamped; "
+    "negative r is shown at 0."
 )
 
 MODEL_DISPLAY_NAMES: dict[str, str] = {
@@ -180,10 +193,54 @@ def relative_scores(raw: pd.DataFrame) -> pd.DataFrame:
     return out
 
 
+def absolute_scores(raw: pd.DataFrame) -> pd.DataFrame:
+    """Map raw metrics to 0–1 radar radius using fixed CY-Bench scales (higher radius = better)."""
+    out = raw.copy()
+    for view in EVALUATION_VIEWS:
+        label = view["label"]
+        metric = view["metric"]
+        spec = RADAR_ABSOLUTE_SCALES.get(metric)
+        if spec is None or metric not in out.columns:
+            out[label] = float("nan")
+            continue
+        lo = float(spec["lo"])
+        hi = float(spec["hi"])
+        higher = bool(spec.get("higher_better", True))
+        vals = out[metric].astype(float).clip(lower=lo, upper=hi)
+        if hi == lo:
+            out[label] = 0.5
+        else:
+            norm = (vals - lo) / (hi - lo)
+            out[label] = norm if higher else 1.0 - norm
+    return out
+
+
+def radar_scales_payload() -> dict[str, Any]:
+    """JSON-serializable absolute radar scale definitions per view label."""
+    by_label: dict[str, Any] = {}
+    for view in EVALUATION_VIEWS:
+        spec = RADAR_ABSOLUTE_SCALES.get(view["metric"], {})
+        lo = float(spec.get("lo", 0.0))
+        hi = float(spec.get("hi", 1.0))
+        higher = bool(spec.get("higher_better", True))
+        by_label[view["label"]] = {
+            "metric": view["metric"],
+            "display": spec.get("display", view["display"]),
+            "lo": lo,
+            "hi": hi,
+            "higher_better": higher,
+            "outer_label": f"{lo:.2f}" if view["metric"] == "nrmse" else f"{lo:g}",
+            "center_label": f"{hi:.2f}" if view["metric"] == "nrmse" else f"{hi:g}",
+        }
+    return {"absolute": by_label}
+
+
 def _records_from_medians(
     medians: pd.DataFrame,
     rel_all: pd.DataFrame,
     entries: list[tuple[str, str, str, str, bool]],
+    *,
+    abs_all: pd.DataFrame | None = None,
 ) -> list[dict[str, Any]]:
     """Build radar rows from median table. *entries*: (index, family, display, color, is_naive)."""
     view_labels = [v["label"] for v in EVALUATION_VIEWS]
@@ -206,6 +263,16 @@ def _records_from_medians(
             )
             for label in view_labels
         }
+        absolute: dict[str, float | None] = {}
+        if abs_all is not None:
+            absolute = {
+                label: (
+                    round(float(abs_all[label].loc[model_key]), 4)
+                    if label in abs_all.columns and model_key in abs_all.index
+                    else None
+                )
+                for label in view_labels
+            }
         rows.append(
             {
                 "family": family,
@@ -215,6 +282,7 @@ def _records_from_medians(
                 "is_naive": is_naive,
                 "raw": raw,
                 "relative": relative,
+                "absolute": absolute,
             }
         )
     return rows
@@ -225,9 +293,12 @@ def _family_records(
     representatives: dict[str, str],
     *,
     rel_all: pd.DataFrame | None = None,
+    abs_all: pd.DataFrame | None = None,
 ) -> list[dict[str, Any]]:
     if rel_all is None:
         rel_all = relative_scores(medians.copy()) if not medians.empty else pd.DataFrame()
+    if abs_all is None:
+        abs_all = absolute_scores(medians.copy()) if not medians.empty else pd.DataFrame()
     entries = [
         (
             representatives[family],
@@ -239,7 +310,7 @@ def _family_records(
         for family in FAMILY_ORDER
         if family in representatives
     ]
-    return _records_from_medians(medians, rel_all, entries)
+    return _records_from_medians(medians, rel_all, entries, abs_all=abs_all)
 
 
 def family_for_model(model: str) -> str | None:
@@ -460,7 +531,8 @@ def build_radar_slice(
     rep_models = list(reps.values())
     medians = _median_for_models(work, rep_models, VIEW_METRICS)
     rel_all = relative_scores(medians.copy()) if not medians.empty else pd.DataFrame()
-    families = _family_records(medians, reps, rel_all=rel_all)
+    abs_all = absolute_scores(medians.copy()) if not medians.empty else pd.DataFrame()
+    families = _family_records(medians, reps, rel_all=rel_all, abs_all=abs_all)
     return {
         "batch_horizon": batch_horizon,
         "crop": crop or "all",
@@ -514,6 +586,9 @@ def build_radar_payload(
         "by_horizon": by_horizon,
         "sample_scatter_metric": SAMPLE_SCATTER_METRIC,
         "sample_scatter": build_sample_scatter_payload(df, representatives=representatives),
+        "radar_scales": radar_scales_payload(),
+        "relative_note": RADAR_NORMALIZATION_NOTE,
+        "absolute_note": RADAR_ABSOLUTE_NOTE,
         "normalization_note": RADAR_NORMALIZATION_NOTE,
         "representative_selection": (
             "One model per family: lowest median NRMSE across datasets in the selection; "

cybench/runs/analysis/publish_dashboard_bundle.py

@@ -257,19 +257,19 @@ def render_cards(items: list[IndexEntry]) -> str:
     if publish_root and (publish_root / "insights.html").is_file():
         insights_card = """
   <section class="block">
-    <h2>Global insights</h2>
+    <h2>Global benchmarks</h2>
     <div class="grid">
       <a class="link-card" href="insights.html">
         <div class="link-card-top"><span class="badge badge-neutral">ALL</span><span class="arrow" aria-hidden="true">→</span></div>
-        <h3>Cross-country model ranking</h3>
-        <p class="muted">NRMSE leaderboard and end-of-season vs mid-season comparison</p>
+        <h3>Global insights</h3>
+        <p class="muted">All models — leaderboard, model×country heatmap, end-of-season vs mid-season</p>
       </a>"""
         if (publish_root / "model_families.html").is_file():
             insights_card += """
       <a class="link-card" href="model_families.html">
         <div class="link-card-top"><span class="badge badge-neutral">ALL</span><span class="arrow" aria-hidden="true">→</span></div>
-        <h3>Model family radar</h3>
-        <p class="muted">Relative performance across overall, spatial, temporal, and anomaly views</p>
+        <h3>Model families (paper summary)</h3>
+        <p class="muted">Five paradigms — radar chart, median metrics, performance vs training size</p>
       </a>"""
         insights_card += """
     </div>

cybench/runs/viz/global_insights_template.html

@@ -163,6 +163,7 @@
   <div class="page">
     <h1>CY-Bench global insights</h1>
     <p class="lead">Cross-country model comparison and end-of-season vs mid-season walk-forward results.</p>
+    <p class="muted">Every model in the benchmark. For a five-paradigm summary (radar chart), see <a href="model_families.html">Model families (paper summary)</a>.</p>
     <p class="muted" id="meta"></p>
 
     <div class="card">
@@ -245,7 +246,10 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per crop × country × m
       </p>
     </div>
 
-    <p class="muted"><a href="index.html">← Back to country dashboards</a></p>
+    <p class="muted">
+      <a href="model_families.html">Model families (paper summary)</a> ·
+      <a href="index.html">← Back to country dashboards</a>
+    </p>
   </div>
 
   <script>
@@ -372,8 +376,8 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per crop × country × m
           { id: "nrmse", label: "NRMSE", higher_better: false },
         ],
       },
-      { id: "spatial", label: "Spatial", note: "Pearson r on regional means (aggregate).", metrics: [{ id: "r", label: "r", higher_better: true }] },
-      { id: "temporal", label: "Temporal", note: "Pearson r on yearly national means (aggregate).", metrics: [{ id: "r", label: "r", higher_better: true }] },
+      { id: "spatial", label: "Spatial", note: "Median per-year r across regions (typical-year slice).", metrics: [{ id: "r", label: "r", higher_better: true }] },
+      { id: "temporal", label: "Temporal", note: "Median per-region r across years (typical-region slice).", metrics: [{ id: "r", label: "r", higher_better: true }] },
       { id: "anomaly", label: "Anomaly", note: "Pooled Pearson r on de-meaned yields.", metrics: [{ id: "r", label: "r", higher_better: true }] },
     ];
 

cybench/runs/viz/index_map_template.html

@@ -159,10 +159,10 @@ <h3>Select a country</h3>
 
     const topLinks = document.getElementById("top-links");
     if (DATA.has_insights) {
-      topLinks.innerHTML += `<a class="pill" href="insights.html">Global insights</a>`;
+      topLinks.innerHTML += `<a class="pill" href="insights.html">Global insights (all models)</a>`;
     }
     if (DATA.has_model_families) {
-      topLinks.innerHTML += `<a class="pill" href="model_families.html">Model families</a>`;
+      topLinks.innerHTML += `<a class="pill" href="model_families.html">Model families (paper)</a>`;
     }
     if (DATA.has_screening && DATA.screening_href) {
       topLinks.innerHTML += `<a class="pill" href="${DATA.screening_href}">Screening (all)</a>`;