dahboard

michielkallenberg · michielkallenberg · commit e1a59fea3a62 · 2026-07-05T16:20:56.000+02:00
diff --git a/cybench/runs/analysis/global_insights_lib.py b/cybench/runs/analysis/global_insights_lib.py
@@ -24,6 +24,9 @@
     "eos": "End of season (~100% observed)",
 }
 
+# Process baselines evaluated only at end-of-season (no mid/qtr walk-forward runs).
+EOS_ONLY_HORIZON_CURVE_MODELS: frozenset[str] = frozenset({"lpjml_bc"})
+
 
 def horizons_in_data(df: pd.DataFrame) -> tuple[str, ...]:
     """Horizons present in a summary frame, ordered by increasing season progress."""
@@ -774,6 +777,68 @@ def _family_curve_points(
     return points
 
 
+def _eos_only_family_points(
+    work: pd.DataFrame,
+    *,
+    model: str,
+    trend_model: str,
+) -> list[dict[str, Any]]:
+    """Median EOS metrics for models not run at earlier forecast horizons."""
+    eos = work[
+        (work["model"] == model) & (work["batch_horizon"] == "eos") & work["nrmse"].notna()
+    ]
+    if eos.empty:
+        return []
+
+    trend_eos = work[
+        (work["model"] == trend_model) & (work["batch_horizon"] == "eos") & work["nrmse"].notna()
+    ]
+    trend_by_country: dict[str, float] = {}
+    if not trend_eos.empty and "country" in trend_eos.columns:
+        for country, grp in trend_eos.groupby("country"):
+            med = grp["nrmse"].median()
+            if pd.notna(med) and float(med) > 0:
+                trend_by_country[str(country)] = float(med)
+
+    country_nrmse: list[float] = []
+    country_skill: list[float] = []
+    if "country" in eos.columns:
+        for country, grp in eos.groupby("country"):
+            med = grp["nrmse"].median()
+            if pd.isna(med):
+                continue
+            nrmse = float(med)
+            country_nrmse.append(nrmse)
+            trend_val = trend_by_country.get(str(country))
+            if trend_val and trend_val > 0:
+                country_skill.append(1.0 - nrmse / trend_val)
+    else:
+        med = eos["nrmse"].median()
+        if pd.notna(med):
+            country_nrmse.append(float(med))
+
+    nrmse_series = pd.Series(country_nrmse, dtype=float)
+    skill_series = pd.Series(country_skill, dtype=float)
+    return [
+        {
+            "horizon": "eos",
+            "median_nrmse": (
+                round(float(nrmse_series.median()), 4) if not nrmse_series.empty else None
+            ),
+            "q25_nrmse": (
+                round(float(nrmse_series.quantile(0.25)), 4) if not nrmse_series.empty else None
+            ),
+            "q75_nrmse": (
+                round(float(nrmse_series.quantile(0.75)), 4) if not nrmse_series.empty else None
+            ),
+            "median_skill_vs_trend": (
+                round(float(skill_series.median()), 4) if not skill_series.empty else None
+            ),
+            "n_countries": int(len(country_nrmse)),
+        }
+    ]
+
+
 def build_horizon_skill_curves_payload(
     df: pd.DataFrame,
     *,
@@ -830,18 +895,27 @@ def build_horizon_skill_curves_payload(
             model = reps.get(family)
             if not model or model == trend_model:
                 continue
-            points = _family_curve_points(
-                wide, model=model, trend_model=trend_model, horizons=horizons
-            )
+            eos_only = model in EOS_ONLY_HORIZON_CURVE_MODELS
+            if eos_only:
+                points = _eos_only_family_points(work, model=model, trend_model=trend_model)
+            else:
+                points = _family_curve_points(
+                    wide, model=model, trend_model=trend_model, horizons=horizons
+                )
             if not points:
                 continue
+            n_horizons_with_data = sum(
+                1 for p in points if p.get("median_nrmse") is not None or p.get("median_skill_vs_trend") is not None
+            )
             families.append(
                 {
                     "family": family,
                     "model": model,
                     "display": MODEL_DISPLAY_NAMES.get(model, model),
                     "color": FAMILY_COLORS.get(family, "#666"),
                     "points": points,
+                    "eos_only": eos_only,
+                    "plot": (not eos_only) and n_horizons_with_data >= 2,
                 }
             )
 
@@ -865,11 +939,16 @@ def build_horizon_skill_curves_payload(
         "note": (
             "Fixed model per family (representatives chosen at "
             f"{HORIZON_DISPLAY_LABELS.get(rep_source_hz, rep_source_hz)}). "
-            "Only countries with data at every plotted horizon are included "
+            "Curves use only countries with data at every plotted horizon "
             "(inner join on country×model). Each country contributes one median NRMSE "
             "(median across crops when crop=all). IQR band = spread across countries. "
+            "LPJmL is end-of-season only and appears in the table but not on the curve plot. "
             "Hyperparameters are tuned per horizon (screening at each lead time)."
         ),
+        "plot_excluded_note": (
+            "EOS-only baselines (LPJmL) are omitted from the curve plot; see the table for "
+            "their end-of-season median."
+        ),
         "metric_notes": {
             "nrmse": "Median pooled NRMSE across countries (lower is better).",
             "skill_vs_trend": (
diff --git a/cybench/runs/viz/global_insights_template.html b/cybench/runs/viz/global_insights_template.html
@@ -267,6 +267,8 @@ <h2>Forecast horizon vs performance</h2>
       <p class="muted" id="horizon-curves-meta"></p>
       <div class="curve-chart-wrap" id="horizon-curves-chart"></div>
       <div class="curve-legend" id="horizon-curves-legend"></div>
+      <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">By family (median across countries)</h3>
+      <div class="table-scroll" id="horizon-curves-table-wrap"></div>
       <p class="muted note" id="horizon-curves-metric-note"></p>
     </div>
 
@@ -791,11 +793,14 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
     const hscMeta = document.getElementById("horizon-curves-meta");
     const hscChart = document.getElementById("horizon-curves-chart");
     const hscLegend = document.getElementById("horizon-curves-legend");
+    const hscTableWrap = document.getElementById("horizon-curves-table-wrap");
     const hscMetricNote = document.getElementById("horizon-curves-metric-note");
     const hscCropSelect = document.getElementById("horizon-curves-crop");
     const hscMetricSelect = document.getElementById("horizon-curves-metric");
     let hscCrop = "all";
     let hscMetric = "nrmse";
+    let hscTableSort = "family";
+    let hscTableDir = "asc";
 
     hscNote.textContent = HSC.note || "Horizon skill curves require at least two collected horizons.";
 
@@ -825,7 +830,13 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
       const slice = (HSC.by_crop || {})[hscCrop] || { families: [], n_countries: 0, countries: [] };
       const horizons = HSC.horizons || [];
       const metricNotes = HSC.metric_notes || {};
-      hscMetricNote.textContent = metricNotes[hscMetric] || "";
+      const plotFamilies = (slice.families || []).filter(fam => fam.plot !== false);
+      const tableFamilies = slice.families || [];
+      let metricNote = metricNotes[hscMetric] || "";
+      if (HSC.plot_excluded_note && (slice.families || []).some(fam => fam.eos_only)) {
+        metricNote = metricNote ? `${metricNote} ${HSC.plot_excluded_note}` : HSC.plot_excluded_note;
+      }
+      hscMetricNote.textContent = metricNote;
 
       const reps = HSC.representatives_summary || "";
       const excluded = slice.excluded_countries || [];
@@ -837,9 +848,10 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
       }
       hscMeta.textContent = meta;
 
-      if (!slice.families || !slice.families.length || horizons.length < 2) {
+      if (!tableFamilies.length || horizons.length < 2) {
         hscChart.innerHTML = `<p class="muted">Not enough paired horizon data for this crop.</p>`;
         hscLegend.innerHTML = "";
+        hscTableWrap.innerHTML = `<p class="muted">No horizon table for this crop.</p>`;
         return;
       }
 
@@ -851,8 +863,12 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
       const nX = horizons.length;
       const xAt = (i) => pad.l + (nX <= 1 ? innerW / 2 : (i / (nX - 1)) * innerW);
 
+      if (!plotFamilies.length) {
+        hscChart.innerHTML = `<p class="muted">No multi-horizon families to plot for this crop.</p>`;
+        hscLegend.innerHTML = "";
+      } else {
       const allVals = [];
-      for (const fam of slice.families) {
+      for (const fam of plotFamilies) {
         for (const pt of fam.points || []) {
           const v = hscMetricValue(pt, hscMetric);
           if (v !== null && v !== undefined && !Number.isNaN(v)) allVals.push(v);
@@ -864,8 +880,7 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
       if (!allVals.length) {
         hscChart.innerHTML = `<p class="muted">No metric values for this selection.</p>`;
         hscLegend.innerHTML = "";
-        return;
-      }
+      } else {
 
       const yMinRaw = Math.min(...allVals);
       const yMaxRaw = Math.max(...allVals);
@@ -892,7 +907,7 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
       svg += `<text x="${pad.l + innerW / 2}" y="${height - 6}" text-anchor="middle" font-size="11" fill="#1f2328">Season progress (more of season observed →)</text>`;
       svg += `<text x="14" y="${pad.t + innerH / 2}" text-anchor="middle" font-size="11" fill="#1f2328" transform="rotate(-90 14 ${pad.t + innerH / 2})">${yLabel}</text>`;
 
-      for (const fam of slice.families) {
+      for (const fam of plotFamilies) {
         const color = fam.color || "#457b9d";
         const pts = (fam.points || []).map((pt, i) => {
           const hzIdx = horizons.findIndex(h => h.id === pt.horizon);
@@ -922,9 +937,61 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
       svg += "</svg>";
       hscChart.innerHTML = svg;
 
-      hscLegend.innerHTML = slice.families.map(fam =>
+      hscLegend.innerHTML = plotFamilies.map(fam =>
         `<span><span class="swatch" style="background:${fam.color}"></span>${fam.family} (${fam.display})</span>`
       ).join("");
+      }
+      }
+
+      const tableRows = tableFamilies.map(fam => {
+        const byHz = Object.fromEntries((fam.points || []).map(pt => [pt.horizon, pt]));
+        const row = {
+          family: fam.family,
+          model: fam.display || fam.model,
+          eos_only: !!fam.eos_only,
+        };
+        for (const hz of horizons) {
+          const pt = byHz[hz.id];
+          row[`hz_${hz.id}`] = pt ? hscMetricValue(pt, hscMetric) : null;
+        }
+        return row;
+      });
+      const sortedRows = [...tableRows].sort((a, b) => {
+        const cmp = compareSortValues(a, b, hscTableSort);
+        return hscTableDir === "asc" ? cmp : -cmp;
+      });
+      const tableCols = [
+        { key: "family", label: "Family", sortable: true },
+        { key: "model", label: "Model", sortable: true },
+        ...horizons.map(hz => ({
+          key: `hz_${hz.id}`,
+          label: hz.label || hz.id,
+          sortable: true,
+          format: (v, row) => {
+            if (v === null || v === undefined) {
+              return row.eos_only ? '<span class="muted">EOS only</span>' : "—";
+            }
+            return fmt(v);
+          },
+          bg: (v) => (hscMetric === "nrmse" ? fixedNrmseBg(v) : null),
+        })),
+      ];
+      renderSortableTable(
+        hscTableWrap,
+        tableCols,
+        sortedRows,
+        hscTableSort,
+        hscTableDir,
+        (key) => {
+          if (hscTableSort === key) {
+            hscTableDir = hscTableDir === "asc" ? "desc" : "asc";
+          } else {
+            hscTableSort = key;
+            hscTableDir = key === "family" || key === "model" ? "asc" : "desc";
+          }
+          renderHorizonSkillCurves();
+        },
+      );
     }
 
     hscCropSelect.addEventListener("change", () => {
diff --git a/tests/runs/test_global_insights.py b/tests/runs/test_global_insights.py
@@ -534,10 +534,41 @@ def test_horizon_skill_curves_inner_join_countries():
         assert "US" in maize["excluded_countries"]
 
         xgb = next(f for f in maize["families"] if f["model"] == "xgboost")
+        assert xgb["plot"] is True
+        assert xgb.get("eos_only") is not True
         nrmse_by_hz = {p["horizon"]: p["median_nrmse"] for p in xgb["points"]}
         assert nrmse_by_hz["mid"] > nrmse_by_hz["qtr"] > nrmse_by_hz["eos"]
 
 
+def test_horizon_skill_curves_eos_only_lpjml_in_table_not_plot(tmp_path: Path):
+    df = _three_horizon_fixture(tmp_path)
+    lpj_rows = [
+        {
+            "crop": "maize",
+            "country": cc,
+            "model": "lpjml_bc",
+            "batch_horizon": "eos",
+            "nrmse": 0.19,
+            "r2": 0.5,
+            "n_samples": 40,
+        }
+        for cc in ("DE", "FR")
+    ]
+    df = pd.concat([df, pd.DataFrame(lpj_rows)], ignore_index=True)
+
+    payload = build_horizon_skill_curves_payload(df)
+    maize = payload["by_crop"]["maize"]
+    lpj = next(f for f in maize["families"] if f["model"] == "lpjml_bc")
+    assert lpj["eos_only"] is True
+    assert lpj["plot"] is False
+    assert len(lpj["points"]) == 1
+    assert lpj["points"][0]["horizon"] == "eos"
+    assert lpj["points"][0]["median_nrmse"] == 0.19
+    plot_models = {f["model"] for f in maize["families"] if f["plot"]}
+    assert "lpjml_bc" not in plot_models
+    assert "plot_excluded_note" in payload
+
+
 def test_build_insights_payload_includes_qtr_and_curves(tmp_path: Path):
     _three_horizon_fixture(tmp_path)
     payload = build_insights_payload(tmp_path, version=1)
