dashboard

Author: michielkallenberg

cybench/runs/analysis/collect_walk_forward_results.py

@@ -395,7 +395,9 @@ def summary_rows_to_dashboard_records(
         ("region_year", "r2", "r2"),
         ("region_year", "nrmse", "nrmse"),
         ("spatial", "r2", "r2_spatial"),
+        ("spatial", "r2_agg", "r2_spatial_agg"),
         ("temporal", "r2", "r2_temporal"),
+        ("temporal", "r2_agg", "r2_temporal_agg"),
         ("anomaly", "r2", "r2_anomaly"),
     ]
     records: list[dict[str, Any]] = []

cybench/runs/analysis/model_family_radar_lib.py

@@ -273,25 +273,80 @@ def build_sample_scatter_slice(
     return families_out
 
 
+def summarize_sample_scatter(families: list[dict[str, Any]]) -> dict[str, Any]:
+    """Headline stats for the training-size scatter (percentiles + rank correlation)."""
+    rows: list[dict[str, Any]] = []
+    for fam in families:
+        for p in fam.get("points") or []:
+            rel = p.get("relative_nrmse")
+            n_train = p.get("n_train")
+            if rel is None or n_train is None:
+                continue
+            rows.append(
+                {
+                    "family": fam.get("family"),
+                    "n_train": int(n_train),
+                    "relative_nrmse": float(rel),
+                }
+            )
+    if not rows:
+        return {}
+
+    frame = pd.DataFrame(rows)
+    n = len(frame)
+    p05 = float(frame["n_train"].quantile(0.05))
+    p95 = float(frame["n_train"].quantile(0.95))
+    core = frame[(frame["n_train"] >= p05) & (frame["n_train"] <= p95)]
+    rho = float(core["n_train"].corr(core["relative_nrmse"], method="spearman")) if len(core) >= 5 else float("nan")
+
+    per_family: dict[str, float | None] = {}
+    for family, grp in core.groupby("family"):
+        if len(grp) < 4:
+            per_family[str(family)] = None
+        else:
+            per_family[str(family)] = round(
+                float(grp["n_train"].corr(grp["relative_nrmse"], method="spearman")), 3
+            )
+
+    return {
+        "n_points": n,
+        "n_outliers_x": int(n - len(core)),
+        "x_p05": int(round(p05)),
+        "x_p50": int(round(float(frame["n_train"].median()))),
+        "x_p95": int(round(p95)),
+        "x_min": int(frame["n_train"].min()),
+        "x_max": int(frame["n_train"].max()),
+        "spearman_rho_core": None if pd.isna(rho) else round(rho, 3),
+        "spearman_family_core": per_family,
+    }
+
+
 def build_sample_scatter_payload(
     df: pd.DataFrame,
     *,
     representatives: dict[str, str] | None = None,
-) -> dict[str, dict[str, list[dict[str, Any]]]]:
+) -> dict[str, dict[str, Any]]:
     by_horizon: dict[str, dict[str, list[dict[str, Any]]]] = {}
     for hz in ("eos", "mid"):
         if "batch_horizon" in df.columns and hz not in set(df["batch_horizon"].astype(str)):
             continue
-        by_crop: dict[str, list[dict[str, Any]]] = {
-            "all": build_sample_scatter_slice(
-                df, batch_horizon=hz, representatives=representatives
-            ),
+        by_crop: dict[str, Any] = {}
+        all_fams = build_sample_scatter_slice(
+            df, batch_horizon=hz, representatives=representatives
+        )
+        by_crop["all"] = {
+            "families": all_fams,
+            "summary": summarize_sample_scatter(all_fams),
         }
         if "crop" in df.columns:
             for crop in sorted({str(c) for c in df["crop"].dropna().unique()}):
-                by_crop[crop] = build_sample_scatter_slice(
+                fams = build_sample_scatter_slice(
                     df, batch_horizon=hz, crop=crop, representatives=representatives
                 )
+                by_crop[crop] = {
+                    "families": fams,
+                    "summary": summarize_sample_scatter(fams),
+                }
         by_horizon[hz] = by_crop
     return by_horizon
 

cybench/runs/viz/dashboard_template.html

@@ -106,6 +106,20 @@
       text-transform: uppercase;
       letter-spacing: 0.03em;
     }
+    th.group-head .group-hint {
+      display: block;
+      font-weight: 400;
+      font-size: 0.65rem;
+      text-transform: none;
+      letter-spacing: 0;
+      margin-top: 0.12rem;
+      line-height: 1.2;
+    }
+    th .metric-sub {
+      font-weight: 500;
+      font-size: 0.72rem;
+      color: var(--muted);
+    }
     tr.metric-row { cursor: pointer; }
     tr.metric-row:hover td { filter: brightness(0.97); }
     tr.metric-row.selected td {
@@ -251,11 +265,41 @@ <h3>Drill-down plots</h3>
       { key: "region_year_r", label: "r", group: "Region-Year", higherBetter: true, title: "Pearson r on all region-year rows" },
       { key: "region_year_r2", label: "R²", group: "Region-Year", higherBetter: true, title: "R² on all region-year rows" },
       { key: "region_year_nrmse", label: "NRMSE", group: "Region-Year", higherBetter: false, title: "Normalized RMSE on absolute yields" },
-      { key: "spatial_r2", label: "R² (med/yr)", group: "Spatial", higherBetter: true, title: "Median across years of within-year cross-region R² (years with ≥3 regions)" },
-      { key: "spatial_r2_agg", label: "R² (agg)", group: "Spatial", higherBetter: true, title: "R² on regional mean yields (mean across years per region)" },
-      { key: "temporal_r2", label: "R² (med/reg)", group: "Temporal", higherBetter: true, title: "Median across regions of within-region cross-year R² (regions with ≥3 years)" },
-      { key: "temporal_r2_agg", label: "R² (agg)", group: "Temporal", higherBetter: true, title: "R² on national mean yield series across years" },
-      { key: "anomaly_r2", label: "R² (med/reg)", group: "Anomaly", higherBetter: true, title: "Median across regions of R² on yield minus regional mean" },
+      {
+        key: "spatial_r2",
+        label: "R²<br><span class=\"metric-sub\">med/yr</span>",
+        group: "Spatial",
+        higherBetter: true,
+        title: "Slice by year (aggregate across time); R² across regions each year; then median across years (years with ≥3 regions)",
+      },
+      {
+        key: "spatial_r2_agg",
+        label: "R²<br><span class=\"metric-sub\">agg</span>",
+        group: "Spatial",
+        higherBetter: true,
+        title: "Mean yield across years per region (aggregate across time); then one R² across regions",
+      },
+      {
+        key: "temporal_r2",
+        label: "R²<br><span class=\"metric-sub\">med/reg</span>",
+        group: "Temporal",
+        higherBetter: true,
+        title: "Slice by region (aggregate across space); R² across years per region; then median across regions (regions with ≥3 years)",
+      },
+      {
+        key: "temporal_r2_agg",
+        label: "R²<br><span class=\"metric-sub\">agg</span>",
+        group: "Temporal",
+        higherBetter: true,
+        title: "Mean yield across regions each year (aggregate across space); then one R² across years",
+      },
+      {
+        key: "anomaly_r2",
+        label: "R²<br><span class=\"metric-sub\">med/reg</span>",
+        group: "Anomaly",
+        higherBetter: true,
+        title: "Per region: R² on yield minus regional mean; then median across regions",
+      },
     ];
 
     // Absolute color scales — comparable across rows and dashboards.
@@ -442,8 +486,8 @@ <h3>Drill-down plots</h3>
       <th rowspan="2" class="sortable" data-sort="n_years">Years ${sortMark("n_years")}</th>
       <th rowspan="2" class="sortable" data-sort="n_samples">Samples ${sortMark("n_samples")}</th>
       <th colspan="3" class="group-head">Region-Year</th>
-      <th colspan="1" class="group-head">Spatial</th>
-      <th colspan="1" class="group-head">Temporal</th>
+      <th colspan="2" class="group-head" title="Spatial pattern recovery">Spatial<span class="group-hint">slice by year → R² → median</span></th>
+      <th colspan="2" class="group-head" title="Temporal dynamics recovery">Temporal<span class="group-hint">slice by region → R² → median</span></th>
       <th colspan="1" class="group-head">Anomaly</th>
     </tr>
     <tr>`;

cybench/runs/viz/model_family_radar_template.html

@@ -177,7 +177,7 @@ <h2>Raw metrics (median across datasets)</h2>
 
     <div class="card">
       <h2>Performance vs training set size</h2>
-      <p class="muted" id="scatter-note">Each point is one crop×country dataset for the family representative. Y-axis: model NRMSE divided by average yield NRMSE (1.0 = baseline; lower is better). X-axis: mean training rows across walk-forward splits.</p>
+      <p class="muted" id="scatter-note">Each point is one crop×country dataset for the family representative. Y-axis: model NRMSE ÷ average yield NRMSE (1.0 = baseline; lower is better). X-axis: log₁₀(mean training rows); view spans the 5th–95th percentile range.</p>
       <div class="scatter-panel" id="scatter-panel"></div>
     </div>
   </div>
@@ -296,82 +296,150 @@ <h2>Performance vs training set size</h2>
       `).join("");
     }
 
-    function currentScatterFamilies() {
+    function currentScatterSlice() {
       const hz = horizonSelect.value;
       const crop = cropSelect.value;
-      return ((DATA.sample_scatter || {})[hz] || {})[crop] || [];
+      const raw = ((DATA.sample_scatter || {})[hz] || {})[crop] || [];
+      if (Array.isArray(raw)) {
+        return { families: raw, summary: {} };
+      }
+      return { families: raw.families || [], summary: raw.summary || {} };
+    }
+
+    function percentile(sorted, p) {
+      if (!sorted.length) return 0;
+      const idx = (sorted.length - 1) * p;
+      const lo = Math.floor(idx);
+      const hi = Math.ceil(idx);
+      if (lo === hi) return sorted[lo];
+      return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
+    }
+
+    function spearmanRho(xs, ys) {
+      if (xs.length < 5) return null;
+      const rank = arr => {
+        const order = arr.map((v, i) => [v, i]).sort((a, b) => a[0] - b[0]);
+        const out = new Array(arr.length);
+        order.forEach(([_, i], r) => { out[i] = r + 1; });
+        return out;
+      };
+      const rx = rank(xs);
+      const ry = rank(ys);
+      const mx = rx.reduce((a, b) => a + b, 0) / rx.length;
+      const my = ry.reduce((a, b) => a + b, 0) / ry.length;
+      let num = 0, dx = 0, dy = 0;
+      for (let i = 0; i < rx.length; i++) {
+        const vx = rx[i] - mx;
+        const vy = ry[i] - my;
+        num += vx * vy;
+        dx += vx * vx;
+        dy += vy * vy;
+      }
+      return dx && dy ? num / Math.sqrt(dx * dy) : null;
     }
 
     function renderScatter() {
       const metricKey = scatterMetric.key || "relative_nrmse";
       const refY = scatterMetric.reference ?? 1.0;
-      const families = currentScatterFamilies();
+      const { families, summary } = currentScatterSlice();
       const series = families
         .map(fam => ({
           ...fam,
           points: (fam.points || []).filter(
-            p => p[metricKey] !== null && p[metricKey] !== undefined
+            p => p[metricKey] !== null && p[metricKey] !== undefined && p.n_train > 0
           ),
         }))
         .filter(fam => fam.points.length);
 
       if (!series.length) {
         scatterPanel.innerHTML =
           '<p class="muted">No scatter data for this selection (re-collect summaries with average_yield baseline and n_train).</p>';
+        scatterNote.textContent =
+          "Each point is one crop×country dataset for the family representative.";
         return;
       }
 
+      const allPoints = series.flatMap(fam =>
+        fam.points.map(p => ({ ...p, family: fam.family, color: fam.color, rep: fam.display_name }))
+      );
+      const nTrainSorted = allPoints.map(p => p.n_train).sort((a, b) => a - b);
+      const xP05 = summary.x_p05 ?? Math.round(percentile(nTrainSorted, 0.05));
+      const xP95 = summary.x_p95 ?? Math.round(percentile(nTrainSorted, 0.95));
+      const xP50 = summary.x_p50 ?? Math.round(percentile(nTrainSorted, 0.5));
+      const log = v => Math.log10(Math.max(v, 1));
+      const logP05 = log(xP05);
+      const logP95 = log(xP95);
+      const logPad = 0.04 * (logP95 - logP05 || 1);
+
+      const corePts = allPoints.filter(p => p.n_train >= xP05 && p.n_train <= xP95);
+      const rho = summary.spearman_rho_core ?? spearmanRho(
+        corePts.map(p => p.n_train),
+        corePts.map(p => p[metricKey])
+      );
+      const nOut = summary.n_outliers_x ?? allPoints.length - corePts.length;
+
       const width = 720;
-      const height = 360;
-      const pad = { l: 56, r: 20, t: 20, b: 48 };
+      const height = 380;
+      const pad = { l: 58, r: 24, t: 20, b: 52 };
       const innerW = width - pad.l - pad.r;
       const innerH = height - pad.t - pad.b;
 
-      const allPoints = series.flatMap(fam =>
-        fam.points.map(p => ({ ...p, family: fam.family, color: fam.color, rep: fam.display_name }))
-      );
-      const xs = allPoints.map(p => p.n_train);
       const ys = allPoints.map(p => p[metricKey]);
-      const xMin = Math.min(...xs);
-      const xMax = Math.max(...xs);
       const yMin = Math.max(0, Math.min(...ys, refY) - 0.08);
       const yMax = Math.max(...ys, refY) + 0.08;
-      const xScale = v => pad.l + ((v - xMin) / (xMax - xMin || 1)) * innerW;
+      const xScale = v => pad.l + ((log(v) - logP05 + logPad) / (logP95 - logP05 + 2 * logPad || 1)) * innerW;
       const yScale = v => pad.t + innerH - ((v - yMin) / (yMax - yMin || 1)) * innerH;
 
       let svg = `<svg viewBox="0 0 ${width} ${height}" role="img" aria-label="Performance vs training size">`;
       svg += `<rect x="${pad.l}" y="${pad.t}" width="${innerW}" height="${innerH}" fill="#fff" stroke="#e6eaef"/>`;
 
       const yRef = yScale(refY);
       svg += `<line x1="${pad.l}" y1="${yRef}" x2="${pad.l + innerW}" y2="${yRef}" stroke="#bbb" stroke-dasharray="5 4"/>`;
-      svg += `<text x="${pad.l + innerW + 4}" y="${yRef + 3}" font-size="10" fill="#656d76">baseline</text>`;
+      svg += `<text x="${pad.l + innerW + 4}" y="${yRef + 3}" font-size="10" fill="#656d76">1.0</text>`;
 
+      const tickVals = [xP05, xP50, xP95].filter((v, i, a) => a.indexOf(v) === i);
+      tickVals.forEach(tx => {
+        svg += `<text x="${xScale(tx)}" y="${height - 12}" text-anchor="middle" font-size="10" fill="#656d76">${tx >= 1000 ? `${Math.round(tx / 1000)}k` : tx}</text>`;
+      });
       for (let i = 0; i <= 4; i++) {
-        const tx = xMin + (i / 4) * (xMax - xMin);
         const ty = yMin + (i / 4) * (yMax - yMin);
-        svg += `<text x="${xScale(tx)}" y="${height - 10}" text-anchor="middle" font-size="10" fill="#656d76">${Math.round(tx)}</text>`;
         svg += `<text x="10" y="${yScale(ty) + 3}" text-anchor="start" font-size="10" fill="#656d76">${ty.toFixed(2)}</text>`;
       }
-      svg += `<text x="${pad.l + innerW / 2}" y="${height - 2}" text-anchor="middle" font-size="11" fill="#1f2328">Training rows (mean)</text>`;
+      svg += `<text x="${pad.l + innerW / 2}" y="${height - 2}" text-anchor="middle" font-size="11" fill="#1f2328">Training rows (mean, log scale; ${xP05.toLocaleString()}–${xP95.toLocaleString()})</text>`;
       svg += `<text x="16" y="${pad.t + innerH / 2}" text-anchor="middle" font-size="11" fill="#1f2328" transform="rotate(-90 16 ${pad.t + innerH / 2})">${scatterMetric.label || metricKey}</text>`;
 
-      series.forEach(fam => {
-        fam.points.forEach(p => {
-          const cx = xScale(p.n_train);
-          const cy = yScale(p[metricKey]);
-          svg += `<circle cx="${cx}" cy="${cy}" r="6" fill="${fam.color}" fill-opacity="0.82" stroke="#fff" stroke-width="1.2">
-            <title>${fam.family} · ${p.display_name} · ${p.dataset}
-Train=${p.n_train}, rel NRMSE=${p[metricKey]}
+      allPoints.forEach(p => {
+        const isOut = p.n_train < xP05 || p.n_train > xP95;
+        const cx = xScale(Math.min(Math.max(p.n_train, xP05), xP95));
+        const cy = yScale(p[metricKey]);
+        const r = isOut ? 5 : 6;
+        const stroke = isOut ? p.color : "#fff";
+        const sw = isOut ? 2 : 1.2;
+        const fill = isOut ? "#fff" : p.color;
+        const outLabel = isOut ? " [outside range]" : "";
+        svg += `<circle cx="${cx}" cy="${cy}" r="${r}" fill="${fill}" fill-opacity="${isOut ? 1 : 0.82}" stroke="${stroke}" stroke-width="${sw}">
+            <title>${p.family} · ${p.display_name} · ${p.dataset}${outLabel}
+Train=${p.n_train.toLocaleString()}, rel NRMSE=${p[metricKey]}
 (model=${p.nrmse}, baseline=${p.baseline_nrmse})</title>
           </circle>`;
-        });
       });
       svg += "</svg>";
 
       const legend = series.map(fam =>
         `<span style="color:${fam.color}"><strong>${fam.family}</strong> · ${fam.display_name}</span>`
       ).join("");
 
+      let statLine = "";
+      if (rho !== null && rho !== undefined) {
+        const strength = Math.abs(rho) < 0.15 ? "no clear" : Math.abs(rho) < 0.35 ? "weak" : "moderate";
+        const dir = rho < -0.05 ? "more data → lower relative NRMSE" : rho > 0.05 ? "more data → higher relative NRMSE" : "flat trend";
+        statLine = `Spearman ρ = ${Number(rho).toFixed(2)} across datasets in range (${strength} trend; ${dir}). `;
+      }
+      scatterNote.textContent =
+        `Y-axis: model NRMSE ÷ average yield NRMSE (1.0 = baseline). X-axis uses log₁₀ scale over the 5th–95th percentile of training size (median ${xP50.toLocaleString()} rows). `
+        + `${nOut} point${nOut === 1 ? "" : "s"} outside that range shown as open circles at the axis edge. `
+        + statLine;
+
       scatterPanel.innerHTML = `${svg}<div class="scatter-legend">${legend}</div>`;
     }
 

tests/runs/test_collect_walk_forward_results.py

@@ -225,7 +225,7 @@ def test_summary_rows_to_dashboard_records(tmp_path: Path):
     (assets / "maize_NL_scatter.png").write_bytes(b"png")
 
     records = summary_rows_to_dashboard_records(rows, tmp_path)
-    assert len(records) == 6
+    assert len(records) == 8
     assert records[0]["view"] == "region_year"
     nrmse_rec = next(r for r in records if r["view"] == "region_year" and r["metric"] == "nrmse")
     assert nrmse_rec["value"] == 0.31
@@ -234,6 +234,10 @@ def test_summary_rows_to_dashboard_records(tmp_path: Path):
     assert r_rec["value_std"] == 0.004
     r2_rec = next(r for r in records if r["view"] == "region_year" and r["metric"] == "r2")
     assert r2_rec["value_std"] is None
+    sp_agg = next(r for r in records if r["view"] == "spatial" and r["metric"] == "r2_agg")
+    assert sp_agg["value"] == -0.93
+    tm_agg = next(r for r in records if r["view"] == "temporal" and r["metric"] == "r2_agg")
+    assert tm_agg["value"] == -5.10
     scatter_recs = [r for r in records if r.get("images", {}).get("scatter")]
     assert scatter_recs
     assert "maize_NL" in scatter_recs[0]["images"]["scatter"]