continent

Author: michielkallenberg

cybench/runs/analysis/global_insights_lib.py

@@ -1230,6 +1230,204 @@ def build_horizon_skill_curves_payload(
     }
 
 
+CONTINENT_REGION_ORDER: tuple[str, ...] = (
+    "Africa",
+    "Asia",
+    "Europe",
+    "North America",
+    "South America",
+)
+
+
+def _country_to_continent() -> dict[str, str]:
+    from cybench.config import CONTINENT_DICT
+
+    out: dict[str, str] = {}
+    for continent, codes in CONTINENT_DICT.items():
+        for cc in codes:
+            out[str(cc).upper()] = continent
+    return out
+
+
+def _continent_region_id(label: str) -> str:
+    if label == "Global":
+        return "global"
+    return label.lower().replace(" ", "_")
+
+
+def _region_country_sets(work: pd.DataFrame) -> list[tuple[str, str, set[str]]]:
+    """(region_id, label, countries) for Global and each continent with data."""
+    if work.empty or "country" not in work.columns:
+        return [("global", "Global", set())]
+    all_countries = {str(c).upper() for c in work["country"].unique()}
+    cc_to_cont = _country_to_continent()
+    from cybench.config import CONTINENT_DICT
+
+    regions: list[tuple[str, str, set[str]]] = [("global", "Global", all_countries)]
+    for continent in CONTINENT_REGION_ORDER:
+        codes = {str(c).upper() for c in CONTINENT_DICT.get(continent, [])}
+        present = all_countries & codes
+        if present:
+            regions.append((_continent_region_id(continent), continent, present))
+    return regions
+
+
+def _filter_work_to_countries(work: pd.DataFrame, countries: set[str]) -> pd.DataFrame:
+    if work.empty or not countries:
+        return work.iloc[0:0].copy()
+    upper = {str(c).upper() for c in countries}
+    return work[work["country"].astype(str).str.upper().isin(upper)].copy()
+
+
+def build_continent_horizon_payload(
+    df: pd.DataFrame,
+    *,
+    representatives: dict[str, str] | None = None,
+) -> dict[str, Any]:
+    """Per-region horizon medians for EOS family representatives."""
+    from cybench.runs.analysis.model_family_radar_lib import (
+        FAMILY_COLORS,
+        FAMILY_ORDER,
+        MODEL_DISPLAY_NAMES,
+        pick_representatives,
+    )
+
+    horizons = horizons_in_data(df)
+    if len(horizons) < 2:
+        return {
+            "horizons": [],
+            "by_crop": {},
+            "note": "Need at least two forecast horizons with collected summaries.",
+        }
+
+    rep_source_hz = "eos" if "eos" in horizons else horizons[-1]
+    rep_work = _filter_summary_work(df, batch_horizon=rep_source_hz)
+    reps = pick_representatives(rep_work, overrides=representatives)
+    trend_model = reps.get("Naive baselines", "trend")
+
+    value_columns = tuple(c for c in HORIZON_SKILL_VALUE_COLUMNS if c in df.columns)
+    crop_keys = ["all", *_crop_keys(df)]
+    by_crop: dict[str, Any] = {}
+
+    for crop_key in crop_keys:
+        crop_filter = None if crop_key == "all" else crop_key
+        crop_work = _filter_summary_work(df, crop=crop_filter)
+        region_specs = _region_country_sets(crop_work)
+        regions_meta: list[dict[str, Any]] = []
+        entries: list[dict[str, Any]] = []
+
+        for region_id, region_label, countries in region_specs:
+            regional_work = _filter_work_to_countries(crop_work, countries)
+            regions_meta.append(
+                {
+                    "id": region_id,
+                    "label": region_label,
+                    "n_countries": len(countries),
+                }
+            )
+            if regional_work.empty:
+                continue
+            for family in FAMILY_ORDER:
+                model = reps.get(family)
+                if not model or model == trend_model:
+                    continue
+                entry = _build_model_horizon_entry_independent(
+                    regional_work,
+                    model=model,
+                    trend_model=trend_model,
+                    horizons=horizons,
+                    value_columns=value_columns,
+                )
+                if entry is None:
+                    continue
+                entries.append(
+                    {
+                        "region": region_label,
+                        "region_id": region_id,
+                        "family": family,
+                        "display": MODEL_DISPLAY_NAMES.get(model, model),
+                        "color": FAMILY_COLORS.get(family, "#666"),
+                        **entry,
+                    }
+                )
+
+        by_crop[crop_key] = {
+            "regions": regions_meta,
+            "entries": entries,
+        }
+
+    rep_labels = ", ".join(f"{f}: {m}" for f, m in reps.items())
+    return {
+        "horizons": [
+            {"id": hz, "label": HORIZON_DISPLAY_LABELS.get(hz, hz), "order": i}
+            for i, hz in enumerate(horizons)
+        ],
+        "representatives_summary": rep_labels,
+        "by_crop": by_crop,
+        "note": (
+            "EOS family representatives only. Median across countries within each region "
+            "(each country weighs equally). Horizons scored independently — blank cells mean "
+            "no runs at that horizon in that region. Small regions (e.g. Asia, Americas) may "
+            "have few countries; check n in tooltips via country counts per horizon in the "
+            "exported table."
+        ),
+    }
+
+
+def build_continent_horizon_table(
+    df: pd.DataFrame,
+    *,
+    crop: str | None = None,
+    horizons: tuple[str, ...] | None = None,
+    metrics: tuple[str, ...] = ("nrmse", "r2", "r_spatial", "r_temporal", "r_res"),
+    representatives: dict[str, str] | None = None,
+) -> pd.DataFrame:
+    """Flatten :func:`build_continent_horizon_payload` to a CSV-friendly table."""
+    payload = build_continent_horizon_payload(df, representatives=representatives)
+    horizons = horizons or tuple(h["id"] for h in payload.get("horizons", []))
+    crop_key = crop or "all"
+    slice_data = payload.get("by_crop", {}).get(crop_key, {})
+    present_metrics = tuple(m for m in metrics if m in HORIZON_SKILL_VALUE_COLUMNS)
+
+    rows: list[dict[str, Any]] = []
+    for entry in slice_data.get("entries", []):
+        row: dict[str, Any] = {
+            "region": entry.get("region"),
+            "region_id": entry.get("region_id"),
+            "family": entry.get("family"),
+            "model": entry.get("model"),
+            "display_name": entry.get("display"),
+            "eos_only": entry.get("eos_only", False),
+        }
+        points_by_hz = {p["horizon"]: p for p in entry.get("points", [])}
+        for hz in horizons:
+            pt = points_by_hz.get(hz)
+            row[f"n_countries_{hz}"] = pt.get("n_countries") if pt else None
+            for metric in present_metrics:
+                stats = (pt.get("metrics") or {}).get(metric) if pt else None
+                if isinstance(stats, dict):
+                    row[f"{hz}_{metric}_median"] = stats.get("median")
+                    row[f"{hz}_{metric}_q25"] = stats.get("q25")
+                    row[f"{hz}_{metric}_q75"] = stats.get("q75")
+                else:
+                    row[f"{hz}_{metric}_median"] = float("nan")
+                    row[f"{hz}_{metric}_q25"] = float("nan")
+                    row[f"{hz}_{metric}_q75"] = float("nan")
+        rows.append(row)
+
+    if not rows:
+        return pd.DataFrame()
+    region_order = ["Global", *list(CONTINENT_REGION_ORDER)]
+    ord_map = {name: i for i, name in enumerate(region_order)}
+    out = pd.DataFrame(rows)
+    out["_region_ord"] = out["region"].map(lambda r: ord_map.get(r, 99))
+    return (
+        out.sort_values(["_region_ord", "family", "model"])
+        .drop(columns=["_region_ord"])
+        .reset_index(drop=True)
+    )
+
+
 def _model_country_count(work: pd.DataFrame, model: str) -> int:
     sub = work[work["model"] == model]
     if sub.empty or "country" not in sub.columns:
@@ -1523,6 +1721,7 @@ def build_insights_payload(output_root: Path, *, version: int = 2) -> dict[str,
         "model_country": model_country,
         "model_country_skilled": model_country_skilled,
         "horizon_skill_curves": build_horizon_skill_curves_payload(df),
+        "continent_horizons": build_continent_horizon_payload(df),
         "crop_comparison": build_crop_comparison_payload(df),
         "country_map_cc": country_map_cc,
         "benchmark_map_isos": benchmark_map_isos,

cybench/runs/viz/global_insights_template.html

@@ -502,6 +502,10 @@ <h2>Forecast horizon vs performance</h2>
       <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">By family (median across countries)</h3>
       <div class="table-scroll" id="horizon-curves-table-wrap"></div>
       <p class="muted note" id="horizon-curves-metric-note"></p>
+      <h3 style="font-size:1rem;margin:1.5rem 0 0.5rem;">By region (median across countries)</h3>
+      <p class="muted" id="horizon-continent-note"></p>
+      <p class="muted" id="horizon-continent-meta"></p>
+      <div class="table-scroll" id="horizon-continent-table-wrap"></div>
     </div>
 
     <div class="card">
@@ -2032,11 +2036,16 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
     const hscAxisToolbar = document.getElementById("horizon-curves-axis-toolbar");
     const hscMetricWrap = document.getElementById("horizon-curves-metric-wrap");
     const hscMetricSelect = document.getElementById("horizon-curves-metric");
+    const hscContinentNote = document.getElementById("horizon-continent-note");
+    const hscContinentMeta = document.getElementById("horizon-continent-meta");
+    const hscContinentTableWrap = document.getElementById("horizon-continent-table-wrap");
     let hscCrop = "all";
     let hscAxis = "overall";
     let hscMetric = "nrmse";
     let hscTableSort = "family";
     let hscTableDir = "asc";
+    let hscContinentTableSort = "region";
+    let hscContinentTableDir = "asc";
 
     const HSC_AXES = HSC.axes || MATRIX_AXES.map(axis => ({
       ...axis,
@@ -2160,6 +2169,11 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
     }
 
     hscNote.textContent = HSC.note || "Horizon skill curves require at least two collected horizons.";
+    if (hscContinentNote) {
+      hscContinentNote.textContent = (DATA.continent_horizons || {}).note || (
+        "Family representatives by continent (median across countries in each region)."
+      );
+    }
 
     const hscCropKeys = Object.keys(HSC.by_crop || {});
     if (hscCropKeys.length) {
@@ -2202,6 +2216,7 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
         hscChart.innerHTML = `<p class="muted">Not enough paired horizon data for this crop.</p>`;
         hscLegend.innerHTML = "";
         hscTableWrap.innerHTML = `<p class="muted">No horizon table for this crop.</p>`;
+        renderHorizonContinentTable();
         return;
       }
 
@@ -2252,6 +2267,55 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
         metricValueFn: hscMetricValue,
         cellBgFn: hscCellBg,
       });
+      renderHorizonContinentTable();
+    }
+
+    function renderHorizonContinentTable() {
+      if (!hscContinentTableWrap) return;
+      const CONT = DATA.continent_horizons || {};
+      const slice = (CONT.by_crop || {})[hscCrop] || { entries: [], regions: [] };
+      const horizons = CONT.horizons || HSC.horizons || [];
+      const metricCol = hscMetricColumn(hscMetric);
+      const entries = slice.entries || [];
+
+      const regionBits = (slice.regions || [])
+        .filter(r => (r.n_countries || 0) > 0)
+        .map(r => `${r.label} (${r.n_countries})`);
+      let meta = regionBits.length ? regionBits.join(" · ") : "No regional breakdown for this crop.";
+      const reps = CONT.representatives_summary || HSC.representatives_summary || "";
+      if (reps) meta += ` · Representatives: ${reps}`;
+      if (hscContinentMeta) hscContinentMeta.textContent = meta;
+
+      if (!entries.length || horizons.length < 2) {
+        hscContinentTableWrap.innerHTML = `<p class="muted">No regional horizon table for this crop.</p>`;
+        return;
+      }
+
+      renderHorizonPointsTable({
+        container: hscContinentTableWrap,
+        horizons,
+        entries,
+        leadingCols: [
+          { key: "region", label: "Region", value: entry => entry.region },
+          { key: "family", label: "Family", value: entry => entry.family },
+          { key: "model", label: "Model", value: entry => entry.display || entry.model },
+        ],
+        sortKey: hscContinentTableSort,
+        sortDir: hscContinentTableDir,
+        onSort: (key) => {
+          if (hscContinentTableSort === key) {
+            hscContinentTableDir = hscContinentTableDir === "asc" ? "desc" : "asc";
+          } else {
+            hscContinentTableSort = key;
+            hscContinentTableDir = key === "region" || key === "family" || key === "model" ? "asc" : "desc";
+          }
+          renderHorizonContinentTable();
+        },
+        metricId: hscMetric,
+        metricCol,
+        metricValueFn: hscMetricValue,
+        cellBgFn: hscCellBg,
+      });
     }
 
     hscCropSelect.addEventListener("change", () => {

tests/runs/test_global_insights.py

@@ -12,6 +12,8 @@
     aggregate_model_leaderboard,
     attach_baseline_metrics,
     build_dashboard_hrefs,
+    build_continent_horizon_payload,
+    build_continent_horizon_table,
     build_family_models_horizon_table,
     build_horizon_skill_curves_payload,
     build_insights_payload,
@@ -583,6 +585,73 @@ def test_horizon_skill_curves_eos_only_lpjml_in_table_not_plot(tmp_path: Path):
     assert "plot_excluded_note" in payload
 
 
+def test_continent_horizon_payload_splits_regions():
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        df = _three_horizon_fixture(Path(tmp))
+        za_rows = [
+            {
+                "crop": "maize",
+                "country": "ZA",
+                "model": model,
+                "batch_horizon": hz,
+                "nrmse": nrmse,
+                "r2": 0.5,
+                "r_spatial": 0.5,
+                "r_temporal": 0.4,
+                "r_res": 0.3,
+                "n_samples": 30,
+            }
+            for hz, nrmse_by_model in [
+                ("mid", {"ridge": 0.40, "trend": 0.44, "xgboost": 0.38, "average": 0.46, "lstm_lf": 0.45}),
+                ("qtr", {"ridge": 0.32, "trend": 0.40, "xgboost": 0.30, "average": 0.42, "lstm_lf": 0.37}),
+                ("eos", {"ridge": 0.25, "trend": 0.38, "xgboost": 0.24, "average": 0.40, "lstm_lf": 0.30}),
+            ]
+            for model, nrmse in nrmse_by_model.items()
+        ]
+        df = pd.concat([df, pd.DataFrame(za_rows)], ignore_index=True)
+
+        payload = build_continent_horizon_payload(df)
+        assert len(payload["horizons"]) == 3
+        maize = payload["by_crop"]["maize"]
+        regions = {r["label"]: r["n_countries"] for r in maize["regions"]}
+        assert regions["Global"] == 4
+        assert regions["Africa"] == 1
+        assert regions["Europe"] == 2
+        assert regions["North America"] == 1
+
+        europe_xgb = next(
+            e
+            for e in maize["entries"]
+            if e["region"] == "Europe" and e["model"] == "xgboost"
+        )
+        africa_xgb = next(
+            e
+            for e in maize["entries"]
+            if e["region"] == "Africa" and e["model"] == "xgboost"
+        )
+        europe_eos = next(
+            p["metrics"]["nrmse"]["median"]
+            for p in europe_xgb["points"]
+            if p["horizon"] == "eos"
+        )
+        africa_eos = next(
+            p["metrics"]["nrmse"]["median"]
+            for p in africa_xgb["points"]
+            if p["horizon"] == "eos"
+        )
+        assert europe_eos < africa_eos
+
+        table = build_continent_horizon_table(df, crop="maize")
+        assert "region" in table.columns
+        assert set(table["region"]) >= {"Global", "Europe", "Africa"}
+
+        insights = build_insights_payload(Path(tmp), version=1)
+        assert "continent_horizons" in insights
+        assert insights["continent_horizons"]["by_crop"]["maize"]["regions"]
+
+
 def test_horizon_models_include_partial_early_coverage():
     df = pd.DataFrame(
         [