Skip to content

Commit 0f53d1f

Browse files
michielkallenbergmichielkallenberg
committed
dashboard
1 parent e1a59fe commit 0f53d1f

2 files changed

Lines changed: 134 additions & 57 deletions

File tree

‎cybench/runs/viz/global_insights_template.html‎

Lines changed: 99 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -257,9 +257,17 @@ <h2>Forecast horizon vs performance</h2>
257257
<label>Crop
258258
<select id="horizon-curves-crop"></select>
259259
</label>
260-
<label>Metric
260+
<span class="muted">Axis</span>
261+
<div class="toolbar" id="horizon-curves-axis-toolbar">
262+
<button type="button" data-axis="overall" class="active">Overall</button>
263+
<button type="button" data-axis="spatial">Spatial</button>
264+
<button type="button" data-axis="temporal">Temporal</button>
265+
<button type="button" data-axis="anomaly">Anomaly</button>
266+
</div>
267+
<label id="horizon-curves-metric-wrap">Metric
261268
<select id="horizon-curves-metric">
262-
<option value="nrmse" selected>Median NRMSE</option>
269+
<option value="nrmse" selected>NRMSE (median)</option>
270+
<option value="r2">R² (median)</option>
263271
<option value="skill_vs_trend">Skill vs trend</option>
264272
</select>
265273
</label>
@@ -796,12 +804,85 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
796804
const hscTableWrap = document.getElementById("horizon-curves-table-wrap");
797805
const hscMetricNote = document.getElementById("horizon-curves-metric-note");
798806
const hscCropSelect = document.getElementById("horizon-curves-crop");
807+
const hscAxisToolbar = document.getElementById("horizon-curves-axis-toolbar");
808+
const hscMetricWrap = document.getElementById("horizon-curves-metric-wrap");
799809
const hscMetricSelect = document.getElementById("horizon-curves-metric");
800810
let hscCrop = "all";
811+
let hscAxis = "overall";
801812
let hscMetric = "nrmse";
802813
let hscTableSort = "family";
803814
let hscTableDir = "asc";
804815

816+
const HSC_AXES = HSC.axes || MATRIX_AXES.map(axis => ({
817+
...axis,
818+
metrics: axis.id === "overall"
819+
? [
820+
{ id: "nrmse", column: "nrmse", label: "NRMSE", higher_better: false, has_iqr: true },
821+
{ id: "r2", column: "r2", label: "R²", higher_better: true, has_iqr: true },
822+
{ id: "skill_vs_trend", column: "skill_vs_trend", label: "Skill vs trend", higher_better: true, has_iqr: false },
823+
]
824+
: (axis.metrics || []).map(m => ({
825+
...m,
826+
column: m.column || (axis.id === "spatial" ? "r_spatial" : axis.id === "temporal" ? "r_temporal" : "r_res"),
827+
has_iqr: true,
828+
})),
829+
}));
830+
831+
function hscAxisDef(axisId) {
832+
return HSC_AXES.find(a => a.id === axisId) || HSC_AXES[0] || { id: "overall", metrics: [] };
833+
}
834+
835+
function hscMetricSpec() {
836+
const axis = hscAxisDef(hscAxis);
837+
const metrics = axis.metrics || [];
838+
return metrics.find(m => m.id === hscMetric) || metrics[0] || { id: "nrmse", column: "nrmse", higher_better: false, has_iqr: true };
839+
}
840+
841+
function hscMetricColumn(metricId) {
842+
const axis = hscAxisDef(hscAxis);
843+
const spec = (axis.metrics || []).find(m => m.id === metricId);
844+
return spec?.column || metricId;
845+
}
846+
847+
function syncHscMetricControl() {
848+
const axis = hscAxisDef(hscAxis);
849+
const metrics = axis.metrics || [];
850+
const showPicker = metrics.length > 1;
851+
hscMetricWrap.style.display = showPicker ? "" : "none";
852+
if (!showPicker) {
853+
hscMetric = metrics[0]?.id || "nrmse";
854+
return;
855+
}
856+
const options = metrics.map(m => `<option value="${m.id}">${m.label} (median)</option>`).join("");
857+
const prev = hscMetric;
858+
hscMetricSelect.innerHTML = options;
859+
hscMetric = metrics.some(m => m.id === prev) ? prev : metrics[0].id;
860+
hscMetricSelect.value = hscMetric;
861+
}
862+
863+
function hscMetricValue(point, metricId) {
864+
const col = hscMetricColumn(metricId);
865+
const stats = (point.metrics || {})[col];
866+
return stats?.median ?? null;
867+
}
868+
869+
function hscMetricIqr(point, metricId) {
870+
const spec = hscMetricSpec();
871+
if (spec.has_iqr === false) return { low: null, high: null };
872+
const col = hscMetricColumn(metricId);
873+
const stats = (point.metrics || {})[col];
874+
if (!stats) return { low: null, high: null };
875+
return { low: stats.q25 ?? null, high: stats.q75 ?? null };
876+
}
877+
878+
function hscCellBg(v, metricId) {
879+
if (v === null || v === undefined || Number.isNaN(v)) return null;
880+
if (metricId === "nrmse") return fixedNrmseBg(v);
881+
if (metricId === "r2" || metricId === "skill_vs_trend") return fixedR2Bg(v);
882+
if (metricId === "r" || String(metricId).startsWith("r_")) return fixedRBg(v);
883+
return null;
884+
}
885+
805886
hscNote.textContent = HSC.note || "Horizon skill curves require at least two collected horizons.";
806887

807888
const hscCropKeys = Object.keys(HSC.by_crop || {});
@@ -816,23 +897,16 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
816897
hscCropSelect.innerHTML = `<option value="all">All crops</option>`;
817898
}
818899

819-
function hscMetricValue(point, metric) {
820-
if (metric === "skill_vs_trend") return point.median_skill_vs_trend;
821-
return point.median_nrmse;
822-
}
823-
824-
function hscMetricIqr(point, metric) {
825-
if (metric === "skill_vs_trend") return { low: null, high: null };
826-
return { low: point.q25_nrmse, high: point.q75_nrmse };
827-
}
828-
829900
function renderHorizonSkillCurves() {
830901
const slice = (HSC.by_crop || {})[hscCrop] || { families: [], n_countries: 0, countries: [] };
831902
const horizons = HSC.horizons || [];
832903
const metricNotes = HSC.metric_notes || {};
904+
const metricCol = hscMetricColumn(hscMetric);
905+
const axisDef = hscAxisDef(hscAxis);
906+
const metricDef = hscMetricSpec();
833907
const plotFamilies = (slice.families || []).filter(fam => fam.plot !== false);
834908
const tableFamilies = slice.families || [];
835-
let metricNote = metricNotes[hscMetric] || "";
909+
let metricNote = metricNotes[metricCol] || metricNotes[hscMetric] || axisDef.note || "";
836910
if (HSC.plot_excluded_note && (slice.families || []).some(fam => fam.eos_only)) {
837911
metricNote = metricNote ? `${metricNote} ${HSC.plot_excluded_note}` : HSC.plot_excluded_note;
838912
}
@@ -889,7 +963,7 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
889963
const yMax = yMaxRaw + yPad;
890964
const yAt = (v) => pad.t + innerH - ((v - yMin) / (yMax - yMin || 1)) * innerH;
891965

892-
const yLabel = hscMetric === "skill_vs_trend" ? "Skill vs trend (median)" : "Median NRMSE";
966+
const yLabel = `${metricDef.label || hscMetric} (median)`;
893967
let svg = `<svg viewBox="0 0 ${width} ${height}" width="${width}" height="${height}" role="img" aria-label="Horizon skill curves">`;
894968
svg += `<rect x="0" y="0" width="${width}" height="${height}" fill="#fff"/>`;
895969
for (let t = 0; t <= 4; t++) {
@@ -973,7 +1047,7 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
9731047
}
9741048
return fmt(v);
9751049
},
976-
bg: (v) => (hscMetric === "nrmse" ? fixedNrmseBg(v) : null),
1050+
bg: (v) => hscCellBg(v, metricCol),
9771051
})),
9781052
];
9791053
renderSortableTable(
@@ -998,10 +1072,20 @@ <h3 style="font-size:1rem;margin:1rem 0 0.5rem;">Detail per country × model</h3
9981072
hscCrop = hscCropSelect.value || "all";
9991073
renderHorizonSkillCurves();
10001074
});
1075+
hscAxisToolbar.querySelectorAll("button").forEach(btn => {
1076+
btn.addEventListener("click", () => {
1077+
hscAxisToolbar.querySelectorAll("button").forEach(b => b.classList.remove("active"));
1078+
btn.classList.add("active");
1079+
hscAxis = btn.dataset.axis || "overall";
1080+
syncHscMetricControl();
1081+
renderHorizonSkillCurves();
1082+
});
1083+
});
10011084
hscMetricSelect.addEventListener("change", () => {
10021085
hscMetric = hscMetricSelect.value || "nrmse";
10031086
renderHorizonSkillCurves();
10041087
});
1088+
syncHscMetricControl();
10051089
renderHorizonSkillCurves();
10061090

10071091
// Horizon overall (RQ2)

‎tests/runs/test_global_insights.py‎

Lines changed: 35 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -451,52 +451,37 @@ def _three_horizon_fixture(tmp_path: Path) -> pd.DataFrame:
451451
"qtr": {"ridge": 0.22, "trend": 0.30, "xgboost": 0.20},
452452
"eos": {"ridge": 0.15, "trend": 0.28, "xgboost": 0.14},
453453
}
454+
pattern_by_hz = {
455+
"mid": {"r_spatial": 0.55, "r_temporal": 0.35, "r_res": 0.25},
456+
"qtr": {"r_spatial": 0.60, "r_temporal": 0.50, "r_res": 0.30},
457+
"eos": {"r_spatial": 0.65, "r_temporal": 0.70, "r_res": 0.35},
458+
}
459+
460+
def _row(model: str, cc: str, hz: str, nrmse: float, r2: float) -> dict:
461+
pat = pattern_by_hz[hz]
462+
return {
463+
"crop": "maize",
464+
"country": cc.upper(),
465+
"model": model,
466+
"nrmse": nrmse,
467+
"r2": r2,
468+
"r_spatial": pat["r_spatial"],
469+
"r_temporal": pat["r_temporal"],
470+
"r_res": pat["r_res"],
471+
"n_samples": 40,
472+
}
473+
454474
for cc, hz in [("de", "eos"), ("de", "mid"), ("de", "qtr"), ("fr", "eos"), ("fr", "mid"), ("fr", "qtr")]:
455475
d = tmp_path / f"paper_walk_forward_{cc}_{hz}_v1"
456476
d.mkdir(parents=True, exist_ok=True)
457477
metrics = rows_by_hz[hz]
458478
pd.DataFrame(
459479
[
460-
{
461-
"crop": "maize",
462-
"country": cc.upper(),
463-
"model": "ridge",
464-
"nrmse": metrics["ridge"],
465-
"r2": 0.7,
466-
"n_samples": 40,
467-
},
468-
{
469-
"crop": "maize",
470-
"country": cc.upper(),
471-
"model": "trend",
472-
"nrmse": metrics["trend"],
473-
"r2": 0.3,
474-
"n_samples": 40,
475-
},
476-
{
477-
"crop": "maize",
478-
"country": cc.upper(),
479-
"model": "xgboost",
480-
"nrmse": metrics["xgboost"],
481-
"r2": 0.75,
482-
"n_samples": 40,
483-
},
484-
{
485-
"crop": "maize",
486-
"country": cc.upper(),
487-
"model": "average",
488-
"nrmse": metrics["trend"] + 0.02,
489-
"r2": 0.2,
490-
"n_samples": 40,
491-
},
492-
{
493-
"crop": "maize",
494-
"country": cc.upper(),
495-
"model": "lstm_lf",
496-
"nrmse": metrics["ridge"] + 0.05,
497-
"r2": 0.6,
498-
"n_samples": 40,
499-
},
480+
_row("ridge", cc, hz, metrics["ridge"], 0.7),
481+
_row("trend", cc, hz, metrics["trend"], 0.3),
482+
_row("xgboost", cc, hz, metrics["xgboost"], 0.75),
483+
_row("average", cc, hz, metrics["trend"] + 0.02, 0.2),
484+
_row("lstm_lf", cc, hz, metrics["ridge"] + 0.05, 0.6),
500485
]
501486
).to_csv(d / "walk_forward_summary.csv", index=False)
502487
# US: eos only
@@ -536,8 +521,16 @@ def test_horizon_skill_curves_inner_join_countries():
536521
xgb = next(f for f in maize["families"] if f["model"] == "xgboost")
537522
assert xgb["plot"] is True
538523
assert xgb.get("eos_only") is not True
539-
nrmse_by_hz = {p["horizon"]: p["median_nrmse"] for p in xgb["points"]}
524+
nrmse_by_hz = {
525+
p["horizon"]: p["metrics"]["nrmse"]["median"] for p in xgb["points"]
526+
}
540527
assert nrmse_by_hz["mid"] > nrmse_by_hz["qtr"] > nrmse_by_hz["eos"]
528+
temporal_by_hz = {
529+
p["horizon"]: p["metrics"]["r_temporal"]["median"] for p in xgb["points"]
530+
}
531+
assert temporal_by_hz["mid"] < temporal_by_hz["qtr"] < temporal_by_hz["eos"]
532+
assert len(payload["axes"]) == 4
533+
assert payload["axes"][2]["id"] == "temporal"
541534

542535

543536
def test_horizon_skill_curves_eos_only_lpjml_in_table_not_plot(tmp_path: Path):
@@ -563,7 +556,7 @@ def test_horizon_skill_curves_eos_only_lpjml_in_table_not_plot(tmp_path: Path):
563556
assert lpj["plot"] is False
564557
assert len(lpj["points"]) == 1
565558
assert lpj["points"][0]["horizon"] == "eos"
566-
assert lpj["points"][0]["median_nrmse"] == 0.19
559+
assert lpj["points"][0]["metrics"]["nrmse"]["median"] == 0.19
567560
plot_models = {f["model"] for f in maize["families"] if f["plot"]}
568561
assert "lpjml_bc" not in plot_models
569562
assert "plot_excluded_note" in payload

0 commit comments

Comments
 (0)