slurm

Author: michielkallenberg

cybench/runs/analysis/country_significance_lib.py

@@ -112,6 +112,8 @@ def bootstrap_country_ai_metrics(
     """Bootstrap countries; report CIs for median Δ_abs, median Δ%, and win rate."""
     if frame.empty:
         return {
+            "median_nrmse_trad": None,
+            "median_nrmse_ai": None,
             "median_delta_abs": None,
             "delta_abs_ci_lo": None,
             "delta_abs_ci_hi": None,
@@ -127,9 +129,23 @@ def bootstrap_country_ai_metrics(
     delta_pct = frame["delta_pct"].to_numpy(dtype=float)
     ai_wins = frame["ai_wins"].to_numpy(dtype=bool)
     n = delta_abs.size
+    nrmse_trad = (
+        frame["nrmse_trad"].to_numpy(dtype=float)
+        if "nrmse_trad" in frame.columns
+        else np.full(n, np.nan)
+    )
+    nrmse_ai = (
+        frame["nrmse_ai"].to_numpy(dtype=float)
+        if "nrmse_ai" in frame.columns
+        else np.full(n, np.nan)
+    )
+    med_trad = float(np.median(nrmse_trad)) if np.isfinite(nrmse_trad).any() else None
+    med_ai = float(np.median(nrmse_ai)) if np.isfinite(nrmse_ai).any() else None
 
     if n == 1:
         return {
+            "median_nrmse_trad": med_trad,
+            "median_nrmse_ai": med_ai,
             "median_delta_abs": float(delta_abs[0]),
             "delta_abs_ci_lo": float(delta_abs[0]),
             "delta_abs_ci_hi": float(delta_abs[0]),
@@ -157,6 +173,8 @@ def bootstrap_country_ai_metrics(
     wr_lo, wr_hi = _percentile_ci(boot_wr, ci=ci)
 
     return {
+        "median_nrmse_trad": med_trad,
+        "median_nrmse_ai": med_ai,
         "median_delta_abs": float(np.median(delta_abs)),
         "delta_abs_ci_lo": abs_lo,
         "delta_abs_ci_hi": abs_hi,
@@ -256,9 +274,12 @@ def bootstrap_stats_json(result: dict[str, Any]) -> dict[str, Any]:
 
 
 COUNTRY_BOOTSTRAP_NOTE = (
-    "Best data-driven vs best traditional baseline per country (Average, Trend, LPJmL). "
-    "NRMSE is averaged over walk-forward seeds before comparison. Countries are resampled "
-    "with replacement; intervals are percentile 95% bootstrap CIs."
+    "Per country: best traditional NRMSE = min(Average, Trend, LPJmL); best data-driven NRMSE = "
+    "min(feature-engineered, sequence, foundation). NRMSE columns are medians of those "
+    "country-level values. Median improvement (%) is the median of "
+    "100×(traditional−data-driven)/traditional per country. Win rate: fraction of countries "
+    "where data-driven NRMSE is lower. Seed-averaged walk-forward NRMSE; bootstrap resamples "
+    "countries (95% percentile CIs on improvement and win rate)."
 )
 
 
@@ -325,11 +346,18 @@ def _fmt_ci(lo: float | None, hi: float | None, *, digits: int = 1, percent: boo
     return f"[{lo:.{digits}f}, {hi:.{digits}f}]"
 
 
+def _fmt_nrmse_pct(value: float | None) -> str:
+    """Format pooled NRMSE fraction as percent (matches Table 1)."""
+    if value is None or not np.isfinite(value):
+        return "---"
+    return f"{100.0 * float(value):.1f}"
+
+
 def format_results_markdown_table(results: dict[str, dict[str, Any]]) -> str:
     """Paper-style summary table (markdown)."""
     lines = [
-        "| Crop | Median ΔNRMSE (%) | 95% CI | AI win rate | 95% CI |",
-        "|------|-------------------|--------|-------------|--------|",
+        "| Crop | Trad. NRMSE (%) | AI NRMSE (%) | Median improvement (%) | 95% CI | AI win rate | 95% CI |",
+        "|------|-----------------|--------------|------------------------|--------|-------------|--------|",
     ]
     for crop, res in results.items():
         med = res.get("median_delta_pct")
@@ -341,6 +369,8 @@ def format_results_markdown_table(results: dict[str, dict[str, Any]]) -> str:
             + " | ".join(
                 [
                     crop.capitalize(),
+                    _fmt_nrmse_pct(res.get("median_nrmse_trad")),
+                    _fmt_nrmse_pct(res.get("median_nrmse_ai")),
                     _fmt_num(med),
                     _fmt_ci(lo, hi).strip("[]") if lo is not None else "---",
                     f"{100 * wr:.0f}%" if wr is not None else "---",
@@ -363,29 +393,34 @@ def format_results_latex_table(
     """LaTeX table for main paper or supplement."""
     if caption is None:
         caption = (
-            "Country-level bootstrap summary of AI advantage over traditional baselines "
-            "(best data-driven vs.\\ best among Average, Trend, and LPJmL per country). "
-            "Median $\\Delta$NRMSE is reported as percent improvement; win rate is the "
-            "fraction of countries where AI achieved lower NRMSE. Countries were resampled "
-            "with replacement ($B=10{,}000$); intervals are percentile 95\\% bootstrap CIs."
+            "Country-level bootstrap summary of AI advantage over traditional baselines. "
+            "Traditional and data-driven NRMSE are medians across countries of the best "
+            "traditional (Average, Trend, LPJmL) and best data-driven model per country. "
+            "Median improvement (\\%) is the median of "
+            "$100\\times(\\mathrm{NRMSE}_{\\mathrm{trad}}-\\mathrm{NRMSE}_{\\mathrm{AI}})"
+            "/\\mathrm{NRMSE}_{\\mathrm{trad}}$ per country. Win rate: fraction of countries "
+            "where data-driven NRMSE is lower. Bootstrap resamples countries ($B=10{,}000$); "
+            "intervals are percentile 95\\% CIs."
         )
     body_rows: list[str] = []
     for crop, res in results.items():
+        trad = _fmt_nrmse_pct(res.get("median_nrmse_trad"))
+        ai = _fmt_nrmse_pct(res.get("median_nrmse_ai"))
         med = _fmt_num(res.get("median_delta_pct"))
         pct_ci = _fmt_ci(res.get("delta_pct_ci_lo"), res.get("delta_pct_ci_hi"))
         wr = _fmt_pct_rate(res.get("win_rate"))
         wr_ci = _fmt_ci(res.get("win_rate_ci_lo"), res.get("win_rate_ci_hi"), percent=True)
         body_rows.append(
-            f"{crop.capitalize()} & {med} & {pct_ci} & {wr} & {wr_ci} \\\\"
+            f"{crop.capitalize()} & {trad} & {ai} & {med} & {pct_ci} & {wr} & {wr_ci} \\\\"
         )
     body = "\n".join(body_rows)
     return f"""\\begin{{table}}[t]
 \\centering
 \\caption{{{caption}}}
 \\label{{{label}}}
-\\begin{{tabular}}{{lcccc}}
+\\begin{{tabular}}{{lcccccc}}
 \\toprule
-Crop & Median $\\Delta$NRMSE (\\%) & 95\\% CI & AI win rate & 95\\% CI \\\\
+Crop & Trad.\\ NRMSE (\\%) & AI NRMSE (\\%) & Median impr. (\\%) & 95\\% CI & AI win rate & 95\\% CI \\\\
 \\midrule
 {body}
 \\bottomrule

cybench/runs/slurm/benchmark_completion_lib.py

@@ -431,6 +431,11 @@ def filter_jobs_by_models(
     return [job for job in jobs if job.model.lower() in allowed]
 
 
+def filter_jobs_cpu_only(jobs: list[JobRow]) -> list[JobRow]:
+    """Keep naive + CPU-tabular rows (needs_gpu=no); exclude torch / TabPFN / TabDPT."""
+    return [job for job in jobs if job.needs_gpu == "no"]
+
+
 def job_row_key(job: JobRow) -> tuple[str, str, str, str, str, str, str]:
     return (
         job.crop,

cybench/runs/slurm/orchestrate_benchmark_complete.py

@@ -47,6 +47,7 @@
     expand_all_country_targets,
     expand_target_batches,
     filter_jobs_by_models,
+    filter_jobs_cpu_only,
     jobs_for_phase,
     merge_jobs_for_models,
     resolve_country_list,
@@ -260,6 +261,13 @@ def _process_batch(
             )
             return 0
 
+    if args.only_cpu:
+        jobs = filter_jobs_cpu_only(jobs)
+        manifest_source = f"{manifest_source} | cpu-models-only"
+        if not jobs:
+            print(f"[WARN] No CPU/naive jobs in manifest for {batch}", file=sys.stderr)
+            return 0
+
     if not baselines_dir.is_dir():
         print(
             f"[WARN] Baselines dir missing (all jobs treated as incomplete): {baselines_dir}",
@@ -454,6 +462,14 @@ def main(argv: list[str] | None = None) -> int:
         action="store_true",
         help="Force GPU manifest group to CPU (--cpu on submit_benchmark.sh)",
     )
+    parser.add_argument(
+        "--only-cpu",
+        action="store_true",
+        help=(
+            "Assess/submit only CPU/naive models (needs_gpu=no); "
+            "exclude tst_lf, tabdpt, TabPFN, etc."
+        ),
+    )
     parser.add_argument(
         "--force-gpu",
         action="store_true",

cybench/runs/slurm/orchestrate_benchmark_complete.sh

@@ -38,6 +38,7 @@ Options:
   --submit            Write retry manifest and call submit_benchmark.sh
   --dry-run           With --submit: no sbatch
   --cpu               Force torch/TabPFN group to CPU partition
+  --only-cpu          Complete/submit only CPU+naive models (needs_gpu=no)
   --force-gpu         Use gpu partition even when regions < threshold
   --region-threshold N  gpu when country has >= N regions (default: 600)
 
@@ -80,6 +81,7 @@ LIST=false
 SUBMIT=false
 DRY_RUN=false
 FORCE_CPU=false
+ONLY_CPU=false
 FORCE_GPU=false
 REGION_THRESHOLD=""
 
@@ -177,6 +179,10 @@ while [[ $# -gt 0 ]]; do
       FORCE_CPU=true
       shift
       ;;
+    --only-cpu)
+      ONLY_CPU=true
+      shift
+      ;;
     --force-gpu)
       FORCE_GPU=true
       shift
@@ -225,6 +231,7 @@ done
 [[ "${SUBMIT}" == true ]] && cmd+=(--submit)
 [[ "${DRY_RUN}" == true ]] && cmd+=(--dry-run)
 [[ "${FORCE_CPU}" == true ]] && cmd+=(--cpu)
+[[ "${ONLY_CPU}" == true ]] && cmd+=(--only-cpu)
 [[ "${FORCE_GPU}" == true ]] && cmd+=(--force-gpu)
 [[ -n "${REGION_THRESHOLD}" ]] && cmd+=(--region-threshold "${REGION_THRESHOLD}")
 

cybench/runs/viz/model_family_radar_template.html

@@ -310,7 +310,9 @@ <h2>Country-level AI advantage (bootstrap)</h2>
             <tr>
               <th>Crop</th>
               <th>Countries</th>
-              <th>Median ΔNRMSE (%)</th>
+              <th>Traditional NRMSE (%)</th>
+              <th>Data-driven NRMSE (%)</th>
+              <th>Median improvement (%)</th>
               <th>95% CI</th>
               <th>AI win rate</th>
               <th>95% CI</th>
@@ -879,6 +881,11 @@ <h2>Country-level AI advantage (bootstrap)</h2>
       return (DATA.crops || []).slice().sort();
     }
 
+    function formatBootstrapNrmsePct(value) {
+      if (value == null || !Number.isFinite(value)) return "—";
+      return (100 * Number(value)).toFixed(1);
+    }
+
     function formatBootstrapNum(value, digits = 1) {
       if (value == null || !Number.isFinite(value)) return "—";
       return Number(value).toFixed(digits);
@@ -918,14 +925,16 @@ <h2>Country-level AI advantage (bootstrap)</h2>
       if (!bootstrapTable) return;
       const tbody = bootstrapTable.querySelector("tbody");
       if (!currentBootstrapRows.length) {
-        tbody.innerHTML = `<tr><td colspan="6" class="muted">No bootstrap summary for this horizon/crop.</td></tr>`;
+        tbody.innerHTML = `<tr><td colspan="8" class="muted">No bootstrap summary for this horizon/crop.</td></tr>`;
         return;
       }
       tbody.innerHTML = currentBootstrapRows.map(row => {
         const wr = row.win_rate;
         return `<tr>
           <td>${row.crop.charAt(0).toUpperCase() + row.crop.slice(1)}</td>
           <td>${row.n_ai_wins ?? "—"}/${row.n_countries ?? "—"}</td>
+          <td>${formatBootstrapNrmsePct(row.median_nrmse_trad)}</td>
+          <td>${formatBootstrapNrmsePct(row.median_nrmse_ai)}</td>
           <td>${formatBootstrapNum(row.median_delta_pct)}</td>
           <td>${formatBootstrapCi(row.delta_pct_ci_lo, row.delta_pct_ci_hi)}</td>
           <td>${wr == null ? "—" : `${Math.round(100 * wr)}%`}</td>
@@ -952,21 +961,23 @@ <h2>Country-level AI advantage (bootstrap)</h2>
       const { hzLabel, cropLabel } = bootstrapExportMeta();
       const nBoot = (bootstrapPayloadRoot().n_bootstrap || 10000).toLocaleString();
       const bodyRows = rows.map(row => {
+        const trad = formatBootstrapNrmsePct(row.median_nrmse_trad);
+        const ai = formatBootstrapNrmsePct(row.median_nrmse_ai);
         const wr = row.win_rate == null ? "---" : `${Math.round(100 * row.win_rate)}\\%`;
         const wrCi = formatBootstrapCi(row.win_rate_ci_lo, row.win_rate_ci_hi, { percent: true });
         const pctCi = formatBootstrapCi(row.delta_pct_ci_lo, row.delta_pct_ci_hi);
-        return `${row.crop.charAt(0).toUpperCase() + row.crop.slice(1)} & `
+        return `${row.crop.charAt(0).toUpperCase() + row.crop.slice(1)} & ${trad} & ${ai} & `
           + `${formatBootstrapNum(row.median_delta_pct)} & ${pctCi} & ${wr} & ${wrCi} \\\\`;
       }).join("\n");
       return `% CY-Bench country-level AI bootstrap table (generated from dashboard)
 % Requires: \\usepackage{booktabs}
 \\begin{table}[t]
 \\centering
-\\caption{Country-level bootstrap summary of AI advantage over traditional baselines (best data-driven vs.\\ best among Average, Trend, and LPJmL per country). Median $\\Delta$NRMSE is reported as percent improvement; win rate is the fraction of countries where AI achieved lower NRMSE. Horizon: ${escapeLatex(hzLabel)}; crops: ${escapeLatex(cropLabel)}. Countries resampled with replacement ($B=${nBoot}$); intervals are percentile 95\\% bootstrap CIs.}
+\\caption{Country-level bootstrap summary of AI advantage over traditional baselines. Traditional and data-driven NRMSE are medians across countries of the best traditional (Average, Trend, LPJmL) and best data-driven model per country. Median improvement (\\%) is the median of $100\\times(\\mathrm{NRMSE}_{\\mathrm{trad}}-\\mathrm{NRMSE}_{\\mathrm{AI}})/\\mathrm{NRMSE}_{\\mathrm{trad}}$ per country. Horizon: ${escapeLatex(hzLabel)}; crops: ${escapeLatex(cropLabel)}. Bootstrap resamples countries ($B=${nBoot}$).}
 \\label{tab:ai_country_bootstrap}
-\\begin{tabular}{lcccc}
+\\begin{tabular}{lcccccc}
 \\toprule
-Crop & Median $\\Delta$NRMSE (\\%) & 95\\% CI & AI win rate & 95\\% CI \\\\
+Crop & Trad.\\ NRMSE (\\%) & AI NRMSE (\\%) & Median impr. (\\%) & 95\\% CI & AI win rate & 95\\% CI \\\\
 \\midrule
 ${bodyRows}
 \\bottomrule

tests/runs/test_benchmark_completion.py

@@ -9,6 +9,7 @@
     assess_job,
     check_screening_years,
     filter_jobs_by_models,
+    filter_jobs_cpu_only,
     jobs_for_phase,
     read_manifest,
     screening_complete,
@@ -111,6 +112,17 @@ def test_filter_jobs_by_models():
     assert filter_jobs_by_models(jobs, None) == jobs
 
 
+def test_filter_jobs_cpu_only():
+    jobs = [
+        JobRow("maize", "DE", "xgboost", "pandas", "yes", "yes", "no"),
+        JobRow("maize", "DE", "lpjml_bc", "pandas", "no", "no", "no"),
+        JobRow("maize", "DE", "tst_lf", "torch", "yes", "no", "yes"),
+        JobRow("wheat", "DE", "tabdpt", "pandas", "yes", "yes", "yes"),
+    ]
+    cpu = filter_jobs_cpu_only(jobs)
+    assert cpu == jobs[:2]
+
+
 def test_jobs_for_phase_force_rerun_includes_complete():
     job = JobRow("maize", "BE", "lpjml_bc", "pandas", "no", "no", "no")
     from cybench.runs.slurm.benchmark_completion_lib import JobAssessment

tests/runs/test_country_significance.py

@@ -93,5 +93,7 @@ def test_analyze_country_ai_benefit_returns_bootstrap_fields():
     assert res["win_rate"] == pytest.approx(2 / 3)
     assert res["median_delta_pct"] is not None
     assert res["delta_pct_ci_lo"] is not None
+    assert res["median_nrmse_trad"] == pytest.approx(0.20)
+    assert res["median_nrmse_ai"] == pytest.approx(0.17)
     assert res["median_delta_abs"] is not None
     assert res["delta_abs_ci_lo"] is not None