early bird

Author: michielkallenberg

cybench/runs/slurm/README.md

@@ -178,26 +178,26 @@ cybench/runs/slurm/orchestrate_benchmark_complete.sh \
   --all-countries --horizons qtr --version 2 --list
 ```
 
-**Early season (~25% observed), family representatives only** (xgboost, lpjml_bc, tst_lf, tabdpt; 20 HPO trials, 5 WF seeds — same as v2 benchmark):
+**Early season (~25% observed)** — same orchestration as eos/mid/qtr:
 
 ```bash
 export HP_TRIALS=20 WF_REPETITIONS=5
-export CYBENCH_OUTPUT_ROOT=/lustre/backup/SHARED/AIN/agml/output
 
-# Manifests (pass countries explicitly if baselines_*_early_v2 dirs do not exist yet)
-cybench/runs/slurm/refresh_batch_manifests.sh \
-  --horizon early-season --version 2 \
-  --models cybench/runs/slurm/models_family_reps.txt \
-  --countries DE FR NL   # or omit once early batch dirs exist on lustre
+# Preview plan (all countries with yield data)
+cybench/runs/slurm/orchestrate_benchmark_submit.sh --list --horizon early --version 2
+
+# Submit all countries
+cybench/runs/slurm/orchestrate_benchmark_submit.sh \
+  --horizon early --version 2 --skip-naive --repetitions 5
 
-# Submit per country (or loop over the same country list)
-cybench/runs/slurm/submit_benchmark.sh all \
-  --horizon early-season --batch baselines_DE_early_v2 \
-  --skip-naive --repetitions 5
+# Family representatives only (xgboost, lpjml_bc, tst_lf, tabdpt)
+cybench/runs/slurm/orchestrate_benchmark_submit.sh \
+  --horizon early --version 2 --skip-naive --repetitions 5 \
+  --models cybench/runs/slurm/models_family_reps.txt
 
 # Audit / partial reruns
 cybench/runs/slurm/orchestrate_benchmark_complete.sh \
-  --country DE --horizons early-season --version 2 --list
+  --all-countries --horizons early --version 2 --list
 ```
 
 Manual per-manifest submits (fine-grained control) are below.

cybench/runs/slurm/orchestrate_benchmark_submit.sh

@@ -6,6 +6,7 @@
 # Usage (from repo root on anunna):
 #   cybench/runs/slurm/orchestrate_benchmark_submit.sh --list
 #   cybench/runs/slurm/orchestrate_benchmark_submit.sh --dry-run
+#   cybench/runs/slurm/orchestrate_benchmark_submit.sh --horizon early --version 2 --dry-run
 #   cybench/runs/slurm/orchestrate_benchmark_submit.sh --countries PL IT --horizon eos
 #   cybench/runs/slurm/orchestrate_benchmark_submit.sh --region-threshold 50 --max 3
 #
@@ -15,26 +16,39 @@ usage() {
   cat <<'EOF'
 Usage: orchestrate_benchmark_submit.sh [options]
 
-Discover countries / horizons without manifests yet, then call submit_benchmark.sh
-per batch. Tabular (cpu) and naive jobs always use the cpu/main arrays; only the
-gpu manifest group is routed to the gpu partition or --cpu based on region count.
+Discover countries with yield data and horizons without manifests yet, then call
+submit_benchmark.sh per batch. Tabular (cpu) and naive jobs always use the cpu/main
+arrays; only the gpu manifest group is routed to the gpu partition or --cpu based
+on region count.
+
+By default every country with data on disk is included (no --countries needed).
 
 Options:
   --list              Show plan (country, regions, gpu vs cpu) and exit
   --dry-run           Print submit_benchmark.sh commands without sbatch
   --countries CC ...  Limit to these countries (default: all with data on disk)
-  --horizon H ...     Default: eos middle-of-season (alias mid → middle-of-season)
+  --horizon H ...     eos | mid | qtr | early (alias early-season; repeatable)
   --region-threshold N  gpu partition when country has >= N regions (default: 50)
   --version N         Batch version suffix (default: 3)
   --phase MODE        screening | walk_forward | all (default: all)
+  --repetitions N     Walk-forward seeds 42..42+N-1 (passed to submit_benchmark.sh)
+  --skip-naive        Omit average/trend jobs
+  --models FILE       Model catalogue for manifest regeneration (default: models.txt)
   --force             Submit even if manifest batch dir already exists
-  --all-countries     Include already-filed batches in --list
+  --all-countries     With --list: include batches that already have manifests
   --max N             Submit at most N batches this run
   --manifest-root DIR Override slurm/manifests parent
   --data-dir DIR      Override cybench/data
 
+Environment:
+  HP_TRIALS           Optuna trials in screening (default: 100)
+  WF_REPETITIONS      Default for --repetitions when omitted (default: 1)
+
 Examples:
-  orchestrate_benchmark_submit.sh --list
+  orchestrate_benchmark_submit.sh --list --horizon early --version 2
+  orchestrate_benchmark_submit.sh --horizon early --version 2 --dry-run
+  export HP_TRIALS=20 WF_REPETITIONS=5
+  orchestrate_benchmark_submit.sh --horizon early --version 2 --skip-naive --repetitions 5
   orchestrate_benchmark_submit.sh --countries IN --horizon eos
   orchestrate_benchmark_submit.sh --region-threshold 50 --max 3
 EOF
@@ -63,6 +77,9 @@ MANIFEST_ROOT="${SLURM_DIR}/manifests"
 DATA_DIR=""
 COUNTRIES=()
 HORIZONS=()
+WF_REPETITIONS="${WF_REPETITIONS:-}"
+SKIP_NAIVE=false
+MODELS_FILE=""
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -120,6 +137,18 @@ while [[ $# -gt 0 ]]; do
       DATA_DIR=$2
       shift 2
       ;;
+    --repetitions)
+      WF_REPETITIONS=$2
+      shift 2
+      ;;
+    --skip-naive)
+      SKIP_NAIVE=true
+      shift
+      ;;
+    --models)
+      MODELS_FILE=$2
+      shift 2
+      ;;
     -h|--help)
       usage
       exit 0
@@ -184,6 +213,15 @@ for line in "${PLAN_LINES[@]}"; do
   if [[ "${GPU_MODE}" == "cpu" ]]; then
     cmd+=(--cpu)
   fi
+  if [[ -n "${WF_REPETITIONS}" ]]; then
+    cmd+=(--repetitions "${WF_REPETITIONS}")
+  fi
+  if [[ "${SKIP_NAIVE}" == true ]]; then
+    cmd+=(--skip-naive)
+  fi
+  if [[ -n "${MODELS_FILE}" ]]; then
+    cmd+=(--models "${MODELS_FILE}")
+  fi
   if [[ "${DRY_RUN}" == true ]]; then
     cmd+=(--dry-run)
   fi

tests/runs/test_benchmark_submit_lib.py

@@ -16,8 +16,8 @@
 def test_normalize_horizon():
     assert normalize_horizon("eos") == "eos"
     assert normalize_horizon("mid") == "middle-of-season"
-    assert normalize_horizon("early-season") == "early-season"
-    assert horizon_batch_suffix("early-season") == "early"
+    assert normalize_horizon("early") == "early-season"
+    assert horizon_batch_suffix("early") == "early"
 
 
 def test_batch_name():