Merge pull request #334 from alan-turing-institute/2026-04-18/runs

Update config timings and naming resolution

Author: sgreenbury

slurm_scripts/comparison/README.md

@@ -99,14 +99,32 @@ extracted per-dataset from `*_timing.sh` via
 `uv run autocast time-epochs --from-checkpoint <path>/timing.ckpt -b 24`
 and live in `COSINE_EPOCHS_BY_DATASET` at the top of each `*_large.sh`.
 
-Ambient (timing 2026-04-17):
+Ambient (timing 2026-04-18; CRPS from `timing_efficient_crps/`, FM from `timing/`):
 
 | variant | gray_scott | gpe_laser_only_wake | cond_navier_stokes | advection_diffusion |
 |---|---|---|---|---|
-| CRPS ambient | **398** (212.3 s/ep) | 477 (177.3) | 471 (179.5) | 486 (174.0) |
-| FM ambient | **2619** (32.3 s/ep) | 3097 (27.3) | 2917 (29.0) | 3279 (25.8) |
+| CRPS ambient (permute_concat) | **399** (212.0 s/ep) | 477 (177.2) | 473 (178.8) | 478 (177.0) |
+| CRPS-via-AE ambient (EPD)     | **49**  (1724.6 s/ep) | 85  (991.0) | 85  (985.0) | 58  (1436.9) |
+| FM ambient                    | **2631** (32.2 s/ep)  | 3171 (26.7) | 2982 (28.4) | 3264 (25.9) |
 
-Latent: placeholders (1080) pending `submit_{crps_latent,fm}_timing.sh`.
+Latent (timing 2026-04-18, FM only — full 4-dataset CRPS-latent timing pending):
+
+| variant | gray_scott | gpe_laser_only_wake | cond_navier_stokes | advection_diffusion |
+|---|---|---|---|---|
+| FM latent (cached) | **2830** (29.9 s/ep) | 3411 (24.8) | 3223 (26.3) | 3314 (25.6) |
+| CRPS latent (cached) | 1080 (placeholder) | 1080 | 1080 | 1080 |
+
+CNS-only ablations (timing 2026-04-18):
+
+| variant | conditioned_navier_stokes |
+|---|---|
+| CRPS ambient (identity + global_cond AdaLN) | 469 (180.3 s/ep) |
+| CRPS latent (cached, ablation)              | 345 (245.0 s/ep) |
+
+All `s/ep` values are the mean across the n=5 epoch durations recorded by
+`TrainingTimerCallback` in `timing.ckpt` (saved after `on_train_end`, so
+it captures the final epoch — unlike `last.ckpt`, which is saved during
+`on_train_epoch_end` and only holds the first 4 durations).
 
 Each script saves quarter-schedule checkpoints (every `cosine_epochs / 4`)
 plus `last.ckpt` at train-end (guaranteed final state). Quarter boundaries

slurm_scripts/comparison/cached_latents/submit_crps_latent_cns_large.sh

@@ -14,7 +14,7 @@ set -euo pipefail
 DATAMODULE="conditioned_navier_stokes"
 EXPERIMENT="processor/conditioned_navier_stokes/crps_vit_azula_large"
 AE_RUN_DIR="$HOME/autocast/outputs/2026-04-17/ae_cns64_3a7999b_b9c29f8"
-COSINE_EPOCHS=1080  # placeholder
+COSINE_EPOCHS=345  # 245.0 s/ep (timing_efficient_crps, 2026-04-18)
 
 BUDGET_MAX_TIME="00:23:59:00"
 # SLURM timeout with 1-min buffer beyond the 24h budget.
@@ -58,5 +58,5 @@ for run_dry in "${RUN_DRY_STATES[@]}"; do
         +trainer.max_epochs="${COSINE_EPOCHS}" \
         trainer.callbacks.0.every_n_epochs="${quarter_epochs}" \
         trainer.callbacks.0.save_top_k=-1 \
-        trainer.callbacks.0.filename="quarter-{epoch:04d}"
+        trainer.callbacks.0.filename=\"quarter-{epoch:04d}\"
 done

slurm_scripts/comparison/cached_latents/submit_crps_latent_large.sh

@@ -83,6 +83,6 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
             +trainer.max_epochs="${cosine_epochs}" \
             trainer.callbacks.0.every_n_epochs="${quarter_epochs}" \
             trainer.callbacks.0.save_top_k=-1 \
-            trainer.callbacks.0.filename="quarter-{epoch:04d}"
+            trainer.callbacks.0.filename=\"quarter-{epoch:04d}\"
     done
 done

slurm_scripts/comparison/cached_latents/submit_fm_large.sh

@@ -9,18 +9,17 @@ set -euo pipefail
 # the authoritative hyperparameters.
 #
 # Per-dataset cosine schedule: each (method, dataset) pair fills its own
-# 24h budget so each model gets its best shot within budget. PLACEHOLDERS
-# pending submit_fm_timing.sh — extract per-dataset values via
+# 24h budget so each model gets its best shot within budget. Values from
+# submit_fm_timing.sh (2026-04-18) via
 #   uv run autocast time-epochs --from-checkpoint <path>/timing.ckpt -b 24
-# and replace each entry below.
 #
 # learning_rate (1e-4) and warmup (0) are baked into each per-dataset
 # local_experiment config; adjust the yaml to change them.
 declare -A COSINE_EPOCHS_BY_DATASET=(
-    ["gray_scott"]=1080                 # placeholder
-    ["gpe_laser_only_wake"]=1080        # placeholder
-    ["conditioned_navier_stokes"]=1080  # placeholder
-    ["advection_diffusion"]=1080        # placeholder
+    ["gray_scott"]=2830                 # 29.9 s/ep
+    ["gpe_laser_only_wake"]=3411        # 24.8 s/ep
+    ["conditioned_navier_stokes"]=3223  # 26.3 s/ep
+    ["advection_diffusion"]=3314        # 25.6 s/ep
 )
 BUDGET_MAX_TIME="00:23:59:00"
 # SLURM timeout with 1-min buffer beyond the 24h budget.
@@ -82,6 +81,6 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
             +trainer.max_epochs="${cosine_epochs}" \
             trainer.callbacks.0.every_n_epochs="${quarter_epochs}" \
             trainer.callbacks.0.save_top_k=-1 \
-            trainer.callbacks.0.filename="quarter-{epoch:04d}"
+            trainer.callbacks.0.filename=\"quarter-{epoch:04d}\"
     done
 done

slurm_scripts/comparison/epd/submit_crps_ae_ambient_large.sh

@@ -11,27 +11,27 @@ set -euo pipefail
 #   uv run autocast time-epochs --from-checkpoint <path>/timing.ckpt -b 24
 
 declare -A COSINE_EPOCHS_BY_DATASET=(
-    ["gray_scott"]=1080                 # placeholder
-    ["gpe_laser_only_wake"]=1080        # placeholder
-    ["conditioned_navier_stokes"]=1080  # placeholder
-    ["advection_diffusion"]=1080        # placeholder
+    # ["gray_scott"]=49                   # 1724.6 s/ep (timing_efficient_crps, 2026-04-18)
+    # ["gpe_laser_only_wake"]=85          #  991.0 s/ep (timing_efficient_crps, 2026-04-18)
+    ["conditioned_navier_stokes"]=85    #  985.0 s/ep (timing_efficient_crps, 2026-04-18)
+    # ["advection_diffusion"]=58          # 1436.9 s/ep (timing_efficient_crps, 2026-04-18)
 )
 BUDGET_MAX_TIME="00:23:59:00"
 # SLURM timeout with 1-min buffer beyond the 24h budget.
 TIMEOUT_MIN=1439
 RUN_DRY_STATES=("true" "false")
 
 declare -A EXPERIMENTS=(
-    ["gray_scott"]="epd/gray_scott/crps_vit_azula_large_ae_ambient"
-    ["gpe_laser_only_wake"]="epd/gpe_laser_wake_only/crps_vit_azula_large_ae_ambient"
+    # ["gray_scott"]="epd/gray_scott/crps_vit_azula_large_ae_ambient"
+    # ["gpe_laser_only_wake"]="epd/gpe_laser_wake_only/crps_vit_azula_large_ae_ambient"
     ["conditioned_navier_stokes"]="epd/conditioned_navier_stokes/crps_vit_azula_large_ae_ambient"
-    ["advection_diffusion"]="epd/advection_diffusion/crps_vit_azula_large_ae_ambient"
+    # ["advection_diffusion"]="epd/advection_diffusion/crps_vit_azula_large_ae_ambient"
 )
 declare -A AE_RUN_DIRS=(
-    ["gray_scott"]="$HOME/autocast/outputs/2026-04-17/ae_gs64_3a7999b_ed36b8e"
-    ["gpe_laser_only_wake"]="$HOME/autocast/outputs/2026-04-17/ae_gpe64_3a7999b_31e1c9f"
+    # ["gray_scott"]="$HOME/autocast/outputs/2026-04-17/ae_gs64_3a7999b_ed36b8e"
+    # ["gpe_laser_only_wake"]="$HOME/autocast/outputs/2026-04-17/ae_gpe64_3a7999b_31e1c9f"
     ["conditioned_navier_stokes"]="$HOME/autocast/outputs/2026-04-17/ae_cns64_3a7999b_b9c29f8"
-    ["advection_diffusion"]="$HOME/autocast/outputs/2026-04-17/ae_ad64_3a7999b_1a1e300"
+    # ["advection_diffusion"]="$HOME/autocast/outputs/2026-04-17/ae_ad64_3a7999b_1a1e300"
 )
 
 for datamodule in "${!EXPERIMENTS[@]}"; do
@@ -70,6 +70,7 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
         echo "  cosine_epochs: ${cosine_epochs}"
 
         uv run autocast epd --mode slurm "${dry_run_arg[@]}" \
+            datamodule="${datamodule}" \
             local_experiment="${experiment}" \
             autoencoder_checkpoint="${ckpt}" \
             logging.wandb.enabled=true \
@@ -79,6 +80,6 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
             +trainer.max_epochs="${cosine_epochs}" \
             trainer.callbacks.0.every_n_epochs="${quarter_epochs}" \
             trainer.callbacks.0.save_top_k=-1 \
-            trainer.callbacks.0.filename="quarter-{epoch:04d}"
+            trainer.callbacks.0.filename=\"quarter-{epoch:04d}\"
     done
 done

slurm_scripts/comparison/epd/submit_crps_ambient_identity_global_cond_large.sh

@@ -14,7 +14,7 @@ set -euo pipefail
 # and then extracting:
 #   uv run autocast time-epochs --from-checkpoint <path>/timing.ckpt -b 24
 declare -A COSINE_EPOCHS_BY_DATASET=(
-    ["conditioned_navier_stokes"]=471  # seed from CRPS ambient baseline; update from timing
+    ["conditioned_navier_stokes"]=469  # 180.3 s/ep (timing_efficient_crps, 2026-04-18)
 )
 BUDGET_MAX_TIME="00:23:59:00"
 # SLURM timeout with 1-min buffer beyond the 24h budget.
@@ -48,6 +48,7 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
         echo "  cosine_epochs: ${cosine_epochs}"
 
         uv run autocast epd --mode slurm "${dry_run_arg[@]}" \
+            datamodule="${datamodule}" \
             local_experiment="${experiment}" \
             logging.wandb.enabled=true \
             optimizer.cosine_epochs="${cosine_epochs}" \
@@ -56,6 +57,6 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
             +trainer.max_epochs="${cosine_epochs}" \
             trainer.callbacks.0.every_n_epochs="${quarter_epochs}" \
             trainer.callbacks.0.save_top_k=-1 \
-            trainer.callbacks.0.filename="quarter-{epoch:04d}"
+            trainer.callbacks.0.filename=\"quarter-{epoch:04d}\"
     done
 done

slurm_scripts/comparison/epd/submit_crps_large.sh

@@ -9,16 +9,16 @@ set -euo pipefail
 #
 # Per-dataset cosine schedule: each (method, dataset) pair fills its own
 # 24h budget so each model gets its best shot within budget. Values from
-# submit_crps_timing.sh (2026-04-17) via
+# submit_crps_timing.sh (2026-04-18) via
 #   uv run autocast time-epochs --from-checkpoint <path>/timing.ckpt -b 24
 #
 # learning_rate (2e-4) and warmup (0) are baked into each per-dataset
 # local_experiment config; adjust the yaml to change them.
 declare -A COSINE_EPOCHS_BY_DATASET=(
-    ["gray_scott"]=398                  # 212.3s/epoch
-    ["gpe_laser_only_wake"]=477         # 177.3s/epoch
-    ["conditioned_navier_stokes"]=471   # 179.5s/epoch
-    ["advection_diffusion"]=486         # 174.0s/epoch
+    ["gray_scott"]=399                  # 212.0 s/ep
+    ["gpe_laser_only_wake"]=477         # 177.2 s/ep
+    ["conditioned_navier_stokes"]=473   # 178.8 s/ep
+    ["advection_diffusion"]=478         # 177.0 s/ep
 )
 BUDGET_MAX_TIME="00:23:59:00"
 # SLURM timeout with 1-min buffer beyond the 24h budget.
@@ -56,6 +56,7 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
         echo "  cosine_epochs: ${cosine_epochs}"
 
         uv run autocast epd --mode slurm "${dry_run_arg[@]}" \
+            datamodule="${datamodule}" \
             local_experiment="${experiment}" \
             logging.wandb.enabled=true \
             optimizer.cosine_epochs="${cosine_epochs}" \
@@ -64,6 +65,6 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
             +trainer.max_epochs="${cosine_epochs}" \
             trainer.callbacks.0.every_n_epochs="${quarter_epochs}" \
             trainer.callbacks.0.save_top_k=-1 \
-            trainer.callbacks.0.filename="quarter-{epoch:04d}"
+            trainer.callbacks.0.filename=\"quarter-{epoch:04d}\"
     done
 done

slurm_scripts/comparison/epd/submit_fm_ambient_large.sh

@@ -13,16 +13,16 @@ set -euo pipefail
 #
 # Per-dataset cosine schedule: each (method, dataset) pair fills its own
 # 24h budget so each model gets its best shot within budget. Values from
-# submit_fm_ambient_timing.sh (2026-04-17) via
+# submit_fm_ambient_timing.sh (2026-04-18) via
 #   uv run autocast time-epochs --from-checkpoint <path>/timing.ckpt -b 24
 #
 # learning_rate (1e-4) and warmup (0) are baked into each per-dataset
 # local_experiment config; adjust the yaml to change them.
 declare -A COSINE_EPOCHS_BY_DATASET=(
-    ["gray_scott"]=2619                 # 32.3s/epoch
-    ["gpe_laser_only_wake"]=3097        # 27.3s/epoch
-    ["conditioned_navier_stokes"]=2917  # 29.0s/epoch
-    ["advection_diffusion"]=3279        # 25.8s/epoch
+    ["gray_scott"]=2631                 # 32.2 s/ep
+    ["gpe_laser_only_wake"]=3171        # 26.7 s/ep
+    ["conditioned_navier_stokes"]=2982  # 28.4 s/ep
+    ["advection_diffusion"]=3264        # 25.9 s/ep
 )
 BUDGET_MAX_TIME="00:23:59:00"
 # SLURM timeout with 1-min buffer beyond the 24h budget.
@@ -60,6 +60,7 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
         echo "  cosine_epochs: ${cosine_epochs}"
 
         uv run autocast epd --mode slurm "${dry_run_arg[@]}" \
+            datamodule="${datamodule}" \
             local_experiment="${experiment}" \
             logging.wandb.enabled=true \
             optimizer.cosine_epochs="${cosine_epochs}" \
@@ -68,6 +69,6 @@ for datamodule in "${!EXPERIMENTS[@]}"; do
             +trainer.max_epochs="${cosine_epochs}" \
             trainer.callbacks.0.every_n_epochs="${quarter_epochs}" \
             trainer.callbacks.0.save_top_k=-1 \
-            trainer.callbacks.0.filename="quarter-{epoch:04d}"
+            trainer.callbacks.0.filename=\"quarter-{epoch:04d}\"
     done
 done

src/autocast/scripts/workflow/naming.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 
 from omegaconf import OmegaConf
+from omegaconf.errors import OmegaConfBaseException
 
 from autocast.scripts.workflow.constants import DATASET_NAME_TOKENS, NAMING_DEFAULT_KEYS
 from autocast.scripts.workflow.overrides import extract_override_value
@@ -85,7 +86,13 @@ def _extract_naming_hints_from_preset(path: Path) -> list[str]:
     if not path.exists():
         return []
 
-    loaded = OmegaConf.to_container(OmegaConf.load(path), resolve=True)
+    try:
+        # Naming only needs a few literal fields; avoid resolving unrelated
+        # interpolations that can fail outside full Hydra composition.
+        loaded = OmegaConf.to_container(OmegaConf.load(path), resolve=False)
+    except OmegaConfBaseException:
+        return []
+
     if not isinstance(loaded, dict):
         return []
 

tests/scripts/test_workflow.py

@@ -332,6 +332,40 @@ def test_auto_run_name_hidden_dim_included():
     assert "256" in name
 
 
+def test_auto_run_name_local_experiment_ignores_unresolved_interpolation(
+    tmp_path: Path, monkeypatch
+):
+    local_cfg = tmp_path / "local_hydra" / "local_experiment" / "repro.yaml"
+    local_cfg.parent.mkdir(parents=True, exist_ok=True)
+    local_cfg.write_text(
+        "\n".join(
+            [
+                "model:",
+                "  processor:",
+                "    _target_: autocast.nn.vit.TemporalViTBackbone",
+                "    n_steps_input: ${datamodule.n_steps_input}",
+                "  loss_func:",
+                "    _target_: autocast.losses.ensemble.CRPSLoss",
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.chdir(tmp_path)
+
+    with (
+        patch("autocast.scripts.workflow.naming._git_hash", return_value="abc1234"),
+        patch("autocast.scripts.workflow.naming._short_uuid", return_value="xyz7890"),
+    ):
+        name = auto_run_name(
+            "epd",
+            "reaction_diffusion",
+            ["local_experiment=repro"],
+        )
+
+    assert name == "crps_rd64_vit_abc1234_xyz7890"
+
+
 # ---------------------------------------------------------------------------
 # commands
 # ---------------------------------------------------------------------------