models_fullextend: config #3 (random/always/detachF) + uniform MC9 plumbing

- add config #3 ghost run script (random coeff mode; completes the sweep)
- register config #3 in CE-curves export
- add uniform run to convert_to_hf (HF coeff-mode now derived from runname),
  launch_mc9_eval (idempotent: skips already-scored configs), and the MC9
  results export columns
- README status: config #2 complete (CE 2.690, neutral), config #3 launched

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: ryanyxw

scripts/models_fullextend/README.md

@@ -147,7 +147,13 @@ hist = {int(x["_step"]): x["train/CE loss"] for x in r.scan_history(keys=["_step
   hard-stop, **final CE 2.654** vs the no-ghost baseline's **2.689** at the same
   step (mean gap over the run ≈ −0.005). The ghost adds no convergence
   penalty (slight edge at 50B; within run-to-run noise).
-- In progress: config #2 (`uniform / always / detachF`); remaining
-  coefficient-mode / `detach_coeff` configs chosen incrementally.
-- Not yet done: the downstream "actually add a new expert and measure
-  degradation" evaluation; the `topk` route.
+- **Config #2 (`uniform / always / detachF`) complete** — trained to the 50B
+  hard-stop, **final CE 2.690** vs the no-ghost baseline's **2.689** at the same
+  step (essentially identical, +0.001). A naive uniform pool-average ghost is
+  convergence-neutral; the usage-weighted blend's slight edge does not transfer
+  to the uniform mode.
+- **Config #3 (`random / always / detachF`) launched** — completes the
+  coefficient-mode sweep. `detach_coeff` is a no-op for `random` (alpha constant).
+- Not yet done: MC9 distribution-shift probe for config #2 (HF conversion + eval
+  in flight); the downstream "actually add a new expert and measure degradation"
+  evaluation; the `topk` route.

scripts/models_fullextend/convert_to_hf.sh

@@ -12,6 +12,7 @@ MODELS_DIR="${MODELS_DIR:-models_fullextend}"
 # "<olmo-core checkpoint dir>|<HF output dir>". Add configs as they finish.
 MODELS=(
     "${MODELS_DIR}/emo_1b14b_130b_ghost_usage_always_detachF/step11921|${MODELS_DIR}/emo_1b14b_130b_ghost_usage_always_detachF/step11921-hf"
+    "${MODELS_DIR}/emo_1b14b_130b_ghost_uniform_always_detachF/step11921|${MODELS_DIR}/emo_1b14b_130b_ghost_uniform_always_detachF/step11921-hf"
 )
 
 for PAIR in "${MODELS[@]}"; do
@@ -32,10 +33,17 @@ for PAIR in "${MODELS[@]}"; do
 done
 
 # Materialize the ghost-on variant (symlinked weights + config.json with the toggle) for
-# each converted checkpoint, so the MC9 eval can run the with-ghost arm.
+# each converted checkpoint, so the MC9 eval can run the with-ghost arm. The eval-time
+# ghost coeff_mode is matched to the run's training coeff_mode, read off the runname
+# (..._ghost_<mode>_always_detachF/...): usage -> usage, uniform -> uniform, random -> random.
 for PAIR in "${MODELS[@]}"; do
     OUTPUT="${PAIR##*|}"
     if [ -f "${OUTPUT}/config.json" ]; then
-        python scripts/models_fullextend/make_ghost_hf_variant.py --src "${OUTPUT}" --coeff-mode usage
+        case "$OUTPUT" in
+            *_ghost_uniform_*) COEFF_MODE=uniform ;;
+            *_ghost_random_*)  COEFF_MODE=random ;;
+            *)                 COEFF_MODE=usage ;;
+        esac
+        python scripts/models_fullextend/make_ghost_hf_variant.py --src "${OUTPUT}" --coeff-mode "${COEFF_MODE}"
     fi
 done

scripts/models_fullextend/emo_1b14b_130b_ghost_random_always_detachF.sh

@@ -0,0 +1,70 @@
+# PARENT: "scripts/models_fullextend/emo_1b14b_130b_ghost_uniform_always_detachF.sh"
+# DESCRIPTION:
+#     - models_fullextend ghost-expert SWEEP run, config #3.
+#       Ghost knobs: coeff_mode=random, route=always, detach_coeff=false, num=1, random_k=8.
+#       Completes the coeff_mode sweep (usage / uniform / random). Each ghost is the
+#       uniform average over a random sample of random_k=8 document-pool experts
+#       (re-sampled per forward). detach_coeff is a no-op for random (alpha is constant).
+#     - 16 nodes, max_duration=130B, hard_stop=50B. 2-deep rolling checkpoints + final;
+#       pre_train_checkpoint disabled (skip the useless random-init step0).
+##############################################################
+source "$(dirname "${BASH_SOURCE[0]}")/../launch_common.sh"
+
+EXPERIMENT_NAME="models_fullextend"
+MODELS_DIR="/weka/oe-training-default/ryanwang/EMO/${EXPERIMENT_NAME}"
+DATA_ROOT="s3://ai2-llm"
+
+BEAKER_NODES=16
+BEAKER_GPUS=8
+
+min_document_expert_pool=8
+max_document_expert_pool=128
+eval_document_expert_pool=32
+lr=4e-3
+lb=1e-1
+
+num_shared_experts=1 # 1 out of 8 will be shared experts
+
+# --- ghost-expert knobs (this run) ---
+ghost_extend_num=1               # fixed across the sweep
+ghost_extend_random_k=8          # fixed across the sweep
+ghost_extend_coeff_mode="random" # swept: usage | uniform | random
+ghost_extend_route="always"      # swept: always (topk not implemented)
+ghost_extend_detach_coeff=false  # swept: false | true (no-op for random)
+
+runname="emo_1b14b_130b_ghost_random_always_detachF"
+
+launch src/scripts/train/olmoe-1B-7B_fsl.py $runname \
+		--save-folder="${MODELS_DIR}/$runname" \
+		--dataset.mix=OLMoE-mix-0824 \
+		--work-dir="${DATASET_CACHE}" \
+		--trainer.max_duration='{value: 130_000_000_000, unit: tokens}' \
+		--trainer.hard_stop='{value: 50_000_000_000, unit: tokens}' \
+		--trainer.callbacks.checkpointer.save_interval=1000000 \
+		--trainer.callbacks.checkpointer.ephemeral_save_interval=500 \
+		--trainer.callbacks.checkpointer.keep_ephemeral=2 \
+		--trainer.callbacks.checkpointer.pre_train_checkpoint=false \
+		--trainer.callbacks.wandb.enabled=true \
+		--trainer.callbacks.wandb.entity=ryanyxw \
+		--trainer.callbacks.wandb.project=emo-extension \
+		--trainer.callbacks.wandb.name="${runname}" \
+		--trainer.callbacks.wandb.tags="[pretraining, ${EXPERIMENT_NAME}]" \
+		--model.block.feed_forward_moe.num_experts=128 \
+		--dataset.generate_doc_lengths=true \
+		--model.block.sequence_mixer.backend=flash_2 \
+		--model-type="two-level_lb-batch_reduce-dp_sharedexp_randpool" \
+		--min_document_expert_pool=${min_document_expert_pool} \
+		--max_document_expert_pool=${max_document_expert_pool} \
+		--eval_document_expert_pool=${eval_document_expert_pool} \
+		--num_shared_experts=$num_shared_experts \
+		--dataset.instance_filter_config='{repetition_max_period: 13, repetition_min_period: 1, repetition_max_count: 32}' \
+		--model.block.name="moe" \
+		--model.block.sequence_mixer.qk_norm=null \
+		--lr=${lr} \
+		--model.block.feed_forward_moe.lb_loss_weight=${lb} \
+		--model.block.feed_forward_moe.router.ghost_extend_mode=true \
+		--model.block.feed_forward_moe.router.ghost_extend_num=${ghost_extend_num} \
+		--model.block.feed_forward_moe.router.ghost_extend_coeff_mode=${ghost_extend_coeff_mode} \
+		--model.block.feed_forward_moe.router.ghost_extend_random_k=${ghost_extend_random_k} \
+		--model.block.feed_forward_moe.router.ghost_extend_route=${ghost_extend_route} \
+		--model.block.feed_forward_moe.router.ghost_extend_detach_coeff=${ghost_extend_detach_coeff}

scripts/models_fullextend/export_ce_curves.py

@@ -34,6 +34,7 @@
     ),
     ("ryanyxw/emo-extension", "emo_1b14b_130b_ghost_usage_always_detachF", "usage / always / detachF"),
     ("ryanyxw/emo-extension", "emo_1b14b_130b_ghost_uniform_always_detachF", "uniform / always / detachF"),
+    ("ryanyxw/emo-extension", "emo_1b14b_130b_ghost_random_always_detachF", "random / always / detachF"),
 ]
 
 # (chart key, wandb metric key, chart title, make_chart). make_chart=False metrics are

scripts/models_fullextend/export_mc9_results.py

@@ -17,8 +17,10 @@
 # (run dir, mode, column label). Order = display order.
 COLUMNS = [
     ("no_ghost_baseline_130b", "standard", "no-ghost baseline (130B)"),
-    ("ghost_usage_50b", "standard", "ghost-trained, ghost OFF (50B)"),
-    ("ghost_usage_50b", "ghost", "ghost-trained, ghost ON (50B)"),
+    ("ghost_usage_50b", "standard", "usage ghost-trained, ghost OFF (50B)"),
+    ("ghost_usage_50b", "ghost", "usage ghost-trained, ghost ON (50B)"),
+    ("ghost_uniform_50b", "standard", "uniform ghost-trained, ghost OFF (50B)"),
+    ("ghost_uniform_50b", "ghost", "uniform ghost-trained, ghost ON (50B)"),
 ]
 
 

scripts/models_fullextend/launch_mc9_eval.sh

@@ -35,6 +35,7 @@ DRY_RUN="${DRY_RUN:-0}"
 # "name|hf_path|modes"  (modes = space-separated subset of {standard, ghost})
 MODELS=(
   "ghost_usage_50b|${WEKA_ROOT}/models_fullextend/emo_1b14b_130b_ghost_usage_always_detachF/step11921-hf|standard ghost"
+  "ghost_uniform_50b|${WEKA_ROOT}/models_fullextend/emo_1b14b_130b_ghost_uniform_always_detachF/step11921-hf|standard ghost"
   "no_ghost_baseline_130b|${WEKA_ROOT}/models_sizescaling/emo_1b14b_130b/step30995-hf|standard"
 )
 
@@ -68,6 +69,13 @@ launch_one() {
   job="mc9-${name}-${mode}-${safe_task}"
   job=$(echo "$job" | sed 's/[^a-zA-Z0-9_-]//g' | cut -c1-80)
 
+  # Idempotent: skip configs already scored (so re-running to add a new model doesn't
+  # re-submit the finished usage/baseline jobs). Set FORCE=1 to re-run regardless.
+  if [ "${FORCE:-0}" != "1" ] && [ -f "${out}/metrics.json" ]; then
+    echo "=== skip ${name} | ${mode} | ${task} (metrics.json exists) ==="
+    return
+  fi
+
   echo ">>> ${name} | ${mode} | ${task}  ->  ${out}"
   if [ "$DRY_RUN" = "1" ]; then
     echo "    model: ${hf_path}"