prepare for release

Author: bminixhofer

src/examples/bolmo/compute_entropies.py

@@ -12,10 +12,6 @@
         --batch-size 32 \
         --sequence-length 2048
 
-Environment variables:
-    DATA_SOURCE: One of 'dclm', 'dolmino', 'dolma2_code_string', 'dolmino_code_string', 'tulu3'
-    HAS_WEKA: Set to '1' if running on Weka filesystem
-
 For each input shard like "part-0-00000.npy", this will create:
     - output_dir/entropy/path/to/part-0-00000.npy (entropy values as raw binary float16)
     - output_dir/cross_entropy/path/to/part-0-00000.npy (cross-entropy values as raw binary float16)

src/examples/bolmo/instructify.py

@@ -33,14 +33,10 @@
 
 def parse_args():
     parser = argparse.ArgumentParser(description="Instructify OLMo model merging script")
-    parser.add_argument("--checkpoint-dir", type=str, default="/weka/oe-training-default/benjaminm/runs/stage2n_hnet_v8-e1d4-w-ee-no-sm-150k-half-local-lr_wd01gc05/step150000",
-                        help="Path to the main checkpoint directory")
-    parser.add_argument("--output", type=str, default="/weka/oe-training-default/benjaminm/merges/stage2n_hnet_v8-e1d4-w-ee-no-sm-150k-half-local-lr_wd01gc05_zeroshot_instruct_alpha1",
-                        help="Path to output directory")
-    parser.add_argument("--base-checkpoint-dir", type=str, default="/weka/oe-training-default/benjaminm/checkpoints/olmo2_1b",
-                        help="Path to base checkpoint directory")
-    parser.add_argument("--instruct-checkpoint-dir", type=str, default="/weka/oe-training-default/benjaminm/checkpoints/olmo2_1b_instruct",
-                        help="Path to instruct checkpoint directory")
+    parser.add_argument("--checkpoint-dir", type=str, help="Path to the main checkpoint directory")
+    parser.add_argument("--output", type=str, help="Path to output directory")
+    parser.add_argument("--base-checkpoint-dir", type=str, help="Path to base checkpoint directory")
+    parser.add_argument("--instruct-checkpoint-dir", type=str, help="Path to instruct checkpoint directory")
     parser.add_argument("--include-instruct-teacher", action="store_true",
                         default=True,
                         help="Include instruct teacher in the merged model")