Repository navigation
Expand file tree
/
Copy pathtrain_args.py
More file actions
1656 lines (1475 loc) · 103 KB
/
Copy pathtrain_args.py
File metadata and controls
1656 lines (1475 loc) · 103 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# train_args.py
import argparse
import math
import re
import json
from train_variations.loss_variants import LOSS_VARIANTS
from train_variations.distillation_loss_variants import (
DISTILLATION_LOSS_VARIANTS,
)
def clean_dataset_path(dataset_name):
"""Removes leading './data/' or 'data/' from dataset paths."""
return re.sub(r'^(?:\./)?data/', '', dataset_name)
def parse_args():
parser = argparse.ArgumentParser()
# argparse groups
model_group = parser.add_argument_group('model_group')
training_group = parser.add_argument_group('training_group')
logging_group = parser.add_argument_group('logging_group')
model_group.add_argument(
'--attention_residual_variant', default='standard', choices=['standard', 'full'],
help='Residual stream implementation: ordinary addition or full depth-wise attention.',
)
model_group.add_argument('--attention_residual_eps', default=1e-6, type=float,
help='RMSNorm epsilon used for Full Attention Residual routing keys.')
# MLP Bias Configuration
model_group.add_argument('--mlp_up_bias', default=None, action=argparse.BooleanOptionalAction, help='Whether to use bias in MLP up projections. If None, uses global bias setting.')
model_group.add_argument('--mlp_down_bias', default=None, action=argparse.BooleanOptionalAction, help='Whether to use bias in MLP down projections. If None, uses global bias setting.')
# MLP x y Offset
model_group.add_argument('--mlp_x_offset', type=float, default=0.0, help='X-axis offset for mlp activation functions')
model_group.add_argument('--mlp_y_offset', type=float, default=0.0, help='Y-axis offset for mlp activation functions')
model_group.add_argument('--learn_mlp_x_offset', default=False, action=argparse.BooleanOptionalAction, help='Whether to learn the x-axis offset for mlp')
model_group.add_argument('--learn_mlp_y_offset', default=False, action=argparse.BooleanOptionalAction, help='Whether to learn the y-axis offset for mlp')
# L2 Normalization options
model_group.add_argument('--l2_norm_mlp_up', default=False, action=argparse.BooleanOptionalAction,
help='L2 normalize MLP up projection weights along embedding dimension')
model_group.add_argument('--l2_norm_mlp_down', default=False, action=argparse.BooleanOptionalAction,
help='L2 normalize MLP down projection weights along embedding dimension')
model_group.add_argument('--l2_norm_mlp_up_dim', type=str, default='embed', choices=['embed', 'hidden'],
help="Dimension for L2-normalizing MLP up projections: 'embed' or 'hidden'")
model_group.add_argument('--l2_norm_mlp_down_dim', type=str, default='hidden', choices=['embed', 'hidden'],
help="Dimension for L2-normalizing MLP down projections: 'embed' or 'hidden'")
model_group.add_argument('--l2_norm_print_dims', default=False, action=argparse.BooleanOptionalAction,
help='Print the dimension size used for L2 normalization when enabled')
# Attention L2 Normalization options (Infinite attention)
model_group.add_argument('--l2_norm_attn_q', default=False, action=argparse.BooleanOptionalAction,
help='L2 normalize attention query projection weights')
model_group.add_argument('--l2_norm_attn_k', default=False, action=argparse.BooleanOptionalAction,
help='L2 normalize attention key projection weights')
model_group.add_argument('--l2_norm_attn_v', default=False, action=argparse.BooleanOptionalAction,
help='L2 normalize attention value projection weights')
model_group.add_argument('--l2_norm_attn_cproj', default=False, action=argparse.BooleanOptionalAction,
help='L2 normalize attention output projection weights')
model_group.add_argument('--l2_norm_attn_q_dim', type=str, default='embed', choices=['embed', 'hidden'],
help="Dimension for L2-normalizing attention query projections: 'embed' or 'hidden'")
model_group.add_argument('--l2_norm_attn_k_dim', type=str, default='embed', choices=['embed', 'hidden'],
help="Dimension for L2-normalizing attention key projections: 'embed' or 'hidden'")
model_group.add_argument('--l2_norm_attn_v_dim', type=str, default='embed', choices=['embed', 'hidden'],
help="Dimension for L2-normalizing attention value projections: 'embed' or 'hidden'")
model_group.add_argument('--l2_norm_attn_cproj_dim', type=str, default='embed', choices=['embed', 'hidden'],
help="Dimension for L2-normalizing attention output projections: 'embed' or 'hidden'")
# Export Args
## Factored WTE
model_group.add_argument('--import_wte_npy', default=None, type=str, help='Path to import the embedding table as a .npy file')
model_group.add_argument('--export_wte_npy', default=None, type=str, help='Path to export the embedding table as a .npy file')
model_group.add_argument('--export_wte_each_eval', default=False, action=argparse.BooleanOptionalAction, help="Requires --export_wte is not None. If this is so, will always export embedding to numpy after evaluation")
model_group.add_argument('--import_wte_freeze', default=False, action=argparse.BooleanOptionalAction, help="Whether to freeze an imported wte")
## Factored Scale Matrices
model_group.add_argument('--import_scale_matrices_npz', default=None, type=str, help='Path to import the scale matrices as a .npz file')
model_group.add_argument('--export_scale_matrices_npz', default=None, type=str, help='Path to export the scale matrices as a .npz file')
model_group.add_argument('--export_scale_matrices_each_eval', default=False, action=argparse.BooleanOptionalAction, help="Requires --export_scale_matrices_npz is not None. If this is so, will always export to npz after evaluation")
model_group.add_argument('--import_scale_matrices_freeze', default=False, action=argparse.BooleanOptionalAction, help="Whether to freeze scaled_matrices")
# I/O args
training_group.add_argument('--out_dir', default='out', type=str)
training_group.add_argument('--eval_interval', default=250, type=int)
training_group.add_argument('--log_interval', default=10, type=int)
training_group.add_argument('--eval_iters', default=200, type=int)
training_group.add_argument('--eval_only', default=False, action=argparse.BooleanOptionalAction)
# Validation-time LM-head minimum-angle graph export args
training_group.add_argument('--export_min_angle_graph_dir', default=None, type=str,
help='Optional directory for per-validation LM-head minimum-angle graph exports.')
training_group.add_argument('--export_min_angle_graph_each_eval', default=False,
action=argparse.BooleanOptionalAction,
help='Export the LM-head minimum-angle graph after every validation loss.')
training_group.add_argument('--export_min_angle_graph_block_size', default=2048, type=int,
help='Row/column block size used when exporting the minimum-angle graph.')
training_group.add_argument('--export_min_angle_graph_device', default='auto', type=str,
help="Compute device for minimum-angle graph export: 'auto', 'cpu', or a CUDA device such as 'cuda:0'.")
training_group.add_argument('--export_min_angle_graph_label', default=None, type=str,
help='Optional label inserted into minimum-angle graph export filenames.')
training_group.add_argument(
'--mezo_epsilon',
type=float,
default=1e-3,
help='Perturbation scale for MeZO forward-only training.',
)
training_group.add_argument(
'--mezo_seed',
type=int,
default=None,
help='Optional fixed seed for MeZO perturbations (defaults to random per step).',
)
# latency / ETA estimate options
training_group.add_argument('--eta_variant', choices=['iteration', 'eval_cycle'], default='eval_cycle', help="iteration - estimates only based on training iterations -- use if doing one eval at the end; eval_cycle -- use if doing multiple evals, will use a single cycle for the estimation.")
training_group.add_argument('--iteration_window', default=100, type=int)
training_group.add_argument('--eval_cycle_window', default=5, type=int)
# Loss variations
training_group.add_argument(
'--loss_fn',
type=str,
default='cross_entropy',
choices=sorted(LOSS_VARIANTS.keys()),
help='Loss function to use during training.',
)
training_group.add_argument(
'--loss_schedule',
type=str,
default=None,
help='Comma-separated schedule of step:loss_fn pairs to switch loss during training.',
)
training_group.add_argument(
'--rank_scale',
type=float,
default=1.0,
help='Base scaling factor for rank_distance loss.',
)
training_group.add_argument(
'--rank_scale_schedule',
type=str,
default=None,
help="Schedule for rank_distance's scaling factor, e.g. 0:1.0,1000:2.0",
)
# Loss hyperparameters
training_group.add_argument(
'--label_smoothing',
type=float,
default=0.1,
help='Smoothing factor for label_smoothing loss.',
)
training_group.add_argument(
'--focal_gamma',
type=float,
default=2.0,
help='Gamma parameter for focal-style losses.',
)
training_group.add_argument(
'--top1_focus_alpha',
type=float,
default=0.5,
help='Penalty weight for incorrect top-1 predictions in top1_focus loss.',
)
training_group.add_argument(
'--top1_margin',
type=float,
default=0.1,
help='Desired logit margin for top1_margin loss.',
)
training_group.add_argument(
'--entropy_beta',
type=float,
default=0.01,
help='Weight of entropy penalty in entropy-based losses.',
)
training_group.add_argument(
'--top1_ratio_beta',
type=float,
default=0.5,
help='Weight for ratio penalty in top1_ratio loss.',
)
training_group.add_argument(
'--flatness_beta',
type=float,
default=1.0,
help='Scaling for flatness_boost loss when predictions are flat.',
)
training_group.add_argument(
'--bit_loss_weight',
type=float,
default=1e-6,
help='Weight of the bit-usage penalty in bit_balanced_cross_entropy.',
)
training_group.add_argument(
'--bit_loss_normalize',
default=False,
action=argparse.BooleanOptionalAction,
help='Normalize the bit penalty by parameter count in bit_balanced_cross_entropy.',
)
training_group.add_argument(
'--correct_top1_attenuation',
type=float,
default=1.0,
help='Constant multiplier for correctly predicted tokens in attenuated_correct_top1 loss.',
)
training_group.add_argument(
'--distance_top1_strength',
type=float,
default=0.0,
help='Strength of distance-based attenuation in distance_attenuated_top1 loss.',
)
# Distillation options
training_group.add_argument(
'--distillation_teacher_ckpt',
type=str,
default=None,
help='Path to a teacher checkpoint (ckpt.pt) for knowledge distillation.',
)
training_group.add_argument(
'--distillation_loss',
type=str,
default=None,
choices=sorted(DISTILLATION_LOSS_VARIANTS.keys()),
help='Distillation loss variant applied when a teacher checkpoint is provided.',
)
training_group.add_argument(
'--distillation_weight',
type=float,
default=1.0,
help='Scaling factor applied to the distillation loss before combining with the student loss.',
)
training_group.add_argument(
'--distillation_temperature',
type=float,
default=1.0,
help='Logit temperature used when computing distillation losses.',
)
training_group.add_argument(
'--distillation_eps',
type=float,
default=1e-8,
help='Numerical stability epsilon for distillation losses.',
)
training_group.add_argument(
'--passive_distillation_loss_log',
default=False,
action=argparse.BooleanOptionalAction,
help=(
'Compute and log the selected distillation loss without adding it to '
'the training objective. Requires --distillation_teacher_ckpt and --distillation_loss.'
),
)
training_group.add_argument(
'--log_ntp_val_loss_during_distillation',
default=True,
action=argparse.BooleanOptionalAction,
help=(
'When distillation is configured, also log next-token prediction validation '
'loss as a baseline metric.'
),
)
# Sample args
training_group.add_argument('--max_sample_tokens', default=None, type=int, help="If set, maximum number of tokens to sample and print after each validation loss")
training_group.add_argument('--sample_each_eval', default=False, action=argparse.BooleanOptionalAction, help="Produce sample even if the validation loss did not improve. Allows for testing what overtraining looks like.")
training_group.add_argument('--sample_start_tokens', default='\n', type=str)
training_group.add_argument('--sample_only', default=False, action=argparse.BooleanOptionalAction, help="Run only the sampling process and exit")
training_group.add_argument('--dataset_benchmarks', default=False, action=argparse.BooleanOptionalAction, help="Run dataset benchmark metrics on a random slice after each validation")
training_group.add_argument('--sample_metrics', default=False, action=argparse.BooleanOptionalAction, help="Display sample metrics like spelling correctness during sampling")
# Checkpoint args
training_group.add_argument('--save_major_ckpt_interval', default=None, type=int, help="Interval for saving major checkpoints.")
training_group.add_argument('--only_save_checkpoint_at_end', default=False, action=argparse.BooleanOptionalAction)
training_group.add_argument('--always_save_checkpoint', default=False, action=argparse.BooleanOptionalAction)
training_group.add_argument('--never_save_checkpoint', default=False, action=argparse.BooleanOptionalAction, help="If set, disables saving of all checkpoints.")
training_group.add_argument('--patience', default=None, type=int, help="if set, will stop training if the number of evaluations since val loss was seen to decrease exceeds 'patience' setting.")
training_group.add_argument('--init_from', default='scratch', choices=['scratch', 'prev_run', 'resume', 'gpt2'], type=str)
training_group.add_argument('--gpt2_type', default='gpt2', type=str)
training_group.add_argument('--prev_run_ckpt', default='', type=str)
training_group.add_argument('--csv_ckpt_dir', default='', type=str)
training_group.add_argument('--init_from_ckpt', default='ckpt.pt', type=str, help="if save_major_ckpt_interval was set, can use to init from specific ckpts")
training_group.add_argument('--reset_best_val_loss_on_resume', default=False, action=argparse.BooleanOptionalAction,
help="When resuming, reset best_val_loss tracking instead of loading it from the checkpoint.")
# Training modes
# TODO: find a way to merge this with the multicontext arg
training_group.add_argument(
'--training_mode',
default='single',
choices=['single', 'multidataset', 'multicontext'],
help="Training mode to use. 'multidataset' uses sequential sampling from multiple datasets. 'multicontext' processes multiple contexts simultaneously."
)
# Data args
training_group.add_argument('--dataset', default='shakespeare_char', type=str)
training_group.add_argument('--batch_size', default=64, type=int)
training_group.add_argument("--seed", default=1337, type=int)
# Multicontext Training Dataset args
model_group.add_argument('--numerical_multicontext', default=False, action=argparse.BooleanOptionalAction,
help="Interpret multicontext inputs as numerical values and use regression heads")
model_group.add_argument('--numerical_mlp_hidden_dim', default=64, type=int,
help="Hidden dimension for numerical multi-context embedding/output MLPs")
model_group.add_argument('--numerical_mlp_hidden_dims', default=None, nargs='+', type=int,
help="List of hidden dimensions for numerical MLP mappings (overrides --numerical_mlp_num_layers)")
model_group.add_argument('--numerical_mlp_num_layers', default=2, type=int,
help="Number of hidden layers for numerical MLP mappings when hidden dims not provided")
model_group.add_argument('--numerical_mlp_activation_variant', default="relu", type=str,
help="Activation variant for numerical MLP mappings")
model_group.add_argument('--numerical_embedding_variant', default="mlp", type=str,
help="Variant for numerical multicontext input mapping (e.g., mlp, linear)")
model_group.add_argument('--numerical_output_variant', default="mlp", type=str,
help="Variant for numerical multicontext output mapping (e.g., mlp, linear)")
model_group.add_argument('--numerical_mapping_weight_tying', default=True, action=argparse.BooleanOptionalAction,
help="Tie numerical embedding/output mapping weights when supported")
model_group.add_argument(
'--numerical_multicontext_input_format',
default='scalar',
choices=['scalar', 'fp16_bits'],
help=(
"How to interpret raw uint16 values for numerical multicontext inputs/targets. "
"'scalar' uses the integer value directly, 'fp16_bits' decodes IEEE-754 half bits to float32."
),
)
model_group.add_argument(
'--numerical_loss_huber_delta',
default=0.5,
type=float,
help="Delta parameter for numerical multicontext Huber loss.",
)
model_group.add_argument(
'--numerical_loss_use_cosine',
default=False,
action=argparse.BooleanOptionalAction,
help="Enable cosine similarity term in numerical multicontext regression loss.",
)
model_group.add_argument(
'--numerical_loss_cosine_coeff',
default=0.5,
type=float,
help="Coefficient for cosine loss term when --numerical_loss_use_cosine is enabled.",
)
model_group.add_argument(
"--norm_channel_variant",
type=str,
default=None,
choices=[
"krmsnorm",
"prmsnorm",
"rmsnorm",
"layernorm",
"hyperspherenorm",
"dact",
"cappedhyperspherenorm",
"identity",
],
help="Optional post-mapping normalization applied to numerical embedding channels.",
)
model_group.add_argument('--norm_channel_radius', type=float, default=None)
model_group.add_argument('--norm_channel_scale', type=float, default=None)
model_group.add_argument('--norm_channel_gain', type=bool, default=None, action=argparse.BooleanOptionalAction)
model_group.add_argument('--norm_channel_radius_learning', type=bool, default=None, action=argparse.BooleanOptionalAction)
model_group.add_argument('--multicontext', default=False, action=argparse.BooleanOptionalAction,
help="Enable multi-context training on multiple simultaneous datasets")
model_group.add_argument('--multidataset_wte', default=False, action=argparse.BooleanOptionalAction,
help='Use separate token embeddings and lm heads per dataset when training_mode is multidataset')
training_group.add_argument('--multicontext_datasets', default=None, nargs='+', type=str,
help="List of datasets to train on in multi-context mode (e.g., --multicontext_datasets shakespeare wikitext103 openwebtext)")
model_group.add_argument('--vocab_sizes', default=None, nargs='+', type=int,
help="List of vocabulary sizes for each dataset in --multicontext_datasets")
# Batch sampling args
training_group.add_argument('--sampling_method', default="random",
choices=["random", "sequential", "without_replacement"],
help="Sampling method for get_batch: 'random' (with replacement), 'sequential' (without shuffling), or 'without_replacement' (shuffled without replacement)")
# Add a new argument for specifying multiple datasets
training_group.add_argument('--dataset_list', default=None, nargs='+', type=str, help="If not None, training will be done from a list of datasets to train on, e.g. --dataset_list shakespeare wikitext103 openwebtext")
training_group.add_argument('--dataset_interleaving', default=False, action=argparse.BooleanOptionalAction)
training_group.add_argument('--dataset_interleaving_shuffle', default=False, action=argparse.BooleanOptionalAction)
training_group.add_argument('--dataset_sampling_learning_rate', default=None, nargs='+', type=float, help="Sampling learning rates for each dataset in dataset_list.")
training_group.add_argument('--dataset_sampling_probs', action=FlattenListAction, default=None, nargs='+', help="Sampling proportions for each dataset in dataset_list. Probabilities normally but proportions in dataset_interleaving")
training_group.add_argument('--dataset_sampling_probs_final', action=FlattenListAction,default=None, nargs='+', help="If, set final sampling probabilities for each dataset in dataset_list.")
training_group.add_argument('--dataset_sampling_probs_transition_method', default=None, type=str, choices=["linear", "cosine", "exponential"])
# Add GNS settings
training_group.add_argument('--gns_type', type=str, default=None, choices=['sogns', 'exact'], help='Type of gradient norm scaling to use (default: None)')
training_group.add_argument('--gns_ema_beta', type=float, default=0.9, choices=['sogns', 'exact'], help='Type of gradient norm scaling to use (default: None)')
training_group.add_argument('--gns_target', type=float, default=None)
training_group.add_argument('--gns_max_batch', type=int, default=100)
training_group.add_argument('--gns_batch_pct', type=float, default=0.2)
# Optimizer-specific arguments
optimizer_variations = [
"sgd",
"adam",
"adamw",
"adamw_act_reg",
"adamax",
"radam",
"nadam",
"adagrad",
"rmsprop",
"rprop",
"sparseadam",
"asgd",
"lbfgs",
"adabelief",
"orthoadam",
"adams",
"ademamix",
"adan",
"qhadam",
"yogi",
"adamp",
"lion",
"adafactor",
"accsgd",
"adabound",
"adamod",
"aggmo",
"diffgrad",
"lamb",
"lambdiff",
"adamod_diffgrad",
"novograd",
"pid",
"qhm",
"sgdp",
"sgdw",
"shampoo",
"swats",
"sophiag",
"soap",
"var_adaptive_lr",
"lookahead",
"entropy_aware_adamw",
"muon",
]
training_group.add_argument("--optimizer", type=str, default="adamw",
choices=optimizer_variations,
help="Optimizer to use for training.")
# -------- SGD --------------------------------------------------
training_group.add_argument("--sgd_momentum", type=float, default=0.9, help="Momentum for SGD optimizer.")
training_group.add_argument("--sgd_nesterov", type=bool, default=False, action=argparse.BooleanOptionalAction)
# -------- MUON --------------------------------------------------
training_group.add_argument("--muon_momentum", type=float, default=0.95,
help="Momentum for the Muon optimizer.")
training_group.add_argument("--muon_ns_steps", type=int, default=5,
help="Newton-Schulz iteration steps for Muon orthogonalization.")
training_group.add_argument("--muon_nesterov", type=bool, default=True, action=argparse.BooleanOptionalAction,
help="Use Nesterov momentum in Muon update.")
training_group.add_argument("--muon_include_all_weights", type=bool, default=False, action=argparse.BooleanOptionalAction,
help="Route all matrix-shaped parameters, including embeddings and heads, to Muon; vectors remain on auxiliary Adam.")
training_group.add_argument("--muon_min_ndim", type=int, default=2,
help="Minimum tensor ndim eligible for Muon routing (default preserves current behavior).")
training_group.add_argument("--muon_exclude_substrings", type=str, nargs="*", default=["embed", "wte", "wpe", "lm_head"],
help="Parameter-name substrings excluded from Muon routing unless force-included.")
training_group.add_argument("--muon_force_include_substrings", type=str, nargs="*", default=[],
help="Parameter-name substrings force-included into Muon routing, even if excluded.")
# -------- ADAMW --------------------------------------------------
training_group.add_argument("--adamw_betas", type=float, nargs=2, default=[0.9, 0.999], help="Betas for AdamW optimizer.")
training_group.add_argument("--adamw_eps", type=float, default=1e-8, help="Epsilon for AdamW optimizer.")
training_group.add_argument("--adamw_weight_decay", type=float, default=0.01, help="Weight decay for AdamW optimizer.")
training_group.add_argument(
"--entropy_lr_boost",
type=float,
default=1.0,
help="Coefficient for entropy_aware_adamw to scale learning rate when logits are flat.",
)
# -------- ADAMW_ACT_REG --------------------------------------------------
training_group.add_argument(
"--activation_decay",
type=float,
default=0.0,
help="L2 regularization coefficient for activations used by adamw_act_reg optimiser.",
)
training_group.add_argument(
"--activation_stat",
type=str,
default="stdev",
choices=["stdev", "kurtosis", "max", "min", "abs_max"],
help="Statistic to modulate activation regularization for adamw_act_reg optimizer.",
)
# -------- ADAGRAD --------------------------------------------------
training_group.add_argument("--adagrad_lr_decay", type=float, default=0, help="Learning rate decay for Adagrad optimizer.")
# -------- RMSProp --------------------------------------------------
training_group.add_argument("--rmsprop_alpha", type=float, default=0.99, help="Smoothing constant for RMSprop.")
# -------- ADAMS --------------------------------------------------
training_group.add_argument("--adams_beta1", type=float, default=0.9, help="Beta1 for AdamS optimizer.")
training_group.add_argument("--adams_beta2", type=float, default=0.95, help="Beta2 for AdamS optimizer. Paper suggests 0.95 vs 0.999 for numerical stability.")
# -------- ADEMAMIX --------------------------------------------------
training_group.add_argument("--ademamix_beta1", type=float, default=0.9, help="β1 hyper-parameter for the Ademamix optimizer.")
training_group.add_argument("--ademamix_beta2", type=float, default=0.999, help="β2 hyper-parameter for the Ademamix optimizer.")
training_group.add_argument("--ademamix_beta3", type=float, default=0.9999, help="β3 hyper-parameter for the Ademamix optimizer.")
training_group.add_argument("--ademamix_alpha", type=float, default=8, help="α (scaling factor) for the Ademamix optimizer.")
training_group.add_argument("--ademamix_warmup", type=int, default=2000, help="Number of warm-up steps for Ademamix's learning-rate schedule.")
# -------- NADAM --------------------------------------------------
training_group.add_argument("--nadam_betas", type=float, nargs=2, default=[0.9, 0.999], help="Betas for Nadam optimizer.")
training_group.add_argument("--nadam_eps", type=float, default=1e-8, help="Epsilon for Nadam optimizer.")
# -------- ASGD --------------------------------------------------
training_group.add_argument("--asgd_lambda", type=float, default=1e-4, help="Decay term for ASGD.")
training_group.add_argument("--asgd_alpha", type=float, default=0.75, help="Power for eta calculation in ASGD.")
training_group.add_argument("--asgd_t0", type=float, default=1e6, help="Point at which to start averaging in ASGD.")
# -------- OrthoAdam --------------------------------------------------
training_group.add_argument("--ortho_perm_threshold", type=int, default=1_000_000, help="If a tensor has more elements than this, OrthoAdam uses identity rotation for it.")
# -------- LBFGS --------------------------------------------------
training_group.add_argument("--lbfgs_max_iter", type=int, default=20, help="Maximum iterations per LBFGS step.")
training_group.add_argument("--lbfgs_max_eval", type=int, default=None, help="Maximum function evaluations per LBFGS step.")
training_group.add_argument("--lbfgs_tol_grad", type=float, default=1e-7, help="Gradient-norm tolerance for LBFGS convergence.")
training_group.add_argument("--lbfgs_tol_change", type=float, default=1e-9, help="Parameter-change tolerance for LBFGS convergence.")
training_group.add_argument("--lbfgs_history", type=int, default=100, help="History size (m) for LBFGS.")
training_group.add_argument("--lbfgs_line_search", type=str, default=None, choices=[None, "strong_wolfe"], help="Line-search algorithm for LBFGS.")
# -------- Rprop -------------------------------------------------
training_group.add_argument("--rprop_eta_min", type=float, default=1e-6, help="Minimum step size for Rprop.")
training_group.add_argument("--rprop_eta_max", type=float, default=50.0, help="Maximum step size for Rprop.")
# -------- Adadelta ----------------------------------------------
training_group.add_argument("--adadelta_rho", type=float, default=0.9, help="Coefficient used for computing running averages of gradients in Adadelta.")
# -------- Adamax ------------------------------------------------
training_group.add_argument("--adamax_betas", type=float, nargs=2, default=[0.9, 0.999], help="Betas for Adamax optimizer.")
# ---------- More Modern research optimisers ----------
# AdaFactor -----------------------------------------------------------
training_group.add_argument("--adafactor_eps_row", type=float, default=1e-30, help="Row-wise ε₂ for Adafactor.")
training_group.add_argument("--adafactor_eps_col", type=float, default=1e-3, help="Column-wise ε₂ for Adafactor.")
training_group.add_argument("--adafactor_clip", type=float, default=1.0, help="Gradient clipping threshold (χ) for Adafactor.")
training_group.add_argument("--adafactor_decay", type=float, default=-0.8, help="Running-average decay rate (φ) in Adafactor.")
training_group.add_argument("--adafactor_beta1", type=float, default=-1.0, help="β₁ for momentum in Adafactor (<0 to disable momentum).")
training_group.add_argument("--adafactor_scale_param", action=argparse.BooleanOptionalAction, default=True, help="Enable parameter-scale adaptive LR.")
training_group.add_argument("--adafactor_relative_step", action=argparse.BooleanOptionalAction, default=True, help="Use relative-step schedule if learning rate is not supplied.")
training_group.add_argument("--adafactor_warmup_init", action=argparse.BooleanOptionalAction, default=False, help="Use warm-up initialisation of learning rate.")
# Sophia-G
training_group.add_argument("--sophiag_beta1", type=float, default=0.96)
training_group.add_argument("--sophiag_beta2", type=float, default=0.99)
training_group.add_argument("--sophiag_rho", type=float, default=0.04)
training_group.add_argument("--sophiag_update_freq", type=int, default=10)
# SOAP
training_group.add_argument("--soap_graft_lr", type=float, default=1.0)
# Variance Adaptive Lr
training_group.add_argument("--varlr_beta", type=float, default=0.9, help="EMA smoothing for VarianceAdaptiveLR optimizer")
# AdaBelief -----------------------------------------------------------
training_group.add_argument("--adabelief_eps", type=float, default=1e-16, help="AdaBelief epsilon.")
# Adan ---------------------------------------------------------------
training_group.add_argument("--adan_wd", type=float, default=0.0, help="Adan weight decay.")
training_group.add_argument("--adan_eps", type=float, default=1e-8, help="Adan epsilon.")
training_group.add_argument("--lookahead_inner_opt",
type=str,
default="adamw",
choices=optimizer_variations,
help="Inner/fast optimiser that Lookahead will wrap.")
training_group.add_argument("--lookahead_k",
type=int,
default=6,
help="Number of inner-optimiser steps before a slow weight sync.")
training_group.add_argument("--lookahead_alpha",
type=float,
default=0.5,
help="Interpolation factor for the slow update (0 < α ≤ 1).")
# from torch-optimizer (common)
training_group.add_argument("--opt_betas", type=float, nargs=2, default=[0.9, 0.999],
help="Betas used by Adam-family / etc.")
training_group.add_argument("--opt_eps", type=float, default=1e-8)
training_group.add_argument("--opt_weight_decay", type=float, default=0.0)
# AdaBound / AdaMod
training_group.add_argument("--adabound_final_lr", type=float, default=0.1)
training_group.add_argument("--adabound_gamma", type=float, default=1e-3)
training_group.add_argument("--adamod_beta3", type=float, default=0.999)
# Lamb
training_group.add_argument("--lamb_clamp", type=float, default=10.0)
training_group.add_argument("--lamb_adam", action=argparse.BooleanOptionalAction, default=False)
training_group.add_argument("--lamb_debias", action=argparse.BooleanOptionalAction, default=False)
# Shampoo
training_group.add_argument("--shampoo_momentum", type=float, default=0.0)
training_group.add_argument("--shampoo_eps", type=float, default=1e-4)
training_group.add_argument("--shampoo_update_freq", type=int, default=1)
# ASGD-like PID
training_group.add_argument("--pid_momentum", type=float, default=0.0)
training_group.add_argument("--pid_integral", type=float, default=5.0)
training_group.add_argument("--pid_derivative",type=float, default=10.0)
# Learning rate scheduler-specific arguments
scheduler_variations = ["none",
"cosine",
"exponential",
"step",
"plateau",
]
training_group.add_argument("--lr_scheduler", type=str, default="none",
choices= scheduler_variations,
help="Learning rate scheduler to use.")
training_group.add_argument("--cosine_t_max", type=int, default=1000, help="T_max parameter for CosineAnnealingLR.")
training_group.add_argument("--cosine_eta_min", type=float, default=0, help="Minimum learning rate for CosineAnnealingLR.")
training_group.add_argument("--exponential_gamma", type=float, default=0.9, help="Gamma value for ExponentialLR.")
training_group.add_argument("--step_lr_size", type=int, default=1000, help="Step size for StepLR.")
training_group.add_argument("--step_lr_gamma", type=float, default=0.1, help="Gamma value for StepLR.")
training_group.add_argument("--plateau_mode", type=str, default="min", choices=["min", "max"], help="Mode for ReduceLROnPlateau.")
training_group.add_argument("--plateau_factor", type=float, default=0.1, help="Factor by which learning rate is reduced for ReduceLROnPlateau.")
training_group.add_argument("--plateau_patience", type=int, default=10, help="Number of epochs with no improvement for ReduceLROnPlateau.")
# Training sample *args
training_group.add_argument('--interactive', default=False, action=argparse.BooleanOptionalAction, help="Enable interactive generation at the end of training (similar to sample.py --interactive).")
training_group.add_argument('--stop_string', type=str, default='~W', help="String to stop generation and allow user input (used when --interactive).")
training_group.add_argument('--colorize_output', default=True, action=argparse.BooleanOptionalAction, help="Colorize tokens based on predicted probabilities.")
training_group.add_argument('--colorize_mode', type=str, default='minmax',
choices=['minmax', 'softmax', 'softmax_top_k', 'rank', 'dot_product', 'topk', 'all'],
help="Colorization mode for tokens (see sample.py).")
training_group.add_argument('--colorize_topk', type=int, default=10,
help="Number of top predictions to display when colorize_mode='topk'.")
training_group.add_argument('--show_heatmaps', type=bool, default=False, action=argparse.BooleanOptionalAction, help="Show heatmaps (or bar charts) of top-k token probabilities.")
training_group.add_argument('--show_minmax_chart', type=bool, default=False, action=argparse.BooleanOptionalAction, help="show timeseries of raw logit values")
training_group.add_argument('--chart_type', type=str, default='heatmap', choices=['heatmap', 'barchart'], help="Type of chart to display if --show_heatmaps is set.")
training_group.add_argument('--last_k_tokens', type=int, default=10, help="Number of last tokens to display in heatmaps or bar charts.")
training_group.add_argument('--sample_file', type=str, default="sample.txt", help="Output file for inference samples (if you want to save them).")
training_group.add_argument('--token_boundary', type=str, default=None, help="Optional separator string between emitted tokens (for decode).")
training_group.add_argument('--num_samples', type=int, default=1, help="Number of generated samples during sampling.")
training_group.add_argument('--temperature', type=float, default=0.8, help="Temperature for predictions (1.0 = normal, < 1.0 = less random).")
training_group.add_argument('--top_k', type=int, nargs='+', default=[1, 2, 5, 10], help="Retain only the top_k most likely tokens (used in sample.py).")
training_group.add_argument('--softmax_threshold', type=float, default=None, help="Retain only the top_k most likely tokens (used in sample.py).")
training_group.add_argument(
'--cosine_penalty',
type=float,
nargs='*',
default=None,
help="Apply a penalty to logits based on cosine similarity to recent tokens. "
"Use alone for defaults (N=5, alpha=1.0). "
"Optionally provide lookback window N and penalty strength alpha. Ex: --cosine_penalty 5 1.5"
)
training_group.add_argument('--eval_dataset', type=str, default=None, help="Optional dataset name for custom evaluation splits.")
training_group.add_argument('--quantization_data_file', type=str, default=None, help="If set, export quantized weights/activations to a specified file (pkl).")
# Model args
model_group.add_argument('--block_size', default=256, type=int)
model_group.add_argument('--n_layer', default=6, type=int)
model_group.add_argument('--n_head', default=6, type=int)
model_group.add_argument('--n_kv_group', default=None, type=int)
model_group.add_argument('--n_embd', default=384, type=int, help="Size of embeddings in decoder layer and wte unless n_embd_wte is set." )
model_group.add_argument('--mlp_down_projs', default=1, type=int, help="Number of down projections in MLP/SwiGLU")
model_group.add_argument('--n_embd_wte', default=None, type=int, help="If different from n_embd, an adapter table will be automatically created")
model_group.add_argument('--n_embd_wte_scale_tying', default=True, action=argparse.BooleanOptionalAction, help="Enable weight tying for scale up and scale down matrices, only has effects if n_embd_wte is not 'None'.")
model_group.add_argument('--wte_weight_tying', default=True, action=argparse.BooleanOptionalAction, help="Enable weight tying for non-factorized wte")
model_group.add_argument(
'--wte_fixed_norm',
default=False,
action=argparse.BooleanOptionalAction,
help='Initialize every token embedding at a fixed L2 norm and reproject it after every optimizer update.',
)
model_group.add_argument(
'--wte_fixed_norm_value',
default=None,
type=float,
help='Fixed token embedding L2 norm (defaults to sqrt(n_embd_wte or n_embd)).',
)
model_group.add_argument('--dropout', default=0.0, type=float)
model_group.add_argument('--use_pre_ln', default=True, action=argparse.BooleanOptionalAction, help="apply before any attn or mlp")
model_group.add_argument('--use_peri_ln', default=False, action=argparse.BooleanOptionalAction, help="apply directly after each attn and mlp")
model_group.add_argument('--use_post_ln', default=False, action=argparse.BooleanOptionalAction, help="apply after recombining the residual")
model_group.add_argument('--use_pre_ln_attn', default=None, action=argparse.BooleanOptionalAction, help="override pre-LN for attention block")
model_group.add_argument('--use_pre_ln_mlp', default=None, action=argparse.BooleanOptionalAction, help="override pre-LN for MLP block")
model_group.add_argument('--use_peri_ln_attn', default=None, action=argparse.BooleanOptionalAction, help="override peri-LN for attention block")
model_group.add_argument('--use_peri_ln_mlp', default=None, action=argparse.BooleanOptionalAction, help="override peri-LN for MLP block")
model_group.add_argument('--use_post_ln_attn', default=None, action=argparse.BooleanOptionalAction, help="override post-LN for attention block")
model_group.add_argument('--use_post_ln_mlp', default=None, action=argparse.BooleanOptionalAction, help="override post-LN for MLP block")
model_group.add_argument('--window_size', default=None, type=int, help="Sliding window size, note this cannot be greater than block size")
model_group.add_argument('--gate', default=False, action=argparse.BooleanOptionalAction, help="option for gated attention see https://arxiv.org/abs/2306.12929")
model_group.add_argument('--use_moe', default=False, action=argparse.BooleanOptionalAction, help="option for Mixture of Experts (MoE) architecture")
model_group.add_argument('--moe_layer_freq', default=2, type=int, help="set frequency for replacing FFNs with MoE layers")
model_group.add_argument('--n_experts', default=8, type=int, help="set number of experts per MoE layer")
model_group.add_argument('--moe_top_k', default=2, type=int)
model_group.add_argument('--moe_router_scheme', default="softmax", type=str, help="option to set routing scheme for MoE layer, defaults to softmax")
model_group.add_argument('--use_flex_attn', default=None, action=argparse.BooleanOptionalAction, help="option for using flex attention for sliding windows")
model_group.add_argument('--attn_logit_softcapping', default=None, action=argparse.BooleanOptionalAction, help="option for softcapping attention (before masking)")
model_group.add_argument('--final_logit_softcapping', default=None, action=argparse.BooleanOptionalAction, help="option for softcapping final logits")
model_group.add_argument('--use_ln_f_input_mixer', default=False, action=argparse.BooleanOptionalAction, help='blend outputs of all blocks before final layer norm')
model_group.add_argument('--ln_f_input_mixer_variant', default='linear', type=str,
choices=['linear', 'router_top1', 'router_topk', 'decoder'],
help='strategy for combining block outputs before ln_f')
model_group.add_argument('--ln_f_mixer_top_k', default=2, type=int,
help='number of routes to mix when using router_topk')
## Manual Steering Vector Options
### Applying Steering Vectors
model_group.add_argument('--apply_vector_at_layer_idx', default=None, type=int)
model_group.add_argument("--apply_vector_file", type=str, default=None, help="single vector to apply with scaling factor")
model_group.add_argument("--apply_vector_scaling_factor", type=float, default=1.0, help="scaling factor to apply to steering vector")
### Options for intercepting and obtaining vectors
model_group.add_argument('--obtain_vector_at_layer_idx', default=None, type=int)
model_group.add_argument("--obtain_vector_file", type=str, default=None, help="initial KAN activation")
## Learned Steering Vector (LSV) Options
lsv_variations = [
"one_hot",
"linear_comb",
"one_hot_mlp",
"ohmg",
"ohmt",
"ohmm",
"ohma",
"ohmgu",
"ohmh",
"mol",
]
model_group.add_argument("--use_lsv", default=False, action=argparse.BooleanOptionalAction, help="whether to use Learned Steering Vectors")
model_group.add_argument("--lsv_index", default=None, type=int, help="Which steering vector to use")
model_group.add_argument("--lsv_variant", default="one_hot", type=str, choices=lsv_variations, help="Which steering vector to use")
model_group.add_argument('--apply_lsv_at_layer_idx', default=None, type=int)
training_group.add_argument("--lsv_focused_training", default=False, action=argparse.BooleanOptionalAction, help="train but only unfreeze lsv")
## MLP Options
# MLP Variations
mlp_variants = [
"mlp",
"edgellm_asic_mlp",
"kan",
"swiglu",
"dual_path",
"dual_path_swiglu",
"identity",
]
model_group.add_argument('--use_parallel_mlp', default=False, action=argparse.BooleanOptionalAction)
model_group.add_argument("--mlp_variant", type=str, default="mlp", choices=mlp_variants, help="MLP variation type")
model_group.add_argument("--mlp_expansion_factor", type=int, default=4, help="If MLP like variant is used, set the expansion factor for the linear transformations, default is 4.")
model_group.add_argument("--mlp_size", type=int, default=None, help="If not None, is used instead of mlp_expansion_factor")
model_group.add_argument('--mlp_cproj_scale', default=1.0, type=float, help="Divide MLP down projection outputs by this value")
model_group.add_argument('--mlp_post_act_l2_norm', default=False, action=argparse.BooleanOptionalAction, help="L2 normalize MLP activation vectors before down projection")
## KAN Options
model_group.add_argument("--kan_poly_order", type=int, default=3, help="Order of KAN non-linearity")
model_group.add_argument("--kan_base_activation", type=str, default="silu", help="initial KAN activation")
model_group.add_argument("--kan_middle_layers", type=int, nargs='+', help="List of integers", default=[])
# Shared Parameter Settings
model_group.add_argument('--shared_mlp_size', default=1, type=int, help="every 'k' contiguous blocks of mlp are shared")
model_group.add_argument('--shared_mlp_sym', default=False, action=argparse.BooleanOptionalAction)
model_group.add_argument('--shared_mlp_seq', default=1, type=int, help="Sequence length for cyclic sharing of MLP layers")
model_group.add_argument('--shared_attn_size', default=1, type=int, help="every 'k' contiguous blocks of attn are shared")
model_group.add_argument('--shared_attn_sym', default=False, action=argparse.BooleanOptionalAction, help="symmetrical attention sharing")
model_group.add_argument('--shared_attn_seq', default=1, type=int, help="Sequence length for cyclic sharing of attention layers")
## Learned Confidence Residual Scaling
confidence_variants = ["zeros", "ones", "gaussian"]
### Attn scaling
model_group.add_argument('--use_attn_resid_scaling', default=False, action=argparse.BooleanOptionalAction, help='Apply learned confidence scaling to attention outputs')
model_group.add_argument('--attn_confidence_variant', type=str, default='zeros', choices=confidence_variants, help='Initialization for attention residual scaling vector')
model_group.add_argument('--use_attn_resid_const', default=False, action=argparse.BooleanOptionalAction, help='Add constant term to attention residual scaling dot product')
model_group.add_argument('--attn_resid_const', type=float, default=0.0, help='Constant added to attention residual scaling dot product')
model_group.add_argument('--learn_attn_resid_const', default=False, action=argparse.BooleanOptionalAction, help='Learn the attention residual scaling constant')
### MLP scaling
model_group.add_argument('--use_mlp_resid_scaling', default=False, action=argparse.BooleanOptionalAction, help='Apply learned confidence scaling to MLP outputs')
model_group.add_argument('--mlp_confidence_variant', type=str, default='zeros', choices=confidence_variants, help='Initialization for MLP residual scaling vector')
model_group.add_argument('--use_mlp_resid_const', default=False, action=argparse.BooleanOptionalAction, help='Add constant term to MLP residual scaling dot product')
model_group.add_argument('--mlp_resid_const', type=float, default=0.0, help='Constant added to MLP residual scaling dot product')
model_group.add_argument('--learn_mlp_resid_const', default=False, action=argparse.BooleanOptionalAction, help='Learn the MLP residual scaling constant')
### Guassian style settings
model_group.add_argument('--resid_gaussian_mean_init', type=float, default=0.0, help='Gaussian residual init setting, mean value.')
model_group.add_argument('--resid_gaussian_std_init', type=float, default=0.02, help='Gaussian residual init setting, standard deviation.')
# Residual combination options
model_group.add_argument(
'--attn_residual_combination',
type=str,
default='add',
choices=['add', 'rezero', 'lerp', 'slerp'],
help='Residual combination method for attention block'
)
model_group.add_argument(
'--mlp_residual_combination',
type=str,
default='add',
choices=['add', 'rezero', 'lerp', 'slerp'],
help='Residual combination method for MLP block'
)
model_group.add_argument(
'--residual_slerp_eps',
type=float,
default=0.0,
help='Threshold below which LERP is used instead of SLERP; 0 means no fallback to LERP (SLERP is always used)'
)
model_group.add_argument(
'--attn_residual_alpha',
type=float,
default=0.05,
help='Alpha parameter for attention residual LERP/SLERP'
)
model_group.add_argument(
'--mlp_residual_alpha',
type=float,
default=0.05,
help='Alpha parameter for MLP residual LERP/SLERP'
)
model_group.add_argument(
'--attn_residual_alpha_type',
type=str,
default='fixed',
choices=['fixed', 'learned', 'rezero', 'dot'],
help='Alpha mode for attention residual combination'
)
model_group.add_argument(
'--mlp_residual_alpha_type',
type=str,
default='fixed',
choices=['fixed', 'learned', 'rezero', 'dot'],
help='Alpha mode for MLP residual combination'
)
# NORM VARIATIONS
norm_variations = [
"krmsnorm",
"prmsnorm",
"rmsnorm",
"layernorm",
"hyperspherenorm",
"dact",
"identity",
"cappedhyperspherenorm",
]
model_group.add_argument("--norm_variant_attn", type=str, default="rmsnorm", choices=norm_variations)
model_group.add_argument("--norm_variant_output", type=str, default="rmsnorm", choices=norm_variations)
model_group.add_argument("--use_flash_norm", type=bool, default=None, action=argparse.BooleanOptionalAction)
### WTE and Abs Pos Embedding Post Norms (optional, and default None)
model_group.add_argument("--norm_variant_wte", type=str, default=None, choices=norm_variations)
model_group.add_argument("--norm_variant_abs", type=str, default=None, choices=norm_variations)
model_group.add_argument("--norm_variant_lm_head", type=str, default=None, choices=norm_variations)
model_group.add_argument("--norm_wte_radius", type=float, default=None)
model_group.add_argument("--norm_wte_scale", type=float, default=None)
model_group.add_argument("--norm_wte_gain", type=bool, default=None, action=argparse.BooleanOptionalAction)
model_group.add_argument("--norm_wte_radius_learning", type=bool, default=None, action=argparse.BooleanOptionalAction)
model_group.add_argument("--norm_abs_radius", type=float, default=None)
model_group.add_argument("--norm_abs_scale", type=float, default=None)
model_group.add_argument("--norm_abs_gain", type=bool, default=None, action=argparse.BooleanOptionalAction)
model_group.add_argument("--norm_abs_radius_learning", type=bool, default=None, action=argparse.BooleanOptionalAction)
model_group.add_argument("--norm_lm_head_radius", type=float, default=None)
model_group.add_argument("--norm_lm_head_scale", type=float, default=None)
model_group.add_argument("--norm_lm_head_gain", type=bool, default=None, action=argparse.BooleanOptionalAction)
model_group.add_argument("--norm_lm_head_radius_learning", type=bool, default=None, action=argparse.BooleanOptionalAction)
## Layernorm
model_group.add_argument('--bias', default=False, action=argparse.BooleanOptionalAction, help="only used for layernorm variation option")
## PRMSNorm
model_group.add_argument("--prmsnorm_pct", default=0.0625, type=float, help="percentage (1 being 100 percent) of first entries used for partial rms" )
## KRMSNorm
model_group.add_argument("--krmsnorm_num", default=10, type=int, help="max number of first entries for partial rms" )
model_group.add_argument("--krmsnorm_quantize_type", type=str, default="none", choices=["int8", "int16", "none"])
model_group.add_argument('--krmsnorm_enable_gain', default=True, action=argparse.BooleanOptionalAction, help="include gain in kRMSNorm")
model_group.add_argument("--krmsnorm_selection_type", type=str, default="last", choices=["first", "last", "random"])
model_group.add_argument("--krmsnorm_recompute_percentage", type=float, default=None, help="percentage needed within the total RMS to not trigger recompute")
## HyperSphereNorm
model_group.add_argument("--hsnorm_gain", default=False, action=argparse.BooleanOptionalAction)
model_group.add_argument("--hsnorm_scale", type=float, default=1.0)
model_group.add_argument("--hsnorm_radius", type=float, default=None)
model_group.add_argument("--hsnorm_radius_learning", default=False, action=argparse.BooleanOptionalAction)
activation_variations = [
"celu",
"elu",
"gelu",
"gelu_shifted",
"glu",
"leaky_relu",
"learned_spline",
"mish",
"piecewise",
"pfla",
"pfla_le",
"prelu",
"relu",
"relu_power",
"relu6",
"rrelu",
"disco",
"selu",
"sigmoid",
"silu",
"softplus",
"softsign",
"softshrink",
"squared_relu",
"squared_gelu",
"tanh",
"identity",
]
## DynamicActivations
model_group.add_argument("--dact_activation", type=str, default="tanh", choices=activation_variations)
model_group.add_argument("--dact_use_gamma", type=bool, default=True, action=argparse.BooleanOptionalAction)
model_group.add_argument("--dact_use_beta", type=bool, default=True, action=argparse.BooleanOptionalAction)
model_group.add_argument("--dact_alpha_init", default=1.0, type=float)
model_group.add_argument("--dact_use_alpha", type=bool, default=True, action=argparse.BooleanOptionalAction)
model_group.add_argument("--use_embedding_scale", type=bool, default=False, action=argparse.BooleanOptionalAction)
model_group.add_argument("--embedding_scale_init", type=float, default=None)
# ACTIVATION VARIATIONS
model_group.add_argument( "--activation_variant", type=str, default="gelu", choices=activation_variations)
## ReLUPower
model_group.add_argument("--relu_power", type=float, default=2.0)
## Shifted Gelu
model_group.add_argument("--shifted_gelu_learnable_shift", type=bool, default=True, action=argparse.BooleanOptionalAction)
model_group.add_argument("--shifted_gelu_initial_shift", type=float, default=0.0)
## Softshrink
model_group.add_argument("--softshrink_lambda", type=float, default=0.5)
## PiecewiseLearnableActivation - pla
model_group.add_argument("--pla_num_points", type=int, default=7)
model_group.add_argument("--pla_left_bound", type=float, default=-2.0)
model_group.add_argument("--pla_right_bound", type=float, default=2.0)
## PiecewiseFullyLearnableActivation - pfla
model_group.add_argument("--pfla_num_points", type=int, default=50)
model_group.add_argument("--pfla_left_bound", type=float, default=-10.0)
model_group.add_argument("--pfla_right_bound", type=float, default=10.0)
## LearnedSplineActivation - lsa
model_group.add_argument("--lsa_num_knots", type=int, default=30)
# Attention Variations
attention_variants = [
"causal",
"edgellm_asic_attn",
"linear",
"ssm",
"identity",
"infinite",
"mla",
"co4",
]
model_group.add_argument(
"--attention_list",
nargs='+',
type=str,
default=None,
help="List of attention variants to cycle through, e.g. 'causal linear ssm'."
)
model_group.add_argument(
"--attention_variant",
type=str,
default="causal",
choices=attention_variants,
help="Which attention variant to use for the Transformer blocks."
)
## MLA Variations
# ── inside model_group = parser.add_argument_group('model_group') … ──
model_group.add_argument(
'--mla_latent_dim',
type=int,
default=None,
help="d_c: projected latent size for MLA (defaults to n_embd//4)."
)
model_group.add_argument(
'--mla_rotary_dim',
type=int,
default=32,
help="d_r: rotary-branch dimensionality used by MLA."
)