fairseq2/recipes/lm/train/configs/llama3_2_1b_fineweb_edu_10bt.yaml at dd4280c6e48fa1b55cbb498c75e0f3b23a9a25c5 · facebookresearch/fairseq2 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
# LLaMA 3.2 1B Continued Pretraining on FineWeb-Edu 10BT
#
# Loads the pretrained LLaMA 3.2 1B checkpoint and continues training
# on the FineWeb-Edu Sample 10BT educational text dataset.
#
# Prerequisites:
#   1. Convert parquet data to chunked JSONL:
#      cd /checkpoint/smallomnillm/shared/data/fineweb-edu
#      python convert_to_jsonl.py --sample 10BT --chunk-format --lightweight \
#          --num-shards 256 --output-dir jsonl/10BT
#
# Usage:
#   torchrun --standalone --nproc_per_node=8 -m recipes.lm.train \
#     --config-file recipes/lm/train/configs/llama3_2_1b_fineweb_edu_10bt.yaml \
#     /path/to/output_dir

model:
  name: llama3_2_1b
  dtype: bfloat16
  mmap: false
  compile: false
  compile_options:
    fullgraph: false
    dynamic: false
    mode: default
    backend: inductor
    backend_options: null

dataset:
  name: null
  family: lm_train
  config_overrides:
    sources:
      - path: /checkpoint/smallomnillm/shared/data/fineweb-edu/jsonl/10BT
        weight: 1.0
  max_seq_len: 4096
  max_num_tokens: 8192
  prefetch: 4
  sync_ranks: false

tokenizer:
  name: llama3
  path: null
  family: null
  config_overrides: null

gang:
  tensor_parallel_size: 1
  timeout: 15
  high_priority: true

trainer:
  data_parallelism: fsdp
  fsdp:
    version: v2
    granularity: layer
    hybrid: false
    reshard_after_forward: true
    fp32_reduce: true
  mixed_precision:
    mode: static
    dtype: bfloat16
  grad_accumulation:
    num_batches: 1
    no_sync: false
  activation_checkpointing:
    mode: layerwise
    every_nth_layer: 1
  max_grad_norm: 1.0
  fp16_loss_scale:
  - 128.0
  - 0.0001
  gc_every_n_steps: 1000
  grad_check: false
  anomaly_detection: false

optimizer:
  name: adamw
  config:
    lr: 5.0e-5
    betas:
    - 0.9
    - 0.95
    eps: 1.0e-8
    weight_decay: 0.1
    amsgrad: false
    maximize: false
    capturable: false
    differentiable: false
    impl: fused
    groups: []

lr_scheduler:
  name: cosine_annealing
  config:
    cycle_len: null
    num_warmup_steps: 500
    cycle_mul: 1.0
    lr_mul: 1.0
    start_lr: 1.0e-30
    final_lr: null
    final_lr_scale: 0.01

regime:
  num_steps: 76000
  num_data_epochs: null
  validate_at_start: false
  validate_after_n_steps: 0
  validate_every_n_steps: 4000
  validate_after_n_data_epochs: 0
  validate_every_n_data_epochs: null
  score_metric: null
  checkpoint_after_n_steps: 0
  checkpoint_every_n_steps: 2000
  checkpoint_after_n_data_epochs: 0
  checkpoint_every_n_data_epochs: null
  save_model_only: all_but_last
  export_hugging_face: true
  keep_last_n_checkpoints: 3
  keep_best_n_checkpoints: null
  keep_checkpoint_every_n_steps: 10000
  publish_metrics_after_n_steps: 0
  publish_metrics_every_n_steps: 10
  publish_metrics_after_n_data_epochs: 0
  publish_metrics_every_n_data_epochs: null

common:
  torch:
    num_threads: null
    allow_tf32: true
    fp16_reduced_precision: true
    bf16_reduced_precision: true
    default_sdpa: torch
    compiled_region_activation_memory_budget: 0.9
  metric_recorders:
    tensorboard:
      enabled: true
    wandb:
      enabled: true
      entity: yunchaoyang1
      project: llama3_2_1b_fineweb_edu_10bt
      run_id: persistent
      run_name: null
      group: null
      job_type: null
      resume_mode: null
  profilers:
    torch:
      enabled: false
      skip_n_steps: 4
      wait_n_steps: 0
      num_warmup_steps: 1
      num_active_steps: 4
      repeat: 1
  assets:
    extra_paths: []
    prev_checkpoint_dir: null
  seed: 2
  debug: false
  cluster: auto
  no_sweep_dir: false
  sweep_format: ws_{world_size}.{hash}