fairseq2/recipes/lm/sft/configs/qwen35_0.8b_gsm8k.yaml at dd4280c6e48fa1b55cbb498c75e0f3b23a9a25c5 · facebookresearch/fairseq2 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

# Qwen 3.5 0.8B GSM8K SFT Fine-tuning Config
#
# Validates training recipe integration and loss convergence for the Qwen 3.5
# model on the GSM8K math reasoning dataset.
#
# Usage:
#   torchrun --standalone --nproc_per_node=8 -m recipes.lm.sft \
#     --config-file recipes/lm/sft/configs/qwen35_0.8b_gsm8k.yaml \
#     /path/to/output_dir

model:
  name: "qwen35_0.8b"
  dtype: bfloat16
  config_overrides:
    pad_idx: 248044

tokenizer:
  name: "qwen35_0.8b"
  config_overrides:
    use_im_end: true

dataset:
  max_seq_len: 4096
  max_num_tokens: 8192
  valid_split: "sft_test"
  chat_mode: false
  config_overrides:
    sources:
      train:
      - path: "hg://facebook/fairseq2-lm-gsm8k"
        split: "sft_train"
        weight: 1.0
      sft_test:
      - path: "hg://facebook/fairseq2-lm-gsm8k"
        split: "sft_test"
        weight: 1.0

trainer:
  data_parallelism: fsdp
  max_grad_norm: 1.0
  mixed_precision:
    mode: static
    dtype: bfloat16

optimizer:
  name: adamw
  config:
    lr: 2.0e-5
    betas: [0.9, 0.95]
    weight_decay: 0.1
    impl: fused

lr_scheduler:
  name: cosine_annealing
  config:
    final_lr_scale: 0.1
    num_warmup_steps: 100

regime:
  num_steps: 100000
  checkpoint_every_n_steps: 100
  validate_every_n_steps: 100
  keep_last_n_checkpoints: 10
  publish_metrics_every_n_steps: 1
  save_model_only: false

common:
  seed: 0
  metric_recorders:
    wandb:
      enabled: true
      entity: "yunchaoyang1"
      project: "fairseq2"
    tensorboard:
      enabled: false