fairseq2/recipes/lm/sft/configs/gemma4_12b_gsm8k.yaml at d0ecda2022c8c0ccac78871c90544ee9d1c69b40 · facebookresearch/fairseq2 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

# Gemma4 12B (Unified family, dense) GSM8K SFT Fine-tuning Config
#
# 12B: 12B params total, 48 layers, model_dim=3840, head_dim=256,
# 16 attn heads, 8 KV heads, num_global_key_value_heads=1 (MQA global),
# attention_k_eq_v=True, ffn_inner=15360, sliding_window=1024,
# max_seq_len=262144, no PLE, no MoE.
#
# Recommended hardware: 8x H100/H200 (FSDP). Reduce max_seq_len/max_num_tokens
# on smaller setups.
#
# Usage:
#   torchrun --standalone --nproc_per_node=8 -m recipes.lm.sft \
#     --config-file recipes/lm/sft/configs/gemma4_12b_gsm8k.yaml \
#     /path/to/output_dir

model:
  family: "gemma4"
  # Fine-tune the -it variant. SFT on the instruction-tuned checkpoint is the
  # standard pattern for domain adaptation. The base gemma4_12b card is also
  # registered (points at /checkpoint/fairseq2/shared/models/gemma-4-12B) if
  # the user wants to SFT from base — just swap the name below.
  name: "gemma4_12b_it"
  dtype: bfloat16

tokenizer:
  family: "gemma4"
  name: "gemma4_12b_it"

dataset:
  max_seq_len: 4096
  max_num_tokens: 8192
  valid_split: "sft_test"
  # NOTE: chat_mode=true requires the chat_template.jinja to use
  # {% generation %} markers so apply_chat_template can return a non-empty
  # assistant_masks. Google's Gemma 4 chat_template.jinja does NOT use
  # {% generation %} (verified 2026-06-07), so target_mask is all-false and
  # loss collapses to 0 with zero gradient. Until the chat template is
  # patched upstream (or replaced with a fairseq2-side variant), use
  # chat_mode=false: the SFT runs as LM continuation on the src+tgt pair,
  # which still validates the recipe wiring end-to-end and produces a
  # nonzero loss signal.
  chat_mode: false
  config_overrides:
    sources:
      train:
      - path: "hg://facebook/fairseq2-lm-gsm8k"
        split: "sft_train"
        weight: 1.0
      sft_test:
      - path: "hg://facebook/fairseq2-lm-gsm8k"
        split: "sft_test"
        weight: 1.0

trainer:
  data_parallelism: fsdp
  max_grad_norm: 1.0
  mixed_precision:
    mode: static
    dtype: bfloat16

optimizer:
  name: adamw
  config:
    lr: 2.0e-5
    betas: [0.9, 0.95]
    weight_decay: 0.1
    impl: fused

lr_scheduler:
  name: cosine_annealing
  config:
    final_lr_scale: 0.1
    num_warmup_steps: 100

regime:
  num_steps: 100000
  checkpoint_every_n_steps: 100
  validate_every_n_steps: 100
  keep_last_n_checkpoints: 10
  publish_metrics_every_n_steps: 1
  save_model_only: false

common:
  seed: 0
  metric_recorders:
    wandb:
      enabled: true
      project: "gemma-4-fairseq2"
      run_name: "sft_gemma4_12b_gsm8k"
    tensorboard:
      enabled: true