RL/examples/configs/recipes/llm/performance/dapo-deepseek-v3-64n8g.yaml at 2520299036bbfa4e6635bebb3e6cb599330d9692 · NVIDIA-NeMo/RL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
defaults: ../../../grpo_math_1B.yaml
grpo:
  num_prompts_per_step: 256
  num_generations_per_prompt: 16
  batch_multiplier: 3 # Multiplier for dataloader batch size calculation
  max_rollout_turns: 1
  max_num_steps: 10
  use_leave_one_out_baseline: false
  val_period: 5
  max_val_samples: 256
  val_batch_size: 256
  use_dynamic_sampling: true
  dynamic_sampling_max_gen_batches: 10
  reward_scaling:
    enabled: true
    source_min: 0.0
    source_max: 1.0
    target_min: -1.0
    target_max: 1.0
  reward_shaping:
    enabled: true
    overlong_buffer_length: 512
    max_response_length: 1024
loss_fn:
  reference_policy_kl_penalty: 0.0  # Corresponds to actor_rollout_ref.actor.kl_loss_coef
  ratio_clip_max: 0.28  # clip_ratio_high
  ratio_clip_min: 0.2   # clip_ratio_low
  ratio_clip_c: 10.0
checkpointing:
  checkpoint_dir: results/grpo_dapomath17k_dsv3_megatron
  keep_top_k: 100
policy:
  model_name: "Follow instructions in https://github.com/NVIDIA-NeMo/RL/blob/main/docs/guides/deepseek.md"
  hf_config_overrides:
    max_position_embeddings: 1536
  train_micro_batch_size: 1
  logprob_batch_size: 1
  max_total_sequence_length: 1536
  dtensor_cfg:
    enabled: false
  make_sequence_length_divisible_by: ${mul:${policy.megatron_cfg.tensor_model_parallel_size},
    ${mul:2, ${policy.megatron_cfg.context_parallel_size}}}
  megatron_cfg:
    empty_unused_memory_level: 2
    enabled: true
    activation_checkpointing: true
    moe_grouped_gemm: true
    tensor_model_parallel_size: 8
    expert_model_parallel_size: 32
    pipeline_model_parallel_size: 8
    num_layers_in_first_pipeline_stage: 7
    num_layers_in_last_pipeline_stage: 6
    context_parallel_size: 4
    sequence_parallel: true
    apply_rope_fusion: false
    # MTP — disabled
    mtp_num_layers: 0
    optimizer:
      lr: 5.0e-07
      min_lr: 5.0e-08
    scheduler:
      lr_warmup_init: 5.0e-08
  sequence_packing:
    enabled: true
  generation:
    max_new_tokens: 1536
    vllm_cfg:
      async_engine: false
      tensor_parallel_size: 64
      enforce_eager: true
data:
  max_input_seq_length: 512  # max_prompt_length
  prompt_file: null
  dataset_name: DAPOMath17K
env:
  dapo:
    num_workers: 64
  math:
    num_workers: 64
    math_verify_impl: "dapo_math_verify"

logger:
  monitor_gpus: true
  wandb:
    project: DAPO
    name: DAPO-DeepSeek-671b-megatron
  mlflow:
    experiment_name: DAPO
    run_name: DAPO-DeepSeek-671b-megatron
cluster:
  gpus_per_node: 8
  num_nodes: 64