llama3_debug_fsdp.toml

# torchtitan Config.toml
# NOTE: this toml config is a preset for 2x B200 GPUs.

[job]
dump_folder = "./outputs"
description = "Llama 3 Debug FSDP training"

[profiling]
enable_profiling = false
save_traces_folder = "profile_trace"
profile_freq = 100

[metrics]
log_freq = 10
enable_tensorboard = true
save_tb_folder = "tb"

[model]
name = "llama3"
flavor = "debugmodel"
tokenizer_path = "tokenizer.model"
# converters = ["float8"]

[optimizer]
name = "AdamW"
lr = 3e-4
eps = 1e-8

[lr_scheduler]
warmup_steps = 200  # lr scheduler warm up

[training]
local_batch_size = 2
seq_len = 2048
max_norm = 1.0  # grad norm clipping
steps = 1000
dataset = "c4_test"
compile = true

[parallelism]
data_parallel_replicate_degree = 1
data_parallel_shard_degree = -1
tensor_parallel_degree = 1
pipeline_parallel_degree = 1
context_parallel_degree = 1

[checkpoint]
enable_checkpoint = false
folder = "checkpoint"
interval = 500
last_save_model_only = true
export_dtype = "bfloat16"
async_mode = "disabled" # ["disabled", "async", "async_with_pinned_mem"]

[activation_checkpoint]
mode = "selective"  # ["none", "selective", "full"]
selective_ac_option = "op"  # "int" = ac every positive int layer or 'op', ac based on ops policy

[float8]
enable_fsdp_float8_all_gather = false
precompute_float8_dynamic_scale_for_fsdp = false
filter_fqns = ["output"]


[experimental]
enable_torchft = true
ft_group_size = 2