VeOmni/configs/dit/wan_sft.yaml at main · ByteDance-Seed/VeOmni · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
model:
  model_path: Wan-AI/Wan2.1-I2V-14B-480P
  config_path: ./configs/model_configs/wan/wani2v_14b.json
  ops_implementation:
    attn_implementation: flash_attention_2

data:
  train_path: Wanmini480
  train_size: 1000000000000
  dataloader:
    type: native
    drop_last: true
  datasets_type: iterable
  data_type: diffusion
  max_seq_len: 8192
  text_keys: text
  dyn_bsz_buffer_size: 200

train:
  accelerator:
    dp_replicate_size: 1
    ulysses_size: 4
    fsdp_config:
      fsdp_mode: fsdp1
      full_shard: true
      offload: false
    offload:
      enable_activation: false
  gradient_checkpointing:
    enable: true
  global_batch_size: 8
  micro_batch_size: 1
  bsz_warmup_ratio: 0.007
  optimizer:
    type: adamw
    lr: 1.0e-4
    lr_warmup_ratio: 0.007
    lr_decay_style: constant
    lr_decay_ratio: 1.0
    weight_decay: 0.01
    max_grad_norm: 1.0
  vit_lr: 5.0e-5
  enable_mixed_precision: false
  init_device: cuda
  enable_full_determinism: false
  empty_cache_steps: 500
  checkpoint:
    output_dir: wan2.1-i2v-14b-480p
    manager: dcp
    save_epochs: 20
    save_hf_weights: true
  max_steps: 500
  num_train_epochs: 100
  wandb:
    enable: false
    project: Wan2.1-I2V-14B-480P
    name: sft_wan_i2v_14b
  ops_to_save:
    # - aten.addmm.default
    - torch.ops.flash_attn._flash_attn_forward.default
    - aten._scaled_dot_product_flash_attention.default
    - torch.ops.flash_attn_3.fwd.default