FlagScale/examples/llava1_5/conf/train/7b.yaml at main · lxd-cumt/FlagScale · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
system:
  tensor_model_parallel_size: 1
  pipeline_model_parallel_size: 1
  disable_bias_linear: True
  use_flash_attn: True
  use_distributed_optimizer: True
  use_mcore_models: True
  transformer_impl: transformer_engine
  precision:
    bf16: True
    attention_softmax_in_fp32: True
  logging:
    log_interval: 1
    tensorboard_log_interval: 1
    wandb_project: "train-llava-1.5"
    wandb_exp_name: "train-llava-1.5"
    log_params_norm: True
    log_num_zeros_in_grad: True
  checkpoint:
    save_interval: 1000
    pretrained_checkpoint: ${pretrained_checkpoint_path:??}
    dataloader_save: ${experiment.exp_dir}/checkpoints/dataloader
    use_dist_ckpt: False
    ckpt_format: torch
    async_save: False

model:
  num_layers: 32
  hidden_size: 4096
  ffn_hidden_size: 11008
  num_attention_heads: 32
  seq_length: 2048
  max_position_embeddings: 4096
  swiglu: True
  normalization: RMSNorm
  init_method_std: 0.014
  attention_dropout: 0.0
  hidden_dropout: 0.0
  clip_grad: 1.0
  train_iters: 5000
  eval_iters: 10
  eval_interval: 1000
  micro_batch_size: 2
  global_batch_size: 256
  allow_missing_vision_projection_checkpoint: True
  apply_layernorm_1p: True
  use_te: True
  group_query_attention: True
  num_query_groups: 32
  no_masked_softmax_fusion: True
  untie-embeddings-and-output-weights: True
  position_embedding_type: rope
  rotary_percent: 1.0
  rotary_base: 10000
  eod_mask_loss: True
  freeze_LM: True
  freeze_ViT: True
  patch_dim: 14
  img_h: 336
  img_w: 336
  language_model_type: vicuna_7b
  disable_vision_class_token: True
  seed: 1234

  optimizer:
    weight_decay: 0.0
    adam_beta1: 0.9
    adam_beta2: 0.95
    lr_scheduler:
      lr: 1.0e-3
      lr_warmup_fraction: .03
      lr_decay_style: cosine

data:
  data_path: ./examples/multimodal/pretrain_dataset.yaml
  valid_path: ./examples/multimodal/pretrain_dataset.yaml
  prompt-path: ./examples/multimodal/manual_prompts.json
  dataloader_type: external
  split: 100,0,0
  tokenizer:
    tokenizer_type: Llama2Tokenizer
    tokenizer_model: ${tokenizer_model_path:??}
    vocab_size: 32000