add functional tests

Author: heavyrain-lzy

tests/functional_tests/train/qwen3_vl/conf/8b.yaml

@@ -0,0 +1,33 @@
+defaults:
+  - _self_
+  - train: 8b
+
+experiment:
+  exp_name: 8b
+  exp_dir: tests/functional_tests/train/qwen3_vl/test_results/8b
+  task:
+    type: train
+    backend: megatron
+    entrypoint: flagscale/train/megatron/train_qwen3_vl.py
+  runner:
+    ssh_port: null
+  shell_cmds: null
+  envs:
+    HYDRA_FULL_ERROR: 1
+    CUDA_VISIBLE_DEVICES: "0,1,2,3"
+    CUDA_DEVICE_MAX_CONNECTIONS: 1
+    CUBLAS_WORKSPACE_CONFIG: ":4096:8"
+    NCCL_ALGO: "Ring"
+    NVTE_APPLY_QK_LAYER_SCALING: 0
+    NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+    NVTE_FLASH_ATTN: 1
+    NVTE_FUSED_ATTN: 0
+    CUDNN_BENCHMARK: "false"
+    CUDNN_DETERMINISTIC: "true"
+  cmds:
+    before_start: source /root/miniconda3/bin/activate flagscale-train && pip install git+https://github.com/NVIDIA/Megatron-Energon.git@ab40226100830f41de38d1f1204d7848b54b1f3e && pip install transformers==4.57
+action: run
+
+hydra:
+  run:
+    dir: ${experiment.exp_dir}/hydra

tests/functional_tests/train/qwen3_vl/conf/train/8b.yaml

@@ -0,0 +1,105 @@
+system:
+  vision_ration: 0.1
+  num_workers: 1
+  calculate_per_token_loss: true
+  tensor_model_parallel_size: 1
+  pipeline_model_parallel_size: 2
+  context_parallel_size: 1
+  use_flash_attn: True
+  use_distributed_optimizer: True
+  sequence_parallel: True
+  tp_comm_overlap: False
+  overlap_grad_reduce: False # if has text-only must be false
+  overlap_param_gather: False # if has text-only must be false
+  use_mcore_models: True
+  transformer_impl: transformer_engine
+  use_te: True
+  precision:
+    bf16: True
+    attention_softmax_in_fp32: True
+  logging:
+    log_interval: 1
+    tensorboard_log_interval: 1
+    log_throughput: True
+    wandb_project: ${experiment.exp_name}
+    wandb_exp_name: ${experiment.exp_name}
+    log_params_norm: True
+    log_num_zeros_in_grad: True
+  checkpoint:
+    save_interval: 1000
+    # pretrained_checkpoint: /nfs/lizhiyu/embody/data/Qwen3-VL-8B-Instruct-tp2/
+    dataloader_save: ${experiment.exp_dir}/checkpoints/dataloader
+    ckpt_format: torch
+    async_save: False
+
+model:
+  kv_channels: 128 # the weight out_size of prepare qkv
+  qk_layernorm: True
+  attention_backend: flash # don't use "auto(nvte_flash_attn)"
+  disable_bias_linear: True
+  num_layers: 36
+  hidden_size: 4096
+  ffn_hidden_size: 12288
+  num_attention_heads: 32
+  num_query_groups: 8
+  seq_length: 2048
+  max_padding_length: 2048 # (cutoff_len)max 262144, change according the dataset
+  # especial for qwen3-vl
+  enable_variable_seq_lengths: True
+  max_position_embeddings: 262144 # only useful for additional position embedding
+  swiglu: True
+  normalization: RMSNorm
+  norm_epsilon: 1e-6
+  init_method_std: 0.02
+  attention_dropout: 0.0
+  hidden_dropout: 0.0
+  clip_grad: 1.0
+  train_iters: 10
+  eval_iters: 0 # no valid
+  eval_interval: 1000
+  micro_batch_size: 1
+  global_batch_size: 4
+  allow_missing_vision_projection_checkpoint: False
+  apply_layernorm_1p: False
+  group_query_attention: True
+  no_masked_softmax_fusion: True
+  untie_embeddings_and_output_weights: True
+
+  # position embedding
+  position_embedding_type: mrope
+  rotary_percent: 1.0
+  rotary_base: 5000000
+  rotary_seq_len_interpolation_factor: 1
+  no_rope_fusion: False
+  mrope_section: [24, 20, 20]
+  eod_mask_loss: False
+
+  # vision model
+  patch_size: 16
+  freeze_LM: False
+  freeze_ViT: False
+  disable_vision_class_token: True
+  seed: 42
+
+  optimizer:
+    weight_decay: 0.1
+    adam_beta1: 0.9
+    adam_beta2: 0.999
+    lr_scheduler:
+      lr: 1.0e-5
+      min_lr: 1.0e-6
+      lr_warmup_fraction: .03
+      lr_decay_style: cosine
+
+data:
+  no_use_system_prompt: True
+  data_path: /nfs/lizhiyu/embody/data/blip_laion_cc_sbu_558k_first_5k/wds-2/
+  vision_root: /nfs/lizhiyu/embody/data/blip_laion_cc_sbu_558k_first_5k/
+  dataloader_type: external
+  split: 100,0,0
+  tokenizer:
+    tokenizer_type: Qwen2VLTokenizer
+    tokenizer_path: /nfs/lizhiyu/embody/data/Qwen3-VL-8B-Instruct-tp2/
+    # vocab_size: 151936 #
+    extra_vocab_size: 293 # Qwen3-VL specific. total vocab size = 151643 + extra_vocab_size
+    make_vocab_size_divisible_by: 64

tests/functional_tests/train/qwen3_vl/gold_values/8b.json

@@ -0,0 +1 @@
+{"lm loss:": {"values": [12.70535, 12.76372, 11.01310, 12.65629, 13.28216, 13.88037, 11.55830, 11.04979, 11.11519, 9.531566], "rtol": 0.1, "atol": 0.2}}