[model] support hy3 model

Author: mme-benchmark

examples/train_full/hy3_full_sft.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Hy3 Full SFT training script
+# Recommended: 4 nodes with 32x GPUs (at least 80GB memory each)
+# For single-node multi-GPU: use torchrun or deepspeed launcher
+
+set -x
+
+MODEL_PATH=tencent/Hy3
+
+# Use deepspeed zero-3 offload for full fine-tuning
+llamafactory-cli train \
+    --model_name_or_path ${MODEL_PATH} \
+    --trust_remote_code \
+    --stage sft \
+    --do_train \
+    --finetuning_type full \
+    --deepspeed examples/deepspeed/ds_z3_offload_config.json \
+    --dataset identity,alpaca_en_demo \
+    --template hy3 \
+    --cutoff_len 262144 \
+    --max_samples 1000 \
+    --preprocessing_num_workers 16 \
+    --dataloader_num_workers 4 \
+    --output_dir saves/hy3/full/sft \
+    --logging_steps 10 \
+    --save_steps 500 \
+    --plot_loss \
+    --overwrite_output_dir \
+    --save_only_model false \
+    --report_to none \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 2 \
+    --learning_rate 1e-5 \
+    --num_train_epochs 3.0 \
+    --lr_scheduler_type cosine \
+    --warmup_ratio 0.1 \
+    --bf16 \
+    --ddp_timeout 180000000 \
+    --flash_attn fa2

examples/train_full/hy3_full_sft.yaml

@@ -0,0 +1,45 @@
+### model
+model_name_or_path: tencent/Hy3
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+deepspeed: examples/deepspeed/ds_z3_offload_config.json
+
+### dataset
+dataset: identity,alpaca_en_demo
+template: hy3
+cutoff_len: 262144
+max_samples: 1000
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/hy3/full/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 2
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+flash_attn: fa2
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/train_lora/hy3_lora_sft.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Hy3 LoRA SFT training script
+# Recommended: 8x GPUs with at least 80GB memory each
+# For single-node multi-GPU: use torchrun or deepspeed launcher
+
+set -x
+
+MODEL_PATH=tencent/Hy3
+
+# Use deepspeed for multi-GPU training
+llamafactory-cli train \
+    --model_name_or_path ${MODEL_PATH} \
+    --trust_remote_code \
+    --stage sft \
+    --do_train \
+    --finetuning_type lora \
+    --lora_rank 64 \
+    --lora_alpha 128 \
+    --lora_dropout 0.05 \
+    --lora_target q_proj,k_proj,v_proj,o_proj \
+    --deepspeed examples/deepspeed/ds_z2_offload_config.json \
+    --dataset identity,alpaca_en_demo \
+    --template hy3 \
+    --cutoff_len 8192 \
+    --max_samples 1000 \
+    --preprocessing_num_workers 16 \
+    --dataloader_num_workers 4 \
+    --output_dir saves/hy3/lora/sft \
+    --logging_steps 10 \
+    --save_steps 500 \
+    --plot_loss \
+    --overwrite_output_dir \
+    --save_only_model false \
+    --report_to none \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 8 \
+    --learning_rate 2e-4 \
+    --num_train_epochs 3.0 \
+    --lr_scheduler_type cosine \
+    --warmup_ratio 0.1 \
+    --bf16 \
+    --ddp_timeout 180000000 \
+    --flash_attn fa2

examples/train_lora/hy3_lora_sft.yaml

@@ -0,0 +1,49 @@
+### model
+model_name_or_path: tencent/Hy3
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: q_proj,k_proj,v_proj,o_proj
+deepspeed: examples/deepspeed/ds_z2_offload_config.json
+
+### dataset
+dataset: identity,alpaca_en_demo
+template: hy3
+cutoff_len: 8192
+max_samples: 1000
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/hy3/lora/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 2.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+flash_attn: fa2
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

src/llamafactory/data/template.py

@@ -843,7 +843,7 @@ def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args:
     format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
     stop_words=["<｜hy_eos｜>"],
     replace_eos=True,
-    thought_words=("<think>", "</think>"),
+    thought_words=("<think:opensource>", "</think:opensource>"),
     template_class=ReasoningTemplate,
 )
 

src/llamafactory/extras/constants.py

@@ -1287,6 +1287,10 @@ def register_model_group(
             DownloadSource.DEFAULT: "tencent/Hy3-preview",
             DownloadSource.MODELSCOPE: "tencent/Hy3-preview",
         },
+        "Hy3": {
+            DownloadSource.DEFAULT: "tencent/Hy3",
+            DownloadSource.MODELSCOPE: "tencent/Hy3",
+        },
     },
     template="hy3",
 )