testnet-training-node-quickstart/training_args.yaml at main · mikethoue0/testnet-training-node-quickstart · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#Qwen/Qwen1.5-4B-Chat:
 # per_device_train_batch_size: 1
  #gradient_accumulation_steps: 16
  #num_train_epochs: 8
  #lora_rank: 8
  #lora_alpha: 32
  #lora_dropout: 0.5


# google/gemma-2b:
  # per_device_train_batch_size: 1
  # gradient_accumulation_steps: 8
  # num_train_epochs: 10
  # lora_rank: 4
  # lora_alpha: 8
  # lora_dropout: 0.1


microsoft/Phi-3.5-mini-instruct:
  per_device_train_batch_size: 1
  gradient_accumulation_steps: 1
  num_train_epochs: 1
  lora_rank: 1
  lora_alpha: 1
  lora_dropout: 0.005

microsoft/Phi-3-small-8k-instruct:
  per_device_train_batch_size: 1
  gradient_accumulation_steps: 16
  num_train_epochs: 8
  lora_rank: 8
  lora_alpha: 32
  lora_dropout: 0.5

microsoft/Phi-3-medium-4k-instruct:
  per_device_train_batch_size: 1
  gradient_accumulation_steps: 16
  num_train_epochs: 8
  lora_rank: 8
  lora_alpha: 32
  lora_dropout: 0.5


#Qwen/Qwen2.5-3B-Instruct:
 # per_device_train_batch_size: 1
  #gradient_accumulation_steps: 8
  #num_train_epochs: 10
  #lora_rank: 4
  #lora_alpha: 8
  #lora_dropout: 0.1