local path

Author: Begunner

tests/utils/dataset/test_rl_dataset_on_cpu.py

@@ -30,7 +30,7 @@ def test_rl_dataset():
     from verl.utils import hf_tokenizer
     from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn
 
-    tokenizer = hf_tokenizer("deepseek-ai/deepseek-coder-1.3b-instruct")
+    tokenizer = hf_tokenizer(os.path.expanduser("~/models/deepseek-ai/deepseek-coder-1.3b-instruct"))
     local_path = get_gsm8k_data()
     config = OmegaConf.create(
         {
@@ -70,7 +70,7 @@ def test_rl_dataset_with_max_samples():
     from verl.utils import hf_tokenizer
     from verl.utils.dataset.rl_dataset import RLHFDataset
 
-    tokenizer = hf_tokenizer("deepseek-ai/deepseek-coder-1.3b-instruct")
+    tokenizer = hf_tokenizer(os.path.expanduser("~/models/deepseek-ai/deepseek-coder-1.3b-instruct"))
     local_path = get_gsm8k_data()
     config = OmegaConf.create(
         {
@@ -89,8 +89,8 @@ def test_image_rl_data():
     from verl.utils import hf_processor, hf_tokenizer
     from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn
 
-    tokenizer = hf_tokenizer("Qwen/Qwen2-VL-2B-Instruct")
-    processor = hf_processor("Qwen/Qwen2-VL-2B-Instruct")
+    tokenizer = hf_tokenizer(os.path.expanduser("~/models/Qwen/Qwen2-VL-2B-Instruct"))
+    processor = hf_processor(os.path.expanduser("~/models/Qwen/Qwen2-VL-2B-Instruct"))
     config = OmegaConf.create(
         {
             "prompt_key": "prompt",

tests/utils/dataset/test_sft_dataset_on_cpu.py

@@ -25,7 +25,7 @@ def get_gsm8k_data():
 
 
 def test_sft_cot_dataset():
-    tokenizer = hf_tokenizer("deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct")
+    tokenizer = hf_tokenizer(os.path.expanduser("~/models/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"))
     local_path = get_gsm8k_data()
     from omegaconf import OmegaConf
 
@@ -50,7 +50,7 @@ def test_sft_cot_dataset():
 
 
 def test_sft_dataset():
-    tokenizer = hf_tokenizer("deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct")
+    tokenizer = hf_tokenizer(os.path.expanduser("~/models/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"))
     local_path = get_gsm8k_data()
     from omegaconf import OmegaConf
 
@@ -75,7 +75,7 @@ def test_sft_dataset():
 
 
 def test_sft_dataset_with_max_samples():
-    tokenizer = hf_tokenizer("deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct")
+    tokenizer = hf_tokenizer(os.path.expanduser("~/models/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"))
     local_path = get_gsm8k_data()
     from omegaconf import OmegaConf