1
1
import glob
2
2
import importlib .util
3
3
import os
4
- import shutil
5
- import tempfile
6
4
import unittest
7
5
8
- from src .unitxt import register_local_catalog
9
6
from src .unitxt .loaders import MissingKaggleCredentialsError
10
7
from src .unitxt .logging_utils import get_logger
11
8
from src .unitxt .test_utils .catalog import register_local_catalog_for_tests
17
14
project_dir = os .path .dirname (
18
15
os .path .dirname (os .path .dirname (os .path .abspath (__file__ )))
19
16
)
17
+ prepration_dir = os .path .join (project_dir , "prepare" )
20
18
catalog_dir = os .path .join (project_dir , "src" , "unitxt" , "catalog" )
21
- glob_query = os .path .join (project_dir , "prepare" , "**" , "*.py" )
22
- all_preparation_files = glob .glob (glob_query , recursive = True )
19
+
20
+
21
+ def test_all_python_modules_in_dir (dir , tester ):
22
+ glob_query = os .path .join (dir , "**" , "*.py" )
23
+ all_preparation_files = glob .glob (glob_query , recursive = True )
24
+
25
+ logger .info (glob_query )
26
+ logger .info (f"Testing preparation files: { all_preparation_files } " )
27
+ # Make sure the order in which the tests are run is deterministic
28
+ # Having a different order for local testing and github testing may cause diffs in results.
29
+ all_preparation_files .sort ()
30
+ for file in all_preparation_files :
31
+ with tester .subTest (file = file ):
32
+ logger .info (f"Testing preparation file: { file } ." )
33
+ try :
34
+ import_module_from_file (file )
35
+ except MissingKaggleCredentialsError as e :
36
+ logger .info (f"Skipping file { file } due to ignored error { e } " )
37
+ continue
38
+ logger .info (f"Testing preparation file: { file } passed" )
39
+ tester .assertTrue (True )
23
40
24
41
25
42
def import_module_from_file (file_path ):
@@ -38,43 +55,22 @@ def import_module_from_file(file_path):
38
55
return module
39
56
40
57
41
- def move_all (src_dir , dst_dir ):
42
- # Create the destination directory if it does not exist
43
- if not os .path .exists (dst_dir ):
44
- os .makedirs (dst_dir )
45
-
46
- for item in os .listdir (src_dir ):
47
- src_path = os .path .join (src_dir , item )
48
- dst_path = os .path .join (dst_dir , item )
49
-
50
- # Move each item in the source directory to the destination directory
51
- shutil .move (src_path , dst_path )
52
-
53
-
54
58
class TestExamples (unittest .TestCase ):
55
59
@classmethod
56
60
def setUpClass (cls ):
57
61
register_local_catalog_for_tests ()
58
62
59
- def test_preprations (self ):
60
- with tempfile .TemporaryDirectory () as tmp_directory :
61
- move_all (catalog_dir , tmp_directory )
62
-
63
- register_local_catalog (tmp_directory )
64
- register_local_catalog (catalog_dir )
65
-
66
- logger .info (glob_query )
67
- logger .info (f"Testing preparation files: { all_preparation_files } " )
68
- # Make sure the order in which the tests are run is deterministic
69
- # Having a different order for local testing and github testing may cause diffs in results.
70
- all_preparation_files .sort ()
71
- for file in all_preparation_files :
72
- with self .subTest (file = file ):
73
- logger .info (f"Testing preparation file: { file } ." )
74
- try :
75
- import_module_from_file (file )
76
- except MissingKaggleCredentialsError as e :
77
- logger .info (f"Skipping file { file } due to ignored error { e } " )
78
- continue
79
- logger .info (f"Testing preparation file: { file } passed" )
80
- self .assertTrue (True )
63
+ def test_prepration (self ):
64
+ dirs = [
65
+ "metrics" ,
66
+ "processors" ,
67
+ "splitters" ,
68
+ "instructions" ,
69
+ "augmentors" ,
70
+ "tasks" ,
71
+ "templates" ,
72
+ "formats" ,
73
+ "cards" ,
74
+ ]
75
+ for dir in dirs :
76
+ test_all_python_modules_in_dir (os .path .join (prepration_dir , dir ), self )
0 commit comments