File tree 2 files changed +8
-4
lines changed
2 files changed +8
-4
lines changed Original file line number Diff line number Diff line change @@ -447,9 +447,13 @@ def load_iterables(self):
447
447
return iterables
448
448
449
449
def split_generator (self , split : str ) -> Generator :
450
- dataset = pd .read_csv (
451
- self .files [split ], nrows = self .get_limit (), sep = self .sep
452
- ).to_dict ("records" )
450
+ dataset = self .__class__ ._loader_cache .get (str (self ) + "_" + split , None )
451
+ if dataset is None :
452
+ reader = self .get_reader ()
453
+ dataset = reader (self .files [split ], ** self .get_args ()).to_dict ("records" )
454
+ self .__class__ ._loader_cache .max_size = settings .loader_cache_size
455
+ self .__class__ ._loader_cache [str (self ) + "_" + split ] = dataset
456
+
453
457
yield from dataset
454
458
455
459
Original file line number Diff line number Diff line change 151
151
"filename": "src/unitxt/loaders.py",
152
152
"hashed_secret": "840268f77a57d5553add023cfa8a4d1535f49742",
153
153
"is_verified": false,
154
- "line_number": 595 ,
154
+ "line_number": 599 ,
155
155
"is_secret": false
156
156
}
157
157
],
You can’t perform that action at this time.
0 commit comments