Skip to content

Commit e00617f

Browse files
committed
adjust to new readers for csv
Signed-off-by: dafnapension <[email protected]>
1 parent ef29521 commit e00617f

File tree

2 files changed

+8
-4
lines changed

2 files changed

+8
-4
lines changed

src/unitxt/loaders.py

+7-3
Original file line numberDiff line numberDiff line change
@@ -447,9 +447,13 @@ def load_iterables(self):
447447
return iterables
448448

449449
def split_generator(self, split: str) -> Generator:
450-
dataset = pd.read_csv(
451-
self.files[split], nrows=self.get_limit(), sep=self.sep
452-
).to_dict("records")
450+
dataset = self.__class__._loader_cache.get(str(self) + "_" + split, None)
451+
if dataset is None:
452+
reader = self.get_reader()
453+
dataset = reader(self.files[split], **self.get_args()).to_dict("records")
454+
self.__class__._loader_cache.max_size = settings.loader_cache_size
455+
self.__class__._loader_cache[str(self) + "_" + split] = dataset
456+
453457
yield from dataset
454458

455459

utils/.secrets.baseline

+1-1
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@
151151
"filename": "src/unitxt/loaders.py",
152152
"hashed_secret": "840268f77a57d5553add023cfa8a4d1535f49742",
153153
"is_verified": false,
154-
"line_number": 595,
154+
"line_number": 599,
155155
"is_secret": false
156156
}
157157
],

0 commit comments

Comments
 (0)