try with dataset_builder

Signed-off-by: dafnapension <[email protected]>

Author: dafnapension

src/unitxt/loaders.py

@@ -55,9 +55,12 @@
 import pandas as pd
 import requests
 from datasets import (
+    DatasetBuilder,
+    DownloadConfig,
     IterableDataset,
     IterableDatasetDict,
-    get_dataset_split_names,
+    StreamingDownloadManager,
+    load_dataset_builder,
 )
 from datasets import load_dataset as hf_load_dataset
 from huggingface_hub import HfApi
@@ -316,24 +319,59 @@ def load_iterables(
         if self.get_limit() is not None:
             self.log_limited_loading()
 
-        if self.split is not None:
-            return {
-                self.split: ReusableGenerator(
-                    self.split_generator, gen_kwargs={"split": self.split}
-                )
-            }
-
         try:
-            split_names = get_dataset_split_names(
+            dataset_builder = load_dataset_builder(
+                # split_names = get_dataset_split_names(
                 path=self.path,
-                config_name=self.name,
+                name=self.name,
                 trust_remote_code=settings.allow_unverified_code,
             )
+
+            if self.split is not None:
+                return {
+                    self.split: ReusableGenerator(
+                        self.split_generator,
+                        gen_kwargs={
+                            "split": self.split,
+                            "dataset_builder": dataset_builder,
+                        },
+                    )
+                }
+
+            info = dataset_builder.info
+            if info.splits is None:
+                download_config = DownloadConfig()
+                dataset_builder._check_manual_download(
+                    StreamingDownloadManager(
+                        base_path=dataset_builder.base_path,
+                        download_config=download_config,
+                    )
+                )
+                # try:
+                info.splits = {
+                    split_generator.name: {
+                        "name": split_generator.name,
+                        "dataset_name": self.path,
+                    }
+                    for split_generator in dataset_builder._split_generators(
+                        StreamingDownloadManager(
+                            base_path=dataset_builder.base_path,
+                            download_config=download_config,
+                        )
+                    )
+                }
+                # except Exception as err:
+                #     raise SplitsNotFoundError("The split names could not be parsed from the dataset config.") from err
+
             return {
                 split_name: ReusableGenerator(
-                    self.split_generator, gen_kwargs={"split": split_name}
+                    self.split_generator,
+                    gen_kwargs={
+                        "split": split_name,
+                        "dataset_builder": dataset_builder,
+                    },
                 )
-                for split_name in split_names
+                for split_name in info.splits
             }
 
         except:
@@ -360,13 +398,22 @@ def load_iterables(
 
         return dataset
 
-    def split_generator(self, split: str) -> Generator:
+    def split_generator(self, split: str, dataset_builder: DatasetBuilder) -> Generator:
         dataset = self.__class__._loader_cache.get(str(self) + "_" + split, None)
         if dataset is None:
             try:
-                dataset = self.stream_dataset(split)
+                dataset = dataset_builder.as_streaming_dataset(split=split)
             except NotImplementedError:  # streaming is not supported for zipped files so we load without streaming
-                dataset = self.load_dataset(split)
+                dataset_builder.download_and_prepare(
+                    # download_config=download_config,
+                    # download_mode=download_mode,
+                    # verification_mode=verification_mode,
+                    # num_proc=num_proc,
+                    # storage_options=storage_options,
+                )
+
+                # Build dataset for splits
+                dataset = dataset_builder.as_dataset(split=split)
 
             if self.filtering_lambda is not None:
                 dataset = self.filter_load(dataset)

utils/.secrets.baseline

@@ -151,7 +151,7 @@
         "filename": "src/unitxt/loaders.py",
         "hashed_secret": "840268f77a57d5553add023cfa8a4d1535f49742",
         "is_verified": false,
-        "line_number": 599,
+        "line_number": 646,
         "is_secret": false
       }
     ],
@@ -184,5 +184,5 @@
       }
     ]
   },
-  "generated_at": "2025-01-26T10:03:47Z"
+  "generated_at": "2025-01-27T14:50:50Z"
 }