Turn off streaming for huggingface datasets and set cache to be off

Signed-off-by: elronbandel <[email protected]>

Author: elronbandel

src/unitxt/loaders.py

@@ -210,7 +210,7 @@ class LoadHF(Loader):
         Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]
     ] = None
     revision: Optional[str] = None
-    streaming: bool = True
+    streaming: bool = False
     filtering_lambda: Optional[str] = None
     num_proc: Optional[int] = None
     requirements_list: List[str] = OptionalField(default_factory=list)
@@ -308,11 +308,12 @@ def _maybe_set_classification_policy(self):
             )
 
     def load_iterables(self) -> IterableDatasetDict:
-        try:
-            dataset = self.stream_dataset()
-        except (
-            NotImplementedError
-        ):  # streaming is not supported for zipped files so we load without streaming
+        if self.streaming:
+            try:
+                dataset = self.stream_dataset()
+            except NotImplementedError:  # streaming is not supported for zipped files so we load without streaming
+                dataset = self.load_dataset()
+        else:
             dataset = self.load_dataset()
 
         if self.filtering_lambda is not None:

src/unitxt/settings_utils.py

@@ -149,7 +149,7 @@ def __getattr__(self, key):
     settings.skip_artifacts_prepare_and_verify = (bool, False)
     settings.data_classification_policy = None
     settings.mock_inference_mode = (bool, False)
-    settings.disable_hf_datasets_cache = (bool, True)
+    settings.disable_hf_datasets_cache = (bool, False)
     settings.loader_cache_size = (int, 1)
     settings.task_data_as_text = (bool, True)
     settings.default_provider = "watsonx"
@@ -193,5 +193,5 @@ def get_settings() -> Settings:
     return Settings()
 
 
-def get_constants():
+def get_constants() -> Constants:
     return Constants()

utils/.secrets.baseline

@@ -151,7 +151,7 @@
         "filename": "src/unitxt/loaders.py",
         "hashed_secret": "840268f77a57d5553add023cfa8a4d1535f49742",
         "is_verified": false,
-        "line_number": 531,
+        "line_number": 532,
         "is_secret": false
       }
     ],
@@ -184,5 +184,5 @@
       }
     ]
   },
-  "generated_at": "2025-01-26T10:03:47Z"
+  "generated_at": "2025-01-28T08:40:45Z"
 }