Revisit huggingface cache policy

Signed-off-by: elronbandel <[email protected]>

Author: elronbandel

src/unitxt/api.py

@@ -139,7 +139,7 @@ def load_dataset(
     dataset_query: Optional[str] = None,
     split: Optional[str] = None,
     streaming: bool = False,
-    disable_cache: Optional[bool] = None,
+    disable_cache: Optional[bool] = True,
     **kwargs,
 ) -> Union[DatasetDict, IterableDatasetDict, Dataset, IterableDataset]:
     """Loads dataset.
@@ -188,9 +188,6 @@ def load_dataset(
     if split is not None:
         stream = stream[split]
 
-    if disable_cache is None:
-        disable_cache = settings.disable_hf_datasets_cache
-
     if streaming:
         dataset = stream.to_iterable_dataset(
             features=UNITXT_DATASET_SCHEMA,

src/unitxt/loaders.py

@@ -210,7 +210,7 @@ class LoadHF(Loader):
         Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]
     ] = None
     revision: Optional[str] = None
-    streaming: bool = True
+    streaming: bool = False
     filtering_lambda: Optional[str] = None
     num_proc: Optional[int] = None
     requirements_list: List[str] = OptionalField(default_factory=list)

src/unitxt/settings_utils.py

@@ -149,7 +149,7 @@ def __getattr__(self, key):
     settings.skip_artifacts_prepare_and_verify = (bool, False)
     settings.data_classification_policy = None
     settings.mock_inference_mode = (bool, False)
-    settings.disable_hf_datasets_cache = (bool, True)
+    settings.disable_hf_datasets_cache = (bool, False)
     settings.loader_cache_size = (int, 1)
     settings.task_data_as_text = (bool, True)
     settings.default_provider = "watsonx"