Merge branch 'main' into add-cache-gitignore

Author: martinscooper

src/unitxt/inference.py

@@ -972,16 +972,18 @@ def _get_model_args(self) -> Dict[str, Any]:
         return args
 
     def _create_pipeline(self, model_args: Dict[str, Any]):
-        from transformers import pipeline
+        from transformers import AutoTokenizer, pipeline
 
         path = self.model_name
         if settings.hf_offline_models_path is not None:
             path = os.path.join(settings.hf_offline_models_path, path)
 
+        tokenizer = AutoTokenizer.from_pretrained(self.model_name)
         self.model = pipeline(
             model=path,
             task=self.task,
             use_fast=self.use_fast_tokenizer,
+            tokenizer=tokenizer,
             trust_remote_code=settings.allow_unverified_code,
             **model_args,
             **self.to_dict(

src/unitxt/loaders.py

@@ -307,7 +307,7 @@ def load_dataset(
             self.__class__._loader_cache.max_size = settings.loader_cache_size
             if not disable_memory_caching:
                 self.__class__._loader_cache[dataset_id] = dataset
-        return self.__class__._loader_cache[dataset_id]
+        return dataset
 
     def _maybe_set_classification_policy(self):
         if os.path.exists(self.path):