Merge branch 'main' into fix-azure-openai

Author: martinscooper

src/unitxt/inference.py

@@ -972,16 +972,18 @@ def _get_model_args(self) -> Dict[str, Any]:
         return args
 
     def _create_pipeline(self, model_args: Dict[str, Any]):
-        from transformers import pipeline
+        from transformers import AutoTokenizer, pipeline
 
         path = self.model_name
         if settings.hf_offline_models_path is not None:
             path = os.path.join(settings.hf_offline_models_path, path)
 
+        tokenizer = AutoTokenizer.from_pretrained(self.model_name)
         self.model = pipeline(
             model=path,
             task=self.task,
             use_fast=self.use_fast_tokenizer,
+            tokenizer=tokenizer,
             trust_remote_code=settings.allow_unverified_code,
             **model_args,
             **self.to_dict(