update hub loading with latest transformers

Signed-off-by: nithinraok <nithinrao.koluguri@gmail.com>

Author: nithinraok

nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from typing import List, Optional
 
 from transformers import AutoTokenizer as AUTOTOKENIZER
@@ -189,6 +190,23 @@ def _initialize_tokenizer(
                 use_fast=use_fast,
                 trust_remote_code=trust_remote_code,
             )
+            # In transformers >= 5.0, from_pretrained may ignore the vocab_file kwarg
+            if vocab_file and os.path.isfile(vocab_file):
+                try:
+                    with open(vocab_file, 'r', encoding='utf-8') as f:
+                        expected_vocab_size = sum(1 for line in f if line.strip())
+                    if expected_vocab_size > 0 and len(self.tokenizer) != expected_vocab_size:
+                        tokenizer_class = type(self.tokenizer)
+                        self.tokenizer = tokenizer_class.from_pretrained(
+                            pretrained_model_name_or_path=vocab_file,
+                            use_fast=use_fast,
+                        )
+                        logging.info(
+                            f"Loaded tokenizer from custom vocab_file with {len(self.tokenizer)} tokens "
+                            f"(resolved class: {tokenizer_class.__name__})"
+                        )
+                except Exception:
+                    pass  # Keep the originally loaded tokenizer if fallback fails
         else:
             self.tokenizer = AUTOTOKENIZER.from_pretrained(
                 pretrained_model_name_or_path=pretrained_model_name,

nemo/core/classes/mixins/hf_io_mixin.py

@@ -50,11 +50,8 @@ def get_hf_model_filter(cls) -> Dict[str, Any]:
         """
         model_filter = dict(
             author=None,
-            library='nemo',
-            language=None,
+            filter=['nemo'],
             model_name=None,
-            task=None,
-            tags=None,
             limit=None,
             full=None,
             cardData=False,
@@ -83,9 +80,8 @@ def search_huggingface_models(cls, model_filter: Optional[Dict[str, Any]] = None
             filt = <DomainSubclass>.get_hf_model_filter()
 
             # Make any modifications to the filter as necessary
-            filt['language'] = [...]
-            filt['task'] = ...
-            filt['tags'] = [...]
+            filt['filter'].append('en')  # Add language filter
+            filt['filter'].append('automatic-speech-recognition')  # Add task filter
 
             # Add any metadata to the filter as needed (kwargs to list_models)
             filt['limit'] = 5

tests/core/test_save_restore.py

@@ -1336,7 +1336,7 @@ class MockModelV2(MockModel):
     def test_hf_model_filter(self):
         filt = ModelPT.get_hf_model_filter()
         assert isinstance(filt, dict)
-        assert filt['library'] == 'nemo'
+        assert 'nemo' in filt['filter']
 
     @pytest.mark.with_downloads()
     @pytest.mark.unit