Revert "Serialize tokenizer use_fast setting (#339)" (#355)

This reverts commit 7482d85.

Because this change can cause backwards incompatibility within models
saved by v1.1.x, I think it would be better for it to wait for v1.2.

Author: adrianeboyd

azure-pipelines.yml

@@ -86,10 +86,3 @@ jobs:
       pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.1.0/en_core_web_trf-3.1.0-py3-none-any.whl --no-deps
       python -c "import spacy; nlp = spacy.load('en_core_web_trf'); doc = nlp('test')"
     displayName: 'Test backwards compatibility for v1.0 models'
-    condition: and(startsWith(variables['imageName'], 'ubuntu'), eq(variables['python.version'], '3.9'))
-
-  - script: |
-      pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.4.0/en_core_web_trf-3.4.0-py3-none-any.whl --no-deps
-      python -c "import spacy; nlp = spacy.load('en_core_web_trf'); doc = nlp('test')"
-    displayName: 'Test backwards compatibility for v1.1 models'
-    condition: and(startsWith(variables['imageName'], 'ubuntu'), eq(variables['python.version'], '3.9'))

spacy_transformers/layers/hf_shim.py

@@ -64,7 +64,6 @@ def to_bytes(self):
                     with open(vocab_file_path, "wb") as fileh:
                         fileh.write(hf_model.vocab_file_contents)
                     tokenizer.vocab_file = vocab_file_path
-                tok_dict["kwargs"] = {"use_fast": tokenizer.is_fast}
                 tokenizer.save_pretrained(str(temp_dir.absolute()))
                 for x in temp_dir.glob("**/*"):
                     if x.is_file():
@@ -94,10 +93,9 @@ def from_bytes(self, bytes_data):
                 config_file = temp_dir / "config.json"
                 srsly.write_json(config_file, config_dict)
                 config = self.config_cls.from_pretrained(config_file)
-                tok_kwargs = tok_dict.pop("kwargs", {})
                 for x, x_bytes in tok_dict.items():
                     Path(temp_dir / x).write_bytes(x_bytes)
-                tokenizer = self.tokenizer_cls.from_pretrained(str(temp_dir.absolute()), **tok_kwargs)
+                tokenizer = self.tokenizer_cls.from_pretrained(str(temp_dir.absolute()))
                 vocab_file_contents = None
                 if hasattr(tokenizer, "vocab_file"):
                     vocab_file_name = tokenizer.vocab_files_names["vocab_file"]

spacy_transformers/tests/test_model_wrapper.py

@@ -51,18 +51,11 @@ def trf_model(name, output_attentions, output_hidden_states):
                 "output_hidden_states": output_hidden_states,
             },
         )
-
     else:
-        # test slow tokenizers with distilbert-base-uncased (parameterizing
-        # for all models blows up the memory usage during the test suite)
-        if name == "distilbert-base-uncased":
-            use_fast = False
-        else:
-            use_fast = True
         model = TransformerModel(
             name,
             get_doc_spans,
-            {"use_fast": use_fast},
+            {"use_fast": True},
             {
                 "output_attentions": output_attentions,
                 "output_hidden_states": output_hidden_states,
@@ -74,10 +67,6 @@ def trf_model(name, output_attentions, output_hidden_states):
 
 def test_model_init(name, trf_model):
     assert isinstance(trf_model, Model)
-    if name == "distilbert-base-uncased":
-        assert not trf_model.tokenizer.is_fast
-    else:
-        assert trf_model.tokenizer.is_fast
 
 
 def test_model_predict(docs, trf_model):

spacy_transformers/tests/test_serialize.py

@@ -1,5 +1,4 @@
 import pytest
-import copy
 import spacy
 from spacy import Language
 from spacy.lang.en import English
@@ -19,7 +18,6 @@
     "model": {
         "@architectures": "spacy-transformers.TransformerModel.v3",
         "name": "distilbert-base-uncased",
-        "tokenizer_config": {"use_fast": False},
     }
 }
 
@@ -74,8 +72,6 @@ def test_initialized_transformer_tobytes():
     trf2 = nlp2.add_pipe("transformer", config=DEFAULT_CONFIG)
     trf2.from_bytes(trf_bytes)
 
-    assert trf2.model.tokenizer.is_fast is False
-
 
 def test_initialized_transformer_todisk():
     nlp = Language()
@@ -87,21 +83,6 @@ def test_initialized_transformer_todisk():
         trf2 = nlp2.add_pipe("transformer", config=DEFAULT_CONFIG)
         trf2.from_disk(d)
 
-        assert trf2.model.tokenizer.is_fast is False
-
-    fast_config = copy.deepcopy(DEFAULT_CONFIG)
-    fast_config["model"]["tokenizer_config"]["use_fast"] = True
-    nlp = Language()
-    trf = nlp.add_pipe("transformer", config=fast_config)
-    nlp.initialize()
-    with make_tempdir() as d:
-        trf.to_disk(d)
-        nlp2 = Language()
-        trf2 = nlp2.add_pipe("transformer", config=fast_config)
-        trf2.from_disk(d)
-
-        assert trf2.model.tokenizer.is_fast is True
-
 
 def test_transformer_pipeline_tobytes():
     nlp = Language()