Merge pull request #361 from adrianeboyd/backport/ci-extend-transformers

Backport requirements and CI updates to v1.1.x

Author: adrianeboyd

azure-pipelines.yml

@@ -17,7 +17,7 @@ jobs:
   strategy:
     matrix:
       Python36Linux:
-        imageName: 'ubuntu-latest'
+        imageName: 'ubuntu-20.04'
         python.version: '3.6'
       Python37Mac:
         imageName: 'macos-latest'
@@ -34,6 +34,9 @@ jobs:
       Python310Windows:
         imageName: 'windows-latest'
         python.version: '3.10'
+      Python311Linux:
+        imageName: 'ubuntu-latest'
+        python.version: '3.11'
     maxParallel: 4
   pool:
     vmImage: $(imageName)
@@ -67,11 +70,6 @@ jobs:
     displayName: 'Install oldest supported torch for python 3.6'
     condition: eq(variables['python.version'], '3.6')
 
-  - script: |
-      pip install "torch<1.13.0+cpu" --extra-index-url https://download.pytorch.org/whl/cpu
-    displayName: 'Install newest working torch for python 3.7+'
-    condition: ne(variables['python.version'], '3.6')
-
   - bash: |
       SDIST=$(python -c "import os;print(os.listdir('./dist')[0])" 2>&1)
       python -m pip install dist/$SDIST

requirements.txt

@@ -1,5 +1,5 @@
 spacy>=3.4.0,<4.0.0
-transformers>=3.4.0,<4.25.0
+transformers>=3.4.0,<4.26.0
 torch>=1.6.0
 srsly>=2.4.0,<3.0.0
 dataclasses>=0.6,<1.0; python_version < "3.7"

setup.cfg

@@ -24,14 +24,15 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [options]
 zip_safe = false
 include_package_data = true
 python_requires = >=3.6
 install_requires =
     spacy>=3.4.0,<4.0.0
-    transformers>=3.4.0,<4.25.0
+    transformers>=3.4.0,<4.26.0
     torch>=1.6.0
     srsly>=2.4.0,<3.0.0
     dataclasses>=0.6,<1.0; python_version < "3.7"

spacy_transformers/tests/test_serialize.py

@@ -116,9 +116,12 @@ def test_transformer_pipeline_todisk_settings():
     assert trf.model.transformer.config.output_attentions is False
     assert "attentions" not in nlp("test")._.trf_data.model_output
     # modify model_max_length (note that modifications to
-    # tokenizer.model_max_length are not serialized by save_pretrained
-    # see: https://github.com/explosion/spaCy/discussions/7393)
+    # tokenizer.model_max_length for transformers<4.25 are not serialized by
+    # save_pretrained, see: https://github.com/explosion/spaCy/discussions/7393)
     trf.model.tokenizer.init_kwargs["model_max_length"] = 499
+    # transformer>=4.25, model_max_length is saved and init_kwargs changes are
+    # clobbered, so do both for this test
+    trf.model.tokenizer.model_max_length = 499
     # add attentions on-the-fly
     trf.model.transformer.config.output_attentions = True
     assert nlp("test")._.trf_data.model_output.attentions is not None