Merge branch 'master' into fix/listener_ref

Author: adrianeboyd

azure-pipelines.yml

@@ -12,15 +12,15 @@ jobs:
       Python36Linux:
         imageName: 'ubuntu-16.04'
         python.version: '3.6'
-      Python36Mac:
-        imageName: 'macos-10.14'
-        python.version: '3.6'
-      Python37Linux:
-        imageName: 'ubuntu-16.04'
-        python.version: '3.7'
       Python37Mac:
         imageName: 'macos-10.14'
         python.version: '3.7'
+      Python38Linux:
+        imageName: 'ubuntu-16.04'
+        python.version: '3.8'
+      Python38Mac:
+        imageName: 'macos-10.14'
+        python.version: '3.8'
     maxParallel: 4
   pool:
     vmImage: $(imageName)
@@ -32,15 +32,22 @@ jobs:
       architecture: 'x64'
 
   - script: |
-      python -m pip install --upgrade pip wheel
+      python -m pip install --upgrade pip setuptools
       pip install -r requirements.txt
     displayName: 'Install dependencies'
 
   - script: python setup.py sdist
     displayName: 'Build sdist'
 
+  - script: |
+      pip freeze > installed.txt
+      pip uninstall -y -r installed.txt
+    displayName: "Uninstall all packages"
+
   - script: pip install dist/*.tar.gz
     displayName: 'Install from sdist'
 
-  - script: python -m pytest spacy_transformers --cov=spacy_transformers
+  - script: |
+      pip install -r requirements.txt
+      python -m pytest spacy_transformers --cov=spacy_transformers
     displayName: 'Run tests'

requirements.txt

@@ -1,12 +1,12 @@
 spacy-nightly>=3.0.0a39,<3.1.0
-transformers>=3.0.0,<3.1.0
+transformers>=3.0.0,<4.3.0
 torch>=1.5.0
 torchcontrib>=0.0.2,<0.1.0
 srsly>=2.0.1,<3.0.0
 ftfy>=5.0.0,<6.0.0
 dataclasses>=0.6; python_version < "3.7"
 importlib_metadata>=0.20; python_version < "3.8"
-pytokenizations>=0.2.0
+spacy-alignments>=0.7.2,<1.0.0
 # Testing
 pytest>=5.0.1,<5.1.0
 pytest-cov>=2.7.0,<2.8.0

setup.cfg

@@ -1,5 +1,5 @@
 [metadata]
-version = 1.0.0rc0
+version = 1.0.0rc1
 description = spaCy pipelines for pre-trained BERT and other transformers
 url = https://spacy.io
 author = Explosion
@@ -28,14 +28,14 @@ include_package_data = true
 python_requires = >=3.6
 install_requires =
     spacy-nightly>=3.0.0a39,<3.1.0
-    transformers>=3.0.0,<3.1.0
+    transformers>=3.0.0,<4.3.0
     torch>=1.5.0
     torchcontrib>=0.0.2,<0.1.0
     srsly>=2.0.1,<3.0.0
     ftfy>=5.0.0,<6.0.0
     dataclasses>=0.6; python_version < "3.7"
     importlib_metadata>=0.20; python_version < "3.8"
-    pytokenizations>=0.2.0
+    spacy-alignments>=0.7.2,<1.0.0
 
 setup_requires =
     setuptools
@@ -53,6 +53,14 @@ cuda92 =
     cupy-cuda92>=5.0.0b4
 cuda100 =
     cupy-cuda100>=5.0.0b4
+cuda101 =
+    cupy-cuda101>=5.0.0b4
+cuda102 =
+    cupy-cuda102>=5.0.0b4
+cuda110 =
+    cupy-cuda110>=5.0.0b4
+cuda111 =
+    cupy-cuda111>=5.0.0b4
 
 [options.entry_points]
 spacy_factories =

spacy_transformers/align.py

@@ -1,6 +1,6 @@
 import numpy
-from typing import cast, Dict, List, Tuple, Callable, Set
-import tokenizations
+from typing import cast, Dict, List, Tuple, Callable, Set, Optional
+from spacy_alignments.tokenizations import get_alignments
 from spacy.tokens import Span, Token
 from thinc.api import Ops
 from thinc.types import Ragged, Floats2d, Ints1d
@@ -100,7 +100,7 @@ def get_alignment_via_offset_mapping(spans: List[Span], token_data) -> Ragged:
     return align
 
 
-def get_alignment(spans: List[Span], wordpieces: List[List[str]]) -> Ragged:
+def get_alignment(spans: List[Span], wordpieces: List[List[str]], special_tokens: Optional[List[str]] = None) -> Ragged:
     """Compute a ragged alignment array that records, for each unique token in
     `spans`, the corresponding indices in the flattened `wordpieces` array.
     For instance, imagine you have two overlapping spans:
@@ -135,14 +135,24 @@ def get_alignment(spans: List[Span], wordpieces: List[List[str]]) -> Ragged:
     """
     if len(spans) != len(wordpieces):
         raise ValueError("Cannot align batches of different sizes.")
+    if special_tokens is None:
+        special_tokens = []
     # Tokens can occur more than once, and we need the alignment of each token
     # to its place in the concatenated wordpieces array.
     token_positions = get_token_positions(spans)
     alignment: List[Set[int]] = [set() for _ in range(len(token_positions))]
     wp_start = 0
     for i, (span, wp_toks) in enumerate(zip(spans, wordpieces)):
         sp_toks = [token.text for token in span]
-        span2wp, wp2span = tokenizations.get_alignments(sp_toks, wp_toks)
+        wp_toks_filtered = wp_toks
+        # In the case that the special tokens do not appear in the text, filter
+        # them out for alignment purposes so that special tokens like "<s>" are
+        # not aligned to the character "s" in the text. (If the special tokens
+        # appear in the text, it's not possible to distinguish them from the
+        # added special tokens, so they may be aligned incorrectly.)
+        if not any([special in span.text for special in special_tokens]):
+            wp_toks_filtered = [tok if tok not in special_tokens else "" for tok in wp_toks]
+        span2wp, wp2span = get_alignments(sp_toks, wp_toks_filtered)
         for token, wp_js in zip(span, span2wp):
             position = token_positions[token]
             alignment[position].update(wp_start + j for j in wp_js)

spacy_transformers/architectures.py

@@ -9,7 +9,7 @@
 
 @registry.architectures.register("spacy-transformers.TransformerListener.v1")
 def transformer_listener_tok2vec_v1(
-    pooling: Model[Ragged, Floats2d], grad_factor: float = 1.0
+    pooling: Model[Ragged, Floats2d], grad_factor: float = 1.0, upstream: str = "*"
 ) -> Model[List[Doc], List[Floats2d]]:
     """Create a 'TransformerListener' layer, which will connect to a Transformer
     component earlier in the pipeline.
@@ -30,8 +30,13 @@ def transformer_listener_tok2vec_v1(
         them upstream. You can set this to 0 to "freeze" the transformer weights
         with respect to the component, or use it to make some components more
         significant than others. Leaving it at 1.0 is usually fine.
+    upstream (str): A string to identify the 'upstream' Transformer
+        to communicate with. The upstream name should either be the wildcard
+        string '*', or the name of the `Transformer` component. You'll almost
+        never have multiple upstream Transformer components, so the wildcard
+        string will almost always be fine.
     """
-    listener = TransformerListener("transformer")
+    listener = TransformerListener(upstream_name=upstream)
     model = chain(listener, trfs2arrays(pooling, grad_factor))
     model.set_ref("listener", listener)
     return model

spacy_transformers/data_classes.py

@@ -91,21 +91,21 @@ class FullTransformerBatch:
     tokens: BatchEncoding
     tensors: List[torch.Tensor]
     align: Ragged
-    _doc_data: Optional[List[TransformerData]] = None
+    cached_doc_data: Optional[List[TransformerData]] = None
 
     @classmethod
     def empty(cls, nr_docs) -> "FullTransformerBatch":
         spans = [[] for i in range(nr_docs)]
         doc_data = [TransformerData.empty() for i in range(nr_docs)]
         align = Ragged(numpy.zeros((0,), dtype="i"), numpy.zeros((0,), dtype="i"))
-        return cls(spans=spans, tokens={}, tensors=[], align=align, _doc_data=doc_data)
+        return cls(spans=spans, tokens={}, tensors=[], align=align, cached_doc_data=doc_data)
 
     @property
     def doc_data(self) -> List[TransformerData]:
         """The outputs, split per spaCy Doc object."""
-        if self._doc_data is None:
-            self._doc_data = self.split_by_doc()
-        return self._doc_data
+        if self.cached_doc_data is None:
+            self.cached_doc_data = self.split_by_doc()
+        return self.cached_doc_data
 
     def unsplit_by_doc(self, arrays: List[List[Floats3d]]) -> "FullTransformerBatch":
         """Return a new FullTransformerBatch from a split batch of activations,

spacy_transformers/layers/transformer_model.py

@@ -116,7 +116,7 @@ def forward(
     # if "offset_mapping" in token_data and hasattr(token_data, "char_to_token"):
     #    align = get_alignment_via_offset_mapping(flat_spans, token_data)
     # else:
-    align = get_alignment(flat_spans, token_data["input_texts"])
+    align = get_alignment(flat_spans, token_data["input_texts"], model.attrs["tokenizer"].all_special_tokens)
     output = FullTransformerBatch(
         spans=nested_spans, tokens=token_data, tensors=tensors, align=align
     )
@@ -147,6 +147,8 @@ def _convert_transformer_inputs(model, tokens: BatchEncoding, is_train):
 
 def _convert_transformer_outputs(model, inputs_outputs, is_train):
     _, tensors = inputs_outputs
+    if hasattr(tensors, "to_tuple"):
+        tensors = tensors.to_tuple()
 
     def backprop(d_tensors: List[torch.Tensor]) -> ArgsKwargs:
         return ArgsKwargs(args=(tensors,), kwargs={"grad_tensors": d_tensors})

spacy_transformers/tests/test_pipeline_component.py

@@ -32,6 +32,22 @@ def component(vocab):
     return Transformer(Vocab(), DummyTransformer())
 
 
+@pytest.fixture(scope="module")
+def simple_nlp():
+    nlp = Language()
+    nlp.add_pipe("transformer")
+    train_examples = []
+    for t in TRAIN_DATA:
+        train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))
+
+    optimizer = nlp.initialize()
+    for i in range(2):
+        losses = {}
+        nlp.update(train_examples, sgd=optimizer, losses=losses)
+
+    return nlp
+
+
 def test_init(component):
     assert isinstance(component.vocab, Vocab)
     assert isinstance(component.model, Model)
@@ -83,22 +99,19 @@ def test_listeners(component, docs):
 ]
 
 
-def test_transformer_pipeline_simple():
+def test_transformer_pipeline_simple(simple_nlp):
     """Test that a simple pipeline with just a transformer at least runs"""
-    nlp = Language()
-    nlp.add_pipe("transformer")
-    train_examples = []
-    for t in TRAIN_DATA:
-        train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))
-
-    optimizer = nlp.initialize()
-    for i in range(2):
-        losses = {}
-        nlp.update(train_examples, sgd=optimizer, losses=losses)
-    doc = nlp("We're interested at underwater basket weaving.")
+    doc = simple_nlp("We're interested at underwater basket weaving.")
     assert doc
 
 
+def test_transformer_pipeline_long_token(simple_nlp):
+    """Test that a simple pipeline raises an error on a text that exceeds the
+    model max length."""
+    with pytest.raises(ValueError):
+        doc = simple_nlp("https://example.com/" + "a/" * 1000)
+
+
 cfg_string = """
     [nlp]
     lang = "en"
@@ -116,12 +129,14 @@ def test_transformer_pipeline_simple():
     [components.tagger.model.tok2vec]
     @architectures = "spacy-transformers.TransformerListener.v1"
     grad_factor = 1.0
+    upstream = ${components.transformer.name}
     
     [components.tagger.model.tok2vec.pooling]
     @layers = "reduce_mean.v1"
 
     [components.transformer]
     factory = "transformer"
+    name = "custom_upstream"
     """
 
 
@@ -135,6 +150,7 @@ def test_transformer_pipeline_tagger():
     tagger_trf = tagger.model.get_ref("tok2vec").layers[0]
     assert isinstance(transformer, Transformer)
     assert isinstance(tagger_trf, TransformerListener)
+    assert tagger_trf.upstream_name == "custom_upstream"
     train_examples = []
     for t in TRAIN_DATA:
         train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))

spacy_transformers/tests/util.py

@@ -16,15 +16,21 @@ def __init__(self):
         self.int2str = {}
         self.start_symbol = "<s>"
         self.end_symbol = "</s>"
+        self.model_max_length = 512
 
-    def batch_encode_plus(
+    @property
+    def all_special_tokens(self):
+        return [self.start_symbol, self.end_symbol]
+
+    def __call__(
         self,
         texts,
         add_special_tokens=True,
         max_length=None,
         stride: int = 0,
         truncation_strategy="longest_first",
-        pad_to_max_length=False,
+        padding=False,
+        truncation=False,
         is_pretokenized=False,
         return_tensors=None,
         return_token_type_ids=None,
@@ -48,11 +54,14 @@ def batch_encode_plus(
             output["attention_mask"].append(mask)
             output["token_type_ids"].append(type_ids)
             output["offset_mapping"].append(offsets)
-        output = self._pad(output)
+        if padding:
+            output = self._pad(output)
         if return_tensors == "pt":
             output["input_ids"] = torch.tensor(output["input_ids"])  # type: ignore
             output["attention_mask"] = torch.tensor(output["attention_mask"])  # type: ignore
             output["token_type_ids"] = torch.tensor(output["token_type_ids"])  # type: ignore
+        if return_length:
+            output["length"] = torch.tensor([len(x) for x in output["input_ids"]])  # type: ignore
         return output
 
     def convert_ids_to_tokens(self, ids: Union[List[int], torch.Tensor]) -> List[str]:

spacy_transformers/util.py

@@ -38,19 +38,32 @@ def huggingface_from_pretrained(source: Union[Path, str], config: Dict):
 
 def huggingface_tokenize(tokenizer, texts: List[str]) -> BatchEncoding:
     """Apply a Huggingface tokenizer to a batch of texts."""
-    token_data = tokenizer.batch_encode_plus(
+    token_data = tokenizer(
         texts,
         add_special_tokens=True,
         return_attention_mask=True,
         return_length=True,
         return_offsets_mapping=isinstance(tokenizer, PreTrainedTokenizerFast),
         return_tensors="pt",
         return_token_type_ids=None,  # Sets to model default
-        pad_to_max_length=True,
+        padding="longest",
     )
     token_data["input_texts"] = [
         tokenizer.convert_ids_to_tokens(list(ids)) for ids in token_data["input_ids"]
     ]
+    # Because of padding=longest all sequences contain the same number of
+    # tokens, so simply check whether the first sequence is too long.
+    if (
+        len(token_data["length"]) > 0
+        and int(token_data["length"][0]) > tokenizer.model_max_length
+    ):
+        for text, tokens in zip(texts, token_data["input_texts"]):
+            # The longest text(s) in the batch will have another symbol (the end
+            # symbol) rather than the pad token as the final token in the
+            # sequence. Raise an error on the first such text found.
+            if tokens[-1] != tokenizer.pad_token:
+                msg = f"The following input span is too long for the model max length {tokenizer.model_max_length}. Preprocess the texts to break up long tokens or if you're training a new model, you may want to modify the spaCy tokenizer or the @span_getters in the model config:\n\n{text}"
+                raise ValueError(msg)
     return token_data