Merge branch 'master' into xlmr_mlm

Author: Guanheng Zhang

.circleci/unittest/linux/scripts/environment.yml

@@ -17,5 +17,5 @@ dependencies:
     - sphinx
     - sphinx-rtd-theme
     - tqdm
-    - https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.2.5/de_core_news_sm-2.2.5.tar.gz#egg=de_core_news_sm==2.2.5
-    - https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz#egg=en_core_web_sm==2.2.5
+    - https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.0.0/de_core_news_sm-3.0.0.tar.gz#egg=de_core_news_sm==3.0.0
+    - https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz#egg=en_core_web_sm==3.0.0

.circleci/unittest/linux/scripts/install.sh

@@ -16,3 +16,6 @@ conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch cpuonly
 printf "* Installing torchtext\n"
 git submodule update --init --recursive
 python setup.py develop
+
+printf "* Installing parameterized\n"
+pip install parameterized

.circleci/unittest/linux/scripts/setup_env.sh

@@ -45,4 +45,6 @@ fi
 
 # 4. Download
 printf "* Downloading SpaCy English models\n"
-python -m spacy download en
+python -m spacy download en_core_web_sm
+printf "* Downloading SpaCy German models\n"
+python -m spacy download de_core_news_sm 

.circleci/unittest/windows/scripts/environment.yml

@@ -19,5 +19,5 @@ dependencies:
     - tqdm
     - certifi
     - future
-    - https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.2.5/de_core_news_sm-2.2.5.tar.gz#egg=de_core_news_sm==2.2.5
-    - https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz#egg=en_core_web_sm==2.2.5
+    - https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.0.0/de_core_news_sm-3.0.0.tar.gz#egg=de_core_news_sm==3.0.0
+    - https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz#egg=en_core_web_sm==3.0.0

.circleci/unittest/windows/scripts/install.sh

@@ -21,3 +21,6 @@ conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch cpuonly
 printf "* Installing torchtext\n"
 git submodule update --init --recursive
 "$root_dir/packaging/vc_env_helper.bat" python setup.py develop
+
+printf "* Installing parameterized\n"
+pip install parameterized

.circleci/unittest/windows/scripts/setup_env.sh

@@ -39,4 +39,6 @@ conda env update --file "${this_dir}/environment.yml" --prune
 
 # 4. Download
 printf "* Downloading SpaCy English models\n"
-python -m spacy download en
+python -m spacy download en_core_web_sm
+printf "* Downloading SpaCy German models\n"
+python -m spacy download de_core_news_sm 

benchmark/benchmark_basic_english_normalize.py

@@ -15,7 +15,7 @@ def _run_benchmark_lookup(train, tokenizer):
 
     existing_basic_english_tokenizer = get_tokenizer("basic_english")
     experimental_basic_english_normalize = basic_english_normalize()
-    experimental_jit_basic_english_normalize = torch.jit.script(experimental_basic_english_normalize.to_ivalue())
+    experimental_jit_basic_english_normalize = torch.jit.script(experimental_basic_english_normalize)
 
     # existing eager lookup
     train, _ = AG_NEWS()

benchmark/benchmark_experimental_vectors.py

@@ -42,7 +42,7 @@ def _run_benchmark_lookup(tokens, vector):
 
     # experimental FastText jit lookup
     print("FastText Experimental - Jit Mode")
-    jit_fast_text_experimental = torch.jit.script(fast_text_experimental.to_ivalue())
+    jit_fast_text_experimental = torch.jit.script(fast_text_experimental)
     _run_benchmark_lookup(tokens, jit_fast_text_experimental)
 
 

benchmark/benchmark_experimental_vocab.py

@@ -67,7 +67,7 @@ def benchmark_experimental_vocab_construction(vocab_file_path, is_raw_text=True,
             print("Loading from raw text file with basic_english_normalize tokenizer")
             for _ in range(num_iters):
                 tokenizer = basic_english_normalize()
-                jited_tokenizer = torch.jit.script(tokenizer.to_ivalue())
+                jited_tokenizer = torch.jit.script(tokenizer)
                 build_vocab_from_text_file(f, jited_tokenizer, num_cpus=1)
             print("Construction time:", time.monotonic() - t0)
     else:
@@ -140,7 +140,7 @@ def token_iterator(file_path):
         t0 = time.monotonic()
         v_experimental = VocabExperimental(ordered_dict)
         print("Construction time:", time.monotonic() - t0)
-    jit_v_experimental = torch.jit.script(v_experimental.to_ivalue())
+    jit_v_experimental = torch.jit.script(v_experimental)
 
     # existing Vocab eager lookup
     print("Vocab - Eager Mode")
@@ -154,7 +154,7 @@ def token_iterator(file_path):
     _run_benchmark_lookup([tokens], v_experimental)
     _run_benchmark_lookup(tokens_lists, v_experimental)
 
-    jit_v_experimental = torch.jit.script(v_experimental.to_ivalue())
+    jit_v_experimental = torch.jit.script(v_experimental)
     # experimental Vocab jit lookup
     print("Vocab Experimental - Jit Mode")
     _run_benchmark_lookup(tokens, jit_v_experimental)

benchmark/benchmark_pytext_vocab.py

@@ -150,7 +150,7 @@ def benchmark_experimental_vocab():
     t0 = time.monotonic()
     experimental_script_vocab = ExperimentalScriptVocabulary(ordered_dict, unk_token="<unk>")
     print("Construction time:", time.monotonic() - t0)
-    jit_experimental_script_vocab = torch.jit.script(experimental_script_vocab.to_ivalue())
+    jit_experimental_script_vocab = torch.jit.script(experimental_script_vocab)
 
     # pytext Vocab eager lookup
     print("Pytext Vocabulary - Eager Mode")

docs/source/index.rst

@@ -44,6 +44,7 @@ popular datasets for natural language.
    experimental_transforms
    experimental_vectors
    experimental_vocab
+   models_utils
    examples <examples>
 
 .. automodule:: torchtext

docs/source/models_utils.rst

@@ -0,0 +1,13 @@
+.. role:: hidden
+    :class: hidden-section
+
+torchtext.experimental.models.utils
+===================================
+
+.. automodule:: torchtext.experimental.models.utils
+.. currentmodule:: torchtext.experimental.models.utils
+
+:hidden:`count_model_param`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: count_model_param

examples/BERT/model.py

@@ -43,7 +43,8 @@ def __init__(self, ntoken, ninp, dropout=0.5):
         self.norm = LayerNorm(ninp)
         self.dropout = Dropout(dropout)
 
-    def forward(self, src, token_type_input):
+    def forward(self, seq_inputs):
+        src, token_type_input = seq_inputs
         src = self.embed(src) + self.pos_embed(src) \
             + self.tok_type_embed(src, token_type_input)
         return self.dropout(self.norm(src))
@@ -114,16 +115,16 @@ def forward(self, src, src_mask=None, src_key_padding_mask=None):
 class BertModel(nn.Module):
     """Contain a transformer encoder."""
 
-    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
+    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, embed_layer, dropout=0.5):
         super(BertModel, self).__init__()
         self.model_type = 'Transformer'
-        self.bert_embed = BertEmbedding(ntoken, ninp)
+        self.bert_embed = embed_layer
         encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
         self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
         self.ninp = ninp
 
-    def forward(self, src, token_type_input):
-        src = self.bert_embed(src, token_type_input)
+    def forward(self, seq_inputs):
+        src = self.bert_embed(seq_inputs)
         output = self.transformer_encoder(src)
         return output
 
@@ -150,15 +151,16 @@ class MLMTask(nn.Module):
 
     def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
         super(MLMTask, self).__init__()
-        self.bert_model = BertModel(ntoken, ninp, nhead, nhid, nlayers, dropout=0.5)
+        embed_layer = BertEmbedding(ntoken, ninp)
+        self.bert_model = BertModel(ntoken, ninp, nhead, nhid, nlayers, embed_layer, dropout=0.5)
         self.mlm_span = Linear(ninp, ninp)
         self.activation = F.gelu
         self.norm_layer = LayerNorm(ninp, eps=1e-12)
         self.mlm_head = Linear(ninp, ntoken)
 
     def forward(self, src, token_type_input=None):
         src = src.transpose(0, 1)  # Wrap up by nn.DataParallel
-        output = self.bert_model(src, token_type_input)
+        output = self.bert_model((src, token_type_input))
         output = self.mlm_span(output)
         output = self.activation(output)
         output = self.norm_layer(output)
@@ -199,7 +201,7 @@ def __init__(self, bert_model):
 
     def forward(self, src, token_type_input):
         src = src.transpose(0, 1)  # Wrap up by nn.DataParallel
-        output = self.bert_model(src, token_type_input)
+        output = self.bert_model((src, token_type_input))
         # Send the first <'cls'> seq to a classifier
         output = self.activation(self.linear_layer(output[0]))
         output = self.ns_span(output)
@@ -216,7 +218,7 @@ def __init__(self, bert_model):
         self.qa_span = Linear(bert_model.ninp, 2)
 
     def forward(self, src, token_type_input):
-        output = self.bert_model(src, token_type_input)
+        output = self.bert_model((src, token_type_input))
         # transpose output (S, N, E) to (N, S, E)
         output = output.transpose(0, 1)
         output = self.activation(output)

examples/BERT/ns_task.py

@@ -5,7 +5,7 @@
 import torch.nn as nn
 from torch.nn.parallel import DistributedDataParallel as DDP
 from torch.utils.data import DataLoader
-from model import NextSentenceTask, BertModel
+from model import NextSentenceTask, BertModel, BertEmbedding
 from utils import run_demo, run_ddp, wrap_up
 
 
@@ -149,7 +149,8 @@ def run_main(args, rank=None):
     if args.checkpoint != 'None':
         model = torch.load(args.checkpoint)
     else:
-        pretrained_bert = BertModel(len(vocab), args.emsize, args.nhead, args.nhid, args.nlayers, args.dropout)
+        embed_layer = BertEmbedding(len(vocab), args.emsize)
+        pretrained_bert = BertModel(len(vocab), args.emsize, args.nhead, args.nhid, args.nlayers, embed_layer, args.dropout)
         pretrained_bert.load_state_dict(torch.load(args.bert_model))
         model = NextSentenceTask(pretrained_bert)
 

examples/BERT/qa_task.py

@@ -9,7 +9,7 @@
 from model import QuestionAnswerTask
 from metrics import compute_qa_exact, compute_qa_f1
 from utils import print_loss_log
-from model import BertModel
+from model import BertModel, BertEmbedding
 
 
 def process_raw_data(data):
@@ -174,7 +174,8 @@ def train():
     train_dataset = process_raw_data(train_dataset)
     dev_dataset = process_raw_data(dev_dataset)
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    pretrained_bert = BertModel(len(vocab), args.emsize, args.nhead, args.nhid, args.nlayers, args.dropout)
+    embed_layer = BertEmbedding(len(vocab), args.emsize)
+    pretrained_bert = BertModel(len(vocab), args.emsize, args.nhead, args.nhid, args.nlayers, embed_layer, args.dropout)
     pretrained_bert.load_state_dict(torch.load(args.bert_model))
     model = QuestionAnswerTask(pretrained_bert).to(device)
     criterion = nn.CrossEntropyLoss()

examples/data_pipeline/pipelines.py

@@ -32,11 +32,10 @@ def build_sp_pipeline(spm_file):
     vocab = PretrainedSPVocab(load_sp_model(spm_file))
 
     # Insert token in vocab to match a pretrained vocab
-    vocab.insert_token('<pad>', 1)
     pipeline = TextSequentialTransforms(tokenizer, vocab)
-    jit_pipeline = torch.jit.script(pipeline.to_ivalue())
+    jit_pipeline = torch.jit.script(pipeline)
     print('jit sentencepiece pipeline success!')
-    return pipeline, pipeline.to_ivalue(), jit_pipeline
+    return pipeline, pipeline, jit_pipeline
 
 
 def build_legacy_torchtext_vocab_pipeline(vocab_file):
@@ -59,9 +58,9 @@ def build_experimental_torchtext_pipeline(hf_vocab_file):
     with open(hf_vocab_file, 'r') as f:
         vocab = load_vocab_from_file(f)
         pipeline = TextSequentialTransforms(tokenizer, vocab)
-        jit_pipeline = torch.jit.script(pipeline.to_ivalue())
+        jit_pipeline = torch.jit.script(pipeline)
         print('jit experimental torchtext pipeline success!')
-        return pipeline, pipeline.to_ivalue(), jit_pipeline
+        return pipeline, pipeline, jit_pipeline
 
 
 def build_legacy_batch_torchtext_vocab_pipeline(vocab_file):
@@ -104,9 +103,9 @@ def build_legacy_pytext_script_vocab_pipeline(vocab_file):
         vocab_list.insert(0, "<unk>")
         pipeline = TextSequentialTransforms(tokenizer,
                                             PyTextScriptVocabTransform(ScriptVocabulary(vocab_list)))
-        jit_pipeline = torch.jit.script(pipeline.to_ivalue())
+        jit_pipeline = torch.jit.script(pipeline)
         print('jit legacy PyText pipeline success!')
-        return pipeline, pipeline.to_ivalue(), jit_pipeline
+        return pipeline, pipeline, jit_pipeline
 
 
 def build_experimental_pytext_script_pipeline(vocab_file):
@@ -125,9 +124,9 @@ def build_experimental_pytext_script_pipeline(vocab_file):
     # Insert token in vocab to match a pretrained vocab
     pipeline = TextSequentialTransforms(tokenizer,
                                         PyTextScriptVocabTransform(script_vocab(ordered_dict)))
-    jit_pipeline = torch.jit.script(pipeline.to_ivalue())
+    jit_pipeline = torch.jit.script(pipeline)
     print('jit legacy PyText pipeline success!')
-    return pipeline, pipeline.to_ivalue(), jit_pipeline
+    return pipeline, pipeline, jit_pipeline
 
 
 def build_legacy_fasttext_vector_pipeline():
@@ -143,10 +142,10 @@ def build_experimental_fasttext_vector_pipeline():
     vector = FastTextExperimental()
 
     pipeline = TextSequentialTransforms(tokenizer, vector)
-    jit_pipeline = torch.jit.script(pipeline.to_ivalue())
+    jit_pipeline = torch.jit.script(pipeline)
 
     print('jit legacy fasttext pipeline success!')
-    return pipeline, pipeline.to_ivalue(), jit_pipeline
+    return pipeline, pipeline, jit_pipeline
 
 
 def run_benchmark_lookup(text_classification_dataset, pipeline):

examples/data_pipeline/transforms.py

@@ -24,14 +24,6 @@ def forward(self, tokens: List[str]) -> List[int]:
     def insert_token(self, token: str, index: int) -> None:
         self.vocab.insert_token(token, index)
 
-    def to_ivalue(self):
-        if hasattr(self.vocab, 'to_ivalue'):
-            sp_model = self.sp_model
-            new_module = PretrainedSPVocab(sp_model)
-            new_module.vocab = self.vocab.to_ivalue()
-            return new_module
-        return self
-
 
 class PyTextVocabTransform(nn.Module):
     r"""PyTextVocabTransform transform
@@ -57,12 +49,6 @@ def __init__(self, vocab):
     def forward(self, tokens: List[str]) -> List[int]:
         return self.vocab.lookup_indices_1d(tokens)
 
-    def to_ivalue(self):
-        if hasattr(self.vocab, 'to_ivalue'):
-            vocab = self.vocab.to_ivalue()
-            return PyTextScriptVocabTransform(vocab)
-        return self
-
 
 class ToLongTensor(nn.Module):
     r"""Convert a list of integers to long tensor