Merge branch 'master' into improved_testing

2019-08-30 13:40:35 +02:00
parent f8aace6bcd f7978490b2
commit 50e615f43d
43 changed files with 3103 additions and 56 deletions
--- a/pytorch_transformers/tokenization_xlm.py
+++ b/pytorch_transformers/tokenization_xlm.py
@@ -122,6 +122,10 @@ class XLMTokenizer(PreTrainedTokenizer):
                                           cls_token=cls_token, mask_token=mask_token,
                                           additional_special_tokens=additional_special_tokens,
                                           **kwargs)
+
+        self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
+        self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens
+
        try:
            import ftfy
            from spacy.lang.en import English