update readme and pretrained model weight files

2019-07-16 15:11:29 +02:00
parent 43e0e8fa04
commit 1849aa7d39
6 changed files with 60 additions and 18 deletions
--- a/pytorch_transformers/tokenization_xlm.py
+++ b/pytorch_transformers/tokenization_xlm.py
@@ -59,6 +59,13 @@ PRETRAINED_VOCAB_FILES_MAP = {

 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    'xlm-mlm-en-2048': 512,
+    'xlm-mlm-ende-1024': 512,
+    'xlm-mlm-enfr-1024': 512,
+    'xlm-mlm-enro-1024': 512,
+    'xlm-mlm-tlm-xnli15-1024': 512,
+    'xlm-mlm-xnli15-1024': 512,
+    'xlm-clm-enfr-1024': 512,
+    'xlm-clm-ende-1024': 512,
 }

 def get_pairs(word):