added tokenizers serialization tests

2019-04-15 12:03:56 +02:00
parent 3e65f255dc
commit 870b734bfd
7 changed files with 51 additions and 32 deletions
--- a/pytorch_pretrained_bert/tokenization.py
+++ b/pytorch_pretrained_bert/tokenization.py
@@ -146,6 +146,7 @@ class BertTokenizer(object):
                    index = token_index
                writer.write(token + u'\n')
                index += 1
+        return vocab_file

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, cache_dir=None, *inputs, **kwargs):