added tokenizers serialization tests

2019-04-15 12:03:56 +02:00
parent 3e65f255dc
commit 870b734bfd
7 changed files with 51 additions and 32 deletions
--- a/pytorch_pretrained_bert/tokenization_transfo_xl.py
+++ b/pytorch_pretrained_bert/tokenization_transfo_xl.py
@@ -148,6 +148,7 @@ class TransfoXLTokenizer(object):
        index = 0
        vocab_file = os.path.join(vocab_path, VOCAB_NAME)
        torch.save(self.__dict__, vocab_file)
+        return vocab_file

    def build_vocab(self):
        if self.vocab_file: