[tokenizers] Fixing #8001 - Adding tests on tokenizers serialization (#8006)

* fixing #8001 * make T5 tokenizer serialization more robust - style
2020-10-26 10:27:48 +01:00
parent 7087d9b1c0
commit 79eb391586
17 changed files with 98 additions and 25 deletions
--- a/src/transformers/tokenization_utils_base.py
+++ b/src/transformers/tokenization_utils_base.py
@@ -1673,7 +1673,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
        if (
            "tokenizer_file" not in resolved_vocab_files or resolved_vocab_files["tokenizer_file"] is None
        ) and cls.slow_tokenizer_class is not None:
-            slow_tokenizer = cls.slow_tokenizer_class._from_pretrained(
+            slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained(
                copy.deepcopy(resolved_vocab_files),
                pretrained_model_name_or_path,
                copy.deepcopy(init_configuration),