From 5af8df5afb7a550e18af22094daac97bc4984be9 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Wed, 25 Aug 2021 18:56:16 +0200 Subject: [PATCH] Some `model_type`s cannot be in the mapping (#13259) * Some tokenizers cannot be in the mapping * Style --- src/transformers/tokenization_utils_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index e57299cf77..55670bcc61 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1799,7 +1799,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): break if model_type is not None: - config_tokenizer_class, config_tokenizer_class_fast = TOKENIZER_MAPPING_NAMES[model_type] + config_tokenizer_class, config_tokenizer_class_fast = TOKENIZER_MAPPING_NAMES.get( + model_type, (None, None) + ) if config_tokenizer_class is None: config_tokenizer_class = config_tokenizer_class_fast