Some model_types cannot be in the mapping (#13259)

* Some tokenizers cannot be in the mapping * Style
2021-08-25 18:56:16 +02:00
parent 68b6907290
commit 5af8df5afb
1 changed files with 3 additions and 1 deletions
--- a/src/transformers/tokenization_utils_base.py
+++ b/src/transformers/tokenization_utils_base.py
@@ -1799,7 +1799,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
                            break

                if model_type is not None:
-                    config_tokenizer_class, config_tokenizer_class_fast = TOKENIZER_MAPPING_NAMES[model_type]
+                    config_tokenizer_class, config_tokenizer_class_fast = TOKENIZER_MAPPING_NAMES.get(
+                        model_type, (None, None)
+                    )
                    if config_tokenizer_class is None:
                        config_tokenizer_class = config_tokenizer_class_fast