From eb3479e7cf2c1969897722981bd8e55103f2857f Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Mon, 5 Apr 2021 09:37:49 -0400 Subject: [PATCH] Some models have no tokenizers (#11064) --- tests/test_tokenization_common.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 7aa1bbf443..f1f7afca62 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -69,13 +69,14 @@ def merge_model_tokenizer_mappings( model_tokenizer_mapping = OrderedDict([]) for configuration in configurations: - model = model_mapping[configuration] - tokenizer = tokenizer_mapping[configuration][0] - tokenizer_fast = tokenizer_mapping[configuration][1] + if configuration in model_mapping and configuration in tokenizer_mapping: + model = model_mapping[configuration] + tokenizer = tokenizer_mapping[configuration][0] + tokenizer_fast = tokenizer_mapping[configuration][1] - model_tokenizer_mapping.update({tokenizer: (configuration, model)}) - if tokenizer_fast is not None: - model_tokenizer_mapping.update({tokenizer_fast: (configuration, model)}) + model_tokenizer_mapping.update({tokenizer: (configuration, model)}) + if tokenizer_fast is not None: + model_tokenizer_mapping.update({tokenizer_fast: (configuration, model)}) return model_tokenizer_mapping