From f9257843b5a036618b31c6ab8376f036ade780a7 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 25 Oct 2022 16:02:13 +0200 Subject: [PATCH] Fix incorrect model<->tokenizer mapping in tokenization testing (#19872) * Fix model-tokenizer mapping Co-authored-by: ydshieh --- tests/test_tokenization_common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 9f14e4122b..6ece29b718 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -120,9 +120,12 @@ def merge_model_tokenizer_mappings( tokenizer = tokenizer_mapping[configuration][0] tokenizer_fast = tokenizer_mapping[configuration][1] - model_tokenizer_mapping.update({tokenizer: (configuration, model)}) + if tokenizer is not None: + if configuration.__name__.startswith(tokenizer.__name__.replace("Tokenizer", "")): + model_tokenizer_mapping.update({tokenizer: (configuration, model)}) if tokenizer_fast is not None: - model_tokenizer_mapping.update({tokenizer_fast: (configuration, model)}) + if configuration.__name__.startswith(tokenizer_fast.__name__.replace("TokenizerFast", "")): + model_tokenizer_mapping.update({tokenizer_fast: (configuration, model)}) return model_tokenizer_mapping