Some models have no tokenizers (#11064)

2021-04-05 09:37:49 -04:00
parent 773e4c7263
commit eb3479e7cf
1 changed files with 7 additions and 6 deletions
--- a/tests/test_tokenization_common.py
+++ b/tests/test_tokenization_common.py
@@ -69,6 +69,7 @@ def merge_model_tokenizer_mappings(
    model_tokenizer_mapping = OrderedDict([])

    for configuration in configurations:
+        if configuration in model_mapping and configuration in tokenizer_mapping:
            model = model_mapping[configuration]
            tokenizer = tokenizer_mapping[configuration][0]
            tokenizer_fast = tokenizer_mapping[configuration][1]