SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Some models have no tokenizers (#11064)

Browse Source
This commit is contained in:
Lysandre Debut
2021-04-05 09:37:49 -04:00
committed by GitHub
parent 773e4c7263
commit eb3479e7cf
1 changed files with 7 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tests/test_tokenization_common.py
View File

@@ -69,13 +69,14 @@ def merge_model_tokenizer_mappings(
model_tokenizer_mapping = OrderedDict([]) model_tokenizer_mapping = OrderedDict([])
for configuration in configurations: for configuration in configurations:
model = model_mapping[configuration] if configuration in model_mapping and configuration in tokenizer_mapping:
tokenizer = tokenizer_mapping[configuration][0] model = model_mapping[configuration]
tokenizer_fast = tokenizer_mapping[configuration][1] tokenizer = tokenizer_mapping[configuration][0]
tokenizer_fast = tokenizer_mapping[configuration][1]
model_tokenizer_mapping.update({tokenizer: (configuration, model)}) model_tokenizer_mapping.update({tokenizer: (configuration, model)})
if tokenizer_fast is not None: if tokenizer_fast is not None:
model_tokenizer_mapping.update({tokenizer_fast: (configuration, model)}) model_tokenizer_mapping.update({tokenizer_fast: (configuration, model)})
return model_tokenizer_mapping return model_tokenizer_mapping