fix python2 tests

This commit is contained in:
thomwolf
2019-07-05 15:11:59 +02:00
parent 36bca545ff
commit 6dacc79d39
2 changed files with 7 additions and 5 deletions

View File

@@ -101,8 +101,12 @@ class PreTrainedTokenizer(object):
max_len = cls.max_model_input_sizes[pretrained_model_name_or_path]
kwargs['max_len'] = min(kwargs.get('max_len', int(1e12)), max_len)
# Merge resolved_vocab_files arguments in kwargs.
for args_name, file_path in resolved_vocab_files.items():
kwargs[args_name] = file_path
# Instantiate tokenizer.
tokenizer = cls(*inputs, **resolved_vocab_files, **kwargs)
tokenizer = cls(*inputs, **kwargs)
return tokenizer