SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[tokenizer] use use_auth_token for config (#13523)

Browse Source 
* [tokenizer] use use_auth_token for config

* args order
This commit is contained in:
Stas Bekman
2021-09-13 04:31:35 -07:00
committed by GitHub
parent d2904264ab
commit f1c22dae7d
1 changed files with 14 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/transformers/tokenization_utils_base.py
View File

@@ -1741,12 +1741,23 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
logger.info(f"loading file {file_path} from cache at {resolved_vocab_files[file_id]}") logger.info(f"loading file {file_path} from cache at {resolved_vocab_files[file_id]}")
return cls._from_pretrained( return cls._from_pretrained(
resolved_vocab_files, pretrained_model_name_or_path, init_configuration, *init_inputs, **kwargs resolved_vocab_files,
pretrained_model_name_or_path,
init_configuration,
*init_inputs,
use_auth_token=use_auth_token,
**kwargs,
) )
@classmethod @classmethod
def _from_pretrained( def _from_pretrained(
cls, resolved_vocab_files, pretrained_model_name_or_path, init_configuration, *init_inputs, **kwargs cls,
resolved_vocab_files,
pretrained_model_name_or_path,
init_configuration,
*init_inputs,
use_auth_token=None,
**kwargs
): ):
# We instantiate fast tokenizers based on a slow tokenizer if we don't have access to the tokenizer.json # We instantiate fast tokenizers based on a slow tokenizer if we don't have access to the tokenizer.json
# file or if `from_slow` is set to True. # file or if `from_slow` is set to True.
@@ -1784,7 +1795,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
# Second attempt. If we have not yet found tokenizer_class, let's try to use the config. # Second attempt. If we have not yet found tokenizer_class, let's try to use the config.
try: try:
config = AutoConfig.from_pretrained(pretrained_model_name_or_path) config = AutoConfig.from_pretrained(pretrained_model_name_or_path, use_auth_token=use_auth_token)
config_tokenizer_class = config.tokenizer_class config_tokenizer_class = config.tokenizer_class
except (OSError, ValueError, KeyError): except (OSError, ValueError, KeyError):
# skip if an error occurred. # skip if an error occurred.