SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix from_pretrained kwargs passing (#18387)

Browse Source 
Fix #18385
I don't know whether `use_auth_token`, `cache_dir` and `local_files_only` should be passed to `(cls.slow_tokenizer_class)._from_pretrained`, but I guess it should.
This commit is contained in:
YouJiacheng
2022-08-01 20:16:24 +08:00
committed by GitHub
parent 96b5d7db9c
commit 1cd7c6f154
1 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/tokenization_utils_base.py
View File

@@ -1809,6 +1809,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
*init_inputs, *init_inputs,
use_auth_token=use_auth_token, use_auth_token=use_auth_token,
cache_dir=cache_dir, cache_dir=cache_dir,
local_files_only=local_files_only,
**kwargs, **kwargs,
) )
@@ -1821,6 +1822,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
*init_inputs, *init_inputs,
use_auth_token=None, use_auth_token=None,
cache_dir=None, cache_dir=None,
local_files_only=False,
**kwargs **kwargs
): ):
# We instantiate fast tokenizers based on a slow tokenizer if we don't have access to the tokenizer.json # We instantiate fast tokenizers based on a slow tokenizer if we don't have access to the tokenizer.json
@@ -1833,6 +1835,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
pretrained_model_name_or_path, pretrained_model_name_or_path,
copy.deepcopy(init_configuration), copy.deepcopy(init_configuration),
*init_inputs, *init_inputs,
use_auth_token=use_auth_token,
cache_dir=cache_dir,
local_files_only=local_files_only,
**(copy.deepcopy(kwargs)), **(copy.deepcopy(kwargs)),
) )
else: else:
@@ -1864,6 +1869,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
pretrained_model_name_or_path, pretrained_model_name_or_path,
use_auth_token=use_auth_token, use_auth_token=use_auth_token,
cache_dir=cache_dir, cache_dir=cache_dir,
local_files_only=local_files_only,
) )
config_tokenizer_class = config.tokenizer_class config_tokenizer_class = config.tokenizer_class
except (OSError, ValueError, KeyError): except (OSError, ValueError, KeyError):