SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix pad_token check condition (#25685)

Browse Source 
fix

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2023-08-23 16:39:28 +02:00
committed by GitHub
parent 8657ec68fc
commit 2189a7f54a
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_utils_base.py
View File

@@ -2503,7 +2503,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
max_length = self.model_max_length max_length = self.model_max_length
# Test if we have a padding token # Test if we have a padding token
if padding_strategy != PaddingStrategy.DO_NOT_PAD and (not self.pad_token or self.pad_token_id < 0): if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.pad_token is None or self.pad_token_id < 0):
raise ValueError( raise ValueError(
"Asking to pad but the tokenizer does not have a padding token. " "Asking to pad but the tokenizer does not have a padding token. "
"Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` " "Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` "