SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Avoid erasing the attention mask when double padding (#8915)

Browse Source
This commit is contained in:
Sylvain Gugger
2020-12-03 10:45:07 -05:00
committed by GitHub
parent 0deece9c53
commit 8453201cfe
1 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/transformers/tokenization_utils_base.py
View File

@@ -3047,8 +3047,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"] encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"]
else: else:
raise ValueError("Invalid padding strategy:" + str(self.padding_side)) raise ValueError("Invalid padding strategy:" + str(self.padding_side))
else: elif return_attention_mask and "attention_mask" not in encoded_inputs:
if return_attention_mask:
encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"])
return encoded_inputs return encoded_inputs