SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix tokenizer UnboundLocalError when padding is set to PaddingStrategy.MAX_LENGTH (#7610)

Browse Source 
* Fix UnboundLocalError when PaddingStrategy is MAX_LENGTH

* Fix UnboundLocalError for TruncationStrategy
This commit is contained in:
Gabriele Picco
2020-10-06 23:16:00 +01:00
committed by GitHub
parent adfe6ace88
commit e084089eb9
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/tokenization_utils_base.py
View File

@@ -1781,6 +1781,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch
elif not isinstance(padding, PaddingStrategy): elif not isinstance(padding, PaddingStrategy):
padding_strategy = PaddingStrategy(padding) padding_strategy = PaddingStrategy(padding)
elif isinstance(padding, PaddingStrategy):
padding_strategy = padding
else: else:
padding_strategy = PaddingStrategy.DO_NOT_PAD padding_strategy = PaddingStrategy.DO_NOT_PAD
@@ -1806,6 +1808,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
) # Default to truncate the longest sequences in pairs of inputs ) # Default to truncate the longest sequences in pairs of inputs
elif not isinstance(truncation, TruncationStrategy): elif not isinstance(truncation, TruncationStrategy):
truncation_strategy = TruncationStrategy(truncation) truncation_strategy = TruncationStrategy(truncation)
elif isinstance(truncation, TruncationStrategy):
truncation_strategy = truncation
else: else:
truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE