From e084089eb9d50896fd593224fba1d044efa760e8 Mon Sep 17 00:00:00 2001 From: Gabriele Picco Date: Tue, 6 Oct 2020 23:16:00 +0100 Subject: [PATCH] Fix tokenizer UnboundLocalError when padding is set to PaddingStrategy.MAX_LENGTH (#7610) * Fix UnboundLocalError when PaddingStrategy is MAX_LENGTH * Fix UnboundLocalError for TruncationStrategy --- src/transformers/tokenization_utils_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 3648f5e99b..4330ae0a36 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1781,6 +1781,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch elif not isinstance(padding, PaddingStrategy): padding_strategy = PaddingStrategy(padding) + elif isinstance(padding, PaddingStrategy): + padding_strategy = padding else: padding_strategy = PaddingStrategy.DO_NOT_PAD @@ -1806,6 +1808,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): ) # Default to truncate the longest sequences in pairs of inputs elif not isinstance(truncation, TruncationStrategy): truncation_strategy = TruncationStrategy(truncation) + elif isinstance(truncation, TruncationStrategy): + truncation_strategy = truncation else: truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE