From 026097b9ee7862905ec3f3b0e729c5dbc95a0fd9 Mon Sep 17 00:00:00 2001 From: Funtowicz Morgan Date: Thu, 7 May 2020 14:02:53 +0000 Subject: [PATCH] Ensure fast tokenizer can construct tensor without pad token if only one sample is provided. (#4201) --- src/transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py index b3e5ef3a92..a2d258afad 100644 --- a/src/transformers/tokenization_utils.py +++ b/src/transformers/tokenization_utils.py @@ -2435,7 +2435,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer): ) # Needed if we have to return a tensor - pad_to_max_length = pad_to_max_length or (return_tensors is not None) + pad_to_max_length = pad_to_max_length or (return_tensors is not None and len(batch_text_or_text_pairs) > 1) # Throw an error if we can pad because there is no padding token if pad_to_max_length and self.pad_token_id is None: