[Pipelines] Encode to max length of input not max length of tokenizer for batch input (#3857)

* remove max_length = tokenizer.max_length when encoding

* make style
This commit is contained in:
Patrick von Platen
2020-04-20 20:39:16 +02:00
committed by GitHub
parent 857ccdb259
commit c4158a6314

View File

@@ -425,11 +425,7 @@ class Pipeline(_ScikitCompat):
# Parse arguments # Parse arguments
inputs = self._args_parser(*texts, **kwargs) inputs = self._args_parser(*texts, **kwargs)
inputs = self.tokenizer.batch_encode_plus( inputs = self.tokenizer.batch_encode_plus(
inputs, inputs, add_special_tokens=True, return_tensors=self.framework, pad_to_max_length=pad_to_max_length,
add_special_tokens=True,
return_tensors=self.framework,
max_length=self.tokenizer.max_len,
pad_to_max_length=pad_to_max_length,
) )
return inputs return inputs