[Pipelines] Encode to max length of input not max length of tokenizer for batch input (#3857)
* remove max_length = tokenizer.max_length when encoding * make style
This commit is contained in:
committed by
GitHub
parent
857ccdb259
commit
c4158a6314
@@ -425,11 +425,7 @@ class Pipeline(_ScikitCompat):
|
|||||||
# Parse arguments
|
# Parse arguments
|
||||||
inputs = self._args_parser(*texts, **kwargs)
|
inputs = self._args_parser(*texts, **kwargs)
|
||||||
inputs = self.tokenizer.batch_encode_plus(
|
inputs = self.tokenizer.batch_encode_plus(
|
||||||
inputs,
|
inputs, add_special_tokens=True, return_tensors=self.framework, pad_to_max_length=pad_to_max_length,
|
||||||
add_special_tokens=True,
|
|
||||||
return_tensors=self.framework,
|
|
||||||
max_length=self.tokenizer.max_len,
|
|
||||||
pad_to_max_length=pad_to_max_length,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return inputs
|
return inputs
|
||||||
|
|||||||
Reference in New Issue
Block a user