Code style update

This commit is contained in:
Anthony MOI
2019-12-26 15:13:30 -05:00
parent 835b76a46f
commit 599db139f9
2 changed files with 4 additions and 8 deletions

View File

@@ -22,7 +22,7 @@ import unicodedata
import tokenizers as tk
from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer
from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
logger = logging.getLogger(__name__)
@@ -583,9 +583,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
)
)
if max_length is not None:
self._tokenizer.with_truncation(max_length,
stride=stride,
strategy=truncation_strategy)
self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy)
self._tokenizer.with_padding(
max_length=max_length if pad_to_max_length else None,
direction=self.padding_side,