From 599db139f921f3af535052c860cb685cadae6fcd Mon Sep 17 00:00:00 2001 From: Anthony MOI Date: Thu, 26 Dec 2019 15:13:30 -0500 Subject: [PATCH] Code style update --- src/transformers/tokenization_bert.py | 6 ++---- src/transformers/tokenization_gpt2.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/transformers/tokenization_bert.py b/src/transformers/tokenization_bert.py index 4d41654037..4c2abc7d17 100644 --- a/src/transformers/tokenization_bert.py +++ b/src/transformers/tokenization_bert.py @@ -22,7 +22,7 @@ import unicodedata import tokenizers as tk -from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer +from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast logger = logging.getLogger(__name__) @@ -583,9 +583,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast): ) ) if max_length is not None: - self._tokenizer.with_truncation(max_length, - stride=stride, - strategy=truncation_strategy) + self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy) self._tokenizer.with_padding( max_length=max_length if pad_to_max_length else None, direction=self.padding_side, diff --git a/src/transformers/tokenization_gpt2.py b/src/transformers/tokenization_gpt2.py index 1cdc263e25..c99ec08ffc 100644 --- a/src/transformers/tokenization_gpt2.py +++ b/src/transformers/tokenization_gpt2.py @@ -23,7 +23,7 @@ from functools import lru_cache import regex as re import tokenizers as tk -from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer +from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast logger = logging.getLogger(__name__) @@ -277,9 +277,7 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast): self._tokenizer.with_pre_tokenizer(tk.pre_tokenizers.ByteLevel.new(add_prefix_space=add_prefix_space)) self._tokenizer.with_decoder(tk.decoders.ByteLevel.new()) if max_length: - self._tokenizer.with_truncation(max_length, - stride=stride, - strategy=truncation_strategy) + self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy) self._tokenizer.with_padding( max_length=max_length if pad_to_max_length else None, direction=self.padding_side,