SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Code style update

Browse Source
This commit is contained in:
Anthony MOI
2019-12-26 15:13:30 -05:00
parent 835b76a46f
commit 599db139f9
2 changed files with 4 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/tokenization_bert.py
View File

@@ -22,7 +22,7 @@ import unicodedata
import tokenizers as tk import tokenizers as tk
from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -583,9 +583,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
) )
) )
if max_length is not None: if max_length is not None:
self._tokenizer.with_truncation(max_length, self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy)
stride=stride,
strategy=truncation_strategy)
self._tokenizer.with_padding( self._tokenizer.with_padding(
max_length=max_length if pad_to_max_length else None, max_length=max_length if pad_to_max_length else None,
direction=self.padding_side, direction=self.padding_side,

6
src/transformers/tokenization_gpt2.py
View File

@@ -23,7 +23,7 @@ from functools import lru_cache
import regex as re import regex as re
import tokenizers as tk import tokenizers as tk
from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -277,9 +277,7 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
self._tokenizer.with_pre_tokenizer(tk.pre_tokenizers.ByteLevel.new(add_prefix_space=add_prefix_space)) self._tokenizer.with_pre_tokenizer(tk.pre_tokenizers.ByteLevel.new(add_prefix_space=add_prefix_space))
self._tokenizer.with_decoder(tk.decoders.ByteLevel.new()) self._tokenizer.with_decoder(tk.decoders.ByteLevel.new())
if max_length: if max_length:
self._tokenizer.with_truncation(max_length, self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy)
stride=stride,
strategy=truncation_strategy)
self._tokenizer.with_padding( self._tokenizer.with_padding(
max_length=max_length if pad_to_max_length else None, max_length=max_length if pad_to_max_length else None,
direction=self.padding_side, direction=self.padding_side,