max_len_single_sentence & max_len_sentences_pair as attributes so they can be modified

This commit is contained in:
thomwolf
2019-08-23 22:07:26 +02:00
parent ab7bd5ef98
commit 3bcbebd440
8 changed files with 26 additions and 40 deletions

View File

@@ -87,6 +87,9 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
super(OpenAIGPTTokenizer, self).__init__(unk_token=unk_token, **kwargs)
self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens
self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens
try:
import ftfy
from spacy.lang.en import English