adding max_lengths for single sentences and sentences pairs

This commit is contained in:
thomwolf
2019-08-23 17:31:11 +02:00
parent 90dcd8c05d
commit 47d6853439
5 changed files with 40 additions and 0 deletions

View File

@@ -139,6 +139,14 @@ class BertTokenizer(PreTrainedTokenizer):
tokenize_chinese_chars=tokenize_chinese_chars)
self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token)
@property
def max_len_single_sentence(self):
return self.max_len - 2 # take into account special tokens
@property
def max_len_sentences_pair(self):
return self.max_len - 3 # take into account special tokens
@property
def vocab_size(self):
return len(self.vocab)