adding max_lengths for single sentences and sentences pairs

This commit is contained in:
thomwolf
2019-08-23 17:31:11 +02:00
parent 90dcd8c05d
commit 47d6853439
5 changed files with 40 additions and 0 deletions

View File

@@ -160,6 +160,14 @@ class RobertaTokenizer(PreTrainedTokenizer):
text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
return text
@property
def max_len_single_sentence(self):
return self.max_len - 2 # take into account special tokens
@property
def max_len_sentences_pair(self):
return self.max_len - 4 # take into account special tokens
def add_special_tokens_single_sentence(self, token_ids):
"""
Adds special tokens to a sequence for sequence classification tasks.