adding max_lengths for single sentences and sentences pairs
This commit is contained in:
@@ -160,6 +160,14 @@ class RobertaTokenizer(PreTrainedTokenizer):
|
||||
text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
|
||||
return text
|
||||
|
||||
@property
|
||||
def max_len_single_sentence(self):
|
||||
return self.max_len - 2 # take into account special tokens
|
||||
|
||||
@property
|
||||
def max_len_sentences_pair(self):
|
||||
return self.max_len - 4 # take into account special tokens
|
||||
|
||||
def add_special_tokens_single_sentence(self, token_ids):
|
||||
"""
|
||||
Adds special tokens to a sequence for sequence classification tasks.
|
||||
|
||||
Reference in New Issue
Block a user