SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Close #1304

Browse Source
This commit is contained in:
Julien Chaumond
2019-09-27 17:02:29 -04:00
parent df7cd9e4e4
commit c6acbdd50a
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
transformers/tokenization_roberta.py
View File

@@ -81,6 +81,8 @@ class RobertaTokenizer(GPT2Tokenizer):
bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,
sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,
mask_token=mask_token, **kwargs) mask_token=mask_token, **kwargs)
self.max_len_single_sentence = self.max_len - 2 # take into account special tokens
self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens
def add_special_tokens_single_sequence(self, token_ids): def add_special_tokens_single_sequence(self, token_ids):
""" """