SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix misleading RoBERTa token type ids

Browse Source
This commit is contained in:
Lysandre
2020-01-07 13:44:56 +01:00
committed by Lysandre Debut
parent c024ab98df
commit dfe012ad9d
1 changed files with 2 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/tokenization_roberta.py
View File

@@ -144,9 +144,7 @@ class RobertaTokenizer(GPT2Tokenizer):
def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
"""
Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
A RoBERTa sequence pair mask has the following format:
0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
| first sequence | second sequence
RoBERTa does not make use of token type ids, therefore a list of zeros is returned.
if token_ids_1 is None, only returns the first portion of the mask (0's).
"""
@@ -155,4 +153,4 @@ class RobertaTokenizer(GPT2Tokenizer):
if token_ids_1 is None:
return len(cls + token_ids_0 + sep) * [0]
return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1]
return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]