SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge branch 'master' into tf2

Browse Source
This commit is contained in:
thomwolf
2019-09-26 12:02:54 +02:00
parent 4ddc31ff40 7c0f2d0a6a
commit 481d9c4fb5
10 changed files with 146 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
transformers/tokenization_xlnet.py
View File

@@ -183,8 +183,8 @@ class XLNetTokenizer(PreTrainedTokenizer):
def add_special_tokens_single_sequence(self, token_ids):
"""
Adds special tokens to a sequence pair for sequence classification tasks.
An XLNet sequence pair has the following format: A [SEP] B [SEP][CLS]
Adds special tokens to a sequence for sequence classification tasks.
An XLNet sequence has the following format: X [SEP][CLS]
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
@@ -192,8 +192,8 @@ class XLNetTokenizer(PreTrainedTokenizer):
def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1):
"""
Adds special tokens to a sequence for sequence classification tasks.
An XLNet sequence has the following format: X [SEP][CLS]
Adds special tokens to a sequence pair for sequence classification tasks.
An XLNet sequence pair has the following format: A [SEP] B [SEP][CLS]
"""
sep = [self.sep_token_id]