SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Added documentation and changed parameters for special_tokens_sentences_pair.

Browse Source
This commit is contained in:
LysandreJik
2019-08-12 15:13:53 -04:00
parent 912fdff899
commit 22ac004a7c
5 changed files with 50 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
pytorch_transformers/tokenization_xlnet.py
View File

@@ -178,14 +178,22 @@ class XLNetTokenizer(PreTrainedTokenizer):
return out_string
def add_special_tokens_single_sentence(self, token_ids):
logger.warning("No method was defined for special tokens and single sentence streams in XLNet. "
"Returning token_ids")
return token_ids
def add_special_tokens_sentences_pair(self, *token_ids):
"""
Adds special tokens to a sequence pair for sequence classification tasks.
An XLNet sequence pair has the following format: A [SEP] B [SEP][CLS]
"""
sep = [self._convert_token_to_id(self.sep_token)]
cls = [self._convert_token_to_id(self.cls_token)]
return token_ids[0] + sep + token_ids[1] + sep + cls
return token_ids + sep + cls
def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1):
"""
Adds special tokens to a sequence for sequence classification tasks.
An XLNet sequence has the following format: X [SEP][CLS]
"""
sep = [self._convert_token_to_id(self.sep_token)]
cls = [self._convert_token_to_id(self.cls_token)]
return token_ids_0 + sep + token_ids_1 + sep + cls
def save_vocabulary(self, save_directory):
""" Save the sentencepiece vocabulary (copy original file) and special tokens file