SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Added documentation and changed parameters for special_tokens_sentences_pair.

Browse Source
This commit is contained in:
LysandreJik
2019-08-12 15:13:53 -04:00
parent 912fdff899
commit 22ac004a7c
5 changed files with 50 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pytorch_transformers/tokenization_xlm.py
View File

@@ -215,12 +215,20 @@ class XLMTokenizer(PreTrainedTokenizer):
return out_string
def add_special_tokens_single_sentence(self, token_ids):
"""
Adds special tokens to a sequence for sequence classification tasks.
An XLM sequence has the following format: [CLS] X [SEP]
"""
return [self._convert_token_to_id(self.cls_token)] + token_ids + [self._convert_token_to_id(self.sep_token)]
def add_special_tokens_sentences_pair(self, *token_ids):
def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1):
"""
Adds special tokens to a sequence pair for sequence classification tasks.
An XLM sequence pair has the following format: [CLS] A [SEP] B [SEP]
"""
sep = [self._convert_token_to_id(self.sep_token)]
cls = [self._convert_token_to_id(self.cls_token)]
return cls + token_ids[0] + sep + token_ids[1] + sep
return cls + token_ids_0 + sep + token_ids_1 + sep
def save_vocabulary(self, save_directory):
"""Save the tokenizer vocabulary and merge files to a directory."""