SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

update tokenizers to use self.XX_token_id instead of converting self.XX_token

Browse Source
This commit is contained in:
thomwolf
2019-08-30 13:39:52 +02:00
parent 8faf2e086b
commit f8aace6bcd
4 changed files with 13 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pytorch_transformers/tokenization_bert.py
View File

@@ -171,15 +171,15 @@ class BertTokenizer(PreTrainedTokenizer):
Adds special tokens to the a sequence for sequence classification tasks. Adds special tokens to the a sequence for sequence classification tasks.
A BERT sequence has the following format: [CLS] X [SEP] A BERT sequence has the following format: [CLS] X [SEP]
""" """
return [self._convert_token_to_id(self.cls_token)] + token_ids + [self._convert_token_to_id(self.sep_token)] return [self.cls_token_id] + token_ids + [self.sep_token_id]
def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1):
""" """
Adds special tokens to a sequence pair for sequence classification tasks. Adds special tokens to a sequence pair for sequence classification tasks.
A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP] A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP]
""" """
sep = [self._convert_token_to_id(self.sep_token)] sep = [self.sep_token_id]
cls = [self._convert_token_to_id(self.cls_token)] cls = [self.cls_token_id]
return cls + token_ids_0 + sep + token_ids_1 + sep return cls + token_ids_0 + sep + token_ids_1 + sep
def save_vocabulary(self, vocab_path): def save_vocabulary(self, vocab_path):

6
pytorch_transformers/tokenization_roberta.py
View File

@@ -86,13 +86,13 @@ class RobertaTokenizer(GPT2Tokenizer):
Adds special tokens to a sequence for sequence classification tasks. Adds special tokens to a sequence for sequence classification tasks.
A RoBERTa sequence has the following format: [CLS] X [SEP] A RoBERTa sequence has the following format: [CLS] X [SEP]
""" """
return [self._convert_token_to_id(self.cls_token)] + token_ids + [self._convert_token_to_id(self.sep_token)] return [self.cls_token_id] + token_ids + [self.sep_token_id]
def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1):
""" """
Adds special tokens to a sequence pair for sequence classification tasks. Adds special tokens to a sequence pair for sequence classification tasks.
A RoBERTa sequence pair has the following format: [CLS] A [SEP][SEP] B [SEP] A RoBERTa sequence pair has the following format: [CLS] A [SEP][SEP] B [SEP]
""" """
sep = [self._convert_token_to_id(self.sep_token)] sep = [self.sep_token_id]
cls = [self._convert_token_to_id(self.cls_token)] cls = [self.cls_token_id]
return cls + token_ids_0 + sep + sep + token_ids_1 + sep return cls + token_ids_0 + sep + sep + token_ids_1 + sep

6
pytorch_transformers/tokenization_xlm.py
View File

@@ -220,15 +220,15 @@ class XLMTokenizer(PreTrainedTokenizer):
Adds special tokens to a sequence for sequence classification tasks. Adds special tokens to a sequence for sequence classification tasks.
An XLM sequence has the following format: [CLS] X [SEP] An XLM sequence has the following format: [CLS] X [SEP]
""" """
return [self._convert_token_to_id(self.cls_token)] + token_ids + [self._convert_token_to_id(self.sep_token)] return [self.cls_token_id] + token_ids + [self.sep_token_id]
def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1):
""" """
Adds special tokens to a sequence pair for sequence classification tasks. Adds special tokens to a sequence pair for sequence classification tasks.
An XLM sequence pair has the following format: [CLS] A [SEP] B [SEP] An XLM sequence pair has the following format: [CLS] A [SEP] B [SEP]
""" """
sep = [self._convert_token_to_id(self.sep_token)] sep = [self.sep_token_id]
cls = [self._convert_token_to_id(self.cls_token)] cls = [self.cls_token_id]
return cls + token_ids_0 + sep + token_ids_1 + sep return cls + token_ids_0 + sep + token_ids_1 + sep
def save_vocabulary(self, save_directory): def save_vocabulary(self, save_directory):

8
pytorch_transformers/tokenization_xlnet.py
View File

@@ -182,8 +182,8 @@ class XLNetTokenizer(PreTrainedTokenizer):
Adds special tokens to a sequence pair for sequence classification tasks. Adds special tokens to a sequence pair for sequence classification tasks.
An XLNet sequence pair has the following format: A [SEP] B [SEP][CLS] An XLNet sequence pair has the following format: A [SEP] B [SEP][CLS]
""" """
sep = [self._convert_token_to_id(self.sep_token)] sep = [self.sep_token_id]
cls = [self._convert_token_to_id(self.cls_token)] cls = [self.cls_token_id]
return token_ids + sep + cls return token_ids + sep + cls
def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1):
@@ -191,8 +191,8 @@ class XLNetTokenizer(PreTrainedTokenizer):
Adds special tokens to a sequence for sequence classification tasks. Adds special tokens to a sequence for sequence classification tasks.
An XLNet sequence has the following format: X [SEP][CLS] An XLNet sequence has the following format: X [SEP][CLS]
""" """
sep = [self._convert_token_to_id(self.sep_token)] sep = [self.sep_token_id]
cls = [self._convert_token_to_id(self.cls_token)] cls = [self.cls_token_id]
return token_ids_0 + sep + token_ids_1 + sep + cls return token_ids_0 + sep + token_ids_1 + sep + cls
def save_vocabulary(self, save_directory): def save_vocabulary(self, save_directory):