SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

create_mask_from_sequences -> create_token_type_ids_from_sequences

Browse Source
This commit is contained in:
LysandreJik
2019-09-24 09:09:28 -04:00
parent c832f43a4d
commit d340e2329e
6 changed files with 6 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pytorch_transformers/tokenization_utils.py
View File

@@ -780,7 +780,7 @@ class PreTrainedTokenizer(object):
)
if output_token_type:
information["token_type_ids"] = self.create_mask_from_sequences(text, text_pair)
information["token_type_ids"] = self.create_token_type_ids_from_sequences(text, text_pair)
else:
logger.warning("No special tokens were added. The two sequences have been concatenated.")
sequence = first_sentence_tokens + second_sentence_tokens
@@ -863,7 +863,7 @@ class PreTrainedTokenizer(object):
return information
def create_mask_from_sequences(self, sequence_0, sequence_1):
def create_token_type_ids_from_sequences(self, sequence_0, sequence_1):
logger.warning("This tokenizer does not make use of special tokens.")
return [0] * len(self.encode(sequence_0)) + [1] * len(self.encode(sequence_1))