SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix #1784

Browse Source
This commit is contained in:
Lysandre
2019-11-11 10:15:14 -05:00
parent 7a9aae1044
commit b5d330d118
6 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
templates/adding_a_new_model/tokenization_xxx.py
View File

@@ -172,7 +172,7 @@ class XxxTokenizer(PreTrainedTokenizer):
special tokens for the model special tokens for the model
Returns: Returns:
A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:

2
transformers/tokenization_bert.py
View File

@@ -220,7 +220,7 @@ class BertTokenizer(PreTrainedTokenizer):
special tokens for the model special tokens for the model
Returns: Returns:
A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:

2
transformers/tokenization_roberta.py
View File

@@ -120,7 +120,7 @@ class RobertaTokenizer(GPT2Tokenizer):
special tokens for the model special tokens for the model
Returns: Returns:
A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
if token_ids_1 is not None: if token_ids_1 is not None:

2
transformers/tokenization_utils.py
View File

@@ -951,7 +951,7 @@ class PreTrainedTokenizer(object):
special tokens for the model special tokens for the model
Returns: Returns:
A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0)) return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0))

2
transformers/tokenization_xlm.py
View File

@@ -781,7 +781,7 @@ class XLMTokenizer(PreTrainedTokenizer):
special tokens for the model special tokens for the model
Returns: Returns:
A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:

2
transformers/tokenization_xlnet.py
View File

@@ -208,7 +208,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
special tokens for the model special tokens for the model
Returns: Returns:
A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens: