SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix special token doc (#4292)

Browse Source
This commit is contained in:
Lysandre Debut
2020-05-11 15:05:36 -04:00
committed by GitHub
parent 82601f4c1a
commit cd40cb8879
7 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_albert.py
View File

@@ -274,7 +274,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model
Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens:

2
src/transformers/tokenization_bert.py
View File

@@ -269,7 +269,7 @@ class BertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model
Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens:

2
src/transformers/tokenization_camembert.py
View File

@@ -182,7 +182,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model
Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens:
if token_ids_1 is not None:

2
src/transformers/tokenization_roberta.py
View File

@@ -193,7 +193,7 @@ class RobertaTokenizer(GPT2Tokenizer):
Set to True if the token list is already formatted with special tokens for the model
Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens:
if token_ids_1 is not None:

2
src/transformers/tokenization_xlm.py
View File

@@ -893,7 +893,7 @@ class XLMTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model
Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens:

2
src/transformers/tokenization_xlm_roberta.py
View File

@@ -217,7 +217,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model
Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens:

2
src/transformers/tokenization_xlnet.py
View File

@@ -278,7 +278,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model
Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens: