SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix special token doc (#4292)

Browse Source
This commit is contained in:
Lysandre Debut
2020-05-11 15:05:36 -04:00
committed by GitHub
parent 82601f4c1a
commit cd40cb8879
7 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_albert.py
View File

@@ -274,7 +274,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:

2
src/transformers/tokenization_bert.py
View File

@@ -269,7 +269,7 @@ class BertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:

2
src/transformers/tokenization_camembert.py
View File

@@ -182,7 +182,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
if token_ids_1 is not None: if token_ids_1 is not None:

2
src/transformers/tokenization_roberta.py
View File

@@ -193,7 +193,7 @@ class RobertaTokenizer(GPT2Tokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
if token_ids_1 is not None: if token_ids_1 is not None:

2
src/transformers/tokenization_xlm.py
View File

@@ -893,7 +893,7 @@ class XLMTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:

2
src/transformers/tokenization_xlm_roberta.py
View File

@@ -217,7 +217,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:

2
src/transformers/tokenization_xlnet.py
View File

@@ -278,7 +278,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens: