From cd40cb8879de2bc5d3058edb47fc942484c45081 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Mon, 11 May 2020 15:05:36 -0400 Subject: [PATCH] Fix special token doc (#4292) --- src/transformers/tokenization_albert.py | 2 +- src/transformers/tokenization_bert.py | 2 +- src/transformers/tokenization_camembert.py | 2 +- src/transformers/tokenization_roberta.py | 2 +- src/transformers/tokenization_xlm.py | 2 +- src/transformers/tokenization_xlm_roberta.py | 2 +- src/transformers/tokenization_xlnet.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/transformers/tokenization_albert.py b/src/transformers/tokenization_albert.py index 953c250b3a..f33ce15f78 100644 --- a/src/transformers/tokenization_albert.py +++ b/src/transformers/tokenization_albert.py @@ -274,7 +274,7 @@ class AlbertTokenizer(PreTrainedTokenizer): Set to True if the token list is already formatted with special tokens for the model Returns: - :obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/src/transformers/tokenization_bert.py b/src/transformers/tokenization_bert.py index 376196c314..e28673fa0a 100644 --- a/src/transformers/tokenization_bert.py +++ b/src/transformers/tokenization_bert.py @@ -269,7 +269,7 @@ class BertTokenizer(PreTrainedTokenizer): Set to True if the token list is already formatted with special tokens for the model Returns: - :obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/src/transformers/tokenization_camembert.py b/src/transformers/tokenization_camembert.py index 07feb56c16..5b8fe7ab00 100644 --- a/src/transformers/tokenization_camembert.py +++ b/src/transformers/tokenization_camembert.py @@ -182,7 +182,7 @@ class CamembertTokenizer(PreTrainedTokenizer): Set to True if the token list is already formatted with special tokens for the model Returns: - :obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: if token_ids_1 is not None: diff --git a/src/transformers/tokenization_roberta.py b/src/transformers/tokenization_roberta.py index 6cc96dec4f..6e05f6e60c 100644 --- a/src/transformers/tokenization_roberta.py +++ b/src/transformers/tokenization_roberta.py @@ -193,7 +193,7 @@ class RobertaTokenizer(GPT2Tokenizer): Set to True if the token list is already formatted with special tokens for the model Returns: - :obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: if token_ids_1 is not None: diff --git a/src/transformers/tokenization_xlm.py b/src/transformers/tokenization_xlm.py index b0a5204bfb..0ca13344ac 100644 --- a/src/transformers/tokenization_xlm.py +++ b/src/transformers/tokenization_xlm.py @@ -893,7 +893,7 @@ class XLMTokenizer(PreTrainedTokenizer): Set to True if the token list is already formatted with special tokens for the model Returns: - :obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/src/transformers/tokenization_xlm_roberta.py b/src/transformers/tokenization_xlm_roberta.py index f5331ff166..f2f5f76c79 100644 --- a/src/transformers/tokenization_xlm_roberta.py +++ b/src/transformers/tokenization_xlm_roberta.py @@ -217,7 +217,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer): Set to True if the token list is already formatted with special tokens for the model Returns: - :obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/src/transformers/tokenization_xlnet.py b/src/transformers/tokenization_xlnet.py index 995b35ed8b..93ef2d2bb7 100644 --- a/src/transformers/tokenization_xlnet.py +++ b/src/transformers/tokenization_xlnet.py @@ -278,7 +278,7 @@ class XLNetTokenizer(PreTrainedTokenizer): Set to True if the token list is already formatted with special tokens for the model Returns: - :obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: