From b5d330d11820f4ac2cc8c909b1a6a77e0cd961e0 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 11 Nov 2019 10:15:14 -0500 Subject: [PATCH] Fix #1784 --- templates/adding_a_new_model/tokenization_xxx.py | 2 +- transformers/tokenization_bert.py | 2 +- transformers/tokenization_roberta.py | 2 +- transformers/tokenization_utils.py | 2 +- transformers/tokenization_xlm.py | 2 +- transformers/tokenization_xlnet.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/adding_a_new_model/tokenization_xxx.py b/templates/adding_a_new_model/tokenization_xxx.py index 1b1325aab5..3d6b4ad9df 100644 --- a/templates/adding_a_new_model/tokenization_xxx.py +++ b/templates/adding_a_new_model/tokenization_xxx.py @@ -172,7 +172,7 @@ class XxxTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index 8affdd9036..ded5072e58 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -220,7 +220,7 @@ class BertTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py index df3e12bc7c..b44e004997 100644 --- a/transformers/tokenization_roberta.py +++ b/transformers/tokenization_roberta.py @@ -120,7 +120,7 @@ class RobertaTokenizer(GPT2Tokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: if token_ids_1 is not None: diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index ac765165e2..cd14cc4582 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -951,7 +951,7 @@ class PreTrainedTokenizer(object): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0)) diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index d09ce6b9dc..01f8721d98 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -781,7 +781,7 @@ class XLMTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index deae8de336..a4f1a6e3ba 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -208,7 +208,7 @@ class XLNetTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: