Fixed inconsistency in several fast tokenizers (#26561)

src/transformers/models/bert/tokenization_bert_fast.py

@@ -265,7 +265,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/convbert/tokenization_convbert_fast.py

@@ -159,7 +159,7 @@ class ConvBertTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/deprecated/retribert/tokenization_retribert_fast.py

@@ -164,7 +164,7 @@ class RetriBertTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/distilbert/tokenization_distilbert_fast.py

@@ -190,7 +190,7 @@ class DistilBertTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/electra/tokenization_electra_fast.py

@@ -192,7 +192,7 @@ class ElectraTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/funnel/tokenization_funnel_fast.py

@@ -212,7 +212,7 @@ class FunnelTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/layoutlm/tokenization_layoutlm_fast.py

@@ -166,7 +166,7 @@ class LayoutLMTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/lxmert/tokenization_lxmert_fast.py

@@ -152,7 +152,7 @@ class LxmertTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/mobilebert/tokenization_mobilebert_fast.py

@@ -150,7 +150,7 @@ class MobileBertTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/realm/tokenization_realm_fast.py

@@ -282,7 +282,7 @@ class RealmTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/roformer/tokenization_roformer_fast.py

@@ -163,7 +163,7 @@ class RoFormerTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

src/transformers/models/squeezebert/tokenization_squeezebert_fast.py

@@ -173,7 +173,7 @@ class SqueezeBertTokenizerFast(PreTrainedTokenizerFast):
         """
         output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
 
-        if token_ids_1:
+        if token_ids_1 is not None:
             output += token_ids_1 + [self.sep_token_id]
 
         return output

tests/test_tokenization_common.py

@@ -3209,9 +3209,17 @@ class TokenizerTesterMixin:
                 # output_p = tokenizer_p.build_inputs_with_special_tokens(input_simple, input_pair)
                 # self.assertEqual(output_p, output_r)
 
+                input_pairs = [
+                    ("", ""),
+                    ("", "This is a sample pair"),
+                    ("This is a sample input", ""),
+                    ("This is a sample input", "This is a sample pair"),
+                ]
+
+                for sample_input, sample_pair in input_pairs:
                     # Input tokens id
-                input_simple = tokenizer_p.encode("This is a sample input", add_special_tokens=False)
+                    input_simple = tokenizer_p.encode(sample_input, add_special_tokens=False)
-                input_pair = tokenizer_p.encode("This is a sample pair", add_special_tokens=False)
+                    input_pair = tokenizer_p.encode(sample_pair, add_special_tokens=False)
 
                     # Generate output
                     output_r = tokenizer_r.build_inputs_with_special_tokens(input_simple)