Feature to use the PreTrainedTokenizerFast class as a stand-alone tokenizer (#11810)

* feature for tokenizer without slow/legacy version

* format

* modify common test

* add tests

* add PreTrainedTokenizerFast to AutoTokenizer

* format

* change tokenizer common test in order to be able to run test without a slow version

* update tokenizer fast test in order to use `rust_tokenizer_class` attribute instead of `tokenizer_class`

* add autokenizer test

* replace  `if self.tokenizer_class is not None` with ` if self.tokenizer_class is None`

* remove obsolete change in comment

* Update src/transformers/tokenization_utils_base.py

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

* Update src/transformers/tokenization_utils_fast.py

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

* change `get_main_tokenizer` into `get_tokenizers`

* clarify `get_tokenizers` method

* homogenize with `test_slow_tokenizer` and `test_rust_tokenizer`

* add `test_rust_tokenizer = False` to tokenizer which don't define a fast version

* `test_rust_tokenizer = False` for BertJapaneseTokenizer

* `test_rust_tokenizer = False` for BertJapaneseCharacterTokenizationTest

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>
Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

src/transformers/models/auto/tokenization_auto.py

@@ -157,6 +157,7 @@ else:
     Speech2TextTokenizer = None
 
 if is_tokenizers_available():
+    from ...tokenization_utils_fast import PreTrainedTokenizerFast
     from ..albert.tokenization_albert_fast import AlbertTokenizerFast
     from ..bart.tokenization_bart_fast import BartTokenizerFast
     from ..barthez.tokenization_barthez_fast import BarthezTokenizerFast
@@ -223,6 +224,7 @@ else:
     T5TokenizerFast = None
     XLMRobertaTokenizerFast = None
     XLNetTokenizerFast = None
+    PreTrainedTokenizerFast = None
 
 
 logger = logging.get_logger(__name__)
@@ -297,6 +299,7 @@ NO_CONFIG_TOKENIZER = [
     BarthezTokenizerFast,
     MBart50Tokenizer,
     MBart50TokenizerFast,
+    PreTrainedTokenizerFast,
 ]
 
 

src/transformers/tokenization_utils_base.py

@@ -1872,14 +1872,15 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
             save_directory (:obj:`str` or :obj:`os.PathLike`): The path to a directory where the tokenizer will be saved.
             legacy_format (:obj:`bool`, `optional`):
                 Only applicable for a fast tokenizer. If unset (default), will save the tokenizer in the unified JSON
-                format as well as in legacy format, i.e. with tokenizer specific vocabulary and a separate added_tokens
+                format as well as in legacy format if it exists, i.e. with tokenizer specific vocabulary and a separate
-                files.
+                added_tokens files.
 
                 If :obj:`False`, will only save the tokenizer in the unified JSON format. This format is incompatible
                 with "slow" tokenizers (not powered by the `tokenizers` library), so the tokenizer will not be able to
                 be loaded in the corresponding "slow" tokenizer.
 
-                If :obj:`True`, will save the tokenizer in legacy format.
+                If :obj:`True`, will save the tokenizer in legacy format. If the "slow" tokenizer doesn't exits, a
+                value error is raised.
             filename_prefix: (:obj:`str`, `optional`):
                 A prefix to add to the names of the files saved by the tokenizer.
 

src/transformers/tokenization_utils_fast.py

@@ -525,7 +525,13 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
         """
         save_directory = str(save_directory)
 
-        save_slow = legacy_format is None or legacy_format is True
+        if self.slow_tokenizer_class is None and legacy_format is True:
+            raise ValueError(
+                "Your tokenizer does not have a legacy version defined and therefore cannot register this version. You "
+                "might consider leaving the legacy_format at `None` or setting it to `False`."
+            )
+
+        save_slow = (legacy_format is None or legacy_format is True) and self.slow_tokenizer_class is not None
         save_fast = legacy_format is None or legacy_format is False
 
         if save_slow:

tests/test_tokenization_auto.py

@@ -24,6 +24,7 @@ from transformers import (
     BertTokenizerFast,
     GPT2Tokenizer,
     GPT2TokenizerFast,
+    PreTrainedTokenizerFast,
     RobertaTokenizer,
     RobertaTokenizerFast,
 )
@@ -119,3 +120,12 @@ class AutoTokenizerTest(unittest.TestCase):
         tokenizer = AutoTokenizer.from_pretrained("microsoft/mpnet-base", do_lower_case=False)
         tokens = tokenizer.tokenize(sample)
         self.assertEqual("[UNK]", tokens[0])
+
+    @require_tokenizers
+    def test_PreTrainedTokenizerFast_from_pretrained(self):
+        tokenizer = AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config")
+        self.assertEqual(type(tokenizer), PreTrainedTokenizerFast)
+        self.assertEqual(tokenizer.model_max_length, 512)
+        self.assertEqual(tokenizer.vocab_size, 30000)
+        self.assertEqual(tokenizer.unk_token, "[UNK]")
+        self.assertEqual(tokenizer.padding_side, "right")

tests/test_tokenization_bert_generation.py

@@ -32,6 +32,7 @@ SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixture
 class BertGenerationTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = BertGenerationTokenizer
+    test_rust_tokenizer = False
     test_sentencepiece = True
 
     def setUp(self):

tests/test_tokenization_bert_japanese.py

@@ -35,6 +35,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = BertJapaneseTokenizer
+    test_rust_tokenizer = False
     space_between_special_tokens = True
 
     def setUp(self):
@@ -204,6 +205,7 @@ class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 class BertJapaneseCharacterTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = BertJapaneseTokenizer
+    test_rust_tokenizer = False
 
     def setUp(self):
         super().setUp()

tests/test_tokenization_bertweet.py

@@ -24,6 +24,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 class BertweetTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = BertweetTokenizer
+    test_rust_tokenizer = False
 
     def setUp(self):
         super().setUp()

tests/test_tokenization_clip.py

@@ -30,6 +30,7 @@ class CLIPTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = CLIPTokenizer
     rust_tokenizer_class = CLIPTokenizerFast
+    test_rust_tokenizer = False
     from_pretrained_kwargs = {"add_prefix_space": True}
     test_seq2seq = False
 

tests/test_tokenization_common.py

@@ -94,7 +94,8 @@ class TokenizerTesterMixin:
 
     tokenizer_class = None
     rust_tokenizer_class = None
-    test_rust_tokenizer = False
+    test_slow_tokenizer = True
+    test_rust_tokenizer = True
     space_between_special_tokens = False
     from_pretrained_kwargs = None
     from_pretrained_filter = None
@@ -165,9 +166,14 @@ class TokenizerTesterMixin:
         return output_txt, output_ids
 
     def get_tokenizers(self, fast=True, **kwargs) -> List[PreTrainedTokenizerBase]:
-        if fast and self.test_rust_tokenizer:
+        if fast and self.test_rust_tokenizer and self.test_slow_tokenizer:
             return [self.get_tokenizer(**kwargs), self.get_rust_tokenizer(**kwargs)]
-        return [self.get_tokenizer(**kwargs)]
+        elif fast and self.test_rust_tokenizer:
+            return [self.get_rust_tokenizer(**kwargs)]
+        elif self.test_slow_tokenizer:
+            return [self.get_tokenizer(**kwargs)]
+        else:
+            raise ValueError("This tokenizer class has no tokenizer to be tested.")
 
     def get_tokenizer(self, **kwargs) -> PreTrainedTokenizer:
         return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs)
@@ -366,6 +372,9 @@ class TokenizerTesterMixin:
         self.assertIsNone(signature.parameters["tokenizer_file"].default)
 
     def test_tokenizer_slow_store_full_signature(self):
+        if not self.test_slow_tokenizer:
+            return
+
         signature = inspect.signature(self.tokenizer_class.__init__)
         tokenizer = self.get_tokenizer()
 
@@ -388,6 +397,10 @@ class TokenizerTesterMixin:
         if not self.test_rust_tokenizer:
             return
 
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer()
 
@@ -559,8 +572,8 @@ class TokenizerTesterMixin:
         self.assertEqual(tok1.__getstate__(), tok2.__getstate__())
 
     def test_added_tokens_do_lower_case(self):
-        # TODO(thom) activate fast tokenizer tests once Rust tokenizers accepts white spaces in added tokens
+        # TODO(thom) activate fast tokenizer tests once Rust tokenizers accepts white spaces in added tokens.
-        tokenizers = self.get_tokenizers(fast=False, do_lower_case=True)
+        tokenizers = [self.get_tokenizer(do_lower_case=True)] if self.test_slow_tokenizer else []
         for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                 if not hasattr(tokenizer, "do_lower_case") or not tokenizer.do_lower_case:
@@ -594,7 +607,7 @@ class TokenizerTesterMixin:
                 for special_token in tokenizer.all_special_tokens:
                     self.assertTrue(special_token in tokenized_sequence)
 
-        tokenizers = self.get_tokenizers(fast=False, do_lower_case=False)
+        tokenizers = [self.get_tokenizer(do_lower_case=True)] if self.test_slow_tokenizer else []
         for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                 if hasattr(tokenizer, "do_lower_case") and tokenizer.do_lower_case:
@@ -750,7 +763,7 @@ class TokenizerTesterMixin:
             self.assertListEqual(weights_list, weights_list_2)
 
     def test_mask_output(self):
-        tokenizers = self.get_tokenizers(fast=False, do_lower_case=False)
+        tokenizers = self.get_tokenizers(do_lower_case=False)
         for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
 
@@ -1480,11 +1493,14 @@ class TokenizerTesterMixin:
         # This tests that tokenizers don't impact others. Unfortunately the case where it fails is when
         # we're loading an S3 configuration from a pre-trained identifier, and we have no way of testing those today.
 
-        tokenizer = self.get_tokenizer(random_argument=True)
+        tokenizers = self.get_tokenizers(random_argument=True)
-        assert tokenizer.init_kwargs["random_argument"] is True
+        new_tokenizers = self.get_tokenizers(random_argument=False)
-        new_tokenizer = self.get_tokenizer(random_argument=False)
+
-        assert tokenizer.init_kwargs["random_argument"] is True
+        for tokenizer, new_tokenizer in zip(tokenizers, new_tokenizers):
-        assert new_tokenizer.init_kwargs["random_argument"] is False
+            with self.subTest(f"{tokenizer.__class__.__name__}"):
+                assert tokenizer.init_kwargs["random_argument"] is True
+                assert tokenizer.init_kwargs["random_argument"] is True
+                assert new_tokenizer.init_kwargs["random_argument"] is False
 
     def test_get_vocab(self):
         tokenizers = self.get_tokenizers(do_lower_case=False)
@@ -1992,96 +2008,104 @@ class TokenizerTesterMixin:
 
         MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(MODEL_MAPPING, TOKENIZER_MAPPING)
 
-        tokenizer = self.get_tokenizer()
+        tokenizers = self.get_tokenizers()
-        if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING:
+        for tokenizer in tokenizers:
-            return
+            with self.subTest(f"{tokenizer.__class__.__name__}"):
+                if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING:
+                    return
 
-        config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__]
+                config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__]
-        config = config_class()
+                config = config_class()
 
-        if config.is_encoder_decoder or config.pad_token_id is None:
+                if config.is_encoder_decoder or config.pad_token_id is None:
-            return
+                    return
 
-        # Build sequence
+                # Build sequence
-        first_ten_tokens = list(tokenizer.get_vocab().keys())[:10]
+                first_ten_tokens = list(tokenizer.get_vocab().keys())[:10]
-        sequence = " ".join(first_ten_tokens)
+                sequence = " ".join(first_ten_tokens)
-        encoded_sequence = tokenizer.encode_plus(sequence, return_tensors="np")
+                encoded_sequence = tokenizer.encode_plus(sequence, return_tensors="np")
-        batch_encoded_sequence = tokenizer.batch_encode_plus([sequence, sequence], return_tensors="np")
+                batch_encoded_sequence = tokenizer.batch_encode_plus([sequence, sequence], return_tensors="np")
 
-        # TODO: add forward through JAX/Flax when PR is merged
+                # TODO: add forward through JAX/Flax when PR is merged
-        # This is currently here to make flake8 happy !
+                # This is currently here to make flake8 happy !
-        if encoded_sequence is None:
+                if encoded_sequence is None:
-            raise ValueError("Cannot convert list to numpy tensor on  encode_plus()")
+                    raise ValueError("Cannot convert list to numpy tensor on  encode_plus()")
 
-        if batch_encoded_sequence is None:
+                if batch_encoded_sequence is None:
-            raise ValueError("Cannot convert list to numpy tensor on  batch_encode_plus()")
+                    raise ValueError("Cannot convert list to numpy tensor on  batch_encode_plus()")
 
-        if self.test_rust_tokenizer:
+                if self.test_rust_tokenizer:
-            fast_tokenizer = self.get_rust_tokenizer()
+                    fast_tokenizer = self.get_rust_tokenizer()
-            encoded_sequence_fast = fast_tokenizer.encode_plus(sequence, return_tensors="np")
+                    encoded_sequence_fast = fast_tokenizer.encode_plus(sequence, return_tensors="np")
-            batch_encoded_sequence_fast = fast_tokenizer.batch_encode_plus([sequence, sequence], return_tensors="np")
+                    batch_encoded_sequence_fast = fast_tokenizer.batch_encode_plus(
+                        [sequence, sequence], return_tensors="np"
+                    )
 
-            # TODO: add forward through JAX/Flax when PR is merged
+                    # TODO: add forward through JAX/Flax when PR is merged
-            # This is currently here to make flake8 happy !
+                    # This is currently here to make flake8 happy !
-            if encoded_sequence_fast is None:
+                    if encoded_sequence_fast is None:
-                raise ValueError("Cannot convert list to numpy tensor on  encode_plus() (fast)")
+                        raise ValueError("Cannot convert list to numpy tensor on  encode_plus() (fast)")
 
-            if batch_encoded_sequence_fast is None:
+                    if batch_encoded_sequence_fast is None:
-                raise ValueError("Cannot convert list to numpy tensor on  batch_encode_plus() (fast)")
+                        raise ValueError("Cannot convert list to numpy tensor on  batch_encode_plus() (fast)")
 
     @require_torch
     def test_prepare_seq2seq_batch(self):
         if not self.test_seq2seq:
             return
 
-        tokenizer = self.get_tokenizer()
+        tokenizers = self.get_tokenizers()
+        for tokenizer in tokenizers:
+            with self.subTest(f"{tokenizer.__class__.__name__}"):
+                # Longer text that will definitely require truncation.
+                src_text = [
+                    " UN Chief Says There Is No Military Solution in Syria",
+                    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
+                ]
+                tgt_text = [
+                    "Şeful ONU declară că nu există o soluţie militară în Siria",
+                    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei "
+                    'pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu '
+                    "vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
+                ]
+                try:
+                    batch = tokenizer.prepare_seq2seq_batch(
+                        src_texts=src_text,
+                        tgt_texts=tgt_text,
+                        max_length=3,
+                        max_target_length=10,
+                        return_tensors="pt",
+                        src_lang="en_XX",  # this should be ignored (for all but mbart) but not cause an error
+                    )
+                except NotImplementedError:
+                    return
+                self.assertEqual(batch.input_ids.shape[1], 3)
+                self.assertEqual(batch.labels.shape[1], 10)
+                # max_target_length will default to max_length if not specified
+                batch = tokenizer.prepare_seq2seq_batch(
+                    src_text, tgt_texts=tgt_text, max_length=3, return_tensors="pt"
+                )
+                self.assertEqual(batch.input_ids.shape[1], 3)
+                self.assertEqual(batch.labels.shape[1], 3)
 
-        # Longer text that will definitely require truncation.
+                batch_encoder_only = tokenizer.prepare_seq2seq_batch(
-        src_text = [
+                    src_texts=src_text, max_length=3, max_target_length=10, return_tensors="pt"
-            " UN Chief Says There Is No Military Solution in Syria",
+                )
-            " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
+                self.assertEqual(batch_encoder_only.input_ids.shape[1], 3)
-        ]
+                self.assertEqual(batch_encoder_only.attention_mask.shape[1], 3)
-        tgt_text = [
+                self.assertNotIn("decoder_input_ids", batch_encoder_only)
-            "Şeful ONU declară că nu există o soluţie militară în Siria",
-            "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei "
-            'pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu '
-            "vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
-        ]
-        try:
-            batch = tokenizer.prepare_seq2seq_batch(
-                src_texts=src_text,
-                tgt_texts=tgt_text,
-                max_length=3,
-                max_target_length=10,
-                return_tensors="pt",
-                src_lang="en_XX",  # this should be ignored (for all but mbart) but not cause an error
-            )
-        except NotImplementedError:
-            return
-        self.assertEqual(batch.input_ids.shape[1], 3)
-        self.assertEqual(batch.labels.shape[1], 10)
-        # max_target_length will default to max_length if not specified
-        batch = tokenizer.prepare_seq2seq_batch(src_text, tgt_texts=tgt_text, max_length=3, return_tensors="pt")
-        self.assertEqual(batch.input_ids.shape[1], 3)
-        self.assertEqual(batch.labels.shape[1], 3)
-
-        batch_encoder_only = tokenizer.prepare_seq2seq_batch(
-            src_texts=src_text, max_length=3, max_target_length=10, return_tensors="pt"
-        )
-        self.assertEqual(batch_encoder_only.input_ids.shape[1], 3)
-        self.assertEqual(batch_encoder_only.attention_mask.shape[1], 3)
-        self.assertNotIn("decoder_input_ids", batch_encoder_only)
 
     def test_is_fast(self):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
-                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)
-
                 # Check is_fast is set correctly
-                self.assertFalse(tokenizer_p.is_fast)
                 self.assertTrue(tokenizer_r.is_fast)
 
+                if self.test_slow_tokenizer:
+                    tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)
+                    self.assertFalse(tokenizer_p.is_fast)
+
     def test_fast_only_inputs(self):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
@@ -2320,6 +2344,10 @@ class TokenizerTesterMixin:
                     self.assertIn(None, pair_batch_sequence_ids)
 
     def test_tokenization_python_rust_equals(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2357,6 +2385,10 @@ class TokenizerTesterMixin:
                     self.assertSequenceEqual(input_p[key], input_r[key][0])
 
     def test_num_special_tokens_to_add_equal(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2371,6 +2403,10 @@ class TokenizerTesterMixin:
                 )
 
     def test_max_length_equal(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2381,6 +2417,10 @@ class TokenizerTesterMixin:
                 self.assertEqual(tokenizer_r.max_len_sentences_pair, tokenizer_p.max_len_sentences_pair)
 
     def test_special_tokens_map_equal(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2515,6 +2555,10 @@ class TokenizerTesterMixin:
                     self.assertEqual(tokens[key].shape[-1], 6)
 
     def test_compare_pretokenized_inputs(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2593,6 +2637,10 @@ class TokenizerTesterMixin:
                     self.assertEqual(output_p[key], output_r[key])
 
     def test_create_token_type_ids(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2611,6 +2659,10 @@ class TokenizerTesterMixin:
                 self.assertEqual(output_p, output_r)
 
     def test_build_inputs_with_special_tokens(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2644,6 +2696,10 @@ class TokenizerTesterMixin:
                 self.assertEqual(output_p, output_r)
 
     def test_padding(self, max_length=50):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2862,6 +2918,10 @@ class TokenizerTesterMixin:
                 self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)
 
     def test_padding_different_model_input_name(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2896,6 +2956,10 @@ class TokenizerTesterMixin:
                 )
 
     def test_save_pretrained(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -2962,6 +3026,10 @@ class TokenizerTesterMixin:
                 shutil.rmtree(tmpdirname2)
 
     def test_embeded_special_tokens(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -3026,6 +3094,10 @@ class TokenizerTesterMixin:
                             self.assertEqual(len(i_no), len(i_with) - simple_num_special_tokens_to_add)
 
     def test_compare_prepare_for_model(self):
+        if not self.test_slow_tokenizer:
+            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
+            return
+
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
@@ -3049,24 +3121,28 @@ class TokenizerTesterMixin:
                 tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                     pretrained_name, additional_special_tokens=added_tokens, **kwargs
                 )
-                tokenizer_cr = self.rust_tokenizer_class.from_pretrained(
-                    pretrained_name, additional_special_tokens=added_tokens, **kwargs, from_slow=True
-                )
-                tokenizer_p = self.tokenizer_class.from_pretrained(
-                    pretrained_name, additional_special_tokens=added_tokens, **kwargs
-                )
-
-                p_output = tokenizer_p.encode("Hey this is a <special> token")
                 r_output = tokenizer_r.encode("Hey this is a <special> token")
-                cr_output = tokenizer_cr.encode("Hey this is a <special> token")
 
                 special_token_id = tokenizer_r.encode("<special>", add_special_tokens=False)[0]
 
-                self.assertEqual(p_output, r_output)
-                self.assertEqual(cr_output, r_output)
-                self.assertTrue(special_token_id in p_output)
                 self.assertTrue(special_token_id in r_output)
-                self.assertTrue(special_token_id in cr_output)
+
+                if self.test_slow_tokenizer:
+                    tokenizer_cr = self.rust_tokenizer_class.from_pretrained(
+                        pretrained_name, additional_special_tokens=added_tokens, **kwargs, from_slow=True
+                    )
+                    tokenizer_p = self.tokenizer_class.from_pretrained(
+                        pretrained_name, additional_special_tokens=added_tokens, **kwargs
+                    )
+
+                    p_output = tokenizer_p.encode("Hey this is a <special> token")
+
+                    cr_output = tokenizer_cr.encode("Hey this is a <special> token")
+
+                    self.assertEqual(p_output, r_output)
+                    self.assertEqual(cr_output, r_output)
+                    self.assertTrue(special_token_id in p_output)
+                    self.assertTrue(special_token_id in cr_output)
 
 
 @is_staging_test

tests/test_tokenization_fast.py

@@ -0,0 +1,53 @@
+# coding=utf-8
+# Copyright 2019 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from transformers import PreTrainedTokenizerFast
+from transformers.testing_utils import require_tokenizers
+
+from .test_tokenization_common import TokenizerTesterMixin
+
+
+@require_tokenizers
+class PreTrainedTokenizationFastTest(TokenizerTesterMixin, unittest.TestCase):
+    rust_tokenizer_class = PreTrainedTokenizerFast
+    test_slow_tokenizer = False
+    test_rust_tokenizer = True
+    from_pretrained_vocab_key = "tokenizer_file"
+
+    def setUp(self):
+        self.test_rust_tokenizer = False  # because we don't have pretrained_vocab_files_map
+        super().setUp()
+        self.test_rust_tokenizer = True
+
+        self.tokenizers_list = [(PreTrainedTokenizerFast, "robot-test/dummy-tokenizer-fast", {})]
+
+        tokenizer = PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-fast")
+        tokenizer.save_pretrained(self.tmpdirname)
+
+    def test_pretrained_model_lists(self):
+        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
+        # model
+        pass
+
+    def test_prepare_for_model(self):
+        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
+        # model
+        pass
+
+    def test_rust_tokenizer_signature(self):
+        # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
+        pass

tests/test_tokenization_fsmt.py

@@ -31,6 +31,7 @@ FSMT_TINY2 = "stas/tiny-wmt19-en-ru"
 
 class FSMTTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     tokenizer_class = FSMTTokenizer
+    test_rust_tokenizer = False
 
     def setUp(self):
         super().setUp()

tests/test_tokenization_luke.py

@@ -24,6 +24,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 
 class Luke(TokenizerTesterMixin, unittest.TestCase):
     tokenizer_class = LukeTokenizer
+    test_rust_tokenizer = False
     from_pretrained_kwargs = {"cls_token": "<s>"}
 
     def setUp(self):

tests/test_tokenization_phobert.py

@@ -24,6 +24,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 class PhobertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = PhobertTokenizer
+    test_rust_tokenizer = False
 
     def setUp(self):
         super().setUp()

tests/test_tokenization_small_blenderbot.py

@@ -29,6 +29,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 class BlenderbotSmallTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = BlenderbotSmallTokenizer
+    test_rust_tokenizer = False
 
     def setUp(self):
         super().setUp()