From 476ba679dd21c25aa6c7b57e6849835b9700db0c Mon Sep 17 00:00:00 2001 From: SaulLu <55560583+SaulLu@users.noreply.github.com> Date: Mon, 14 Jun 2021 11:58:44 +0200 Subject: [PATCH] Feature to use the PreTrainedTokenizerFast class as a stand-alone tokenizer (#11810) * feature for tokenizer without slow/legacy version * format * modify common test * add tests * add PreTrainedTokenizerFast to AutoTokenizer * format * change tokenizer common test in order to be able to run test without a slow version * update tokenizer fast test in order to use `rust_tokenizer_class` attribute instead of `tokenizer_class` * add autokenizer test * replace `if self.tokenizer_class is not None` with ` if self.tokenizer_class is None` * remove obsolete change in comment * Update src/transformers/tokenization_utils_base.py Co-authored-by: Lysandre Debut * Update src/transformers/tokenization_utils_fast.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * change `get_main_tokenizer` into `get_tokenizers` * clarify `get_tokenizers` method * homogenize with `test_slow_tokenizer` and `test_rust_tokenizer` * add `test_rust_tokenizer = False` to tokenizer which don't define a fast version * `test_rust_tokenizer = False` for BertJapaneseTokenizer * `test_rust_tokenizer = False` for BertJapaneseCharacterTokenizationTest Co-authored-by: Lysandre Debut Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- .../models/auto/tokenization_auto.py | 3 + src/transformers/tokenization_utils_base.py | 7 +- src/transformers/tokenization_utils_fast.py | 8 +- tests/test_tokenization_auto.py | 10 + tests/test_tokenization_bert_generation.py | 1 + tests/test_tokenization_bert_japanese.py | 2 + tests/test_tokenization_bertweet.py | 1 + tests/test_tokenization_clip.py | 1 + tests/test_tokenization_common.py | 260 +++++++++++------- tests/test_tokenization_fast.py | 53 ++++ tests/test_tokenization_fsmt.py | 1 + tests/test_tokenization_luke.py | 1 + tests/test_tokenization_phobert.py | 1 + tests/test_tokenization_small_blenderbot.py | 1 + 14 files changed, 254 insertions(+), 96 deletions(-) create mode 100644 tests/test_tokenization_fast.py diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 7434a7d663..f0fe4ea3a9 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -157,6 +157,7 @@ else: Speech2TextTokenizer = None if is_tokenizers_available(): + from ...tokenization_utils_fast import PreTrainedTokenizerFast from ..albert.tokenization_albert_fast import AlbertTokenizerFast from ..bart.tokenization_bart_fast import BartTokenizerFast from ..barthez.tokenization_barthez_fast import BarthezTokenizerFast @@ -223,6 +224,7 @@ else: T5TokenizerFast = None XLMRobertaTokenizerFast = None XLNetTokenizerFast = None + PreTrainedTokenizerFast = None logger = logging.get_logger(__name__) @@ -297,6 +299,7 @@ NO_CONFIG_TOKENIZER = [ BarthezTokenizerFast, MBart50Tokenizer, MBart50TokenizerFast, + PreTrainedTokenizerFast, ] diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 9e449fb2ef..4f3129e16d 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1872,14 +1872,15 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): save_directory (:obj:`str` or :obj:`os.PathLike`): The path to a directory where the tokenizer will be saved. legacy_format (:obj:`bool`, `optional`): Only applicable for a fast tokenizer. If unset (default), will save the tokenizer in the unified JSON - format as well as in legacy format, i.e. with tokenizer specific vocabulary and a separate added_tokens - files. + format as well as in legacy format if it exists, i.e. with tokenizer specific vocabulary and a separate + added_tokens files. If :obj:`False`, will only save the tokenizer in the unified JSON format. This format is incompatible with "slow" tokenizers (not powered by the `tokenizers` library), so the tokenizer will not be able to be loaded in the corresponding "slow" tokenizer. - If :obj:`True`, will save the tokenizer in legacy format. + If :obj:`True`, will save the tokenizer in legacy format. If the "slow" tokenizer doesn't exits, a + value error is raised. filename_prefix: (:obj:`str`, `optional`): A prefix to add to the names of the files saved by the tokenizer. diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py index c62ecdf82a..dbf6223cca 100644 --- a/src/transformers/tokenization_utils_fast.py +++ b/src/transformers/tokenization_utils_fast.py @@ -525,7 +525,13 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase): """ save_directory = str(save_directory) - save_slow = legacy_format is None or legacy_format is True + if self.slow_tokenizer_class is None and legacy_format is True: + raise ValueError( + "Your tokenizer does not have a legacy version defined and therefore cannot register this version. You " + "might consider leaving the legacy_format at `None` or setting it to `False`." + ) + + save_slow = (legacy_format is None or legacy_format is True) and self.slow_tokenizer_class is not None save_fast = legacy_format is None or legacy_format is False if save_slow: diff --git a/tests/test_tokenization_auto.py b/tests/test_tokenization_auto.py index 64c3e72eff..72db79d1c5 100644 --- a/tests/test_tokenization_auto.py +++ b/tests/test_tokenization_auto.py @@ -24,6 +24,7 @@ from transformers import ( BertTokenizerFast, GPT2Tokenizer, GPT2TokenizerFast, + PreTrainedTokenizerFast, RobertaTokenizer, RobertaTokenizerFast, ) @@ -119,3 +120,12 @@ class AutoTokenizerTest(unittest.TestCase): tokenizer = AutoTokenizer.from_pretrained("microsoft/mpnet-base", do_lower_case=False) tokens = tokenizer.tokenize(sample) self.assertEqual("[UNK]", tokens[0]) + + @require_tokenizers + def test_PreTrainedTokenizerFast_from_pretrained(self): + tokenizer = AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config") + self.assertEqual(type(tokenizer), PreTrainedTokenizerFast) + self.assertEqual(tokenizer.model_max_length, 512) + self.assertEqual(tokenizer.vocab_size, 30000) + self.assertEqual(tokenizer.unk_token, "[UNK]") + self.assertEqual(tokenizer.padding_side, "right") diff --git a/tests/test_tokenization_bert_generation.py b/tests/test_tokenization_bert_generation.py index 7a2767b610..40d3f1bae8 100644 --- a/tests/test_tokenization_bert_generation.py +++ b/tests/test_tokenization_bert_generation.py @@ -32,6 +32,7 @@ SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixture class BertGenerationTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = BertGenerationTokenizer + test_rust_tokenizer = False test_sentencepiece = True def setUp(self): diff --git a/tests/test_tokenization_bert_japanese.py b/tests/test_tokenization_bert_japanese.py index 2fcd841fef..b42a14314a 100644 --- a/tests/test_tokenization_bert_japanese.py +++ b/tests/test_tokenization_bert_japanese.py @@ -35,6 +35,7 @@ from .test_tokenization_common import TokenizerTesterMixin class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = BertJapaneseTokenizer + test_rust_tokenizer = False space_between_special_tokens = True def setUp(self): @@ -204,6 +205,7 @@ class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase): class BertJapaneseCharacterTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = BertJapaneseTokenizer + test_rust_tokenizer = False def setUp(self): super().setUp() diff --git a/tests/test_tokenization_bertweet.py b/tests/test_tokenization_bertweet.py index 14d926e094..bf7d5c7798 100644 --- a/tests/test_tokenization_bertweet.py +++ b/tests/test_tokenization_bertweet.py @@ -24,6 +24,7 @@ from .test_tokenization_common import TokenizerTesterMixin class BertweetTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = BertweetTokenizer + test_rust_tokenizer = False def setUp(self): super().setUp() diff --git a/tests/test_tokenization_clip.py b/tests/test_tokenization_clip.py index f7911d0f25..2f5ab7bd4a 100644 --- a/tests/test_tokenization_clip.py +++ b/tests/test_tokenization_clip.py @@ -30,6 +30,7 @@ class CLIPTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = CLIPTokenizer rust_tokenizer_class = CLIPTokenizerFast + test_rust_tokenizer = False from_pretrained_kwargs = {"add_prefix_space": True} test_seq2seq = False diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 7abf5bef26..5171b88d3b 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -94,7 +94,8 @@ class TokenizerTesterMixin: tokenizer_class = None rust_tokenizer_class = None - test_rust_tokenizer = False + test_slow_tokenizer = True + test_rust_tokenizer = True space_between_special_tokens = False from_pretrained_kwargs = None from_pretrained_filter = None @@ -165,9 +166,14 @@ class TokenizerTesterMixin: return output_txt, output_ids def get_tokenizers(self, fast=True, **kwargs) -> List[PreTrainedTokenizerBase]: - if fast and self.test_rust_tokenizer: + if fast and self.test_rust_tokenizer and self.test_slow_tokenizer: return [self.get_tokenizer(**kwargs), self.get_rust_tokenizer(**kwargs)] - return [self.get_tokenizer(**kwargs)] + elif fast and self.test_rust_tokenizer: + return [self.get_rust_tokenizer(**kwargs)] + elif self.test_slow_tokenizer: + return [self.get_tokenizer(**kwargs)] + else: + raise ValueError("This tokenizer class has no tokenizer to be tested.") def get_tokenizer(self, **kwargs) -> PreTrainedTokenizer: return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs) @@ -366,6 +372,9 @@ class TokenizerTesterMixin: self.assertIsNone(signature.parameters["tokenizer_file"].default) def test_tokenizer_slow_store_full_signature(self): + if not self.test_slow_tokenizer: + return + signature = inspect.signature(self.tokenizer_class.__init__) tokenizer = self.get_tokenizer() @@ -388,6 +397,10 @@ class TokenizerTesterMixin: if not self.test_rust_tokenizer: return + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + tokenizer = self.get_tokenizer() rust_tokenizer = self.get_rust_tokenizer() @@ -559,8 +572,8 @@ class TokenizerTesterMixin: self.assertEqual(tok1.__getstate__(), tok2.__getstate__()) def test_added_tokens_do_lower_case(self): - # TODO(thom) activate fast tokenizer tests once Rust tokenizers accepts white spaces in added tokens - tokenizers = self.get_tokenizers(fast=False, do_lower_case=True) + # TODO(thom) activate fast tokenizer tests once Rust tokenizers accepts white spaces in added tokens. + tokenizers = [self.get_tokenizer(do_lower_case=True)] if self.test_slow_tokenizer else [] for tokenizer in tokenizers: with self.subTest(f"{tokenizer.__class__.__name__}"): if not hasattr(tokenizer, "do_lower_case") or not tokenizer.do_lower_case: @@ -594,7 +607,7 @@ class TokenizerTesterMixin: for special_token in tokenizer.all_special_tokens: self.assertTrue(special_token in tokenized_sequence) - tokenizers = self.get_tokenizers(fast=False, do_lower_case=False) + tokenizers = [self.get_tokenizer(do_lower_case=True)] if self.test_slow_tokenizer else [] for tokenizer in tokenizers: with self.subTest(f"{tokenizer.__class__.__name__}"): if hasattr(tokenizer, "do_lower_case") and tokenizer.do_lower_case: @@ -750,7 +763,7 @@ class TokenizerTesterMixin: self.assertListEqual(weights_list, weights_list_2) def test_mask_output(self): - tokenizers = self.get_tokenizers(fast=False, do_lower_case=False) + tokenizers = self.get_tokenizers(do_lower_case=False) for tokenizer in tokenizers: with self.subTest(f"{tokenizer.__class__.__name__}"): @@ -1480,11 +1493,14 @@ class TokenizerTesterMixin: # This tests that tokenizers don't impact others. Unfortunately the case where it fails is when # we're loading an S3 configuration from a pre-trained identifier, and we have no way of testing those today. - tokenizer = self.get_tokenizer(random_argument=True) - assert tokenizer.init_kwargs["random_argument"] is True - new_tokenizer = self.get_tokenizer(random_argument=False) - assert tokenizer.init_kwargs["random_argument"] is True - assert new_tokenizer.init_kwargs["random_argument"] is False + tokenizers = self.get_tokenizers(random_argument=True) + new_tokenizers = self.get_tokenizers(random_argument=False) + + for tokenizer, new_tokenizer in zip(tokenizers, new_tokenizers): + with self.subTest(f"{tokenizer.__class__.__name__}"): + assert tokenizer.init_kwargs["random_argument"] is True + assert tokenizer.init_kwargs["random_argument"] is True + assert new_tokenizer.init_kwargs["random_argument"] is False def test_get_vocab(self): tokenizers = self.get_tokenizers(do_lower_case=False) @@ -1992,96 +2008,104 @@ class TokenizerTesterMixin: MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(MODEL_MAPPING, TOKENIZER_MAPPING) - tokenizer = self.get_tokenizer() - if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING: - return + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING: + return - config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__] - config = config_class() + config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__] + config = config_class() - if config.is_encoder_decoder or config.pad_token_id is None: - return + if config.is_encoder_decoder or config.pad_token_id is None: + return - # Build sequence - first_ten_tokens = list(tokenizer.get_vocab().keys())[:10] - sequence = " ".join(first_ten_tokens) - encoded_sequence = tokenizer.encode_plus(sequence, return_tensors="np") - batch_encoded_sequence = tokenizer.batch_encode_plus([sequence, sequence], return_tensors="np") + # Build sequence + first_ten_tokens = list(tokenizer.get_vocab().keys())[:10] + sequence = " ".join(first_ten_tokens) + encoded_sequence = tokenizer.encode_plus(sequence, return_tensors="np") + batch_encoded_sequence = tokenizer.batch_encode_plus([sequence, sequence], return_tensors="np") - # TODO: add forward through JAX/Flax when PR is merged - # This is currently here to make flake8 happy ! - if encoded_sequence is None: - raise ValueError("Cannot convert list to numpy tensor on encode_plus()") + # TODO: add forward through JAX/Flax when PR is merged + # This is currently here to make flake8 happy ! + if encoded_sequence is None: + raise ValueError("Cannot convert list to numpy tensor on encode_plus()") - if batch_encoded_sequence is None: - raise ValueError("Cannot convert list to numpy tensor on batch_encode_plus()") + if batch_encoded_sequence is None: + raise ValueError("Cannot convert list to numpy tensor on batch_encode_plus()") - if self.test_rust_tokenizer: - fast_tokenizer = self.get_rust_tokenizer() - encoded_sequence_fast = fast_tokenizer.encode_plus(sequence, return_tensors="np") - batch_encoded_sequence_fast = fast_tokenizer.batch_encode_plus([sequence, sequence], return_tensors="np") + if self.test_rust_tokenizer: + fast_tokenizer = self.get_rust_tokenizer() + encoded_sequence_fast = fast_tokenizer.encode_plus(sequence, return_tensors="np") + batch_encoded_sequence_fast = fast_tokenizer.batch_encode_plus( + [sequence, sequence], return_tensors="np" + ) - # TODO: add forward through JAX/Flax when PR is merged - # This is currently here to make flake8 happy ! - if encoded_sequence_fast is None: - raise ValueError("Cannot convert list to numpy tensor on encode_plus() (fast)") + # TODO: add forward through JAX/Flax when PR is merged + # This is currently here to make flake8 happy ! + if encoded_sequence_fast is None: + raise ValueError("Cannot convert list to numpy tensor on encode_plus() (fast)") - if batch_encoded_sequence_fast is None: - raise ValueError("Cannot convert list to numpy tensor on batch_encode_plus() (fast)") + if batch_encoded_sequence_fast is None: + raise ValueError("Cannot convert list to numpy tensor on batch_encode_plus() (fast)") @require_torch def test_prepare_seq2seq_batch(self): if not self.test_seq2seq: return - tokenizer = self.get_tokenizer() + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + # Longer text that will definitely require truncation. + src_text = [ + " UN Chief Says There Is No Military Solution in Syria", + " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.", + ] + tgt_text = [ + "Şeful ONU declară că nu există o soluţie militară în Siria", + "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei " + 'pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu ' + "vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.", + ] + try: + batch = tokenizer.prepare_seq2seq_batch( + src_texts=src_text, + tgt_texts=tgt_text, + max_length=3, + max_target_length=10, + return_tensors="pt", + src_lang="en_XX", # this should be ignored (for all but mbart) but not cause an error + ) + except NotImplementedError: + return + self.assertEqual(batch.input_ids.shape[1], 3) + self.assertEqual(batch.labels.shape[1], 10) + # max_target_length will default to max_length if not specified + batch = tokenizer.prepare_seq2seq_batch( + src_text, tgt_texts=tgt_text, max_length=3, return_tensors="pt" + ) + self.assertEqual(batch.input_ids.shape[1], 3) + self.assertEqual(batch.labels.shape[1], 3) - # Longer text that will definitely require truncation. - src_text = [ - " UN Chief Says There Is No Military Solution in Syria", - " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.", - ] - tgt_text = [ - "Şeful ONU declară că nu există o soluţie militară în Siria", - "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei " - 'pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu ' - "vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.", - ] - try: - batch = tokenizer.prepare_seq2seq_batch( - src_texts=src_text, - tgt_texts=tgt_text, - max_length=3, - max_target_length=10, - return_tensors="pt", - src_lang="en_XX", # this should be ignored (for all but mbart) but not cause an error - ) - except NotImplementedError: - return - self.assertEqual(batch.input_ids.shape[1], 3) - self.assertEqual(batch.labels.shape[1], 10) - # max_target_length will default to max_length if not specified - batch = tokenizer.prepare_seq2seq_batch(src_text, tgt_texts=tgt_text, max_length=3, return_tensors="pt") - self.assertEqual(batch.input_ids.shape[1], 3) - self.assertEqual(batch.labels.shape[1], 3) - - batch_encoder_only = tokenizer.prepare_seq2seq_batch( - src_texts=src_text, max_length=3, max_target_length=10, return_tensors="pt" - ) - self.assertEqual(batch_encoder_only.input_ids.shape[1], 3) - self.assertEqual(batch_encoder_only.attention_mask.shape[1], 3) - self.assertNotIn("decoder_input_ids", batch_encoder_only) + batch_encoder_only = tokenizer.prepare_seq2seq_batch( + src_texts=src_text, max_length=3, max_target_length=10, return_tensors="pt" + ) + self.assertEqual(batch_encoder_only.input_ids.shape[1], 3) + self.assertEqual(batch_encoder_only.attention_mask.shape[1], 3) + self.assertNotIn("decoder_input_ids", batch_encoder_only) def test_is_fast(self): for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) - tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs) - # Check is_fast is set correctly - self.assertFalse(tokenizer_p.is_fast) self.assertTrue(tokenizer_r.is_fast) + if self.test_slow_tokenizer: + tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs) + self.assertFalse(tokenizer_p.is_fast) + def test_fast_only_inputs(self): for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): @@ -2320,6 +2344,10 @@ class TokenizerTesterMixin: self.assertIn(None, pair_batch_sequence_ids) def test_tokenization_python_rust_equals(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2357,6 +2385,10 @@ class TokenizerTesterMixin: self.assertSequenceEqual(input_p[key], input_r[key][0]) def test_num_special_tokens_to_add_equal(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2371,6 +2403,10 @@ class TokenizerTesterMixin: ) def test_max_length_equal(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2381,6 +2417,10 @@ class TokenizerTesterMixin: self.assertEqual(tokenizer_r.max_len_sentences_pair, tokenizer_p.max_len_sentences_pair) def test_special_tokens_map_equal(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2515,6 +2555,10 @@ class TokenizerTesterMixin: self.assertEqual(tokens[key].shape[-1], 6) def test_compare_pretokenized_inputs(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2593,6 +2637,10 @@ class TokenizerTesterMixin: self.assertEqual(output_p[key], output_r[key]) def test_create_token_type_ids(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2611,6 +2659,10 @@ class TokenizerTesterMixin: self.assertEqual(output_p, output_r) def test_build_inputs_with_special_tokens(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2644,6 +2696,10 @@ class TokenizerTesterMixin: self.assertEqual(output_p, output_r) def test_padding(self, max_length=50): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2862,6 +2918,10 @@ class TokenizerTesterMixin: self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id) def test_padding_different_model_input_name(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2896,6 +2956,10 @@ class TokenizerTesterMixin: ) def test_save_pretrained(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -2962,6 +3026,10 @@ class TokenizerTesterMixin: shutil.rmtree(tmpdirname2) def test_embeded_special_tokens(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -3026,6 +3094,10 @@ class TokenizerTesterMixin: self.assertEqual(len(i_no), len(i_with) - simple_num_special_tokens_to_add) def test_compare_prepare_for_model(self): + if not self.test_slow_tokenizer: + # as we don't have a slow version, we can't compare the outputs between slow and fast versions + return + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) @@ -3049,24 +3121,28 @@ class TokenizerTesterMixin: tokenizer_r = self.rust_tokenizer_class.from_pretrained( pretrained_name, additional_special_tokens=added_tokens, **kwargs ) - tokenizer_cr = self.rust_tokenizer_class.from_pretrained( - pretrained_name, additional_special_tokens=added_tokens, **kwargs, from_slow=True - ) - tokenizer_p = self.tokenizer_class.from_pretrained( - pretrained_name, additional_special_tokens=added_tokens, **kwargs - ) - - p_output = tokenizer_p.encode("Hey this is a token") r_output = tokenizer_r.encode("Hey this is a token") - cr_output = tokenizer_cr.encode("Hey this is a token") special_token_id = tokenizer_r.encode("", add_special_tokens=False)[0] - self.assertEqual(p_output, r_output) - self.assertEqual(cr_output, r_output) - self.assertTrue(special_token_id in p_output) self.assertTrue(special_token_id in r_output) - self.assertTrue(special_token_id in cr_output) + + if self.test_slow_tokenizer: + tokenizer_cr = self.rust_tokenizer_class.from_pretrained( + pretrained_name, additional_special_tokens=added_tokens, **kwargs, from_slow=True + ) + tokenizer_p = self.tokenizer_class.from_pretrained( + pretrained_name, additional_special_tokens=added_tokens, **kwargs + ) + + p_output = tokenizer_p.encode("Hey this is a token") + + cr_output = tokenizer_cr.encode("Hey this is a token") + + self.assertEqual(p_output, r_output) + self.assertEqual(cr_output, r_output) + self.assertTrue(special_token_id in p_output) + self.assertTrue(special_token_id in cr_output) @is_staging_test diff --git a/tests/test_tokenization_fast.py b/tests/test_tokenization_fast.py new file mode 100644 index 0000000000..796a3f07c2 --- /dev/null +++ b/tests/test_tokenization_fast.py @@ -0,0 +1,53 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from transformers import PreTrainedTokenizerFast +from transformers.testing_utils import require_tokenizers + +from .test_tokenization_common import TokenizerTesterMixin + + +@require_tokenizers +class PreTrainedTokenizationFastTest(TokenizerTesterMixin, unittest.TestCase): + rust_tokenizer_class = PreTrainedTokenizerFast + test_slow_tokenizer = False + test_rust_tokenizer = True + from_pretrained_vocab_key = "tokenizer_file" + + def setUp(self): + self.test_rust_tokenizer = False # because we don't have pretrained_vocab_files_map + super().setUp() + self.test_rust_tokenizer = True + + self.tokenizers_list = [(PreTrainedTokenizerFast, "robot-test/dummy-tokenizer-fast", {})] + + tokenizer = PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-fast") + tokenizer.save_pretrained(self.tmpdirname) + + def test_pretrained_model_lists(self): + # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any + # model + pass + + def test_prepare_for_model(self): + # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any + # model + pass + + def test_rust_tokenizer_signature(self): + # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature + pass diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py index 276941f594..05c80ee3df 100644 --- a/tests/test_tokenization_fsmt.py +++ b/tests/test_tokenization_fsmt.py @@ -31,6 +31,7 @@ FSMT_TINY2 = "stas/tiny-wmt19-en-ru" class FSMTTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = FSMTTokenizer + test_rust_tokenizer = False def setUp(self): super().setUp() diff --git a/tests/test_tokenization_luke.py b/tests/test_tokenization_luke.py index ee5af69eef..84bf52a0f3 100644 --- a/tests/test_tokenization_luke.py +++ b/tests/test_tokenization_luke.py @@ -24,6 +24,7 @@ from .test_tokenization_common import TokenizerTesterMixin class Luke(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = LukeTokenizer + test_rust_tokenizer = False from_pretrained_kwargs = {"cls_token": ""} def setUp(self): diff --git a/tests/test_tokenization_phobert.py b/tests/test_tokenization_phobert.py index 1f7e88deeb..b5d42c8a24 100644 --- a/tests/test_tokenization_phobert.py +++ b/tests/test_tokenization_phobert.py @@ -24,6 +24,7 @@ from .test_tokenization_common import TokenizerTesterMixin class PhobertTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = PhobertTokenizer + test_rust_tokenizer = False def setUp(self): super().setUp() diff --git a/tests/test_tokenization_small_blenderbot.py b/tests/test_tokenization_small_blenderbot.py index e4ee8254e1..9169d21b43 100644 --- a/tests/test_tokenization_small_blenderbot.py +++ b/tests/test_tokenization_small_blenderbot.py @@ -29,6 +29,7 @@ from .test_tokenization_common import TokenizerTesterMixin class BlenderbotSmallTokenizerTest(TokenizerTesterMixin, unittest.TestCase): tokenizer_class = BlenderbotSmallTokenizer + test_rust_tokenizer = False def setUp(self): super().setUp()