From 6c2ad00c4678d56aad4f34ee8d05e699f2ef10c5 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 14 Mar 2023 10:03:02 +0100 Subject: [PATCH] Move `is_pipeline_test_to_skip` to specific model test classes (#21999) * Move `is_pipeline_test_to_skip` to specific model test classes --------- Co-authored-by: ydshieh --- .../commands/add_new_model_like.py | 100 +++++++++++++++- tests/models/ctrl/test_modeling_ctrl.py | 12 ++ tests/models/ctrl/test_modeling_tf_ctrl.py | 12 ++ .../models/flaubert/test_modeling_flaubert.py | 20 ++++ .../flaubert/test_modeling_tf_flaubert.py | 20 ++++ tests/models/fnet/test_modeling_fnet.py | 9 ++ tests/models/gptj/test_modeling_gptj.py | 16 +++ tests/models/gptj/test_modeling_tf_gptj.py | 16 +++ .../models/layoutlm/test_modeling_layoutlm.py | 14 +++ .../layoutlmv2/test_modeling_layoutlmv2.py | 23 ++++ .../layoutlmv3/test_modeling_layoutlmv3.py | 6 + .../layoutlmv3/test_modeling_tf_layoutlmv3.py | 6 + tests/models/lilt/test_modeling_lilt.py | 6 + .../longformer/test_modeling_longformer.py | 16 +++ .../longformer/test_modeling_tf_longformer.py | 16 +++ tests/models/m2m_100/test_modeling_m2m_100.py | 11 ++ tests/models/mvp/test_modeling_mvp.py | 16 +++ tests/models/openai/test_modeling_openai.py | 12 ++ .../models/openai/test_modeling_tf_openai.py | 12 ++ tests/models/opt/test_modeling_opt.py | 16 +++ tests/models/plbart/test_modeling_plbart.py | 11 ++ .../prophetnet/test_modeling_prophetnet.py | 12 ++ .../models/reformer/test_modeling_reformer.py | 16 +++ .../models/roc_bert/test_modeling_roc_bert.py | 18 +++ .../roformer/test_modeling_tf_roformer.py | 9 ++ tests/models/tapas/test_modeling_tapas.py | 6 + tests/models/tapas/test_modeling_tf_tapas.py | 6 + .../transfo_xl/test_modeling_tf_transfo_xl.py | 12 ++ .../transfo_xl/test_modeling_transfo_xl.py | 12 ++ tests/models/xlm/test_modeling_tf_xlm.py | 20 ++++ tests/models/xlm/test_modeling_xlm.py | 20 ++++ tests/test_pipeline_mixin.py | 113 +----------------- 32 files changed, 506 insertions(+), 108 deletions(-) diff --git a/src/transformers/commands/add_new_model_like.py b/src/transformers/commands/add_new_model_like.py index 30d273bec5..0525ad2eb6 100644 --- a/src/transformers/commands/add_new_model_like.py +++ b/src/transformers/commands/add_new_model_like.py @@ -173,6 +173,56 @@ def parse_module_content(content: str) -> List[str]: return objects +def extract_block(content: str, indent_level: int = 0) -> str: + """Return the first block in `content` with the indent level `indent_level`. + + The first line in `content` should be indented at `indent_level` level, otherwise an error will be thrown. + + This method will immediately stop the search when a (non-empty) line with indent level less than `indent_level` is + encountered. + + Args: + content (`str`): The content to parse + indent_level (`int`, *optional*, default to 0): The indent level of the blocks to search for + + Returns: + `str`: The first block in `content` with the indent level `indent_level`. + """ + current_object = [] + lines = content.split("\n") + # Doc-styler takes everything between two triple quotes in docstrings, so we need a fake """ here to go with this. + end_markers = [")", "]", "}", '"""'] + + for idx, line in enumerate(lines): + if idx == 0 and indent_level > 0 and not is_empty_line(line) and find_indent(line) != indent_level: + raise ValueError( + f"When `indent_level > 0`, the first line in `content` should have indent level {indent_level}. Got " + f"{find_indent(line)} instead." + ) + + if find_indent(line) < indent_level and not is_empty_line(line): + break + + # End of an object + is_valid_object = len(current_object) > 0 + if ( + not is_empty_line(line) + and not line.endswith(":") + and find_indent(line) == indent_level + and is_valid_object + ): + # Closing parts should be included in current object + if line.lstrip() in end_markers: + current_object.append(line) + return "\n".join(current_object) + else: + current_object.append(line) + + # Add last object + if len(current_object) > 0: + return "\n".join(current_object) + + def add_content_to_text( text: str, content: str, @@ -402,12 +452,53 @@ SPECIAL_PATTERNS = { _re_class_func = re.compile(r"^(?:class|def)\s+([^\s:\(]+)\s*(?:\(|\:)", flags=re.MULTILINE) +def remove_attributes(obj, target_attr): + """Remove `target_attr` in `obj`.""" + lines = obj.split(os.linesep) + + target_idx = None + for idx, line in enumerate(lines): + # search for assignment + if line.lstrip().startswith(f"{target_attr} = "): + target_idx = idx + break + # search for function/method definition + elif line.lstrip().startswith(f"def {target_attr}("): + target_idx = idx + break + + # target not found + if target_idx is None: + return obj + + line = lines[target_idx] + indent_level = find_indent(line) + # forward pass to find the ending of the block (including empty lines) + parsed = extract_block("\n".join(lines[target_idx:]), indent_level) + num_lines = len(parsed.split("\n")) + for idx in range(num_lines): + lines[target_idx + idx] = None + + # backward pass to find comments or decorator + for idx in range(target_idx - 1, -1, -1): + line = lines[idx] + if (line.lstrip().startswith("#") or line.lstrip().startswith("@")) and find_indent(line) == indent_level: + lines[idx] = None + else: + break + + new_obj = os.linesep.join([x for x in lines if x is not None]) + + return new_obj + + def duplicate_module( module_file: Union[str, os.PathLike], old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns, dest_file: Optional[str] = None, add_copied_from: bool = True, + attrs_to_remove: List[str] = None, ): """ Create a new module from an existing one and adapting all function and classes names from old patterns to new ones. @@ -491,8 +582,14 @@ def duplicate_module( new_objects.append(obj) + content = "\n".join(new_objects) + # Remove some attributes that we don't want to copy to the new file(s) + if attrs_to_remove is not None: + for attr in attrs_to_remove: + content = remove_attributes(content, target_attr=attr) + with open(dest_file, "w", encoding="utf-8") as f: - content = f.write("\n".join(new_objects)) + f.write(content) def filter_framework_files( @@ -1294,6 +1391,7 @@ def create_new_model_like( new_model_patterns, dest_file=dest_file, add_copied_from=False, + attrs_to_remove=["pipeline_model_mapping", "is_pipeline_test_to_skip"], ) disabled_fx_test = disabled_fx_test | disable_fx_test(dest_file) diff --git a/tests/models/ctrl/test_modeling_ctrl.py b/tests/models/ctrl/test_modeling_ctrl.py index b42db21aa6..87d719853a 100644 --- a/tests/models/ctrl/test_modeling_ctrl.py +++ b/tests/models/ctrl/test_modeling_ctrl.py @@ -210,6 +210,18 @@ class CTRLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_resize_embeddings = False test_head_masking = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny + # config could not be created. + return True + + return False + def setUp(self): self.model_tester = CTRLModelTester(self) self.config_tester = ConfigTester(self, config_class=CTRLConfig, n_embd=37) diff --git a/tests/models/ctrl/test_modeling_tf_ctrl.py b/tests/models/ctrl/test_modeling_tf_ctrl.py index fb48ba93ee..c71c96bc9d 100644 --- a/tests/models/ctrl/test_modeling_tf_ctrl.py +++ b/tests/models/ctrl/test_modeling_tf_ctrl.py @@ -185,6 +185,18 @@ class TFCTRLModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCase test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny + # config could not be created. + return True + + return False + def setUp(self): self.model_tester = TFCTRLModelTester(self) self.config_tester = ConfigTester(self, config_class=CTRLConfig, n_embd=37) diff --git a/tests/models/flaubert/test_modeling_flaubert.py b/tests/models/flaubert/test_modeling_flaubert.py index 0ac3ced935..40a9f68f69 100644 --- a/tests/models/flaubert/test_modeling_flaubert.py +++ b/tests/models/flaubert/test_modeling_flaubert.py @@ -390,6 +390,26 @@ class FlaubertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `FlaubertConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + # Flaubert has 2 QA models -> need to manually set the correct labels for one of them here def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/flaubert/test_modeling_tf_flaubert.py b/tests/models/flaubert/test_modeling_tf_flaubert.py index 996c7323ee..1174002035 100644 --- a/tests/models/flaubert/test_modeling_tf_flaubert.py +++ b/tests/models/flaubert/test_modeling_tf_flaubert.py @@ -306,6 +306,26 @@ class TFFlaubertModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Test test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `FlaubertConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFFlaubertModelTester(self) self.config_tester = ConfigTester(self, config_class=FlaubertConfig, emb_dim=37) diff --git a/tests/models/fnet/test_modeling_fnet.py b/tests/models/fnet/test_modeling_fnet.py index 056d8237b1..836da26021 100644 --- a/tests/models/fnet/test_modeling_fnet.py +++ b/tests/models/fnet/test_modeling_fnet.py @@ -299,6 +299,15 @@ class FNetModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_head_masking = False test_pruning = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "QAPipelineTests": + return True + + return False + # special case for ForPreTraining model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/gptj/test_modeling_gptj.py b/tests/models/gptj/test_modeling_gptj.py index 067b84ab7d..5fe0fec391 100644 --- a/tests/models/gptj/test_modeling_gptj.py +++ b/tests/models/gptj/test_modeling_gptj.py @@ -385,6 +385,22 @@ class GPTJModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_model_parallel = False test_head_masking = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + # special case for DoubleHeads model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/gptj/test_modeling_tf_gptj.py b/tests/models/gptj/test_modeling_tf_gptj.py index c230429738..3aa63d2790 100644 --- a/tests/models/gptj/test_modeling_tf_gptj.py +++ b/tests/models/gptj/test_modeling_tf_gptj.py @@ -318,6 +318,22 @@ class TFGPTJModelTest(TFModelTesterMixin, TFCoreModelTesterMixin, PipelineTester test_missing_keys = False test_head_masking = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFGPTJModelTester(self) self.config_tester = ConfigTester(self, config_class=GPTJConfig, n_embd=37) diff --git a/tests/models/layoutlm/test_modeling_layoutlm.py b/tests/models/layoutlm/test_modeling_layoutlm.py index 0535fbf4e1..d2aad061c3 100644 --- a/tests/models/layoutlm/test_modeling_layoutlm.py +++ b/tests/models/layoutlm/test_modeling_layoutlm.py @@ -246,6 +246,20 @@ class LayoutLMModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase ) fx_compatible = True + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "DocumentQuestionAnsweringPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # This pipeline uses `sequence_ids()` which is only available for fast tokenizers. + return True + + return False + def setUp(self): self.model_tester = LayoutLMModelTester(self) self.config_tester = ConfigTester(self, config_class=LayoutLMConfig, hidden_size=37) diff --git a/tests/models/layoutlmv2/test_modeling_layoutlmv2.py b/tests/models/layoutlmv2/test_modeling_layoutlmv2.py index 812bff4aba..6c82a34a62 100644 --- a/tests/models/layoutlmv2/test_modeling_layoutlmv2.py +++ b/tests/models/layoutlmv2/test_modeling_layoutlmv2.py @@ -282,6 +282,29 @@ class LayoutLMv2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name in [ + "QAPipelineTests", + "TextClassificationPipelineTests", + "TokenClassificationPipelineTests", + "ZeroShotClassificationPipelineTests", + ]: + # `LayoutLMv2Config` was never used in pipeline tests (`test_pt_LayoutLMv2Config_XXX`) due to lack of tiny + # config. With new tiny model creation, it is available, but we need to fix the failed tests. + return True + elif ( + pipeline_test_casse_name == "DocumentQuestionAnsweringPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # This pipeline uses `sequence_ids()` which is only available for fast tokenizers. + return True + + return False + def setUp(self): self.model_tester = LayoutLMv2ModelTester(self) self.config_tester = ConfigTester(self, config_class=LayoutLMv2Config, hidden_size=37) diff --git a/tests/models/layoutlmv3/test_modeling_layoutlmv3.py b/tests/models/layoutlmv3/test_modeling_layoutlmv3.py index f62fff0048..5f1046d534 100644 --- a/tests/models/layoutlmv3/test_modeling_layoutlmv3.py +++ b/tests/models/layoutlmv3/test_modeling_layoutlmv3.py @@ -298,6 +298,12 @@ class LayoutLMv3ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def setUp(self): self.model_tester = LayoutLMv3ModelTester(self) self.config_tester = ConfigTester(self, config_class=LayoutLMv3Config, hidden_size=37) diff --git a/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py b/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py index 09ca417595..df103194ab 100644 --- a/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py +++ b/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py @@ -291,6 +291,12 @@ class TFLayoutLMv3ModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Te test_resize_embeddings = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False) -> dict: inputs_dict = copy.deepcopy(inputs_dict) diff --git a/tests/models/lilt/test_modeling_lilt.py b/tests/models/lilt/test_modeling_lilt.py index e78d5d13d4..1bb92300c3 100644 --- a/tests/models/lilt/test_modeling_lilt.py +++ b/tests/models/lilt/test_modeling_lilt.py @@ -244,6 +244,12 @@ class LiltModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin fx_compatible = False test_pruning = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def setUp(self): self.model_tester = LiltModelTester(self) self.config_tester = ConfigTester(self, config_class=LiltConfig, hidden_size=37) diff --git a/tests/models/longformer/test_modeling_longformer.py b/tests/models/longformer/test_modeling_longformer.py index 1dc97a114a..21853e4420 100644 --- a/tests/models/longformer/test_modeling_longformer.py +++ b/tests/models/longformer/test_modeling_longformer.py @@ -326,6 +326,22 @@ class LongformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = LongformerModelTester(self) self.config_tester = ConfigTester(self, config_class=LongformerConfig, hidden_size=37) diff --git a/tests/models/longformer/test_modeling_tf_longformer.py b/tests/models/longformer/test_modeling_tf_longformer.py index ae11946cb9..b5452bc80a 100644 --- a/tests/models/longformer/test_modeling_tf_longformer.py +++ b/tests/models/longformer/test_modeling_tf_longformer.py @@ -299,6 +299,22 @@ class TFLongformerModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Te test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFLongformerModelTester(self) self.config_tester = ConfigTester(self, config_class=LongformerConfig, hidden_size=37) diff --git a/tests/models/m2m_100/test_modeling_m2m_100.py b/tests/models/m2m_100/test_modeling_m2m_100.py index 2457eea21a..e37a023307 100644 --- a/tests/models/m2m_100/test_modeling_m2m_100.py +++ b/tests/models/m2m_100/test_modeling_m2m_100.py @@ -246,6 +246,17 @@ class M2M100ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TranslationPipelineTests": + # Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`. + # `M2M100Config` was never used in pipeline tests: cannot create a simple tokenizer. + return True + + return False + def setUp(self): self.model_tester = M2M100ModelTester(self) self.config_tester = ConfigTester(self, config_class=M2M100Config) diff --git a/tests/models/mvp/test_modeling_mvp.py b/tests/models/mvp/test_modeling_mvp.py index 5513f34b46..7ade831659 100644 --- a/tests/models/mvp/test_modeling_mvp.py +++ b/tests/models/mvp/test_modeling_mvp.py @@ -433,6 +433,22 @@ class MvpModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = MvpModelTester(self) self.config_tester = ConfigTester(self, config_class=MvpConfig) diff --git a/tests/models/openai/test_modeling_openai.py b/tests/models/openai/test_modeling_openai.py index a9d76d3770..0e8ba6d9ce 100644 --- a/tests/models/openai/test_modeling_openai.py +++ b/tests/models/openai/test_modeling_openai.py @@ -210,6 +210,18 @@ class OpenAIGPTModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a + # tiny config could not be created. + return True + + return False + # special case for DoubleHeads model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/openai/test_modeling_tf_openai.py b/tests/models/openai/test_modeling_tf_openai.py index f059487f21..a4cf71bf1a 100644 --- a/tests/models/openai/test_modeling_tf_openai.py +++ b/tests/models/openai/test_modeling_tf_openai.py @@ -214,6 +214,18 @@ class TFOpenAIGPTModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Tes test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a + # tiny config could not be created. + return True + + return False + def setUp(self): self.model_tester = TFOpenAIGPTModelTester(self) self.config_tester = ConfigTester(self, config_class=OpenAIGPTConfig, n_embd=37) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 4805f1c215..f8f217790c 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -207,6 +207,22 @@ class OPTModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = OPTModelTester(self) self.config_tester = ConfigTester(self, config_class=OPTConfig) diff --git a/tests/models/plbart/test_modeling_plbart.py b/tests/models/plbart/test_modeling_plbart.py index 28278f9a46..4c2b2b84ad 100644 --- a/tests/models/plbart/test_modeling_plbart.py +++ b/tests/models/plbart/test_modeling_plbart.py @@ -237,6 +237,17 @@ class PLBartModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TranslationPipelineTests": + # Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`. + # `PLBartConfig` was never used in pipeline tests: cannot create a simple tokenizer. + return True + + return False + def setUp(self): self.model_tester = PLBartModelTester(self) self.config_tester = ConfigTester(self, config_class=PLBartConfig) diff --git a/tests/models/prophetnet/test_modeling_prophetnet.py b/tests/models/prophetnet/test_modeling_prophetnet.py index 2579be5396..1d4b45e23c 100644 --- a/tests/models/prophetnet/test_modeling_prophetnet.py +++ b/tests/models/prophetnet/test_modeling_prophetnet.py @@ -904,6 +904,18 @@ class ProphetNetModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste test_resize_embeddings = False is_encoder_decoder = True + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `ProphetNetConfig` was never used in pipeline tests: cannot create a simple + # tokenizer. + return True + + return False + def setUp(self): self.model_tester = ProphetNetModelTester(self) self.config_tester = ConfigTester(self, config_class=ProphetNetConfig) diff --git a/tests/models/reformer/test_modeling_reformer.py b/tests/models/reformer/test_modeling_reformer.py index 4793f4ea43..a7f4f2f454 100644 --- a/tests/models/reformer/test_modeling_reformer.py +++ b/tests/models/reformer/test_modeling_reformer.py @@ -709,6 +709,22 @@ class ReformerLSHAttnModelTest( test_headmasking = False test_torchscript = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = ReformerModelTester( self, diff --git a/tests/models/roc_bert/test_modeling_roc_bert.py b/tests/models/roc_bert/test_modeling_roc_bert.py index afc0f63122..2efd9b7992 100644 --- a/tests/models/roc_bert/test_modeling_roc_bert.py +++ b/tests/models/roc_bert/test_modeling_roc_bert.py @@ -586,6 +586,24 @@ class RoCBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) else {} ) + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name in [ + "FillMaskPipelineTests", + "FeatureExtractionPipelineTests", + "TextClassificationPipelineTests", + "TokenClassificationPipelineTests", + ]: + # Get error: IndexError: index out of range in self. + # `word_shape_file` and `word_pronunciation_file` should be shrunk during tiny model creation, + # otherwise `IndexError` could occur in some embedding layers. Skip for now until this model has + # more usage. + return True + + return False + # special case for ForPreTraining model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/roformer/test_modeling_tf_roformer.py b/tests/models/roformer/test_modeling_tf_roformer.py index f5f3875db1..0a632e39a2 100644 --- a/tests/models/roformer/test_modeling_tf_roformer.py +++ b/tests/models/roformer/test_modeling_tf_roformer.py @@ -271,6 +271,15 @@ class TFRoFormerModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Test test_head_masking = False test_onnx = False + # TODO: add `prepare_inputs_for_generation` for `TFRoFormerForCausalLM` + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + return True + + return False + def setUp(self): self.model_tester = TFRoFormerModelTester(self) self.config_tester = ConfigTester(self, config_class=RoFormerConfig, hidden_size=37) diff --git a/tests/models/tapas/test_modeling_tapas.py b/tests/models/tapas/test_modeling_tapas.py index b3b9217a00..644307e3f9 100644 --- a/tests/models/tapas/test_modeling_tapas.py +++ b/tests/models/tapas/test_modeling_tapas.py @@ -486,6 +486,12 @@ class TapasModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): ) return inputs_dict + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def setUp(self): self.model_tester = TapasModelTester(self) self.config_tester = ConfigTester(self, config_class=TapasConfig, dim=37) diff --git a/tests/models/tapas/test_modeling_tf_tapas.py b/tests/models/tapas/test_modeling_tf_tapas.py index 380b7605b8..c3cc5fae3a 100644 --- a/tests/models/tapas/test_modeling_tf_tapas.py +++ b/tests/models/tapas/test_modeling_tf_tapas.py @@ -443,6 +443,12 @@ class TFTapasModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCas test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False) -> dict: inputs_dict = copy.deepcopy(inputs_dict) diff --git a/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py b/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py index 88bfbc1c7b..47880013b9 100644 --- a/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py +++ b/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py @@ -177,6 +177,18 @@ class TFTransfoXLModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Tes test_onnx = False test_mismatched_shapes = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple + # tokenizer. + return True + + return False + def setUp(self): self.model_tester = TFTransfoXLModelTester(self) self.config_tester = ConfigTester(self, config_class=TransfoXLConfig, d_embed=37) diff --git a/tests/models/transfo_xl/test_modeling_transfo_xl.py b/tests/models/transfo_xl/test_modeling_transfo_xl.py index df9647ae20..c86cd704f1 100644 --- a/tests/models/transfo_xl/test_modeling_transfo_xl.py +++ b/tests/models/transfo_xl/test_modeling_transfo_xl.py @@ -271,6 +271,18 @@ class TransfoXLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester test_resize_embeddings = True test_mismatched_shapes = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple + # tokenizer. + return True + + return False + def check_cutoffs_and_n_token( self, copied_cutoffs, layer, model_embed, model, model_class, resized_value, vocab_size ): diff --git a/tests/models/xlm/test_modeling_tf_xlm.py b/tests/models/xlm/test_modeling_tf_xlm.py index e34978495c..3f27dad5d4 100644 --- a/tests/models/xlm/test_modeling_tf_xlm.py +++ b/tests/models/xlm/test_modeling_tf_xlm.py @@ -309,6 +309,26 @@ class TFXLMModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `XLMConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFXLMModelTester(self) self.config_tester = ConfigTester(self, config_class=XLMConfig, emb_dim=37) diff --git a/tests/models/xlm/test_modeling_xlm.py b/tests/models/xlm/test_modeling_xlm.py index 58d2a26466..c86210e599 100644 --- a/tests/models/xlm/test_modeling_xlm.py +++ b/tests/models/xlm/test_modeling_xlm.py @@ -391,6 +391,26 @@ class XLMModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `XLMConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + # XLM has 2 QA models -> need to manually set the correct labels for one of them here def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/test_pipeline_mixin.py b/tests/test_pipeline_mixin.py index cef7062ba6..fbeac086e1 100644 --- a/tests/test_pipeline_mixin.py +++ b/tests/test_pipeline_mixin.py @@ -172,7 +172,7 @@ class PipelineTesterMixin: for tokenizer_name in tokenizer_names: for processor_name in processor_names: - if is_test_to_skip( + if self.is_pipeline_test_to_skip( pipeline_test_class_name, model_architecture.config_class, model_architecture, @@ -404,6 +404,11 @@ class PipelineTesterMixin: def test_pipeline_zero_shot_object_detection(self): self.run_task_tests(task="zero-shot-object-detection") + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return False + def validate_test_components(test_case, task, model, tokenizer, processor): # TODO: Move this to tiny model creation script @@ -431,109 +436,3 @@ def validate_test_components(test_case, task, model, tokenizer, processor): f"(`{tokenizer.__class__.__name__}`) has {len(tokenizer)} tokens which is greater than " f"`config_vocab_size` ({config_vocab_size}). Something is wrong." ) - - -def is_test_to_skip(test_casse_name, config_class, model_architecture, tokenizer_name, processor_name): - """Some tests are just not working""" - - to_skip = False - - if config_class.__name__ == "RoCBertConfig" and test_casse_name in [ - "FillMaskPipelineTests", - "FeatureExtractionPipelineTests", - "TextClassificationPipelineTests", - "TokenClassificationPipelineTests", - ]: - # Get error: IndexError: index out of range in self. - # `word_shape_file` and `word_pronunciation_file` should be shrunk during tiny model creation, - # otherwise `IndexError` could occur in some embedding layers. Skip for now until this model has - # more usage. - to_skip = True - elif config_class.__name__ in ["LayoutLMv3Config", "LiltConfig"]: - # Get error: ValueError: Words must be of type `List[str]`. Previously, `LayoutLMv3` is not - # used in pipeline tests as it could not find a checkpoint - # TODO: check and fix if possible - to_skip = True - # config/model class we decide to skip - elif config_class.__name__ in ["TapasConfig"]: - # Get error: AssertionError: Table must be of type pd.DataFrame. Also, the tiny model has large - # vocab size as the fast tokenizer could not be converted. Previous, `Tapas` is not used in - # pipeline tests due to the same reason. - # TODO: check and fix if possible - to_skip = True - - # TODO: check and fix if possible - if not to_skip and tokenizer_name is not None: - if ( - test_casse_name == "QAPipelineTests" - and not tokenizer_name.endswith("Fast") - and config_class.__name__ - in [ - "FlaubertConfig", - "GPTJConfig", - "LongformerConfig", - "MvpConfig", - "OPTConfig", - "ReformerConfig", - "XLMConfig", - ] - ): - # `QAPipelineTests` fails for a few models when the slower tokenizer are used. - # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) - # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer - to_skip = True - elif test_casse_name == "ZeroShotClassificationPipelineTests" and config_class.__name__ in [ - "CTRLConfig", - "OpenAIGPTConfig", - ]: - # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. - # `CTRLConfig` and `OpenAIGPTConfig` were never used in pipeline tests, either because of a missing - # checkpoint or because a tiny config could not be created - to_skip = True - elif test_casse_name == "TranslationPipelineTests" and config_class.__name__ in [ - "M2M100Config", - "PLBartConfig", - ]: - # Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`. - # `M2M100Config` and `PLBartConfig` were never used in pipeline tests: cannot create a simple tokenizer - to_skip = True - elif test_casse_name == "TextGenerationPipelineTests" and config_class.__name__ in [ - "ProphetNetConfig", - "TransfoXLConfig", - ]: - # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. - # `TransfoXLConfig` and `ProphetNetConfig` were never used in pipeline tests: cannot create a simple - # tokenizer. - to_skip = True - elif test_casse_name == "FillMaskPipelineTests" and config_class.__name__ in [ - "FlaubertConfig", - "XLMConfig", - ]: - # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. - # `FlaubertConfig` and `TransfoXLConfig` were never used in pipeline tests: cannot create a simple - # tokenizer - to_skip = True - elif test_casse_name == "TextGenerationPipelineTests" and model_architecture.__name__ in [ - "TFRoFormerForCausalLM" - ]: - # TODO: add `prepare_inputs_for_generation` for `TFRoFormerForCausalLM` - to_skip = True - elif test_casse_name == "QAPipelineTests" and model_architecture.__name__ in ["FNetForQuestionAnswering"]: - # TODO: The change in `base.py` in the PR #21132 (https://github.com/huggingface/transformers/pull/21132) - # fails this test case. Skip for now - a fix for this along with the initial changes in PR #20426 is - # too much. Let `ydshieh` to fix it ASAP once #20426 is merged. - to_skip = True - elif config_class.__name__ == "LayoutLMv2Config" and test_casse_name in [ - "QAPipelineTests", - "TextClassificationPipelineTests", - "TokenClassificationPipelineTests", - "ZeroShotClassificationPipelineTests", - ]: - # `LayoutLMv2Config` was never used in pipeline tests (`test_pt_LayoutLMv2Config_XXX`) due to lack of tiny - # config. With new tiny model creation, it is available, but we need to fix the failed tests. - to_skip = True - elif test_casse_name == "DocumentQuestionAnsweringPipelineTests" and not tokenizer_name.endswith("Fast"): - # This pipeline uses `sequence_ids()` which is only available for fast tokenizers. - to_skip = True - - return to_skip