Move is_pipeline_test_to_skip to specific model test classes (#21999)

* Move `is_pipeline_test_to_skip` to specific model test classes

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/models/flaubert/test_modeling_tf_flaubert.py

@@ -306,6 +306,26 @@ class TFFlaubertModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Test
     test_head_masking = False
     test_onnx = False
 
+    # TODO: Fix the failed tests
+    def is_pipeline_test_to_skip(
+        self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
+    ):
+        if pipeline_test_casse_name == "FillMaskPipelineTests":
+            # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
+            # `FlaubertConfig` was never used in pipeline tests: cannot create a simple tokenizer
+            return True
+        elif (
+            pipeline_test_casse_name == "QAPipelineTests"
+            and tokenizer_name is not None
+            and not tokenizer_name.endswith("Fast")
+        ):
+            # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
+            # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
+            # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
+            return True
+
+        return False
+
     def setUp(self):
         self.model_tester = TFFlaubertModelTester(self)
         self.config_tester = ConfigTester(self, config_class=FlaubertConfig, emb_dim=37)