Moving question_answering tests to the new testing scheme. Had to tweak a little some ModelTesterConfig for pipelines. (#13277)

* Moving question_answering tests to the new testing scheme. Had to tweak
a little some ModelTesterConfig for pipelines.

* Removing commented code.

tests/test_modeling_bart.py

@@ -147,6 +147,11 @@ class BartModelTester:
             pad_token_id=self.pad_token_id,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.max_position_embeddings = 100
+        return config
+
     def prepare_config_and_inputs_for_common(self):
         config, inputs_dict = self.prepare_config_and_inputs()
         return config, inputs_dict

tests/test_modeling_reformer.py

@@ -189,6 +189,7 @@ class ReformerModelTester:
     def get_pipeline_config(self):
         config = self.get_config()
         config.vocab_size = 100
+        config.axial_pos_shape = (4, 25)
         config.is_decoder = False
         return config
 

tests/test_pipelines_question_answering.py

@@ -14,107 +14,126 @@
 
 import unittest
 
-from transformers import is_tf_available, is_torch_available
+from transformers import (
+    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+    LxmertConfig,
+    QuestionAnsweringPipeline,
+)
 from transformers.data.processors.squad import SquadExample
-from transformers.pipelines import Pipeline, QuestionAnsweringArgumentHandler, pipeline
-from transformers.testing_utils import slow
+from transformers.pipelines import QuestionAnsweringArgumentHandler, pipeline
+from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow
 
-from .test_pipelines_common import CustomInputPipelineCommonMixin
+from .test_pipelines_common import ANY, PipelineTestCaseMeta
 
 
-class QAPipelineTests(CustomInputPipelineCommonMixin, unittest.TestCase):
-    pipeline_task = "question-answering"
-    pipeline_running_kwargs = {
-        "padding": "max_length",
-        "max_seq_len": 25,
-        "doc_stride": 5,
-    }  # Default is 'longest' but we use 'max_length' to test equivalence between slow/fast tokenizers
-    small_models = [
-        "sshleifer/tiny-distilbert-base-cased-distilled-squad"
-    ]  # Models tested without the @slow decorator
-    large_models = []  # Models tested with the @slow decorator
-    valid_inputs = [
-        {"question": "Where was HuggingFace founded ?", "context": "HuggingFace was founded in Paris."},
-        {
-            "question": "In what field is HuggingFace working ?",
-            "context": "HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.",
-        },
-        {
-            "question": ["In what field is HuggingFace working ?", "In what field is HuggingFace working ?"],
-            "context": "HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.",
-        },
-        {
-            "question": ["In what field is HuggingFace working ?", "In what field is HuggingFace working ?"],
-            "context": [
-                "HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.",
-                "HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.",
+@is_pipeline_test
+class QAPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
+    model_mapping = MODEL_FOR_QUESTION_ANSWERING_MAPPING
+    tf_model_mapping = TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING
+
+    def run_pipeline_test(self, model, tokenizer, feature_extractor):
+        if isinstance(model.config, LxmertConfig):
+            # This is an bimodal model, we need to find a more consistent way
+            # to switch on those models.
+            return
+        question_answerer = QuestionAnsweringPipeline(model, tokenizer)
+
+        outputs = question_answerer(
+            question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris."
+        )
+        self.assertEqual(outputs, {"answer": ANY(str), "start": ANY(int), "end": ANY(int), "score": ANY(float)})
+
+        outputs = question_answerer(
+            question=["In what field is HuggingFace working ?", "In what field is HuggingFace working ?"],
+            context="HuggingFace was founded in Paris.",
+        )
+        self.assertEqual(
+            outputs,
+            [
+                {"answer": ANY(str), "start": ANY(int), "end": ANY(int), "score": ANY(float)},
+                {"answer": ANY(str), "start": ANY(int), "end": ANY(int), "score": ANY(float)},
             ],
-        },
-    ]
+        )
 
-    def get_pipelines(self):
-        question_answering_pipelines = [
-            pipeline(
-                task=self.pipeline_task,
-                model=model,
-                tokenizer=model,
-                framework="pt" if is_torch_available() else "tf",
-                **self.pipeline_loading_kwargs,
-            )
-            for model in self.small_models
-        ]
-        return question_answering_pipelines
+        outputs = question_answerer(
+            question=["What field is HuggingFace working ?", "In what field is HuggingFace ?"],
+            context=[
+                "HuggingFace is a startup based in New-York",
+                "HuggingFace is a startup founded in Paris",
+            ],
+        )
+        self.assertEqual(
+            outputs,
+            [
+                {"answer": ANY(str), "start": ANY(int), "end": ANY(int), "score": ANY(float)},
+                {"answer": ANY(str), "start": ANY(int), "end": ANY(int), "score": ANY(float)},
+            ],
+        )
+
+        with self.assertRaises(ValueError):
+            question_answerer(question="", context="HuggingFace was founded in Paris.")
+        with self.assertRaises(ValueError):
+            question_answerer(question=None, context="HuggingFace was founded in Paris.")
+        with self.assertRaises(ValueError):
+            question_answerer(question="In what field is HuggingFace working ?", context="")
+        with self.assertRaises(ValueError):
+            question_answerer(question="In what field is HuggingFace working ?", context=None)
+
+        outputs = question_answerer(
+            question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris.", topk=20
+        )
+        self.assertEqual(
+            outputs, [{"answer": ANY(str), "start": ANY(int), "end": ANY(int), "score": ANY(float)} for i in range(20)]
+        )
+
+    @require_torch
+    def test_small_model_pt(self):
+        question_answerer = pipeline(
+            "question-answering", model="sshleifer/tiny-distilbert-base-cased-distilled-squad"
+        )
+        outputs = question_answerer(
+            question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris."
+        )
+
+        self.assertEqual(nested_simplify(outputs), {"score": 0.01, "start": 0, "end": 11, "answer": "HuggingFace"})
+
+    @require_tf
+    def test_small_model_tf(self):
+        question_answerer = pipeline(
+            "question-answering", model="sshleifer/tiny-distilbert-base-cased-distilled-squad", framework="tf"
+        )
+        outputs = question_answerer(
+            question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris."
+        )
+
+        self.assertEqual(nested_simplify(outputs), {"score": 0.011, "start": 0, "end": 11, "answer": "HuggingFace"})
 
     @slow
-    @unittest.skipIf(not is_torch_available() and not is_tf_available(), "Either torch or TF must be installed.")
-    def test_high_topk_small_context(self):
-        self.pipeline_running_kwargs.update({"topk": 20})
-        valid_inputs = [
-            {"question": "Where was HuggingFace founded ?", "context": "Paris"},
-        ]
-        question_answering_pipelines = self.get_pipelines()
-        output_keys = {"score", "answer", "start", "end"}
-        for question_answering_pipeline in question_answering_pipelines:
-            result = question_answering_pipeline(valid_inputs, **self.pipeline_running_kwargs)
-            self.assertIsInstance(result, dict)
+    @require_torch
+    def test_large_model_pt(self):
+        question_answerer = pipeline(
+            "question-answering",
+        )
+        outputs = question_answerer(
+            question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris."
+        )
 
-            for key in output_keys:
-                self.assertIn(key, result)
+        self.assertEqual(nested_simplify(outputs), {"score": 0.979, "start": 27, "end": 32, "answer": "Paris"})
 
-    def _test_pipeline(self, question_answering_pipeline: Pipeline):
-        output_keys = {"score", "answer", "start", "end"}
-        valid_inputs = [
-            {"question": "Where was HuggingFace founded ?", "context": "HuggingFace was founded in Paris."},
-            {
-                "question": "In what field is HuggingFace working ?",
-                "context": "HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.",
-            },
-        ]
-        invalid_inputs = [
-            {"question": "", "context": "This is a test to try empty question edge case"},
-            {"question": None, "context": "This is a test to try empty question edge case"},
-            {"question": "What is does with empty context ?", "context": ""},
-            {"question": "What is does with empty context ?", "context": None},
-        ]
-        self.assertIsNotNone(question_answering_pipeline)
+    @slow
+    @require_tf
+    def test_large_model_tf(self):
+        question_answerer = pipeline("question-answering", framework="tf")
+        outputs = question_answerer(
+            question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris."
+        )
 
-        mono_result = question_answering_pipeline(valid_inputs[0])
-        self.assertIsInstance(mono_result, dict)
+        self.assertEqual(nested_simplify(outputs), {"score": 0.979, "start": 27, "end": 32, "answer": "Paris"})
 
-        for key in output_keys:
-            self.assertIn(key, mono_result)
-
-        multi_result = question_answering_pipeline(valid_inputs)
-        self.assertIsInstance(multi_result, list)
-        self.assertIsInstance(multi_result[0], dict)
-
-        for result in multi_result:
-            for key in output_keys:
-                self.assertIn(key, result)
-        for bad_input in invalid_inputs:
-            self.assertRaises(ValueError, question_answering_pipeline, bad_input)
-        self.assertRaises(ValueError, question_answering_pipeline, invalid_inputs)
 
+@is_pipeline_test
+class QuestionAnsweringArgumentHandlerTests(unittest.TestCase):
     def test_argument_handler(self):
         qa = QuestionAnsweringArgumentHandler()