Improve QA pipeline error handling (#8286)
- The issue is that with previous code we would have the following: ```python qa_pipeline = (...) qa_pipeline(question="Where was he born ?", context="") -> IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1) ``` The goal here is to improve this to actually return a ValueError wherever possible. While at it, I tried to simplify QuestionArgumentHandler's code to make it smaller and more compat while keeping backward compat.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import unittest
|
||||
|
||||
from transformers.pipelines import Pipeline
|
||||
from transformers.data.processors.squad import SquadExample
|
||||
from transformers.pipelines import Pipeline, QuestionAnsweringArgumentHandler
|
||||
|
||||
from .test_pipelines_common import CustomInputPipelineCommonMixin
|
||||
|
||||
@@ -43,5 +44,116 @@ class QAPipelineTests(CustomInputPipelineCommonMixin, unittest.TestCase):
|
||||
for key in output_keys:
|
||||
self.assertIn(key, result)
|
||||
for bad_input in invalid_inputs:
|
||||
self.assertRaises(Exception, nlp, bad_input)
|
||||
self.assertRaises(Exception, nlp, invalid_inputs)
|
||||
self.assertRaises(ValueError, nlp, bad_input)
|
||||
self.assertRaises(ValueError, nlp, invalid_inputs)
|
||||
|
||||
def test_argument_handler(self):
|
||||
qa = QuestionAnsweringArgumentHandler()
|
||||
|
||||
Q = "Where was HuggingFace founded ?"
|
||||
C = "HuggingFace was founded in Paris"
|
||||
|
||||
normalized = qa(Q, C)
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa(question=Q, context=C)
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa(question=Q, context=C)
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa({"question": Q, "context": C})
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa([{"question": Q, "context": C}])
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa([{"question": Q, "context": C}, {"question": Q, "context": C}])
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 2)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa(X={"question": Q, "context": C})
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa(X=[{"question": Q, "context": C}])
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
normalized = qa(data={"question": Q, "context": C})
|
||||
self.assertEqual(type(normalized), list)
|
||||
self.assertEqual(len(normalized), 1)
|
||||
self.assertEqual({type(el) for el in normalized}, {SquadExample})
|
||||
|
||||
def test_argument_handler_error_handling(self):
|
||||
qa = QuestionAnsweringArgumentHandler()
|
||||
|
||||
Q = "Where was HuggingFace founded ?"
|
||||
C = "HuggingFace was founded in Paris"
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
qa({"context": C})
|
||||
with self.assertRaises(KeyError):
|
||||
qa({"question": Q})
|
||||
with self.assertRaises(KeyError):
|
||||
qa([{"context": C}])
|
||||
with self.assertRaises(ValueError):
|
||||
qa(None, C)
|
||||
with self.assertRaises(ValueError):
|
||||
qa("", C)
|
||||
with self.assertRaises(ValueError):
|
||||
qa(Q, None)
|
||||
with self.assertRaises(ValueError):
|
||||
qa(Q, "")
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
qa(question=None, context=C)
|
||||
with self.assertRaises(ValueError):
|
||||
qa(question="", context=C)
|
||||
with self.assertRaises(ValueError):
|
||||
qa(question=Q, context=None)
|
||||
with self.assertRaises(ValueError):
|
||||
qa(question=Q, context="")
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
qa({"question": None, "context": C})
|
||||
with self.assertRaises(ValueError):
|
||||
qa({"question": "", "context": C})
|
||||
with self.assertRaises(ValueError):
|
||||
qa({"question": Q, "context": None})
|
||||
with self.assertRaises(ValueError):
|
||||
qa({"question": Q, "context": ""})
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
qa([{"question": Q, "context": C}, {"question": None, "context": C}])
|
||||
with self.assertRaises(ValueError):
|
||||
qa([{"question": Q, "context": C}, {"question": "", "context": C}])
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
qa([{"question": Q, "context": C}, {"question": Q, "context": None}])
|
||||
with self.assertRaises(ValueError):
|
||||
qa([{"question": Q, "context": C}, {"question": Q, "context": ""}])
|
||||
|
||||
def test_argument_handler_error_handling_odd(self):
|
||||
qa = QuestionAnsweringArgumentHandler()
|
||||
with self.assertRaises(ValueError):
|
||||
qa(None)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
qa(Y=None)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
qa(1)
|
||||
|
||||
Reference in New Issue
Block a user