From 6b586ed18c137a152c0f77af2d21fee346b4c760 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 26 Aug 2021 11:52:49 +0200 Subject: [PATCH] Move `image-classification` pipeline to new testing (#13272) - Enforce `test_small_models_{tf,pt}` methods to exist (enforce checking actual values in small tests) - Add support for non RGB image for the pipeline. --- .../pipelines/image_classification.py | 20 +- tests/test_pipelines_common.py | 51 ++-- tests/test_pipelines_conversational.py | 21 +- tests/test_pipelines_feature_extraction.py | 29 ++- tests/test_pipelines_fill_mask.py | 4 +- tests/test_pipelines_image_classification.py | 231 +++++++++--------- 6 files changed, 199 insertions(+), 157 deletions(-) diff --git a/src/transformers/pipelines/image_classification.py b/src/transformers/pipelines/image_classification.py index 76a519a988..3ebb264999 100644 --- a/src/transformers/pipelines/image_classification.py +++ b/src/transformers/pipelines/image_classification.py @@ -61,15 +61,21 @@ class ImageClassificationPipeline(Pipeline): if image.startswith("http://") or image.startswith("https://"): # We need to actually check for a real protocol, otherwise it's impossible to use a local file # like http_huggingface_co.png - return Image.open(requests.get(image, stream=True).raw) + image = Image.open(requests.get(image, stream=True).raw) elif os.path.isfile(image): - return Image.open(image) + image = Image.open(image) + else: + raise ValueError( + f"Incorrect path or url, URLs must start with `http://` or `https://`, and {image} is not a valid path" + ) elif isinstance(image, Image.Image): - return image - - raise ValueError( - "Incorrect format used for image. Should be an url linking to an image, a local path, or a PIL image." - ) + image = image + else: + raise ValueError( + "Incorrect format used for image. Should be an url linking to an image, a local path, or a PIL image." + ) + image = image.convert("RGB") + return image def __call__(self, images: Union[str, List[str], "Image", List["Image"]], top_k=5): """ diff --git a/tests/test_pipelines_common.py b/tests/test_pipelines_common.py index 8ff66f2780..76f8a2b611 100644 --- a/tests/test_pipelines_common.py +++ b/tests/test_pipelines_common.py @@ -15,6 +15,7 @@ import importlib import logging import string +from abc import abstractmethod from functools import lru_cache from typing import List, Optional from unittest import mock, skipIf @@ -123,15 +124,18 @@ class PipelineTestCaseMeta(type): model = ModelClass(tiny_config) if hasattr(model, "eval"): model = model.eval() - try: - tokenizer = get_tiny_tokenizer_from_checkpoint(checkpoint) - if hasattr(model.config, "max_position_embeddings"): - tokenizer.model_max_length = model.config.max_position_embeddings - # Rust Panic exception are NOT Exception subclass - # Some test tokenizer contain broken vocabs or custom PreTokenizer, so we - # provide some default tokenizer and hope for the best. - except: # noqa: E722 - self.skipTest(f"Ignoring {ModelClass}, cannot create a simple tokenizer") + if tokenizer_class is not None: + try: + tokenizer = get_tiny_tokenizer_from_checkpoint(checkpoint) + if hasattr(model.config, "max_position_embeddings"): + tokenizer.model_max_length = model.config.max_position_embeddings + # Rust Panic exception are NOT Exception subclass + # Some test tokenizer contain broken vocabs or custom PreTokenizer, so we + # provide some default tokenizer and hope for the best. + except: # noqa: E722 + self.skipTest(f"Ignoring {ModelClass}, cannot create a simple tokenizer") + else: + tokenizer = None feature_extractor = get_tiny_feature_extractor_from_checkpoint(checkpoint, tiny_config) self.run_pipeline_test(model, tokenizer, feature_extractor) @@ -149,16 +153,21 @@ class PipelineTestCaseMeta(type): tiny_config = get_tiny_config_from_class(configuration) tokenizer_classes = TOKENIZER_MAPPING.get(configuration, []) feature_extractor_class = FEATURE_EXTRACTOR_MAPPING.get(configuration, None) + feature_extractor_name = ( + feature_extractor_class.__name__ if feature_extractor_class else "nofeature_extractor" + ) + if not tokenizer_classes: + # We need to test even if there are no tokenizers. + tokenizer_classes = [None] for tokenizer_class in tokenizer_classes: - if tokenizer_class is not None and tokenizer_class.__name__.endswith("Fast"): + if tokenizer_class is not None: + tokenizer_name = tokenizer_class.__name__ + else: + tokenizer_name = "notokenizer" - tokenizer_name = tokenizer_class.__name__ if tokenizer_class else "notokenizer" - feature_extractor_name = ( - feature_extractor_class.__name__ - if feature_extractor_class - else "nofeature_extractor" - ) - test_name = f"test_{prefix}_{configuration.__name__}_{model_architecture.__name__}_{tokenizer_name}_{feature_extractor_name}" + test_name = f"test_{prefix}_{configuration.__name__}_{model_architecture.__name__}_{tokenizer_name}_{feature_extractor_name}" + + if tokenizer_class is not None or feature_extractor_class is not None: dct[test_name] = gen_test( model_architecture, checkpoint, @@ -167,6 +176,14 @@ class PipelineTestCaseMeta(type): feature_extractor_class, ) + @abstractmethod + def inner(self): + raise NotImplementedError("Not implemented test") + + # Force these 2 methods to exist + dct["test_small_model_pt"] = dct.get("test_small_model_pt", inner) + dct["test_small_model_tf"] = dct.get("test_small_model_tf", inner) + return type.__new__(mcs, name, bases, dct) diff --git a/tests/test_pipelines_conversational.py b/tests/test_pipelines_conversational.py index 9a495d65f1..3895c7e6f4 100644 --- a/tests/test_pipelines_conversational.py +++ b/tests/test_pipelines_conversational.py @@ -26,9 +26,10 @@ from transformers import ( BlenderbotSmallTokenizer, Conversation, ConversationalPipeline, + TFAutoModelForCausalLM, pipeline, ) -from transformers.testing_utils import is_pipeline_test, require_torch, slow, torch_device +from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow, torch_device from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -160,6 +161,24 @@ class ConversationalPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseM self.assertEqual(result.past_user_inputs[1], "Is it an action movie?") self.assertEqual(result.generated_responses[1], "It's a comedy.") + @require_torch + def test_small_model_pt(self): + tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small") + model = AutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-small") + conversation_agent = ConversationalPipeline(model=model, tokenizer=tokenizer) + conversation = Conversation("hello") + output = conversation_agent(conversation) + self.assertEqual(output, Conversation(past_user_inputs=["hello"], generated_responses=["Hi"])) + + @require_tf + def test_small_model_tf(self): + tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small") + model = TFAutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-small") + conversation_agent = ConversationalPipeline(model=model, tokenizer=tokenizer) + conversation = Conversation("hello") + output = conversation_agent(conversation) + self.assertEqual(output, Conversation(past_user_inputs=["hello"], generated_responses=["Hi"])) + @require_torch @slow def test_integration_torch_conversation_dialogpt_input_ids(self): diff --git a/tests/test_pipelines_feature_extraction.py b/tests/test_pipelines_feature_extraction.py index a0c228da27..f57c5e87c3 100644 --- a/tests/test_pipelines_feature_extraction.py +++ b/tests/test_pipelines_feature_extraction.py @@ -14,7 +14,7 @@ import unittest -from transformers import MODEL_MAPPING, TF_MODEL_MAPPING, FeatureExtractionPipeline, LxmertConfig, pipeline +from transformers import MODEL_MAPPING, TF_MODEL_MAPPING, CLIPConfig, FeatureExtractionPipeline, LxmertConfig, pipeline from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch from .test_pipelines_common import PipelineTestCaseMeta @@ -62,20 +62,29 @@ class FeatureExtractionPipelineTests(unittest.TestCase, metaclass=PipelineTestCa return shape def run_pipeline_test(self, model, tokenizer, feature_extractor): - if isinstance(model.config, LxmertConfig): - # This is an bimodal model, we need to find a more consistent way - # to switch on those models. + if tokenizer is None: + self.skipTest("No tokenizer") + return + + elif isinstance(model.config, (LxmertConfig, CLIPConfig)): + self.skipTest( + "This is an Lxmert bimodal model, we need to find a more consistent way to switch on those models." + ) + return + elif model.config.is_encoder_decoder: + self.skipTest( + """encoder_decoder models are trickier for this pipeline. + Do we want encoder + decoder inputs to get some featues? + Do we want encoder only features ? + For now ignore those. + """ + ) + return feature_extractor = FeatureExtractionPipeline( model=model, tokenizer=tokenizer, feature_extractor=feature_extractor ) - if feature_extractor.model.config.is_encoder_decoder: - # encoder_decoder models are trickier for this pipeline. - # Do we want encoder + decoder inputs to get some featues? - # Do we want encoder only features ? - # For now ignore those. - return outputs = feature_extractor("This is a test") diff --git a/tests/test_pipelines_fill_mask.py b/tests/test_pipelines_fill_mask.py index a612e9ba22..8d78c31add 100644 --- a/tests/test_pipelines_fill_mask.py +++ b/tests/test_pipelines_fill_mask.py @@ -169,8 +169,8 @@ class FillMaskPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): self.run_pipeline_test(unmasker.model, unmasker.tokenizer, None) def run_pipeline_test(self, model, tokenizer, feature_extractor): - if tokenizer.mask_token_id is None: - self.skipTest("The provided tokenizer has no mask token, (probably reformer)") + if tokenizer is None or tokenizer.mask_token_id is None: + self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)") fill_masker = FillMaskPipeline(model=model, tokenizer=tokenizer) diff --git a/tests/test_pipelines_image_classification.py b/tests/test_pipelines_image_classification.py index 0306523255..e06dec9d0b 100644 --- a/tests/test_pipelines_image_classification.py +++ b/tests/test_pipelines_image_classification.py @@ -14,15 +14,18 @@ import unittest -from transformers import ( - AutoConfig, - AutoFeatureExtractor, - AutoModelForImageClassification, - PreTrainedTokenizer, - is_vision_available, -) +from transformers import MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING, PreTrainedTokenizer, is_vision_available from transformers.pipelines import ImageClassificationPipeline, pipeline -from transformers.testing_utils import require_torch, require_vision +from transformers.testing_utils import ( + is_pipeline_test, + nested_simplify, + require_datasets, + require_tf, + require_torch, + require_vision, +) + +from .test_pipelines_common import ANY, PipelineTestCaseMeta if is_vision_available(): @@ -35,127 +38,115 @@ else: pass +@is_pipeline_test @require_vision @require_torch -class ImageClassificationPipelineTests(unittest.TestCase): - pipeline_task = "image-classification" - small_models = ["lysandre/tiny-vit-random"] # Models tested without the @slow decorator - valid_inputs = [ - {"images": "http://images.cocodataset.org/val2017/000000039769.jpg"}, - { - "images": [ +class ImageClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): + model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING + + @require_datasets + def run_pipeline_test(self, model, tokenizer, feature_extractor): + image_classifier = ImageClassificationPipeline(model=model, feature_extractor=feature_extractor) + outputs = image_classifier("./tests/fixtures/tests_samples/COCO/000000039769.png") + + self.assertEqual( + outputs, + [ + {"score": ANY(float), "label": ANY(str)}, + {"score": ANY(float), "label": ANY(str)}, + ], + ) + + import datasets + + dataset = datasets.load_dataset("Narsil/image_dummy", "image", split="test") + + # Accepts URL + PIL.Image + lists + outputs = image_classifier( + [ + Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"), + "http://images.cocodataset.org/val2017/000000039769.jpg", + # RGBA + dataset[0]["file"], + # LA + dataset[1]["file"], + # L + dataset[2]["file"], + ] + ) + self.assertEqual( + outputs, + [ + [ + {"score": ANY(float), "label": ANY(str)}, + {"score": ANY(float), "label": ANY(str)}, + ], + [ + {"score": ANY(float), "label": ANY(str)}, + {"score": ANY(float), "label": ANY(str)}, + ], + [ + {"score": ANY(float), "label": ANY(str)}, + {"score": ANY(float), "label": ANY(str)}, + ], + [ + {"score": ANY(float), "label": ANY(str)}, + {"score": ANY(float), "label": ANY(str)}, + ], + [ + {"score": ANY(float), "label": ANY(str)}, + {"score": ANY(float), "label": ANY(str)}, + ], + ], + ) + + @require_torch + def test_small_model_pt(self): + small_model = "lysandre/tiny-vit-random" + image_classifier = pipeline("image-classification", model=small_model) + + outputs = image_classifier("http://images.cocodataset.org/val2017/000000039769.jpg") + self.assertEqual( + nested_simplify(outputs, decimals=4), + [ + {"score": 0.0015, "label": "chambered nautilus, pearly nautilus, nautilus"}, + {"score": 0.0015, "label": "pajama, pyjama, pj's, jammies"}, + {"score": 0.0014, "label": "trench coat"}, + {"score": 0.0014, "label": "handkerchief, hankie, hanky, hankey"}, + {"score": 0.0014, "label": "baboon"}, + ], + ) + + outputs = image_classifier( + [ "http://images.cocodataset.org/val2017/000000039769.jpg", "http://images.cocodataset.org/val2017/000000039769.jpg", - ] - }, - {"images": "./tests/fixtures/tests_samples/COCO/000000039769.png"}, - { - "images": [ - "./tests/fixtures/tests_samples/COCO/000000039769.png", - "./tests/fixtures/tests_samples/COCO/000000039769.png", - ] - }, - {"images": Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")}, - { - "images": [ - Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"), - Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"), - ] - }, - { - "images": [ - Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"), - "./tests/fixtures/tests_samples/COCO/000000039769.png", - ] - }, - ] + ], + top_k=2, + ) + self.assertEqual( + nested_simplify(outputs, decimals=4), + [ + [ + {"score": 0.0015, "label": "chambered nautilus, pearly nautilus, nautilus"}, + {"score": 0.0015, "label": "pajama, pyjama, pj's, jammies"}, + ], + [ + {"score": 0.0015, "label": "chambered nautilus, pearly nautilus, nautilus"}, + {"score": 0.0015, "label": "pajama, pyjama, pj's, jammies"}, + ], + ], + ) - def test_small_model_from_factory(self): - for small_model in self.small_models: - - image_classifier = pipeline("image-classification", model=small_model) - - for valid_input in self.valid_inputs: - output = image_classifier(**valid_input) - top_k = valid_input.get("top_k", 5) - - def assert_valid_pipeline_output(pipeline_output): - self.assertTrue(isinstance(pipeline_output, list)) - self.assertEqual(len(pipeline_output), top_k) - for label_result in pipeline_output: - self.assertTrue(isinstance(label_result, dict)) - self.assertIn("label", label_result) - self.assertIn("score", label_result) - - if isinstance(valid_input["images"], list): - self.assertEqual(len(valid_input["images"]), len(output)) - for individual_output in output: - assert_valid_pipeline_output(individual_output) - else: - assert_valid_pipeline_output(output) - - def test_small_model_from_pipeline(self): - for small_model in self.small_models: - - model = AutoModelForImageClassification.from_pretrained(small_model) - feature_extractor = AutoFeatureExtractor.from_pretrained(small_model) - image_classifier = ImageClassificationPipeline(model=model, feature_extractor=feature_extractor) - - for valid_input in self.valid_inputs: - output = image_classifier(**valid_input) - top_k = valid_input.get("top_k", 5) - - def assert_valid_pipeline_output(pipeline_output): - self.assertTrue(isinstance(pipeline_output, list)) - self.assertEqual(len(pipeline_output), top_k) - for label_result in pipeline_output: - self.assertTrue(isinstance(label_result, dict)) - self.assertIn("label", label_result) - self.assertIn("score", label_result) - - if isinstance(valid_input["images"], list): - # When images are batched, pipeline output is a list of lists of dictionaries - self.assertEqual(len(valid_input["images"]), len(output)) - for individual_output in output: - assert_valid_pipeline_output(individual_output) - else: - # When images are batched, pipeline output is a list of dictionaries - assert_valid_pipeline_output(output) + @require_tf + @unittest.skip("Image classification is not implemented for TF") + def test_small_model_tf(self): + pass def test_custom_tokenizer(self): tokenizer = PreTrainedTokenizer() # Assert that the pipeline can be initialized with a feature extractor that is not in any mapping - image_classifier = pipeline("image-classification", model=self.small_models[0], tokenizer=tokenizer) + image_classifier = pipeline("image-classification", model="lysandre/tiny-vit-random", tokenizer=tokenizer) self.assertIs(image_classifier.tokenizer, tokenizer) - - def test_num_labels_inferior_to_topk(self): - for small_model in self.small_models: - - num_labels = 2 - model = AutoModelForImageClassification.from_config( - AutoConfig.from_pretrained(small_model, num_labels=num_labels) - ) - feature_extractor = AutoFeatureExtractor.from_pretrained(small_model) - image_classifier = ImageClassificationPipeline(model=model, feature_extractor=feature_extractor) - - for valid_input in self.valid_inputs: - output = image_classifier(**valid_input) - - def assert_valid_pipeline_output(pipeline_output): - self.assertTrue(isinstance(pipeline_output, list)) - self.assertEqual(len(pipeline_output), num_labels) - for label_result in pipeline_output: - self.assertTrue(isinstance(label_result, dict)) - self.assertIn("label", label_result) - self.assertIn("score", label_result) - - if isinstance(valid_input["images"], list): - # When images are batched, pipeline output is a list of lists of dictionaries - self.assertEqual(len(valid_input["images"]), len(output)) - for individual_output in output: - assert_valid_pipeline_output(individual_output) - else: - # When images are batched, pipeline output is a list of dictionaries - assert_valid_pipeline_output(output)