SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Move image-classification pipeline to new testing (#13272)

Browse Source 
- Enforce `test_small_models_{tf,pt}` methods to exist (enforce checking
actual values in small tests)
- Add support for non RGB image for the pipeline.
This commit is contained in:
Nicolas Patry
2021-08-26 11:52:49 +02:00
committed by GitHub
parent 401377e679
commit 6b586ed18c
6 changed files with 199 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
tests/test_pipelines_common.py
View File

@@ -15,6 +15,7 @@
import importlib
import logging
import string
from abc import abstractmethod
from functools import lru_cache
from typing import List, Optional
from unittest import mock, skipIf
@@ -123,15 +124,18 @@ class PipelineTestCaseMeta(type):
model = ModelClass(tiny_config)
if hasattr(model, "eval"):
model = model.eval()
try:
tokenizer = get_tiny_tokenizer_from_checkpoint(checkpoint)
if hasattr(model.config, "max_position_embeddings"):
tokenizer.model_max_length = model.config.max_position_embeddings
# Rust Panic exception are NOT Exception subclass
# Some test tokenizer contain broken vocabs or custom PreTokenizer, so we
# provide some default tokenizer and hope for the best.
except: # noqa: E722
self.skipTest(f"Ignoring {ModelClass}, cannot create a simple tokenizer")
if tokenizer_class is not None:
try:
tokenizer = get_tiny_tokenizer_from_checkpoint(checkpoint)
if hasattr(model.config, "max_position_embeddings"):
tokenizer.model_max_length = model.config.max_position_embeddings
# Rust Panic exception are NOT Exception subclass
# Some test tokenizer contain broken vocabs or custom PreTokenizer, so we
# provide some default tokenizer and hope for the best.
except: # noqa: E722
self.skipTest(f"Ignoring {ModelClass}, cannot create a simple tokenizer")
else:
tokenizer = None
feature_extractor = get_tiny_feature_extractor_from_checkpoint(checkpoint, tiny_config)
self.run_pipeline_test(model, tokenizer, feature_extractor)
@@ -149,16 +153,21 @@ class PipelineTestCaseMeta(type):
tiny_config = get_tiny_config_from_class(configuration)
tokenizer_classes = TOKENIZER_MAPPING.get(configuration, [])
feature_extractor_class = FEATURE_EXTRACTOR_MAPPING.get(configuration, None)
feature_extractor_name = (
feature_extractor_class.__name__ if feature_extractor_class else "nofeature_extractor"
)
if not tokenizer_classes:
# We need to test even if there are no tokenizers.
tokenizer_classes = [None]
for tokenizer_class in tokenizer_classes:
if tokenizer_class is not None and tokenizer_class.__name__.endswith("Fast"):
if tokenizer_class is not None:
tokenizer_name = tokenizer_class.__name__
else:
tokenizer_name = "notokenizer"
tokenizer_name = tokenizer_class.__name__ if tokenizer_class else "notokenizer"
feature_extractor_name = (
feature_extractor_class.__name__
if feature_extractor_class
else "nofeature_extractor"
)
test_name = f"test_{prefix}_{configuration.__name__}_{model_architecture.__name__}_{tokenizer_name}_{feature_extractor_name}"
test_name = f"test_{prefix}_{configuration.__name__}_{model_architecture.__name__}_{tokenizer_name}_{feature_extractor_name}"
if tokenizer_class is not None or feature_extractor_class is not None:
dct[test_name] = gen_test(
model_architecture,
checkpoint,
@@ -167,6 +176,14 @@ class PipelineTestCaseMeta(type):
feature_extractor_class,
)
@abstractmethod
def inner(self):
raise NotImplementedError("Not implemented test")
# Force these 2 methods to exist
dct["test_small_model_pt"] = dct.get("test_small_model_pt", inner)
dct["test_small_model_tf"] = dct.get("test_small_model_tf", inner)
return type.__new__(mcs, name, bases, dct)

21
tests/test_pipelines_conversational.py
View File

@@ -26,9 +26,10 @@ from transformers import (
BlenderbotSmallTokenizer,
Conversation,
ConversationalPipeline,
TFAutoModelForCausalLM,
pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_torch, slow, torch_device
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow, torch_device
from .test_pipelines_common import ANY, PipelineTestCaseMeta
@@ -160,6 +161,24 @@ class ConversationalPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseM
self.assertEqual(result.past_user_inputs[1], "Is it an action movie?")
self.assertEqual(result.generated_responses[1], "It's a comedy.")
@require_torch
def test_small_model_pt(self):
tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small")
model = AutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-small")
conversation_agent = ConversationalPipeline(model=model, tokenizer=tokenizer)
conversation = Conversation("hello")
output = conversation_agent(conversation)
self.assertEqual(output, Conversation(past_user_inputs=["hello"], generated_responses=["Hi"]))
@require_tf
def test_small_model_tf(self):
tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small")
model = TFAutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-small")
conversation_agent = ConversationalPipeline(model=model, tokenizer=tokenizer)
conversation = Conversation("hello")
output = conversation_agent(conversation)
self.assertEqual(output, Conversation(past_user_inputs=["hello"], generated_responses=["Hi"]))
@require_torch
@slow
def test_integration_torch_conversation_dialogpt_input_ids(self):

29
tests/test_pipelines_feature_extraction.py
View File

@@ -14,7 +14,7 @@
import unittest
from transformers import MODEL_MAPPING, TF_MODEL_MAPPING, FeatureExtractionPipeline, LxmertConfig, pipeline
from transformers import MODEL_MAPPING, TF_MODEL_MAPPING, CLIPConfig, FeatureExtractionPipeline, LxmertConfig, pipeline
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch
from .test_pipelines_common import PipelineTestCaseMeta
@@ -62,20 +62,29 @@ class FeatureExtractionPipelineTests(unittest.TestCase, metaclass=PipelineTestCa
return shape
def run_pipeline_test(self, model, tokenizer, feature_extractor):
if isinstance(model.config, LxmertConfig):
# This is an bimodal model, we need to find a more consistent way
# to switch on those models.
if tokenizer is None:
self.skipTest("No tokenizer")
return
elif isinstance(model.config, (LxmertConfig, CLIPConfig)):
self.skipTest(
"This is an Lxmert bimodal model, we need to find a more consistent way to switch on those models."
)
return
elif model.config.is_encoder_decoder:
self.skipTest(
"""encoder_decoder models are trickier for this pipeline.
Do we want encoder + decoder inputs to get some featues?
Do we want encoder only features ?
For now ignore those.
"""
)
return
feature_extractor = FeatureExtractionPipeline(
model=model, tokenizer=tokenizer, feature_extractor=feature_extractor
)
if feature_extractor.model.config.is_encoder_decoder:
# encoder_decoder models are trickier for this pipeline.
# Do we want encoder + decoder inputs to get some featues?
# Do we want encoder only features ?
# For now ignore those.
return
outputs = feature_extractor("This is a test")

4
tests/test_pipelines_fill_mask.py
View File

@@ -169,8 +169,8 @@ class FillMaskPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
self.run_pipeline_test(unmasker.model, unmasker.tokenizer, None)
def run_pipeline_test(self, model, tokenizer, feature_extractor):
if tokenizer.mask_token_id is None:
self.skipTest("The provided tokenizer has no mask token, (probably reformer)")
if tokenizer is None or tokenizer.mask_token_id is None:
self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)")
fill_masker = FillMaskPipeline(model=model, tokenizer=tokenizer)

231
tests/test_pipelines_image_classification.py
View File

@@ -14,15 +14,18 @@
import unittest
from transformers import (
AutoConfig,
AutoFeatureExtractor,
AutoModelForImageClassification,
PreTrainedTokenizer,
is_vision_available,
)
from transformers import MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING, PreTrainedTokenizer, is_vision_available
from transformers.pipelines import ImageClassificationPipeline, pipeline
from transformers.testing_utils import require_torch, require_vision
from transformers.testing_utils import (
is_pipeline_test,
nested_simplify,
require_datasets,
require_tf,
require_torch,
require_vision,
)
from .test_pipelines_common import ANY, PipelineTestCaseMeta
if is_vision_available():
@@ -35,127 +38,115 @@ else:
pass
@is_pipeline_test
@require_vision
@require_torch
class ImageClassificationPipelineTests(unittest.TestCase):
pipeline_task = "image-classification"
small_models = ["lysandre/tiny-vit-random"] # Models tested without the @slow decorator
valid_inputs = [
{"images": "http://images.cocodataset.org/val2017/000000039769.jpg"},
{
"images": [
class ImageClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
@require_datasets
def run_pipeline_test(self, model, tokenizer, feature_extractor):
image_classifier = ImageClassificationPipeline(model=model, feature_extractor=feature_extractor)
outputs = image_classifier("./tests/fixtures/tests_samples/COCO/000000039769.png")
self.assertEqual(
outputs,
[
{"score": ANY(float), "label": ANY(str)},
{"score": ANY(float), "label": ANY(str)},
],
)
import datasets
dataset = datasets.load_dataset("Narsil/image_dummy", "image", split="test")
# Accepts URL + PIL.Image + lists
outputs = image_classifier(
[
Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),
"http://images.cocodataset.org/val2017/000000039769.jpg",
# RGBA
dataset[0]["file"],
# LA
dataset[1]["file"],
# L
dataset[2]["file"],
]
)
self.assertEqual(
outputs,
[
[
{"score": ANY(float), "label": ANY(str)},
{"score": ANY(float), "label": ANY(str)},
],
[
{"score": ANY(float), "label": ANY(str)},
{"score": ANY(float), "label": ANY(str)},
],
[
{"score": ANY(float), "label": ANY(str)},
{"score": ANY(float), "label": ANY(str)},
],
[
{"score": ANY(float), "label": ANY(str)},
{"score": ANY(float), "label": ANY(str)},
],
[
{"score": ANY(float), "label": ANY(str)},
{"score": ANY(float), "label": ANY(str)},
],
],
)
@require_torch
def test_small_model_pt(self):
small_model = "lysandre/tiny-vit-random"
image_classifier = pipeline("image-classification", model=small_model)
outputs = image_classifier("http://images.cocodataset.org/val2017/000000039769.jpg")
self.assertEqual(
nested_simplify(outputs, decimals=4),
[
{"score": 0.0015, "label": "chambered nautilus, pearly nautilus, nautilus"},
{"score": 0.0015, "label": "pajama, pyjama, pj's, jammies"},
{"score": 0.0014, "label": "trench coat"},
{"score": 0.0014, "label": "handkerchief, hankie, hanky, hankey"},
{"score": 0.0014, "label": "baboon"},
],
)
outputs = image_classifier(
[
"http://images.cocodataset.org/val2017/000000039769.jpg",
"http://images.cocodataset.org/val2017/000000039769.jpg",
]
},
{"images": "./tests/fixtures/tests_samples/COCO/000000039769.png"},
{
"images": [
"./tests/fixtures/tests_samples/COCO/000000039769.png",
"./tests/fixtures/tests_samples/COCO/000000039769.png",
]
},
{"images": Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")},
{
"images": [
Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),
Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),
]
},
{
"images": [
Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),
"./tests/fixtures/tests_samples/COCO/000000039769.png",
]
},
]
],
top_k=2,
)
self.assertEqual(
nested_simplify(outputs, decimals=4),
[
[
{"score": 0.0015, "label": "chambered nautilus, pearly nautilus, nautilus"},
{"score": 0.0015, "label": "pajama, pyjama, pj's, jammies"},
],
[
{"score": 0.0015, "label": "chambered nautilus, pearly nautilus, nautilus"},
{"score": 0.0015, "label": "pajama, pyjama, pj's, jammies"},
],
],
)
def test_small_model_from_factory(self):
for small_model in self.small_models:
image_classifier = pipeline("image-classification", model=small_model)
for valid_input in self.valid_inputs:
output = image_classifier(**valid_input)
top_k = valid_input.get("top_k", 5)
def assert_valid_pipeline_output(pipeline_output):
self.assertTrue(isinstance(pipeline_output, list))
self.assertEqual(len(pipeline_output), top_k)
for label_result in pipeline_output:
self.assertTrue(isinstance(label_result, dict))
self.assertIn("label", label_result)
self.assertIn("score", label_result)
if isinstance(valid_input["images"], list):
self.assertEqual(len(valid_input["images"]), len(output))
for individual_output in output:
assert_valid_pipeline_output(individual_output)
else:
assert_valid_pipeline_output(output)
def test_small_model_from_pipeline(self):
for small_model in self.small_models:
model = AutoModelForImageClassification.from_pretrained(small_model)
feature_extractor = AutoFeatureExtractor.from_pretrained(small_model)
image_classifier = ImageClassificationPipeline(model=model, feature_extractor=feature_extractor)
for valid_input in self.valid_inputs:
output = image_classifier(**valid_input)
top_k = valid_input.get("top_k", 5)
def assert_valid_pipeline_output(pipeline_output):
self.assertTrue(isinstance(pipeline_output, list))
self.assertEqual(len(pipeline_output), top_k)
for label_result in pipeline_output:
self.assertTrue(isinstance(label_result, dict))
self.assertIn("label", label_result)
self.assertIn("score", label_result)
if isinstance(valid_input["images"], list):
# When images are batched, pipeline output is a list of lists of dictionaries
self.assertEqual(len(valid_input["images"]), len(output))
for individual_output in output:
assert_valid_pipeline_output(individual_output)
else:
# When images are batched, pipeline output is a list of dictionaries
assert_valid_pipeline_output(output)
@require_tf
@unittest.skip("Image classification is not implemented for TF")
def test_small_model_tf(self):
pass
def test_custom_tokenizer(self):
tokenizer = PreTrainedTokenizer()
# Assert that the pipeline can be initialized with a feature extractor that is not in any mapping
image_classifier = pipeline("image-classification", model=self.small_models[0], tokenizer=tokenizer)
image_classifier = pipeline("image-classification", model="lysandre/tiny-vit-random", tokenizer=tokenizer)
self.assertIs(image_classifier.tokenizer, tokenizer)
def test_num_labels_inferior_to_topk(self):
for small_model in self.small_models:
num_labels = 2
model = AutoModelForImageClassification.from_config(
AutoConfig.from_pretrained(small_model, num_labels=num_labels)
)
feature_extractor = AutoFeatureExtractor.from_pretrained(small_model)
image_classifier = ImageClassificationPipeline(model=model, feature_extractor=feature_extractor)
for valid_input in self.valid_inputs:
output = image_classifier(**valid_input)
def assert_valid_pipeline_output(pipeline_output):
self.assertTrue(isinstance(pipeline_output, list))
self.assertEqual(len(pipeline_output), num_labels)
for label_result in pipeline_output:
self.assertTrue(isinstance(label_result, dict))
self.assertIn("label", label_result)
self.assertIn("score", label_result)
if isinstance(valid_input["images"], list):
# When images are batched, pipeline output is a list of lists of dictionaries
self.assertEqual(len(valid_input["images"]), len(output))
for individual_output in output:
assert_valid_pipeline_output(individual_output)
else:
# When images are batched, pipeline output is a list of dictionaries
assert_valid_pipeline_output(output)