SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fast image processor (#28847)

Browse Source 
* Draft fast image processors

* Draft working fast version

* py3.8 compatible cache

* Enable loading fast image processors through auto

* Tidy up; rescale behaviour based on input type

* Enable tests for fast image processors

* Smarter rescaling

* Don't default to Fast

* Safer imports

* Add necessary Pillow requirement

* Woops

* Add AutoImageProcessor test

* Fix up

* Fix test for imagegpt

* Fix test

* Review comments

* Add warning for TF and JAX input types

* Rearrange

* Return transforms

* NumpyToTensor transformation

* Rebase - include changes from upstream in ImageProcessingMixin

* Safe typing

* Fix up

* convert mean/std to tesnor to rescale

* Don't store transforms in state

* Fix up

* Update src/transformers/image_processing_utils_fast.py

Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com>

* Update src/transformers/models/auto/image_processing_auto.py

Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com>

* Update src/transformers/models/auto/image_processing_auto.py

Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com>

* Update src/transformers/models/auto/image_processing_auto.py

Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com>

* Warn if fast image processor available

* Update src/transformers/models/vit/image_processing_vit_fast.py

* Transpose incoming numpy images to be in CHW format

* Update mapping names based on packages, auto set fast to None

* Fix up

* Fix

* Add AutoImageProcessor.from_pretrained(checkpoint, use_fast=True) test

* Update src/transformers/models/vit/image_processing_vit_fast.py

Co-authored-by: Pavel Iakubovskii <qubvel@gmail.com>

* Add equivalence and speed tests

* Fix up

---------

Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com>
Co-authored-by: Pavel Iakubovskii <qubvel@gmail.com>
This commit is contained in:
amyeroberts
2024-06-11 15:47:38 +01:00
committed by GitHub
parent edc1dffd00
commit f53fe35b29
64 changed files with 1645 additions and 813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
tests/models/auto/test_image_processing_auto.py
View File

@@ -27,8 +27,10 @@ from transformers import (
AutoImageProcessor,
CLIPConfig,
CLIPImageProcessor,
ViTImageProcessor,
ViTImageProcessorFast,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_torchvision, require_vision
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))
@@ -133,6 +135,23 @@ class AutoImageProcessorTest(unittest.TestCase):
):
_ = AutoImageProcessor.from_pretrained("hf-internal-testing/config-no-model")
@require_vision
@require_torchvision
def test_use_fast_selection(self):
checkpoint = "hf-internal-testing/tiny-random-vit"
# Slow image processor is selected by default
image_processor = AutoImageProcessor.from_pretrained(checkpoint)
self.assertIsInstance(image_processor, ViTImageProcessor)
# Fast image processor is selected when use_fast=True
image_processor = AutoImageProcessor.from_pretrained(checkpoint, use_fast=True)
self.assertIsInstance(image_processor, ViTImageProcessorFast)
# Slow image processor is selected when use_fast=False
image_processor = AutoImageProcessor.from_pretrained(checkpoint, use_fast=False)
self.assertIsInstance(image_processor, ViTImageProcessor)
def test_from_pretrained_dynamic_image_processor(self):
# If remote code is not set, we will time out when asking whether to load the model.
with self.assertRaises(ValueError):

1
tests/models/beit/test_image_processing_beit.py
View File

@@ -121,6 +121,7 @@ class BeitImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = BeitImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = BeitImageProcessingTester(self)
@property

2
tests/models/blip/test_image_processing_blip.py
View File

@@ -90,6 +90,7 @@ class BlipImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = BlipImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = BlipImageProcessingTester(self)
@property
@@ -112,6 +113,7 @@ class BlipImageProcessingTestFourChannels(ImageProcessingTestMixin, unittest.Tes
image_processing_class = BlipImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = BlipImageProcessingTester(self, num_channels=4)
self.expected_encoded_image_num_channels = 3

1
tests/models/bridgetower/test_image_processing_bridgetower.py
View File

@@ -136,6 +136,7 @@ class BridgeTowerImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase
image_processing_class = BridgeTowerImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = BridgeTowerImageProcessingTester(self)
@property

2
tests/models/chinese_clip/test_image_processing_chinese_clip.py
View File

@@ -98,6 +98,7 @@ class ChineseCLIPImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase
image_processing_class = ChineseCLIPImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = ChineseCLIPImageProcessingTester(self, do_center_crop=True)
@property
@@ -135,6 +136,7 @@ class ChineseCLIPImageProcessingTestFourChannels(ImageProcessingTestMixin, unitt
image_processing_class = ChineseCLIPImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = ChineseCLIPImageProcessingTester(self, num_channels=4, do_center_crop=True)
self.expected_encoded_image_num_channels = 3

1
tests/models/clip/test_image_processing_clip.py
View File

@@ -94,6 +94,7 @@ class CLIPImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = CLIPImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = CLIPImageProcessingTester(self)
@property

1
tests/models/conditional_detr/test_image_processing_conditional_detr.py
View File

@@ -131,6 +131,7 @@ class ConditionalDetrImageProcessingTest(AnnotationFormatTestMixin, ImageProcess
image_processing_class = ConditionalDetrImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = ConditionalDetrImageProcessingTester(self)
@property

1
tests/models/convnext/test_image_processing_convnext.py
View File

@@ -87,6 +87,7 @@ class ConvNextImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = ConvNextImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = ConvNextImageProcessingTester(self)
@property

1
tests/models/deformable_detr/test_image_processing_deformable_detr.py
View File

@@ -131,6 +131,7 @@ class DeformableDetrImageProcessingTest(AnnotationFormatTestMixin, ImageProcessi
image_processing_class = DeformableDetrImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = DeformableDetrImageProcessingTester(self)
@property

1
tests/models/deit/test_image_processing_deit.py
View File

@@ -93,6 +93,7 @@ class DeiTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
test_cast_dtype = True
def setUp(self):
super().setUp()
self.image_processor_tester = DeiTImageProcessingTester(self)
@property

1
tests/models/detr/test_image_processing_detr.py
View File

@@ -130,6 +130,7 @@ class DetrImageProcessingTest(AnnotationFormatTestMixin, ImageProcessingTestMixi
image_processing_class = DetrImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = DetrImageProcessingTester(self)
@property

1
tests/models/donut/test_image_processing_donut.py
View File

@@ -99,6 +99,7 @@ class DonutImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = DonutImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = DonutImageProcessingTester(self)
@property

1
tests/models/dpt/test_image_processing_dpt.py
View File

@@ -86,6 +86,7 @@ class DPTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = DPTImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = DPTImageProcessingTester(self)
@property

1
tests/models/efficientnet/test_image_processing_efficientnet.py
View File

@@ -86,6 +86,7 @@ class EfficientNetImageProcessorTest(ImageProcessingTestMixin, unittest.TestCase
image_processing_class = EfficientNetImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = EfficientNetImageProcessorTester(self)
@property

1
tests/models/flava/test_image_processing_flava.py
View File

@@ -175,6 +175,7 @@ class FlavaImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
maxDiff = None
def setUp(self):
super().setUp()
self.image_processor_tester = FlavaImageProcessingTester(self)
@property

1
tests/models/glpn/test_image_processing_glpn.py
View File

@@ -93,6 +93,7 @@ class GLPNImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = GLPNImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = GLPNImageProcessingTester(self)
@property

1
tests/models/grounding_dino/test_image_processing_grounding_dino.py
View File

@@ -146,6 +146,7 @@ class GroundingDinoImageProcessingTest(AnnotationFormatTestMixin, ImageProcessin
image_processing_class = GroundingDinoImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = GroundingDinoImageProcessingTester(self)
@property

1
tests/models/idefics/test_image_processing_idefics.py
View File

@@ -127,6 +127,7 @@ class IdeficsImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = IdeficsImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = IdeficsImageProcessingTester(self)
@property

1
tests/models/idefics2/test_image_processing_idefics2.py
View File

@@ -185,6 +185,7 @@ class Idefics2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = Idefics2ImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = Idefics2ImageProcessingTester(self)
@property

44
tests/models/imagegpt/test_image_processing_imagegpt.py
View File

@@ -22,7 +22,8 @@ import unittest
import numpy as np
from datasets import load_dataset
from transformers.testing_utils import require_torch, require_vision, slow
from transformers import AutoImageProcessor
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available
from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs
@@ -96,6 +97,7 @@ class ImageGPTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = ImageGPTImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = ImageGPTImageProcessingTester(self)
@property
@@ -141,18 +143,38 @@ class ImageGPTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
self.assertEqual(image_processor_first[key], value)
def test_image_processor_from_and_save_pretrained(self):
image_processor_first = self.image_processing_class(**self.image_processor_dict)
for image_processing_class in self.image_processor_list:
image_processor_first = self.image_processing_class(**self.image_processor_dict)
with tempfile.TemporaryDirectory() as tmpdirname:
image_processor_first.save_pretrained(tmpdirname)
image_processor_second = self.image_processing_class.from_pretrained(tmpdirname).to_dict()
with tempfile.TemporaryDirectory() as tmpdirname:
image_processor_first.save_pretrained(tmpdirname)
image_processor_second = self.image_processing_class.from_pretrained(tmpdirname).to_dict()
image_processor_first = image_processor_first.to_dict()
for key, value in image_processor_first.items():
if key == "clusters":
self.assertTrue(np.array_equal(value, image_processor_second[key]))
else:
self.assertEqual(image_processor_first[key], value)
image_processor_first = image_processor_first.to_dict()
for key, value in image_processor_first.items():
if key == "clusters":
self.assertTrue(np.array_equal(value, image_processor_second[key]))
else:
self.assertEqual(image_processor_first[key], value)
def test_image_processor_save_load_with_autoimageprocessor(self):
for image_processing_class in self.image_processor_list:
image_processor_first = image_processing_class(**self.image_processor_dict)
with tempfile.TemporaryDirectory() as tmpdirname:
saved_file = image_processor_first.save_pretrained(tmpdirname)[0]
check_json_file_has_correct_format(saved_file)
image_processor_second = AutoImageProcessor.from_pretrained(tmpdirname)
image_processor_first = image_processor_first.to_dict()
image_processor_second = image_processor_second.to_dict()
for key, value in image_processor_first.items():
if key == "clusters":
self.assertTrue(np.array_equal(value, image_processor_second[key]))
else:
self.assertEqual(image_processor_first[key], value)
@unittest.skip("ImageGPT requires clusters at initialization")
def test_init_without_params(self):

1
tests/models/layoutlmv2/test_image_processing_layoutlmv2.py
View File

@@ -76,6 +76,7 @@ class LayoutLMv2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
image_processing_class = LayoutLMv2ImageProcessor if is_pytesseract_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = LayoutLMv2ImageProcessingTester(self)
@property

1
tests/models/layoutlmv3/test_image_processing_layoutlmv3.py
View File

@@ -76,6 +76,7 @@ class LayoutLMv3ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
image_processing_class = LayoutLMv3ImageProcessor if is_pytesseract_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = LayoutLMv3ImageProcessingTester(self)
@property

1
tests/models/levit/test_image_processing_levit.py
View File

@@ -91,6 +91,7 @@ class LevitImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = LevitImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = LevitImageProcessingTester(self)
@property

1
tests/models/llava_next/test_image_processor_llava_next.py
View File

@@ -105,6 +105,7 @@ class LlavaNextImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
# Copied from tests.models.clip.test_image_processing_clip.CLIPImageProcessingTest.setUp with CLIP->LlavaNext
def setUp(self):
super().setUp()
self.image_processor_tester = LlavaNextImageProcessingTester(self)
@property

1
tests/models/mask2former/test_image_processing_mask2former.py
View File

@@ -149,6 +149,7 @@ class Mask2FormerImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase
image_processing_class = Mask2FormerImageProcessor if (is_vision_available() and is_torch_available()) else None
def setUp(self):
super().setUp()
self.image_processor_tester = Mask2FormerImageProcessingTester(self)
@property

1
tests/models/maskformer/test_image_processing_maskformer.py
View File

@@ -149,6 +149,7 @@ class MaskFormerImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
image_processing_class = MaskFormerImageProcessor if (is_vision_available() and is_torch_available()) else None
def setUp(self):
super().setUp()
self.image_processor_tester = MaskFormerImageProcessingTester(self)
@property

1
tests/models/mobilenet_v1/test_image_processing_mobilenet_v1.py
View File

@@ -82,6 +82,7 @@ class MobileNetV1ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase
image_processing_class = MobileNetV1ImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = MobileNetV1ImageProcessingTester(self)
@property

1
tests/models/mobilenet_v2/test_image_processing_mobilenet_v2.py
View File

@@ -82,6 +82,7 @@ class MobileNetV2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase
image_processing_class = MobileNetV2ImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = MobileNetV2ImageProcessingTester(self)
@property

1
tests/models/mobilevit/test_image_processing_mobilevit.py
View File

@@ -112,6 +112,7 @@ class MobileViTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = MobileViTImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = MobileViTImageProcessingTester(self)
@property

1
tests/models/nougat/test_image_processing_nougat.py
View File

@@ -111,6 +111,7 @@ class NougatImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = NougatImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = NougatImageProcessingTester(self)
@property

1
tests/models/oneformer/test_image_processing_oneformer.py
View File

@@ -159,6 +159,7 @@ class OneFormerImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = image_processing_class
def setUp(self):
super().setUp()
self.image_processor_tester = OneFormerImageProcessorTester(self)
@property

1
tests/models/owlv2/test_image_processor_owlv2.py
View File

@@ -90,6 +90,7 @@ class Owlv2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = Owlv2ImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = Owlv2ImageProcessingTester(self)
@property

1
tests/models/owlvit/test_image_processing_owlvit.py
View File

@@ -92,6 +92,7 @@ class OwlViTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = OwlViTImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = OwlViTImageProcessingTester(self)
@property

2
tests/models/pix2struct/test_image_processing_pix2struct.py
View File

@@ -87,6 +87,7 @@ class Pix2StructImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
image_processing_class = Pix2StructImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = Pix2StructImageProcessingTester(self)
@property
@@ -288,6 +289,7 @@ class Pix2StructImageProcessingTestFourChannels(ImageProcessingTestMixin, unitte
image_processing_class = Pix2StructImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = Pix2StructImageProcessingTester(self, num_channels=4)
self.expected_encoded_image_num_channels = 3

1
tests/models/poolformer/test_image_processing_poolformer.py
View File

@@ -88,6 +88,7 @@ class PoolFormerImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
image_processing_class = PoolFormerImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = PoolFormerImageProcessingTester(self)
@property

1
tests/models/pvt/test_image_processing_pvt.py
View File

@@ -84,6 +84,7 @@ class PvtImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = PvtImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = PvtImageProcessingTester(self)
@property

1
tests/models/segformer/test_image_processing_segformer.py
View File

@@ -112,6 +112,7 @@ class SegformerImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = SegformerImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = SegformerImageProcessingTester(self)
@property

1
tests/models/seggpt/test_image_processing_seggpt.py
View File

@@ -114,6 +114,7 @@ class SegGptImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = SegGptImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = SegGptImageProcessingTester(self)
@property

1
tests/models/siglip/test_image_processor_siglip.py
View File

@@ -91,6 +91,7 @@ class SiglipImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = SiglipImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = SiglipImageProcessingTester(self)
@property

1
tests/models/superpoint/test_image_processing_superpoint.py
View File

@@ -77,6 +77,7 @@ class SuperPointImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
image_processing_class = SuperPointImageProcessor if is_vision_available() else None
def setUp(self) -> None:
super().setUp()
self.image_processor_tester = SuperPointImageProcessingTester(self)
@property

1
tests/models/swin2sr/test_image_processing_swin2sr.py
View File

@@ -98,6 +98,7 @@ class Swin2SRImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = Swin2SRImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = Swin2SRImageProcessingTester(self)
@property

1
tests/models/tvp/test_image_processing_tvp.py
View File

@@ -127,6 +127,7 @@ class TvpImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = TvpImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = TvpImageProcessingTester(self)
@property

1
tests/models/video_llava/test_image_processing_video_llava.py
View File

@@ -128,6 +128,7 @@ class VideoLlavaImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
# Copied from tests.models.clip.test_image_processing_clip.CLIPImageProcessingTest.setUp with CLIP->VideoLlava
def setUp(self):
super().setUp()
self.image_processor_tester = VideoLlavaImageProcessingTester(self)
@property

1
tests/models/videomae/test_image_processing_videomae.py
View File

@@ -99,6 +99,7 @@ class VideoMAEImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = VideoMAEImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = VideoMAEImageProcessingTester(self)
@property

1
tests/models/vilt/test_image_processing_vilt.py
View File

@@ -130,6 +130,7 @@ class ViltImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = ViltImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = ViltImageProcessingTester(self)
@property

23
tests/models/vit/test_image_processing_vit.py
View File

@@ -84,6 +84,7 @@ class ViTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = ViTImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = ViTImageProcessingTester(self)
@property
@@ -91,16 +92,18 @@ class ViTImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
return self.image_processor_tester.prepare_image_processor_dict()
def test_image_processor_properties(self):
image_processing = self.image_processing_class(**self.image_processor_dict)
self.assertTrue(hasattr(image_processing, "image_mean"))
self.assertTrue(hasattr(image_processing, "image_std"))
self.assertTrue(hasattr(image_processing, "do_normalize"))
self.assertTrue(hasattr(image_processing, "do_resize"))
self.assertTrue(hasattr(image_processing, "size"))
for image_processing_class in self.image_processor_list:
image_processing = image_processing_class(**self.image_processor_dict)
self.assertTrue(hasattr(image_processing, "image_mean"))
self.assertTrue(hasattr(image_processing, "image_std"))
self.assertTrue(hasattr(image_processing, "do_normalize"))
self.assertTrue(hasattr(image_processing, "do_resize"))
self.assertTrue(hasattr(image_processing, "size"))
def test_image_processor_from_dict_with_kwargs(self):
image_processor = self.image_processing_class.from_dict(self.image_processor_dict)
self.assertEqual(image_processor.size, {"height": 18, "width": 18})
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class.from_dict(self.image_processor_dict)
self.assertEqual(image_processor.size, {"height": 18, "width": 18})
image_processor = self.image_processing_class.from_dict(self.image_processor_dict, size=42)
self.assertEqual(image_processor.size, {"height": 42, "width": 42})
image_processor = image_processing_class.from_dict(self.image_processor_dict, size=42)
self.assertEqual(image_processor.size, {"height": 42, "width": 42})

1
tests/models/vitmatte/test_image_processing_vitmatte.py
View File

@@ -94,6 +94,7 @@ class VitMatteImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = VitMatteImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = VitMatteImageProcessingTester(self)
@property

1
tests/models/vivit/test_image_processing_vivit.py
View File

@@ -99,6 +99,7 @@ class VivitImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = VivitImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = VivitImageProcessingTester(self)
@property

1
tests/models/yolos/test_image_processing_yolos.py
View File

@@ -143,6 +143,7 @@ class YolosImageProcessingTest(AnnotationFormatTestMixin, ImageProcessingTestMix
image_processing_class = YolosImageProcessor if is_vision_available() else None
def setUp(self):
super().setUp()
self.image_processor_tester = YolosImageProcessingTester(self)
@property

373
tests/test_image_processing_common.py
View File

@@ -19,7 +19,9 @@ import os
import pathlib
import tempfile
from transformers import BatchFeature
import requests
from transformers import AutoImageProcessor, BatchFeature
from transformers.image_utils import AnnotationFormat, AnnotionFormat
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available
@@ -129,176 +131,263 @@ def prepare_video_inputs(
class ImageProcessingTestMixin:
test_cast_dtype = None
image_processing_class = None
fast_image_processing_class = None
image_processors_list = None
test_slow_image_processor = True
test_fast_image_processor = True
def setUp(self):
image_processor_list = []
if self.test_slow_image_processor and self.image_processing_class:
image_processor_list.append(self.image_processing_class)
if self.test_fast_image_processor and self.fast_image_processing_class:
image_processor_list.append(self.fast_image_processing_class)
self.image_processor_list = image_processor_list
@require_vision
@require_torch
def test_slow_fast_equivalence(self):
dummy_image = Image.open(
requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw
)
if not self.test_slow_image_processor or not self.test_fast_image_processor:
self.skipTest("Skipping slow/fast equivalence test")
if self.image_processing_class is None or self.fast_image_processing_class is None:
self.skipTest("Skipping slow/fast equivalence test as one of the image processors is not defined")
image_processor_slow = self.image_processing_class(**self.image_processor_dict)
image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict)
encoding_slow = image_processor_slow(dummy_image, return_tensors="pt")
encoding_fast = image_processor_fast(dummy_image, return_tensors="pt")
self.assertTrue(torch.allclose(encoding_slow.pixel_values, encoding_fast.pixel_values, atol=1e-3))
@require_vision
@require_torch
def test_fast_is_faster_than_slow(self):
import time
def measure_time(self, image_processor, dummy_image):
start = time.time()
_ = image_processor(dummy_image, return_tensors="pt")
return time.time() - start
dummy_image = Image.open(
requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw
)
if not self.test_slow_image_processor or not self.test_fast_image_processor:
self.skipTest("Skipping speed test")
if self.image_processing_class is None or self.fast_image_processing_class is None:
self.skipTest("Skipping speed test as one of the image processors is not defined")
image_processor_slow = self.image_processing_class(**self.image_processor_dict)
image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict)
slow_time = self.measure_time(image_processor_slow, dummy_image)
fast_time = self.measure_time(image_processor_fast, dummy_image)
self.assertLessEqual(fast_time, slow_time)
def test_image_processor_to_json_string(self):
image_processor = self.image_processing_class(**self.image_processor_dict)
obj = json.loads(image_processor.to_json_string())
for key, value in self.image_processor_dict.items():
self.assertEqual(obj[key], value)
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class(**self.image_processor_dict)
obj = json.loads(image_processor.to_json_string())
for key, value in self.image_processor_dict.items():
self.assertEqual(obj[key], value)
def test_image_processor_to_json_file(self):
image_processor_first = self.image_processing_class(**self.image_processor_dict)
for image_processing_class in self.image_processor_list:
image_processor_first = image_processing_class(**self.image_processor_dict)
with tempfile.TemporaryDirectory() as tmpdirname:
json_file_path = os.path.join(tmpdirname, "image_processor.json")
image_processor_first.to_json_file(json_file_path)
image_processor_second = self.image_processing_class.from_json_file(json_file_path)
with tempfile.TemporaryDirectory() as tmpdirname:
json_file_path = os.path.join(tmpdirname, "image_processor.json")
image_processor_first.to_json_file(json_file_path)
image_processor_second = image_processing_class.from_json_file(json_file_path)
self.assertEqual(image_processor_second.to_dict(), image_processor_first.to_dict())
self.assertEqual(image_processor_second.to_dict(), image_processor_first.to_dict())
def test_image_processor_from_and_save_pretrained(self):
image_processor_first = self.image_processing_class(**self.image_processor_dict)
for image_processing_class in self.image_processor_list:
image_processor_first = image_processing_class(**self.image_processor_dict)
with tempfile.TemporaryDirectory() as tmpdirname:
saved_file = image_processor_first.save_pretrained(tmpdirname)[0]
check_json_file_has_correct_format(saved_file)
image_processor_second = self.image_processing_class.from_pretrained(tmpdirname)
with tempfile.TemporaryDirectory() as tmpdirname:
saved_file = image_processor_first.save_pretrained(tmpdirname)[0]
check_json_file_has_correct_format(saved_file)
image_processor_second = image_processing_class.from_pretrained(tmpdirname)
self.assertEqual(image_processor_second.to_dict(), image_processor_first.to_dict())
self.assertEqual(image_processor_second.to_dict(), image_processor_first.to_dict())
def test_image_processor_save_load_with_autoimageprocessor(self):
for image_processing_class in self.image_processor_list:
image_processor_first = image_processing_class(**self.image_processor_dict)
with tempfile.TemporaryDirectory() as tmpdirname:
saved_file = image_processor_first.save_pretrained(tmpdirname)[0]
check_json_file_has_correct_format(saved_file)
image_processor_second = AutoImageProcessor.from_pretrained(tmpdirname)
self.assertEqual(image_processor_second.to_dict(), image_processor_first.to_dict())
def test_init_without_params(self):
image_processor = self.image_processing_class()
self.assertIsNotNone(image_processor)
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class()
self.assertIsNotNone(image_processor)
@require_torch
@require_vision
def test_cast_dtype_device(self):
if self.test_cast_dtype is not None:
# Initialize image_processor
image_processor = self.image_processing_class(**self.image_processor_dict)
for image_processing_class in self.image_processor_list:
if self.test_cast_dtype is not None:
# Initialize image_processor
image_processor = image_processing_class(**self.image_processor_dict)
# create random PyTorch tensors
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True)
encoding = image_processor(image_inputs, return_tensors="pt")
# for layoutLM compatiblity
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.float32)
encoding = image_processor(image_inputs, return_tensors="pt").to(torch.float16)
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.float16)
encoding = image_processor(image_inputs, return_tensors="pt").to("cpu", torch.bfloat16)
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.bfloat16)
with self.assertRaises(TypeError):
_ = image_processor(image_inputs, return_tensors="pt").to(torch.bfloat16, "cpu")
# Try with text + image feature
encoding = image_processor(image_inputs, return_tensors="pt")
encoding.update({"input_ids": torch.LongTensor([[1, 2, 3], [4, 5, 6]])})
encoding = encoding.to(torch.float16)
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.float16)
self.assertEqual(encoding.input_ids.dtype, torch.long)
def test_call_pil(self):
for image_processing_class in self.image_processor_list:
# Initialize image_processing
image_processing = image_processing_class(**self.image_processor_dict)
# create random PIL images
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False)
for image in image_inputs:
self.assertIsInstance(image, Image.Image)
# Test not batched input
encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test batched
encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
self.assertEqual(
tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape)
)
def test_call_numpy(self):
for image_processing_class in self.image_processor_list:
# Initialize image_processing
image_processing = image_processing_class(**self.image_processor_dict)
# create random numpy tensors
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True)
for image in image_inputs:
self.assertIsInstance(image, np.ndarray)
# Test not batched input
encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test batched
encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
self.assertEqual(
tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape)
)
def test_call_pytorch(self):
for image_processing_class in self.image_processor_list:
# Initialize image_processing
image_processing = image_processing_class(**self.image_processor_dict)
# create random PyTorch tensors
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True)
encoding = image_processor(image_inputs, return_tensors="pt")
# for layoutLM compatiblity
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.float32)
for image in image_inputs:
self.assertIsInstance(image, torch.Tensor)
encoding = image_processor(image_inputs, return_tensors="pt").to(torch.float16)
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.float16)
# Test not batched input
encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
encoding = image_processor(image_inputs, return_tensors="pt").to("cpu", torch.bfloat16)
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.bfloat16)
with self.assertRaises(TypeError):
_ = image_processor(image_inputs, return_tensors="pt").to(torch.bfloat16, "cpu")
# Try with text + image feature
encoding = image_processor(image_inputs, return_tensors="pt")
encoding.update({"input_ids": torch.LongTensor([[1, 2, 3], [4, 5, 6]])})
encoding = encoding.to(torch.float16)
self.assertEqual(encoding.pixel_values.device, torch.device("cpu"))
self.assertEqual(encoding.pixel_values.dtype, torch.float16)
self.assertEqual(encoding.input_ids.dtype, torch.long)
def test_call_pil(self):
# Initialize image_processing
image_processing = self.image_processing_class(**self.image_processor_dict)
# create random PIL images
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False)
for image in image_inputs:
self.assertIsInstance(image, Image.Image)
# Test not batched input
encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test batched
encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
self.assertEqual(
tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape)
)
def test_call_numpy(self):
# Initialize image_processing
image_processing = self.image_processing_class(**self.image_processor_dict)
# create random numpy tensors
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True)
for image in image_inputs:
self.assertIsInstance(image, np.ndarray)
# Test not batched input
encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test batched
encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
self.assertEqual(
tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape)
)
def test_call_pytorch(self):
# Initialize image_processing
image_processing = self.image_processing_class(**self.image_processor_dict)
# create random PyTorch tensors
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True)
for image in image_inputs:
self.assertIsInstance(image, torch.Tensor)
# Test not batched input
encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test batched
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values
self.assertEqual(
tuple(encoded_images.shape),
(self.image_processor_tester.batch_size, *expected_output_image_shape),
)
# Test batched
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values
self.assertEqual(
tuple(encoded_images.shape),
(self.image_processor_tester.batch_size, *expected_output_image_shape),
)
def test_call_numpy_4_channels(self):
# Test that can process images which have an arbitrary number of channels
# Initialize image_processing
image_processor = self.image_processing_class(**self.image_processor_dict)
for image_processing_class in self.image_processor_list:
# Test that can process images which have an arbitrary number of channels
# Initialize image_processing
image_processor = image_processing_class(**self.image_processor_dict)
# create random numpy tensors
self.image_processor_tester.num_channels = 4
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True)
# create random numpy tensors
self.image_processor_tester.num_channels = 4
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True)
# Test not batched input
encoded_images = image_processor(
image_inputs[0],
return_tensors="pt",
input_data_format="channels_first",
image_mean=0,
image_std=1,
).pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test not batched input
encoded_images = image_processor(
image_inputs[0],
return_tensors="pt",
input_data_format="channels_first",
image_mean=0,
image_std=1,
).pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]])
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test batched
encoded_images = image_processor(
image_inputs,
return_tensors="pt",
input_data_format="channels_first",
image_mean=0,
image_std=1,
).pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
self.assertEqual(
tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape)
)
# Test batched
encoded_images = image_processor(
image_inputs,
return_tensors="pt",
input_data_format="channels_first",
image_mean=0,
image_std=1,
).pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
self.assertEqual(
tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape)
)
def test_image_processor_preprocess_arguments(self):
image_processor = self.image_processing_class(**self.image_processor_dict)
if hasattr(image_processor, "_valid_processor_keys") and hasattr(image_processor, "preprocess"):
preprocess_parameter_names = inspect.getfullargspec(image_processor.preprocess).args
preprocess_parameter_names.remove("self")
preprocess_parameter_names.sort()
valid_processor_keys = image_processor._valid_processor_keys
valid_processor_keys.sort()
self.assertEqual(preprocess_parameter_names, valid_processor_keys)
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class(**self.image_processor_dict)
if hasattr(image_processor, "_valid_processor_keys") and hasattr(image_processor, "preprocess"):
preprocess_parameter_names = inspect.getfullargspec(image_processor.preprocess).args
preprocess_parameter_names.remove("self")
preprocess_parameter_names.sort()
valid_processor_keys = image_processor._valid_processor_keys
valid_processor_keys.sort()
self.assertEqual(preprocess_parameter_names, valid_processor_keys)
class AnnotationFormatTestMixin: