Refactoring of ImageProcessorFast (#35069)
* add init and base image processing functions * add add_fast_image_processor to transformers-cli * add working fast image processor clip * add fast image processor to doc, working tests * remove "to be implemented" SigLip * fix unprotected import * fix unprotected vision import * update ViTImageProcessorFast * increase threshold slow fast ewuivalence * add fast img blip * add fast class in tests with cli * improve cli * add fast image processor convnext * add LlavaPatchingMixin and fast image processor for llava_next and llava_onevision * add device kwarg to ImagesKwargs for fast processing on cuda * cleanup * fix unprotected import * group images by sizes and add batch processing * Add batch equivalence tests, skip when center_crop is used * cleanup * update init and cli * fix-copies * refactor convnext, cleanup base * fix * remove patching mixins, add piped torchvision transforms for ViT * fix unbatched processing * fix f strings * protect imports * change llava onevision to class transforms (test) * fix convnext * improve formatting (following Pavel review) * fix handling device arg * improve cli * fix * fix inits * Add distinction between preprocess and _preprocess, and support for arbitrary kwargs through valid_extra_kwargs * uniformize qwen2_vl fast * fix docstrings * add add fast image processor llava * remove min_pixels max_pixels from accepted size * nit * nit * refactor fast image processors docstrings * cleanup and remove fast class transforms * update add fast image processor transformers cli * cleanup docstring * uniformize pixtral fast and make _process_image explicit * fix prepare image structure llava next/onevision * Use typed kwargs instead of explicit args * nit fix import Unpack * clearly separate pops and gets in base preprocess. Use explicit typed kwargs * make qwen2_vl preprocess arguments hashable
This commit is contained in:
Yoni Gozlan
@@ -165,23 +165,50 @@ class ImageProcessingTestMixin:
|
||||
@require_vision
|
||||
@require_torch
|
||||
def test_slow_fast_equivalence(self):
|
||||
dummy_image = Image.open(
|
||||
requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw
|
||||
)
|
||||
|
||||
if not self.test_slow_image_processor or not self.test_fast_image_processor:
|
||||
self.skipTest(reason="Skipping slow/fast equivalence test")
|
||||
|
||||
if self.image_processing_class is None or self.fast_image_processing_class is None:
|
||||
self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined")
|
||||
|
||||
dummy_image = Image.open(
|
||||
requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw
|
||||
)
|
||||
image_processor_slow = self.image_processing_class(**self.image_processor_dict)
|
||||
image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict)
|
||||
|
||||
encoding_slow = image_processor_slow(dummy_image, return_tensors="pt")
|
||||
encoding_fast = image_processor_fast(dummy_image, return_tensors="pt")
|
||||
self.assertTrue(torch.allclose(encoding_slow.pixel_values, encoding_fast.pixel_values, atol=1e-1))
|
||||
self.assertLessEqual(
|
||||
torch.mean(torch.abs(encoding_slow.pixel_values - encoding_fast.pixel_values)).item(), 1e-3
|
||||
)
|
||||
|
||||
torch.testing.assert_close(encoding_slow.pixel_values, encoding_fast.pixel_values, rtol=1e-1, atol=1e-2)
|
||||
@require_vision
|
||||
@require_torch
|
||||
def test_slow_fast_equivalence_batched(self):
|
||||
if not self.test_slow_image_processor or not self.test_fast_image_processor:
|
||||
self.skipTest(reason="Skipping slow/fast equivalence test")
|
||||
|
||||
if self.image_processing_class is None or self.fast_image_processing_class is None:
|
||||
self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined")
|
||||
|
||||
if hasattr(self.image_processor_tester, "do_center_crop") and self.image_processor_tester.do_center_crop:
|
||||
self.skipTest(
|
||||
reason="Skipping as do_center_crop is True and center_crop functions are not equivalent for fast and slow processors"
|
||||
)
|
||||
|
||||
dummy_images = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True)
|
||||
image_processor_slow = self.image_processing_class(**self.image_processor_dict)
|
||||
image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict)
|
||||
|
||||
encoding_slow = image_processor_slow(dummy_images, return_tensors="pt")
|
||||
encoding_fast = image_processor_fast(dummy_images, return_tensors="pt")
|
||||
|
||||
self.assertTrue(torch.allclose(encoding_slow.pixel_values, encoding_fast.pixel_values, atol=1e-1))
|
||||
self.assertLessEqual(
|
||||
torch.mean(torch.abs(encoding_slow.pixel_values - encoding_fast.pixel_values)).item(), 1e-3
|
||||
)
|
||||
|
||||
@require_vision
|
||||
@require_torch
|
||||
@@ -194,7 +221,8 @@ class ImageProcessingTestMixin:
|
||||
|
||||
def measure_time(image_processor, image):
|
||||
# Warmup
|
||||
_ = image_processor(image, return_tensors="pt")
|
||||
for _ in range(5):
|
||||
_ = image_processor(image, return_tensors="pt")
|
||||
start = time.time()
|
||||
_ = image_processor(image, return_tensors="pt")
|
||||
return time.time() - start
|
||||
@@ -270,8 +298,31 @@ class ImageProcessingTestMixin:
|
||||
image_processor_fast_1.save_pretrained(tmpdirname)
|
||||
image_processor_slow_1 = self.image_processing_class.from_pretrained(tmpdirname)
|
||||
|
||||
self.assertEqual(image_processor_slow_0.to_dict(), image_processor_slow_1.to_dict())
|
||||
self.assertEqual(image_processor_fast_0.to_dict(), image_processor_fast_1.to_dict())
|
||||
dict_slow_0 = image_processor_slow_0.to_dict()
|
||||
dict_slow_1 = image_processor_slow_1.to_dict()
|
||||
difference = {
|
||||
key: dict_slow_0.get(key) if key in dict_slow_0 else dict_slow_1.get(key)
|
||||
for key in set(dict_slow_0) ^ set(dict_slow_1)
|
||||
}
|
||||
dict_slow_0 = {key: dict_slow_0[key] for key in set(dict_slow_0) & set(dict_slow_1)}
|
||||
dict_slow_1 = {key: dict_slow_1[key] for key in set(dict_slow_0) & set(dict_slow_1)}
|
||||
# check that all additional keys are None, except for `default_to_square` which is only set in fast processors
|
||||
self.assertTrue(all(value is None for key, value in difference.items() if key not in ["default_to_square"]))
|
||||
# check that the remaining keys are the same
|
||||
self.assertEqual(dict_slow_0, dict_slow_1)
|
||||
|
||||
dict_fast_0 = image_processor_fast_0.to_dict()
|
||||
dict_fast_1 = image_processor_fast_1.to_dict()
|
||||
difference = {
|
||||
key: dict_fast_0.get(key) if key in dict_fast_0 else dict_fast_1.get(key)
|
||||
for key in set(dict_fast_0) ^ set(dict_fast_1)
|
||||
}
|
||||
dict_fast_0 = {key: dict_fast_0[key] for key in set(dict_fast_0) & set(dict_fast_1)}
|
||||
dict_fast_1 = {key: dict_fast_1[key] for key in set(dict_fast_0) & set(dict_fast_1)}
|
||||
# check that all additional keys are None, except for `default_to_square` which is only set in fast processors
|
||||
self.assertTrue(all(value is None for key, value in difference.items() if key not in ["default_to_square"]))
|
||||
# check that the remaining keys are the same
|
||||
self.assertEqual(dict_fast_0, dict_fast_1)
|
||||
|
||||
def test_save_load_fast_slow_auto(self):
|
||||
"Test that we can load a fast image processor from a slow one and vice-versa using AutoImageProcessor."
|
||||
@@ -293,8 +344,31 @@ class ImageProcessingTestMixin:
|
||||
image_processor_fast_1.save_pretrained(tmpdirname)
|
||||
image_processor_slow_1 = AutoImageProcessor.from_pretrained(tmpdirname, use_fast=False)
|
||||
|
||||
self.assertEqual(image_processor_slow_0.to_dict(), image_processor_slow_1.to_dict())
|
||||
self.assertEqual(image_processor_fast_0.to_dict(), image_processor_fast_1.to_dict())
|
||||
dict_slow_0 = image_processor_slow_0.to_dict()
|
||||
dict_slow_1 = image_processor_slow_1.to_dict()
|
||||
difference = {
|
||||
key: dict_slow_0.get(key) if key in dict_slow_0 else dict_slow_1.get(key)
|
||||
for key in set(dict_slow_0) ^ set(dict_slow_1)
|
||||
}
|
||||
dict_slow_0 = {key: dict_slow_0[key] for key in set(dict_slow_0) & set(dict_slow_1)}
|
||||
dict_slow_1 = {key: dict_slow_1[key] for key in set(dict_slow_0) & set(dict_slow_1)}
|
||||
# check that all additional keys are None, except for `default_to_square` which is only set in fast processors
|
||||
self.assertTrue(all(value is None for key, value in difference.items() if key not in ["default_to_square"]))
|
||||
# check that the remaining keys are the same
|
||||
self.assertEqual(dict_slow_0, dict_slow_1)
|
||||
|
||||
dict_fast_0 = image_processor_fast_0.to_dict()
|
||||
dict_fast_1 = image_processor_fast_1.to_dict()
|
||||
difference = {
|
||||
key: dict_fast_0.get(key) if key in dict_fast_0 else dict_fast_1.get(key)
|
||||
for key in set(dict_fast_0) ^ set(dict_fast_1)
|
||||
}
|
||||
dict_fast_0 = {key: dict_fast_0[key] for key in set(dict_fast_0) & set(dict_fast_1)}
|
||||
dict_fast_1 = {key: dict_fast_1[key] for key in set(dict_fast_0) & set(dict_fast_1)}
|
||||
# check that all additional keys are None, except for `default_to_square` which is only set in fast processors
|
||||
self.assertTrue(all(value is None for key, value in difference.items() if key not in ["default_to_square"]))
|
||||
# check that the remaining keys are the same
|
||||
self.assertEqual(dict_fast_0, dict_fast_1)
|
||||
|
||||
def test_init_without_params(self):
|
||||
for image_processing_class in self.image_processor_list:
|
||||
|
||||
Reference in New Issue
Block a user