SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

add fast image processor nougat (#37661)

Browse Source 
* add fast image processor nougat

* test fixes

* docstring white space

* last fixes

* docstring_type

* tolerance unit test

* fix tolerance

* fix rtol

* remove traling white space

* remove white space

* note for tolerance unit test

* fix tests

* remove print

---------

Co-authored-by: yonigozlan <yoni.gozlan@huggingface.co>
Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com>
This commit is contained in:
Nahieli
2025-06-27 16:39:43 +02:00
committed by GitHub
parent 0c35280e58
commit 4336ecd1ea
6 changed files with 498 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
tests/models/nougat/test_image_processing_nougat.py
View File

@@ -16,10 +16,12 @@
import unittest
import numpy as np
import requests
from huggingface_hub import hf_hub_download
from transformers.image_utils import SizeDict
from transformers.testing_utils import require_torch, require_vision
from transformers.utils import cached_property, is_torch_available, is_vision_available
from transformers.utils import cached_property, is_torch_available, is_torchvision_available, is_vision_available
from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs
@@ -32,6 +34,9 @@ if is_vision_available():
from transformers import NougatImageProcessor
if is_torchvision_available():
from transformers import NougatImageProcessorFast
class NougatImageProcessingTester:
def __init__(
@@ -68,6 +73,8 @@ class NougatImageProcessingTester:
self.do_normalize = do_normalize
self.image_mean = image_mean
self.image_std = image_std
self.data_format = "channels_first"
self.input_data_format = "channels_first"
def prepare_image_processor_dict(self):
return {
@@ -112,6 +119,7 @@ class NougatImageProcessingTester:
@require_vision
class NougatImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = NougatImageProcessor if is_vision_available() else None
fast_image_processing_class = NougatImageProcessorFast if is_torchvision_available() else None
def setUp(self):
super().setUp()
@@ -126,61 +134,106 @@ class NougatImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
return self.image_processing_class(**self.image_processor_dict)
def test_image_processor_properties(self):
image_processing = self.image_processing_class(**self.image_processor_dict)
self.assertTrue(hasattr(image_processing, "do_resize"))
self.assertTrue(hasattr(image_processing, "size"))
self.assertTrue(hasattr(image_processing, "do_normalize"))
self.assertTrue(hasattr(image_processing, "image_mean"))
self.assertTrue(hasattr(image_processing, "image_std"))
for image_processing_class in self.image_processor_list:
image_processing = image_processing_class(**self.image_processor_dict)
self.assertTrue(hasattr(image_processing, "do_resize"))
self.assertTrue(hasattr(image_processing, "size"))
self.assertTrue(hasattr(image_processing, "do_normalize"))
self.assertTrue(hasattr(image_processing, "image_mean"))
self.assertTrue(hasattr(image_processing, "image_std"))
def test_image_processor_from_dict_with_kwargs(self):
image_processor = self.image_processing_class.from_dict(self.image_processor_dict)
self.assertEqual(image_processor.size, {"height": 20, "width": 20})
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class(**self.image_processor_dict)
self.assertEqual(image_processor.size, {"height": 20, "width": 20})
image_processor = self.image_processing_class.from_dict(self.image_processor_dict, size=42)
self.assertEqual(image_processor.size, {"height": 42, "width": 42})
kwargs = dict(self.image_processor_dict)
kwargs.pop("size", None)
image_processor = self.image_processing_class(**kwargs, size=42)
self.assertEqual(image_processor.size, {"height": 42, "width": 42})
def test_expected_output(self):
dummy_image = self.image_processor_tester.prepare_dummy_image()
image_processor = self.image_processor
inputs = image_processor(dummy_image, return_tensors="pt")
torch.testing.assert_close(inputs["pixel_values"].mean(), torch.tensor(0.4906), rtol=1e-3, atol=1e-3)
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class(**self.image_processor_dict)
inputs = image_processor(dummy_image, return_tensors="pt")
torch.testing.assert_close(inputs["pixel_values"].mean(), torch.tensor(0.4906), rtol=1e-3, atol=1e-3)
def test_crop_margin_all_white(self):
image = np.uint8(np.ones((100, 100, 3)) * 255)
image_processor = self.image_processor
cropped_image = image_processor.crop_margin(image)
self.assertTrue(np.array_equal(image, cropped_image))
image = np.uint8(np.ones((3, 100, 100)) * 255)
for image_processing_class in self.image_processor_list:
if image_processing_class == NougatImageProcessorFast:
image = torch.from_numpy(image)
image_processor = image_processing_class(**self.image_processor_dict)
cropped_image = image_processor.crop_margin(image)
self.assertTrue(torch.equal(image, cropped_image))
else:
image_processor = image_processing_class(**self.image_processor_dict)
cropped_image = image_processor.crop_margin(image)
self.assertTrue(np.array_equal(image, cropped_image))
def test_crop_margin_centered_black_square(self):
image = np.ones((100, 100, 3), dtype=np.uint8) * 255
image[45:55, 45:55, :] = 0
image_processor = self.image_processor
cropped_image = image_processor.crop_margin(image)
expected_cropped = image[45:55, 45:55, :]
self.assertTrue(np.array_equal(expected_cropped, cropped_image))
image = np.ones((3, 100, 100), dtype=np.uint8) * 255
image[:, 45:55, 45:55] = 0
expected_cropped = image[:, 45:55, 45:55]
for image_processing_class in self.image_processor_list:
if image_processing_class == NougatImageProcessorFast:
image = torch.from_numpy(image)
expected_cropped = torch.from_numpy(expected_cropped)
image_processor = image_processing_class(**self.image_processor_dict)
cropped_image = image_processor.crop_margin(image)
self.assertTrue(torch.equal(expected_cropped, cropped_image))
else:
image_processor = image_processing_class(**self.image_processor_dict)
cropped_image = image_processor.crop_margin(image)
self.assertTrue(np.array_equal(expected_cropped, cropped_image))
def test_align_long_axis_no_rotation(self):
image = np.uint8(np.ones((100, 200, 3)) * 255)
image_processor = self.image_processor
size = {"height": 200, "width": 300}
aligned_image = image_processor.align_long_axis(image, size)
self.assertEqual(image.shape, aligned_image.shape)
image = np.uint8(np.ones((3, 100, 200)) * 255)
for image_processing_class in self.image_processor_list:
if image_processing_class == NougatImageProcessorFast:
image = torch.from_numpy(image)
size = SizeDict(height=200, width=300)
image_processor = image_processing_class(**self.image_processor_dict)
aligned_image = image_processor.align_long_axis(image, size)
self.assertEqual(image.shape, aligned_image.shape)
else:
size = {"height": 200, "width": 300}
image_processor = image_processing_class(**self.image_processor_dict)
aligned_image = image_processor.align_long_axis(image, size)
self.assertEqual(image.shape, aligned_image.shape)
def test_align_long_axis_with_rotation(self):
image = np.uint8(np.ones((200, 100, 3)) * 255)
image_processor = self.image_processor
size = {"height": 300, "width": 200}
aligned_image = image_processor.align_long_axis(image, size)
self.assertEqual((200, 100, 3), aligned_image.shape)
image = np.uint8(np.ones((3, 200, 100)) * 255)
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class(**self.image_processor_dict)
if image_processing_class == NougatImageProcessorFast:
image = torch.from_numpy(image)
size = SizeDict(height=300, width=200)
image_processor = image_processing_class(**self.image_processor_dict)
aligned_image = image_processor.align_long_axis(image, size)
self.assertEqual(torch.Size([3, 200, 100]), aligned_image.shape)
else:
size = {"height": 300, "width": 200}
image_processor = image_processing_class(**self.image_processor_dict)
aligned_image = image_processor.align_long_axis(image, size)
self.assertEqual((3, 200, 100), aligned_image.shape)
def test_align_long_axis_data_format(self):
image = np.uint8(np.ones((100, 200, 3)) * 255)
data_format = "channels_first"
size = {"height": 200, "width": 300}
image_processor = self.image_processor
aligned_image = image_processor.align_long_axis(image, size, data_format=data_format)
self.assertEqual((3, 100, 200), aligned_image.shape)
image = np.uint8(np.ones((3, 100, 200)) * 255)
for image_processing_class in self.image_processor_list:
if image_processing_class == NougatImageProcessorFast:
image = torch.from_numpy(image)
image_processor = image_processing_class(**self.image_processor_dict)
size = SizeDict(height=200, width=300)
aligned_image = image_processor.align_long_axis(image, size)
self.assertEqual(torch.Size([3, 100, 200]), aligned_image.shape)
else:
size = {"height": 200, "width": 300}
data_format = "channels_first"
image_processor = image_processing_class(**self.image_processor_dict)
aligned_image = image_processor.align_long_axis(image, size, data_format)
self.assertEqual((3, 100, 200), aligned_image.shape)
def prepare_dummy_np_image(self):
revision = "ec57bf8c8b1653a209c13f6e9ee66b12df0fc2db"
@@ -191,12 +244,77 @@ class NougatImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
revision=revision,
)
image = Image.open(filepath).convert("RGB")
return np.array(image)
return np.array(image).transpose(2, 0, 1)
def test_crop_margin_equality_cv2_python(self):
image = self.prepare_dummy_np_image()
image_processor = self.image_processor
image_cropped_python = image_processor.crop_margin(image)
for image_processing_class in self.image_processor_list:
if image_processing_class == NougatImageProcessorFast:
image = torch.from_numpy(image)
image_processor = image_processing_class(**self.image_processor_dict)
image_cropped_python = image_processor.crop_margin(image)
self.assertEqual(image_cropped_python.shape, torch.Size([3, 850, 685]))
self.assertAlmostEqual(image_cropped_python.float().mean().item(), 237.43881150708458, delta=0.001)
else:
image_processor = image_processing_class(**self.image_processor_dict)
image_cropped_python = image_processor.crop_margin(image)
self.assertEqual(image_cropped_python.shape, (3, 850, 685))
self.assertAlmostEqual(image_cropped_python.mean(), 237.43881150708458, delta=0.001)
self.assertEqual(image_cropped_python.shape, (850, 685, 3))
self.assertEqual(image_cropped_python.mean(), 237.43881150708458)
def test_call_numpy_4_channels(self):
for image_processing_class in self.image_processor_list:
if image_processing_class == NougatImageProcessor:
# Test that can process images which have an arbitrary number of channels
# Initialize image_processing
image_processor = image_processing_class(**self.image_processor_dict)
# create random numpy tensors
self.image_processor_tester.num_channels = 4
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True)
# Test not batched input
encoded_images = image_processor(
image_inputs[0],
return_tensors="pt",
input_data_format="channels_last",
image_mean=0,
image_std=1,
).pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(
[image_inputs[0]]
)
self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape))
# Test batched
encoded_images = image_processor(
image_inputs,
return_tensors="pt",
input_data_format="channels_last",
image_mean=0,
image_std=1,
).pixel_values
expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs)
self.assertEqual(
tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape)
)
def test_slow_fast_equivalence(self):
if not self.test_slow_image_processor or not self.test_fast_image_processor:
self.skipTest(reason="Skipping slow/fast equivalence test")
if self.image_processing_class is None or self.fast_image_processing_class is None:
self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined")
dummy_image = Image.open(
requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw
)
image_processor_slow = self.image_processing_class(**self.image_processor_dict)
image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict)
encoding_slow = image_processor_slow(dummy_image, return_tensors="pt")
encoding_fast = image_processor_fast(dummy_image, return_tensors="pt")
# Adding a larget than usual tolerance because the slow processor uses reducing_gap=2.0 during resizing.
torch.testing.assert_close(encoding_slow.pixel_values, encoding_fast.pixel_values, atol=2e-1, rtol=0)
self.assertLessEqual(
torch.mean(torch.abs(encoding_slow.pixel_values - encoding_fast.pixel_values)).item(), 2e-2
)