Add Fast Image Processor for LayoutLMv2 (#37203)

* add support layoutlmv2

* make style

* Apply suggestions from code review

Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com>

* add warning and clean up

* make style

* Update src/transformers/models/layoutlmv2/image_processing_layoutlmv2_fast.py

Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com>

---------

Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com>

docs/source/en/model_doc/layoutlmv2.md

@@ -310,6 +310,11 @@ print(encoding.keys())
 [[autodoc]] LayoutLMv2ImageProcessor
     - preprocess
 
+## LayoutLMv2ImageProcessorFast
+
+[[autodoc]] LayoutLMv2ImageProcessorFast
+    - preprocess
+
 ## LayoutLMv2Tokenizer
 
 [[autodoc]] LayoutLMv2Tokenizer

src/transformers/models/auto/image_processing_auto.py

@@ -102,7 +102,7 @@ else:
             ("instructblip", ("BlipImageProcessor", "BlipImageProcessorFast")),
             ("instructblipvideo", ("InstructBlipVideoImageProcessor",)),
             ("kosmos-2", ("CLIPImageProcessor", "CLIPImageProcessorFast")),
-            ("layoutlmv2", ("LayoutLMv2ImageProcessor",)),
+            ("layoutlmv2", ("LayoutLMv2ImageProcessor", "LayoutLMv2ImageProcessorFast")),
             ("layoutlmv3", ("LayoutLMv3ImageProcessor",)),
             ("levit", ("LevitImageProcessor",)),
             ("llama4", ("Llama4ImageProcessor", "Llama4ImageProcessorFast")),

src/transformers/models/layoutlmv2/__init__.py

@@ -21,6 +21,7 @@ if TYPE_CHECKING:
     from .configuration_layoutlmv2 import *
     from .feature_extraction_layoutlmv2 import *
     from .image_processing_layoutlmv2 import *
+    from .image_processing_layoutlmv2_fast import *
     from .modeling_layoutlmv2 import *
     from .processing_layoutlmv2 import *
     from .tokenization_layoutlmv2 import *

src/transformers/models/layoutlmv2/image_processing_layoutlmv2_fast.py

@@ -0,0 +1,164 @@
+# coding=utf-8
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Fast Image processor class for LayoutLMv2."""
+
+from typing import Optional, Union
+
+from ...image_processing_utils_fast import (
+    BASE_IMAGE_PROCESSOR_FAST_DOCSTRING,
+    BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS,
+    BaseImageProcessorFast,
+    BatchFeature,
+    DefaultFastImageProcessorKwargs,
+)
+from ...image_transforms import ChannelDimension, group_images_by_shape, reorder_images
+from ...image_utils import ImageInput, PILImageResampling, SizeDict
+from ...processing_utils import Unpack
+from ...utils import (
+    TensorType,
+    add_start_docstrings,
+    is_torch_available,
+    is_torchvision_available,
+    is_torchvision_v2_available,
+    logging,
+    requires_backends,
+)
+from .image_processing_layoutlmv2 import apply_tesseract
+
+
+logger = logging.get_logger(__name__)
+
+if is_torch_available():
+    import torch
+
+if is_torchvision_available():
+    if is_torchvision_v2_available():
+        from torchvision.transforms.v2 import functional as F
+    else:
+        from torchvision.transforms import functional as F
+
+
+class LayoutLMv2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
+    apply_ocr: Optional[bool]
+    ocr_lang: Optional[str]
+    tesseract_config: Optional[str]
+
+
+@add_start_docstrings(
+    "Constructs a fast LayoutLMv2 image processor.",
+    BASE_IMAGE_PROCESSOR_FAST_DOCSTRING,
+    """
+        apply_ocr (`bool`, *optional*, defaults to `True`):
+            Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by
+            the `apply_ocr` parameter in the `preprocess` method.
+        ocr_lang (`str`, *optional*):
+            The language, specified by its ISO code, to be used by the Tesseract OCR engine. By default, English is
+            used. Can be overridden by the `ocr_lang` parameter in the `preprocess` method.
+        tesseract_config (`str`, *optional*):
+            Any additional custom configuration flags that are forwarded to the `config` parameter when calling
+            Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the
+            `preprocess` method.
+    """,
+)
+class LayoutLMv2ImageProcessorFast(BaseImageProcessorFast):
+    resample = PILImageResampling.BILINEAR
+    size = {"height": 224, "width": 224}
+    rescale_factor = None
+    do_resize = True
+    apply_ocr = True
+    ocr_lang = None
+    tesseract_config = ""
+    valid_kwargs = LayoutLMv2FastImageProcessorKwargs
+
+    def __init__(self, **kwargs: Unpack[LayoutLMv2FastImageProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    @add_start_docstrings(
+        BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS,
+        """
+            apply_ocr (`bool`, *optional*, defaults to `True`):
+                Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by
+                the `apply_ocr` parameter in the `preprocess` method.
+            ocr_lang (`str`, *optional*):
+                The language, specified by its ISO code, to be used by the Tesseract OCR engine. By default, English is
+                used. Can be overridden by the `ocr_lang` parameter in the `preprocess` method.
+            tesseract_config (`str`, *optional*):
+                Any additional custom configuration flags that are forwarded to the `config` parameter when calling
+                Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the
+                `preprocess` method.
+        """,
+    )
+    def preprocess(self, images: ImageInput, **kwargs: Unpack[LayoutLMv2FastImageProcessorKwargs]) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def _preprocess(
+        self,
+        images: list["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        interpolation: Optional["F.InterpolationMode"],
+        apply_ocr: bool,
+        ocr_lang: Optional[str],
+        tesseract_config: Optional[str],
+        return_tensors: Optional[Union[str, TensorType]],
+        **kwargs,
+    ) -> BatchFeature:
+        # Tesseract OCR to get words + normalized bounding boxes
+        if apply_ocr:
+            requires_backends(self, "pytesseract")
+            words_batch = []
+            boxes_batch = []
+            for image in images:
+                if image.is_cuda:
+                    logger.warning_once(
+                        "apply_ocr can only be performed on cpu. Tensors will be transferred to cpu before processing."
+                    )
+                words, boxes = apply_tesseract(
+                    image.cpu(), ocr_lang, tesseract_config, input_data_format=ChannelDimension.FIRST
+                )
+                words_batch.append(words)
+                boxes_batch.append(boxes)
+
+        # Group images by size for batched resizing
+        grouped_images, grouped_images_index = group_images_by_shape(images)
+        resized_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_resize:
+                stacked_images = self.resize(image=stacked_images, size=size, interpolation=interpolation)
+            resized_images_grouped[shape] = stacked_images
+        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+        # Group images by size for further processing
+        # Needed in case do_resize is False, or resize returns images with different sizes
+        grouped_images, grouped_images_index = group_images_by_shape(resized_images)
+        processed_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            # flip color channels from RGB to BGR (as Detectron2 requires this)
+            stacked_images = stacked_images.flip(1)
+            processed_images_grouped[shape] = stacked_images
+
+        processed_images = reorder_images(processed_images_grouped, grouped_images_index)
+        processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images
+
+        data = BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)
+
+        if apply_ocr:
+            data["words"] = words_batch
+            data["boxes"] = boxes_batch
+
+        return data
+
+
+__all__ = ["LayoutLMv2ImageProcessorFast"]

tests/models/layoutlmv2/test_image_processing_layoutlmv2.py

@@ -12,20 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import unittest
 
-from transformers.testing_utils import require_pytesseract, require_torch
+import requests
-from transformers.utils import is_pytesseract_available
+
+from transformers.testing_utils import require_pytesseract, require_torch, require_vision
+from transformers.utils import is_pytesseract_available, is_torch_available, is_torchvision_available
 
 from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs
 
 
+if is_torch_available():
+    import torch
+
 if is_pytesseract_available():
     from PIL import Image
 
     from transformers import LayoutLMv2ImageProcessor
 
+    if is_torchvision_available():
+        from transformers import LayoutLMv2ImageProcessorFast
+
 
 class LayoutLMv2ImageProcessingTester:
     def __init__(
@@ -73,6 +80,9 @@ class LayoutLMv2ImageProcessingTester:
 @require_pytesseract
 class LayoutLMv2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
     image_processing_class = LayoutLMv2ImageProcessor if is_pytesseract_available() else None
+    fast_image_processing_class = (
+        LayoutLMv2ImageProcessorFast if (is_torchvision_available() and is_pytesseract_available()) else None
+    )
 
     def setUp(self):
         super().setUp()
@@ -83,27 +93,30 @@ class LayoutLMv2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
         return self.image_processor_tester.prepare_image_processor_dict()
 
     def test_image_processor_properties(self):
-        image_processing = self.image_processing_class(**self.image_processor_dict)
+        for image_processing_class in self.image_processor_list:
+            image_processing = image_processing_class(**self.image_processor_dict)
             self.assertTrue(hasattr(image_processing, "do_resize"))
             self.assertTrue(hasattr(image_processing, "size"))
             self.assertTrue(hasattr(image_processing, "apply_ocr"))
 
     def test_image_processor_from_dict_with_kwargs(self):
-        image_processor = self.image_processing_class.from_dict(self.image_processor_dict)
+        for image_processing_class in self.image_processor_list:
+            image_processor = image_processing_class.from_dict(self.image_processor_dict)
             self.assertEqual(image_processor.size, {"height": 18, "width": 18})
 
-        image_processor = self.image_processing_class.from_dict(self.image_processor_dict, size=42)
+            image_processor = image_processing_class.from_dict(self.image_processor_dict, size=42)
             self.assertEqual(image_processor.size, {"height": 42, "width": 42})
 
     @unittest.skip(reason="Tesseract version is not correct in ci. @Arthur FIXME")
     def test_layoutlmv2_integration_test(self):
-        # with apply_OCR = True
-        image_processing = LayoutLMv2ImageProcessor()
-
         from datasets import load_dataset
 
         ds = load_dataset("hf-internal-testing/fixtures_docvqa", split="test", trust_remote_code=True)
 
+        for image_processing_class in self.image_processor_list:
+            # with apply_OCR = True
+            image_processing = image_processing_class()
+
             image = Image.open(ds[0]["file"]).convert("RGB")
 
             encoding = image_processing(image, return_tensors="pt")
@@ -121,8 +134,70 @@ class LayoutLMv2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
             self.assertListEqual(encoding.boxes, expected_boxes)
 
             # with apply_OCR = False
-        image_processing = LayoutLMv2ImageProcessor(apply_ocr=False)
+            image_processing = image_processing_class(apply_ocr=False)
 
             encoding = image_processing(image, return_tensors="pt")
 
             self.assertEqual(encoding.pixel_values.shape, (1, 3, 224, 224))
+
+    @require_vision
+    @require_torch
+    def test_slow_fast_equivalence(self):
+        if not self.test_slow_image_processor or not self.test_fast_image_processor:
+            self.skipTest(reason="Skipping slow/fast equivalence test")
+
+        if self.image_processing_class is None or self.fast_image_processing_class is None:
+            self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined")
+
+        dummy_image = Image.open(
+            requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw
+        )
+        image_processor_slow = self.image_processing_class(**self.image_processor_dict)
+        image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict)
+
+        encoding_slow = image_processor_slow(dummy_image, return_tensors="pt")
+        encoding_fast = image_processor_fast(dummy_image, return_tensors="pt")
+        self.assertTrue(
+            torch.allclose(
+                encoding_slow.pixel_values.float() / 255, encoding_fast.pixel_values.float() / 255, atol=1e-1
+            )
+        )
+        self.assertLessEqual(
+            torch.mean(
+                torch.abs(encoding_slow.pixel_values.float() - encoding_fast.pixel_values.float()) / 255
+            ).item(),
+            1e-3,
+        )
+
+    @require_vision
+    @require_torch
+    def test_slow_fast_equivalence_batched(self):
+        if not self.test_slow_image_processor or not self.test_fast_image_processor:
+            self.skipTest(reason="Skipping slow/fast equivalence test")
+
+        if self.image_processing_class is None or self.fast_image_processing_class is None:
+            self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined")
+
+        if hasattr(self.image_processor_tester, "do_center_crop") and self.image_processor_tester.do_center_crop:
+            self.skipTest(
+                reason="Skipping as do_center_crop is True and center_crop functions are not equivalent for fast and slow processors"
+            )
+
+        dummy_images = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True)
+        image_processor_slow = self.image_processing_class(**self.image_processor_dict)
+        image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict)
+
+        encoding_slow = image_processor_slow(dummy_images, return_tensors="pt")
+        encoding_fast = image_processor_fast(dummy_images, return_tensors="pt")
+
+        self.assertTrue(
+            torch.allclose(
+                encoding_slow.pixel_values.float() / 255, encoding_fast.pixel_values.float() / 255, atol=1e-1
+            )
+        )
+        self.assertLessEqual(
+            torch.mean(
+                torch.abs(encoding_slow.pixel_values.float() - encoding_fast.pixel_values.float()) / 255
+            ).item(),
+            1e-3,
+        )