SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add Fast Image Processor for LayoutLMv3 (#37201)

Browse Source 
* support fast image processor layoutlmv3

* make style

* add warning and update test

* make style

* Update src/transformers/models/layoutlmv3/image_processing_layoutlmv3_fast.py

* Update image_processing_auto.py

---------

Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com>
This commit is contained in:
Vinh H. Pham
2025-04-14 20:42:11 +07:00
committed by GitHub
parent 7bff4bdcf6
commit 1897a02d83
5 changed files with 222 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/source/en/model_doc/layoutlmv3.md
View File

@@ -88,6 +88,11 @@ LayoutLMv3 is nearly identical to LayoutLMv2, so we've also included LayoutLMv2
[[autodoc]] LayoutLMv3ImageProcessor
- preprocess
## LayoutLMv3ImageProcessorFast
[[autodoc]] LayoutLMv3ImageProcessorFast
- preprocess
## LayoutLMv3Tokenizer
[[autodoc]] LayoutLMv3Tokenizer

4
src/transformers/models/auto/image_processing_auto.py
View File

@@ -103,7 +103,7 @@ else:
("instructblipvideo", ("InstructBlipVideoImageProcessor",)),
("kosmos-2", ("CLIPImageProcessor", "CLIPImageProcessorFast")),
("layoutlmv2", ("LayoutLMv2ImageProcessor", "LayoutLMv2ImageProcessorFast")),
("layoutlmv3", ("LayoutLMv3ImageProcessor",)),
("layoutlmv3", ("LayoutLMv3ImageProcessor", "LayoutLMv3ImageProcessorFast")),
("levit", ("LevitImageProcessor",)),
("llama4", ("Llama4ImageProcessor", "Llama4ImageProcessorFast")),
("llava", ("LlavaImageProcessor", "LlavaImageProcessorFast")),
@@ -154,7 +154,7 @@ else:
("timm_wrapper", ("TimmWrapperImageProcessor",)),
("tvlt", ("TvltImageProcessor",)),
("tvp", ("TvpImageProcessor",)),
("udop", ("LayoutLMv3ImageProcessor",)),
("udop", ("LayoutLMv3ImageProcessor", "LayoutLMv3ImageProcessorFast")),
("upernet", ("SegformerImageProcessor",)),
("van", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")),
("videomae", ("VideoMAEImageProcessor",)),

1
src/transformers/models/layoutlmv3/__init__.py
View File

@@ -21,6 +21,7 @@ if TYPE_CHECKING:
from .configuration_layoutlmv3 import *
from .feature_extraction_layoutlmv3 import *
from .image_processing_layoutlmv3 import *
from .image_processing_layoutlmv3_fast import *
from .modeling_layoutlmv3 import *
from .modeling_tf_layoutlmv3 import *
from .processing_layoutlmv3 import *

178
src/transformers/models/layoutlmv3/image_processing_layoutlmv3_fast.py Normal file
View File

@@ -0,0 +1,178 @@
# coding=utf-8
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fast Image processor class for LayoutLMv3."""
from typing import Optional, Union
from ...image_processing_utils_fast import (
BASE_IMAGE_PROCESSOR_FAST_DOCSTRING,
BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS,
BaseImageProcessorFast,
BatchFeature,
DefaultFastImageProcessorKwargs,
)
from ...image_transforms import ChannelDimension, group_images_by_shape, reorder_images
from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, ImageInput, PILImageResampling, SizeDict
from ...processing_utils import Unpack
from ...utils import (
TensorType,
add_start_docstrings,
is_torch_available,
is_torchvision_available,
is_torchvision_v2_available,
logging,
requires_backends,
)
from .image_processing_layoutlmv3 import apply_tesseract
logger = logging.get_logger(__name__)
if is_torch_available():
import torch
if is_torchvision_available():
if is_torchvision_v2_available():
from torchvision.transforms.v2 import functional as F
else:
from torchvision.transforms import functional as F
class LayoutLMv3FastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
apply_ocr: Optional[bool]
ocr_lang: Optional[str]
tesseract_config: Optional[str]
@add_start_docstrings(
"Constructs a fast LayoutLMv3 image processor.",
BASE_IMAGE_PROCESSOR_FAST_DOCSTRING,
"""
apply_ocr (`bool`, *optional*, defaults to `True`):
Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by
the `apply_ocr` parameter in the `preprocess` method.
ocr_lang (`str`, *optional*):
The language, specified by its ISO code, to be used by the Tesseract OCR engine. By default, English is
used. Can be overridden by the `ocr_lang` parameter in the `preprocess` method.
tesseract_config (`str`, *optional*):
Any additional custom configuration flags that are forwarded to the `config` parameter when calling
Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the
`preprocess` method.
""",
)
class LayoutLMv3ImageProcessorFast(BaseImageProcessorFast):
resample = PILImageResampling.BILINEAR
image_mean = IMAGENET_STANDARD_MEAN
image_std = IMAGENET_STANDARD_STD
size = {"height": 224, "width": 224}
do_resize = True
do_rescale = True
do_normalize = True
apply_ocr = True
ocr_lang = None
tesseract_config = ""
valid_kwargs = LayoutLMv3FastImageProcessorKwargs
def __init__(self, **kwargs: Unpack[LayoutLMv3FastImageProcessorKwargs]):
super().__init__(**kwargs)
@add_start_docstrings(
BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS,
"""
apply_ocr (`bool`, *optional*, defaults to `True`):
Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by
the `apply_ocr` parameter in the `preprocess` method.
ocr_lang (`str`, *optional*):
The language, specified by its ISO code, to be used by the Tesseract OCR engine. By default, English is
used. Can be overridden by the `ocr_lang` parameter in the `preprocess` method.
tesseract_config (`str`, *optional*):
Any additional custom configuration flags that are forwarded to the `config` parameter when calling
Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the
`preprocess` method.
""",
)
def preprocess(self, images: ImageInput, **kwargs: Unpack[LayoutLMv3FastImageProcessorKwargs]) -> BatchFeature:
return super().preprocess(images, **kwargs)
def _preprocess(
self,
images: list["torch.Tensor"],
do_resize: bool,
size: SizeDict,
interpolation: Optional["F.InterpolationMode"],
do_center_crop: bool,
crop_size: SizeDict,
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: Optional[Union[float, list[float]]],
image_std: Optional[Union[float, list[float]]],
apply_ocr: bool,
ocr_lang: Optional[str],
tesseract_config: Optional[str],
return_tensors: Optional[Union[str, TensorType]],
**kwargs,
) -> BatchFeature:
# Tesseract OCR to get words + normalized bounding boxes
if apply_ocr:
requires_backends(self, "pytesseract")
words_batch = []
boxes_batch = []
for image in images:
if image.is_cuda:
logger.warning_once(
"apply_ocr can only be performed on cpu. Tensors will be transferred to cpu before processing."
)
words, boxes = apply_tesseract(
image.cpu(), ocr_lang, tesseract_config, input_data_format=ChannelDimension.FIRST
)
words_batch.append(words)
boxes_batch.append(boxes)
# Group images by size for batched resizing
grouped_images, grouped_images_index = group_images_by_shape(images)
resized_images_grouped = {}
for shape, stacked_images in grouped_images.items():
if do_resize:
stacked_images = self.resize(image=stacked_images, size=size, interpolation=interpolation)
resized_images_grouped[shape] = stacked_images
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
# Group images by size for further processing
# Needed in case do_resize is False, or resize returns images with different sizes
grouped_images, grouped_images_index = group_images_by_shape(resized_images)
processed_images_grouped = {}
for shape, stacked_images in grouped_images.items():
if do_center_crop:
stacked_images = self.center_crop(stacked_images, crop_size)
# Fused rescale and normalize
stacked_images = self.rescale_and_normalize(
stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
)
processed_images_grouped[shape] = stacked_images
processed_images = reorder_images(processed_images_grouped, grouped_images_index)
processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images
data = BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)
if apply_ocr:
data["words"] = words_batch
data["boxes"] = boxes_batch
return data
__all__ = ["LayoutLMv3ImageProcessorFast"]

29
tests/models/layoutlmv3/test_image_processing_layoutlmv3.py
View File

@@ -16,7 +16,7 @@
import unittest
from transformers.testing_utils import require_pytesseract, require_torch
from transformers.utils import is_pytesseract_available
from transformers.utils import is_pytesseract_available, is_torchvision_available
from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs
@@ -26,6 +26,9 @@ if is_pytesseract_available():
from transformers import LayoutLMv3ImageProcessor
if is_torchvision_available():
from transformers import LayoutLMv3ImageProcessorFast
class LayoutLMv3ImageProcessingTester:
def __init__(
@@ -73,6 +76,9 @@ class LayoutLMv3ImageProcessingTester:
@require_pytesseract
class LayoutLMv3ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
image_processing_class = LayoutLMv3ImageProcessor if is_pytesseract_available() else None
fast_image_processing_class = (
LayoutLMv3ImageProcessorFast if (is_torchvision_available() and is_pytesseract_available()) else None
)
def setUp(self):
super().setUp()
@@ -83,29 +89,32 @@ class LayoutLMv3ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
return self.image_processor_tester.prepare_image_processor_dict()
def test_image_processor_properties(self):
image_processing = self.image_processing_class(**self.image_processor_dict)
for image_processing_class in self.image_processor_list:
image_processing = image_processing_class(**self.image_processor_dict)
self.assertTrue(hasattr(image_processing, "do_resize"))
self.assertTrue(hasattr(image_processing, "size"))
self.assertTrue(hasattr(image_processing, "apply_ocr"))
def test_image_processor_from_dict_with_kwargs(self):
image_processor = self.image_processing_class.from_dict(self.image_processor_dict)
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class.from_dict(self.image_processor_dict)
self.assertEqual(image_processor.size, {"height": 18, "width": 18})
image_processor = self.image_processing_class.from_dict(self.image_processor_dict, size=42)
image_processor = image_processing_class.from_dict(self.image_processor_dict, size=42)
self.assertEqual(image_processor.size, {"height": 42, "width": 42})
def test_LayoutLMv3_integration_test(self):
# with apply_OCR = True
image_processing = LayoutLMv3ImageProcessor()
from datasets import load_dataset
ds = load_dataset("hf-internal-testing/fixtures_docvqa", split="test", trust_remote_code=True)
# with apply_OCR = True
for image_processing_class in self.image_processor_list:
image_processor = image_processing_class()
image = Image.open(ds[0]["file"]).convert("RGB")
encoding = image_processing(image, return_tensors="pt")
encoding = image_processor(image, return_tensors="pt")
self.assertEqual(encoding.pixel_values.shape, (1, 3, 224, 224))
self.assertEqual(len(encoding.words), len(encoding.boxes))
@@ -120,8 +129,8 @@ class LayoutLMv3ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase)
self.assertListEqual(encoding.boxes, expected_boxes)
# with apply_OCR = False
image_processing = LayoutLMv3ImageProcessor(apply_ocr=False)
image_processor = image_processing_class(apply_ocr=False)
encoding = image_processing(image, return_tensors="pt")
encoding = image_processor(image, return_tensors="pt")
self.assertEqual(encoding.pixel_values.shape, (1, 3, 224, 224))