From 565c035a2eb610de9a208ee598c47c75cd21fc70 Mon Sep 17 00:00:00 2001 From: lmarshall12 Date: Thu, 24 Jul 2025 22:40:11 -0400 Subject: [PATCH] Add owlv2 fast processor (#39041) * add owlv2 fast image processor * add Owlv2ImageProcessorFast to Owlv2Processor image_processor_class * add Owlv2ImageProcessorFast to Owlv2Processor image_processor_class * change references to owlVit to owlv2 in docstrings for post process methods * change type hints from List, Dict, Tuple to list, dict, tuple * remove unused typing imports * add disable grouping argument to group images by shape * run make quality and repo-consistency * use modular * fix auto_docstring --------- Co-authored-by: Lewis Marshall Co-authored-by: yonigozlan --- docs/source/en/model_doc/owlv2.md | 7 + .../models/auto/image_processing_auto.py | 2 +- src/transformers/models/owlv2/__init__.py | 1 + .../owlv2/image_processing_owlv2_fast.py | 427 ++++++++++++++++++ .../models/owlv2/modular_owlv2.py | 240 ++++++++++ .../models/owlv2/processing_owlv2.py | 6 +- .../owlv2/test_image_processing_owlv2.py | 115 ++--- 7 files changed, 737 insertions(+), 61 deletions(-) create mode 100644 src/transformers/models/owlv2/image_processing_owlv2_fast.py create mode 100644 src/transformers/models/owlv2/modular_owlv2.py diff --git a/docs/source/en/model_doc/owlv2.md b/docs/source/en/model_doc/owlv2.md index b7ab61cc98..b3b444d58f 100644 --- a/docs/source/en/model_doc/owlv2.md +++ b/docs/source/en/model_doc/owlv2.md @@ -106,6 +106,13 @@ Usage of OWLv2 is identical to [OWL-ViT](owlvit) with a new, updated image proce - post_process_object_detection - post_process_image_guided_detection +## Owlv2ImageProcessorFast + +[[autodoc]] Owlv2ImageProcessorFast + - preprocess + - post_process_object_detection + - post_process_image_guided_detection + ## Owlv2Processor [[autodoc]] Owlv2Processor diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index f169cf413a..6857eda317 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -131,7 +131,7 @@ else: ("nat", ("ViTImageProcessor", "ViTImageProcessorFast")), ("nougat", ("NougatImageProcessor", "NougatImageProcessorFast")), ("oneformer", ("OneFormerImageProcessor", "OneFormerImageProcessorFast")), - ("owlv2", ("Owlv2ImageProcessor",)), + ("owlv2", ("Owlv2ImageProcessor", "Owlv2ImageProcessorFast")), ("owlvit", ("OwlViTImageProcessor", "OwlViTImageProcessorFast")), ("paligemma", ("SiglipImageProcessor", "SiglipImageProcessorFast")), ("perceiver", ("PerceiverImageProcessor", "PerceiverImageProcessorFast")), diff --git a/src/transformers/models/owlv2/__init__.py b/src/transformers/models/owlv2/__init__.py index 6ce3294208..c3d6deae9c 100644 --- a/src/transformers/models/owlv2/__init__.py +++ b/src/transformers/models/owlv2/__init__.py @@ -20,6 +20,7 @@ from ...utils.import_utils import define_import_structure if TYPE_CHECKING: from .configuration_owlv2 import * from .image_processing_owlv2 import * + from .image_processing_owlv2_fast import * from .modeling_owlv2 import * from .processing_owlv2 import * else: diff --git a/src/transformers/models/owlv2/image_processing_owlv2_fast.py b/src/transformers/models/owlv2/image_processing_owlv2_fast.py new file mode 100644 index 0000000000..fd46f12f28 --- /dev/null +++ b/src/transformers/models/owlv2/image_processing_owlv2_fast.py @@ -0,0 +1,427 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/owlv2/modular_owlv2.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_owlv2.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from typing import TYPE_CHECKING, Optional, Union + +from ...image_processing_utils_fast import BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs +from ...image_transforms import center_to_corners_format, group_images_by_shape, reorder_images +from ...image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ChannelDimension, + ImageInput, + PILImageResampling, + SizeDict, +) +from ...processing_utils import Unpack +from ...utils import ( + TensorType, + auto_docstring, + is_torch_available, + is_torchvision_available, + is_torchvision_v2_available, +) + + +if is_torch_available(): + import torch + + +if is_torchvision_v2_available(): + from torchvision.transforms.v2 import functional as F +elif is_torchvision_available(): + from torchvision.transforms import functional as F + + +if TYPE_CHECKING: + from .modeling_owlv2 import Owlv2ObjectDetectionOutput + + +if is_torch_available(): + from .image_processing_owlv2 import _scale_boxes, box_iou + + +class Owlv2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with grey pixels. + """ + + do_pad: Optional[bool] + + +@auto_docstring +class Owlv2ImageProcessorFast(BaseImageProcessorFast): + resample = PILImageResampling.BILINEAR + image_mean = OPENAI_CLIP_MEAN + image_std = OPENAI_CLIP_STD + size = {"height": 960, "width": 960} + default_to_square = True + crop_size = None + do_resize = True + do_center_crop = None + do_rescale = True + do_normalize = True + do_convert_rgb = None + model_input_names = ["pixel_values"] + rescale_factor = 1 / 255 + do_pad = True + valid_kwargs = Owlv2FastImageProcessorKwargs + + def post_process(self, outputs, target_sizes): + """ + Converts the raw output of [`Owlv2ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + bottom_right_x, bottom_right_y) format. + + Args: + outputs ([`Owlv2ObjectDetectionOutput`]): + Raw outputs of the model. + target_sizes (`torch.Tensor` of shape `(batch_size, 2)`): + Tensor containing the size (h, w) of each image of the batch. For evaluation, this must be the original + image size (before any data augmentation). For visualization, this should be the image size after data + augment, but before padding. + Returns: + `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image + in the batch as predicted by the model. + """ + # TODO: (amy) add support for other frameworks + warnings.warn( + "`post_process` is deprecated and will be removed in v5 of Transformers, please use" + " `post_process_object_detection` instead, with `threshold=0.` for equivalent results.", + FutureWarning, + ) + + logits, boxes = outputs.logits, outputs.pred_boxes + + if len(logits) != len(target_sizes): + raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") + if target_sizes.shape[1] != 2: + raise ValueError("Each element of target_sizes must contain the size (h, w) of each image of the batch") + + probs = torch.max(logits, dim=-1) + scores = torch.sigmoid(probs.values) + labels = probs.indices + + # Convert to [x0, y0, x1, y1] format + boxes = center_to_corners_format(boxes) + + # Convert from relative [0, 1] to absolute [0, height] coordinates + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + results = [{"scores": s, "labels": l, "boxes": b} for s, l, b in zip(scores, labels, boxes)] + + return results + + def post_process_object_detection( + self, + outputs: "Owlv2ObjectDetectionOutput", + threshold: float = 0.1, + target_sizes: Optional[Union[TensorType, list[tuple]]] = None, + ): + """ + Converts the raw output of [`Owlv2ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + bottom_right_x, bottom_right_y) format. + + Args: + outputs ([`Owlv2ObjectDetectionOutput`]): + Raw outputs of the model. + threshold (`float`, *optional*, defaults to 0.1): + Score threshold to keep object detection predictions. + target_sizes (`torch.Tensor` or `list[tuple[int, int]]`, *optional*): + Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size + `(height, width)` of each image in the batch. If unset, predictions will not be resized. + + Returns: + `list[Dict]`: A list of dictionaries, each dictionary containing the following keys: + - "scores": The confidence scores for each predicted box on the image. + - "labels": Indexes of the classes predicted by the model on the image. + - "boxes": Image bounding boxes in (top_left_x, top_left_y, bottom_right_x, bottom_right_y) format. + """ + batch_logits, batch_boxes = outputs.logits, outputs.pred_boxes + batch_size = len(batch_logits) + + if target_sizes is not None and len(target_sizes) != batch_size: + raise ValueError("Make sure that you pass in as many target sizes as images") + + # batch_logits of shape (batch_size, num_queries, num_classes) + batch_class_logits = torch.max(batch_logits, dim=-1) + batch_scores = torch.sigmoid(batch_class_logits.values) + batch_labels = batch_class_logits.indices + + # Convert to [x0, y0, x1, y1] format + batch_boxes = center_to_corners_format(batch_boxes) + + # Convert from relative [0, 1] to absolute [0, height] coordinates + if target_sizes is not None: + batch_boxes = _scale_boxes(batch_boxes, target_sizes) + + results = [] + for scores, labels, boxes in zip(batch_scores, batch_labels, batch_boxes): + keep = scores > threshold + scores = scores[keep] + labels = labels[keep] + boxes = boxes[keep] + results.append({"scores": scores, "labels": labels, "boxes": boxes}) + + return results + + def post_process_image_guided_detection(self, outputs, threshold=0.0, nms_threshold=0.3, target_sizes=None): + """ + Converts the output of [`Owlv2ForObjectDetection.image_guided_detection`] into the format expected by the COCO + api. + + Args: + outputs ([`Owlv2ImageGuidedObjectDetectionOutput`]): + Raw outputs of the model. + threshold (`float`, *optional*, defaults to 0.0): + Minimum confidence threshold to use to filter out predicted boxes. + nms_threshold (`float`, *optional*, defaults to 0.3): + IoU threshold for non-maximum suppression of overlapping boxes. + target_sizes (`torch.Tensor`, *optional*): + Tensor of shape (batch_size, 2) where each entry is the (height, width) of the corresponding image in + the batch. If set, predicted normalized bounding boxes are rescaled to the target sizes. If left to + None, predictions will not be unnormalized. + + Returns: + `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image + in the batch as predicted by the model. All labels are set to None as + `Owlv2ForObjectDetection.image_guided_detection` perform one-shot object detection. + """ + logits, target_boxes = outputs.logits, outputs.target_pred_boxes + + if target_sizes is not None and len(logits) != len(target_sizes): + raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") + if target_sizes is not None and target_sizes.shape[1] != 2: + raise ValueError("Each element of target_sizes must contain the size (h, w) of each image of the batch") + + probs = torch.max(logits, dim=-1) + scores = torch.sigmoid(probs.values) + + # Convert to [x0, y0, x1, y1] format + target_boxes = center_to_corners_format(target_boxes) + + # Apply non-maximum suppression (NMS) + if nms_threshold < 1.0: + for idx in range(target_boxes.shape[0]): + for i in torch.argsort(-scores[idx]): + if not scores[idx][i]: + continue + + ious = box_iou(target_boxes[idx][i, :].unsqueeze(0), target_boxes[idx])[0][0] + ious[i] = -1.0 # Mask self-IoU. + scores[idx][ious > nms_threshold] = 0.0 + + # Convert from relative [0, 1] to absolute [0, height] coordinates + if target_sizes is not None: + target_boxes = _scale_boxes(target_boxes, target_sizes) + + # Compute box display alphas based on prediction scores + results = [] + alphas = torch.zeros_like(scores) + + for idx in range(target_boxes.shape[0]): + # Select scores for boxes matching the current query: + query_scores = scores[idx] + if not query_scores.nonzero().numel(): + continue + + # Apply threshold on scores before scaling + query_scores[query_scores < threshold] = 0.0 + + # Scale box alpha such that the best box for each query has alpha 1.0 and the worst box has alpha 0.1. + # All other boxes will either belong to a different query, or will not be shown. + max_score = torch.max(query_scores) + 1e-6 + query_alphas = (query_scores - (max_score * 0.1)) / (max_score * 0.9) + query_alphas = torch.clip(query_alphas, 0.0, 1.0) + alphas[idx] = query_alphas + + mask = alphas[idx] > 0 + box_scores = alphas[idx][mask] + boxes = target_boxes[idx][mask] + results.append({"scores": box_scores, "labels": None, "boxes": boxes}) + + return results + + def __init__(self, **kwargs: Unpack[Owlv2FastImageProcessorKwargs]): + super().__init__(**kwargs) + + @auto_docstring + def preprocess(self, images: ImageInput, **kwargs: Unpack[Owlv2FastImageProcessorKwargs]): + return super().preprocess(images, **kwargs) + + def _pad_images(self, images: "torch.Tensor", constant_value: float = 0.5) -> "torch.Tensor": + """ + Pad an image with zeros to the given size. + """ + height, width = images.shape[-2:] + size = max(height, width) + pad_bottom = size - height + pad_right = size - width + + padding = (0, 0, pad_right, pad_bottom) + padded_image = F.pad(images, padding, fill=constant_value) + return padded_image + + def pad( + self, + images: list["torch.Tensor"], + disable_grouping: Optional[bool], + constant_value: float = 0.5, + ) -> list["torch.Tensor"]: + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + stacked_images = self._pad_images( + stacked_images, + constant_value=constant_value, + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + return processed_images + + def resize( + self, + image: "torch.Tensor", + size: SizeDict, + anti_aliasing: bool = True, + anti_aliasing_sigma=None, + **kwargs, + ) -> "torch.Tensor": + """ + Resize an image as per the original implementation. + + Args: + image (`Tensor`): + Image to resize. + size (`dict[str, int]`): + Dictionary containing the height and width to resize the image to. + anti_aliasing (`bool`, *optional*, defaults to `True`): + Whether to apply anti-aliasing when downsampling the image. + anti_aliasing_sigma (`float`, *optional*, defaults to `None`): + Standard deviation for Gaussian kernel when downsampling the image. If `None`, it will be calculated + automatically. + """ + output_shape = (size.height, size.width) + + input_shape = image.shape + + # select height and width from input tensor + factors = torch.tensor(input_shape[2:]).to(image.device) / torch.tensor(output_shape).to(image.device) + + if anti_aliasing: + if anti_aliasing_sigma is None: + anti_aliasing_sigma = ((factors - 1) / 2).clamp(min=0) + else: + anti_aliasing_sigma = torch.atleast_1d(anti_aliasing_sigma) * torch.ones_like(factors) + if torch.any(anti_aliasing_sigma < 0): + raise ValueError("Anti-aliasing standard deviation must be greater than or equal to zero") + elif torch.any((anti_aliasing_sigma > 0) & (factors <= 1)): + warnings.warn( + "Anti-aliasing standard deviation greater than zero but not down-sampling along all axes" + ) + if torch.any(anti_aliasing_sigma == 0): + filtered = image + else: + kernel_sizes = 2 * torch.ceil(3 * anti_aliasing_sigma).int() + 1 + + filtered = F.gaussian_blur( + image, (kernel_sizes[0], kernel_sizes[1]), sigma=anti_aliasing_sigma.tolist() + ) + + else: + filtered = image + + out = F.resize(filtered, size=(size.height, size.width), antialias=False) + + return out + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + interpolation: Optional["F.InterpolationMode"], + do_pad: bool, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Optional[Union[float, list[float]]], + image_std: Optional[Union[float, list[float]]], + disable_grouping: Optional[bool], + return_tensors: Optional[Union[str, TensorType]], + **kwargs, + ) -> BatchFeature: + # Group images by size for batched resizing + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + processed_images_grouped = {} + + for shape, stacked_images in grouped_images.items(): + # Rescale images before other operations as done in original implementation + stacked_images = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, False, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + if do_pad: + processed_images = self.pad(processed_images, disable_grouping=disable_grouping) + + grouped_images, grouped_images_index = group_images_by_shape( + processed_images, disable_grouping=disable_grouping + ) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_resize: + resized_stack = self.resize( + image=stacked_images, + size=size, + interpolation=interpolation, + input_data_format=ChannelDimension.FIRST, + ) + resized_images_grouped[shape] = resized_stack + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + # Group images by size for further processing + # Needed in case do_resize is False, or resize returns images with different sizes + grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + # Fused rescale and normalize + stacked_images = self.rescale_and_normalize( + stacked_images, False, rescale_factor, do_normalize, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images + + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + + +__all__ = ["Owlv2ImageProcessorFast"] diff --git a/src/transformers/models/owlv2/modular_owlv2.py b/src/transformers/models/owlv2/modular_owlv2.py new file mode 100644 index 0000000000..16a27cc01c --- /dev/null +++ b/src/transformers/models/owlv2/modular_owlv2.py @@ -0,0 +1,240 @@ +# coding=utf-8 +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fast Image processor class for OWLv2.""" + +import warnings +from typing import Optional, Union + +from transformers.models.owlvit.image_processing_owlvit_fast import OwlViTImageProcessorFast + +from ...image_processing_utils_fast import ( + BatchFeature, + DefaultFastImageProcessorKwargs, +) +from ...image_transforms import group_images_by_shape, reorder_images +from ...image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ChannelDimension, + ImageInput, + PILImageResampling, + SizeDict, +) +from ...processing_utils import Unpack +from ...utils import ( + TensorType, + auto_docstring, + is_torch_available, + is_torchvision_available, + is_torchvision_v2_available, +) + + +if is_torch_available(): + import torch + + +if is_torchvision_v2_available(): + from torchvision.transforms.v2 import functional as F +elif is_torchvision_available(): + from torchvision.transforms import functional as F + + +class Owlv2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with grey pixels. + """ + + do_pad: Optional[bool] + + +@auto_docstring +class Owlv2ImageProcessorFast(OwlViTImageProcessorFast): + resample = PILImageResampling.BILINEAR + image_mean = OPENAI_CLIP_MEAN + image_std = OPENAI_CLIP_STD + size = {"height": 960, "width": 960} + rescale_factor = 1 / 255 + do_resize = True + do_rescale = True + do_normalize = True + do_pad = True + valid_kwargs = Owlv2FastImageProcessorKwargs + crop_size = None + do_center_crop = None + + def __init__(self, **kwargs: Unpack[Owlv2FastImageProcessorKwargs]): + OwlViTImageProcessorFast().__init__(**kwargs) + + @auto_docstring + def preprocess(self, images: ImageInput, **kwargs: Unpack[Owlv2FastImageProcessorKwargs]): + return OwlViTImageProcessorFast().preprocess(images, **kwargs) + + def _pad_images(self, images: "torch.Tensor", constant_value: float = 0.5) -> "torch.Tensor": + """ + Pad an image with zeros to the given size. + """ + height, width = images.shape[-2:] + size = max(height, width) + pad_bottom = size - height + pad_right = size - width + + padding = (0, 0, pad_right, pad_bottom) + padded_image = F.pad(images, padding, fill=constant_value) + return padded_image + + def pad( + self, + images: list["torch.Tensor"], + disable_grouping: Optional[bool], + constant_value: float = 0.5, + ) -> list["torch.Tensor"]: + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + stacked_images = self._pad_images( + stacked_images, + constant_value=constant_value, + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + return processed_images + + def resize( + self, + image: "torch.Tensor", + size: SizeDict, + anti_aliasing: bool = True, + anti_aliasing_sigma=None, + **kwargs, + ) -> "torch.Tensor": + """ + Resize an image as per the original implementation. + + Args: + image (`Tensor`): + Image to resize. + size (`dict[str, int]`): + Dictionary containing the height and width to resize the image to. + anti_aliasing (`bool`, *optional*, defaults to `True`): + Whether to apply anti-aliasing when downsampling the image. + anti_aliasing_sigma (`float`, *optional*, defaults to `None`): + Standard deviation for Gaussian kernel when downsampling the image. If `None`, it will be calculated + automatically. + """ + output_shape = (size.height, size.width) + + input_shape = image.shape + + # select height and width from input tensor + factors = torch.tensor(input_shape[2:]).to(image.device) / torch.tensor(output_shape).to(image.device) + + if anti_aliasing: + if anti_aliasing_sigma is None: + anti_aliasing_sigma = ((factors - 1) / 2).clamp(min=0) + else: + anti_aliasing_sigma = torch.atleast_1d(anti_aliasing_sigma) * torch.ones_like(factors) + if torch.any(anti_aliasing_sigma < 0): + raise ValueError("Anti-aliasing standard deviation must be greater than or equal to zero") + elif torch.any((anti_aliasing_sigma > 0) & (factors <= 1)): + warnings.warn( + "Anti-aliasing standard deviation greater than zero but not down-sampling along all axes" + ) + if torch.any(anti_aliasing_sigma == 0): + filtered = image + else: + kernel_sizes = 2 * torch.ceil(3 * anti_aliasing_sigma).int() + 1 + + filtered = F.gaussian_blur( + image, (kernel_sizes[0], kernel_sizes[1]), sigma=anti_aliasing_sigma.tolist() + ) + + else: + filtered = image + + out = F.resize(filtered, size=(size.height, size.width), antialias=False) + + return out + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + interpolation: Optional["F.InterpolationMode"], + do_pad: bool, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Optional[Union[float, list[float]]], + image_std: Optional[Union[float, list[float]]], + disable_grouping: Optional[bool], + return_tensors: Optional[Union[str, TensorType]], + **kwargs, + ) -> BatchFeature: + # Group images by size for batched resizing + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + processed_images_grouped = {} + + for shape, stacked_images in grouped_images.items(): + # Rescale images before other operations as done in original implementation + stacked_images = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, False, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + if do_pad: + processed_images = self.pad(processed_images, disable_grouping=disable_grouping) + + grouped_images, grouped_images_index = group_images_by_shape( + processed_images, disable_grouping=disable_grouping + ) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_resize: + resized_stack = self.resize( + image=stacked_images, + size=size, + interpolation=interpolation, + input_data_format=ChannelDimension.FIRST, + ) + resized_images_grouped[shape] = resized_stack + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + # Group images by size for further processing + # Needed in case do_resize is False, or resize returns images with different sizes + grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + # Fused rescale and normalize + stacked_images = self.rescale_and_normalize( + stacked_images, False, rescale_factor, do_normalize, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images + + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + + +__all__ = ["Owlv2ImageProcessorFast"] diff --git a/src/transformers/models/owlv2/processing_owlv2.py b/src/transformers/models/owlv2/processing_owlv2.py index de8f23d3d6..5f45c24695 100644 --- a/src/transformers/models/owlv2/processing_owlv2.py +++ b/src/transformers/models/owlv2/processing_owlv2.py @@ -56,19 +56,19 @@ class Owlv2ProcessorKwargs(ProcessingKwargs, total=False): class Owlv2Processor(ProcessorMixin): r""" - Constructs an Owlv2 processor which wraps [`Owlv2ImageProcessor`] and [`CLIPTokenizer`]/[`CLIPTokenizerFast`] into + Constructs an Owlv2 processor which wraps [`Owlv2ImageProcessor`]/[`Owlv2ImageProcessorFast`] and [`CLIPTokenizer`]/[`CLIPTokenizerFast`] into a single processor that inherits both the image processor and tokenizer functionalities. See the [`~OwlViTProcessor.__call__`] and [`~OwlViTProcessor.decode`] for more information. Args: - image_processor ([`Owlv2ImageProcessor`]): + image_processor ([`Owlv2ImageProcessor`, `Owlv2ImageProcessorFast`]): The image processor is a required input. tokenizer ([`CLIPTokenizer`, `CLIPTokenizerFast`]): The tokenizer is a required input. """ attributes = ["image_processor", "tokenizer"] - image_processor_class = "Owlv2ImageProcessor" + image_processor_class = ("Owlv2ImageProcessor", "Owlv2ImageProcessorFast") tokenizer_class = ("CLIPTokenizer", "CLIPTokenizerFast") def __init__(self, image_processor, tokenizer, **kwargs): diff --git a/tests/models/owlv2/test_image_processing_owlv2.py b/tests/models/owlv2/test_image_processing_owlv2.py index da679c6b08..2306650877 100644 --- a/tests/models/owlv2/test_image_processing_owlv2.py +++ b/tests/models/owlv2/test_image_processing_owlv2.py @@ -16,7 +16,7 @@ import unittest from transformers.testing_utils import require_torch, require_vision, slow -from transformers.utils import is_torch_available, is_vision_available +from transformers.utils import is_torch_available, is_torchvision_available, is_vision_available from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs @@ -29,6 +29,8 @@ if is_vision_available(): if is_torch_available(): import torch + from transformers import Owlv2ImageProcessorFast + class Owlv2ImageProcessingTester: def __init__( @@ -87,6 +89,7 @@ class Owlv2ImageProcessingTester: @require_vision class Owlv2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase): image_processing_class = Owlv2ImageProcessor if is_vision_available() else None + fast_image_processing_class = Owlv2ImageProcessorFast if is_torchvision_available() else None def setUp(self): super().setUp() @@ -97,76 +100,74 @@ class Owlv2ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase): return self.image_processor_tester.prepare_image_processor_dict() def test_image_processor_properties(self): - image_processing = self.image_processing_class(**self.image_processor_dict) - self.assertTrue(hasattr(image_processing, "do_resize")) - self.assertTrue(hasattr(image_processing, "size")) - self.assertTrue(hasattr(image_processing, "do_normalize")) - self.assertTrue(hasattr(image_processing, "image_mean")) - self.assertTrue(hasattr(image_processing, "image_std")) + for image_processing_class in self.image_processor_list: + image_processing = image_processing_class(**self.image_processor_dict) + self.assertTrue(hasattr(image_processing, "do_resize")) + self.assertTrue(hasattr(image_processing, "size")) + self.assertTrue(hasattr(image_processing, "do_normalize")) + self.assertTrue(hasattr(image_processing, "image_mean")) + self.assertTrue(hasattr(image_processing, "image_std")) def test_image_processor_from_dict_with_kwargs(self): - image_processor = self.image_processing_class.from_dict(self.image_processor_dict) - self.assertEqual(image_processor.size, {"height": 18, "width": 18}) + for image_processing_class in self.image_processor_list: + image_processor = image_processing_class.from_dict(self.image_processor_dict) + self.assertEqual(image_processor.size, {"height": 18, "width": 18}) - image_processor = self.image_processing_class.from_dict( - self.image_processor_dict, size={"height": 42, "width": 42} - ) - self.assertEqual(image_processor.size, {"height": 42, "width": 42}) + image_processor = image_processing_class.from_dict( + self.image_processor_dict, size={"height": 42, "width": 42} + ) + self.assertEqual(image_processor.size, {"height": 42, "width": 42}) @slow def test_image_processor_integration_test(self): - processor = Owlv2ImageProcessor() + for image_processing_class in self.image_processor_list: + processor = image_processing_class() - image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") - pixel_values = processor(image, return_tensors="pt").pixel_values + image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") + pixel_values = processor(image, return_tensors="pt").pixel_values - mean_value = round(pixel_values.mean().item(), 4) - self.assertEqual(mean_value, 0.2353) + mean_value = round(pixel_values.mean().item(), 4) + self.assertEqual(mean_value, 0.2353) @slow def test_image_processor_integration_test_resize(self): - checkpoint = "google/owlv2-base-patch16-ensemble" - processor = AutoProcessor.from_pretrained(checkpoint) - model = Owlv2ForObjectDetection.from_pretrained(checkpoint) + for use_fast in [False, True]: + checkpoint = "google/owlv2-base-patch16-ensemble" + processor = AutoProcessor.from_pretrained(checkpoint, use_fast=use_fast) + model = Owlv2ForObjectDetection.from_pretrained(checkpoint) - image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") - text = ["cat"] - target_size = image.size[::-1] - expected_boxes = torch.tensor( - [ - [341.66656494140625, 23.38756561279297, 642.321044921875, 371.3482971191406], - [6.753320693969727, 51.96149826049805, 326.61810302734375, 473.12982177734375], - ] - ) - - # single image - inputs = processor(text=[text], images=[image], return_tensors="pt") - with torch.no_grad(): - outputs = model(**inputs) - - results = processor.post_process_object_detection(outputs, threshold=0.2, target_sizes=[target_size])[0] - - boxes = results["boxes"] - self.assertTrue( - torch.allclose(boxes, expected_boxes, atol=1e-2), - f"Single image bounding boxes fail. Expected {expected_boxes}, got {boxes}", - ) - - # batch of images - inputs = processor(text=[text, text], images=[image, image], return_tensors="pt") - with torch.no_grad(): - outputs = model(**inputs) - results = processor.post_process_object_detection( - outputs, threshold=0.2, target_sizes=[target_size, target_size] - ) - - for result in results: - boxes = result["boxes"] - self.assertTrue( - torch.allclose(boxes, expected_boxes, atol=1e-2), - f"Batch image bounding boxes fail. Expected {expected_boxes}, got {boxes}", + image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") + text = ["cat"] + target_size = image.size[::-1] + expected_boxes = torch.tensor( + [ + [341.66656494140625, 23.38756561279297, 642.321044921875, 371.3482971191406], + [6.753320693969727, 51.96149826049805, 326.61810302734375, 473.12982177734375], + ] ) + # single image + inputs = processor(text=[text], images=[image], return_tensors="pt") + with torch.no_grad(): + outputs = model(**inputs) + + results = processor.post_process_object_detection(outputs, threshold=0.2, target_sizes=[target_size])[0] + + boxes = results["boxes"] + torch.testing.assert_close(boxes, expected_boxes, atol=1e-1, rtol=1e-1) + + # batch of images + inputs = processor(text=[text, text], images=[image, image], return_tensors="pt") + with torch.no_grad(): + outputs = model(**inputs) + results = processor.post_process_object_detection( + outputs, threshold=0.2, target_sizes=[target_size, target_size] + ) + + for result in results: + boxes = result["boxes"] + torch.testing.assert_close(boxes, expected_boxes, atol=1e-1, rtol=1e-1) + @unittest.skip(reason="OWLv2 doesn't treat 4 channel PIL and numpy consistently yet") # FIXME Amy def test_call_numpy_4_channels(self): pass