From 0527c1c0ea79554a63b3809050980dab85c13f0d Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:07:34 +0200 Subject: [PATCH] Add a check in `ImageToTextPipeline._forward` (#24373) * fix * fix * fix * Update src/transformers/pipelines/image_to_text.py Co-authored-by: Nicolas Patry * fix --------- Co-authored-by: ydshieh Co-authored-by: Nicolas Patry --- src/transformers/pipelines/image_to_text.py | 9 +++++++++ tests/models/git/test_modeling_git.py | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/transformers/pipelines/image_to_text.py b/src/transformers/pipelines/image_to_text.py index 1c082c2ecb..25feeb1c0c 100644 --- a/src/transformers/pipelines/image_to_text.py +++ b/src/transformers/pipelines/image_to_text.py @@ -145,6 +145,15 @@ class ImageToTextPipeline(Pipeline): return model_inputs def _forward(self, model_inputs, generate_kwargs=None): + # Git model sets `model_inputs["input_ids"] = None` in `preprocess` (when `prompt=None`). In batch model, the + # pipeline will group them into a list of `None`, which fail `_forward`. Avoid this by checking it first. + if ( + "input_ids" in model_inputs + and isinstance(model_inputs["input_ids"], list) + and all(x is None for x in model_inputs["input_ids"]) + ): + model_inputs["input_ids"] = None + if generate_kwargs is None: generate_kwargs = {} # FIXME: We need to pop here due to a difference in how `generation.py` and `generation.tf_utils.py` diff --git a/tests/models/git/test_modeling_git.py b/tests/models/git/test_modeling_git.py index 45b4457fdc..1c39b61e47 100644 --- a/tests/models/git/test_modeling_git.py +++ b/tests/models/git/test_modeling_git.py @@ -390,15 +390,6 @@ class GitModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, fx_compatible = False test_torchscript = False - # `GitForCausalLM` doesn't fit into image-to-text pipeline. We might need to overwrite its `generate` function. - def is_pipeline_test_to_skip( - self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name - ): - if pipeline_test_casse_name == "ImageToTextPipelineTests": - return True - - return False - # special case for GitForCausalLM model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)