SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Uniformize kwargs for Idefics/2 processors (#32568)

Browse Source 
* Add uniformize idefics processor kwargs and tests

* Uniformize idefics2 processor kwargs

* add image_processor tests idefics

* add BC args order change idefics2 processor and update doc

* Add support for multiple images per prompt in image-text-to-text mode idefics

* Fix processor input args in idefics tests

* improve test processing common, remove unnecessary tests, update process uniformization

* fix doctrings idefics

* fix tests processors idefics/2
This commit is contained in:
Yoni Gozlan
2024-10-03 18:08:24 +02:00
committed by GitHub
parent b0c5660e88
commit 074aa3b3fd
6 changed files with 409 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/models/idefics/test_modeling_idefics.py
View File

@@ -662,7 +662,7 @@ class IdeficsModelIntegrationTest(TestCasePlus):
"HuggingFaceM4/idefics-9b", quantization_config=quantization_config, device_map="auto"
)
processor = self.default_processor
inputs = processor(prompts, return_tensors="pt", padding="longest").to(torch_device)
inputs = processor(text=prompts, return_tensors="pt", padding="longest").to(torch_device)
generated_ids = model.generate(**inputs, max_length=100)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)

192
tests/models/idefics/test_processor_idefics.py
View File

@@ -12,11 +12,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import shutil
import tempfile
import unittest
import numpy as np
from transformers.testing_utils import TestCasePlus, require_torch, require_vision
from transformers import (
AutoProcessor,
IdeficsImageProcessor,
IdeficsProcessor,
LlamaTokenizerFast,
PreTrainedTokenizerFast,
)
from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available
from ...test_processing_common import ProcessorTesterMixin
if is_torch_available():
import torch
@@ -24,37 +37,32 @@ if is_torch_available():
if is_vision_available():
from PIL import Image
from transformers import (
AutoProcessor,
IdeficsImageProcessor,
IdeficsProcessor,
LlamaTokenizerFast,
PreTrainedTokenizerFast,
)
@require_torch
@require_vision
class IdeficsProcessorTest(TestCasePlus):
def setUp(self):
super().setUp()
class IdeficsProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = IdeficsProcessor
self.checkpoint_path = self.get_auto_remove_tmp_dir()
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
image_processor = IdeficsImageProcessor(return_tensors="pt")
tokenizer = LlamaTokenizerFast.from_pretrained("HuggingFaceM4/tiny-random-idefics")
processor = IdeficsProcessor(image_processor, tokenizer)
processor.save_pretrained(self.checkpoint_path)
processor.save_pretrained(self.tmpdirname)
self.input_keys = ["pixel_values", "input_ids", "attention_mask", "image_attention_mask"]
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.checkpoint_path, **kwargs).tokenizer
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.checkpoint_path, **kwargs).image_processor
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
def prepare_prompts(self):
"""This function prepares a list of PIL images"""
@@ -100,13 +108,13 @@ class IdeficsProcessorTest(TestCasePlus):
def test_save_load_pretrained_additional_features(self):
processor = IdeficsProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.checkpoint_path)
processor.save_pretrained(self.tmpdirname)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = IdeficsProcessor.from_pretrained(
self.checkpoint_path, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
@@ -124,7 +132,7 @@ class IdeficsProcessorTest(TestCasePlus):
prompts = self.prepare_prompts()
# test that all prompts succeeded
input_processor = processor(prompts, return_tensors="pt", padding="longest")
input_processor = processor(text=prompts, return_tensors="pt", padding="longest")
for key in self.input_keys:
assert torch.is_tensor(input_processor[key])
@@ -157,8 +165,8 @@ class IdeficsProcessorTest(TestCasePlus):
]
prompts = [[prompt] for prompt in self.prepare_prompts()[2]]
max_length = processor(prompts, padding="max_length", truncation=True, max_length=20, return_tensors="pt")
longest = processor(prompts, padding="longest", truncation=True, max_length=30, return_tensors="pt")
max_length = processor(text=prompts, padding="max_length", truncation=True, max_length=20, return_tensors="pt")
longest = processor(text=prompts, padding="longest", truncation=True, max_length=30, return_tensors="pt")
decoded_max_length = processor.tokenizer.decode(max_length["input_ids"][-1])
decoded_longest = processor.tokenizer.decode(longest["input_ids"][-1])
@@ -185,8 +193,8 @@ class IdeficsProcessorTest(TestCasePlus):
([0] * 10) + ([1] * 10),
]
prompts = [[prompt] for prompt in self.prepare_prompts()[2]]
max_length = processor(prompts, padding="max_length", truncation=True, max_length=20)
longest = processor(prompts, padding="longest", truncation=True, max_length=30)
max_length = processor(text=prompts, padding="max_length", truncation=True, max_length=20)
longest = processor(text=prompts, padding="longest", truncation=True, max_length=30)
decoded_max_length = processor.tokenizer.decode(max_length["input_ids"][-1])
decoded_longest = processor.tokenizer.decode(longest["input_ids"][-1])
@@ -204,7 +212,143 @@ class IdeficsProcessorTest(TestCasePlus):
processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor)
prompts = self.prepare_prompts()
inputs = processor(prompts, padding="longest", return_tensors="pt")
inputs = processor(text=prompts, padding="longest", return_tensors="pt")
# For now the processor supports only ['pixel_values', 'input_ids', 'attention_mask']
self.assertSetEqual(set(inputs.keys()), set(self.input_keys))
# Override the following tests as Idefics image processor does not accept do_rescale and rescale_factor
@require_torch
@require_vision
def test_image_processor_defaults_preserved_by_image_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component("image_processor", image_size=234)
tokenizer = self.get_component("tokenizer", max_length=117)
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()
inputs = processor(text=input_str, images=image_input)
self.assertEqual(len(inputs["pixel_values"][0][0][0]), 234)
@require_torch
@require_vision
def test_kwargs_overrides_default_image_processor_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component("image_processor", image_size=234)
tokenizer = self.get_component("tokenizer", max_length=117)
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()
inputs = processor(text=input_str, images=image_input, image_size=224)
self.assertEqual(len(inputs["pixel_values"][0][0][0]), 224)
@require_torch
@require_vision
def test_unstructured_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component("image_processor")
tokenizer = self.get_component("tokenizer")
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()
inputs = processor(
text=input_str,
images=image_input,
return_tensors="pt",
image_size=214,
padding="max_length",
max_length=76,
)
self.assertEqual(inputs["pixel_values"].shape[3], 214)
self.assertEqual(len(inputs["input_ids"][0]), 76)
@require_torch
@require_vision
def test_unstructured_kwargs_batched(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component("image_processor")
tokenizer = self.get_component("tokenizer")
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = self.prepare_text_inputs(batch_size=2)
image_input = self.prepare_image_inputs(batch_size=2)
inputs = processor(
text=input_str,
images=image_input,
return_tensors="pt",
image_size=214,
padding="longest",
max_length=76,
)
self.assertEqual(inputs["pixel_values"].shape[3], 214)
self.assertEqual(len(inputs["input_ids"][0]), 8)
@require_torch
@require_vision
def test_structured_kwargs_nested(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component("image_processor")
tokenizer = self.get_component("tokenizer")
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()
# Define the kwargs for each modality
all_kwargs = {
"common_kwargs": {"return_tensors": "pt"},
"images_kwargs": {"image_size": 214},
"text_kwargs": {"padding": "max_length", "max_length": 76},
}
inputs = processor(text=input_str, images=image_input, **all_kwargs)
self.skip_processor_without_typed_kwargs(processor)
self.assertEqual(inputs["pixel_values"].shape[3], 214)
self.assertEqual(len(inputs["input_ids"][0]), 76)
@require_torch
@require_vision
def test_structured_kwargs_nested_from_dict(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component("image_processor")
tokenizer = self.get_component("tokenizer")
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()
# Define the kwargs for each modality
all_kwargs = {
"common_kwargs": {"return_tensors": "pt"},
"images_kwargs": {"image_size": 214},
"text_kwargs": {"padding": "max_length", "max_length": 76},
}
inputs = processor(text=input_str, images=image_input, **all_kwargs)
self.assertEqual(inputs["pixel_values"].shape[3], 214)
self.assertEqual(len(inputs["input_ids"][0]), 76)

122
tests/models/idefics2/test_processor_idefics2.py
View File

@@ -13,8 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import shutil
import tempfile
import unittest
from io import BytesIO
from typing import Optional
import requests
@@ -22,16 +25,30 @@ from transformers import Idefics2Processor
from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_vision_available
from ...test_processing_common import ProcessorTesterMixin
if is_vision_available():
from PIL import Image
from transformers import (
AutoProcessor,
Idefics2Processor,
)
@require_torch
@require_vision
class Idefics2ProcessorTest(unittest.TestCase):
class Idefics2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Idefics2Processor
def setUp(self):
self.processor = Idefics2Processor.from_pretrained("HuggingFaceM4/idefics2-8b", image_seq_len=2)
self.tmpdirname = tempfile.mkdtemp()
processor = Idefics2Processor.from_pretrained("HuggingFaceM4/idefics2-8b", image_seq_len=2)
processor.save_pretrained(self.tmpdirname)
self.image1 = Image.open(
BytesIO(
requests.get(
@@ -49,22 +66,35 @@ class Idefics2ProcessorTest(unittest.TestCase):
).content
)
)
self.bos_token = self.processor.tokenizer.bos_token
self.image_token = self.processor.image_token.content
self.fake_image_token = self.processor.fake_image_token.content
self.bos_token = processor.tokenizer.bos_token
self.image_token = processor.image_token.content
self.fake_image_token = processor.fake_image_token.content
self.bos_token_id = self.processor.tokenizer.convert_tokens_to_ids(self.bos_token)
self.image_token_id = self.processor.tokenizer.convert_tokens_to_ids(self.image_token)
self.fake_image_token_id = self.processor.tokenizer.convert_tokens_to_ids(self.fake_image_token)
self.image_seq_len = self.processor.image_seq_len
self.bos_token_id = processor.tokenizer.convert_tokens_to_ids(self.bos_token)
self.image_token_id = processor.tokenizer.convert_tokens_to_ids(self.image_token)
self.fake_image_token_id = processor.tokenizer.convert_tokens_to_ids(self.fake_image_token)
self.image_seq_len = processor.image_seq_len
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def get_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
def test_process_interleaved_images_prompts_no_image_splitting(self):
old_image_splitting = self.processor.image_processor.do_image_splitting
tokenizer = self.get_tokenizer()
processor = self.get_processor()
self.processor.image_processor.do_image_splitting = False
processor.image_processor.do_image_splitting = False
# Test that a single image is processed correctly
inputs = self.processor(images=self.image1)
inputs = processor(images=self.image1)
self.assertEqual(inputs["pixel_values"].shape, (1, 1, 3, 653, 980))
self.assertEqual(inputs["pixel_attention_mask"].shape, (1, 1, 653, 980))
# fmt: on
@@ -73,10 +103,10 @@ class Idefics2ProcessorTest(unittest.TestCase):
image_str = "<image>"
text_str = "In this image, we see"
text = image_str + text_str
inputs = self.processor(text=text, images=self.image1)
inputs = processor(text=text, images=self.image1)
# fmt: off
tokenized_sentence = self.processor.tokenizer(text_str, add_special_tokens=False)
tokenized_sentence = tokenizer(text_str, add_special_tokens=False)
expected_input_ids = [[self.bos_token_id] + [self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id] + tokenized_sentence["input_ids"]]
self.assertEqual(inputs["input_ids"], expected_input_ids)
self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids[0])])
@@ -95,11 +125,11 @@ class Idefics2ProcessorTest(unittest.TestCase):
]
images = [[self.image1], [self.image2, self.image3]]
inputs = self.processor(text=text, images=images, padding=True)
inputs = processor(text=text, images=images, padding=True)
# fmt: off
tokenized_sentence_1 = self.processor.tokenizer(text_str_1, add_special_tokens=False)
tokenized_sentence_2 = self.processor.tokenizer(text_str_2, add_special_tokens=False)
tokenized_sentence_1 = tokenizer(text_str_1, add_special_tokens=False)
tokenized_sentence_2 = tokenizer(text_str_2, add_special_tokens=False)
expected_input_ids_1 = [self.bos_token_id] + [self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id] + tokenized_sentence_1["input_ids"]
expected_input_ids_2 = [self.bos_token_id] + tokenized_sentence_2["input_ids"] + [self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id]
# Pad the first input to match the second input
@@ -117,15 +147,13 @@ class Idefics2ProcessorTest(unittest.TestCase):
self.assertEqual(inputs['pixel_attention_mask'].shape, (2, 2, 767, 980))
# fmt: on
self.processor.image_processor.do_image_splitting = old_image_splitting
def test_process_interleaved_images_prompts_image_splitting(self):
old_image_splitting = self.processor.image_processor.do_image_splitting
self.processor.image_processor.do_image_splitting = True
processor = self.get_processor()
tokenizer = self.get_tokenizer()
processor.image_processor.do_image_splitting = True
# Test that a single image is processed correctly
inputs = self.processor(images=self.image1)
inputs = processor(images=self.image1)
self.assertEqual(inputs["pixel_values"].shape, (1, 5, 3, 653, 980))
self.assertEqual(inputs["pixel_attention_mask"].shape, (1, 5, 653, 980))
# fmt: on
@@ -134,10 +162,10 @@ class Idefics2ProcessorTest(unittest.TestCase):
image_str = "<image>"
text_str = "In this image, we see"
text = image_str + text_str
inputs = self.processor(text=text, images=self.image1)
inputs = processor(text=text, images=self.image1)
# fmt: off
tokenized_sentence = self.processor.tokenizer(text_str, add_special_tokens=False)
tokenized_sentence = tokenizer(text_str, add_special_tokens=False)
expected_input_ids = [[self.bos_token_id] + ([self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len) * 5 + [self.fake_image_token_id] + tokenized_sentence["input_ids"]]
self.assertEqual(inputs["input_ids"], expected_input_ids)
self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids[0])])
@@ -156,11 +184,11 @@ class Idefics2ProcessorTest(unittest.TestCase):
]
images = [[self.image1], [self.image2, self.image3]]
inputs = self.processor(text=text, images=images, padding=True)
inputs = processor(text=text, images=images, padding=True)
# fmt: off
tokenized_sentence_1 = self.processor.tokenizer(text_str_1, add_special_tokens=False)
tokenized_sentence_2 = self.processor.tokenizer(text_str_2, add_special_tokens=False)
tokenized_sentence_1 = tokenizer(text_str_1, add_special_tokens=False)
tokenized_sentence_2 = tokenizer(text_str_2, add_special_tokens=False)
expected_input_ids_1 = [self.bos_token_id] + ([self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len) * 5 + [self.fake_image_token_id] + tokenized_sentence_1["input_ids"]
expected_input_ids_2 = [self.bos_token_id] + tokenized_sentence_2["input_ids"] + ([self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len) * 5 + ([self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len) * 5 + [self.fake_image_token_id]
# Pad the first input to match the second input
@@ -178,22 +206,22 @@ class Idefics2ProcessorTest(unittest.TestCase):
self.assertEqual(inputs['pixel_attention_mask'].shape, (2, 10, 767, 980))
# fmt: on
self.processor.image_processor.do_image_splitting = old_image_splitting
def test_add_special_tokens_processor(self):
processor = self.get_processor()
tokenizer = self.get_tokenizer()
image_str = "<image>"
text_str = "In this image, we see"
text = text_str + image_str
n_image_repeat = 5 if self.processor.image_processor.do_image_splitting else 1
n_image_repeat = 5 if processor.image_processor.do_image_splitting else 1
# fmt: off
inputs = self.processor(text=text, images=self.image1, add_special_tokens=False)
tokenized_sentence = self.processor.tokenizer(text_str, add_special_tokens=False)
inputs = processor(text=text, images=self.image1, add_special_tokens=False)
tokenized_sentence = tokenizer(text_str, add_special_tokens=False)
expected_input_ids = [tokenized_sentence["input_ids"] + ([self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len) * n_image_repeat + [self.fake_image_token_id]]
self.assertEqual(inputs["input_ids"], expected_input_ids)
inputs = self.processor(text=text, images=self.image1)
inputs = processor(text=text, images=self.image1)
expected_input_ids = [[self.bos_token_id] + tokenized_sentence["input_ids"] + ([self.fake_image_token_id] + [self.image_token_id] * self.image_seq_len) * n_image_repeat + [self.fake_image_token_id]]
self.assertEqual(inputs["input_ids"], expected_input_ids)
# fmt: on
@@ -222,7 +250,7 @@ class Idefics2ProcessorTest(unittest.TestCase):
{"role": "user", "content": [{"type": "text", "text": "And who is that?"}]},
]
processor = self.processor
processor = self.get_processor()
# Make short sequence length to test that the fake tokens are added correctly
rendered = processor.apply_chat_template(messages, add_generation_prompt=True)
@@ -233,3 +261,27 @@ class Idefics2ProcessorTest(unittest.TestCase):
"Assistant:"
)
self.assertEqual(rendered, expected_rendered)
# Override as Idefics2Processor needs image tokens in prompts
def prepare_text_inputs(self, batch_size: Optional[int] = None):
if batch_size is None:
return "lower newer <image>"
if batch_size < 1:
raise ValueError("batch_size must be greater than 0")
if batch_size == 1:
return ["lower newer <image>"]
return ["lower newer <image>", "<image> upper older longer string"] + ["<image> lower newer"] * (
batch_size - 2
)
# Override as PixtralProcessor needs nested images to work properly with batched inputs
@require_vision
def prepare_image_inputs(self, batch_size: Optional[int] = None):
"""This function prepares a list of PIL images for testing"""
if batch_size is None:
return super().prepare_image_inputs()
if batch_size < 1:
raise ValueError("batch_size must be greater than 0")
return [[super().prepare_image_inputs()]] * batch_size