SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

rename all test_processing_*.py to test_processor_*.py (#33878)

Browse Source 
* rename all test_processing_*.py to test_processor_*.py ans fix duplicate test processor paligemma

* fix copies

* fix broken tests

* fix-copies

* fix test processor bridgetower
This commit is contained in:
Yoni Gozlan
2024-10-02 16:43:43 +02:00
committed by GitHub
parent 2f25ab95db
commit 62e8c759c3
14 changed files with 46 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
tests/models/paligemma/test_processor_paligemma.py
View File

@@ -16,7 +16,7 @@ import shutil
import tempfile
import unittest
from transformers import GemmaTokenizer
from transformers import GemmaTokenizer, PaliGemmaProcessor
from transformers.testing_utils import get_tests_dir, require_torch, require_vision
from transformers.utils import is_vision_available
@@ -24,11 +24,7 @@ from ...test_processing_common import ProcessorTesterMixin
if is_vision_available():
from transformers import (
PaliGemmaProcessor,
SiglipImageProcessor,
is_vision_available,
)
from transformers import SiglipImageProcessor
SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
@@ -61,3 +57,37 @@ class PaliGemmaProcessorTest(ProcessorTesterMixin, unittest.TestCase):
text=input_str, images=image_input, return_tensors="pt", max_length=112, padding="max_length"
)
self.assertEqual(len(inputs["input_ids"][0]), 112 + 14)
def test_text_with_image_tokens(self):
image_processor = self.get_component("image_processor")
tokenizer = self.get_component("tokenizer")
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
text_multi_images = "<image><image><bos>Dummy text!"
text_single_image = "<image><bos>Dummy text!"
text_no_image = "Dummy text!"
image = self.prepare_image_inputs()
out_noimage = processor(text=text_no_image, images=image, return_tensors="np")
out_singlimage = processor(text=text_single_image, images=image, return_tensors="np")
for k in out_noimage:
self.assertTrue(out_noimage[k].tolist() == out_singlimage[k].tolist())
out_multiimages = processor(text=text_multi_images, images=[image, image], return_tensors="np")
out_noimage = processor(text=text_no_image, images=[[image, image]], return_tensors="np")
# We can't be sure what is users intention, whether user want "one text + two images" or user forgot to add the second text
with self.assertRaises(ValueError):
out_noimage = processor(text=text_no_image, images=[image, image], return_tensors="np")
for k in out_noimage:
self.assertTrue(out_noimage[k].tolist() == out_multiimages[k].tolist())
text_batched = ["Dummy text!", "Dummy text!"]
text_batched_with_image = ["<image><bos>Dummy text!", "<image><bos>Dummy text!"]
out_images = processor(text=text_batched_with_image, images=[image, image], return_tensors="np")
out_noimage_nested = processor(text=text_batched, images=[[image], [image]], return_tensors="np")
out_noimage = processor(text=text_batched, images=[image, image], return_tensors="np")
for k in out_noimage:
self.assertTrue(out_noimage[k].tolist() == out_images[k].tolist() == out_noimage_nested[k].tolist())