[processor] clean up mulitmodal tests (#37362)
* clkea up mulitmodal processor tests * fixup * fix tests * fix one last test * forgot
This commit is contained in:
committed by
GitHub
parent
3c39c07939
commit
a563999a02
@@ -15,7 +15,6 @@
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from typing import Optional
|
||||
|
||||
from transformers import AutoProcessor, Llama4Processor, PreTrainedTokenizerFast
|
||||
from transformers.testing_utils import require_vision
|
||||
@@ -38,9 +37,10 @@ class Llama4ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
|
||||
|
||||
image_processor = Llama4ImageProcessorFast(max_patches=1, size={"height": 20, "width": 20})
|
||||
tokenizer = PreTrainedTokenizerFast.from_pretrained("unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit")
|
||||
processor_kwargs = {}
|
||||
processor_kwargs = cls.prepare_processor_dict()
|
||||
processor = Llama4Processor(image_processor, tokenizer, **processor_kwargs)
|
||||
processor.save_pretrained(cls.tmpdirname)
|
||||
cls.image_token = processor.image_token
|
||||
|
||||
def get_tokenizer(self, **kwargs):
|
||||
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
|
||||
@@ -51,21 +51,3 @@ class Llama4ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
shutil.rmtree(cls.tmpdirname)
|
||||
|
||||
# Override as Llama4Processor needs image tokens in prompts
|
||||
def prepare_text_inputs(self, batch_size: Optional[int] = None):
|
||||
if batch_size is None:
|
||||
return "lower newer <|image|>"
|
||||
|
||||
if batch_size < 1:
|
||||
raise ValueError("batch_size must be greater than 0")
|
||||
|
||||
if batch_size == 1:
|
||||
return ["lower newer <|image|>"]
|
||||
return ["lower newer <|image|>", "<|image|> upper older longer string"] + ["<|image|> lower newer"] * (
|
||||
batch_size - 2
|
||||
)
|
||||
|
||||
@unittest.skip("This test uses return_tensors='np' which is not supported")
|
||||
def test_image_chat_template_accepts_processing_kwargs(self):
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user