SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

🚨 🚨 Setup -> setupclass conversion (#37282)

Browse Source 
* More limited setup -> setupclass conversion

* make fixup

* Trigger tests

* Fixup UDOP

* Missed a spot

* tearDown -> tearDownClass where appropriate

* Couple more class fixes

* Fixups for UDOP and VisionTextDualEncoder

* Ignore errors when removing the tmpdir, in case it already got cleaned up somewhere

* CLIP fixes

* More correct classmethods

* Wav2Vec2Bert fixes

* More methods become static

* More class methods

* More class methods

* Revert changes for integration tests / modeling files

* Use a different tempdir for tests that actually write to it

* Remove addClassCleanup and just use teardownclass

* Remove changes in modeling files

* Cleanup get_processor_dict() for got_ocr2

* Fix regression on Wav2Vec2BERT test that was masked by this before

* Rework tests that modify the tmpdir

* make fix-copies

* revert clvp modeling test changes

* Fix CLIP processor test

* make fix-copies
This commit is contained in:
Matt
2025-04-08 17:15:37 +01:00
committed by GitHub
parent c15a7adb28
commit 4d0de5f73a
49 changed files with 740 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tests/models/altclip/test_processor_altclip.py
View File

@@ -26,15 +26,16 @@ from ...test_processing_common import ProcessorTesterMixin
class AltClipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = AltCLIPProcessor
def setUp(self):
self.model_id = "BAAI/AltCLIP"
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.model_id = "BAAI/AltCLIP"
cls.tmpdirname = tempfile.mkdtemp()
image_processor = CLIPImageProcessor()
tokenizer = XLMRobertaTokenizer.from_pretrained(self.model_id)
tokenizer = XLMRobertaTokenizer.from_pretrained(cls.model_id)
processor = self.processor_class(image_processor, tokenizer)
processor = cls.processor_class(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return XLMRobertaTokenizer.from_pretrained(self.model_id, **kwargs)

2
tests/models/aria/test_processor_aria.py
View File

@@ -87,7 +87,7 @@ class AriaProcessorTest(ProcessorTesterMixin, unittest.TestCase):
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname)
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_kwargs_overrides_default_image_processor_kwargs(self):
if "image_processor" not in self.processor_class.attributes:

17
tests/models/aya_vision/test_processor_aya_vision.py
View File

@@ -37,8 +37,9 @@ if is_vision_available():
class AyaVisionProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = AyaVisionProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = GotOcr2ImageProcessor(
do_resize=True,
@@ -52,16 +53,17 @@ class AyaVisionProcessorTest(ProcessorTesterMixin, unittest.TestCase):
do_convert_rgb=True,
)
tokenizer = AutoTokenizer.from_pretrained("CohereForAI/aya-vision-8b", padding_side="left")
processor_kwargs = self.prepare_processor_dict()
processor_kwargs = cls.prepare_processor_dict()
processor = AyaVisionProcessor.from_pretrained(
"CohereForAI/aya-vision-8b",
image_processor=image_processor,
tokenizer=tokenizer,
**processor_kwargs,
)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def prepare_processor_dict(self):
@staticmethod
def prepare_processor_dict():
return {"patch_size": 10, "img_size": 20}
def get_tokenizer(self, **kwargs):
@@ -73,8 +75,9 @@ class AyaVisionProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
# todo: yoni, fix this test
@unittest.skip("Chat template has long system prompt")

27
tests/models/blip/test_processor_blip.py
View File

@@ -31,15 +31,16 @@ if is_vision_available():
class BlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = BlipProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = BlipImageProcessor()
tokenizer = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-BertModel")
processor = BlipProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -47,19 +48,21 @@ class BlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_additional_features(self):
processor = BlipProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = BlipProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = BlipProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = BlipProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

27
tests/models/blip_2/test_processor_blip_2.py
View File

@@ -31,15 +31,16 @@ if is_vision_available():
class Blip2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Blip2Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = BlipImageProcessor()
tokenizer = GPT2Tokenizer.from_pretrained("hf-internal-testing/tiny-random-GPT2Model")
processor = Blip2Processor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -47,19 +48,21 @@ class Blip2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_additional_features(self):
processor = Blip2Processor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = Blip2Processor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = Blip2Processor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = Blip2Processor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

12
tests/models/bridgetower/test_processor_bridgetower.py
View File

@@ -34,15 +34,16 @@ if is_vision_available():
class BridgeTowerProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = BridgeTowerProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = BridgeTowerImageProcessor()
tokenizer = RobertaTokenizerFast.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
processor = BridgeTowerProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -50,8 +51,9 @@ class BridgeTowerProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
# Some kwargs tests are overriden from common tests to handle shortest_edge
# and size_divisor behaviour

9
tests/models/chameleon/test_processor_chameleon.py
View File

@@ -33,11 +33,12 @@ SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
class ChameleonProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = ChameleonProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = ChameleonImageProcessor()
tokenizer = LlamaTokenizer(vocab_file=SAMPLE_VOCAB)
tokenizer.pad_token_id = 0
tokenizer.sep_token_id = 1
processor = self.processor_class(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor = cls.processor_class(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(cls.tmpdirname)

69
tests/models/chinese_clip/test_processor_chinese_clip.py
View File

@@ -36,8 +36,9 @@ if is_vision_available():
class ChineseCLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = ChineseCLIPProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
vocab_tokens = [
"[UNK]",
@@ -59,8 +60,8 @@ class ChineseCLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"t",
"shirt",
]
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(cls.vocab_file, "w", encoding="utf-8") as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
image_processor_map = {
@@ -73,39 +74,44 @@ class ChineseCLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"image_std": [0.26862954, 0.26130258, 0.27577711],
"do_convert_rgb": True,
}
self.image_processor_file = os.path.join(self.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(self.image_processor_file, "w", encoding="utf-8") as fp:
cls.image_processor_file = os.path.join(cls.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(cls.image_processor_file, "w", encoding="utf-8") as fp:
json.dump(image_processor_map, fp)
tokenizer = self.get_tokenizer()
image_processor = self.get_image_processor()
tokenizer = cls.get_tokenizer()
image_processor = cls.get_image_processor()
processor = ChineseCLIPProcessor(tokenizer=tokenizer, image_processor=image_processor)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return BertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_tokenizer(cls, **kwargs):
return BertTokenizer.from_pretrained(cls.tmpdirname, **kwargs)
def get_rust_tokenizer(self, **kwargs):
return BertTokenizerFast.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_rust_tokenizer(cls, **kwargs):
return BertTokenizerFast.from_pretrained(cls.tmpdirname, **kwargs)
def get_image_processor(self, **kwargs):
return ChineseCLIPImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_image_processor(cls, **kwargs):
return ChineseCLIPImageProcessor.from_pretrained(cls.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer_slow = self.get_tokenizer()
tokenizer_fast = self.get_rust_tokenizer()
image_processor = self.get_image_processor()
processor_slow = ChineseCLIPProcessor(tokenizer=tokenizer_slow, image_processor=image_processor)
processor_slow.save_pretrained(self.tmpdirname)
processor_slow = ChineseCLIPProcessor.from_pretrained(self.tmpdirname, use_fast=False)
with tempfile.TemporaryDirectory() as tmpdir:
processor_slow = ChineseCLIPProcessor(tokenizer=tokenizer_slow, image_processor=image_processor)
processor_slow.save_pretrained(tmpdir)
processor_slow = ChineseCLIPProcessor.from_pretrained(self.tmpdirname, use_fast=False)
processor_fast = ChineseCLIPProcessor(tokenizer=tokenizer_fast, image_processor=image_processor)
processor_fast.save_pretrained(self.tmpdirname)
processor_fast = ChineseCLIPProcessor.from_pretrained(self.tmpdirname)
processor_fast = ChineseCLIPProcessor(tokenizer=tokenizer_fast, image_processor=image_processor)
processor_fast.save_pretrained(tmpdir)
processor_fast = ChineseCLIPProcessor.from_pretrained(self.tmpdirname)
self.assertEqual(processor_slow.tokenizer.get_vocab(), tokenizer_slow.get_vocab())
self.assertEqual(processor_fast.tokenizer.get_vocab(), tokenizer_fast.get_vocab())
@@ -119,15 +125,18 @@ class ChineseCLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertIsInstance(processor_fast.image_processor, ChineseCLIPImageProcessor)
def test_save_load_pretrained_additional_features(self):
processor = ChineseCLIPProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = ChineseCLIPProcessor(
tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor()
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(cls_token="(CLS)", sep_token="(SEP)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False)
tokenizer_add_kwargs = self.get_tokenizer(cls_token="(CLS)", sep_token="(SEP)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False)
processor = ChineseCLIPProcessor.from_pretrained(
self.tmpdirname, cls_token="(CLS)", sep_token="(SEP)", do_normalize=False
)
processor = ChineseCLIPProcessor.from_pretrained(
tmpdir, cls_token="(CLS)", sep_token="(SEP)", do_normalize=False
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, BertTokenizerFast)

69
tests/models/clip/test_processor_clip.py
View File

@@ -36,19 +36,20 @@ if is_vision_available():
class CLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = CLIPProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
vocab = ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"] # fmt: skip
vocab_tokens = dict(zip(vocab, range(len(vocab))))
merges = ["#version: 0.2", "l o", "lo w</w>", "e r</w>", ""]
self.special_tokens_map = {"unk_token": "<unk>"}
cls.special_tokens_map = {"unk_token": "<unk>"}
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"])
with open(self.vocab_file, "w", encoding="utf-8") as fp:
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
cls.merges_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["merges_file"])
with open(cls.vocab_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(vocab_tokens) + "\n")
with open(self.merges_file, "w", encoding="utf-8") as fp:
with open(cls.merges_file, "w", encoding="utf-8") as fp:
fp.write("\n".join(merges))
image_processor_map = {
@@ -60,34 +61,39 @@ class CLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"image_mean": [0.48145466, 0.4578275, 0.40821073],
"image_std": [0.26862954, 0.26130258, 0.27577711],
}
self.image_processor_file = os.path.join(self.tmpdirname, IMAGE_PROCESSOR_NAME)
with open(self.image_processor_file, "w", encoding="utf-8") as fp:
cls.image_processor_file = os.path.join(cls.tmpdirname, IMAGE_PROCESSOR_NAME)
with open(cls.image_processor_file, "w", encoding="utf-8") as fp:
json.dump(image_processor_map, fp)
def get_tokenizer(self, **kwargs):
return CLIPTokenizer.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_tokenizer(cls, **kwargs):
return CLIPTokenizer.from_pretrained(cls.tmpdirname, **kwargs)
def get_rust_tokenizer(self, **kwargs):
return CLIPTokenizerFast.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_rust_tokenizer(cls, **kwargs):
return CLIPTokenizerFast.from_pretrained(cls.tmpdirname, **kwargs)
def get_image_processor(self, **kwargs):
return CLIPImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_image_processor(cls, **kwargs):
return CLIPImageProcessor.from_pretrained(cls.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname)
def test_save_load_pretrained_default(self):
tokenizer_slow = self.get_tokenizer()
tokenizer_fast = self.get_rust_tokenizer()
image_processor = self.get_image_processor()
processor_slow = CLIPProcessor(tokenizer=tokenizer_slow, image_processor=image_processor)
processor_slow.save_pretrained(self.tmpdirname)
processor_slow = CLIPProcessor.from_pretrained(self.tmpdirname, use_fast=False)
with tempfile.TemporaryDirectory() as tmpdir:
processor_slow = CLIPProcessor(tokenizer=tokenizer_slow, image_processor=image_processor)
processor_slow.save_pretrained(tmpdir)
processor_slow = CLIPProcessor.from_pretrained(tmpdir, use_fast=False)
processor_fast = CLIPProcessor(tokenizer=tokenizer_fast, image_processor=image_processor)
processor_fast.save_pretrained(self.tmpdirname)
processor_fast = CLIPProcessor.from_pretrained(self.tmpdirname)
processor_fast = CLIPProcessor(tokenizer=tokenizer_fast, image_processor=image_processor)
processor_fast.save_pretrained(tmpdir)
processor_fast = CLIPProcessor.from_pretrained(tmpdir)
self.assertEqual(processor_slow.tokenizer.get_vocab(), tokenizer_slow.get_vocab())
self.assertEqual(processor_fast.tokenizer.get_vocab(), tokenizer_fast.get_vocab())
@@ -101,15 +107,18 @@ class CLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertIsInstance(processor_fast.image_processor, CLIPImageProcessor)
def test_save_load_pretrained_additional_features(self):
processor = CLIPProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = CLIPProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = CLIPTokenizer.from_pretrained(tmpdir, bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = CLIPImageProcessor.from_pretrained(
tmpdir, do_normalize=False, padding_value=1.0
)
processor = CLIPProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = CLIPProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, CLIPTokenizerFast)

12
tests/models/colpali/test_processing_colpali.py
View File

@@ -27,16 +27,18 @@ SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
class ColPaliProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = ColPaliProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = SiglipImageProcessor.from_pretrained("google/siglip-so400m-patch14-384")
image_processor.image_seq_length = 0
tokenizer = GemmaTokenizer(SAMPLE_VOCAB, keep_accents=True)
processor = PaliGemmaProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
@require_torch
@require_vision

11
tests/models/donut/test_processor_donut.py
View File

@@ -25,16 +25,17 @@ class DonutProcessorTest(ProcessorTesterMixin, unittest.TestCase):
from_pretrained_id = "naver-clova-ix/donut-base"
processor_class = DonutProcessor
def setUp(self):
self.processor = DonutProcessor.from_pretrained(self.from_pretrained_id)
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.processor = DonutProcessor.from_pretrained(cls.from_pretrained_id)
cls.tmpdirname = tempfile.mkdtemp()
image_processor = DonutImageProcessor()
tokenizer = XLMRobertaTokenizerFast.from_pretrained(self.from_pretrained_id)
tokenizer = XLMRobertaTokenizerFast.from_pretrained(cls.from_pretrained_id)
processor = DonutProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def test_token2json(self):
expected_json = {

9
tests/models/emu3/test_processor_emu3.py
View File

@@ -31,8 +31,9 @@ if is_vision_available():
class Emu3ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Emu3Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = Emu3ImageProcessor()
extra_special_tokens = extra_special_tokens = {
"image_token": "<image>",
@@ -46,10 +47,10 @@ class Emu3ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
)
tokenizer.pad_token_id = 0
tokenizer.sep_token_id = 1
processor = self.processor_class(
processor = cls.processor_class(
image_processor=image_processor, tokenizer=tokenizer, chat_template="dummy_template"
)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def prepare_processor_dict(self):
return {

6
tests/models/fuyu/test_processor_fuyu.py
View File

@@ -36,8 +36,6 @@ class FuyuProcessingTest(ProcessorTesterMixin, unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
# Ensure tempdir is cleaned up even if there's a failure
cls.addClassCleanup(lambda tempdir=cls.tmpdirname: rmtree(tempdir))
image_processor = FuyuImageProcessor()
tokenizer = AutoTokenizer.from_pretrained("adept/fuyu-8b")
@@ -49,6 +47,10 @@ class FuyuProcessingTest(ProcessorTesterMixin, unittest.TestCase):
bus_image_url = "https://huggingface.co/datasets/hf-internal-testing/fixtures-captioning/resolve/main/bus.png"
cls.bus_image_pil = Image.open(io.BytesIO(requests.get(bus_image_url).content))
@classmethod
def tearDownClass(cls):
rmtree(cls.tmpdirname)
def get_processor(self):
image_processor = FuyuImageProcessor()
tokenizer = AutoTokenizer.from_pretrained("adept/fuyu-8b")

17
tests/models/gemma3/test_processing_gemma3.py
View File

@@ -34,8 +34,9 @@ SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
class Gemma3ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Gemma3Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
gemma3_image_processor_kwargs = {
"do_pan_and_scan": True,
"pan_and_scan_min_crop_size": 256,
@@ -52,15 +53,17 @@ class Gemma3ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"eoi_token": "<end_of_image>",
}
tokenizer = GemmaTokenizer(SAMPLE_VOCAB, keep_accents=True, extra_special_tokens=extra_special_tokens)
processor_kwargs = self.prepare_processor_dict()
processor_kwargs = cls.prepare_processor_dict()
processor = Gemma3Processor(image_processor=image_processor, tokenizer=tokenizer, **processor_kwargs)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
# TODO: raushan or arthur: add the real chat template
def prepare_processor_dict(self):
@staticmethod
def prepare_processor_dict():
return {
"chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n {%- set first_user_prefix = \"\" -%}\n {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif -%}\n {%- if (message['role'] == 'assistant') -%}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n {%- if message['content'] is string -%}\n {{ message['content'] | trim }}\n {%- elif message['content'] is iterable -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'image' -%}\n {{ '<start_of_image>' }}\n {%- elif item['type'] == 'text' -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type\") }}\n {%- endif -%}\n {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{'<start_of_turn>model\n'}}\n{%- endif -%}\n", "image_seq_length": 3,
} # fmt: skip

27
tests/models/git/test_processor_git.py
View File

@@ -31,8 +31,9 @@ if is_vision_available():
class GitProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = GitProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = CLIPImageProcessor()
tokenizer = BertTokenizer.from_pretrained(
@@ -41,7 +42,7 @@ class GitProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = GitProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -49,19 +50,21 @@ class GitProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_additional_features(self):
processor = GitProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = GitProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = GitProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = GitProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

14
tests/models/got_ocr2/test_processor_got_ocr2.py
View File

@@ -31,14 +31,15 @@ if is_vision_available():
class GotOcr2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = GotOcr2Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = GotOcr2ImageProcessor()
tokenizer = PreTrainedTokenizerFast.from_pretrained("stepfun-ai/GOT-OCR-2.0-hf")
processor_kwargs = self.prepare_processor_dict()
processor_kwargs = {}
processor = GotOcr2Processor(image_processor, tokenizer, **processor_kwargs)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -46,8 +47,9 @@ class GotOcr2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_ocr_queries(self):
processor = self.get_processor()

78
tests/models/grounding_dino/test_processor_grounding_dino.py
View File

@@ -44,12 +44,13 @@ class GroundingDinoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
from_pretrained_id = "IDEA-Research/grounding-dino-base"
processor_class = GroundingDinoProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
vocab_tokens = ["[UNK]","[CLS]","[SEP]","[PAD]","[MASK]","want","##want","##ed","wa","un","runn","##ing",",","low","lowest"] # fmt: skip
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(cls.vocab_file, "w", encoding="utf-8") as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
image_processor_map = {
@@ -62,21 +63,21 @@ class GroundingDinoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"rescale_factor": 1 / 255,
"do_pad": True,
}
self.image_processor_file = os.path.join(self.tmpdirname, IMAGE_PROCESSOR_NAME)
with open(self.image_processor_file, "w", encoding="utf-8") as fp:
cls.image_processor_file = os.path.join(cls.tmpdirname, IMAGE_PROCESSOR_NAME)
with open(cls.image_processor_file, "w", encoding="utf-8") as fp:
json.dump(image_processor_map, fp)
image_processor = GroundingDinoImageProcessor()
tokenizer = BertTokenizer.from_pretrained(self.from_pretrained_id)
tokenizer = BertTokenizer.from_pretrained(cls.from_pretrained_id)
processor = GroundingDinoProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
self.batch_size = 7
self.num_queries = 5
self.embed_dim = 5
self.seq_length = 5
cls.batch_size = 7
cls.num_queries = 5
cls.embed_dim = 5
cls.seq_length = 5
def prepare_text_inputs(self, batch_size: Optional[int] = None):
labels = ["a cat", "remote control"]
@@ -92,21 +93,24 @@ class GroundingDinoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
return [labels]
return [labels, labels_longer] + [labels] * (batch_size - 2)
@classmethod
# Copied from tests.models.clip.test_processor_clip.CLIPProcessorTest.get_tokenizer with CLIP->Bert
def get_tokenizer(self, **kwargs):
return BertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
def get_tokenizer(cls, **kwargs):
return BertTokenizer.from_pretrained(cls.tmpdirname, **kwargs)
@classmethod
# Copied from tests.models.clip.test_processor_clip.CLIPProcessorTest.get_rust_tokenizer with CLIP->Bert
def get_rust_tokenizer(self, **kwargs):
return BertTokenizerFast.from_pretrained(self.tmpdirname, **kwargs)
def get_rust_tokenizer(cls, **kwargs):
return BertTokenizerFast.from_pretrained(cls.tmpdirname, **kwargs)
@classmethod
# Copied from tests.models.clip.test_processor_clip.CLIPProcessorTest.get_image_processor with CLIP->GroundingDino
def get_image_processor(self, **kwargs):
return GroundingDinoImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
def get_image_processor(cls, **kwargs):
return GroundingDinoImageProcessor.from_pretrained(cls.tmpdirname, **kwargs)
# Copied from tests.models.clip.test_processor_clip.CLIPProcessorTest.tearDown
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def get_fake_grounding_dino_output(self):
torch.manual_seed(42)
@@ -147,13 +151,14 @@ class GroundingDinoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
tokenizer_fast = self.get_rust_tokenizer()
image_processor = self.get_image_processor()
processor_slow = GroundingDinoProcessor(tokenizer=tokenizer_slow, image_processor=image_processor)
processor_slow.save_pretrained(self.tmpdirname)
processor_slow = GroundingDinoProcessor.from_pretrained(self.tmpdirname, use_fast=False)
with tempfile.TemporaryDirectory() as tmpdir:
processor_slow = GroundingDinoProcessor(tokenizer=tokenizer_slow, image_processor=image_processor)
processor_slow.save_pretrained(tmpdir)
processor_slow = GroundingDinoProcessor.from_pretrained(tmpdir, use_fast=False)
processor_fast = GroundingDinoProcessor(tokenizer=tokenizer_fast, image_processor=image_processor)
processor_fast.save_pretrained(self.tmpdirname)
processor_fast = GroundingDinoProcessor.from_pretrained(self.tmpdirname)
processor_fast = GroundingDinoProcessor(tokenizer=tokenizer_fast, image_processor=image_processor)
processor_fast.save_pretrained(tmpdir)
processor_fast = GroundingDinoProcessor.from_pretrained(tmpdir)
self.assertEqual(processor_slow.tokenizer.get_vocab(), tokenizer_slow.get_vocab())
self.assertEqual(processor_fast.tokenizer.get_vocab(), tokenizer_fast.get_vocab())
@@ -168,15 +173,20 @@ class GroundingDinoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
# Copied from tests.models.clip.test_processor_clip.CLIPProcessorTest.test_save_load_pretrained_additional_features with CLIP->GroundingDino,GroundingDinoTokenizer->BertTokenizer
def test_save_load_pretrained_additional_features(self):
processor = GroundingDinoProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = GroundingDinoProcessor(
tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor()
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = BertTokenizer.from_pretrained(tmpdir, bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = GroundingDinoImageProcessor.from_pretrained(
tmpdir, do_normalize=False, padding_value=1.0
)
processor = GroundingDinoProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = GroundingDinoProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, BertTokenizerFast)

29
tests/models/idefics/test_processor_idefics.py
View File

@@ -43,17 +43,18 @@ if is_vision_available():
class IdeficsProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = IdeficsProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = IdeficsImageProcessor(return_tensors="pt")
tokenizer = LlamaTokenizerFast.from_pretrained("HuggingFaceM4/tiny-random-idefics")
processor = IdeficsProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
self.input_keys = ["pixel_values", "input_ids", "attention_mask", "image_attention_mask"]
cls.input_keys = ["pixel_values", "input_ids", "attention_mask", "image_attention_mask"]
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -61,8 +62,9 @@ class IdeficsProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def prepare_prompts(self):
"""This function prepares a list of PIL images"""
@@ -107,15 +109,16 @@ class IdeficsProcessorTest(ProcessorTesterMixin, unittest.TestCase):
return prompts
def test_save_load_pretrained_additional_features(self):
processor = IdeficsProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = IdeficsProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = IdeficsProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = IdeficsProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

32
tests/models/idefics2/test_processor_idefics2.py
View File

@@ -41,38 +41,39 @@ if is_vision_available():
class Idefics2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Idefics2Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
processor = Idefics2Processor.from_pretrained("HuggingFaceM4/idefics2-8b", image_seq_len=2)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
self.image1 = Image.open(
cls.image1 = Image.open(
BytesIO(
requests.get(
"https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
).content
)
)
self.image2 = Image.open(
cls.image2 = Image.open(
BytesIO(requests.get("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg").content)
)
self.image3 = Image.open(
cls.image3 = Image.open(
BytesIO(
requests.get(
"https://thumbs.dreamstime.com/b/golden-gate-bridge-san-francisco-purple-flowers-california-echium-candicans-36805947.jpg"
).content
)
)
self.bos_token = processor.tokenizer.bos_token
self.image_token = processor.image_token.content
self.fake_image_token = processor.fake_image_token.content
cls.bos_token = processor.tokenizer.bos_token
cls.image_token = processor.image_token.content
cls.fake_image_token = processor.fake_image_token.content
self.bos_token_id = processor.tokenizer.convert_tokens_to_ids(self.bos_token)
self.image_token_id = processor.tokenizer.convert_tokens_to_ids(self.image_token)
self.fake_image_token_id = processor.tokenizer.convert_tokens_to_ids(self.fake_image_token)
self.image_seq_len = processor.image_seq_len
cls.bos_token_id = processor.tokenizer.convert_tokens_to_ids(cls.bos_token)
cls.image_token_id = processor.tokenizer.convert_tokens_to_ids(cls.image_token)
cls.fake_image_token_id = processor.tokenizer.convert_tokens_to_ids(cls.fake_image_token)
cls.image_seq_len = processor.image_seq_len
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -83,8 +84,9 @@ class Idefics2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_process_interleaved_images_prompts_no_image_splitting(self):
tokenizer = self.get_tokenizer()

2
tests/models/idefics3/test_processor_idefics3.py
View File

@@ -104,7 +104,7 @@ class Idefics3ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname)
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_process_interleaved_images_prompts_no_image_splitting(self):
processor = self.get_processor()

25
tests/models/instructblip/test_processor_instructblip.py
View File

@@ -38,8 +38,9 @@ if is_vision_available():
class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = InstructBlipProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = BlipImageProcessor()
tokenizer = GPT2Tokenizer.from_pretrained("hf-internal-testing/tiny-random-GPT2Model")
@@ -47,7 +48,7 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = InstructBlipProcessor(image_processor, tokenizer, qformer_tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -58,8 +59,9 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_qformer_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).qformer_tokenizer
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_additional_features(self):
processor = InstructBlipProcessor(
@@ -67,14 +69,15 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
image_processor=self.get_image_processor(),
qformer_tokenizer=self.get_qformer_tokenizer(),
)
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = InstructBlipProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = InstructBlipProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

25
tests/models/instructblipvideo/test_processor_instructblipvideo.py
View File

@@ -39,8 +39,9 @@ if is_vision_available():
class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = InstructBlipVideoProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = InstructBlipVideoImageProcessor()
tokenizer = GPT2Tokenizer.from_pretrained("hf-internal-testing/tiny-random-GPT2Model")
@@ -48,7 +49,7 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = InstructBlipVideoProcessor(image_processor, tokenizer, qformer_tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -59,8 +60,9 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_qformer_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).qformer_tokenizer
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_additional_features(self):
processor = InstructBlipVideoProcessor(
@@ -68,14 +70,15 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
image_processor=self.get_image_processor(),
qformer_tokenizer=self.get_qformer_tokenizer(),
)
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = InstructBlipVideoProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = InstructBlipVideoProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

27
tests/models/kosmos2/test_processor_kosmos2.py
View File

@@ -57,8 +57,9 @@ SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
class Kosmos2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Kosmos2Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = CLIPImageProcessor(do_center_crop=False)
@@ -67,7 +68,7 @@ class Kosmos2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
fast_tokenizer = XLMRobertaTokenizerFast(__slow_tokenizer=slow_tokenizer)
processor = Kosmos2Processor(image_processor, fast_tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
# We override this method to take the fast tokenizer by default
def get_component(self, attribute, **kwargs):
@@ -92,8 +93,9 @@ class Kosmos2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_image_procesor_load_save_reload(self):
# make sure load from Hub repo. -> save -> reload locally work
@@ -105,15 +107,16 @@ class Kosmos2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
assert image_processor.to_json_string() == reloaded_image_processor.to_json_string()
def test_save_load_pretrained_additional_features(self):
processor = Kosmos2Processor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = Kosmos2Processor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = Kosmos2Processor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = Kosmos2Processor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

74
tests/models/layoutxlm/test_processor_layoutxlm.py
View File

@@ -46,40 +46,46 @@ class LayoutXLMProcessorTest(ProcessorTesterMixin, unittest.TestCase):
rust_tokenizer_class = LayoutXLMTokenizerFast
processor_class = LayoutXLMProcessor
def setUp(self):
@classmethod
def setUpClass(cls):
image_processor_map = {
"do_resize": True,
"size": 224,
"apply_ocr": True,
}
self.tmpdirname = tempfile.mkdtemp()
self.feature_extraction_file = os.path.join(self.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(self.feature_extraction_file, "w", encoding="utf-8") as fp:
cls.tmpdirname = tempfile.mkdtemp()
cls.feature_extraction_file = os.path.join(cls.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(cls.feature_extraction_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(image_processor_map) + "\n")
# taken from `test_tokenization_layoutxlm.LayoutXLMTokenizationTest.test_save_pretrained`
self.tokenizer_pretrained_name = "hf-internal-testing/tiny-random-layoutxlm"
cls.tokenizer_pretrained_name = "hf-internal-testing/tiny-random-layoutxlm"
tokenizer = self.get_tokenizer()
image_processor = self.get_image_processor()
tokenizer = cls.get_tokenizer()
image_processor = cls.get_image_processor()
processor = LayoutXLMProcessor(tokenizer=tokenizer, image_processor=image_processor)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs) -> PreTrainedTokenizer:
return self.tokenizer_class.from_pretrained(self.tokenizer_pretrained_name, **kwargs)
@classmethod
def get_tokenizer(cls, **kwargs) -> PreTrainedTokenizer:
return cls.tokenizer_class.from_pretrained(cls.tokenizer_pretrained_name, **kwargs)
def get_rust_tokenizer(self, **kwargs) -> PreTrainedTokenizerFast:
return self.rust_tokenizer_class.from_pretrained(self.tokenizer_pretrained_name, **kwargs)
@classmethod
def get_rust_tokenizer(cls, **kwargs) -> PreTrainedTokenizerFast:
return cls.rust_tokenizer_class.from_pretrained(cls.tokenizer_pretrained_name, **kwargs)
def get_tokenizers(self, **kwargs) -> list[PreTrainedTokenizerBase]:
return [self.get_tokenizer(**kwargs), self.get_rust_tokenizer(**kwargs)]
@classmethod
def get_tokenizers(cls, **kwargs) -> list[PreTrainedTokenizerBase]:
return [cls.get_tokenizer(**kwargs), cls.get_rust_tokenizer(**kwargs)]
def get_image_processor(self, **kwargs):
return LayoutLMv2ImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_image_processor(cls, **kwargs):
return LayoutLMv2ImageProcessor.from_pretrained(cls.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
image_processor = self.get_image_processor()
@@ -87,8 +93,9 @@ class LayoutXLMProcessorTest(ProcessorTesterMixin, unittest.TestCase):
for tokenizer in tokenizers:
processor = LayoutXLMProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor = LayoutXLMProcessor.from_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
processor = LayoutXLMProcessor.from_pretrained(tmpdir)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, (LayoutXLMTokenizer, LayoutXLMTokenizerFast))
@@ -97,21 +104,22 @@ class LayoutXLMProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertIsInstance(processor.image_processor, LayoutLMv2ImageProcessor)
def test_save_load_pretrained_additional_features(self):
processor = LayoutXLMProcessor(image_processor=self.get_image_processor(), tokenizer=self.get_tokenizer())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = LayoutXLMProcessor(image_processor=self.get_image_processor(), tokenizer=self.get_tokenizer())
processor.save_pretrained(tmpdir)
# slow tokenizer
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_resize=False, size=30)
# slow tokenizer
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_resize=False, size=30)
processor = LayoutXLMProcessor.from_pretrained(
self.tmpdirname,
use_fast=False,
bos_token="(BOS)",
eos_token="(EOS)",
do_resize=False,
size=30,
)
processor = LayoutXLMProcessor.from_pretrained(
tmpdir,
use_fast=False,
bos_token="(BOS)",
eos_token="(EOS)",
do_resize=False,
size=30,
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, LayoutXLMTokenizer)

17
tests/models/llava/test_processor_llava.py
View File

@@ -34,14 +34,15 @@ if is_torch_available:
class LlavaProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = LlavaProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = CLIPImageProcessor(do_center_crop=False)
tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b")
processor_kwargs = self.prepare_processor_dict()
processor_kwargs = cls.prepare_processor_dict()
processor = LlavaProcessor(image_processor, tokenizer, **processor_kwargs)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -49,10 +50,12 @@ class LlavaProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def prepare_processor_dict(self):
@staticmethod
def prepare_processor_dict():
return {
"chat_template": "{% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>\n' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}",
"patch_size": 3,

12
tests/models/llava_next/test_processor_llava_next.py
View File

@@ -34,14 +34,15 @@ if is_vision_available():
class LlavaNextProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = LlavaNextProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = LlavaNextImageProcessor()
tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b")
processor_kwargs = self.prepare_processor_dict()
processor_kwargs = cls.prepare_processor_dict()
processor = LlavaNextProcessor(image_processor, tokenizer, **processor_kwargs)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return LlavaNextProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -49,7 +50,8 @@ class LlavaNextProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return LlavaNextProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def prepare_processor_dict(self):
@staticmethod
def prepare_processor_dict():
return {
"chat_template": "{% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>\n' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}",
"patch_size": 3,

17
tests/models/llava_next_video/test_processor_llava_next_video.py
View File

@@ -35,17 +35,18 @@ if is_torch_available:
class LlavaNextVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = LlavaNextVideoProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = LlavaNextImageProcessor()
video_processor = LlavaNextVideoImageProcessor()
tokenizer = LlamaTokenizerFast.from_pretrained("llava-hf/LLaVA-NeXT-Video-7B-hf")
processor_kwargs = self.prepare_processor_dict()
processor_kwargs = cls.prepare_processor_dict()
processor = LlavaNextVideoProcessor(
video_processor=video_processor, image_processor=image_processor, tokenizer=tokenizer, **processor_kwargs
)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -56,7 +57,8 @@ class LlavaNextVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_video_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).video_processor
def prepare_processor_dict(self):
@classmethod
def prepare_processor_dict(cls):
return {
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + ' '}}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>' }}{% endfor %}{# Render all video then #}{% for content in message['content'] | selectattr('type', 'equalto', 'video') %}{{ '<video>' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ '\n' + content['text'] }}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ '\n' + content['text'] }}{% endgeneration %}{% endfor %}{% endif %}{{'<|im_end|>'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"num_additional_image_tokens": 6,
@@ -85,8 +87,9 @@ class LlavaNextVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_dict = self.prepare_processor_dict()
self.assertTrue(processor_loaded.chat_template == processor_dict.get("chat_template", None))
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_chat_template(self):
processor = AutoProcessor.from_pretrained("llava-hf/LLaVA-NeXT-Video-7B-hf")

5
tests/models/llava_onevision/test_processor_llava_onevision.py
View File

@@ -42,7 +42,6 @@ class LlavaOnevisionProcessorTest(ProcessorTesterMixin, unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
cls.addClassCleanup(lambda tempdir=cls.tmpdirname: shutil.rmtree(tempdir))
image_processor = LlavaOnevisionImageProcessor()
video_processor = LlavaOnevisionVideoProcessor()
tokenizer = Qwen2TokenizerFast.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
@@ -62,6 +61,10 @@ class LlavaOnevisionProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_video_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).video_processor
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
@staticmethod
def prepare_processor_dict():
return {

30
tests/models/mllama/test_processor_mllama.py
View File

@@ -36,21 +36,23 @@ if is_vision_available():
class MllamaProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = MllamaProcessor
def setUp(self):
self.checkpoint = "hf-internal-testing/mllama-11b"
processor = MllamaProcessor.from_pretrained(self.checkpoint)
self.image1 = Image.new("RGB", (224, 220))
self.image2 = Image.new("RGB", (512, 128))
self.image_token = processor.image_token
self.image_token_id = processor.image_token_id
self.pad_token_id = processor.tokenizer.pad_token_id
self.bos_token = processor.bos_token
self.bos_token_id = processor.tokenizer.bos_token_id
self.tmpdirname = tempfile.mkdtemp()
processor.save_pretrained(self.tmpdirname)
@classmethod
def setUpClass(cls):
cls.checkpoint = "hf-internal-testing/mllama-11b"
processor = MllamaProcessor.from_pretrained(cls.checkpoint)
cls.image1 = Image.new("RGB", (224, 220))
cls.image2 = Image.new("RGB", (512, 128))
cls.image_token = processor.image_token
cls.image_token_id = processor.image_token_id
cls.pad_token_id = processor.tokenizer.pad_token_id
cls.bos_token = processor.bos_token
cls.bos_token_id = processor.tokenizer.bos_token_id
cls.tmpdirname = tempfile.mkdtemp()
processor.save_pretrained(cls.tmpdirname)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def prepare_processor_dict(self):
return {"chat_template": "{% for message in messages %}{% if loop.index0 == 0 %}{{ bos_token }}{% endif %}{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<|image|>' }}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{ '<|eot_id|>' }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}"} # fmt: skip

5
tests/models/myt5/test_tokenization_myt5.py
View File

@@ -38,8 +38,9 @@ def str_to_hex(line: str, sep: str = " ") -> str:
class TestByteRewriter(unittest.TestCase):
def setUp(self) -> None:
self.tokenizer = MyT5Tokenizer.from_pretrained("Tomlim/myt5-base")
@classmethod
def setUpClass(cls) -> None:
cls.tokenizer = MyT5Tokenizer.from_pretrained("Tomlim/myt5-base")
def test_simple_decompose(self):
decompose_rewriter = self.tokenizer.decompose_rewriter

35
tests/models/omdet_turbo/test_processor_omdet_turbo.py
View File

@@ -44,15 +44,16 @@ class OmDetTurboProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = OmDetTurboProcessor
text_input_name = "classes_input_ids"
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = DetrImageProcessor()
tokenizer = CLIPTokenizerFast.from_pretrained("openai/clip-vit-base-patch32")
processor = OmDetTurboProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
self.input_keys = [
cls.input_keys = [
"tasks_input_ids",
"tasks_attention_mask",
"classes_input_ids",
@@ -62,9 +63,9 @@ class OmDetTurboProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"pixel_mask",
]
self.batch_size = 5
self.num_queries = 5
self.embed_dim = 3
cls.batch_size = 5
cls.num_queries = 5
cls.embed_dim = 3
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -72,8 +73,9 @@ class OmDetTurboProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def get_fake_omdet_turbo_output(self):
classes = self.get_fake_omdet_turbo_classes()
@@ -112,15 +114,16 @@ class OmDetTurboProcessorTest(ProcessorTesterMixin, unittest.TestCase):
torch.testing.assert_close(post_processed[0]["boxes"][0], expected_box_slice, rtol=1e-4, atol=1e-4)
def test_save_load_pretrained_additional_features(self):
processor = OmDetTurboProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = OmDetTurboProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = OmDetTurboProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = OmDetTurboProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, CLIPTokenizerFast)

14
tests/models/owlv2/test_processor_owlv2.py
View File

@@ -14,13 +14,15 @@ from ...test_processing_common import ProcessorTesterMixin
class Owlv2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Owlv2Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
processor = self.processor_class.from_pretrained("google/owlv2-base-patch16-ensemble")
processor.save_pretrained(self.tmpdirname)
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
processor = cls.processor_class.from_pretrained("google/owlv2-base-patch16-ensemble")
processor.save_pretrained(cls.tmpdirname)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_processor_query_images_positional(self):
processor_components = self.prepare_components()

12
tests/models/paligemma/test_processor_paligemma.py
View File

@@ -33,16 +33,18 @@ SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
class PaliGemmaProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = PaliGemmaProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = SiglipImageProcessor.from_pretrained("google/siglip-so400m-patch14-384")
image_processor.image_seq_length = 0
tokenizer = GemmaTokenizer(SAMPLE_VOCAB, keep_accents=True)
processor = PaliGemmaProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
@require_torch
@require_vision

27
tests/models/pix2struct/test_processor_pix2struct.py
View File

@@ -40,15 +40,16 @@ class Pix2StructProcessorTest(ProcessorTesterMixin, unittest.TestCase):
text_input_name = "decoder_input_ids"
images_input_name = "flattened_patches"
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = Pix2StructImageProcessor()
tokenizer = T5Tokenizer.from_pretrained("google-t5/t5-small")
processor = Pix2StructProcessor(image_processor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -56,19 +57,21 @@ class Pix2StructProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_additional_features(self):
processor = Pix2StructProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = Pix2StructProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = Pix2StructProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = Pix2StructProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

51
tests/models/pop2piano/test_processor_pop2piano.py
View File

@@ -62,14 +62,15 @@ if requirements_available:
@require_essentia
@require_pretty_midi
class Pop2PianoProcessorTest(unittest.TestCase):
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
feature_extractor = Pop2PianoFeatureExtractor.from_pretrained("sweetcocoa/pop2piano")
tokenizer = Pop2PianoTokenizer.from_pretrained("sweetcocoa/pop2piano")
processor = Pop2PianoProcessor(feature_extractor, tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return Pop2PianoTokenizer.from_pretrained(self.tmpdirname, **kwargs)
@@ -77,31 +78,33 @@ class Pop2PianoProcessorTest(unittest.TestCase):
def get_feature_extractor(self, **kwargs):
return Pop2PianoFeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_additional_features(self):
processor = Pop2PianoProcessor(
tokenizer=self.get_tokenizer(),
feature_extractor=self.get_feature_extractor(),
)
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = Pop2PianoProcessor(
tokenizer=self.get_tokenizer(),
feature_extractor=self.get_feature_extractor(),
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(
unk_token="-1",
eos_token="1",
pad_token="0",
bos_token="2",
)
feature_extractor_add_kwargs = self.get_feature_extractor()
tokenizer_add_kwargs = self.get_tokenizer(
unk_token="-1",
eos_token="1",
pad_token="0",
bos_token="2",
)
feature_extractor_add_kwargs = self.get_feature_extractor()
processor = Pop2PianoProcessor.from_pretrained(
self.tmpdirname,
unk_token="-1",
eos_token="1",
pad_token="0",
bos_token="2",
)
processor = Pop2PianoProcessor.from_pretrained(
tmpdir,
unk_token="-1",
eos_token="1",
pad_token="0",
bos_token="2",
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, Pop2PianoTokenizer)

12
tests/models/qwen2_5_vl/test_processor_qwen2_5_vl.py
View File

@@ -36,10 +36,11 @@ if is_vision_available():
class Qwen2_5_VLProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Qwen2_5_VLProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
processor = Qwen2_5_VLProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct", patch_size=4)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -50,8 +51,9 @@ class Qwen2_5_VLProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def prepare_processor_dict(self):
return {"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"} # fmt: skip
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()

26
tests/models/qwen2_audio/test_processor_qwen2_audio.py
View File

@@ -32,13 +32,14 @@ if is_torch_available:
class Qwen2AudioProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Qwen2AudioProcessor
def setUp(self):
self.checkpoint = "Qwen/Qwen2-Audio-7B-Instruct"
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.checkpoint = "Qwen/Qwen2-Audio-7B-Instruct"
cls.tmpdirname = tempfile.mkdtemp()
processor_kwargs = self.prepare_processor_dict()
processor = Qwen2AudioProcessor.from_pretrained(self.checkpoint, **processor_kwargs)
processor.save_pretrained(self.tmpdirname)
processor_kwargs = cls.prepare_processor_dict()
processor = Qwen2AudioProcessor.from_pretrained(cls.checkpoint, **processor_kwargs)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -46,10 +47,12 @@ class Qwen2AudioProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_audio_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).audio_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def prepare_processor_dict(self):
@staticmethod
def prepare_processor_dict():
return {
"chat_template": "{% set audio_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if 'audio' in content or 'audio_url' in content or message['type'] == 'audio' %}{% set audio_count.value = audio_count.value + 1 %}Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\n{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
}
@@ -80,8 +83,9 @@ class Qwen2AudioProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = Qwen2AudioProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor.save_pretrained(self.tmpdirname)
processor = Qwen2AudioProcessor.from_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
processor = Qwen2AudioProcessor.from_pretrained(tmpdir)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor.to_json_string())

12
tests/models/qwen2_vl/test_processor_qwen2_vl.py
View File

@@ -36,10 +36,11 @@ if is_vision_available():
class Qwen2VLProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Qwen2VLProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
processor = Qwen2VLProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct", patch_size=4)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer
@@ -50,8 +51,9 @@ class Qwen2VLProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def prepare_processor_dict(self):
return {"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"} # fmt: skip
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()

21
tests/models/sam/test_processor_sam.py
View File

@@ -49,17 +49,19 @@ if is_tf_available():
class SamProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = SamProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = SamImageProcessor()
processor = SamProcessor(image_processor)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def prepare_mask_inputs(self):
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
@@ -85,12 +87,13 @@ class SamProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.skipTest("SamProcessor does not have a tokenizer")
def test_save_load_pretrained_additional_features(self):
processor = SamProcessor(image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = SamProcessor(image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = SamProcessor.from_pretrained(self.tmpdirname, do_normalize=False, padding_value=1.0)
processor = SamProcessor.from_pretrained(tmpdir, do_normalize=False, padding_value=1.0)
self.assertEqual(processor.image_processor.to_json_string(), image_processor_add_kwargs.to_json_string())
self.assertIsInstance(processor.image_processor, SamImageProcessor)

17
tests/models/shieldgemma2/test_processing_shieldgemma2.py
View File

@@ -72,8 +72,9 @@ _SHIELDGEMMA2_POLICIES: Mapping[str, str] = {
class ShieldGemma2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = ShieldGemma2Processor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = Gemma3ImageProcessor.from_pretrained("google/siglip-so400m-patch14-384")
extra_special_tokens = {
@@ -83,14 +84,16 @@ class ShieldGemma2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
}
tokenizer = GemmaTokenizer(SAMPLE_VOCAB, keep_accents=True, extra_special_tokens=extra_special_tokens)
processor_kwargs = self.prepare_processor_dict()
processor_kwargs = cls.prepare_processor_dict()
processor = ShieldGemma2Processor(image_processor=image_processor, tokenizer=tokenizer, **processor_kwargs)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def prepare_processor_dict(self):
@classmethod
def prepare_processor_dict(cls):
return {
"chat_template": _CHAT_TEMPLATE,
"policy_definitions": _SHIELDGEMMA2_POLICIES,

2
tests/models/smolvlm/test_processor_smolvlm.py
View File

@@ -111,7 +111,7 @@ class SmolVLMProcessorTest(ProcessorTesterMixin, unittest.TestCase):
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname)
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_process_interleaved_images_prompts_no_image_splitting(self):
processor_components = self.prepare_components()

37
tests/models/speech_to_text/test_processor_speech_to_text.py
View File

@@ -33,18 +33,19 @@ SAMPLE_SP = get_tests_dir("fixtures/test_sentencepiece.model")
@require_torchaudio
@require_sentencepiece
class Speech2TextProcessorTest(unittest.TestCase):
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
vocab = ["<s>", "<pad>", "</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est"]
vocab_tokens = dict(zip(vocab, range(len(vocab))))
save_dir = Path(self.tmpdirname)
save_dir = Path(cls.tmpdirname)
save_json(vocab_tokens, save_dir / VOCAB_FILES_NAMES["vocab_file"])
if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
copyfile(SAMPLE_SP, save_dir / VOCAB_FILES_NAMES["spm_file"])
tokenizer = Speech2TextTokenizer.from_pretrained(self.tmpdirname)
tokenizer.save_pretrained(self.tmpdirname)
tokenizer = Speech2TextTokenizer.from_pretrained(cls.tmpdirname)
tokenizer.save_pretrained(cls.tmpdirname)
feature_extractor_map = {
"feature_size": 24,
@@ -62,8 +63,9 @@ class Speech2TextProcessorTest(unittest.TestCase):
def get_feature_extractor(self, **kwargs):
return Speech2TextFeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()
@@ -81,17 +83,20 @@ class Speech2TextProcessorTest(unittest.TestCase):
self.assertIsInstance(processor.feature_extractor, Speech2TextFeatureExtractor)
def test_save_load_pretrained_additional_features(self):
processor = Speech2TextProcessor(
tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()
)
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = Speech2TextProcessor(
tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = Speech2TextTokenizer.from_pretrained(tmpdir, bos_token="(BOS)", eos_token="(EOS)")
feature_extractor_add_kwargs = Speech2TextFeatureExtractor.from_pretrained(
tmpdir, do_normalize=False, padding_value=1.0
)
processor = Speech2TextProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = Speech2TextProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, Speech2TextTokenizer)

35
tests/models/speecht5/test_processor_speecht5.py
View File

@@ -36,11 +36,12 @@ SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")
@require_torch
class SpeechT5ProcessorTest(unittest.TestCase):
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
tokenizer = SpeechT5Tokenizer(SAMPLE_VOCAB)
tokenizer.save_pretrained(self.tmpdirname)
tokenizer.save_pretrained(cls.tmpdirname)
feature_extractor_map = {
"feature_size": 1,
@@ -58,8 +59,8 @@ class SpeechT5ProcessorTest(unittest.TestCase):
"return_attention_mask": True,
}
self.feature_extraction_file = os.path.join(self.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(self.feature_extraction_file, "w", encoding="utf-8") as fp:
cls.feature_extraction_file = os.path.join(cls.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(cls.feature_extraction_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(feature_extractor_map) + "\n")
def get_tokenizer(self, **kwargs):
@@ -68,8 +69,9 @@ class SpeechT5ProcessorTest(unittest.TestCase):
def get_feature_extractor(self, **kwargs):
return SpeechT5FeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()
@@ -87,15 +89,20 @@ class SpeechT5ProcessorTest(unittest.TestCase):
self.assertIsInstance(processor.feature_extractor, SpeechT5FeatureExtractor)
def test_save_load_pretrained_additional_features(self):
processor = SpeechT5Processor(tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = SpeechT5Processor(
tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = SpeechT5Tokenizer.from_pretrained(tmpdir, bos_token="(BOS)", eos_token="(EOS)")
feature_extractor_add_kwargs = SpeechT5FeatureExtractor.from_pretrained(
tmpdir, do_normalize=False, padding_value=1.0
)
processor = SpeechT5Processor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = SpeechT5Processor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, SpeechT5Tokenizer)

42
tests/models/trocr/test_processor_trocr.py
View File

@@ -27,21 +27,23 @@ class TrOCRProcessorTest(ProcessorTesterMixin, unittest.TestCase):
text_input_name = "labels"
processor_class = TrOCRProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "want", "##want", "##ed", "wa", "un", "runn", "##ing", ",", "low", "lowest"] # fmt: skip
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(cls.vocab_file, "w", encoding="utf-8") as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
image_processor = ViTImageProcessor.from_pretrained("hf-internal-testing/tiny-random-vit")
tokenizer = XLMRobertaTokenizerFast.from_pretrained("FacebookAI/xlm-roberta-base")
processor = TrOCRProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def get_tokenizer(self, **kwargs):
return XLMRobertaTokenizerFast.from_pretrained(self.tmpdirname, **kwargs)
@@ -50,12 +52,13 @@ class TrOCRProcessorTest(ProcessorTesterMixin, unittest.TestCase):
return ViTImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
def test_save_load_pretrained_default(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor = TrOCRProcessor(image_processor=image_processor, tokenizer=tokenizer)
with tempfile.TemporaryDirectory() as tmpdir:
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor = TrOCRProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor = TrOCRProcessor.from_pretrained(self.tmpdirname)
processor.save_pretrained(tmpdir)
processor = TrOCRProcessor.from_pretrained(tmpdir)
self.assertIsInstance(processor.tokenizer, XLMRobertaTokenizerFast)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
@@ -63,14 +66,15 @@ class TrOCRProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertEqual(processor.image_processor.to_json_string(), image_processor.to_json_string())
def test_save_load_pretrained_additional_features(self):
processor = TrOCRProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(self.tmpdirname)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
with tempfile.TemporaryDirectory() as tmpdir:
processor = TrOCRProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = TrOCRProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = TrOCRProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertIsInstance(processor.tokenizer, XLMRobertaTokenizerFast)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())

73
tests/models/udop/test_processor_udop.py
View File

@@ -55,8 +55,9 @@ class UdopProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = UdopProcessor
maxDiff = None
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
image_processor = LayoutLMv3ImageProcessor(
do_resize=True,
size=224,
@@ -64,38 +65,43 @@ class UdopProcessorTest(ProcessorTesterMixin, unittest.TestCase):
)
tokenizer = UdopTokenizer.from_pretrained("microsoft/udop-large")
processor = UdopProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
self.tokenizer_pretrained_name = "microsoft/udop-large"
cls.tokenizer_pretrained_name = "microsoft/udop-large"
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizers()[0]
image_processor = cls.get_image_processor()
tokenizer = cls.get_tokenizers()[0]
processor = UdopProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs) -> PreTrainedTokenizer:
return self.tokenizer_class.from_pretrained(self.tokenizer_pretrained_name, **kwargs)
@classmethod
def get_tokenizer(cls, **kwargs) -> PreTrainedTokenizer:
return cls.tokenizer_class.from_pretrained(cls.tokenizer_pretrained_name, **kwargs)
def get_image_processor(self, **kwargs):
return LayoutLMv3ImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_image_processor(cls, **kwargs):
return LayoutLMv3ImageProcessor.from_pretrained(cls.tmpdirname, **kwargs)
def get_rust_tokenizer(self, **kwargs) -> PreTrainedTokenizerFast:
return self.rust_tokenizer_class.from_pretrained(self.tokenizer_pretrained_name, **kwargs)
@classmethod
def get_rust_tokenizer(cls, **kwargs) -> PreTrainedTokenizerFast:
return cls.rust_tokenizer_class.from_pretrained(cls.tokenizer_pretrained_name, **kwargs)
def get_tokenizers(self, **kwargs) -> list[PreTrainedTokenizerBase]:
return [self.get_tokenizer(**kwargs), self.get_rust_tokenizer(**kwargs)]
@classmethod
def get_tokenizers(cls, **kwargs) -> list[PreTrainedTokenizerBase]:
return [cls.get_tokenizer(**kwargs), cls.get_rust_tokenizer(**kwargs)]
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
image_processor = self.get_image_processor()
tokenizers = self.get_tokenizers()
for tokenizer in tokenizers:
processor = UdopProcessor(image_processor=image_processor, tokenizer=tokenizer)
processor.save_pretrained(self.tmpdirname)
processor = UdopProcessor.from_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
processor = UdopProcessor.from_pretrained(tmpdir)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, (UdopTokenizer, UdopTokenizerFast))
@@ -104,21 +110,22 @@ class UdopProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertIsInstance(processor.image_processor, LayoutLMv3ImageProcessor)
def test_save_load_pretrained_additional_features(self):
processor = UdopProcessor(image_processor=self.get_image_processor(), tokenizer=self.get_tokenizer())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = UdopProcessor(image_processor=self.get_image_processor(), tokenizer=self.get_tokenizer())
processor.save_pretrained(tmpdir)
# slow tokenizer
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_resize=False, size=30)
# slow tokenizer
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_resize=False, size=30)
processor = UdopProcessor.from_pretrained(
self.tmpdirname,
use_fast=False,
bos_token="(BOS)",
eos_token="(EOS)",
do_resize=False,
size=30,
)
processor = UdopProcessor.from_pretrained(
tmpdir,
use_fast=False,
bos_token="(BOS)",
eos_token="(EOS)",
do_resize=False,
size=30,
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, UdopTokenizer)

61
tests/models/vision_text_dual_encoder/test_processor_vision_text_dual_encoder.py
View File

@@ -35,12 +35,13 @@ if is_vision_available():
class VisionTextDualEncoderProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = VisionTextDualEncoderProcessor
def setUp(self):
self.tmpdirname = tempfile.mkdtemp()
@classmethod
def setUpClass(cls):
cls.tmpdirname = tempfile.mkdtemp()
vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "want", "##want", "##ed", "wa", "un", "runn", "##ing", ",", "low", "lowest"] # fmt: skip
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
with open(cls.vocab_file, "w", encoding="utf-8") as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
image_processor_map = {
@@ -50,34 +51,37 @@ class VisionTextDualEncoderProcessorTest(ProcessorTesterMixin, unittest.TestCase
"image_mean": [0.5, 0.5, 0.5],
"image_std": [0.5, 0.5, 0.5],
}
self.image_processor_file = os.path.join(self.tmpdirname, IMAGE_PROCESSOR_NAME)
with open(self.image_processor_file, "w", encoding="utf-8") as fp:
cls.image_processor_file = os.path.join(cls.tmpdirname, IMAGE_PROCESSOR_NAME)
with open(cls.image_processor_file, "w", encoding="utf-8") as fp:
json.dump(image_processor_map, fp)
tokenizer = self.get_tokenizer()
image_processor = self.get_image_processor()
tokenizer = cls.get_tokenizer()
image_processor = cls.get_image_processor()
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, image_processor=image_processor)
processor.save_pretrained(self.tmpdirname)
processor.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs):
return BertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
@classmethod
def get_tokenizer(cls, **kwargs):
return BertTokenizer.from_pretrained(cls.tmpdirname, **kwargs)
def get_image_processor(self, **kwargs):
@classmethod
def get_image_processor(cls, **kwargs):
if is_torchvision_available():
return ViTImageProcessorFast.from_pretrained(self.tmpdirname, **kwargs)
return ViTImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
return ViTImageProcessorFast.from_pretrained(cls.tmpdirname, **kwargs)
return ViTImageProcessor.from_pretrained(cls.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()
image_processor = self.get_image_processor()
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, image_processor=image_processor)
processor.save_pretrained(self.tmpdirname)
processor = VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
processor = VisionTextDualEncoderProcessor.from_pretrained(tmpdir)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, (BertTokenizer, BertTokenizerFast))
@@ -86,17 +90,18 @@ class VisionTextDualEncoderProcessorTest(ProcessorTesterMixin, unittest.TestCase
self.assertIsInstance(processor.image_processor, (ViTImageProcessor, ViTImageProcessorFast))
def test_save_load_pretrained_additional_features(self):
processor = VisionTextDualEncoderProcessor(
tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor()
)
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = VisionTextDualEncoderProcessor(
tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor()
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = VisionTextDualEncoderProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = VisionTextDualEncoderProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, (BertTokenizer, BertTokenizerFast))

57
tests/models/wav2vec2/test_processor_wav2vec2.py
View File

@@ -31,11 +31,12 @@ class Wav2Vec2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
audio_input_name = "input_values"
text_input_name = "labels"
def setUp(self):
@classmethod
def setUpClass(cls):
vocab = "<pad> <s> </s> <unk> | E T A O N I H S R D L U M W C F G Y P B V K ' X J Q Z".split(" ")
vocab_tokens = dict(zip(vocab, range(len(vocab))))
self.add_kwargs_tokens_map = {
cls.add_kwargs_tokens_map = {
"pad_token": "<pad>",
"unk_token": "<unk>",
"bos_token": "<s>",
@@ -49,28 +50,30 @@ class Wav2Vec2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"do_normalize": True,
}
self.tmpdirname = tempfile.mkdtemp()
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
self.feature_extraction_file = os.path.join(self.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(self.vocab_file, "w", encoding="utf-8") as fp:
cls.tmpdirname = tempfile.mkdtemp()
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
cls.feature_extraction_file = os.path.join(cls.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(cls.vocab_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(vocab_tokens) + "\n")
with open(self.feature_extraction_file, "w", encoding="utf-8") as fp:
with open(cls.feature_extraction_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(feature_extractor_map) + "\n")
tokenizer = self.get_tokenizer()
tokenizer.save_pretrained(self.tmpdirname)
tokenizer = cls.get_tokenizer()
tokenizer.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs_init):
kwargs = self.add_kwargs_tokens_map.copy()
@classmethod
def get_tokenizer(cls, **kwargs_init):
kwargs = cls.add_kwargs_tokens_map.copy()
kwargs.update(kwargs_init)
return Wav2Vec2CTCTokenizer.from_pretrained(self.tmpdirname, **kwargs)
return Wav2Vec2CTCTokenizer.from_pretrained(cls.tmpdirname, **kwargs)
def get_feature_extractor(self, **kwargs):
return Wav2Vec2FeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()
@@ -78,8 +81,9 @@ class Wav2Vec2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = Wav2Vec2Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor.save_pretrained(self.tmpdirname)
processor = Wav2Vec2Processor.from_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
processor = Wav2Vec2Processor.from_pretrained(tmpdir)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, Wav2Vec2CTCTokenizer)
@@ -88,15 +92,22 @@ class Wav2Vec2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertIsInstance(processor.feature_extractor, Wav2Vec2FeatureExtractor)
def test_save_load_pretrained_additional_features(self):
processor = Wav2Vec2Processor(tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor())
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = Wav2Vec2Processor(
tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = Wav2Vec2CTCTokenizer.from_pretrained(
tmpdir, **(self.add_kwargs_tokens_map | {"bos_token": "(BOS)", "eos_token": "(EOS)"})
)
feature_extractor_add_kwargs = Wav2Vec2FeatureExtractor.from_pretrained(
tmpdir, do_normalize=False, padding_value=1.0
)
processor = Wav2Vec2Processor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = Wav2Vec2Processor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, Wav2Vec2CTCTokenizer)

59
tests/models/wav2vec2_bert/test_processor_wav2vec2_bert.py
View File

@@ -32,11 +32,12 @@ class Wav2Vec2BertProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor_class = Wav2Vec2BertProcessor
text_input_name = "labels"
def setUp(self):
@classmethod
def setUpClass(cls):
vocab = "<pad> <s> </s> <unk> | E T A O N I H S R D L U M W C F G Y P B V K ' X J Q Z".split(" ")
vocab_tokens = dict(zip(vocab, range(len(vocab))))
self.add_kwargs_tokens_map = {
cls.add_kwargs_tokens_map = {
"pad_token": "<pad>",
"unk_token": "<unk>",
"bos_token": "<s>",
@@ -50,28 +51,30 @@ class Wav2Vec2BertProcessorTest(ProcessorTesterMixin, unittest.TestCase):
"do_normalize": True,
}
self.tmpdirname = tempfile.mkdtemp()
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
self.feature_extraction_file = os.path.join(self.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(self.vocab_file, "w", encoding="utf-8") as fp:
cls.tmpdirname = tempfile.mkdtemp()
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
cls.feature_extraction_file = os.path.join(cls.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(cls.vocab_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(vocab_tokens) + "\n")
with open(self.feature_extraction_file, "w", encoding="utf-8") as fp:
with open(cls.feature_extraction_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(feature_extractor_map) + "\n")
tokenizer = self.get_tokenizer()
tokenizer.save_pretrained(self.tmpdirname)
tokenizer = cls.get_tokenizer()
tokenizer.save_pretrained(cls.tmpdirname)
def get_tokenizer(self, **kwargs_init):
kwargs = self.add_kwargs_tokens_map.copy()
@classmethod
def get_tokenizer(cls, **kwargs_init):
kwargs = cls.add_kwargs_tokens_map.copy()
kwargs.update(kwargs_init)
return Wav2Vec2CTCTokenizer.from_pretrained(self.tmpdirname, **kwargs)
return Wav2Vec2CTCTokenizer.from_pretrained(cls.tmpdirname, **kwargs)
def get_feature_extractor(self, **kwargs):
return SeamlessM4TFeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()
@@ -79,8 +82,9 @@ class Wav2Vec2BertProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = Wav2Vec2BertProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor.save_pretrained(self.tmpdirname)
processor = Wav2Vec2BertProcessor.from_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor.save_pretrained(tmpdir)
processor = Wav2Vec2BertProcessor.from_pretrained(tmpdir)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, Wav2Vec2CTCTokenizer)
@@ -89,17 +93,22 @@ class Wav2Vec2BertProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertIsInstance(processor.feature_extractor, SeamlessM4TFeatureExtractor)
def test_save_load_pretrained_additional_features(self):
processor = Wav2Vec2BertProcessor(
tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()
)
processor.save_pretrained(self.tmpdirname)
with tempfile.TemporaryDirectory() as tmpdir:
processor = Wav2Vec2BertProcessor(
tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()
)
processor.save_pretrained(tmpdir)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0)
tokenizer_add_kwargs = Wav2Vec2CTCTokenizer.from_pretrained(
tmpdir, **(self.add_kwargs_tokens_map | {"bos_token": "(BOS)", "eos_token": "(EOS)"})
)
feature_extractor_add_kwargs = SeamlessM4TFeatureExtractor.from_pretrained(
tmpdir, do_normalize=False, padding_value=1.0
)
processor = Wav2Vec2BertProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
processor = Wav2Vec2BertProcessor.from_pretrained(
tmpdir, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, Wav2Vec2CTCTokenizer)

2
tests/test_tokenization_common.py
View File

@@ -259,7 +259,7 @@ class TokenizerTesterMixin:
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname)
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
def get_input_output_texts(self, tokenizer):
input_txt = self.get_clean_sequence(tokenizer)[0]