SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix padding for IDEFICS (#26396)

Browse Source 
* fix

* fixup

* tests

* fixup
This commit is contained in:
Shauray Singh
2023-09-27 14:26:07 +05:30
committed by GitHub
parent 408b2b3c50
commit abd2531034
2 changed files with 30 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tests/models/idefics/test_processor_idefics.py
View File

@@ -141,6 +141,25 @@ class IdeficsProcessorTest(TestCasePlus):
self.assertListEqual(decoded_tok, decoded_processor)
def test_tokenizer_padding(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer(padding_side="right")
processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor)
predicted_tokens = [
"<s>Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk>",
"<s>Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk>",
]
prompts = [[prompt] for prompt in self.prepare_prompts()[2]]
max_length = processor(prompts, padding="max_length", truncation=True, max_length=20)
longest = processor(prompts, padding="longest", truncation=True, max_length=30)
decoded_max_length = processor.tokenizer.decode(max_length["input_ids"][-1])
decoded_longest = processor.tokenizer.decode(longest["input_ids"][-1])
self.assertEqual(decoded_max_length, predicted_tokens[1])
self.assertEqual(decoded_longest, predicted_tokens[0])
def test_model_input_names(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()