SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

VLMs: patch_size -> num_image_tokens in processing (#33424)

Browse Source 
* use num additional tokens

* fix copies + docs

* another fix copies :)

* add docs

* move order for BC
This commit is contained in:
Raushan Turganbay
2024-11-18 13:21:07 +01:00
committed by GitHub
parent 3ee24e2208
commit 1646ffb4d1
17 changed files with 131 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/models/vipllava/test_modeling_vipllava.py
View File

@@ -374,12 +374,14 @@ class VipLlavaForConditionalGenerationIntegrationTest(unittest.TestCase):
# check processing with expansion of inputs
processor.vision_feature_select_strategy = "default"
processor.patch_size = 14
processor.num_additional_image_tokens = 1
inputs_expanded = processor(prompt, raw_image, return_tensors="pt").to(torch_device, torch.float16)
self.assertTrue(inputs_expanded.input_ids.shape[-1] == 593)
# check processing without expansion of inputs (legacy behavior)
processor.vision_feature_select_strategy = None
processor.patch_size = None
processor.num_additional_image_tokens = None
inputs = processor(prompt, raw_image, return_tensors="pt").to(torch_device, torch.float16)
self.assertTrue(inputs.input_ids.shape[-1] == 18)