SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

VLMs: enable generation tests (#33533)

Browse Source 
* add tests

* fix whisper

* update

* nit

* add qwen2-vl

* more updates!

* better this way

* fix this one

* fix more tests

* fix final tests, hope so

* fix led

* Update tests/generation/test_utils.py

Co-authored-by: Joao Gante <joaofranciscocardosogante@gmail.com>

* pr comments

* not pass pixels and extra for low-mem tests, very flaky because of visio tower

---------

Co-authored-by: Joao Gante <joaofranciscocardosogante@gmail.com>
This commit is contained in:
Raushan Turganbay
2024-09-19 12:04:24 +02:00
committed by GitHub
parent e40bb4845e
commit d7975a5874
22 changed files with 500 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tests/models/llava_onevision/test_modeling_llava_onevision.py
View File

@@ -60,7 +60,7 @@ class LlavaOnevisionVisionText2TextModelTester:
self,
parent,
ignore_index=-100,
image_token_index=0,
image_token_index=1,
projector_hidden_act="gelu",
seq_length=7,
vision_feature_select_strategy="full",
@@ -92,7 +92,7 @@ class LlavaOnevisionVisionText2TextModelTester:
is_training=True,
vision_config={
"image_size": 16,
"patch_size": 2,
"patch_size": 8,
"num_channels": 3,
"is_training": True,
"hidden_size": 32,
@@ -113,7 +113,9 @@ class LlavaOnevisionVisionText2TextModelTester:
self.vision_feature_layer = vision_feature_layer
self.text_config = text_config
self.vision_config = vision_config
self.seq_length = seq_length
self.pad_token_id = text_config["pad_token_id"]
self.num_image_tokens = 10
self.seq_length = seq_length + self.num_image_tokens
self.num_hidden_layers = text_config["num_hidden_layers"]
self.vocab_size = text_config["vocab_size"]
@@ -124,8 +126,7 @@ class LlavaOnevisionVisionText2TextModelTester:
self.batch_size = 3
self.num_channels = 3
self.image_size = 30
self.encoder_seq_length = 7
self.image_grid_pinpoints = [[32, 32]]
self.image_grid_pinpoints = [[16, 16]]
def get_config(self):
return LlavaOnevisionConfig(
@@ -143,7 +144,7 @@ class LlavaOnevisionVisionText2TextModelTester:
pixel_values = floats_tensor(
[
self.batch_size,
9,
3,
self.vision_config["num_channels"],
self.vision_config["image_size"],
self.vision_config["image_size"],
@@ -158,16 +159,16 @@ class LlavaOnevisionVisionText2TextModelTester:
config, pixel_values = config_and_inputs
input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 2) + 2
attention_mask = torch.ones(input_ids.shape, dtype=torch.long).to(torch_device)
# we are giving 3 images let's make sure we pass in 3 image tokens
input_ids[:, 1] = config.image_token_index
input_ids[input_ids == config.image_token_index] = self.pad_token_id
input_ids[:, : self.num_image_tokens] = config.image_token_index
labels = torch.zeros((self.batch_size, self.seq_length), dtype=torch.long, device=torch_device)
# maskout where the image token is
labels[:, 1] == self.ignore_index
labels[:, : self.num_image_tokens] == self.ignore_index
inputs_dict = {
"pixel_values": pixel_values,
"image_sizes": torch.tensor(
[[self.vision_config["image_size"], self.vision_config["image_size"]]] * self.batch_size
),
"image_sizes": torch.tensor([[45, 45]] * self.batch_size),
"input_ids": input_ids,
"attention_mask": attention_mask,
"labels": labels,