SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

BLIPs clean-up (#35560)

Browse Source 
* blips clean up

* update processor

* readability

* fix processor length

* fix copies

* tmp

* update and fix copies

* why keep these, delete?

* fix test fetcher

* irrelevant comment

* fix tests

* fix tests

* fix copies
This commit is contained in:
Raushan Turganbay
2025-07-29 10:03:06 +02:00
committed by GitHub
parent 4f8f51be4e
commit 75794792ad
15 changed files with 301 additions and 592 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
src/transformers/models/blip_2/modeling_blip_2.py
View File

@@ -1539,16 +1539,25 @@ class Blip2Model(Blip2PreTrainedModel):
# step 3: use the language model, conditioned on the query outputs and the prompt
language_model_inputs = self.language_projection(query_output)
language_model_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
inputs_embeds = self.language_model.get_input_embeddings()(input_ids)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds], dim=1)
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
expected_device = language_model_attention_mask.device
attention_mask = torch.cat([language_model_attention_mask, attention_mask.to(expected_device)], dim=1)
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
)
special_image_mask = special_image_mask.all(-1)
else:
special_image_mask = input_ids == self.config.image_token_id
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(language_model_inputs.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.to(language_model_inputs.device).masked_scatter(
special_image_mask, language_model_inputs
)
if self.config.use_decoder_only_language_model:
outputs = self.language_model(
@@ -2026,9 +2035,6 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin):
)
vision_outputs = vision_outputs.to_tuple() if not return_dict else vision_outputs
query_outputs = query_outputs.to_tuple() if not return_dict else query_outputs
language_model_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings()(input_ids)
@@ -2036,9 +2042,6 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin):
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "image_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "image_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -2047,23 +2050,11 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin):
else:
special_image_mask = input_ids == self.config.image_token_id
special_image_mask = (
special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(language_model_inputs.device)
)
language_model_inputs = language_model_inputs.to(inputs_embeds.dtype)
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(language_model_inputs.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.to(language_model_inputs.device).masked_scatter(
special_image_mask, language_model_inputs
)
else:
logger.warning_once(
"Expanding inputs for image tokens in BLIP-2 should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/e9f20b054fa322f84ac9311d9ab67042) to update your BLIP-2 model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.50."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_model_attention_mask, attention_mask.to(language_model_attention_mask.device)], dim=1
)
if self.config.use_decoder_only_language_model:
outputs = self.language_model(
@@ -2172,15 +2163,11 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin):
query_output = query_output.to(image_embeds.dtype)
language_model_inputs = self.language_projection(query_output)
language_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
if input_ids is None:
start_tokens = [self.config.text_config.bos_token_id]
if getattr(self.config, "image_token_id", None) is not None:
start_tokens = [self.config.image_token_id] * self.config.num_query_tokens + start_tokens
image_tokens = [self.config.image_token_index] * self.config.num_query_tokens
start_tokens = image_tokens + [self.config.text_config.bos_token_id]
input_ids = torch.tensor([start_tokens], dtype=torch.long, device=image_embeds.device)
input_ids = input_ids.repeat(batch_size, 1)
inputs_embeds = self.get_input_embeddings()(input_ids)
@@ -2188,9 +2175,6 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin):
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "image_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "image_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -2199,42 +2183,16 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin):
else:
special_image_mask = input_ids == self.config.image_token_id
special_image_mask = (
special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(language_model_inputs.device)
)
language_model_inputs = language_model_inputs.to(inputs_embeds.dtype)
inputs_embeds = inputs_embeds.to(language_model_inputs.device).masked_scatter(
special_image_mask, language_model_inputs
)
attention_mask = attention_mask.to(language_attention_mask.device)
else:
logger.warning_once(
"Expanding inputs for image tokens in BLIP-2 should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/e9f20b054fa322f84ac9311d9ab67042) to update your BLIP-2 model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.50."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_attention_mask, attention_mask.to(language_attention_mask.device)], dim=1
)
# add image_embeds length to max_length, so that the final max_length in counted only on token embeds
# -1 is to account for the prepended BOS after `generate.`
# TODO (joao, raushan): refactor `generate` to avoid these operations with VLMs
if not self.language_model.config.is_encoder_decoder:
generate_kwargs["max_length"] = (
generate_kwargs.get("max_length", 20) + language_model_inputs.shape[1] - 1
)
generate_kwargs["min_length"] = generate_kwargs.get("min_length", 0) + language_model_inputs.shape[1]
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
inputs = {"inputs_embeds": inputs_embeds, "attention_mask": attention_mask}
if not self.language_model.config.is_encoder_decoder:
if input_ids is not None:
input_ids = input_ids.to(language_model_inputs.device)
inputs["input_ids"] = input_ids
outputs = self.language_model.generate(**inputs, **generate_kwargs)
return outputs
@@ -2362,7 +2320,12 @@ class Blip2ForImageTextRetrieval(Blip2PreTrainedModel):
if use_image_text_matching_head:
query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
query_attention_mask = torch.ones(query_tokens.size()[:-1], dtype=torch.long, device=query_tokens.device)
if self.config.image_token_index is not None:
input_ids = input_ids[:, self.config.num_query_tokens :]
else:
query_attention_mask = torch.ones(
query_tokens.size()[:-1], dtype=torch.long, device=query_tokens.device
)
attention_mask = torch.cat([query_attention_mask, attention_mask], dim=1)
query_embeds = self.embeddings(
@@ -2395,6 +2358,10 @@ class Blip2ForImageTextRetrieval(Blip2PreTrainedModel):
image_embeds = query_outputs[0] if not return_dict else query_outputs.last_hidden_state
image_embeds = image_embeds.to(dtype=self.vision_projection.weight.dtype)
if self.config.image_token_index is not None:
input_ids = input_ids[:, self.config.num_query_tokens :]
attention_mask = attention_mask[:, self.config.num_query_tokens :]
query_embeds = self.embeddings(
input_ids=input_ids,
)

52
src/transformers/models/blip_2/processing_blip_2.py
View File

@@ -112,11 +112,13 @@ class Blip2Processor(ProcessorMixin):
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
# BC for explicit return_tensors
if "return_tensors" in output_kwargs["common_kwargs"]:
return_tensors = output_kwargs["common_kwargs"].pop("return_tensors", None)
else:
return_tensors = None
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
max_length = output_kwargs["text_kwargs"].pop("max_length", None)
if max_length is not None:
output_kwargs["text_kwargs"]["max_length"] = max_length - self.num_query_tokens
encoding = BatchFeature(tensor_type=return_tensors)
if text is not None:
if isinstance(text, str):
@@ -124,40 +126,28 @@ class Blip2Processor(ProcessorMixin):
elif not isinstance(text, list) and not isinstance(text[0], str):
raise ValueError("Invalid input text. Please provide a string, or a list of strings")
text_encoding = {}
# We need this hacky manipulation because BLIP expects image tokens to be at the beginning even before BOS token
text_encoding = self.tokenizer(text, **output_kwargs["text_kwargs"])
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
_text_encoding = self.tokenizer(text, **output_kwargs["text_kwargs"], return_tensors=None)
output_kwargs["text_kwargs"]["return_tensors"] = return_tensors
# if we know how many query tokens, expand text inside processor. We need this hacky manipulation
# because BLIP expects image tokens to be at the beginning even before BOS token
if self.num_query_tokens is not None:
if images is not None and self.num_query_tokens is not None:
# Image tokens should not be padded/truncated or prepended with special BOS token
image_tokens = self.image_token.content * self.num_query_tokens
image_token_encoding = self.tokenizer(
[image_tokens] * len(text), add_special_tokens=False, return_tensors=None
)
for k in _text_encoding:
text_encoding[k] = [
img_encoding + txt_encoding
for img_encoding, txt_encoding in zip(image_token_encoding[k], _text_encoding[k])
]
else:
text_encoding = _text_encoding
logger.warning_once(
"Expanding inputs for image tokens in BLIP-2 should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/e9f20b054fa322f84ac9311d9ab67042) to update your BLIP-2 model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.50."
)
output_kwargs["text_kwargs"]["add_special_tokens"] = False
output_kwargs["text_kwargs"]["padding"] = False
output_kwargs["text_kwargs"]["truncation"] = False
image_text_encoding = self.tokenizer(image_tokens, **output_kwargs["text_kwargs"])
for k in text_encoding:
text_encoding[k] = [image_text_encoding[k] + sample for sample in text_encoding[k]]
encoding.update(text_encoding)
# cast to desired return tensors type
encoding.update(BatchEncoding(text_encoding, tensor_type=return_tensors))
# add pixel_values encoding. If we also have text_encoding, update image encoding and return it.
# Now add pixel_values encoding. If we also have text_encoding, update image encoding and return it.
# else, return the text encoding.
if images is not None:
image_encoding = self.image_processor(images, **output_kwargs["images_kwargs"])
encoding.update(image_encoding)
# Cast to desired return tensors type
encoding = BatchFeature(encoding, tensor_type=return_tensors)
return encoding
# Copied from transformers.models.blip.processing_blip.BlipProcessor.batch_decode with BertTokenizerFast->PreTrainedTokenizer

49
src/transformers/models/instructblip/modeling_instructblip.py
View File

@@ -799,7 +799,7 @@ class InstructBlipQFormerLayer(GradientCheckpointingLayer):
self.chunk_size_feed_forward,
self.seq_len_dim,
attention_output[:, query_length:, :],
)
).to(layer_output.device)
layer_output = torch.cat([layer_output, layer_output_text], dim=1)
else:
layer_output = apply_chunking_to_forward(
@@ -1560,9 +1560,6 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, Generati
)
vision_outputs = vision_outputs.to_tuple() if not return_dict else vision_outputs
query_outputs = query_outputs.to_tuple() if not return_dict else query_outputs
language_model_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings()(input_ids)
@@ -1570,9 +1567,6 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, Generati
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "image_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "image_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -1584,16 +1578,6 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, Generati
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
else:
logger.warning_once(
"Expanding inputs for image tokens in InstructBLIP should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/e9f20b054fa322f84ac9311d9ab67042) to update your InstructBLIP model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.50."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_model_attention_mask, attention_mask.to(language_model_attention_mask.device)], dim=1
)
if self.config.use_decoder_only_language_model:
outputs = self.language_model(
@@ -1682,25 +1666,18 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, Generati
interpolate_pos_encoding=interpolate_pos_encoding,
return_dict=True,
)
language_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
if input_ids is None:
start_tokens = [self.config.text_config.bos_token_id]
if getattr(self.config, "image_token_id", None) is not None:
start_tokens = [self.config.image_token_id] * self.config.num_query_tokens + start_tokens
input_ids = torch.tensor([start_tokens], dtype=torch.long, device=language_model_inputs.device)
image_tokens = [self.config.image_token_index] * self.config.num_query_tokens
start_tokens = image_tokens + [self.config.text_config.bos_token_id]
input_ids = torch.tensor([start_tokens], dtype=torch.long, device=pixel_values.device)
input_ids = input_ids.repeat(batch_size, 1)
inputs_embeds = self.get_input_embeddings()(input_ids)
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "image_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "image_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -1712,24 +1689,6 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, Generati
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
else:
logger.warning_once(
"Expanding inputs for image tokens in InstructBLIP should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/e9f20b054fa322f84ac9311d9ab67042) to update your InstructBLIP model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.50."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_attention_mask, attention_mask.to(language_attention_mask.device)], dim=1
)
# add image_embeds length to max_length, so that the final max_length in counted only on token embeds
# -1 is to account for the prepended BOS after `generate.`
if not self.language_model.config.is_encoder_decoder:
generate_kwargs["max_length"] = (
generate_kwargs.get("max_length", 20) + language_model_inputs.shape[1] - 1
)
generate_kwargs["min_length"] = generate_kwargs.get("min_length", 0) + language_model_inputs.shape[1]
inputs = {"inputs_embeds": inputs_embeds, "attention_mask": attention_mask}
if not self.language_model.config.is_encoder_decoder:

55
src/transformers/models/instructblip/processing_instructblip.py
View File

@@ -22,7 +22,7 @@ from typing import Union
from ...image_processing_utils import BatchFeature
from ...image_utils import ImageInput
from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
from ...tokenization_utils_base import AddedToken, BatchEncoding, PreTokenizedInput, TextInput
from ...tokenization_utils_base import AddedToken, PreTokenizedInput, TextInput
from ...utils import logging
from ..auto import AutoTokenizer
@@ -78,6 +78,7 @@ class InstructBlipProcessor(ProcessorMixin):
else:
self.image_token = tokenizer.image_token
self.num_query_tokens = num_query_tokens
super().__init__(image_processor, tokenizer, qformer_tokenizer)
def __call__(
@@ -111,52 +112,40 @@ class InstructBlipProcessor(ProcessorMixin):
**kwargs,
)
encoding = BatchFeature()
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
encoding = {}
if text is not None:
if isinstance(text, str):
text = [text]
elif not isinstance(text, list) and not isinstance(text[0], str):
raise ValueError("Invalid input text. Please provide a string, or a list of strings")
# we have to concatenate lists - so we keep track of return_tensors here
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
_text_encoding = self.tokenizer(text, **output_kwargs["text_kwargs"], return_tensors=None)
output_kwargs["text_kwargs"]["return_tensors"] = return_tensors
# if we know how many query tokens, expand text inside processor. We need this hacky manipulation
# because BLIP expects image tokens to be at the beginning even before BOS token
if self.num_query_tokens is not None and images is not None:
text_encoding = {}
image_tokens = self.image_token.content * self.num_query_tokens
image_token_encoding = self.tokenizer(
[image_tokens] * len(text), add_special_tokens=False, return_tensors=None
)
for k in _text_encoding:
text_encoding[k] = [
img_encoding + txt_encoding
for img_encoding, txt_encoding in zip(image_token_encoding[k], _text_encoding[k])
]
else:
text_encoding = _text_encoding
if images is not None:
logger.warning_once(
"Expanding inputs for image tokens in InstructBLIP should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/e9f20b054fa322f84ac9311d9ab67042) to update your InstructBLIP model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.50."
)
# cast to desired return tensors type after concatenating
text_encoding = BatchEncoding(text_encoding, tensor_type=return_tensors)
encoding.update(text_encoding)
qformer_text_encoding = self.qformer_tokenizer(text, **output_kwargs["text_kwargs"])
encoding["qformer_input_ids"] = qformer_text_encoding.pop("input_ids")
encoding["qformer_attention_mask"] = qformer_text_encoding.pop("attention_mask")
# We need this hacky manipulation because BLIP expects image tokens to be at the beginning even before BOS token
if output_kwargs["text_kwargs"].get("max_length") is not None:
output_kwargs["text_kwargs"]["max_length"] -= self.num_query_tokens
text_encoding = self.tokenizer(text, **output_kwargs["text_kwargs"])
if images is not None:
# Image tokens should not be padded/truncated or prepended with special BOS token
image_tokens = self.image_token.content * self.num_query_tokens
output_kwargs["text_kwargs"]["add_special_tokens"] = False
output_kwargs["text_kwargs"]["padding"] = False
output_kwargs["text_kwargs"]["truncation"] = False
image_text_encoding = self.tokenizer(image_tokens, **output_kwargs["text_kwargs"])
for k in text_encoding:
text_encoding[k] = [image_text_encoding[k] + sample for sample in text_encoding[k]]
encoding.update(text_encoding)
if images is not None:
image_encoding = self.image_processor(images, **output_kwargs["images_kwargs"])
encoding.update(image_encoding)
# Cast to desired return tensors type
encoding = BatchFeature(encoding, tensor_type=return_tensors)
return encoding
# Copied from transformers.models.blip.processing_blip.BlipProcessor.batch_decode with BertTokenizerFast->PreTrainedTokenizer

50
src/transformers/models/instructblipvideo/modeling_instructblipvideo.py
View File

@@ -660,7 +660,7 @@ class InstructBlipVideoQFormerLayer(GradientCheckpointingLayer):
self.chunk_size_feed_forward,
self.seq_len_dim,
attention_output[:, query_length:, :],
)
).to(layer_output.device)
layer_output = torch.cat([layer_output, layer_output_text], dim=1)
else:
layer_output = apply_chunking_to_forward(
@@ -1527,9 +1527,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel
)
vision_outputs = vision_outputs.to_tuple() if not return_dict else vision_outputs
query_outputs = query_outputs.to_tuple() if not return_dict else query_outputs
language_model_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings()(input_ids)
@@ -1537,9 +1534,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "video_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "video_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -1551,16 +1545,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
else:
logger.warning_once(
"Expanding inputs for video tokens in InstructBLIPVideo should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/65f22892b054dc0d68228af56fbeaac2) to update your InstructBLIPVideo model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.54."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_model_attention_mask, attention_mask.to(language_model_attention_mask.device)], dim=1
)
if self.config.use_decoder_only_language_model:
outputs = self.language_model(
@@ -1650,25 +1634,17 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel
return_dict=True,
)
language_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
if input_ids is None:
start_tokens = [self.config.text_config.bos_token_id]
if getattr(self.config, "video_token_id", None) is not None:
start_tokens = [self.config.video_token_id] * self.config.num_query_tokens * 4 + start_tokens
input_ids = torch.tensor([start_tokens], dtype=torch.long, device=language_model_inputs.device)
video_tokens = [self.config.video_token_index] * self.config.num_query_tokens * 4
start_tokens = video_tokens + [self.config.text_config.bos_token_id]
input_ids = torch.tensor([start_tokens], dtype=torch.long, device=pixel_values.device)
input_ids = input_ids.repeat(batch_size, 1)
inputs_embeds = self.get_input_embeddings()(input_ids)
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "video_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "video_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -1680,24 +1656,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
else:
logger.warning_once(
"Expanding inputs for video tokens in InstructBLIPVideo should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/65f22892b054dc0d68228af56fbeaac2) to update your InstructBLIPVideo model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.54."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_attention_mask, attention_mask.to(language_attention_mask.device)], dim=1
)
# add image_embeds length to max_length, so that the final max_length in counted only on token embeds
# -1 is to account for the prepended BOS after `generate.`
if not self.language_model.config.is_encoder_decoder:
generate_kwargs["max_length"] = (
generate_kwargs.get("max_length", 20) + language_model_inputs.shape[1] - 1
)
generate_kwargs["min_length"] = generate_kwargs.get("min_length", 0) + language_model_inputs.shape[1]
inputs = {"inputs_embeds": inputs_embeds, "attention_mask": attention_mask}
if not self.language_model.config.is_encoder_decoder:

48
src/transformers/models/instructblipvideo/modular_instructblipvideo.py
View File

@@ -464,9 +464,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGenera
)
vision_outputs = vision_outputs.to_tuple() if not return_dict else vision_outputs
query_outputs = query_outputs.to_tuple() if not return_dict else query_outputs
language_model_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings()(input_ids)
@@ -474,9 +471,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGenera
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "video_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "video_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -488,16 +482,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGenera
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
else:
logger.warning_once(
"Expanding inputs for video tokens in InstructBLIPVideo should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/65f22892b054dc0d68228af56fbeaac2) to update your InstructBLIPVideo model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.54."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_model_attention_mask, attention_mask.to(language_model_attention_mask.device)], dim=1
)
if self.config.use_decoder_only_language_model:
outputs = self.language_model(
@@ -587,25 +571,17 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGenera
return_dict=True,
)
language_attention_mask = torch.ones(
language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
)
if inputs_embeds is None:
if input_ids is None:
start_tokens = [self.config.text_config.bos_token_id]
if getattr(self.config, "video_token_id", None) is not None:
start_tokens = [self.config.video_token_id] * self.config.num_query_tokens * 4 + start_tokens
input_ids = torch.tensor([start_tokens], dtype=torch.long, device=language_model_inputs.device)
video_tokens = [self.config.video_token_index] * self.config.num_query_tokens * 4
start_tokens = video_tokens + [self.config.text_config.bos_token_id]
input_ids = torch.tensor([start_tokens], dtype=torch.long, device=pixel_values.device)
input_ids = input_ids.repeat(batch_size, 1)
inputs_embeds = self.get_input_embeddings()(input_ids)
if attention_mask is None:
attention_mask = torch.ones_like(input_ids)
# if the model already has "video_token_id" then the input is expanded to account for image embeds
# otherwise we expand manually by concatenating
if getattr(self.config, "video_token_id", None) is not None:
if input_ids is None:
special_image_mask = inputs_embeds == self.get_input_embeddings()(
torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)
@@ -617,24 +593,6 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGenera
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
else:
logger.warning_once(
"Expanding inputs for video tokens in InstructBLIPVideo should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/65f22892b054dc0d68228af56fbeaac2) to update your InstructBLIPVideo model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.54."
)
inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
attention_mask = torch.cat(
[language_attention_mask, attention_mask.to(language_attention_mask.device)], dim=1
)
# add image_embeds length to max_length, so that the final max_length in counted only on token embeds
# -1 is to account for the prepended BOS after `generate.`
if not self.language_model.config.is_encoder_decoder:
generate_kwargs["max_length"] = (
generate_kwargs.get("max_length", 20) + language_model_inputs.shape[1] - 1
)
generate_kwargs["min_length"] = generate_kwargs.get("min_length", 0) + language_model_inputs.shape[1]
inputs = {"inputs_embeds": inputs_embeds, "attention_mask": attention_mask}
if not self.language_model.config.is_encoder_decoder:

91
src/transformers/models/instructblipvideo/processing_instructblipvideo.py
View File

@@ -23,7 +23,6 @@ from ...image_processing_utils import BatchFeature
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import (
AddedToken,
BatchEncoding,
PaddingStrategy,
PreTokenizedInput,
TextInput,
@@ -99,60 +98,13 @@ class InstructBlipVideoProcessor(ProcessorMixin):
if images is None and text is None:
raise ValueError("You have to specify at least one of images or text.")
encoding = BatchFeature()
encoding = {}
if text is not None:
if isinstance(text, str):
text = [text]
elif not isinstance(text, list) and not isinstance(text[0], str):
raise ValueError("Invalid input text. Please provide a string, or a list of strings")
_text_encoding = self.tokenizer(
text=text,
add_special_tokens=add_special_tokens,
padding=padding,
truncation=truncation,
max_length=max_length,
stride=stride,
pad_to_multiple_of=pad_to_multiple_of,
return_attention_mask=return_attention_mask,
return_overflowing_tokens=return_overflowing_tokens,
return_special_tokens_mask=return_special_tokens_mask,
return_offsets_mapping=return_offsets_mapping,
return_token_type_ids=return_token_type_ids,
return_length=return_length,
verbose=verbose,
return_tensors=None, # required to concatenate below
**kwargs,
)
# if we know how many query tokens, expand text inside processor. We need this hacky manipulation
# because BLIP expects image tokens to be at the beginning even before BOS token
if self.num_query_tokens is not None and images is not None:
text_encoding = {}
video_tokens = (
self.video_token.content * self.num_query_tokens * 4
) # InstrucBLIP works with 4 frames only
video_token_encoding = self.tokenizer(
[video_tokens] * len(text), add_special_tokens=False, return_tensors=None
)
for k in _text_encoding:
text_encoding[k] = [
img_encoding + txt_encoding
for img_encoding, txt_encoding in zip(video_token_encoding[k], _text_encoding[k])
]
else:
text_encoding = _text_encoding
if images is not None:
logger.warning_once(
"Expanding inputs for video tokens in InstructBLIPVideo should be done in processing. "
"Please follow instruction here (https://gist.github.com/zucchini-nlp/65f22892b054dc0d68228af56fbeaac2) to update your InstructBLIPVideo model. "
"Using processors without these attributes in the config is deprecated and will throw an error in v4.54."
)
# cast to desired return tensors type after concatenating
text_encoding = BatchEncoding(text_encoding, tensor_type=return_tensors)
encoding.update(text_encoding)
qformer_text_encoding = self.qformer_tokenizer(
text=text,
add_special_tokens=add_special_tokens,
@@ -174,10 +126,51 @@ class InstructBlipVideoProcessor(ProcessorMixin):
encoding["qformer_input_ids"] = qformer_text_encoding.pop("input_ids")
encoding["qformer_attention_mask"] = qformer_text_encoding.pop("attention_mask")
# We need this hacky manipulation because BLIP expects image tokens to be at the beginning even before BOS token
# InstrucBLIP works with 4 frames only
if max_length is not None:
max_length -= self.num_query_tokens
text_encoding = self.tokenizer(
text=text,
add_special_tokens=add_special_tokens,
padding=padding,
truncation=truncation,
max_length=max_length,
stride=stride,
pad_to_multiple_of=pad_to_multiple_of,
return_attention_mask=return_attention_mask,
return_overflowing_tokens=return_overflowing_tokens,
return_special_tokens_mask=return_special_tokens_mask,
return_offsets_mapping=return_offsets_mapping,
return_token_type_ids=return_token_type_ids,
return_length=return_length,
verbose=verbose,
return_tensors=None, # required to concatenate below
**kwargs,
)
if images is not None:
video_tokens = self.video_token.content * self.num_query_tokens * 4
video_text_encoding = self.tokenizer(
video_tokens,
add_special_tokens=False, # required to concatenate below
return_attention_mask=return_attention_mask,
return_overflowing_tokens=return_overflowing_tokens,
return_special_tokens_mask=return_special_tokens_mask,
return_offsets_mapping=return_offsets_mapping,
return_token_type_ids=return_token_type_ids,
return_length=return_length,
return_tensors=None,
)
for k in text_encoding:
text_encoding[k] = [video_text_encoding[k] + sample for sample in text_encoding[k]]
encoding.update(text_encoding)
if images is not None:
image_encoding = self.video_processor(images, return_tensors=return_tensors)
encoding.update(image_encoding)
encoding = BatchFeature(encoding, tensor_type=return_tensors)
return encoding
# Copied from transformers.models.blip.processing_blip.BlipProcessor.batch_decode with BertTokenizerFast->PreTrainedTokenizer

48
tests/models/blip_2/test_modeling_blip_2.py
View File

@@ -29,6 +29,7 @@ from transformers.testing_utils import (
require_torch,
require_torch_accelerator,
require_torch_fp16,
require_torch_gpu,
require_torch_multi_accelerator,
require_torch_sdpa,
require_vision,
@@ -777,7 +778,14 @@ class Blip2TextModelTester:
# this model tester uses an encoder-decoder language model (T5)
class Blip2ModelTester:
def __init__(
self, parent, vision_kwargs=None, qformer_kwargs=None, text_kwargs=None, is_training=True, num_query_tokens=10
self,
parent,
vision_kwargs=None,
qformer_kwargs=None,
text_kwargs=None,
is_training=True,
num_query_tokens=10,
image_token_index=4,
):
if vision_kwargs is None:
vision_kwargs = {}
@@ -792,11 +800,10 @@ class Blip2ModelTester:
self.text_model_tester = Blip2TextModelTester(parent, **text_kwargs)
self.batch_size = self.text_model_tester.batch_size # need bs for batching_equivalence test
self.seq_length = self.text_model_tester.seq_length # need seq_length for common tests
self.encoder_seq_length = (
self.text_model_tester.encoder_seq_length + num_query_tokens
) # need enc seq_length for gen tests
self.encoder_seq_length = self.text_model_tester.encoder_seq_length
self.is_training = is_training
self.num_query_tokens = num_query_tokens
self.image_token_index = image_token_index
def prepare_config_and_inputs(self):
_, pixel_values = self.vision_model_tester.prepare_config_and_inputs()
@@ -819,6 +826,7 @@ class Blip2ModelTester:
qformer_config=self.qformer_model_tester.get_config(),
text_config=self.text_model_tester.get_config(),
num_query_tokens=self.num_query_tokens,
image_token_index=self.image_token_index,
)
def create_and_check_for_conditional_generation(
@@ -1872,37 +1880,7 @@ class Blip2ModelIntegrationTest(unittest.TestCase):
self.assertEqual(predictions[0].tolist(), expected_ids_and_text[0])
self.assertEqual(generated_text, expected_ids_and_text[1])
def test_expansion_in_processing(self):
processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
model = Blip2ForConditionalGeneration.from_pretrained(
"Salesforce/blip2-opt-2.7b", torch_dtype=torch.float16
).to(torch_device)
image = prepare_img()
prompt = "Question: which city is this? Answer:"
# Make sure we will go the legacy path by setting these args to None
processor.num_query_tokens = None
model.config.image_token_index = None
inputs = processor(images=image, text=prompt, return_tensors="pt").to(torch_device, dtype=torch.float16)
predictions = model.generate(**inputs, do_sample=False, max_new_tokens=15)
generated_text = processor.batch_decode(predictions, skip_special_tokens=True)[0].strip()
# Add args to the config to trigger new logic when inputs are expanded in processing file
processor.num_query_tokens = model.config.num_query_tokens
processor.tokenizer.add_special_tokens({"additional_special_tokens": ["<image>"]})
model.config.image_token_index = len(processor.tokenizer) - 1
model.resize_token_embeddings(processor.tokenizer.vocab_size, pad_to_multiple_of=64)
# Generate again with new inputs
inputs = processor(images=image, text=prompt, return_tensors="pt").to(torch_device, dtype=torch.float16)
predictions_expanded = model.generate(**inputs, do_sample=False, max_new_tokens=15)
generated_text_expanded = processor.batch_decode(predictions_expanded, skip_special_tokens=True)[0].strip()
self.assertTrue(generated_text_expanded == generated_text)
@require_torch_accelerator
@require_torch_gpu
def test_inference_itm(self):
model_name = "Salesforce/blip2-itm-vit-g"
processor = Blip2Processor.from_pretrained(model_name)

27
tests/models/blip_2/test_processor_blip_2.py
View File

@@ -48,6 +48,9 @@ class Blip2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_image_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor
def prepare_processor_dict(self):
return {"num_query_tokens": 1}
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
@@ -84,26 +87,12 @@ class Blip2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
for key in input_feat_extract.keys():
self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2)
def test_tokenizer(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor = Blip2Processor(tokenizer=tokenizer, image_processor=image_processor)
input_str = "lower newer"
encoded_processor = processor(text=input_str)
encoded_tok = tokenizer(input_str, return_token_type_ids=False)
for key in encoded_tok.keys():
self.assertListEqual(encoded_tok[key], encoded_processor[key][0])
def test_processor(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = Blip2Processor(tokenizer=tokenizer, image_processor=image_processor)
processor = Blip2Processor(tokenizer=tokenizer, image_processor=image_processor, **processor_kwargs)
input_str = "lower newer"
image_input = self.prepare_image_inputs()
@@ -119,8 +108,9 @@ class Blip2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def test_tokenizer_decode(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = Blip2Processor(tokenizer=tokenizer, image_processor=image_processor)
processor = Blip2Processor(tokenizer=tokenizer, image_processor=image_processor, **processor_kwargs)
predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
@@ -132,8 +122,9 @@ class Blip2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def test_model_input_names(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = Blip2Processor(tokenizer=tokenizer, image_processor=image_processor)
processor = Blip2Processor(tokenizer=tokenizer, image_processor=image_processor, **processor_kwargs)
input_str = "lower newer"
image_input = self.prepare_image_inputs()

31
tests/models/instructblip/test_modeling_instructblip.py
View File

@@ -809,34 +809,3 @@ class InstructBlipModelIntegrationTest(unittest.TestCase):
predictions[0].tolist(), [0, 37, 1023, 753, 3, 9, 2335, 3823, 30, 8, 2608, 28, 3, 9, 1782, 5, 1]
)
self.assertEqual(generated_text, "The image features a woman sitting on the beach with a dog.")
def test_expansion_in_processing(self):
processor = InstructBlipProcessor.from_pretrained("Salesforce/instructblip-flan-t5-xl")
model = InstructBlipForConditionalGeneration.from_pretrained(
"Salesforce/instructblip-flan-t5-xl",
torch_dtype=torch.bfloat16,
).to(torch_device)
image = prepare_img()
prompt = "What's in the image?"
# Make sure we will go the legacy path by setting these args to None
processor.num_query_tokens = None
model.config.image_token_index = None
inputs = processor(images=image, text=prompt, return_tensors="pt").to(torch_device, dtype=torch.float16)
predictions = model.generate(**inputs, do_sample=False, max_new_tokens=15)
generated_text = processor.batch_decode(predictions, skip_special_tokens=True)[0].strip()
# Add args to the config to trigger new logic when inputs are expanded in processing file
processor.num_query_tokens = model.config.num_query_tokens
processor.tokenizer.add_special_tokens({"additional_special_tokens": ["<image>"]})
model.config.image_token_index = len(processor.tokenizer) - 2
model.resize_token_embeddings(processor.tokenizer.vocab_size, pad_to_multiple_of=64)
# Generate again with new inputs
inputs = processor(images=image, text=prompt, return_tensors="pt").to(torch_device, dtype=torch.float16)
predictions_expanded = model.generate(**inputs, do_sample=False, max_new_tokens=15)
generated_text_expanded = processor.batch_decode(predictions_expanded, skip_special_tokens=True)[0].strip()
self.assertTrue(generated_text_expanded == generated_text)

53
tests/models/instructblip/test_processor_instructblip.py
View File

@@ -59,6 +59,9 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_qformer_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).qformer_tokenizer
def prepare_processor_dict(self):
return {"num_query_tokens": 1}
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
@@ -90,9 +93,13 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipProcessor(
tokenizer=tokenizer, image_processor=image_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
image_processor=image_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
image_input = self.prepare_image_inputs()
@@ -103,35 +110,17 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
for key in input_feat_extract.keys():
self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2)
def test_tokenizer(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor = InstructBlipProcessor(
tokenizer=tokenizer, image_processor=image_processor, qformer_tokenizer=qformer_tokenizer
)
input_str = ["lower newer"]
encoded_processor = processor(text=input_str)
encoded_tokens = tokenizer(input_str, return_token_type_ids=False)
encoded_tokens_qformer = qformer_tokenizer(input_str, return_token_type_ids=False)
for key in encoded_tokens.keys():
self.assertListEqual(encoded_tokens[key], encoded_processor[key])
for key in encoded_tokens_qformer.keys():
self.assertListEqual(encoded_tokens_qformer[key], encoded_processor["qformer_" + key])
def test_processor(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipProcessor(
tokenizer=tokenizer, image_processor=image_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
image_processor=image_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
input_str = "lower newer"
@@ -141,7 +130,7 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertListEqual(
list(inputs.keys()),
["input_ids", "attention_mask", "qformer_input_ids", "qformer_attention_mask", "pixel_values"],
["qformer_input_ids", "qformer_attention_mask", "input_ids", "attention_mask", "pixel_values"],
)
# test if it raises when no input is passed
@@ -152,9 +141,13 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipProcessor(
tokenizer=tokenizer, image_processor=image_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
image_processor=image_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
@@ -168,9 +161,13 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipProcessor(
tokenizer=tokenizer, image_processor=image_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
image_processor=image_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
input_str = "lower newer"
@@ -180,5 +177,5 @@ class InstructBlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertListEqual(
list(inputs.keys()),
["input_ids", "attention_mask", "qformer_input_ids", "qformer_attention_mask", "pixel_values"],
["qformer_input_ids", "qformer_attention_mask", "input_ids", "attention_mask", "pixel_values"],
)

31
tests/models/instructblipvideo/test_modeling_instructblipvideo.py
View File

@@ -750,34 +750,3 @@ class InstructBlipVideoModelIntegrationTest(unittest.TestCase):
generated_text,
"Explain what is happening in this short video. a baby girl wearing glasses is reading a book on the bed 1080p",
)
def test_expansion_in_processing(self):
processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")
model = InstructBlipVideoForConditionalGeneration.from_pretrained(
"Salesforce/instructblip-vicuna-7b",
load_in_8bit=True,
)
clip = prepare_video()
prompt = "Explain what is happening in this short video."
# Make sure we will go the legacy path by setting these args to None
processor.num_query_tokens = None
model.config.video_token_index = None
inputs = processor(images=clip, text=prompt, return_tensors="pt").to(torch_device, dtype=torch.float16)
predictions = model.generate(**inputs, do_sample=False, max_new_tokens=15)
generated_text = processor.batch_decode(predictions, skip_special_tokens=True)[0].strip()
# Add args to the config to trigger new logic when inputs are expanded in processing file
processor.num_query_tokens = model.config.num_query_tokens
processor.tokenizer.add_special_tokens({"additional_special_tokens": ["<video>"]})
model.config.video_token_index = len(processor.tokenizer) - 1
model.resize_token_embeddings(len(processor.tokenizer), pad_to_multiple_of=64)
# Generate again with new inputs
inputs = processor(images=clip, text=prompt, return_tensors="pt").to(torch_device, dtype=torch.float16)
predictions_expanded = model.generate(**inputs, do_sample=False, max_new_tokens=15)
generated_text_expanded = processor.batch_decode(predictions_expanded, skip_special_tokens=True)[0].strip()
self.assertTrue(generated_text_expanded == generated_text)

39
tests/models/instructblipvideo/test_processor_instructblipvideo.py
View File

@@ -59,6 +59,9 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
def get_qformer_tokenizer(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).qformer_tokenizer
def prepare_processor_dict(self):
return {"num_query_tokens": 1}
def get_video_processor(self, **kwargs):
return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).video_processor
@@ -93,9 +96,13 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
video_processor = self.get_video_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipVideoProcessor(
tokenizer=tokenizer, video_processor=video_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
video_processor=video_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
image_input = self.prepare_image_inputs()
@@ -110,15 +117,17 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
video_processor = self.get_video_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipVideoProcessor(
tokenizer=tokenizer, video_processor=video_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
video_processor=video_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
input_str = ["lower newer"]
encoded_processor = processor(text=input_str)
encoded_tokens = tokenizer(input_str, return_token_type_ids=False)
encoded_tokens_qformer = qformer_tokenizer(input_str, return_token_type_ids=False)
@@ -132,9 +141,13 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
video_processor = self.get_video_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipVideoProcessor(
tokenizer=tokenizer, video_processor=video_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
video_processor=video_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
input_str = "lower newer"
@@ -144,7 +157,7 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertListEqual(
list(inputs.keys()),
["input_ids", "attention_mask", "qformer_input_ids", "qformer_attention_mask", "pixel_values"],
["qformer_input_ids", "qformer_attention_mask", "input_ids", "attention_mask", "pixel_values"],
)
# test if it raises when no input is passed
@@ -155,9 +168,13 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
video_processor = self.get_video_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipVideoProcessor(
tokenizer=tokenizer, video_processor=video_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
video_processor=video_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
@@ -171,9 +188,13 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
video_processor = self.get_video_processor()
tokenizer = self.get_tokenizer()
qformer_tokenizer = self.get_qformer_tokenizer()
processor_kwargs = self.prepare_processor_dict()
processor = InstructBlipVideoProcessor(
tokenizer=tokenizer, video_processor=video_processor, qformer_tokenizer=qformer_tokenizer
tokenizer=tokenizer,
video_processor=video_processor,
qformer_tokenizer=qformer_tokenizer,
**processor_kwargs,
)
input_str = "lower newer"
@@ -183,5 +204,5 @@ class InstructBlipVideoProcessorTest(ProcessorTesterMixin, unittest.TestCase):
self.assertListEqual(
list(inputs.keys()),
["input_ids", "attention_mask", "qformer_input_ids", "qformer_attention_mask", "pixel_values"],
["qformer_input_ids", "qformer_attention_mask", "input_ids", "attention_mask", "pixel_values"],
)

2
tests/utils/tiny_model_summary.json
View File

@@ -626,7 +626,7 @@
"model_classes": [
"Blip2ForConditionalGeneration"
],
"sha": "35e1ef43da3554af62eb29a7b3dbbef3f3bef48e"
"sha": "d0de11fd1f8ca481231c07ee0934924be96cb281"
},
"Blip2Model": {
"tokenizer_classes": [

46
utils/tests_fetcher.py
View File

@@ -52,11 +52,9 @@ python utils/tests_fetcher.py --diff_with_last_commit
import argparse
import collections
import glob
import importlib.util
import json
import os
import re
import tempfile
from contextlib import contextmanager
from pathlib import Path
from typing import Optional, Union
@@ -323,41 +321,15 @@ def get_impacted_files_from_tiny_model_summary(diff_with_last_commit: bool = Fal
if key in new_keys:
impacted_model_classes.extend(new_content[key]["model_classes"])
# get the module where the model classes are defined. We want to use the main `__init__` file, but it requires
# all the framework being installed, which is not ideal for a simple script like test fetcher.
# So we create a temporary and modified main `__init__` and access its `_import_structure`.
with open(folder / "src/transformers/__init__.py") as fp:
lines = fp.readlines()
new_lines = []
# Get all the code related to `_import_structure`
for line in lines:
if line == "_import_structure = {\n":
new_lines.append(line)
elif line == "# Direct imports for type-checking\n":
break
elif len(new_lines) > 0:
# bypass the framework check so we can get all the information even if frameworks are not available
line = re.sub(r"is_.+_available\(\)", "True", line)
line = line.replace("OptionalDependencyNotAvailable", "Exception")
line = line.replace("Exception()", "Exception")
new_lines.append(line)
# Add imports via `define_import_structure` after the #35167 as we remove explicit import in `__init__.py`
from transformers.utils.import_utils import define_import_structure
# create and load the temporary module
with tempfile.TemporaryDirectory() as tmpdirname:
with open(os.path.join(tmpdirname, "temp_init.py"), "w") as fp:
fp.write("".join(new_lines))
spec = importlib.util.spec_from_file_location("temp_init", os.path.join(tmpdirname, "temp_init.py"))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# Finally, get `_import_structure` that we need
import_structure = module._import_structure
# map model classes to their defined module
reversed_structure = {}
for key, values in import_structure.items():
for value in values:
reversed_structure[value] = key
new_imported_modules_from_import_structure = define_import_structure("src/transformers/__init__.py")
for mapping in new_imported_modules_from_import_structure.values():
for _module, _imports in mapping.items():
for _import in _imports:
reversed_structure[_import] = _module
# Get the corresponding modeling file path
for model_class in impacted_model_classes:
@@ -372,9 +344,7 @@ def get_impacted_files_from_tiny_model_summary(diff_with_last_commit: bool = Fal
if framework == ""
else f"modeling_{framework}_{module.split('.')[-1]}.py"
)
files.add(
f"src.transformers.{module}.{fn}".replace(".", os.path.sep).replace(f"{os.path.sep}py", ".py")
)
files.add(f"src.transformers.{module}.{fn}".replace(".", os.path.sep).replace(f"{os.path.sep}py", ".py"))
return sorted(files)