From 998b5bb56f161ef9270d17674dec693d40e8fde5 Mon Sep 17 00:00:00 2001 From: Zhihao Lin <36994684+LZHgrla@users.noreply.github.com> Date: Tue, 26 Mar 2024 20:51:00 +0800 Subject: [PATCH] Allow `bos_token_id is None` during the generation with `inputs_embeds` (#29772) * update * add ut * update --- src/transformers/generation/utils.py | 7 ++++--- tests/generation/test_utils.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index b2f330548f..c0d3b1dd60 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -436,9 +436,6 @@ class GenerationMixin: shape = encoder_outputs.last_hidden_state.size()[:-1] return torch.ones(shape, dtype=torch.long, device=self.device) * -100 - if bos_token_id is None: - raise ValueError("`bos_token_id` has to be defined when no `input_ids` are provided.") - # If there is some tensor in `model_kwargs`, we can infer the batch size from it. This is helpful with # soft-prompting or in multimodal implementations built on top of decoder-only language models. batch_size = 1 @@ -449,6 +446,10 @@ class GenerationMixin: if "inputs_embeds" in model_kwargs: return torch.ones((batch_size, 0), dtype=torch.long, device=self.device) + + if bos_token_id is None: + raise ValueError("`bos_token_id` has to be defined when no `input_ids` are provided.") + return torch.ones((batch_size, 1), dtype=torch.long, device=self.device) * bos_token_id def _prepare_attention_mask_for_generation( diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index 91e9c3876a..d82f137d95 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -1467,6 +1467,17 @@ class GenerationTesterMixin: past_kv[i][1].shape, (batch_size, num_attention_heads, seq_length, per_head_embed_dim) ) + def test_generate_from_inputs_embeds_with_bos_token_id_is_none(self): + article = "Today a dragon flew over Paris." + model = AutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2").to(torch_device) + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") + input_ids = tokenizer(article, return_tensors="pt").input_ids.to(torch_device) + inputs_embeds = model.get_input_embeddings()(input_ids) + + model.generate(inputs_embeds=inputs_embeds, max_length=20, bos_token_id=None) + with self.assertRaises(ValueError): + model.generate(max_length=20, bos_token_id=None) + def test_generate_from_inputs_embeds_decoder_only(self): # When supported, tests that the decoder model can generate from `inputs_embeds` instead of `input_ids` # if fails, you should probably update the `prepare_inputs_for_generation` function