From 0a8081b03d118da9a8c3fa143a03afe54a5c624e Mon Sep 17 00:00:00 2001 From: Kyle Sayers Date: Thu, 26 Jun 2025 11:56:33 -0400 Subject: [PATCH] [Modeling] Fix encoder CPU offloading for whisper (#38994) * fix cpu offloading for whisper Signed-off-by: Kyle Sayers * unskip offloading tests Signed-off-by: Kyle Sayers * revert small change Signed-off-by: Kyle Sayers * remove tests Signed-off-by: Kyle Sayers --------- Signed-off-by: Kyle Sayers --- .../models/whisper/modeling_whisper.py | 4 ++-- tests/models/whisper/test_modeling_whisper.py | 16 ---------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index 9a2c937e51..d3e9c8e03a 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -687,9 +687,9 @@ class WhisperEncoder(WhisperPreTrainedModel): inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds)) inputs_embeds = inputs_embeds.permute(0, 2, 1) - embed_pos = self.embed_positions.weight + all_positions = torch.arange(self.embed_positions.num_embeddings, device=inputs_embeds.device) - hidden_states = inputs_embeds + embed_pos + hidden_states = inputs_embeds + self.embed_positions(all_positions) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) encoder_states = () if output_hidden_states else None diff --git a/tests/models/whisper/test_modeling_whisper.py b/tests/models/whisper/test_modeling_whisper.py index dbb241f5ad..1b4641f5d4 100644 --- a/tests/models/whisper/test_modeling_whisper.py +++ b/tests/models/whisper/test_modeling_whisper.py @@ -3356,22 +3356,6 @@ class WhisperEncoderModelTest(ModelTesterMixin, unittest.TestCase): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_model_forward(*config_and_inputs, use_weighted_layer_sum=True) - @unittest.skip(reason="Some undefined behavior encountered with tiny versions of this model. Skip for now.") - def test_cpu_offload(self): - pass - - @unittest.skip(reason="Some undefined behavior encountered with tiny versions of this model. Skip for now.") - def test_disk_offload_bin(self): - pass - - @unittest.skip(reason="Some undefined behavior encountered with tiny versions of this model. Skip for now.") - def test_disk_offload_safetensors(self): - pass - - @unittest.skip(reason="Some undefined behavior encountered with tiny versions of this model. Skip for now.") - def test_model_parallelism(self): - pass - @unittest.skip(reason="Not applicable for an encoder-only acoustic model") def test_inputs_embeds(self): # input embeds is meaningless for an encoder-only acoustic model