SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix pad_to_max_length Whisper (#30787)

Browse Source 
* fix pad_to_max_length Whisper

* add tests

* make style
This commit is contained in:
Yoach Lacombe
2024-05-27 16:09:05 +02:00
committed by GitHub
parent b84cd67526
commit d355741eca
2 changed files with 84 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
tests/models/whisper/test_modeling_whisper.py
View File

@@ -35,6 +35,7 @@ from transformers.testing_utils import (
require_torch,
require_torch_fp16,
require_torch_gpu,
require_torch_multi_gpu,
require_torchaudio,
slow,
torch_device,
@@ -2866,6 +2867,81 @@ class WhisperModelIntegrationTests(unittest.TestCase):
for i in range(num_samples):
assert decoded_all[i] == EXPECTED_TEXT[i]
@require_torch_gpu
@slow
def test_whisper_empty_longform(self):
processor = WhisperProcessor.from_pretrained("openai/whisper-tiny")
model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny")
model = model.to(torch_device)
ds = load_dataset("distil-whisper/meanwhile", "default")["test"]
ds = ds.cast_column("audio", Audio(sampling_rate=16000))
num_samples = 8
audio = ds[:num_samples]["audio"]
audios = [x["array"] for x in audio]
audios[0][:] = np.zeros(audios[0].shape)
inputs = processor(
audios,
return_tensors="pt",
truncation=False,
padding="longest",
return_attention_mask=True,
sampling_rate=16_000,
)
inputs = inputs.to(device=torch_device)
gen_kwargs = {
"no_speech_threshold": 0.2,
"temperature": (0.0,),
"logprob_threshold": 0.0, # Ignore logprob, use only no-speech prob
"num_beams": 5,
"language": "fr",
"task": "transcribe",
}
torch.manual_seed(0)
model.generate(**inputs, **gen_kwargs)
@require_torch_multi_gpu
@slow
def test_whisper_empty_longform_multi_gpu(self):
processor = WhisperProcessor.from_pretrained("openai/whisper-tiny")
model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny", device_map="auto")
ds = load_dataset("distil-whisper/meanwhile", "default")["test"]
ds = ds.cast_column("audio", Audio(sampling_rate=16000))
num_samples = 8
audio = ds[:num_samples]["audio"]
audios = [x["array"] for x in audio]
audios[0][:] = np.zeros(audios[0].shape)
inputs = processor(
audios,
return_tensors="pt",
truncation=False,
padding="longest",
return_attention_mask=True,
sampling_rate=16_000,
)
inputs = inputs.to(device=model.device)
gen_kwargs = {
"no_speech_threshold": 0.2,
"temperature": (0.0,),
"logprob_threshold": 0.0, # Ignore logprob, use only no-speech prob
"num_beams": 5,
"language": "fr",
"task": "transcribe",
}
torch.manual_seed(0)
model.generate(**inputs, **gen_kwargs)
def prepare_whisper_encoder_inputs_dict(config, input_features, head_mask=None):
if head_mask is None: